Package Bio :: Package TogoWS
[hide private]
[frames] | no frames]

Source Code for Package Bio.TogoWS

  1  # Copyright 2010-2011 by Peter Cock.  All rights reserved. 
  2  # This code is part of the Biopython distribution and governed by its 
  3  # license.  Please see the LICENSE file that should have been included 
  4  # as part of this package. 
  5   
  6  """Provides code to access the TogoWS integrated websevices of DBCLS, Japan. 
  7   
  8  This module aims to make the TogoWS (from DBCLS, Japan) easier to use. See: 
  9  http://togows.dbcls.jp/ 
 10   
 11  The TogoWS REST service provides simple access to a range of databases, acting 
 12  as a proxy to shield you from all the different provider APIs. This works using 
 13  simple URLs (which this module will construct for you). For more details, see 
 14  http://togows.dbcls.jp/site/en/rest.html 
 15   
 16  The functionality is somewhat similar to Biopython's Bio.Entrez module which 
 17  provides access to the NCBI's Entrez Utilities (E-Utils) which also covers a 
 18  wide range of databases. 
 19   
 20  Currently TogoWS does not provide any usage guidelines (unlike the NCBI whose 
 21  requirements are reasonably clear). To avoid risking overloading the service, 
 22  Biopython will only allow three calls per second. 
 23   
 24  The TogoWS SOAP service offers a more complex API for calling web services 
 25  (essentially calling remote functions) provided by DDBJ, KEGG and PDBj. For 
 26  example, this allows you to run a remote BLAST search at the DDBJ. This is 
 27  not yet covered by this module, however there are lots of Python examples 
 28  on the TogoWS website using the SOAPpy python library. See: 
 29  http://togows.dbcls.jp/site/en/soap.html 
 30  http://soapy.sourceforge.net/ 
 31  """ 
 32   
 33  from __future__ import print_function 
 34   
 35  import time 
 36  from Bio._py3k import _binary_to_string_handle, _as_bytes 
 37   
 38  # Importing these functions with leading underscore as not intended for reuse 
 39  from Bio._py3k import urlopen as _urlopen 
 40  from Bio._py3k import quote as _quote 
 41   
 42  __docformat__ = "restructuredtext en" 
 43   
 44  # Constant 
 45  _BASE_URL = "http://togows.dbcls.jp" 
 46   
 47  # Caches: 
 48  _search_db_names = None 
 49  _entry_db_names = None 
 50  _entry_db_fields = {} 
 51  _entry_db_formats = {} 
 52  _convert_formats = [] 
 53   
 54   
55 -def _get_fields(url):
56 """Queries a TogoWS URL for a plain text list of values (PRIVATE).""" 57 handle = _open(url) 58 fields = handle.read().strip().split() 59 handle.close() 60 return fields
61 62
63 -def _get_entry_dbs():
64 return _get_fields(_BASE_URL + "/entry")
65 66
67 -def _get_entry_fields(db):
68 return _get_fields(_BASE_URL + "/entry/%s?fields" % db)
69 70
71 -def _get_entry_formats(db):
72 return _get_fields(_BASE_URL + "/entry/%s?formats" % db)
73 74
75 -def _get_convert_formats():
76 return [pair.split(".") for pair in 77 _get_fields(_BASE_URL + "/convert/")]
78 79
80 -def entry(db, id, format=None, field=None):
81 """TogoWS fetch entry (returns a handle). 82 83 - db - database (string), see list below. 84 - id - identier (string) or a list of identifiers (either as a list of 85 strings or a single string with comma separators). 86 - format - return data file format (string), options depend on the database 87 e.g. "xml", "json", "gff", "fasta", "ttl" (RDF Turtle) 88 - field - specific field from within the database record (string) 89 e.g. "au" or "authors" for pubmed. 90 91 At the time of writing, this includes the following:: 92 93 KEGG: compound, drug, enzyme, genes, glycan, orthology, reaction, 94 module, pathway 95 DDBj: ddbj, dad, pdb 96 NCBI: nuccore, nucest, nucgss, nucleotide, protein, gene, onim, 97 homologue, snp, mesh, pubmed 98 EBI: embl, uniprot, uniparc, uniref100, uniref90, uniref50 99 100 For the current list, please see http://togows.dbcls.jp/entry/ 101 102 This function is essentially equivalent to the NCBI Entrez service 103 EFetch, available in Biopython as Bio.Entrez.efetch(...), but that 104 does not offer field extraction. 105 """ 106 global _entry_db_names, _entry_db_fields, fetch_db_formats 107 if _entry_db_names is None: 108 _entry_db_names = _get_entry_dbs() 109 if db not in _entry_db_names: 110 raise ValueError("TogoWS entry fetch does not officially support " 111 "database '%s'." % db) 112 if field: 113 try: 114 fields = _entry_db_fields[db] 115 except KeyError: 116 fields = _get_entry_fields(db) 117 _entry_db_fields[db] = fields 118 if db == "pubmed" and field == "ti" and "title" in fields: 119 # Backwards compatibility fix for TogoWS change Nov/Dec 2013 120 field = "title" 121 import warnings 122 warnings.warn("TogoWS dropped 'pubmed' field alias 'ti', please use 'title' instead.") 123 if field not in fields: 124 raise ValueError("TogoWS entry fetch does not explicitly support " 125 "field '%s' for database '%s'. Only: %s" 126 % (field, db, ", ".join(sorted(fields)))) 127 if format: 128 try: 129 formats = _entry_db_formats[db] 130 except KeyError: 131 formats = _get_entry_formats(db) 132 _entry_db_formats[db] = formats 133 if format not in formats: 134 raise ValueError("TogoWS entry fetch does not explicitly support " 135 "format '%s' for database '%s'. Only: %s" 136 % (format, db, ", ".join(sorted(formats)))) 137 138 if isinstance(id, list): 139 id = ",".join(id) 140 url = _BASE_URL + "/entry/%s/%s" % (db, _quote(id)) 141 if field: 142 url += "/" + field 143 if format: 144 url += "." + format 145 return _open(url)
146 147
148 -def search_count(db, query):
149 """TogoWS search count (returns an integer). 150 151 db - database (string), see http://togows.dbcls.jp/search 152 query - search term (string) 153 154 You could then use the count to download a large set of search results in 155 batches using the offset and limit options to Bio.TogoWS.search(). In 156 general however the Bio.TogoWS.search_iter() function is simpler to use. 157 """ 158 global _search_db_names 159 if _search_db_names is None: 160 _search_db_names = _get_fields(_BASE_URL + "/search") 161 if db not in _search_db_names: 162 # TODO - Make this a ValueError? Right now despite the HTML website 163 # claiming to, the "gene" or "ncbi-gene" don't work and are not listed. 164 import warnings 165 warnings.warn("TogoWS search does not officially support database '%s'. " 166 "See %s/search/ for options." % (db, _BASE_URL)) 167 url = _BASE_URL + "/search/%s/%s/count" % (db, _quote(query)) 168 handle = _open(url) 169 data = handle.read() 170 handle.close() 171 try: 172 count = int(data.strip()) 173 except ValueError: 174 raise ValueError("Expected an integer from URL %s, got: %r" % (url, data)) 175 return count
176 177
178 -def search_iter(db, query, limit=None, batch=100):
179 """TogoWS search iteratating over the results (generator function). 180 181 - db - database (string), see http://togows.dbcls.jp/search 182 - query - search term (string) 183 - limit - optional upper bound on number of search results 184 - batch - number of search results to pull back each time talk to 185 TogoWS (currently limited to 100). 186 187 You would use this function within a for loop, e.g. 188 189 >>> for id in search_iter("pubmed", "lung+cancer+drug", limit=10): 190 ... print(id) # maybe fetch data with entry? 191 192 Internally this first calls the Bio.TogoWS.search_count() and then 193 uses Bio.TogoWS.search() to get the results in batches. 194 """ 195 count = search_count(db, query) 196 if not count: 197 raise StopIteration 198 # NOTE - We leave it to TogoWS to enforce any upper bound on each 199 # batch, they currently return an HTTP 400 Bad Request if above 100. 200 remain = count 201 if limit is not None: 202 remain = min(remain, limit) 203 offset = 1 # They don't use zero based counting 204 prev_ids = [] # Just cache the last batch for error checking 205 while remain: 206 batch = min(batch, remain) 207 # print("%r left, asking for %r" % (remain, batch)) 208 ids = search(db, query, offset, batch).read().strip().split() 209 assert len(ids) == batch, "Got %i, expected %i" % (len(ids), batch) 210 # print("offset %i, %s ... %s" % (offset, ids[0], ids[-1])) 211 if ids == prev_ids: 212 raise RuntimeError("Same search results for previous offset") 213 for identifier in ids: 214 if identifier in prev_ids: 215 raise RuntimeError("Result %s was in previous batch" 216 % identifier) 217 yield identifier 218 offset += batch 219 remain -= batch 220 prev_ids = ids
221 222
223 -def search(db, query, offset=None, limit=None, format=None):
224 """TogoWS search (returns a handle). 225 226 This is a low level wrapper for the TogoWS search function, which 227 can return results in a several formats. In general, the search_iter 228 function is more suitable for end users. 229 230 - db - database (string), see http://togows.dbcls.jp/search/ 231 - query - search term (string) 232 - offset, limit - optional integers specifying which result to start from 233 (1 based) and the number of results to return. 234 - format - return data file format (string), e.g. "json", "ttl" (RDF) 235 By default plain text is returned, one result per line. 236 237 At the time of writing, TogoWS applies a default count limit of 100 238 search results, and this is an upper bound. To access more results, 239 use the offset argument or the search_iter(...) function. 240 241 TogoWS supports a long list of databases, including many from the NCBI 242 (e.g. "ncbi-pubmed" or "pubmed", "ncbi-genbank" or "genbank", and 243 "ncbi-taxonomy"), EBI (e.g. "ebi-ebml" or "embl", "ebi-uniprot" or 244 "uniprot, "ebi-go"), and KEGG (e.g. "kegg-compound" or "compound"). 245 For the current list, see http://togows.dbcls.jp/search/ 246 247 The NCBI provide the Entrez Search service (ESearch) which is similar, 248 available in Biopython as the Bio.Entrez.esearch() function. 249 250 See also the function Bio.TogoWS.search_count() which returns the number 251 of matches found, and the Bio.TogoWS.search_iter() function which allows 252 you to iterate over the search results (taking care of batching for you). 253 """ 254 global _search_db_names 255 if _search_db_names is None: 256 _search_db_names = _get_fields(_BASE_URL + "/search") 257 if db not in _search_db_names: 258 # TODO - Make this a ValueError? Right now despite the HTML website 259 # claiming to, the "gene" or "ncbi-gene" don't work and are not listed. 260 import warnings 261 warnings.warn("TogoWS search does not explicitly support database '%s'. " 262 "See %s/search/ for options." % (db, _BASE_URL)) 263 url = _BASE_URL + "/search/%s/%s" % (db, _quote(query)) 264 if offset is not None and limit is not None: 265 try: 266 offset = int(offset) 267 except: 268 raise ValueError("Offset should be an integer (at least one), not %r" % offset) 269 try: 270 limit = int(limit) 271 except: 272 raise ValueError("Limit should be an integer (at least one), not %r" % limit) 273 if offset <= 0: 274 raise ValueError("Offset should be at least one, not %i" % offset) 275 if limit <= 0: 276 raise ValueError("Count should be at least one, not %i" % limit) 277 url += "/%i,%i" % (offset, limit) 278 elif offset is not None or limit is not None: 279 raise ValueError("Expect BOTH offset AND limit to be provided (or neither)") 280 if format: 281 url += "." + format 282 # print(url) 283 return _open(url)
284 285
286 -def convert(data, in_format, out_format):
287 """TogoWS convert (returns a handle). 288 289 data - string or handle containing input record(s) 290 in_format - string describing the input file format (e.g. "genbank") 291 out_format - string describing the requested output format (e.g. "fasta") 292 293 For a list of supported conversions (e.g. "genbank" to "fasta"), see 294 http://togows.dbcls.jp/convert/ 295 296 Note that Biopython has built in support for conversion of sequence and 297 alignnent file formats (functions Bio.SeqIO.convert and Bio.AlignIO.convert) 298 """ 299 global _convert_formats 300 if not _convert_formats: 301 _convert_formats = _get_convert_formats() 302 if [in_format, out_format] not in _convert_formats: 303 msg = "\n".join("%s -> %s" % tuple(pair) for pair in _convert_formats) 304 raise ValueError("Unsupported conversion. Choose from:\n%s" % msg) 305 url = _BASE_URL + "/convert/%s.%s" % (in_format, out_format) 306 # TODO - Should we just accept a string not a handle? What about a filename? 307 if hasattr(data, "read"): 308 # Handle 309 return _open(url, post=data.read()) 310 else: 311 # String 312 return _open(url, post=data)
313 314
315 -def _open(url, post=None):
316 """Helper function to build the URL and open a handle to it (PRIVATE). 317 318 Open a handle to TogoWS, will raise an IOError if it encounters an error. 319 320 In the absense of clear guidelines, this function enforces a limit of 321 "up to three queries per second" to avoid abusing the TogoWS servers. 322 """ 323 delay = 0.333333333 # one third of a second 324 current = time.time() 325 wait = _open.previous + delay - current 326 if wait > 0: 327 time.sleep(wait) 328 _open.previous = current + wait 329 else: 330 _open.previous = current 331 332 # print(url) 333 if post: 334 handle = _urlopen(url, _as_bytes(post)) 335 else: 336 handle = _urlopen(url) 337 338 # We now trust TogoWS to have set an HTTP error code, that 339 # suffices for my current unit tests. Previously we would 340 # examine the start of the data returned back. 341 return _binary_to_string_handle(handle)
342 343 _open.previous = 0 344