diff --git a/.travis.yml b/.travis.yml index 8173790..9e31abd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,11 @@ matrix: - os: linux language: python python: 3.6 + - os: linux + language: python + python: 3.7 + dist: xenial + sudo: true - os: osx language: generic python: 2.7 @@ -36,8 +41,9 @@ matrix: # command to install dependencies install: - python --version - - pip install -r tests/requirements.txt - - python setup.py install + - export PUBS_TESTS_MODE=ONLINE # command to run tests -script: python -m unittest discover +script: + - PUBS_TESTS_MODE=MOCK python setup.py test + - PUBS_TESTS_MODE=COLLECT python setup.py test diff --git a/changelog.md b/changelog.md index 0d8a214..c9ea407 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,10 @@ - Adds `move`, and `link` options for handling of documents during `import` (copy being the default). Makes `copy` the default for document handling during `add`. [(#159)](https://github.com/pubs/pubs/pull/159) +- Support for downloading arXiv reference from their ID ([#146](https://github.com/pubs/pubs/issues/146) by [joe-antognini](https://github.com/joe-antognini)) + +- Better feedback when an error is encountered while adding a reference from a DOI, ISBN or arXiv ID [#155](https://github.com/pubs/pubs/issues/155) + - Better dialog after editing paper [(#142)](https://github.com/pubs/pubs/issues/142) - Add a command to open urls ([#139](https://github.com/pubs/pubs/issues/139) by [ksunden](https://github.com/ksunden)) @@ -28,6 +32,7 @@ - Support year ranges in query [(#102)](https://github.com/pubs/pubs/issues/102) +- Tests can now be run with `python setup.py test` [#155](https://github.com/pubs/pubs/issues/155) ### Fixed bugs diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 0000000..0d91a9e --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1,24 @@ +# if you want to setup your environment for development of the pytest code, +# doing `pip install -r dev_requirements.txt` is the single thing you have to do. +# Alternatively, and perhaps more conveniently, running `python setup.py test` +# will do the same *and* run the tests, but without installing the packages on +# the system. +# Note that if you introduce a new dependency, you need to add it here and, more +# importantly, to the setup.py script so that it is taken into account when +# installing from PyPi. + +-e . +pyyaml +bibtexparser>=1.0 +python-dateutil +requests +configobj +beautifulsoup4 +feedparser +six + +# those are the additional packages required to run the tests +pyfakefs +ddt +mock +pytest # optional (python setup.py test works without it), but possible nonetheless diff --git a/pubs/apis.py b/pubs/apis.py index 0812ba5..070d8fc 100644 --- a/pubs/apis.py +++ b/pubs/apis.py @@ -1,27 +1,198 @@ """Interface for Remote Bibliographic APIs""" +import re +import datetime import requests +import bibtexparser +from bibtexparser.bibdatabase import BibDatabase +import feedparser from bs4 import BeautifulSoup -def doi2bibtex(doi): +class ReferenceNotFoundError(Exception): + pass + + +def get_bibentry_from_api(id_str, id_type, try_doi=True, ui=None): + """Return a bibtex string from various ID methods. + + This is a wrapper around functions that will return a bibtex string given + one of: + + * DOI + * IBSN + * arXiv ID + + Args: + id_str: A string with the ID. + id_type: Name of the ID type. Must be one of `doi`, `isbn`, or `arxiv`. + rp: A `Repository` object. + ui: A UI object. + + Returns: + A bibtex string. + + Raises: + ValueError: if `id_type` is not one of `doi`, `isbn`, or `arxiv`. + apis.ReferenceNotFoundException: if no valid reference could be found. + """ + + id_fns = { + 'doi': doi2bibtex, + 'isbn': isbn2bibtex, + 'arxiv': arxiv2bibtex, + } + + if id_type not in id_fns.keys(): + raise ValueError('id_type must be one of `doi`, `isbn`, or `arxiv`.') + + bibentry_raw = id_fns[id_type](id_str, try_doi=try_doi, ui=ui) + endecoder.EnDecoder().decode_bibdata(bibentry_raw) + if bibentry is None: + raise ReferenceNotFoundException( + 'invalid {} {} or unable to retrieve bibfile from it.'.format(id_type, id_str)) + return bibentry + + + +def _get_request(url, headers=None): + """GET requests to a url. Return the `requests` object. + + :raise ConnectionError: if anything goes bad (connection refused, timeout + http status error (401, 404, etc)). + """ + try: + r = requests.get(url, headers=headers) + r.raise_for_status() + return r + except requests.exceptions.RequestException as e: + raise ReferenceNotFoundError(e.args) + + + ## DOI support + +def doi2bibtex(doi, **kwargs): """Return a bibtex string of metadata from a DOI""" - url = 'http://dx.doi.org/{}'.format(doi) + url = 'https://dx.doi.org/{}'.format(doi) headers = {'accept': 'application/x-bibtex'} - r = requests.get(url, headers=headers) + r = _get_request(url, headers=headers) if r.encoding is None: r.encoding = 'utf8' # Do not rely on guessing from request return r.text -def isbn2bibtex(isbn): + ## ISBN support + + +def isbn2bibtex(isbn, **kwargs): """Return a bibtex string of metadata from an ISBN""" - url = 'http://www.ottobib.com/isbn/{}/bibtex'.format(isbn) - r = requests.get(url) + url = 'https://www.ottobib.com/isbn/{}/bibtex'.format(isbn) + r = _get_request(url) soup = BeautifulSoup(r.text, "html.parser") citation = soup.find("textarea").text return citation + + # Note: apparently ottobib.com uses caracter modifiers for accents instead + # of the correct unicode characters. TODO: Should we convert them? + + + ## arXiv support + +_months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', + 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] + +def _is_arxiv_oldstyle(arxiv_id): + return re.match(r"(arXiv\:)?[a-z\-]+\/[0-9]+(v[0-9]+)?", arxiv_id) is not None + +def _extract_arxiv_id(entry): + pattern = r"http[s]?://arxiv.org/abs/(?P.+)" + return re.search(pattern, entry['id']).groupdict()['entry_id'] + + +def arxiv2bibtex(arxiv_id, try_doi=True, ui=None): + """Return a bibtex string of metadata from an arXiv ID + + :param arxiv_id: arXiv id, with or without the `arXiv:` prefix and version + suffix (e.g. `v1`). Old an new style are accepted. Here are + example of accepted identifiers: `1510.00322`, + `arXiv:1510.00322`, `0901.0512`, `arXiv:0901.0512`, + `hep-ph/9409201` or `arXiv:hep-ph/9409201`. + Note that the `arXiv:` prefix will be automatically + removed, and the version suffix automatically added if + missing. + :param try_doi: if a DOI is referenced in the arXiv metadata, + try to download it instead. If that fails for any reason, + falls back to the arXiv, with a warning message, if the + UI is provided. + :param ui: if not None, will display a warning if the doi request + fails. + """ + ## handle errors + url = 'https://export.arxiv.org/api/query?id_list={}'.format(arxiv_id) + try: + r = requests.get(url) + if r.status_code == 400: # bad request + msg = ("the arXiv server returned a bad request error. The " + "arXiv id {} is possibly invalid or malformed.".format(arxiv_id)) + raise ReferenceNotFoundError(msg) + r.raise_for_status() # raise an exception for HTTP errors: + # 401, 404, 400 if `ui` is None, etc. + except requests.exceptions.RequestException as e: + msg = ("connection error while retrieving arXiv data for " + "'{}': {}".format(arxiv_id, e)) + raise ReferenceNotFoundError(msg) + + feed = feedparser.parse(r.text) + if len(feed.entries) == 0: # no results. + msg = "no results for arXiv id {}".format(arxiv_id) + raise ReferenceNotFoundError(msg) + if len(feed.entries) > 1: # I don't know how that could happen, but let's + # be ready for it. + results = '\n'.join('{}. {}'.format(i, entry['title']) + for entry in feed.entries) + msg = ("multiple results for arXiv id {}:\n{}\nThis is unexpected. " + "Please submit an issue at " + "https://github.com/pubs/pubs/issues").format(arxiv_id, choices) + raise ReferenceNotFoundError(msg) + + entry = feed.entries[0] + + ## try to return a doi instead of the arXiv reference + if try_doi and 'arxiv_doi' in entry: + try: + return doi2bibtex(entry['arxiv_doi']) + except ReferenceNotFoundError as e: + if ui is not None: + ui.warning(str(e)) + + ## create a bibentry from the arXiv response. + db = BibDatabase() + entry_id = _extract_arxiv_id(entry) + author_str = ' and '.join( + [author['name'] for author in entry['authors']]) + db.entries = [{ + 'ENTRYTYPE': 'article', + 'ID': entry_id, + 'author': author_str, + 'title': entry['title'], + 'year': str(entry['published_parsed'].tm_year), + 'month': _months[entry['published_parsed'].tm_mon-1], + 'eprint': entry_id, + 'eprinttype': 'arxiv', + 'date': entry['published'], # not really standard, but a resolution more + # granular than months is increasinlgy relevant. + 'url': entry['link'], + 'urldate': datetime.datetime.utcnow().isoformat() + 'Z' # can't hurt. + }] + # we don't add eprintclass for old-style ids, as it is in the id already. + if not _is_arxiv_oldstyle(entry_id): + db.entries[0]['eprintclass'] = entry['arxiv_primary_category']['term'] + if 'arxiv_doi' in entry: + db.entries[0]['arxiv_doi'] = entry['arxiv_doi'] + + bibtex = bibtexparser.dumps(db) + return bibtex diff --git a/pubs/commands/__init__.py b/pubs/commands/__init__.py index cd17fa3..ed5defa 100644 --- a/pubs/commands/__init__.py +++ b/pubs/commands/__init__.py @@ -6,9 +6,11 @@ from . import add_cmd from . import rename_cmd from . import remove_cmd from . import list_cmd +from . import edit_cmd +from . import tag_cmd +from . import statistics_cmd # doc from . import doc_cmd -from . import tag_cmd from . import note_cmd # bulk from . import export_cmd @@ -16,5 +18,3 @@ from . import import_cmd # bonus from . import websearch_cmd from . import url_cmd - -from . import edit_cmd diff --git a/pubs/commands/add_cmd.py b/pubs/commands/add_cmd.py index dfa9bb1..d02e06e 100644 --- a/pubs/commands/add_cmd.py +++ b/pubs/commands/add_cmd.py @@ -27,8 +27,10 @@ def parser(subparsers, conf): parser = subparsers.add_parser('add', help='add a paper to the repository') parser.add_argument('bibfile', nargs='?', default=None, help='bibtex file') - parser.add_argument('-D', '--doi', help='doi number to retrieve the bibtex entry, if it is not provided', default=None, action=ValidateDOI) - parser.add_argument('-I', '--isbn', help='isbn number to retrieve the bibtex entry, if it is not provided', default=None) + id_arg = parser.add_mutually_exclusive_group() + id_arg.add_argument('-D', '--doi', help='doi number to retrieve the bibtex entry, if it is not provided', default=None, action=ValidateDOI) + id_arg.add_argument('-I', '--isbn', help='isbn number to retrieve the bibtex entry, if it is not provided', default=None) + id_arg.add_argument('-X', '--arxiv', help='arXiv ID to retrieve the bibtex entry, if it is not provided', default=None) parser.add_argument('-d', '--docfile', help='pdf or ps file', default=None) parser.add_argument('-t', '--tags', help='tags associated to the paper, separated by commas', default=None @@ -39,22 +41,25 @@ def parser(subparsers, conf): return parser -def bibentry_from_editor(conf, ui, rp): +def bibentry_from_editor(conf, ui): + again = True - bibstr = templates.add_bib + bibentry_raw = templates.add_bib + decoder = endecoder.EnDecoder() + while again: try: - bibstr = ui.editor_input(initial=bibstr, suffix='.bib') - if bibstr == templates.add_bib: + bibentry_raw = ui.editor_input(initial=bibentry_raw, suffix='.bib') + if bibentry_raw == templates.add_bib: again = ui.input_yn( question='Bibfile not edited. Edit again ?', default='y') if not again: ui.exit(0) else: - bibentry = rp.databroker.verify(bibstr) + bibentry = decoder.decode_bibdata(bibentry_raw) bibstruct.verify_bibdata(bibentry) - # REFACTOR Generate citykey + # REFACTOR Generate citekey again = False except endecoder.EnDecoder.BibDecodingError: @@ -80,30 +85,29 @@ def command(conf, args): citekey = args.citekey rp = repo.Repository(conf) + decoder = endecoder.EnDecoder() # get bibtex entry if bibfile is None: - if args.doi is None and args.isbn is None: - bibentry = bibentry_from_editor(conf, ui, rp) + if args.doi is None and args.isbn is None and args.arxiv is None: + bibentry = bibentry_from_editor(conf, ui) else: - if args.doi is not None: - bibentry_raw = apis.doi2bibtex(args.doi) - bibentry = rp.databroker.verify(bibentry_raw) - if bibentry is None: - ui.error('invalid doi {} or unable to retrieve bibfile from it.'.format(args.doi)) - if args.isbn is None: - ui.exit(1) - if args.isbn is not None: - bibentry_raw = apis.isbn2bibtex(args.isbn) - bibentry = rp.databroker.verify(bibentry_raw) - if bibentry is None: - ui.error('invalid isbn {} or unable to retrieve bibfile from it.'.format(args.isbn)) - ui.exit(1) - # TODO distinguish between cases, offer to open the error page in a webbrowser. - # TODO offer to confirm/change citekey + bibentry = None + try: + if args.doi is not None: + bibentry = apis.get_bibentry_from_api(args.doi, 'doi', ui=ui) + elif args.isbn is not None: + bibentry = apis.get_bibentry_from_api(args.isbn, 'isbn', ui=ui) + # TODO distinguish between cases, offer to open the error page in a webbrowser. + # TODO offer to confirm/change citekey + elif args.arxiv is not None: + bibentry = apis.get_bibentry_from_api(args.arxiv, 'arxiv', ui=ui) + except apis.ReferenceNotFoundException as e: + ui.error(e.message) + ui.exit(1) else: bibentry_raw = content.get_content(bibfile, ui=ui) - bibentry = rp.databroker.verify(bibentry_raw) + bibentry = decoder.decode_bibdata(bibentry_raw) if bibentry is None: ui.error('invalid bibfile {}.'.format(bibfile)) diff --git a/pubs/commands/statistics_cmd.py b/pubs/commands/statistics_cmd.py new file mode 100644 index 0000000..5132010 --- /dev/null +++ b/pubs/commands/statistics_cmd.py @@ -0,0 +1,33 @@ +from ..repo import Repository +from ..uis import get_ui +from .. import color + + +def parser(subparsers, conf): + parser = subparsers.add_parser( + 'statistics', + help="show statistics on the repository.") + return parser + + +def command(conf, args): + ui = get_ui() + rp = Repository(conf) + papers = list(rp.all_papers()) + + paper_count = len(papers) + doc_count = sum([0 if p.docpath is None else 1 for p in papers]) + tag_count = len(list(rp.get_tags())) + papers_with_tags = sum([0 if p.tags else 1 for p in papers]) + + ui.message(color.dye_out('Repository statistics:', 'bold')) + ui.message('Total papers: {}, {} ({}) have a document attached'.format( + color.dye_out('{:d}'.format(paper_count), 'bgreen'), + color.dye_out('{:d}'.format(doc_count), 'bold'), + '{:.0f}%'.format(100. * doc_count / paper_count), + )) + ui.message('Total tags: {}, {} ({}) of papers have at least one tag'.format( + color.dye_out('{:d}'.format(tag_count), 'bgreen'), + color.dye_out('{:d}'.format(papers_with_tags), 'bold'), + '{:.0f}%'.format(100. * papers_with_tags / paper_count), + )) diff --git a/pubs/databroker.py b/pubs/databroker.py index 97c1398..23e8b91 100644 --- a/pubs/databroker.py +++ b/pubs/databroker.py @@ -79,16 +79,6 @@ class DataBroker(object): def listing(self, filestats=True): return self.filebroker.listing(filestats=filestats) - def verify(self, bibdata_raw): - """Will return None if bibdata_raw can't be decoded""" - try: - if bibdata_raw.startswith('\ufeff'): - # remove BOM, because bibtexparser does not support it. - bibdata_raw = bibdata_raw[1:] - return self.endecoder.decode_bibdata(bibdata_raw) - except ValueError as e: - return None - # docbroker def in_docsdir(self, docpath): diff --git a/pubs/datacache.py b/pubs/datacache.py index 2da4fe0..a02a05b 100644 --- a/pubs/datacache.py +++ b/pubs/datacache.py @@ -163,9 +163,6 @@ class DataCache(object): def listing(self, filestats=True): return self.databroker.listing(filestats=filestats) - def verify(self, bibdata_raw): - return self.databroker.verify(bibdata_raw) - # docbroker def in_docsdir(self, docpath): diff --git a/pubs/endecoder.py b/pubs/endecoder.py index 259b8d9..c280460 100644 --- a/pubs/endecoder.py +++ b/pubs/endecoder.py @@ -1,9 +1,16 @@ from __future__ import absolute_import, unicode_literals import copy +import logging + +# both needed to intercept exceptions. +import pyparsing +import bibtexparser try: import bibtexparser as bp + # don't let bibtexparser display stuff + bp.bparser.logger.setLevel(level=logging.CRITICAL) except ImportError: print("error: you need to install bibterxparser; try running 'pip install " "bibtexparser'.") @@ -68,14 +75,14 @@ class EnDecoder(object): class BibDecodingError(Exception): - message = "Could not parse provided bibdata:\n---\n{}\n---" - - def __init__(self, bibdata): + def __init__(self, error_msg, bibdata): + """ + :param error_msg: specific message about what went wrong + :param bibdata: the data that was unsuccessfully decoded. + """ + super(Exception, self).__init__(error_msg) # make `str(self)` work. self.data = bibdata - def __str__(self): - return self.message.format(self.data) - bwriter = bp.bwriter.BibTexWriter() bwriter.display_order = BIBFIELD_ORDER @@ -117,10 +124,12 @@ class EnDecoder(object): If the decoding fails, returns a BibParseError. """ + if len(bibdata) == 0: + error_msg = 'parsing error: the provided string has length zero.' + raise self.BibDecodingError(error_msg, bibdata) try: entries = bp.bparser.BibTexParser( - bibdata, common_strings=True, - customization=customizations, + bibdata, common_strings=True, customization=customizations, homogenize_fields=True).get_entry_dict() # Remove id from bibtexparser attribute which is stored as citekey @@ -131,8 +140,18 @@ class EnDecoder(object): entries[e][TYPE_KEY] = t if len(entries) > 0: return entries - except Exception: - import traceback - traceback.print_exc() - raise self.BibDecodingError(bibdata) - # TODO: filter exceptions from pyparsing and pass reason upstream + except (pyparsing.ParseException, pyparsing.ParseSyntaxException) as e: + error_msg = self._format_parsing_error(e) + raise self.BibDecodingError(error_msg, bibdata) + except bibtexparser.bibdatabase.UndefinedString as e: + error_msg = 'parsing error: undefined string in provided data: {}'.format(e) + raise self.BibDecodingError(error_msg, bibdata) + + + @classmethod + def _format_parsing_error(cls, e): + """Transform a pyparsing exception into an error message + + Does a best effort to be useful, but might need to be improved. + """ + return '{}\n{}^\n{}'.format(e.line, (e.column - 1) * ' ', e) diff --git a/pubs/pubs_cmd.py b/pubs/pubs_cmd.py index 949fd4a..258cb85 100644 --- a/pubs/pubs_cmd.py +++ b/pubs/pubs_cmd.py @@ -20,9 +20,11 @@ CORE_CMDS = collections.OrderedDict([ ('rename', commands.rename_cmd), ('remove', commands.remove_cmd), ('list', commands.list_cmd), + ('edit', commands.edit_cmd), + ('tag', commands.tag_cmd), + ('statistics', commands.statistics_cmd), ('doc', commands.doc_cmd), - ('tag', commands.tag_cmd), ('note', commands.note_cmd), ('export', commands.export_cmd), @@ -30,7 +32,6 @@ CORE_CMDS = collections.OrderedDict([ ('websearch', commands.websearch_cmd), ('url', commands.url_cmd), - ('edit', commands.edit_cmd), ]) diff --git a/readme.md b/readme.md index 4b03dcd..5b43e56 100644 --- a/readme.md +++ b/readme.md @@ -56,6 +56,10 @@ or an ISBN (dashes are ignored): pubs add -I 978-0822324669 -d article.pdf +or an arXiv id (automatically downloading arXiv article is in the works): + + pubs add -X math/9501234 -d article.pdf + ## References always up-to-date diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4e10580..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -pyyaml -bibtexparser>=1.0 -python-dateutil -requests -configobj -beautifulsoup4 diff --git a/setup.py b/setup.py index 9e99694..50058b5 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,16 @@ #!/usr/bin/env python +import unittest from setuptools import setup with open('pubs/version.py') as f: exec(f.read()) # defines __version__ +def pubs_test_suite(): + test_loader = unittest.TestLoader() + test_suite = test_loader.discover('tests', pattern='test_*.py') + return test_suite + setup( name='pubs', version=__version__, @@ -26,9 +32,8 @@ setup( ], }, - install_requires=['pyyaml', 'bibtexparser>=1.0', 'python-dateutil', - 'requests', 'configobj', 'beautifulsoup4'], - tests_require=['pyfakefs>=2.7', 'mock'], + install_requires=['pyyaml', 'bibtexparser>=1.0', 'python-dateutil', 'six', + 'requests', 'configobj', 'beautifulsoup4', 'feedparser'], extras_require={'autocompletion': ['argcomplete'], }, @@ -41,6 +46,9 @@ setup( 'Intended Audience :: Science/Research', ], + test_suite= 'tests', + tests_require=['pyfakefs>=3.4', 'mock', 'ddt'], + # in order to avoid 'zipimport.ZipImportError: bad local file header' zip_safe=False, diff --git a/tests/mock_requests.py b/tests/mock_requests.py new file mode 100644 index 0000000..6660950 --- /dev/null +++ b/tests/mock_requests.py @@ -0,0 +1,106 @@ +""" +Mock the `requests.get` function, and handle collecting data to do so. + +Three modes are available, and controlled via the `PUBS_TESTS_MODE` environment +variable. To modify the variable, under linux or macos, do one of: +$ export PUBS_TESTS_MODE=MOCK +$ export PUBS_TESTS_MODE=COLLECT +$ export PUBS_TESTS_MODE=ONLINE + +The MOCK mode is the default one, active even if PUBS_TESTS_MODE has not been +set. It uses saved data to run pubs units tests relying on the `requests.get` +function without the need of an internet connection (it is also much faster). +The prefected data is save in the `test_apis_data.pickle` file. + +The COLLECT mode does real GET requests, and updates the `test_apis_data.pickle` +file. It is needed if you add or modify the test relying on `requests.get`. + +The ONLINE mode bypasses all this and use the original `requests.get` without +accessing or updating the `test_apis_data.pickle` data. It might be useful when +running tests on Travis for instance. +""" + + +import os +import json +import mock + +import requests + + +_orgininal_requests_get = requests.get +_collected_responses = [] +_data_filepath = os.path.join(os.path.dirname(__file__), 'test_apis_data.json') + + +class MockingResponse: + def __init__(self, text, status_code=200, error_msg=None): + self.text = text + self.status_code = status_code + self.error_msg = error_msg + self.encoding = 'utf8' + + def raise_for_status(self): + if self.status_code != 200: + raise requests.exceptions.RequestException(self.error_msg) + + +def intercept_text(text): + try: + if '10.1103/PhysRevD.89.084044' in text: + # replace with wrong DOI + text = text.replace('PhysRevD', 'INVALIDDOI') + except TypeError: + if b'10.1103/PhysRevD.89.084044' in text: + # replace with wrong DOI + text = text.replace(b'PhysRevD', b'INVALIDDOI') + + return text + + +mode = os.environ.get('PUBS_TESTS_MODE', 'MOCK') + +if mode == 'MOCK': + + with open(os.path.join(_data_filepath), 'r') as fd: + _collected_responses = json.load(fd) + + def mock_requests_get(*args, **kwargs): + for args2, kwargs2, text, status_code, error_msg in _collected_responses: + if list(args) == list(args2) and kwargs == kwargs2: + return MockingResponse(text, status_code, error_msg) + raise KeyError(('No stub data found for requests.get({}, {}).\n You may' + ' need to update the mock data. Look at the ' + 'tests/mock_requests.py file for an explanation').format(args, kwargs)) + +elif mode == 'COLLECT': + + def mock_requests_get(*args, **kwargs): + text, status_code, error_msg = None, None, None + try: + r = _orgininal_requests_get(*args, **kwargs) + text, status_code = r.text, r.status_code + r.raise_for_status() + except requests.exceptions.RequestException as e: + error_msg = str(e) + + text = intercept_text(text) + _collected_responses.append((args, kwargs, text, status_code, error_msg)) + _save_collected_responses() # yes, we save everytime, because it's not + # clear how to run once after all the tests + # have run. If you figure it out... + + return MockingResponse(text, status_code, error_msg) + + def _save_collected_responses(): + with open(os.path.join(_data_filepath), 'w') as fd: + json.dump(sorted(_collected_responses), fd, indent=2) + +elif mode == 'ONLINE': + def mock_requests_get(*args, **kwargs): + # with mock.patch('requests.Response.text', new_callable=mock.PropertyMock) as mock_text: + r = _orgininal_requests_get(*args, **kwargs) + r._content = intercept_text(r.content) + # print(r.content.__class__) + # mock_text.return_value = intercept_text(r.text) + return r diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index f3726e3..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -# those are the additional packages required to run the tests -six -pyfakefs -ddt -mock diff --git a/tests/test_apis.py b/tests/test_apis.py index a83481a..b870f63 100644 --- a/tests/test_apis.py +++ b/tests/test_apis.py @@ -3,24 +3,36 @@ from __future__ import unicode_literals import unittest +import mock + + import dotdot from pubs.p3 import ustr from pubs.endecoder import EnDecoder -from pubs.apis import doi2bibtex, isbn2bibtex +from pubs.apis import ReferenceNotFoundError, arxiv2bibtex, doi2bibtex, isbn2bibtex, _is_arxiv_oldstyle, _extract_arxiv_id + +from pubs import apis + +import mock_requests -class TestDOI2Bibtex(unittest.TestCase): +class APITests(unittest.TestCase): def setUp(self): self.endecoder = EnDecoder() - def test_unicode(self): + +class TestDOI2Bibtex(APITests): + + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_unicode(self, reqget): bib = doi2bibtex('10.1007/BF01700692') self.assertIsInstance(bib, ustr) self.assertIn('Kurt Gödel', bib) - def test_parses_to_bibtex(self): + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_parses_to_bibtex(self, reqget): bib = doi2bibtex('10.1007/BF01700692') b = self.endecoder.decode_bibdata(bib) self.assertEqual(len(b), 1) @@ -30,23 +42,22 @@ class TestDOI2Bibtex(unittest.TestCase): 'Über formal unentscheidbare Sätze der Principia ' 'Mathematica und verwandter Systeme I') - def test_parse_fails_on_incorrect_DOI(self): - bib = doi2bibtex('999999') - with self.assertRaises(EnDecoder.BibDecodingError): - self.endecoder.decode_bibdata(bib) - + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_retrieve_fails_on_incorrect_DOI(self, reqget): + with self.assertRaises(apis.ReferenceNotFoundError): + doi2bibtex('999999') -class TestISBN2Bibtex(unittest.TestCase): - def setUp(self): - self.endecoder = EnDecoder() +class TestISBN2Bibtex(APITests): - def test_unicode(self): + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_unicode(self, reqget): bib = isbn2bibtex('9782081336742') self.assertIsInstance(bib, ustr) self.assertIn('Poincaré, Henri', bib) - def test_parses_to_bibtex(self): + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_parses_to_bibtex(self, reqget): bib = isbn2bibtex('9782081336742') b = self.endecoder.decode_bibdata(bib) self.assertEqual(len(b), 1) @@ -54,11 +65,97 @@ class TestISBN2Bibtex(unittest.TestCase): self.assertEqual(entry['author'][0], 'Poincaré, Henri') self.assertEqual(entry['title'], 'La science et l\'hypothèse') - def test_parse_fails_on_incorrect_ISBN(self): - bib = doi2bibtex('9' * 13) + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_retrieve_fails_on_incorrect_ISBN(self, reqget): + bib = isbn2bibtex('9' * 13) with self.assertRaises(EnDecoder.BibDecodingError): self.endecoder.decode_bibdata(bib) -# Note: apparently ottobib.com uses caracter modifiers for accents instead -# of the correct unicode characters. TODO: Should we convert them? +class TestArxiv2Bibtex(APITests): + + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_new_style(self, reqget): + bib = arxiv2bibtex('astro-ph/9812133') + b = self.endecoder.decode_bibdata(bib) + self.assertEqual(len(b), 1) + entry = b[list(b)[0]] + self.assertEqual(entry['author'][0], 'Perlmutter, S.') + self.assertEqual(entry['year'], '1999') + + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_parses_to_bibtex_with_doi(self, reqget): + bib = arxiv2bibtex('astro-ph/9812133') + b = self.endecoder.decode_bibdata(bib) + self.assertEqual(len(b), 1) + entry = b[list(b)[0]] + self.assertEqual(entry['author'][0], 'Perlmutter, S.') + self.assertEqual(entry['year'], '1999') + + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_parses_to_bibtex_without_doi(self, reqget): + bib = arxiv2bibtex('math/0211159') + b = self.endecoder.decode_bibdata(bib) + self.assertEqual(len(b), 1) + entry = b[list(b)[0]] + self.assertEqual(entry['author'][0], 'Perelman, Grisha') + self.assertEqual(entry['year'], '2002') + self.assertEqual( + entry['title'], + 'The entropy formula for the Ricci flow and its geometric applications') + + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_arxiv_wrong_id(self, reqget): + with self.assertRaises(ReferenceNotFoundError): + bib = arxiv2bibtex('INVALIDID') + + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_arxiv_wrong_doi(self, reqget): + bib = arxiv2bibtex('1312.2021') + b = self.endecoder.decode_bibdata(bib) + entry = b[list(b)[0]] + self.assertEqual(entry['arxiv_doi'], '10.1103/INVALIDDOI.89.084044') + + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_arxiv_good_doi(self, reqget): + """Get the DOI bibtex instead of the arXiv one if possible""" + bib = arxiv2bibtex('1710.08557') + b = self.endecoder.decode_bibdata(bib) + entry = b[list(b)[0]] + self.assertTrue(not 'arxiv_doi' in entry) + self.assertEqual(entry['doi'], '10.1186/s12984-017-0305-3') + self.assertEqual(entry['title'].lower(), 'on neuromechanical approaches for the study of biological and robotic grasp and manipulation') + + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) + def test_arxiv_good_doi_force_arxiv(self, reqget): + bib = arxiv2bibtex('1710.08557', try_doi=False) + b = self.endecoder.decode_bibdata(bib) + entry = b[list(b)[0]] + self.assertEqual(entry['arxiv_doi'], '10.1186/s12984-017-0305-3') + self.assertEqual(entry['title'].lower(), 'on neuromechanical approaches for the study of biological grasp and\nmanipulation') + + +class TestArxiv2BibtexLocal(unittest.TestCase): + """Test arXiv 2 Bibtex connection; those tests don't require a connection""" + + def test_oldstyle_pattern(self): + """Test that we can accurately differentiate between old and new style arXiv ids.""" + # old-style arXiv ids + for arxiv_id in ['cs/9301113', 'math/9201277v3', 'astro-ph/9812133', + 'cond-mat/0604612', 'hep-ph/0702007v10', 'arXiv:physics/9403001' + ]: + self.assertTrue(_is_arxiv_oldstyle(arxiv_id)) + # new-style arXiv ids + for arxiv_id in ['1808.00954', 'arXiv:1808.00953', '1808.0953', + '1808.00954v1', 'arXiv:1808.00953v2', '1808.0953v42']: + self.assertFalse(_is_arxiv_oldstyle(arxiv_id)) + + def test_extract_id(self): + """Test that ids are correctly extracted""" + self.assertEqual(_extract_arxiv_id({'id': "http://arxiv.org/abs/0704.0010v1"}), "0704.0010v1") + self.assertEqual(_extract_arxiv_id({'id': "https://arxiv.org/abs/0704.0010v1"}), "0704.0010v1") + self.assertEqual(_extract_arxiv_id({'id': "https://arxiv.org/abs/astro-ph/9812133v2"}), "astro-ph/9812133v2") + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_apis_data.json b/tests/test_apis_data.json new file mode 100644 index 0000000..8fe73aa --- /dev/null +++ b/tests/test_apis_data.json @@ -0,0 +1,189 @@ +[ + [ + [ + "https://dx.doi.org/10.1007/BF01700692" + ], + { + "headers": { + "accept": "application/x-bibtex" + } + }, + "@article{G_del_1931,\n\tdoi = {10.1007/bf01700692},\n\turl = {https://doi.org/10.1007%2Fbf01700692},\n\tyear = 1931,\n\tmonth = {dec},\n\tpublisher = {Springer Nature},\n\tvolume = {38-38},\n\tnumber = {1},\n\tpages = {173--198},\n\tauthor = {Kurt G\u00f6del},\n\ttitle = {\u00dcber formal unentscheidbare S\u00e4tze der Principia Mathematica und verwandter Systeme I},\n\tjournal = {Monatshefte f\u00fcr Mathematik und Physik}\n}", + 200, + null + ], + [ + [ + "https://dx.doi.org/10.1007/BF01700692" + ], + { + "headers": { + "accept": "application/x-bibtex" + } + }, + "@article{G_del_1931,\n\tdoi = {10.1007/bf01700692},\n\turl = {https://doi.org/10.1007%2Fbf01700692},\n\tyear = 1931,\n\tmonth = {dec},\n\tpublisher = {Springer Nature},\n\tvolume = {38-38},\n\tnumber = {1},\n\tpages = {173--198},\n\tauthor = {Kurt G\u00f6del},\n\ttitle = {\u00dcber formal unentscheidbare S\u00e4tze der Principia Mathematica und verwandter Systeme I},\n\tjournal = {Monatshefte f\u00fcr Mathematik und Physik}\n}", + 200, + null + ], + [ + [ + "https://dx.doi.org/10.1086/307221" + ], + { + "headers": { + "accept": "application/x-bibtex" + } + }, + "@article{Perlmutter_1999,\n\tdoi = {10.1086/307221},\n\turl = {https://doi.org/10.1086%2F307221},\n\tyear = 1999,\n\tmonth = {jun},\n\tpublisher = {{IOP} Publishing},\n\tvolume = {517},\n\tnumber = {2},\n\tpages = {565--586},\n\tauthor = {S. Perlmutter and G. Aldering and G. Goldhaber and R. A. Knop and P. Nugent and P. G. Castro and S. Deustua and S. Fabbro and A. Goobar and D. E. Groom and I. M. Hook and A. G. Kim and M. Y. Kim and J. C. Lee and N. J. Nunes and R. Pain and C. R. Pennypacker and R. Quimby and C. Lidman and R. S. Ellis and M. Irwin and R. G. McMahon and P. Ruiz-Lapuente and N. Walton and B. Schaefer and B. J. Boyle and A. V. Filippenko and T. Matheson and A. S. Fruchter and N. Panagia and H. J. M. Newberg and W. J. Couch and The Supernova Cosmology Project},\n\ttitle = {Measurements of $\\{upOmega}$ and $\\{upLambda}$ from 42 High-Redshift Supernovae},\n\tjournal = {The Astrophysical Journal}\n}", + 200, + null + ], + [ + [ + "https://dx.doi.org/10.1086/307221" + ], + { + "headers": { + "accept": "application/x-bibtex" + } + }, + "@article{Perlmutter_1999,\n\tdoi = {10.1086/307221},\n\turl = {https://doi.org/10.1086%2F307221},\n\tyear = 1999,\n\tmonth = {jun},\n\tpublisher = {{IOP} Publishing},\n\tvolume = {517},\n\tnumber = {2},\n\tpages = {565--586},\n\tauthor = {S. Perlmutter and G. Aldering and G. Goldhaber and R. A. Knop and P. Nugent and P. G. Castro and S. Deustua and S. Fabbro and A. Goobar and D. E. Groom and I. M. Hook and A. G. Kim and M. Y. Kim and J. C. Lee and N. J. Nunes and R. Pain and C. R. Pennypacker and R. Quimby and C. Lidman and R. S. Ellis and M. Irwin and R. G. McMahon and P. Ruiz-Lapuente and N. Walton and B. Schaefer and B. J. Boyle and A. V. Filippenko and T. Matheson and A. S. Fruchter and N. Panagia and H. J. M. Newberg and W. J. Couch and The Supernova Cosmology Project},\n\ttitle = {Measurements of $\\{upOmega}$ and $\\{upLambda}$ from 42 High-Redshift Supernovae},\n\tjournal = {The Astrophysical Journal}\n}", + 200, + null + ], + [ + [ + "https://dx.doi.org/10.1103/INVALIDDOI.89.084044" + ], + { + "headers": { + "accept": "application/x-bibtex" + } + }, + "\n\n\nError: DOI Not Found\n\n\n\n\n \n\n\n\n\n\n\n
\n\"Logo\"\n
\n\n
\n
\n
\n\n\n\n\n\n \n \n \n \n
\"\"\n HOME  | HANDBOOK  | FACTSHEETS  | FAQs  |  RESOURCES  | USERS  | NEWS  | MEMBERS AREA\n
\n\n\n
\n
\n\n\n\n \n\n\n\n\n\n\n\n\n\n\n
\n\"\"\n
\n\n

DOI Not Found

\n\n
 
\n\n\n\n

10.1103/INVALIDDOI.89.084044

\n\n
 
\n\n\n\n\n

This DOI cannot be found in the DOI System. Possible reasons are:

\n\n\n
    \n\n
  • The DOI is incorrect in your source. Search for the item by name, title, or other metadata using a search engine.
    • \n\n
  • The DOI was copied incorrectly. Check to see that the string includes all the characters before and after the slash and no sentence punctuation marks.
    • \n\n
  • The DOI has not been activated yet. Please try again later, and report the problem if the error continues.
    • \n\n
\n\n\n\n
 
\n\n

You may report this error to the responsible DOI Registration Agency using the form below. Include your email address to receive confirmation and feedback.

\n\n
\n\n
\n\n\n\n\n\n\n\n\n\n\n\n\n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Your Email Address:
Additional Information About the Error:
\n\n

\n\n
\n
\n\n\n\n\n		\"\"
\n\n
 
\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n
\nDOI System Proxy Server Documentation\n
\n\"DOI_disc_logo\"\n\n®, DOI®, DOI.ORG®, and shortDOI® are trademarks of the International DOI Foundation.\n
\n\n\n", + 404, + "404 Client Error: for url: https://dx.doi.org/10.1103/INVALIDDOI.89.084044" + ], + [ + [ + "https://dx.doi.org/10.1186/s12984-017-0305-3" + ], + { + "headers": { + "accept": "application/x-bibtex" + } + }, + "@article{Valero_Cuevas_2017,\n\tdoi = {10.1186/s12984-017-0305-3},\n\turl = {https://doi.org/10.1186%2Fs12984-017-0305-3},\n\tyear = 2017,\n\tmonth = {oct},\n\tpublisher = {Springer Nature},\n\tvolume = {14},\n\tnumber = {1},\n\tauthor = {Francisco J. Valero-Cuevas and Marco Santello},\n\ttitle = {On neuromechanical approaches for the study of biological and robotic grasp and manipulation},\n\tjournal = {Journal of {NeuroEngineering} and Rehabilitation}\n}", + 200, + null + ], + [ + [ + "https://dx.doi.org/999999" + ], + { + "headers": { + "accept": "application/x-bibtex" + } + }, + "\n\n\nError: DOI Not Found\n\n\n\n\n \n\n\n\n\n\n\n
\n\"Logo\"\n
\n\n
\n
\n
\n\n\n\n\n\n \n \n \n \n
\"\"\n HOME  | HANDBOOK  | FACTSHEETS  | FAQs  |  RESOURCES  | USERS  | NEWS  | MEMBERS AREA\n
\n\n\n
\n
\n\n\n\n \n\n\n\n\n\n\n\n\n\n\n
\n\"\"\n
\n\n

DOI Not Found

\n\n
 
\n\n\n\n

10/999999

\n\n
 
\n\n\n\n\n

This DOI cannot be found in the DOI System. Possible reasons are:

\n\n\n
    \n\n
  • The DOI is incorrect in your source. Search for the item by name, title, or other metadata using a search engine.
    • \n\n
  • The DOI was copied incorrectly. Check to see that the string includes all the characters before and after the slash and no sentence punctuation marks.
    • \n\n
  • The DOI has not been activated yet. Please try again later, and report the problem if the error continues.
    • \n\n
\n\n\n\n
 
\n\n

You may report this error to the responsible DOI Registration Agency using the form below. Include your email address to receive confirmation and feedback.

\n\n
\n\n
\n\n\n\n\n\n\n\n\n\n\n\n\n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Your Email Address:
Additional Information About the Error:
\n\n

\n\n
\n
\n\n\n\n\n		\"\"
\n\n
 
\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n
\nDOI System Proxy Server Documentation\n
\n\"DOI_disc_logo\"\n\n®, DOI®, DOI.ORG®, and shortDOI® are trademarks of the International DOI Foundation.\n
\n\n\n", + 404, + "404 Client Error: for url: https://dx.doi.org/999999" + ], + [ + [ + "https://export.arxiv.org/api/query?id_list=1312.2021" + ], + {}, + "\n\n \n ArXiv Query: search_query=&id_list=1312.2021&start=0&max_results=10\n http://arxiv.org/api/eXBvi61X4ShcFWF7lwJgRo7KSFk\n 2018-08-14T00:00:00-04:00\n 1\n 0\n 10\n \n http://arxiv.org/abs/1312.2021v1\n 2013-12-06T21:33:40Z\n 2013-12-06T21:33:40Z\n Living with the Wrong Sign\n We describe a UV complete asymptotically fragile Lorentz-invariant theory\nexhibiting superluminal signal propagation. Its low energy effective action\ncontains \"wrong\" sign higher dimensional operators. Nevertheless, the theory\ngives rise to an S-matrix, which is defined at all energies. As expected for a\nnon-local theory, the corresponding scattering amplitudes are not exponentially\nbounded on the physical sheet, but otherwise are healthy. We study some of the\nphysical consequences of this S-matrix.\n\n \n Patrick Cooper\n \n \n Sergei Dubovsky\n \n \n Ali Mohsen\n \n 10.1103/INVALIDDOI.89.084044\n \n Phys. Rev. D 89, 084044 (2014)\n \n \n \n \n \n \n\n", + 200, + null + ], + [ + [ + "https://export.arxiv.org/api/query?id_list=1710.08557" + ], + {}, + "\n\n \n ArXiv Query: search_query=&id_list=1710.08557&start=0&max_results=10\n http://arxiv.org/api/bNB5RPNlUYbELau5YJ4IzaIq1x8\n 2018-08-14T00:00:00-04:00\n 1\n 0\n 10\n \n http://arxiv.org/abs/1710.08557v1\n 2017-10-24T00:21:32Z\n 2017-10-24T00:21:32Z\n On Neuromechanical Approaches for the Study of Biological Grasp and\n Manipulation\n Biological and robotic grasp and manipulation are undeniably similar at the\nlevel of mechanical task performance. However, their underlying fundamental\nbiological vs. engineering mechanisms are, by definition, dramatically\ndifferent and can even be antithetical. Even our approach to each is\ndiametrically opposite: inductive science for the study of biological systems\nvs. engineering synthesis for the design and construction of robotic systems.\nThe past 20 years have seen several conceptual advances in both fields and the\nquest to unify them. Chief among them is the reluctant recognition that their\nunderlying fundamental mechanisms may actually share limited common ground,\nwhile exhibiting many fundamental differences. This recognition is particularly\nliberating because it allows us to resolve and move beyond multiple paradoxes\nand contradictions that arose from the initial reasonable assumption of a large\ncommon ground. Here, we begin by introducing the perspective of neuromechanics,\nwhich emphasizes that real-world behavior emerges from the intimate\ninteractions among the physical structure of the system, the mechanical\nrequirements of a task, the feasible neural control actions to produce it, and\nthe ability of the neuromuscular system to adapt through interactions with the\nenvironment. This allows us to articulate a succinct overview of a few salient\nconceptual paradoxes and contradictions regarding under-determined vs.\nover-determined mechanics, under- vs. over-actuated control, prescribed vs.\nemergent function, learning vs. implementation vs. adaptation, prescriptive vs.\ndescriptive synergies, and optimal vs. habitual performance. We conclude by\npresenting open questions and suggesting directions for future research. We\nhope this frank assessment of the state-of-the-art will encourage and guide\nthese communities to continue to interact and make progress in these important\nareas.\n\n \n Francisco J Valero-Cuevas\n \n \n Marco Santello\n \n 10.1186/s12984-017-0305-3\n \n Journal of NeuroEngineering and Rehabilitation, 2017\n \n \n \n \n \n\n", + 200, + null + ], + [ + [ + "https://export.arxiv.org/api/query?id_list=1710.08557" + ], + {}, + "\n\n \n ArXiv Query: search_query=&id_list=1710.08557&start=0&max_results=10\n http://arxiv.org/api/bNB5RPNlUYbELau5YJ4IzaIq1x8\n 2018-08-14T00:00:00-04:00\n 1\n 0\n 10\n \n http://arxiv.org/abs/1710.08557v1\n 2017-10-24T00:21:32Z\n 2017-10-24T00:21:32Z\n On Neuromechanical Approaches for the Study of Biological Grasp and\n Manipulation\n Biological and robotic grasp and manipulation are undeniably similar at the\nlevel of mechanical task performance. However, their underlying fundamental\nbiological vs. engineering mechanisms are, by definition, dramatically\ndifferent and can even be antithetical. Even our approach to each is\ndiametrically opposite: inductive science for the study of biological systems\nvs. engineering synthesis for the design and construction of robotic systems.\nThe past 20 years have seen several conceptual advances in both fields and the\nquest to unify them. Chief among them is the reluctant recognition that their\nunderlying fundamental mechanisms may actually share limited common ground,\nwhile exhibiting many fundamental differences. This recognition is particularly\nliberating because it allows us to resolve and move beyond multiple paradoxes\nand contradictions that arose from the initial reasonable assumption of a large\ncommon ground. Here, we begin by introducing the perspective of neuromechanics,\nwhich emphasizes that real-world behavior emerges from the intimate\ninteractions among the physical structure of the system, the mechanical\nrequirements of a task, the feasible neural control actions to produce it, and\nthe ability of the neuromuscular system to adapt through interactions with the\nenvironment. This allows us to articulate a succinct overview of a few salient\nconceptual paradoxes and contradictions regarding under-determined vs.\nover-determined mechanics, under- vs. over-actuated control, prescribed vs.\nemergent function, learning vs. implementation vs. adaptation, prescriptive vs.\ndescriptive synergies, and optimal vs. habitual performance. We conclude by\npresenting open questions and suggesting directions for future research. We\nhope this frank assessment of the state-of-the-art will encourage and guide\nthese communities to continue to interact and make progress in these important\nareas.\n\n \n Francisco J Valero-Cuevas\n \n \n Marco Santello\n \n 10.1186/s12984-017-0305-3\n \n Journal of NeuroEngineering and Rehabilitation, 2017\n \n \n \n \n \n\n", + 200, + null + ], + [ + [ + "https://export.arxiv.org/api/query?id_list=INVALIDID" + ], + {}, + "\n\n \n ArXiv Query: search_query=&id_list=INVALIDID\n http://arxiv.org/api//TMQrv7hMz9PNtlUyGrhFtefYFQ\n 2018-08-14T00:00:00-04:00\n 1\n 0\n 1\n \n http://arxiv.org/api/errors#incorrect_id_format_for_INVALIDID\n Error\n incorrect id format for INVALIDID\n 2018-08-14T00:00:00-04:00\n \n \n arXiv api core\n \n \n\n", + 400, + "400 Client Error: Bad Request for url: https://export.arxiv.org/api/query?id_list=INVALIDID" + ], + [ + [ + "https://export.arxiv.org/api/query?id_list=astro-ph/9812133" + ], + {}, + "\n\n \n ArXiv Query: search_query=&id_list=astro-ph/9812133&start=0&max_results=10\n http://arxiv.org/api/SfKekabpSjI/htnxCLpK3q9AsUs\n 2018-08-14T00:00:00-04:00\n 1\n 0\n 10\n \n http://arxiv.org/abs/astro-ph/9812133v1\n 1998-12-08T03:27:34Z\n 1998-12-08T03:27:34Z\n Measurements of Omega and Lambda from 42 High-Redshift Supernovae\n We report measurements of the mass density, Omega_M, and\ncosmological-constant energy density, Omega_Lambda, of the universe based on\nthe analysis of 42 Type Ia supernovae discovered by the Supernova Cosmology\nProject. The magnitude-redshift data for these SNe, at redshifts between 0.18\nand 0.83, are fit jointly with a set of SNe from the Calan/Tololo Supernova\nSurvey, at redshifts below 0.1, to yield values for the cosmological\nparameters. All SN peak magnitudes are standardized using a SN Ia lightcurve\nwidth-luminosity relation. The measurement yields a joint probability\ndistribution of the cosmological parameters that is approximated by the\nrelation 0.8 Omega_M - 0.6 Omega_Lambda ~= -0.2 +/- 0.1 in the region of\ninterest (Omega_M <~ 1.5). For a flat (Omega_M + Omega_Lambda = 1) cosmology we\nfind Omega_M = 0.28{+0.09,-0.08} (1 sigma statistical) {+0.05,-0.04}\n(identified systematics). The data are strongly inconsistent with a Lambda = 0\nflat cosmology, the simplest inflationary universe model. An open, Lambda = 0\ncosmology also does not fit the data well: the data indicate that the\ncosmological constant is non-zero and positive, with a confidence of P(Lambda >\n0) = 99%, including the identified systematic uncertainties. The best-fit age\nof the universe relative to the Hubble time is t_0 = 14.9{+1.4,-1.1} (0.63/h)\nGyr for a flat cosmology. The size of our sample allows us to perform a variety\nof statistical tests to check for possible systematic errors and biases. We\nfind no significant differences in either the host reddening distribution or\nMalmquist bias between the low-redshift Calan/Tololo sample and our\nhigh-redshift sample. The conclusions are robust whether or not a\nwidth-luminosity relation is used to standardize the SN peak magnitudes.\n\n \n S. Perlmutter\n The Supernova Cosmology Project\n \n \n G. Aldering\n The Supernova Cosmology Project\n \n \n G. Goldhaber\n The Supernova Cosmology Project\n \n \n R. A. Knop\n The Supernova Cosmology Project\n \n \n P. Nugent\n The Supernova Cosmology Project\n \n \n P. G. Castro\n The Supernova Cosmology Project\n \n \n S. Deustua\n The Supernova Cosmology Project\n \n \n S. Fabbro\n The Supernova Cosmology Project\n \n \n A. Goobar\n The Supernova Cosmology Project\n \n \n D. E. Groom\n The Supernova Cosmology Project\n \n \n I. M. Hook\n The Supernova Cosmology Project\n \n \n A. G. Kim\n The Supernova Cosmology Project\n \n \n M. Y. Kim\n The Supernova Cosmology Project\n \n \n J. C. Lee\n The Supernova Cosmology Project\n \n \n N. J. Nunes\n The Supernova Cosmology Project\n \n \n R. Pain\n The Supernova Cosmology Project\n \n \n C. R. Pennypacker\n The Supernova Cosmology Project\n \n \n R. Quimby\n The Supernova Cosmology Project\n \n \n C. Lidman\n The Supernova Cosmology Project\n \n \n R. S. Ellis\n The Supernova Cosmology Project\n \n \n M. Irwin\n The Supernova Cosmology Project\n \n \n R. G. McMahon\n The Supernova Cosmology Project\n \n \n P. Ruiz-Lapuente\n The Supernova Cosmology Project\n \n \n N. Walton\n The Supernova Cosmology Project\n \n \n B. Schaefer\n The Supernova Cosmology Project\n \n \n B. J. Boyle\n The Supernova Cosmology Project\n \n \n A. V. Filippenko\n The Supernova Cosmology Project\n \n \n T. Matheson\n The Supernova Cosmology Project\n \n \n A. S. Fruchter\n The Supernova Cosmology Project\n \n \n N. Panagia\n The Supernova Cosmology Project\n \n \n H. J. M. Newberg\n The Supernova Cosmology Project\n \n \n W. J. Couch\n The Supernova Cosmology Project\n \n 10.1086/307221\n \n 21 pages and 10 figures. Accepted for publication in the\n Astrophysical Journal. Individual color figures, supplementary tables, and\n preprint also available at http://www-supernova.lbl.gov/\n Astrophys.J.517:565-586,1999\n \n \n \n \n \n \n \n\n", + 200, + null + ], + [ + [ + "https://export.arxiv.org/api/query?id_list=astro-ph/9812133" + ], + {}, + "\n\n \n ArXiv Query: search_query=&id_list=astro-ph/9812133&start=0&max_results=10\n http://arxiv.org/api/SfKekabpSjI/htnxCLpK3q9AsUs\n 2018-08-14T00:00:00-04:00\n 1\n 0\n 10\n \n http://arxiv.org/abs/astro-ph/9812133v1\n 1998-12-08T03:27:34Z\n 1998-12-08T03:27:34Z\n Measurements of Omega and Lambda from 42 High-Redshift Supernovae\n We report measurements of the mass density, Omega_M, and\ncosmological-constant energy density, Omega_Lambda, of the universe based on\nthe analysis of 42 Type Ia supernovae discovered by the Supernova Cosmology\nProject. The magnitude-redshift data for these SNe, at redshifts between 0.18\nand 0.83, are fit jointly with a set of SNe from the Calan/Tololo Supernova\nSurvey, at redshifts below 0.1, to yield values for the cosmological\nparameters. All SN peak magnitudes are standardized using a SN Ia lightcurve\nwidth-luminosity relation. The measurement yields a joint probability\ndistribution of the cosmological parameters that is approximated by the\nrelation 0.8 Omega_M - 0.6 Omega_Lambda ~= -0.2 +/- 0.1 in the region of\ninterest (Omega_M <~ 1.5). For a flat (Omega_M + Omega_Lambda = 1) cosmology we\nfind Omega_M = 0.28{+0.09,-0.08} (1 sigma statistical) {+0.05,-0.04}\n(identified systematics). The data are strongly inconsistent with a Lambda = 0\nflat cosmology, the simplest inflationary universe model. An open, Lambda = 0\ncosmology also does not fit the data well: the data indicate that the\ncosmological constant is non-zero and positive, with a confidence of P(Lambda >\n0) = 99%, including the identified systematic uncertainties. The best-fit age\nof the universe relative to the Hubble time is t_0 = 14.9{+1.4,-1.1} (0.63/h)\nGyr for a flat cosmology. The size of our sample allows us to perform a variety\nof statistical tests to check for possible systematic errors and biases. We\nfind no significant differences in either the host reddening distribution or\nMalmquist bias between the low-redshift Calan/Tololo sample and our\nhigh-redshift sample. The conclusions are robust whether or not a\nwidth-luminosity relation is used to standardize the SN peak magnitudes.\n\n \n S. Perlmutter\n The Supernova Cosmology Project\n \n \n G. Aldering\n The Supernova Cosmology Project\n \n \n G. Goldhaber\n The Supernova Cosmology Project\n \n \n R. A. Knop\n The Supernova Cosmology Project\n \n \n P. Nugent\n The Supernova Cosmology Project\n \n \n P. G. Castro\n The Supernova Cosmology Project\n \n \n S. Deustua\n The Supernova Cosmology Project\n \n \n S. Fabbro\n The Supernova Cosmology Project\n \n \n A. Goobar\n The Supernova Cosmology Project\n \n \n D. E. Groom\n The Supernova Cosmology Project\n \n \n I. M. Hook\n The Supernova Cosmology Project\n \n \n A. G. Kim\n The Supernova Cosmology Project\n \n \n M. Y. Kim\n The Supernova Cosmology Project\n \n \n J. C. Lee\n The Supernova Cosmology Project\n \n \n N. J. Nunes\n The Supernova Cosmology Project\n \n \n R. Pain\n The Supernova Cosmology Project\n \n \n C. R. Pennypacker\n The Supernova Cosmology Project\n \n \n R. Quimby\n The Supernova Cosmology Project\n \n \n C. Lidman\n The Supernova Cosmology Project\n \n \n R. S. Ellis\n The Supernova Cosmology Project\n \n \n M. Irwin\n The Supernova Cosmology Project\n \n \n R. G. McMahon\n The Supernova Cosmology Project\n \n \n P. Ruiz-Lapuente\n The Supernova Cosmology Project\n \n \n N. Walton\n The Supernova Cosmology Project\n \n \n B. Schaefer\n The Supernova Cosmology Project\n \n \n B. J. Boyle\n The Supernova Cosmology Project\n \n \n A. V. Filippenko\n The Supernova Cosmology Project\n \n \n T. Matheson\n The Supernova Cosmology Project\n \n \n A. S. Fruchter\n The Supernova Cosmology Project\n \n \n N. Panagia\n The Supernova Cosmology Project\n \n \n H. J. M. Newberg\n The Supernova Cosmology Project\n \n \n W. J. Couch\n The Supernova Cosmology Project\n \n 10.1086/307221\n \n 21 pages and 10 figures. Accepted for publication in the\n Astrophysical Journal. Individual color figures, supplementary tables, and\n preprint also available at http://www-supernova.lbl.gov/\n Astrophys.J.517:565-586,1999\n \n \n \n \n \n \n \n\n", + 200, + null + ], + [ + [ + "https://export.arxiv.org/api/query?id_list=math/0211159" + ], + {}, + "\n\n \n ArXiv Query: search_query=&id_list=math/0211159&start=0&max_results=10\n http://arxiv.org/api/4nVRTcAL5Np4oaGFQnqVPG0+c5k\n 2018-08-14T00:00:00-04:00\n 1\n 0\n 10\n \n http://arxiv.org/abs/math/0211159v1\n 2002-11-11T16:11:49Z\n 2002-11-11T16:11:49Z\n The entropy formula for the Ricci flow and its geometric applications\n We present a monotonic expression for the Ricci flow, valid in all dimensions\nand without curvature assumptions. It is interpreted as an entropy for a\ncertain canonical ensemble. Several geometric applications are given. In\nparticular, (1) Ricci flow, considered on the space of riemannian metrics\nmodulo diffeomorphism and scaling, has no nontrivial periodic orbits (that is,\nother than fixed points); (2) In a region, where singularity is forming in\nfinite time, the injectivity radius is controlled by the curvature; (3) Ricci\nflow can not quickly turn an almost euclidean region into a very curved one, no\nmatter what happens far away. We also verify several assertions related to\nRichard Hamilton's program for the proof of Thurston geometrization conjecture\nfor closed three-manifolds, and give a sketch of an eclectic proof of this\nconjecture, making use of earlier results on collapsing with local lower\ncurvature bound.\n\n \n Grisha Perelman\n \n 39 pages\n \n \n \n \n \n \n\n", + 200, + null + ], + [ + [ + "https://www.ottobib.com/isbn/9782081336742/bibtex" + ], + { + "headers": null + }, + "\n\n\n\n \n \n \n \n \n \n \n \n\n \n\n Bibliography and Works Cited Generator for MLA, APA, Wikipedia and Bibtex - OttoBib.com \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n
\n \n
\n
\n

OttoBib

\n

bibtex Format

\n
\n
\n\n
\n
\n \n
\n
\n
\n\n
\n

\n NOTE!\n
\n

    \n
  1. Verify accuracy of the data (particularly authors)
    2. \n
  2. Remember to do a HANGING INDENT (something I cannot do with HTML here)
    3. \n
  3. I strive for accuracy of the citations but you should treat this tool as a starting point in your works cited, because you still need to look it over.
    4. \n
\n

\n
\n
\n\n
\n \n\n \n", + 200, + null + ], + [ + [ + "https://www.ottobib.com/isbn/9782081336742/bibtex" + ], + { + "headers": null + }, + "\n\n\n\n \n \n \n \n \n \n \n \n\n \n\n Bibliography and Works Cited Generator for MLA, APA, Wikipedia and Bibtex - OttoBib.com \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n
\n \n
\n
\n

OttoBib

\n

bibtex Format

\n
\n
\n\n
\n
\n \n
\n
\n
\n\n
\n

\n NOTE!\n
\n

    \n
  1. Verify accuracy of the data (particularly authors)
    2. \n
  2. Remember to do a HANGING INDENT (something I cannot do with HTML here)
    3. \n
  3. I strive for accuracy of the citations but you should treat this tool as a starting point in your works cited, because you still need to look it over.
    4. \n
\n

\n
\n
\n\n
\n \n\n \n", + 200, + null + ], + [ + [ + "https://www.ottobib.com/isbn/9999999999999/bibtex" + ], + { + "headers": null + }, + "\n\n\n\n \n \n \n \n \n \n \n \n\n \n\n Bibliography and Works Cited Generator for MLA, APA, Wikipedia and Bibtex - OttoBib.com \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n
\n \n
\n\t

OttoBib

\n\t

Make a bibliography or works cited with just an ISBN. It's free, easy and FAST

\n\t

\n\t about ottobib and history\n\t \n\t

\n\t \n

\n\t
\n
\n\n
\n
\n\n
No Results for 9999999999999
\n
\n

\n Enter ISBN of book(s) - separate with commas (,)\n

\n \n read more about ISBN on Wikipedia\n \n
\n \n \n \n \n \n\n
\n \n
\n
\n\n
\n
\n\n

Did OttoBib help you? Then help us and Like us on Facebook!

\n
\n
\n\n\n
\n\n\n\n\n
\n \n\n \n", + 200, + null + ] +] \ No newline at end of file diff --git a/tests/test_endecoder.py b/tests/test_endecoder.py index a6e3043..11d5306 100644 --- a/tests/test_endecoder.py +++ b/tests/test_endecoder.py @@ -23,6 +23,11 @@ def compare_yaml_str(s1, s2): class TestEnDecode(unittest.TestCase): + def test_decode_emptystring(self): + decoder = endecoder.EnDecoder() + with self.assertRaises(decoder.BibDecodingError): + entry = decoder.decode_bibdata('') + def test_encode_bibtex_is_unicode(self): decoder = endecoder.EnDecoder() entry = decoder.decode_bibdata(bibtex_raw0) @@ -52,6 +57,18 @@ class TestEnDecode(unittest.TestCase): self.assertEqual(bibraw1, bibraw2) + def test_endecode_bibtex_BOM(self): + """Test that bibtexparser if fine with BOM-prefixed data""" + decoder = endecoder.EnDecoder() + bom_str = '\ufeff' + + entry_1 = decoder.decode_bibdata(bibtex_raw0) + bibraw_1 = decoder.encode_bibdata(entry_1) + entry_2 = decoder.decode_bibdata(bom_str + bibraw_1) + bibraw_2 = decoder.encode_bibdata(entry_2) + + self.assertEqual(bibraw_1, bibraw_2) + def test_endecode_bibtex_converts_month_string(self): """Test if `month=dec` is correctly recognized and transformed into `month={December}`""" diff --git a/tests/test_events.py b/tests/test_events.py index 32e0a4c..9834cfb 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -7,7 +7,7 @@ from pubs.events import Event _output = None -class TestEvent(Event): +class StringEvent(Event): def __init__(self, string): self.string = string @@ -34,20 +34,20 @@ class SpecificInfo(Info): self.specific = specific -@TestEvent.listen(12, 15) -def display(TestEventInstance, nb1, nb2): +@StringEvent.listen(12, 15) +def display(StringEventInstance, nb1, nb2): _output.append("%s %s %s" - % (TestEventInstance.string, nb1, nb2)) + % (StringEventInstance.string, nb1, nb2)) -@TestEvent.listen() -def hello_word(TestEventInstance): +@StringEvent.listen() +def hello_word(StringEventInstance): _output.append('Helloword') -@TestEvent.listen() -def print_it(TestEventInstance): - TestEventInstance.print_one() +@StringEvent.listen() +def print_it(StringEventInstance): + StringEventInstance.print_one() @AddEvent.listen() @@ -56,7 +56,7 @@ def do_it(AddEventInstance): @Info.listen() -def test_info_instance(infoevent): +def collect_info_instance(infoevent): _output.append(infoevent.info) if isinstance(infoevent, SpecificInfo): _output.append(infoevent.specific) @@ -68,9 +68,9 @@ class TestEvents(unittest.TestCase): global _output _output = [] - def test_listen_TestEvent(self): + def test_listen_StringEvent(self): # using the callback system - myevent = TestEvent('abcdefghijklmnopqrstuvwxyz') + myevent = StringEvent('abcdefghijklmnopqrstuvwxyz') myevent.send() # this one call three function correct = ['abcdefghijklmnopqrstuvwxyz 12 15', 'Helloword', diff --git a/tests/test_usecase.py b/tests/test_usecase.py index a2f8f2f..62ce138 100644 --- a/tests/test_usecase.py +++ b/tests/test_usecase.py @@ -982,6 +982,21 @@ class TestUsecase(DataCommandTestCase): self.assertFalse(os.path.isfile(self.default_conf_path)) self.assertTrue(os.path.isfile(alt_conf)) + def test_statistics(self): + cmds = ['pubs init', + 'pubs add data/pagerank.bib', + 'pubs add -d data/turing-mind-1950.pdf data/turing1950.bib', + 'pubs add data/martius.bib', + 'pubs add data/10.1371%2Fjournal.pone.0038236.bib', + 'pubs tag Page99 A+B', + 'pubs tag turing1950computing C', + 'pubs statistics', + ] + out = self.execute_cmds(cmds) + lines = out[-1].splitlines() + self.assertEqual(lines[0], 'Repository statistics:') + self.assertEqual(lines[1], 'Total papers: 4, 1 (25%) have a document attached') + self.assertEqual(lines[2], 'Total tags: 3, 2 (50%) of papers have at least one tag') @ddt.ddt