From efb91b1ff42deac5c9946cc9ff7472452087aeb8 Mon Sep 17 00:00:00 2001 From: "Fabien C. Y. Benureau" Date: Mon, 27 Aug 2018 11:47:25 +0900 Subject: [PATCH 1/4] hotfix for #165 --- pubs/apis.py | 20 +++++++---- pubs/commands/add_cmd.py | 4 +-- pubs/commands/conf_cmd.py | 2 +- pubs/databroker.py | 3 +- pubs/endecoder.py | 2 +- pubs/uis.py | 3 +- pubs/utils.py | 2 +- tests/test_apis.py | 73 ++++++++++++++++----------------------- 8 files changed, 52 insertions(+), 57 deletions(-) diff --git a/pubs/apis.py b/pubs/apis.py index 070d8fc..aef5c5d 100644 --- a/pubs/apis.py +++ b/pubs/apis.py @@ -8,6 +8,8 @@ from bibtexparser.bibdatabase import BibDatabase import feedparser from bs4 import BeautifulSoup +from . import endecoder + class ReferenceNotFoundError(Exception): pass @@ -34,7 +36,7 @@ def get_bibentry_from_api(id_str, id_type, try_doi=True, ui=None): Raises: ValueError: if `id_type` is not one of `doi`, `isbn`, or `arxiv`. - apis.ReferenceNotFoundException: if no valid reference could be found. + apis.ReferenceNotFoundError: if no valid reference could be found. """ id_fns = { @@ -43,13 +45,14 @@ def get_bibentry_from_api(id_str, id_type, try_doi=True, ui=None): 'arxiv': arxiv2bibtex, } + id_type = id_type.lower() if id_type not in id_fns.keys(): raise ValueError('id_type must be one of `doi`, `isbn`, or `arxiv`.') bibentry_raw = id_fns[id_type](id_str, try_doi=try_doi, ui=ui) - endecoder.EnDecoder().decode_bibdata(bibentry_raw) + bibentry = endecoder.EnDecoder().decode_bibdata(bibentry_raw) if bibentry is None: - raise ReferenceNotFoundException( + raise ReferenceNotFoundError( 'invalid {} {} or unable to retrieve bibfile from it.'.format(id_type, id_str)) return bibentry @@ -72,7 +75,7 @@ def _get_request(url, headers=None): ## DOI support def doi2bibtex(doi, **kwargs): - """Return a bibtex string of metadata from a DOI""" + """Return a bibtex string from a DOI""" url = 'https://dx.doi.org/{}'.format(doi) headers = {'accept': 'application/x-bibtex'} @@ -87,13 +90,16 @@ def doi2bibtex(doi, **kwargs): def isbn2bibtex(isbn, **kwargs): - """Return a bibtex string of metadata from an ISBN""" + """Return a bibtex string from an ISBN""" url = 'https://www.ottobib.com/isbn/{}/bibtex'.format(isbn) r = _get_request(url) soup = BeautifulSoup(r.text, "html.parser") citation = soup.find("textarea").text + if len(citation) == 0: + raise ReferenceNotFoundError("No information could be retrieved about ISBN '{}'. ISBN databases are notoriously incomplete. If the ISBN is correct, you may have to enter information manually by invoking 'pubs add' without the '-I' argument.".format(isbn)) + return citation # Note: apparently ottobib.com uses caracter modifiers for accents instead @@ -106,7 +112,7 @@ _months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] def _is_arxiv_oldstyle(arxiv_id): - return re.match(r"(arXiv\:)?[a-z\-]+\/[0-9]+(v[0-9]+)?", arxiv_id) is not None + return re.match(r"(arxiv\:)?[a-z\-]+\/[0-9]+(v[0-9]+)?", arxiv_id.lower()) is not None def _extract_arxiv_id(entry): pattern = r"http[s]?://arxiv.org/abs/(?P.+)" @@ -114,7 +120,7 @@ def _extract_arxiv_id(entry): def arxiv2bibtex(arxiv_id, try_doi=True, ui=None): - """Return a bibtex string of metadata from an arXiv ID + """Return a bibtex string from an arXiv ID :param arxiv_id: arXiv id, with or without the `arXiv:` prefix and version suffix (e.g. `v1`). Old an new style are accepted. Here are diff --git a/pubs/commands/add_cmd.py b/pubs/commands/add_cmd.py index d02e06e..12b87cd 100644 --- a/pubs/commands/add_cmd.py +++ b/pubs/commands/add_cmd.py @@ -102,8 +102,8 @@ def command(conf, args): # TODO offer to confirm/change citekey elif args.arxiv is not None: bibentry = apis.get_bibentry_from_api(args.arxiv, 'arxiv', ui=ui) - except apis.ReferenceNotFoundException as e: - ui.error(e.message) + except apis.ReferenceNotFoundError as e: + ui.error(str(e)) ui.exit(1) else: bibentry_raw = content.get_content(bibfile, ui=ui) diff --git a/pubs/commands/conf_cmd.py b/pubs/commands/conf_cmd.py index 3202e3c..cd3e9e0 100644 --- a/pubs/commands/conf_cmd.py +++ b/pubs/commands/conf_cmd.py @@ -25,7 +25,7 @@ def command(conf, args): ui.message('The configuration file was updated.') break except AssertionError as e: # TODO better error message - ui.error('Error reading the modified configuration file [' + e.message + '].') + ui.error('Error reading the modified configuration file [' + str(e) + '].') options = ['edit_again', 'abort'] choice = options[ui.input_choice( options, ['e', 'a'], diff --git a/pubs/databroker.py b/pubs/databroker.py index 23e8b91..9e78237 100644 --- a/pubs/databroker.py +++ b/pubs/databroker.py @@ -48,7 +48,8 @@ class DataBroker(object): try: return self.endecoder.decode_bibdata(bibdata_raw) except self.endecoder.BibDecodingError as e: - e.message = "Unable to decode bibtex for paper {}.".format(citekey) + # QUESTION: do we really want to obscure a more precise error message here? + e.args = "Unable to decode bibtex for paper {}.".format(citekey) raise e def push_metadata(self, citekey, metadata): diff --git a/pubs/endecoder.py b/pubs/endecoder.py index c280460..abac0f1 100644 --- a/pubs/endecoder.py +++ b/pubs/endecoder.py @@ -80,7 +80,7 @@ class EnDecoder(object): :param error_msg: specific message about what went wrong :param bibdata: the data that was unsuccessfully decoded. """ - super(Exception, self).__init__(error_msg) # make `str(self)` work. + super(Exception, self).__init__(error_msg) # make `str(self)` work. self.data = bibdata bwriter = bp.bwriter.BibTexWriter() diff --git a/pubs/uis.py b/pubs/uis.py index 9a39b32..0ae96e5 100644 --- a/pubs/uis.py +++ b/pubs/uis.py @@ -6,6 +6,7 @@ import shlex import locale import codecs import tempfile +import traceback import subprocess from . import color @@ -87,7 +88,7 @@ class PrintUI(object): if DEBUG_ALL_TRACES: # if an exception has been raised, print the trace. if sys.exc_info()[0] is not None: - traceback.print_exception(*sys.exc_info) + traceback.print_exception(*sys.exc_info()) def exit(self, error_code=1): sys.exit(error_code) diff --git a/pubs/utils.py b/pubs/utils.py index 045deda..eda7480 100644 --- a/pubs/utils.py +++ b/pubs/utils.py @@ -82,7 +82,7 @@ def standardize_doi(doi): match = doi_pattern.search(doi) if not match: - raise ValueError("Not a valid doi: %s", doi) + raise ValueError("Not a valid doi: {}".format(doi)) new_doi = match.group(0) return new_doi diff --git a/tests/test_apis.py b/tests/test_apis.py index b870f63..0a7feb9 100644 --- a/tests/test_apis.py +++ b/tests/test_apis.py @@ -9,34 +9,29 @@ import mock import dotdot from pubs.p3 import ustr -from pubs.endecoder import EnDecoder -from pubs.apis import ReferenceNotFoundError, arxiv2bibtex, doi2bibtex, isbn2bibtex, _is_arxiv_oldstyle, _extract_arxiv_id - from pubs import apis +from pubs.apis import _is_arxiv_oldstyle, _extract_arxiv_id import mock_requests class APITests(unittest.TestCase): - - def setUp(self): - self.endecoder = EnDecoder() + pass class TestDOI2Bibtex(APITests): @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_unicode(self, reqget): - bib = doi2bibtex('10.1007/BF01700692') + bib = apis.doi2bibtex('10.1007/BF01700692') self.assertIsInstance(bib, ustr) self.assertIn('Kurt Gödel', bib) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_parses_to_bibtex(self, reqget): - bib = doi2bibtex('10.1007/BF01700692') - b = self.endecoder.decode_bibdata(bib) - self.assertEqual(len(b), 1) - entry = b[list(b)[0]] + bib = apis.get_bibentry_from_api('10.1007/BF01700692', 'DOI') + self.assertEqual(len(bib), 1) + entry = bib[list(bib)[0]] self.assertEqual(entry['author'][0], 'Gödel, Kurt') self.assertEqual(entry['title'], 'Über formal unentscheidbare Sätze der Principia ' @@ -45,59 +40,54 @@ class TestDOI2Bibtex(APITests): @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_retrieve_fails_on_incorrect_DOI(self, reqget): with self.assertRaises(apis.ReferenceNotFoundError): - doi2bibtex('999999') + apis.get_bibentry_from_api('999999', 'doi') class TestISBN2Bibtex(APITests): @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_unicode(self, reqget): - bib = isbn2bibtex('9782081336742') + bib = apis.isbn2bibtex('9782081336742') self.assertIsInstance(bib, ustr) self.assertIn('Poincaré, Henri', bib) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_parses_to_bibtex(self, reqget): - bib = isbn2bibtex('9782081336742') - b = self.endecoder.decode_bibdata(bib) - self.assertEqual(len(b), 1) - entry = b[list(b)[0]] + bib = apis.get_bibentry_from_api('9782081336742', 'ISBN') + self.assertEqual(len(bib), 1) + entry = bib[list(bib)[0]] self.assertEqual(entry['author'][0], 'Poincaré, Henri') self.assertEqual(entry['title'], 'La science et l\'hypothèse') @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_retrieve_fails_on_incorrect_ISBN(self, reqget): - bib = isbn2bibtex('9' * 13) - with self.assertRaises(EnDecoder.BibDecodingError): - self.endecoder.decode_bibdata(bib) + with self.assertRaises(apis.ReferenceNotFoundError): + apis.get_bibentry_from_api('9' * 13, 'isbn') class TestArxiv2Bibtex(APITests): @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_new_style(self, reqget): - bib = arxiv2bibtex('astro-ph/9812133') - b = self.endecoder.decode_bibdata(bib) - self.assertEqual(len(b), 1) - entry = b[list(b)[0]] + bib = apis.get_bibentry_from_api('astro-ph/9812133', 'arXiv') + self.assertEqual(len(bib), 1) + entry = bib[list(bib)[0]] self.assertEqual(entry['author'][0], 'Perlmutter, S.') self.assertEqual(entry['year'], '1999') @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_parses_to_bibtex_with_doi(self, reqget): - bib = arxiv2bibtex('astro-ph/9812133') - b = self.endecoder.decode_bibdata(bib) - self.assertEqual(len(b), 1) - entry = b[list(b)[0]] + bib = apis.get_bibentry_from_api('astro-ph/9812133', 'arxiv') + self.assertEqual(len(bib), 1) + entry = bib[list(bib)[0]] self.assertEqual(entry['author'][0], 'Perlmutter, S.') self.assertEqual(entry['year'], '1999') @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_parses_to_bibtex_without_doi(self, reqget): - bib = arxiv2bibtex('math/0211159') - b = self.endecoder.decode_bibdata(bib) - self.assertEqual(len(b), 1) - entry = b[list(b)[0]] + bib = apis.get_bibentry_from_api('math/0211159', 'ARXIV') + self.assertEqual(len(bib), 1) + entry = bib[list(bib)[0]] self.assertEqual(entry['author'][0], 'Perelman, Grisha') self.assertEqual(entry['year'], '2002') self.assertEqual( @@ -106,31 +96,28 @@ class TestArxiv2Bibtex(APITests): @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_arxiv_wrong_id(self, reqget): - with self.assertRaises(ReferenceNotFoundError): - bib = arxiv2bibtex('INVALIDID') + with self.assertRaises(apis.ReferenceNotFoundError): + bib = apis.get_bibentry_from_api('INVALIDID', 'arxiv') @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_arxiv_wrong_doi(self, reqget): - bib = arxiv2bibtex('1312.2021') - b = self.endecoder.decode_bibdata(bib) - entry = b[list(b)[0]] + bib = apis.get_bibentry_from_api('1312.2021', 'arXiv') + entry = bib[list(bib)[0]] self.assertEqual(entry['arxiv_doi'], '10.1103/INVALIDDOI.89.084044') @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_arxiv_good_doi(self, reqget): """Get the DOI bibtex instead of the arXiv one if possible""" - bib = arxiv2bibtex('1710.08557') - b = self.endecoder.decode_bibdata(bib) - entry = b[list(b)[0]] + bib = apis.get_bibentry_from_api('1710.08557', 'arXiv') + entry = bib[list(bib)[0]] self.assertTrue(not 'arxiv_doi' in entry) self.assertEqual(entry['doi'], '10.1186/s12984-017-0305-3') self.assertEqual(entry['title'].lower(), 'on neuromechanical approaches for the study of biological and robotic grasp and manipulation') @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_arxiv_good_doi_force_arxiv(self, reqget): - bib = arxiv2bibtex('1710.08557', try_doi=False) - b = self.endecoder.decode_bibdata(bib) - entry = b[list(b)[0]] + bib = apis.get_bibentry_from_api('1710.08557', 'arXiv', try_doi=False) + entry = bib[list(bib)[0]] self.assertEqual(entry['arxiv_doi'], '10.1186/s12984-017-0305-3') self.assertEqual(entry['title'].lower(), 'on neuromechanical approaches for the study of biological grasp and\nmanipulation') From a58f1b1d5e0080b756cb87616745d294f13035df Mon Sep 17 00:00:00 2001 From: "Fabien C. Y. Benureau" Date: Mon, 27 Aug 2018 11:52:52 +0900 Subject: [PATCH 2/4] hotfix for #164 --- pubs/commands/statistics_cmd.py | 32 ++++++++++++++++++-------------- tests/test_usecase.py | 3 +++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pubs/commands/statistics_cmd.py b/pubs/commands/statistics_cmd.py index 5132010..e3b6583 100644 --- a/pubs/commands/statistics_cmd.py +++ b/pubs/commands/statistics_cmd.py @@ -16,18 +16,22 @@ def command(conf, args): papers = list(rp.all_papers()) paper_count = len(papers) - doc_count = sum([0 if p.docpath is None else 1 for p in papers]) - tag_count = len(list(rp.get_tags())) - papers_with_tags = sum([0 if p.tags else 1 for p in papers]) + if paper_count == 0: + ui.message('Your pubs repository is empty.') - ui.message(color.dye_out('Repository statistics:', 'bold')) - ui.message('Total papers: {}, {} ({}) have a document attached'.format( - color.dye_out('{:d}'.format(paper_count), 'bgreen'), - color.dye_out('{:d}'.format(doc_count), 'bold'), - '{:.0f}%'.format(100. * doc_count / paper_count), - )) - ui.message('Total tags: {}, {} ({}) of papers have at least one tag'.format( - color.dye_out('{:d}'.format(tag_count), 'bgreen'), - color.dye_out('{:d}'.format(papers_with_tags), 'bold'), - '{:.0f}%'.format(100. * papers_with_tags / paper_count), - )) + else: + doc_count = sum([0 if p.docpath is None else 1 for p in papers]) + tag_count = len(list(rp.get_tags())) + papers_with_tags = sum([0 if p.tags else 1 for p in papers]) + + ui.message(color.dye_out('Repository statistics:', 'bold')) + ui.message('Total papers: {}, {} ({}) have a document attached'.format( + color.dye_out('{:d}'.format(paper_count), 'bgreen'), + color.dye_out('{:d}'.format(doc_count), 'bold'), + '{:.0f}%'.format(100. * doc_count / paper_count), + )) + ui.message('Total tags: {}, {} ({}) of papers have at least one tag'.format( + color.dye_out('{:d}'.format(tag_count), 'bgreen'), + color.dye_out('{:d}'.format(papers_with_tags), 'bold'), + '{:.0f}%'.format(100. * papers_with_tags / paper_count), + )) diff --git a/tests/test_usecase.py b/tests/test_usecase.py index 62ce138..43083cd 100644 --- a/tests/test_usecase.py +++ b/tests/test_usecase.py @@ -984,6 +984,7 @@ class TestUsecase(DataCommandTestCase): def test_statistics(self): cmds = ['pubs init', + 'pubs statistics', 'pubs add data/pagerank.bib', 'pubs add -d data/turing-mind-1950.pdf data/turing1950.bib', 'pubs add data/martius.bib', @@ -993,6 +994,8 @@ class TestUsecase(DataCommandTestCase): 'pubs statistics', ] out = self.execute_cmds(cmds) + lines = out[1].splitlines() + self.assertEqual(lines[0], 'Your pubs repository is empty.') lines = out[-1].splitlines() self.assertEqual(lines[0], 'Repository statistics:') self.assertEqual(lines[1], 'Total papers: 4, 1 (25%) have a document attached') From 51fa0de520a4f7955040ce0e9d64cf26272f28ba Mon Sep 17 00:00:00 2001 From: "Fabien C. Y. Benureau" Date: Mon, 27 Aug 2018 12:05:57 +0900 Subject: [PATCH 3/4] remove error, improve description --- pubs/pubs_cmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pubs/pubs_cmd.py b/pubs/pubs_cmd.py index 258cb85..740c709 100644 --- a/pubs/pubs_cmd.py +++ b/pubs/pubs_cmd.py @@ -69,10 +69,11 @@ def execute(raw_args=sys.argv): uis.init_ui(conf, force_colors=top_args.force_colors) ui = uis.get_ui() - parser = p3.ArgumentParser(description="research papers repository", + desc = 'Pubs: your bibliography on the command line.\nVisit https://github.com/pubs/pubs for more information.' + parser = p3.ArgumentParser(description=desc, prog="pubs", add_help=True) parser.add_argument('--version', action='version', version=__version__) - subparsers = parser.add_subparsers(title="valid commands", dest="command") + subparsers = parser.add_subparsers(title="commands", dest="command") # Populate the parser with core commands for cmd_name, cmd_mod in CORE_CMDS.items(): @@ -91,7 +92,6 @@ def execute(raw_args=sys.argv): # if no command, print help and exit peacefully (as '--help' does) args = parser.parse_args(remaining_args) if not args.command: - ui.error("Too few arguments!\n") parser.print_help(file=sys.stderr) sys.exit(2) From c642169ec75ad55db900641bcaf9a633b944a29b Mon Sep 17 00:00:00 2001 From: "Fabien C. Y. Benureau" Date: Mon, 27 Aug 2018 12:14:16 +0900 Subject: [PATCH 4/4] add long_description to setup.py for #166. --- setup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup.py b/setup.py index 50058b5..aa0cded 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import os import unittest from setuptools import setup @@ -6,6 +7,10 @@ from setuptools import setup with open('pubs/version.py') as f: exec(f.read()) # defines __version__ +here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, 'readme.md'), 'r') as fd: + long_description = fd.read() + def pubs_test_suite(): test_loader = unittest.TestLoader() test_suite = test_loader.discover('tests', pattern='test_*.py') @@ -20,6 +25,9 @@ setup( url='https://github.com/pubs/pubs', description='command-line scientific bibliography manager', + long_description=long_description, + long_description_content_type='text/markdown', + packages=['pubs', 'pubs.config', 'pubs.commands',