aselimov/pubs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

hotfix for #165

Browse Source
This commit is contained in:
Fabien C. Y. Benureau 2018-08-27 11:47:25 +09:00
parent 3fb951963c
commit efb91b1ff4
No known key found for this signature in database
GPG Key ID: C3FB5E831A249A9A
8 changed files with 52 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pubs/apis.py
View File

@ -8,6 +8,8 @@ from bibtexparser.bibdatabase import BibDatabase
import feedparser import feedparser
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from . import endecoder
class ReferenceNotFoundError(Exception): class ReferenceNotFoundError(Exception):
pass pass
@ -34,7 +36,7 @@ def get_bibentry_from_api(id_str, id_type, try_doi=True, ui=None):
Raises: Raises:
ValueError: if `id_type` is not one of `doi`, `isbn`, or `arxiv`. ValueError: if `id_type` is not one of `doi`, `isbn`, or `arxiv`.
apis.ReferenceNotFoundException: if no valid reference could be found. apis.ReferenceNotFoundError: if no valid reference could be found.
""" """
id_fns = { id_fns = {
@ -43,13 +45,14 @@ def get_bibentry_from_api(id_str, id_type, try_doi=True, ui=None):
'arxiv': arxiv2bibtex, 'arxiv': arxiv2bibtex,
} }
id_type = id_type.lower()
if id_type not in id_fns.keys(): if id_type not in id_fns.keys():
raise ValueError('id_type must be one of `doi`, `isbn`, or `arxiv`.') raise ValueError('id_type must be one of `doi`, `isbn`, or `arxiv`.')
bibentry_raw = id_fns[id_type](id_str, try_doi=try_doi, ui=ui) bibentry_raw = id_fns[id_type](id_str, try_doi=try_doi, ui=ui)
endecoder.EnDecoder().decode_bibdata(bibentry_raw) bibentry = endecoder.EnDecoder().decode_bibdata(bibentry_raw)
if bibentry is None: if bibentry is None:
raise ReferenceNotFoundException( raise ReferenceNotFoundError(
'invalid {} {} or unable to retrieve bibfile from it.'.format(id_type, id_str)) 'invalid {} {} or unable to retrieve bibfile from it.'.format(id_type, id_str))
return bibentry return bibentry
@ -72,7 +75,7 @@ def _get_request(url, headers=None):
## DOI support ## DOI support
def doi2bibtex(doi, **kwargs): def doi2bibtex(doi, **kwargs):
"""Return a bibtex string of metadata from a DOI""" """Return a bibtex string from a DOI"""
url = 'https://dx.doi.org/{}'.format(doi) url = 'https://dx.doi.org/{}'.format(doi)
headers = {'accept': 'application/x-bibtex'} headers = {'accept': 'application/x-bibtex'}
@ -87,13 +90,16 @@ def doi2bibtex(doi, **kwargs):
def isbn2bibtex(isbn, **kwargs): def isbn2bibtex(isbn, **kwargs):
"""Return a bibtex string of metadata from an ISBN""" """Return a bibtex string from an ISBN"""
url = 'https://www.ottobib.com/isbn/{}/bibtex'.format(isbn) url = 'https://www.ottobib.com/isbn/{}/bibtex'.format(isbn)
r = _get_request(url) r = _get_request(url)
soup = BeautifulSoup(r.text, "html.parser") soup = BeautifulSoup(r.text, "html.parser")
citation = soup.find("textarea").text citation = soup.find("textarea").text
if len(citation) == 0:
raise ReferenceNotFoundError("No information could be retrieved about ISBN '{}'. ISBN databases are notoriously incomplete. If the ISBN is correct, you may have to enter information manually by invoking 'pubs add' without the '-I' argument.".format(isbn))
return citation return citation
# Note: apparently ottobib.com uses caracter modifiers for accents instead # Note: apparently ottobib.com uses caracter modifiers for accents instead
@ -106,7 +112,7 @@ _months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun',
'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
def _is_arxiv_oldstyle(arxiv_id): def _is_arxiv_oldstyle(arxiv_id):
return re.match(r"(arXiv\:)?[a-z\-]+\/[0-9]+(v[0-9]+)?", arxiv_id) is not None return re.match(r"(arxiv\:)?[a-z\-]+\/[0-9]+(v[0-9]+)?", arxiv_id.lower()) is not None
def _extract_arxiv_id(entry): def _extract_arxiv_id(entry):
pattern = r"http[s]?://arxiv.org/abs/(?P<entry_id>.+)" pattern = r"http[s]?://arxiv.org/abs/(?P<entry_id>.+)"
@ -114,7 +120,7 @@ def _extract_arxiv_id(entry):
def arxiv2bibtex(arxiv_id, try_doi=True, ui=None): def arxiv2bibtex(arxiv_id, try_doi=True, ui=None):
"""Return a bibtex string of metadata from an arXiv ID """Return a bibtex string from an arXiv ID
:param arxiv_id: arXiv id, with or without the `arXiv:` prefix and version :param arxiv_id: arXiv id, with or without the `arXiv:` prefix and version
suffix (e.g. `v1`). Old an new style are accepted. Here are suffix (e.g. `v1`). Old an new style are accepted. Here are

4
pubs/commands/add_cmd.py
View File

@ -102,8 +102,8 @@ def command(conf, args):
# TODO offer to confirm/change citekey # TODO offer to confirm/change citekey
elif args.arxiv is not None: elif args.arxiv is not None:
bibentry = apis.get_bibentry_from_api(args.arxiv, 'arxiv', ui=ui) bibentry = apis.get_bibentry_from_api(args.arxiv, 'arxiv', ui=ui)
except apis.ReferenceNotFoundException as e: except apis.ReferenceNotFoundError as e:
ui.error(e.message) ui.error(str(e))
ui.exit(1) ui.exit(1)
else: else:
bibentry_raw = content.get_content(bibfile, ui=ui) bibentry_raw = content.get_content(bibfile, ui=ui)

2
pubs/commands/conf_cmd.py
View File

@ -25,7 +25,7 @@ def command(conf, args):
ui.message('The configuration file was updated.') ui.message('The configuration file was updated.')
break break
except AssertionError as e: # TODO better error message except AssertionError as e: # TODO better error message
ui.error('Error reading the modified configuration file [' + e.message + '].') ui.error('Error reading the modified configuration file [' + str(e) + '].')
options = ['edit_again', 'abort'] options = ['edit_again', 'abort']
choice = options[ui.input_choice( choice = options[ui.input_choice(
options, ['e', 'a'], options, ['e', 'a'],

3
pubs/databroker.py
View File

@ -48,7 +48,8 @@ class DataBroker(object):
try: try:
return self.endecoder.decode_bibdata(bibdata_raw) return self.endecoder.decode_bibdata(bibdata_raw)
except self.endecoder.BibDecodingError as e: except self.endecoder.BibDecodingError as e:
e.message = "Unable to decode bibtex for paper {}.".format(citekey) # QUESTION: do we really want to obscure a more precise error message here?
e.args = "Unable to decode bibtex for paper {}.".format(citekey)
raise e raise e
def push_metadata(self, citekey, metadata): def push_metadata(self, citekey, metadata):

3
pubs/uis.py
View File

@ -6,6 +6,7 @@ import shlex
import locale import locale
import codecs import codecs
import tempfile import tempfile
import traceback
import subprocess import subprocess
from . import color from . import color
@ -87,7 +88,7 @@ class PrintUI(object):
if DEBUG_ALL_TRACES: # if an exception has been raised, print the trace. if DEBUG_ALL_TRACES: # if an exception has been raised, print the trace.
if sys.exc_info()[0] is not None: if sys.exc_info()[0] is not None:
traceback.print_exception(*sys.exc_info) traceback.print_exception(*sys.exc_info())
def exit(self, error_code=1): def exit(self, error_code=1):
sys.exit(error_code) sys.exit(error_code)

2
pubs/utils.py
View File

@ -82,7 +82,7 @@ def standardize_doi(doi):
match = doi_pattern.search(doi) match = doi_pattern.search(doi)
if not match: if not match:
raise ValueError("Not a valid doi: %s", doi) raise ValueError("Not a valid doi: {}".format(doi))
new_doi = match.group(0) new_doi = match.group(0)
return new_doi return new_doi

73
tests/test_apis.py
View File

@ -9,34 +9,29 @@ import mock
import dotdot import dotdot
from pubs.p3 import ustr from pubs.p3 import ustr
from pubs.endecoder import EnDecoder
from pubs.apis import ReferenceNotFoundError, arxiv2bibtex, doi2bibtex, isbn2bibtex, _is_arxiv_oldstyle, _extract_arxiv_id
from pubs import apis from pubs import apis
from pubs.apis import _is_arxiv_oldstyle, _extract_arxiv_id
import mock_requests import mock_requests
class APITests(unittest.TestCase): class APITests(unittest.TestCase):
pass
def setUp(self):
self.endecoder = EnDecoder()
class TestDOI2Bibtex(APITests): class TestDOI2Bibtex(APITests):
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_unicode(self, reqget): def test_unicode(self, reqget):
bib = doi2bibtex('10.1007/BF01700692') bib = apis.doi2bibtex('10.1007/BF01700692')
self.assertIsInstance(bib, ustr) self.assertIsInstance(bib, ustr)
self.assertIn('Kurt Gödel', bib) self.assertIn('Kurt Gödel', bib)
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_parses_to_bibtex(self, reqget): def test_parses_to_bibtex(self, reqget):
bib = doi2bibtex('10.1007/BF01700692') bib = apis.get_bibentry_from_api('10.1007/BF01700692', 'DOI')
b = self.endecoder.decode_bibdata(bib) self.assertEqual(len(bib), 1)
self.assertEqual(len(b), 1) entry = bib[list(bib)[0]]
entry = b[list(b)[0]]
self.assertEqual(entry['author'][0], 'Gödel, Kurt') self.assertEqual(entry['author'][0], 'Gödel, Kurt')
self.assertEqual(entry['title'], self.assertEqual(entry['title'],
'Über formal unentscheidbare Sätze der Principia ' 'Über formal unentscheidbare Sätze der Principia '
@ -45,59 +40,54 @@ class TestDOI2Bibtex(APITests):
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_retrieve_fails_on_incorrect_DOI(self, reqget): def test_retrieve_fails_on_incorrect_DOI(self, reqget):
with self.assertRaises(apis.ReferenceNotFoundError): with self.assertRaises(apis.ReferenceNotFoundError):
doi2bibtex('999999') apis.get_bibentry_from_api('999999', 'doi')
class TestISBN2Bibtex(APITests): class TestISBN2Bibtex(APITests):
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_unicode(self, reqget): def test_unicode(self, reqget):
bib = isbn2bibtex('9782081336742') bib = apis.isbn2bibtex('9782081336742')
self.assertIsInstance(bib, ustr) self.assertIsInstance(bib, ustr)
self.assertIn('Poincaré, Henri', bib) self.assertIn('Poincaré, Henri', bib)
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_parses_to_bibtex(self, reqget): def test_parses_to_bibtex(self, reqget):
bib = isbn2bibtex('9782081336742') bib = apis.get_bibentry_from_api('9782081336742', 'ISBN')
b = self.endecoder.decode_bibdata(bib) self.assertEqual(len(bib), 1)
self.assertEqual(len(b), 1) entry = bib[list(bib)[0]]
entry = b[list(b)[0]]
self.assertEqual(entry['author'][0], 'Poincaré, Henri') self.assertEqual(entry['author'][0], 'Poincaré, Henri')
self.assertEqual(entry['title'], 'La science et l\'hypothèse') self.assertEqual(entry['title'], 'La science et l\'hypothèse')
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_retrieve_fails_on_incorrect_ISBN(self, reqget): def test_retrieve_fails_on_incorrect_ISBN(self, reqget):
bib = isbn2bibtex('9' * 13) with self.assertRaises(apis.ReferenceNotFoundError):
with self.assertRaises(EnDecoder.BibDecodingError): apis.get_bibentry_from_api('9' * 13, 'isbn')
self.endecoder.decode_bibdata(bib)
class TestArxiv2Bibtex(APITests): class TestArxiv2Bibtex(APITests):
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_new_style(self, reqget): def test_new_style(self, reqget):
bib = arxiv2bibtex('astro-ph/9812133') bib = apis.get_bibentry_from_api('astro-ph/9812133', 'arXiv')
b = self.endecoder.decode_bibdata(bib) self.assertEqual(len(bib), 1)
self.assertEqual(len(b), 1) entry = bib[list(bib)[0]]
entry = b[list(b)[0]]
self.assertEqual(entry['author'][0], 'Perlmutter, S.') self.assertEqual(entry['author'][0], 'Perlmutter, S.')
self.assertEqual(entry['year'], '1999') self.assertEqual(entry['year'], '1999')
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_parses_to_bibtex_with_doi(self, reqget): def test_parses_to_bibtex_with_doi(self, reqget):
bib = arxiv2bibtex('astro-ph/9812133') bib = apis.get_bibentry_from_api('astro-ph/9812133', 'arxiv')
b = self.endecoder.decode_bibdata(bib) self.assertEqual(len(bib), 1)
self.assertEqual(len(b), 1) entry = bib[list(bib)[0]]
entry = b[list(b)[0]]
self.assertEqual(entry['author'][0], 'Perlmutter, S.') self.assertEqual(entry['author'][0], 'Perlmutter, S.')
self.assertEqual(entry['year'], '1999') self.assertEqual(entry['year'], '1999')
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_parses_to_bibtex_without_doi(self, reqget): def test_parses_to_bibtex_without_doi(self, reqget):
bib = arxiv2bibtex('math/0211159') bib = apis.get_bibentry_from_api('math/0211159', 'ARXIV')
b = self.endecoder.decode_bibdata(bib) self.assertEqual(len(bib), 1)
self.assertEqual(len(b), 1) entry = bib[list(bib)[0]]
entry = b[list(b)[0]]
self.assertEqual(entry['author'][0], 'Perelman, Grisha') self.assertEqual(entry['author'][0], 'Perelman, Grisha')
self.assertEqual(entry['year'], '2002') self.assertEqual(entry['year'], '2002')
self.assertEqual( self.assertEqual(
@ -106,31 +96,28 @@ class TestArxiv2Bibtex(APITests):
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_arxiv_wrong_id(self, reqget): def test_arxiv_wrong_id(self, reqget):
with self.assertRaises(ReferenceNotFoundError): with self.assertRaises(apis.ReferenceNotFoundError):
bib = arxiv2bibtex('INVALIDID') bib = apis.get_bibentry_from_api('INVALIDID', 'arxiv')
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_arxiv_wrong_doi(self, reqget): def test_arxiv_wrong_doi(self, reqget):
bib = arxiv2bibtex('1312.2021') bib = apis.get_bibentry_from_api('1312.2021', 'arXiv')
b = self.endecoder.decode_bibdata(bib) entry = bib[list(bib)[0]]
entry = b[list(b)[0]]
self.assertEqual(entry['arxiv_doi'], '10.1103/INVALIDDOI.89.084044') self.assertEqual(entry['arxiv_doi'], '10.1103/INVALIDDOI.89.084044')
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_arxiv_good_doi(self, reqget): def test_arxiv_good_doi(self, reqget):
"""Get the DOI bibtex instead of the arXiv one if possible""" """Get the DOI bibtex instead of the arXiv one if possible"""
bib = arxiv2bibtex('1710.08557') bib = apis.get_bibentry_from_api('1710.08557', 'arXiv')
b = self.endecoder.decode_bibdata(bib) entry = bib[list(bib)[0]]
entry = b[list(b)[0]]
self.assertTrue(not 'arxiv_doi' in entry) self.assertTrue(not 'arxiv_doi' in entry)
self.assertEqual(entry['doi'], '10.1186/s12984-017-0305-3') self.assertEqual(entry['doi'], '10.1186/s12984-017-0305-3')
self.assertEqual(entry['title'].lower(), 'on neuromechanical approaches for the study of biological and robotic grasp and manipulation') self.assertEqual(entry['title'].lower(), 'on neuromechanical approaches for the study of biological and robotic grasp and manipulation')
@mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
def test_arxiv_good_doi_force_arxiv(self, reqget): def test_arxiv_good_doi_force_arxiv(self, reqget):
bib = arxiv2bibtex('1710.08557', try_doi=False) bib = apis.get_bibentry_from_api('1710.08557', 'arXiv', try_doi=False)
b = self.endecoder.decode_bibdata(bib) entry = bib[list(bib)[0]]
entry = b[list(b)[0]]
self.assertEqual(entry['arxiv_doi'], '10.1186/s12984-017-0305-3') self.assertEqual(entry['arxiv_doi'], '10.1186/s12984-017-0305-3')
self.assertEqual(entry['title'].lower(), 'on neuromechanical approaches for the study of biological grasp and\nmanipulation') self.assertEqual(entry['title'].lower(), 'on neuromechanical approaches for the study of biological grasp and\nmanipulation')