mock mode for test_apis

The mock system relies on the PUBS_TESTS_MODE environmental variable.
See the `tests/mock_requests.py` code for details.
By defaults, fast mock tests are used with data pre-fetched from real
calls.

dev_requirements.txt

@@ -1,7 +1,8 @@
 # if you want to setup your environment for development of the pytest code,
 # doing `pip install -r dev_requirements.txt` is the single thing you have to do.
 # Alternatively, and perhaps more conveniently, running `python setup.py test`
-# will do the same *and* run the tests.
+# will do the same *and* run the tests, but without installing the packages on
+# the system.
 # Note that if you introduce a new dependency, you need to add it here and, more
 # importantly, to the setup.py script so that it is taken into account when
 # installing from PyPi.

pubs/apis.py

@@ -146,7 +146,6 @@ def arxiv2bibtex(arxiv_id, try_doi=True, ui=None):
                "'{}': {}".format(arxiv_id, e))
         raise ReferenceNotFoundError(msg)
 
-    # print("TEXT = '{}'".format(r.text))
     feed = feedparser.parse(r.text)
     if len(feed.entries) == 0:  # no results.
         msg = "no results for arXiv id {}".format(arxiv_id)
@@ -161,12 +160,6 @@ def arxiv2bibtex(arxiv_id, try_doi=True, ui=None):
         raise ReferenceNotFoundError(msg)
 
     entry = feed.entries[0]
-    if 'arxiv.org/api/errors' in entry['id']:  # server is returning an error message.
-        msg = 'the arXiv server returned an error message: {}'.format(entry['summary'])
-        raise ReferenceNotFoundError(msg)
-    # import pprint
-    # pprint.pprint(entry)
-
 
     ## try to return a doi instead of the arXiv reference
     if try_doi and 'arxiv_doi' in entry:
@@ -203,8 +196,3 @@ def arxiv2bibtex(arxiv_id, try_doi=True, ui=None):
 
     bibtex = bibtexparser.dumps(db)
     return bibtex
-
-if __name__ == '__main__':
-    print(arxiv2bibtex("0704.0010"))
-    print(arxiv2bibtex("0704.010*"))
-#    print(arxiv2bibtex("quant-ph/0703266"))

pubs/endecoder.py

@@ -10,7 +10,7 @@ import bibtexparser
 try:
     import bibtexparser as bp
     # don't let bibtexparser display stuff
-#    bp.bparser.logger.setLevel(level=logging.CRITICAL)
+    bp.bparser.logger.setLevel(level=logging.CRITICAL)
 except ImportError:
     print("error: you need to install bibterxparser; try running 'pip install "
           "bibtexparser'.")
@@ -75,8 +75,6 @@ class EnDecoder(object):
 
     class BibDecodingError(Exception):
 
-#        message = "Could not parse provided bibdata:\n---\n{}\n---"
-
         def __init__(self, error_msg, bibdata):
             """
             :param error_msg: specific message about what went wrong

tests/mock_requests.py

@@ -0,0 +1,61 @@
+import os
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+import requests
+
+_orgininal_requests_get = requests.get
+
+_collected_responses = []
+
+class MockingResponse:
+    def __init__(self, text, status_code=200, error_msg=None):
+        self.text = text
+        self.status_code = status_code
+        self.error_msg = error_msg
+        self.encoding = 'utf8'
+
+    def raise_for_status(self):
+        if self.status_code != 200:
+            raise requests.exceptions.RequestException(self.error_msg)
+
+
+mode = os.environ.get('PUBS_TESTS_MODE', 'MOCK')
+
+if mode == 'MOCK':
+
+    with open('test_apis_data.pickle', 'rb') as fd:
+        _collected_responses.extend(pickle.load(fd))
+
+    def mock_requests_get(*args, **kwargs):
+        for args2, kwargs2, text, status_code, error_msg in _collected_responses:
+            if args == args2 and kwargs == kwargs2:
+                return MockingResponse(text, status_code, error_msg)
+        raise KeyError('No stub data found for requests.get({}, {})'.format(args, kwargs))
+
+elif mode == 'COLLECT':
+
+    def mock_requests_get(*args, **kwargs):
+        text, status_code, error_msg = None, None, None
+        try:
+            r = _orgininal_requests_get(*args, **kwargs)
+            text, status_code = r.text, r.status_code
+            r.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            error_msg = str(e)
+        key = (sorted(args), sorted((k, v) for k, v in kwargs.items()))
+
+        _collected_responses.append((args, kwargs, text, status_code, error_msg))
+        _save_collected_responses()
+
+        return MockingResponse(text, status_code, error_msg)
+
+    def _save_collected_responses():
+        with open('test_apis_data.pickle', 'wb') as fd:
+            pickle.dump(_collected_responses, fd, protocol=3)
+
+elif mode == 'ONLINE':
+    def mock_requests_get(*args, **kwargs):
+        return _orgininal_requests_get(*args, **kwargs)

tests/test_apis.py

@@ -4,6 +4,9 @@ from __future__ import unicode_literals
 import unittest
 import socket
 
+import mock
+
+
 import dotdot
 
 from pubs.p3 import ustr
@@ -12,6 +15,8 @@ from pubs.apis import ReferenceNotFoundError, arxiv2bibtex, doi2bibtex, isbn2bib
 
 from pubs import apis
 
+import mock_requests
+
 
 def _is_connected():
     """Return False if no internet connection is detected.
@@ -27,23 +32,25 @@ def _is_connected():
         pass
     return False
 
+
 class APITests(unittest.TestCase):
 
     def setUp(self):
-        if not _is_connected():
-            self.skipTest('no connection detected, skiping test')
+        # if not _is_connected():
+        #     self.skipTest('no connection detected, skiping test')
         self.endecoder = EnDecoder()
 
 
-
 class TestDOI2Bibtex(APITests):
 
-    def test_unicode(self):
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_unicode(self, reqget):
         bib = doi2bibtex('10.1007/BF01700692')
         self.assertIsInstance(bib, ustr)
         self.assertIn('Kurt Gödel', bib)
 
-    def test_parses_to_bibtex(self):
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_parses_to_bibtex(self, reqget):
         bib = doi2bibtex('10.1007/BF01700692')
         b = self.endecoder.decode_bibdata(bib)
         self.assertEqual(len(b), 1)
@@ -53,19 +60,22 @@ class TestDOI2Bibtex(APITests):
                          'Über formal unentscheidbare Sätze der Principia '
                          'Mathematica und verwandter Systeme I')
 
-    def test_retrieve_fails_on_incorrect_DOI(self):
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_retrieve_fails_on_incorrect_DOI(self, reqget):
         with self.assertRaises(apis.ReferenceNotFoundError):
             doi2bibtex('999999')
 
 
 class TestISBN2Bibtex(APITests):
 
-    def test_unicode(self):
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_unicode(self, reqget):
         bib = isbn2bibtex('9782081336742')
         self.assertIsInstance(bib, ustr)
         self.assertIn('Poincaré, Henri', bib)
 
-    def test_parses_to_bibtex(self):
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_parses_to_bibtex(self, reqget):
         bib = isbn2bibtex('9782081336742')
         b = self.endecoder.decode_bibdata(bib)
         self.assertEqual(len(b), 1)
@@ -73,7 +83,8 @@ class TestISBN2Bibtex(APITests):
         self.assertEqual(entry['author'][0], 'Poincaré, Henri')
         self.assertEqual(entry['title'], 'La science et l\'hypothèse')
 
-    def test_retrieve_fails_on_incorrect_ISBN(self):
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_retrieve_fails_on_incorrect_ISBN(self, reqget):
         bib = isbn2bibtex('9' * 13)
         with self.assertRaises(EnDecoder.BibDecodingError):
             self.endecoder.decode_bibdata(bib)
@@ -81,7 +92,8 @@ class TestISBN2Bibtex(APITests):
 
 class TestArxiv2Bibtex(APITests):
 
-    def test_new_style(self):
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_new_style(self, reqget):
         bib = arxiv2bibtex('astro-ph/9812133')
         b = self.endecoder.decode_bibdata(bib)
         self.assertEqual(len(b), 1)
@@ -89,7 +101,8 @@ class TestArxiv2Bibtex(APITests):
         self.assertEqual(entry['author'][0], 'Perlmutter, S.')
         self.assertEqual(entry['year'], '1999')
 
-    def test_parses_to_bibtex_with_doi(self):
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_parses_to_bibtex_with_doi(self, reqget):
         bib = arxiv2bibtex('astro-ph/9812133')
         b = self.endecoder.decode_bibdata(bib)
         self.assertEqual(len(b), 1)
@@ -97,7 +110,8 @@ class TestArxiv2Bibtex(APITests):
         self.assertEqual(entry['author'][0], 'Perlmutter, S.')
         self.assertEqual(entry['year'], '1999')
 
-    def test_parses_to_bibtex_without_doi(self):
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_parses_to_bibtex_without_doi(self, reqget):
         bib = arxiv2bibtex('math/0211159')
         b = self.endecoder.decode_bibdata(bib)
         self.assertEqual(len(b), 1)
@@ -108,6 +122,7 @@ class TestArxiv2Bibtex(APITests):
                 entry['title'],
                 'The entropy formula for the Ricci flow and its geometric applications')
 
+
 class TestArxiv2BibtexLocal(unittest.TestCase):
     """Test arXiv 2 Bibtex connection; those tests don't require a connection"""
 
@@ -129,5 +144,6 @@ class TestArxiv2BibtexLocal(unittest.TestCase):
         self.assertEqual(_extract_arxiv_id({'id': "https://arxiv.org/abs/0704.0010v1"}), "0704.0010v1")
         self.assertEqual(_extract_arxiv_id({'id': "https://arxiv.org/abs/astro-ph/9812133v2"}), "astro-ph/9812133v2")
 
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)