Simplify, update the testsuite

Replace the in-house stubbing by the facilities offered
by pyfakefs for running tests. We don't use `io.open`
anymore, because pyfakefs does not stub it correctly.
We rely on a patched version of pyfakefs, as a few bug
remain in the official repo.

pubs/content.py

@@ -51,9 +51,9 @@ def system_path(path):
 
 def _open(path, mode):
         if mode.find('b') == -1:
-            return io.open(system_path(path), mode, encoding='utf-8')
+            return open(system_path(path), mode, encoding='utf-8')
         else:
-            return io.open(system_path(path), mode)
+            return open(system_path(path), mode)
 
 
 def check_file(path, fail=True):

tests/fake_env.py

@@ -7,7 +7,7 @@ import unittest
 
 import dotdot
 from pyfakefs import (fake_filesystem, fake_filesystem_shutil,
-                      fake_filesystem_glob)
+                      fake_filesystem_glob, fake_filesystem_unittest)
 
 # pyfakefs uses cStringIO under Python 2.x, which does not accept arbitrary unicode strings
 # (see https://docs.python.org/2/library/stringio.html#module-cStringIO)
@@ -25,95 +25,40 @@ from pubs import content, filebroker
     # code for fake fs
 
 real_os     = os
+real_os_path = os.path
 real_open   = open
 real_shutil = shutil
 real_glob   = glob
 real_io     = io
 
 
-ENCODING = 'utf8'
-
-class FakeIO(object):
-
-    def __init__(self, fake_open):
-        self.fake_open = fake_open
-
-    def open(self, file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True):
-        # encoding is ignored by pyfakefs
-        # https://github.com/jmcgeheeiv/pyfakefs/blob/master/pyfakefs/fake_filesystem.py#L2143
-        return self.fake_open(file, mode=mode, buffering=buffering)
-
-    BytesIO = real_io.BytesIO
-    StringIO = real_io.StringIO
-
-
-def create_fake_fs(module_list, nsec_stat=True):
-
-    fake_fs = fake_filesystem.FakeFilesystem(nsec_stat=nsec_stat)
-    fake_os = fake_filesystem.FakeOsModule(fake_fs)
-    fake_open = fake_filesystem.FakeFileOpen(fake_fs)
-    fake_shutil = fake_filesystem_shutil.FakeShutilModule(fake_fs)
-    fake_glob = fake_filesystem_glob.FakeGlobModule(fake_fs)
-    fake_io = FakeIO(fake_open)
-
-    fake_fs.CreateDirectory(fake_os.path.expanduser('~'))
-
-    sys.modules['os']     = fake_os
-    sys.modules['shutil'] = fake_shutil
-    sys.modules['glob']   = fake_glob
-    sys.modules['io']     = fake_io
-
-    for md in module_list:
-        md.os = fake_os
-        md.shutil = fake_shutil
-        md.open = fake_open
-        md.file = None
-        md.io = fake_io
-
-    return {'fs': fake_fs,
-            'os': fake_os,
-            'open': fake_open,
-            'io': fake_io,
-            'shutil': fake_shutil,
-            'glob': fake_glob}
-
-
-def unset_fake_fs(module_list):
-    try:
-        __builtins__.open = real_open
-    except AttributeError:
-        __builtins__['open'] = real_open
-
-    sys.modules['os']     = real_os
-    sys.modules['shutil'] = real_shutil
-    sys.modules['glob']   = real_glob
-    sys.modules['io']     = real_io
-
-    for md in module_list:
-        md.os = real_os
-        md.shutil = real_shutil
-        md.open = real_open
-        md.io = real_io
-
-
-def copy_dir(fs, real_dir, fake_dir = None):
+def copy_dir(fs, real_dir, fake_dir=None):
     """Copy all the data directory into the fake fs"""
     if fake_dir is None:
         fake_dir = real_dir
+
     for filename in real_os.listdir(real_dir):
-        real_path = os.path.abspath(real_os.path.join(real_dir, filename))
-        fake_path = fs['os'].path.join(fake_dir, filename)
+        real_path = real_os.path.join(real_dir, filename)
+        fake_path = os.path.join(fake_dir, filename)
         if real_os.path.isfile(real_path):
             _, ext = real_os.path.splitext(filename)
+            #print('copied into {}'.format(os.path.abspath(fake_path)))
             if ext in ['.yaml', '.bib', '.txt', '.md']:
-                with real_io.open(real_path, 'r', encoding='utf-8') as f:
-                    fs['fs'].CreateFile(fake_path, contents=f.read())
+                with real_open(real_path, 'r', encoding='utf-8') as f:
+                    fs.CreateFile(os.path.abspath(fake_path), contents=f.read())
             else:
-                with real_io.open(real_path, 'rb') as f:
-                    fs['fs'].CreateFile(fake_path, contents=f.read())
-
+                with real_open(real_path, 'rb') as f:
+                    fs.CreateFile(fake_path, contents=f.read())
+
+        #print(real_os.stat)
+        import inspect
+        #print(inspect.getsource(real_os.path.isdir))
+        #print(real_os.stat('/Users/fabien/research/install/pubs/tests/'+real_path))
+        import stat
+        #print('BLA', stat.S_ISDIR(real_os.stat('/Users/fabien/research/install/pubs/tests/'+real_path).st_mode))
+        #print('/Users/fabien/research/install/pubs/tests/'+real_path, real_os_path.isdir('/Users/fabien/research/install/pubs/tests/'+real_path))
         if real_os.path.isdir(real_path):
-            fs['fs'].CreateDirectory(fake_path)
+            fs.CreateDirectory(fake_path)
             copy_dir(fs, real_path, fake_path)
 
 
@@ -179,11 +124,12 @@ class FakeInput():
             raise self.UnexpectedInput('Unexpected user input in test.')
 
 
-class TestFakeFs(unittest.TestCase):
-    """Abstract TestCase intializing the fake filesystem."""
+class TestFakeFs(fake_filesystem_unittest.TestCase):
 
     def setUp(self):
-        self.fs = create_fake_fs([content, filebroker])
+        self.setUpPyfakefs()
+        self.fs.CreateDirectory(os.path.expanduser('~'))
 
-    def tearDown(self):
-        unset_fake_fs([content, filebroker])
+    def reset_fs(self):
+        self._stubber.tearDown() # renew the filesystem
+        self.setUp()

tests/readme.txt

@@ -0,0 +1,7 @@
+1. Install the dependencies using:
+> pip install -r requirements.txt
+
+2. Run the tests using:
+> python -m unittest discover
+
+If you use nosetest, it will complain about addExpectedFailure, which you can safely disregard. 

tests/requirements.txt

@@ -1,4 +1,4 @@
 # those are the additional packages required to run the tests
 six
--e git+http://github.com/humm/pyfakefs@feat/float_mtime#egg=pyfakefs
+-e git+http://github.com/humm/pyfakefs@two_fixes#egg=pyfakefs 
 ddt

tests/test_databroker.py

@@ -12,7 +12,7 @@ import str_fixtures
 from pubs import endecoder
 
 
-class TestDataBroker(unittest.TestCase):
+class TestDataBroker(fake_env.TestFakeFs):
 
     def test_databroker(self):
 
@@ -21,7 +21,7 @@ class TestDataBroker(unittest.TestCase):
         page99_bibentry = ende.decode_bibdata(str_fixtures.bibtex_raw0)
 
         for db_class in [databroker.DataBroker, datacache.DataCache]:
-            self.fs = fake_env.create_fake_fs([content, filebroker, conf])
+            self.reset_fs()
 
             db = db_class('tmp', create=True)
 
@@ -39,7 +39,6 @@ class TestDataBroker(unittest.TestCase):
                 self.assertEqual(pulled[key], page99_bibentry['Page99'][key])
             self.assertEqual(db.pull_bibentry('citekey1'), page99_bibentry)
 
-            fake_env.unset_fake_fs([content, filebroker])
 
     def test_existing_data(self):
 
@@ -47,7 +46,8 @@ class TestDataBroker(unittest.TestCase):
         page99_bibentry = ende.decode_bibdata(str_fixtures.bibtex_raw0)
 
         for db_class in [databroker.DataBroker, datacache.DataCache]:
-            self.fs = fake_env.create_fake_fs([content, filebroker])
+            self.reset_fs()
+
             fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'testrepo'), 'repo')
 
             db = db_class('repo', create=False)
@@ -71,8 +71,6 @@ class TestDataBroker(unittest.TestCase):
 
             db.remove_doc('docsdir://Page99.pdf')
 
-            fake_env.unset_fake_fs([content, filebroker])
-
 
 if __name__ == '__main__':
     unittest.main()

tests/test_usecase.py

@@ -2,9 +2,10 @@
 from __future__ import print_function, unicode_literals
 
 import unittest
-import re
 import os
+import re
 import sys
+import shutil
 
 import six
 import ddt
@@ -65,15 +66,17 @@ class TestFakeInput(unittest.TestCase):
             color.input()
 
 
-class CommandTestCase(unittest.TestCase):
+class CommandTestCase(fake_env.TestFakeFs):
     """Abstract TestCase intializing the fake filesystem."""
 
     maxDiff = 1000000
 
     def setUp(self, nsec_stat=True):
-        self.fs = fake_env.create_fake_fs([content, filebroker, conf, init_cmd, import_cmd, configobj, update],             nsec_stat=nsec_stat)
-        self.default_pubs_dir = self.fs['os'].path.expanduser('~/.pubs')
-        self.default_conf_path = self.fs['os'].path.expanduser('~/.pubsrc')
+        super(CommandTestCase, self).setUp()
+        os.stat_float_times(nsec_stat)
+        # self.fs = fake_env.create_fake_fs([content, filebroker, conf, init_cmd, import_cmd, configobj, update], nsec_stat=nsec_stat)
+        self.default_pubs_dir = os.path.expanduser('~/.pubs')
+        self.default_conf_path = os.path.expanduser('~/.pubsrc')
 
     def execute_cmds(self, cmds, capture_output=CAPTURE_OUTPUT):
         """ Execute a list of commands, and capture their output
@@ -131,7 +134,7 @@ class CommandTestCase(unittest.TestCase):
                 raise FakeSystemExit(exc).with_traceback(tb)
 
     def tearDown(self):
-        fake_env.unset_fake_fs([content, filebroker, conf, init_cmd, import_cmd, configobj])
+        pass
 
 
 class DataCommandTestCase(CommandTestCase):
@@ -152,18 +155,18 @@ class TestInit(CommandTestCase):
     def test_init(self):
         pubsdir = os.path.expanduser('~/pubs_test2')
         self.execute_cmds(['pubs init -p {}'.format(pubsdir)])
-        self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
+        self.assertEqual(set(os.listdir(pubsdir)),
                          {'bib', 'doc', 'meta', 'notes', '.cache'})
 
     def test_init2(self):
         pubsdir = os.path.expanduser('~/.pubs')
         self.execute_cmds(['pubs init'])
-        self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
+        self.assertEqual(set(os.listdir(pubsdir)),
                          {'bib', 'doc', 'meta', 'notes', '.cache'})
 
     def test_init_config(self):
         self.execute_cmds(['pubs init'])
-        self.assertTrue(self.fs['os'].path.isfile(self.default_conf_path))
+        self.assertTrue(os.path.isfile(self.default_conf_path))
 
 
 class TestAdd(DataCommandTestCase):
@@ -173,35 +176,35 @@ class TestAdd(DataCommandTestCase):
                 'pubs add /data/pagerank.bib -d /data/pagerank.pdf',
                 ]
         self.execute_cmds(cmds)
-        bib_dir = self.fs['os'].path.join(self.default_pubs_dir, 'bib')
-        meta_dir = self.fs['os'].path.join(self.default_pubs_dir, 'meta')
-        doc_dir = self.fs['os'].path.join(self.default_pubs_dir, 'doc')
-        self.assertEqual(set(self.fs['os'].listdir(bib_dir)), {'Page99.bib'})
-        self.assertEqual(set(self.fs['os'].listdir(meta_dir)), {'Page99.yaml'})
-        self.assertEqual(set(self.fs['os'].listdir(doc_dir)), {'Page99.pdf'})
+        bib_dir = os.path.join(self.default_pubs_dir, 'bib')
+        meta_dir = os.path.join(self.default_pubs_dir, 'meta')
+        doc_dir = os.path.join(self.default_pubs_dir, 'doc')
+        self.assertEqual(set(os.listdir(bib_dir)), {'Page99.bib'})
+        self.assertEqual(set(os.listdir(meta_dir)), {'Page99.yaml'})
+        self.assertEqual(set(os.listdir(doc_dir)), {'Page99.pdf'})
 
     def test_add_bibutils(self):
         cmds = ['pubs init',
                 'pubs add /bibexamples/bibutils.bib',
                 ]
         self.execute_cmds(cmds)
-        bib_dir = self.fs['os'].path.join(self.default_pubs_dir, 'bib')
-        self.assertEqual(set(self.fs['os'].listdir(bib_dir)), {'Page99.bib'})
+        bib_dir = os.path.join(self.default_pubs_dir, 'bib')
+        self.assertEqual(set(os.listdir(bib_dir)), {'Page99.bib'})
 
     def test_add2(self):
         cmds = ['pubs init -p /not_default',
                 'pubs add /data/pagerank.bib -d /data/pagerank.pdf',
                 ]
         self.execute_cmds(cmds)
-        self.assertEqual(set(self.fs['os'].listdir('/not_default/doc')), {'Page99.pdf'})
+        self.assertEqual(set(os.listdir('/not_default/doc')), {'Page99.pdf'})
 
     def test_add_citekey(self):
         cmds = ['pubs init',
                 'pubs add -k CustomCitekey /data/pagerank.bib',
                 ]
         self.execute_cmds(cmds)
-        bib_dir = self.fs['os'].path.join(self.default_pubs_dir, 'bib')
-        self.assertEqual(set(self.fs['os'].listdir(bib_dir)), {'CustomCitekey.bib'})
+        bib_dir = os.path.join(self.default_pubs_dir, 'bib')
+        self.assertEqual(set(os.listdir(bib_dir)), {'CustomCitekey.bib'})
 
     def test_add_utf8_citekey(self):
         err = ("error: Invalid `hausdorff1949grundzüge` citekey; "
@@ -218,8 +221,8 @@ class TestAdd(DataCommandTestCase):
                 'pubs add /data/pagerank.bib --link -d /data/pagerank.pdf',
                 ]
         self.execute_cmds(cmds)
-        self.assertEqual(self.fs['os'].listdir(
-                self.fs['os'].path.join(self.default_pubs_dir, 'doc')),
+        self.assertEqual(os.listdir(
+                os.path.join(self.default_pubs_dir, 'doc')),
             [])
 
     def test_add_move_removes_doc(self):
@@ -227,7 +230,7 @@ class TestAdd(DataCommandTestCase):
                 'pubs add /data/pagerank.bib --move -d /data/pagerank.pdf',
                 ]
         self.execute_cmds(cmds)
-        self.assertFalse(self.fs['os'].path.exists('/data/pagerank.pdf'))
+        self.assertFalse(os.path.exists('/data/pagerank.pdf'))
 
     def test_add_twice_fails(self):
         cmds = ['pubs init',
@@ -261,7 +264,7 @@ class TestList(DataCommandTestCase):
 
     def test_list_several_no_date(self):
         self.execute_cmds(['pubs init -p /testrepo'])
-        self.fs['shutil'].rmtree('testrepo')
+        shutil.rmtree('testrepo')
         testrepo = os.path.join(os.path.dirname(__file__), 'testrepo')
         fake_env.copy_dir(self.fs, testrepo, 'testrepo')
         cmds = ['pubs list',
@@ -423,8 +426,8 @@ class TestUsecase(DataCommandTestCase):
                 'pubs remove -f turing1950computing',
                ]
         self.execute_cmds(cmds)
-        docdir = self.fs['os'].path.expanduser('~/.pubs/doc/')
-        self.assertNotIn('turing-mind-1950.pdf', self.fs['os'].listdir(docdir))
+        docdir = os.path.expanduser('~/.pubs/doc/')
+        self.assertNotIn('turing-mind-1950.pdf', os.listdir(docdir))
 
     def test_tag_list(self):
         correct = ['Initializing pubs in /paper_first\n',
@@ -550,12 +553,12 @@ class TestUsecase(DataCommandTestCase):
                 'pubs doc add data/pagerank.pdf Page99'
                ]
         self.execute_cmds(cmds)
-        self.assertTrue(self.fs['os'].path.exists(
-            self.fs['os'].path.join(self.default_pubs_dir,
+        self.assertTrue(os.path.exists(
+            os.path.join(self.default_pubs_dir,
                                     'doc',
                                     'Page99.pdf')))
         # Also test that do not remove original
-        self.assertTrue(self.fs['os'].path.exists('/data/pagerank.pdf'))
+        self.assertTrue(os.path.exists('/data/pagerank.pdf'))
 
     def test_doc_add_with_move(self):
         cmds = ['pubs init -p paper_second/',
@@ -563,7 +566,7 @@ class TestUsecase(DataCommandTestCase):
                 'pubs doc add --move data/pagerank.pdf Page99'
                ]
         self.execute_cmds(cmds)
-        self.assertFalse(self.fs['os'].path.exists('/data/pagerank.pdf'))
+        self.assertFalse(os.path.exists('/data/pagerank.pdf'))
 
     def test_doc_remove(self):
         cmds = ['pubs init',
@@ -572,8 +575,8 @@ class TestUsecase(DataCommandTestCase):
                 ('pubs doc remove Page99', ['y']),
                ]
         self.execute_cmds(cmds)
-        docdir = self.fs['os'].path.expanduser('~/.pubs/doc/')
-        self.assertNotIn('turing-mind-1950.pdf', self.fs['os'].listdir(docdir))
+        docdir = os.path.expanduser('~/.pubs/doc/')
+        self.assertNotIn('turing-mind-1950.pdf', os.listdir(docdir))
 
     def test_doc_export(self):
         cmds = ['pubs init',
@@ -583,11 +586,11 @@ class TestUsecase(DataCommandTestCase):
                 'pubs doc export page100 /'
                 ]
         self.execute_cmds(cmds)
-        self.assertIn('page100.pdf', self.fs['os'].listdir('/'))
+        self.assertIn('page100.pdf', os.listdir('/'))
 
 
     def test_alternate_config(self):
-        alt_conf = self.fs['os'].path.expanduser('~/.alt_conf')
+        alt_conf = os.path.expanduser('~/.alt_conf')
         cmds = ['pubs -c ' + alt_conf + ' init',
                 'pubs --config ' + alt_conf + ' import data/ Page99',
                 'pubs list -c ' + alt_conf
@@ -596,8 +599,8 @@ class TestUsecase(DataCommandTestCase):
         # check if pubs works as expected
         self.assertEqual(1 + 1, len(outs[-1].split('\n')))
         # check whether we actually changed the config file
-        self.assertFalse(self.fs['os'].path.isfile(self.default_conf_path))
-        self.assertTrue(self.fs['os'].path.isfile(alt_conf))
+        self.assertFalse(os.path.isfile(self.default_conf_path))
+        self.assertTrue(os.path.isfile(alt_conf))