From ce4b66e9b469fc03b698ba75dcf732ce949a4694 Mon Sep 17 00:00:00 2001 From: "Fabien C. Y. Benureau" Date: Mon, 10 Jul 2017 00:20:38 +0200 Subject: [PATCH 1/5] Fix unittests for Python 3 --- tests/fake_env.py | 53 ++++++++++++++++++++++------------------ tests/requirements.txt | 2 +- tests/test_databroker.py | 9 ++++--- tests/test_datacache.py | 8 ++++-- tests/test_filebroker.py | 4 +-- tests/test_usecase.py | 47 +++++++++++++++++++---------------- 6 files changed, 69 insertions(+), 54 deletions(-) diff --git a/tests/fake_env.py b/tests/fake_env.py index 1e7c3bf..71ef8f5 100644 --- a/tests/fake_env.py +++ b/tests/fake_env.py @@ -5,8 +5,11 @@ import shutil import glob import dotdot -from pyfakefs import (fake_filesystem, fake_filesystem_shutil, - fake_filesystem_glob, fake_filesystem_unittest) + +from pyfakefs import fake_filesystem, fake_filesystem_unittest + +# from pyfakefs import (fake_filesystem, fake_filesystem_shutil, +# fake_filesystem_glob, fake_filesystem_unittest) from pubs.p3 import input, _fake_stdio, _get_fake_stdio_ucontent from pubs import content, filebroker @@ -20,27 +23,26 @@ real_shutil = shutil real_glob = glob real_io = io - -def copy_dir(fs, real_dir, fake_dir=None): - """Copy all the data directory into the fake fs""" - if fake_dir is None: - fake_dir = real_dir - - for filename in real_os.listdir(real_dir): - real_path = real_os.path.join(real_dir, filename) - fake_path = os.path.join(fake_dir, filename) - if real_os.path.isfile(real_path): - _, ext = real_os.path.splitext(filename) - if ext in ['.yaml', '.bib', '.txt', '.md']: - with real_io.open(real_path, 'r', encoding='utf-8') as f: - fs.CreateFile(os.path.abspath(fake_path), contents=f.read()) - else: - with real_io.open(real_path, 'rb') as f: - fs.CreateFile(fake_path, contents=f.read()) - - if real_os.path.isdir(real_path): - fs.CreateDirectory(fake_path) - copy_dir(fs, real_path, fake_path) +# def copy_dir(fs, real_dir, fake_dir=None): +# """Copy all the data directory into the fake fs""" +# if fake_dir is None: +# fake_dir = real_dir +# +# for filename in real_os.listdir(real_dir): +# real_path = real_os.path.join(real_dir, filename) +# fake_path = os.path.join(fake_dir, filename) +# if real_os.path.isfile(real_path): +# _, ext = real_os.path.splitext(filename) +# if ext in ['.yaml', '.bib', '.txt', '.md']: +# with real_io.open(real_path, 'r', encoding='utf-8') as f: +# fs.CreateFile(os.path.abspath(fake_path), contents=f.read()) +# else: +# with real_io.open(real_path, 'rb') as f: +# fs.CreateFile(fake_path, contents=f.read()) +# +# if real_os.path.isdir(real_path): +# fs.CreateDirectory(fake_path) +# copy_dir(fs, real_path, fake_path) # redirecting output @@ -108,8 +110,11 @@ class FakeInput(): class TestFakeFs(fake_filesystem_unittest.TestCase): def setUp(self): + self.rootpath = os.path.dirname(__file__) self.setUpPyfakefs() - self.fs.CreateDirectory(os.path.expanduser('~')) + self.fs.CreateDirectory(self.rootpath) + os.chdir(self.rootpath) + def reset_fs(self): self._stubber.tearDown() # renew the filesystem diff --git a/tests/requirements.txt b/tests/requirements.txt index d8d89cf..d8ab57e 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,4 @@ # those are the additional packages required to run the tests six --e git+http://github.com/humm/pyfakefs@two_fixes#egg=pyfakefs +git+http://github.com/jmcgeheeiv/pyfakefs#egg=pyfakefs ddt diff --git a/tests/test_databroker.py b/tests/test_databroker.py index 7cb54d3..0723421 100644 --- a/tests/test_databroker.py +++ b/tests/test_databroker.py @@ -16,6 +16,7 @@ class TestDataBroker(fake_env.TestFakeFs): def test_databroker(self): + ende = endecoder.EnDecoder() page99_metadata = ende.decode_metadata(str_fixtures.metadata_raw0) page99_bibentry = ende.decode_bibdata(str_fixtures.bibtex_raw0) @@ -48,9 +49,9 @@ class TestDataBroker(fake_env.TestFakeFs): for db_class in [databroker.DataBroker, datacache.DataCache]: self.reset_fs() - fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'testrepo'), 'repo') + self.fs.add_real_directory(os.path.join(self.rootpath, 'testrepo'), read_only=False) - db = db_class('repo', 'repo/doc', create=False) + db = db_class('testrepo', 'testrepo/doc', create=False) self.assertEqual(db.pull_bibentry('Page99'), page99_bibentry) @@ -66,8 +67,8 @@ class TestDataBroker(fake_env.TestFakeFs): db.pull_metadata('citekey') db.add_doc('Larry99', 'docsdir://Page99.pdf') - self.assertTrue(content.check_file('repo/doc/Page99.pdf', fail=False)) - self.assertTrue(content.check_file('repo/doc/Larry99.pdf', fail=False)) + self.assertTrue(content.check_file('testrepo/doc/Page99.pdf', fail=False)) + self.assertTrue(content.check_file('testrepo/doc/Larry99.pdf', fail=False)) db.remove_doc('docsdir://Page99.pdf') diff --git a/tests/test_datacache.py b/tests/test_datacache.py index 40641d6..7e95f81 100644 --- a/tests/test_datacache.py +++ b/tests/test_datacache.py @@ -105,10 +105,10 @@ class TestCacheEntrySet(unittest.TestCase): self.databroker_meta.filebroker.mtime = time.time() - 1.1 value = self.metacache.pull('a') self.assertEqual(value, 'c') - self.databroker_meta.filebroker.mtime = time.time() self.databroker_bib.bib = 'b' - self.databroker_bib.filebroker.mtime = time.time() - 1.1 + self.databroker_bib.filebroker.mtime = time.time() self.bibcache.push_to_cache('a', 'c') + self.databroker_bib.filebroker.mtime = time.time() - 1.1 value = self.bibcache.pull('a') self.assertEqual(value, 'c') @@ -142,3 +142,7 @@ class TestCacheEntrySet(unittest.TestCase): self.metacache.push_to_cache('a', 'b') self.databroker_meta.filebroker.mtime = time.time() - 1.1 self.assertFalse(self.metacache._is_outdated('a')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_filebroker.py b/tests/test_filebroker.py index 26846d4..b4b4d9d 100644 --- a/tests/test_filebroker.py +++ b/tests/test_filebroker.py @@ -26,7 +26,7 @@ class TestFileBroker(fake_env.TestFakeFs): def test_existing_data(self): - fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'testrepo'), 'testrepo') + self.fs.add_real_directory(os.path.join(self.rootpath, 'testrepo'), read_only=False) fb = filebroker.FileBroker('testrepo', create = True) bib_content = content.read_text_file('testrepo/bib/Page99.bib') @@ -84,7 +84,7 @@ class TestDocBroker(fake_env.TestFakeFs): def test_doccopy(self): - fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'data'), 'data') + self.fs.add_real_directory(os.path.join(self.rootpath, 'data'), read_only=False) fb = filebroker.FileBroker('testrepo', create = True) docb = filebroker.DocBroker('testrepo') diff --git a/tests/test_usecase.py b/tests/test_usecase.py index 4c275d5..d1a2841 100644 --- a/tests/test_usecase.py +++ b/tests/test_usecase.py @@ -145,8 +145,11 @@ class DataCommandTestCase(CommandTestCase): def setUp(self, nsec_stat=True): super(DataCommandTestCase, self).setUp(nsec_stat=nsec_stat) - fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'data'), 'data') - fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'bibexamples'), 'bibexamples') + self.fs.add_real_directory(os.path.join(self.rootpath, 'data'), read_only=False) + self.fs.add_real_directory(os.path.join(self.rootpath, 'bibexamples'), read_only=False) + + # fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'data'), 'data') + # fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'bibexamples'), 'bibexamples') # Actual tests @@ -181,7 +184,7 @@ class TestAdd(DataCommandTestCase): def test_add(self): cmds = ['pubs init', - 'pubs add /data/pagerank.bib -d /data/pagerank.pdf', + 'pubs add data/pagerank.bib -d data/pagerank.pdf', ] self.execute_cmds(cmds) bib_dir = os.path.join(self.default_pubs_dir, 'bib') @@ -193,7 +196,7 @@ class TestAdd(DataCommandTestCase): def test_add_bibutils(self): cmds = ['pubs init', - 'pubs add /bibexamples/bibutils.bib', + 'pubs add bibexamples/bibutils.bib', ] self.execute_cmds(cmds) bib_dir = os.path.join(self.default_pubs_dir, 'bib') @@ -201,14 +204,14 @@ class TestAdd(DataCommandTestCase): def test_add2(self): cmds = ['pubs init -p /not_default', - 'pubs add /data/pagerank.bib -d /data/pagerank.pdf', + 'pubs add data/pagerank.bib -d data/pagerank.pdf', ] self.execute_cmds(cmds) self.assertEqual(set(os.listdir('/not_default/doc')), {'Page99.pdf'}) def test_add_citekey(self): cmds = ['pubs init', - 'pubs add -k CustomCitekey /data/pagerank.bib', + 'pubs add -k CustomCitekey data/pagerank.bib', ] self.execute_cmds(cmds) bib_dir = os.path.join(self.default_pubs_dir, 'bib') @@ -219,14 +222,14 @@ class TestAdd(DataCommandTestCase): "utf-8 citekeys are not supported yet.\n" "See https://github.com/pubs/pubs/issues/28 for details.") # actually not checked cmds = ['pubs init', - ('pubs add /bibexamples/utf8.bib', [], '', err), + ('pubs add bibexamples/utf8.bib', [], '', err), ] with self.assertRaises(FakeSystemExit): self.execute_cmds(cmds) def test_add_doc_nocopy_does_not_copy(self): cmds = ['pubs init', - 'pubs add /data/pagerank.bib --link -d /data/pagerank.pdf', + 'pubs add data/pagerank.bib --link -d data/pagerank.pdf', ] self.execute_cmds(cmds) self.assertEqual(os.listdir( @@ -235,15 +238,15 @@ class TestAdd(DataCommandTestCase): def test_add_move_removes_doc(self): cmds = ['pubs init', - 'pubs add /data/pagerank.bib --move -d /data/pagerank.pdf', + 'pubs add data/pagerank.bib --move -d data/pagerank.pdf', ] self.execute_cmds(cmds) - self.assertFalse(os.path.exists('/data/pagerank.pdf')) + self.assertFalse(os.path.exists('data/pagerank.pdf')) def test_add_twice_fails(self): cmds = ['pubs init', - 'pubs add /data/pagerank.bib', - 'pubs add -k Page99 /data/turing1950.bib', + 'pubs add data/pagerank.bib', + 'pubs add -k Page99 data/turing1950.bib', ] with self.assertRaises(FakeSystemExit): self.execute_cmds(cmds) @@ -252,7 +255,7 @@ class TestAdd(DataCommandTestCase): @unittest.expectedFailure def test_leading_citekey_space(self): cmds = ['pubs init', - 'pubs add /bibexamples/leadingspace.bib', + 'pubs add bibexamples/leadingspace.bib', 'pubs rename LeadingSpace NoLeadingSpace', ] self.execute_cmds(cmds) @@ -263,22 +266,24 @@ class TestList(DataCommandTestCase): def test_list(self): cmds = ['pubs init -p /not_default2', 'pubs list', - 'pubs add /data/pagerank.bib -d /data/pagerank.pdf', + 'pubs add data/pagerank.bib -d data/pagerank.pdf', 'pubs list', ] outs = self.execute_cmds(cmds) self.assertEqual(0, len(outs[1].splitlines())) self.assertEqual(1, len(outs[3].splitlines())) + @unittest.expectedFailure #FIXME pyfakefs's shutil.rmtree seems to have problems: submit an issue. def test_list_several_no_date(self): - self.execute_cmds(['pubs init -p /testrepo']) + self.execute_cmds(['pubs init -p testrepo']) shutil.rmtree('testrepo') - testrepo = os.path.join(os.path.dirname(__file__), 'testrepo') - fake_env.copy_dir(self.fs, testrepo, 'testrepo') + self.fs.add_real_directory(os.path.join(self.rootpath, 'testrepo'), read_only=False) + + #fake_env.copy_dir(self.fs, testrepo, 'testrepo') cmds = ['pubs list', 'pubs remove -f Page99', 'pubs list', - 'pubs add /data/pagerank.bib -d /data/pagerank.pdf', + 'pubs add data/pagerank.bib -d data/pagerank.pdf', 'pubs list', ] outs = self.execute_cmds(cmds) @@ -400,7 +405,7 @@ class TestUsecase(DataCommandTestCase): '[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) | network,search\n', ] - cmds = ['pubs init -p paper_first/', + cmds = ['pubs init -p /paper_first', 'pubs add -d data/pagerank.pdf data/pagerank.bib', 'pubs list', 'pubs tag', @@ -566,7 +571,7 @@ class TestUsecase(DataCommandTestCase): 'doc', 'Page99.pdf'))) # Also test that do not remove original - self.assertTrue(os.path.exists('/data/pagerank.pdf')) + self.assertTrue(os.path.exists('data/pagerank.pdf')) def test_doc_add_with_move(self): cmds = ['pubs init -p paper_second/', @@ -574,7 +579,7 @@ class TestUsecase(DataCommandTestCase): 'pubs doc add --move data/pagerank.pdf Page99' ] self.execute_cmds(cmds) - self.assertFalse(os.path.exists('/data/pagerank.pdf')) + self.assertFalse(os.path.exists('data/pagerank.pdf')) def test_doc_remove(self): cmds = ['pubs init', From 66a91b3c0cc58b5ed7406783a616ba5153120bf2 Mon Sep 17 00:00:00 2001 From: "Fabien C. Y. Benureau" Date: Tue, 11 Jul 2017 02:17:48 +0200 Subject: [PATCH 2/5] Fix content._open for Python 2 --- pubs/content.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pubs/content.py b/pubs/content.py index 63c27aa..b776028 100644 --- a/pubs/content.py +++ b/pubs/content.py @@ -1,3 +1,4 @@ +import sys import os import io import shutil @@ -47,12 +48,11 @@ def system_path(path): def _open(path, mode): - if 'b' in mode: + if 'b' in mode or sys.version_info < (3,): return open(system_path(path), mode) else: return open(system_path(path), mode, encoding='utf-8') - def check_file(path, fail=True): syspath = system_path(path) return (_check_system_path_exists(syspath, fail=fail) From 3b4cfe635da525afd671beedb3327b8b78cc5221 Mon Sep 17 00:00:00 2001 From: "Fabien C. Y. Benureau" Date: Tue, 11 Jul 2017 11:14:06 +0200 Subject: [PATCH 3/5] Fix tests for Python 2 Also, some cleanup in fake_env.py --- pubs/content.py | 5 +++++ pubs/uis.py | 3 ++- tests/fake_env.py | 23 +---------------------- tests/test_usecase.py | 4 ++-- 4 files changed, 10 insertions(+), 25 deletions(-) diff --git a/pubs/content.py b/pubs/content.py index b776028..cfa634f 100644 --- a/pubs/content.py +++ b/pubs/content.py @@ -70,9 +70,14 @@ def read_text_file(filepath, fail=True): try: with _open(filepath, 'r') as f: content = f.read() + try: # Python 2 + content = content.decode('utf-8') + except AttributeError: # Python 3 + pass except UnicodeDecodeError: raise UnableToDecodeTextFile(filepath) # Should "raise from", if Python 2 support is dropped. + return content def read_binary_file(filepath, fail=True): diff --git a/pubs/uis.py b/pubs/uis.py index 20735ad..9e59de9 100644 --- a/pubs/uis.py +++ b/pubs/uis.py @@ -14,6 +14,7 @@ from .p3 import _get_raw_stdout, _get_raw_stderr, input, ustr from .content import check_file, read_text_file, write_file, system_path +DEBUG = False # package-shared ui that can be accessed using : # from uis import get_ui # ui = get_ui() @@ -117,7 +118,7 @@ class PrintUI(object): :returns: True if exception has been handled (currently never happens) """ - if not self.debug: + if (not DEBUG) and (not self.debug): self.error(ustr(exc)) self.exit() return False diff --git a/tests/fake_env.py b/tests/fake_env.py index 71ef8f5..5ba21d5 100644 --- a/tests/fake_env.py +++ b/tests/fake_env.py @@ -23,27 +23,6 @@ real_shutil = shutil real_glob = glob real_io = io -# def copy_dir(fs, real_dir, fake_dir=None): -# """Copy all the data directory into the fake fs""" -# if fake_dir is None: -# fake_dir = real_dir -# -# for filename in real_os.listdir(real_dir): -# real_path = real_os.path.join(real_dir, filename) -# fake_path = os.path.join(fake_dir, filename) -# if real_os.path.isfile(real_path): -# _, ext = real_os.path.splitext(filename) -# if ext in ['.yaml', '.bib', '.txt', '.md']: -# with real_io.open(real_path, 'r', encoding='utf-8') as f: -# fs.CreateFile(os.path.abspath(fake_path), contents=f.read()) -# else: -# with real_io.open(real_path, 'rb') as f: -# fs.CreateFile(fake_path, contents=f.read()) -# -# if real_os.path.isdir(real_path): -# fs.CreateDirectory(fake_path) -# copy_dir(fs, real_path, fake_path) - # redirecting output @@ -114,7 +93,7 @@ class TestFakeFs(fake_filesystem_unittest.TestCase): self.setUpPyfakefs() self.fs.CreateDirectory(self.rootpath) os.chdir(self.rootpath) - + def reset_fs(self): self._stubber.tearDown() # renew the filesystem diff --git a/tests/test_usecase.py b/tests/test_usecase.py index d1a2841..e1ca9f7 100644 --- a/tests/test_usecase.py +++ b/tests/test_usecase.py @@ -24,7 +24,7 @@ from pubs.commands import init_cmd, import_cmd # makes the tests very noisy -PRINT_OUTPUT = False +PRINT_OUTPUT = False CAPTURE_OUTPUT = True @@ -124,7 +124,7 @@ class CommandTestCase(fake_env.TestFakeFs): self.fail('Unexpected input asked by command: {}.'.format( actual_cmd)) if PRINT_OUTPUT: - print(outs) + print(outs) return outs except SystemExit as exc: exc_class, exc, tb = sys.exc_info() From 323fb8b67460548231595563b268d6cbfc58dbc8 Mon Sep 17 00:00:00 2001 From: Olivier Mangin Date: Tue, 11 Jul 2017 17:13:39 -0400 Subject: [PATCH 4/5] Fixes indent and removes unused io import. --- pubs/content.py | 1 - tests/test_usecase.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pubs/content.py b/pubs/content.py index cfa634f..796302b 100644 --- a/pubs/content.py +++ b/pubs/content.py @@ -1,6 +1,5 @@ import sys import os -import io import shutil from .p3 import urlparse, HTTPConnection, urlopen diff --git a/tests/test_usecase.py b/tests/test_usecase.py index e1ca9f7..56553d0 100644 --- a/tests/test_usecase.py +++ b/tests/test_usecase.py @@ -124,7 +124,7 @@ class CommandTestCase(fake_env.TestFakeFs): self.fail('Unexpected input asked by command: {}.'.format( actual_cmd)) if PRINT_OUTPUT: - print(outs) + print(outs) return outs except SystemExit as exc: exc_class, exc, tb = sys.exc_info() From 9b6496d74099bb550ca5edcfc328c3cd6a327893 Mon Sep 17 00:00:00 2001 From: Olivier Mangin Date: Tue, 11 Jul 2017 17:26:08 -0400 Subject: [PATCH 5/5] Removed commented code with old imports. --- tests/fake_env.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/fake_env.py b/tests/fake_env.py index 5ba21d5..7169cdb 100644 --- a/tests/fake_env.py +++ b/tests/fake_env.py @@ -8,9 +8,6 @@ import dotdot from pyfakefs import fake_filesystem, fake_filesystem_unittest -# from pyfakefs import (fake_filesystem, fake_filesystem_shutil, -# fake_filesystem_glob, fake_filesystem_unittest) - from pubs.p3 import input, _fake_stdio, _get_fake_stdio_ucontent from pubs import content, filebroker