From 8c54b19207c62e2850c1c0f5c03fe91293ac0344 Mon Sep 17 00:00:00 2001 From: Fabien Benureau Date: Sun, 10 Nov 2013 19:35:21 +0100 Subject: [PATCH] updated import and added rename command --- papers/commands/__init__.py | 14 ++- papers/commands/import_cmd.py | 73 +++++++++++-- papers/commands/rename_cmd.py | 27 +++++ papers/databroker.py | 2 +- papers/datacache.py | 2 +- papers/endecoder.py | 2 + papers/papers_cmd.py | 12 ++- papers/repo.py | 18 ++-- tests/fake_env.py | 4 +- tests/str_fixtures.py | 16 +++ tests/test_usecase.py | 192 +++++++--------------------------- 11 files changed, 179 insertions(+), 183 deletions(-) create mode 100644 papers/commands/rename_cmd.py diff --git a/papers/commands/__init__.py b/papers/commands/__init__.py index d5b7100..f832ba3 100644 --- a/papers/commands/__init__.py +++ b/papers/commands/__init__.py @@ -1,12 +1,18 @@ +# core import init_cmd import add_cmd +import rename_cmd +import remove_cmd import list_cmd +# doc +import attach_cmd import open_cmd -import websearch_cmd -import remove_cmd import tag_cmd -import attach_cmd +# bulk import export_cmd -# import import_cmd +import import_cmd +# bonus +import websearch_cmd + # import edit_cmd # import update_cmd diff --git a/papers/commands/import_cmd.py b/papers/commands/import_cmd.py index c917949..0f94ad4 100644 --- a/papers/commands/import_cmd.py +++ b/papers/commands/import_cmd.py @@ -1,10 +1,17 @@ +import os + +from pybtex.database import Entry, BibliographyData, FieldDict, Person + from .. import repo +from .. import endecoder +from .. import bibstruct +from .. import color from ..paper import Paper -from .helpers import add_paper_with_docfile, extract_doc_path_from_bibdata from ..configs import config from ..uis import get_ui + def parser(subparsers): parser = subparsers.add_parser('import', help='import paper(s) to the repository') @@ -19,9 +26,46 @@ def parser(subparsers): return parser +def many_from_path(bibpath): + """Extract list of papers found in bibliographic files in path. + + The behavior is to: + - ignore wrong entries, + - overwrite duplicated entries. + :returns: dictionary of (key, paper | exception) + if loading of entry failed, the excpetion is returned in the + dictionary in place of the paper + """ + coder = endecoder.EnDecoder() + + bibpath = os.path.expanduser(bibpath) + if os.path.isdir(bibpath): + all_files = [os.path.join(bibpath, f) for f in os.listdir(bibpath) + if os.path.splitext(f)[-1][1:] in list(coder.decode_fmt.keys())] + else: + all_files = [bibpath] + + biblist = [] + for filepath in all_files: + with open(filepath, 'r') as f: + biblist.append(coder.decode_bibdata(f.read())) + + papers = {} + for b in biblist: + for k in b.entries: + try: + bibdata = BibliographyData() + bibdata.entries[k] = b.entries[k] + + papers[k] = Paper(bibdata, citekey=k) + except ValueError, e: + papers[k] = e + return papers + + def command(args): """ - :param bibpath: path (no url yet) to a bibliography file + :param bibpath: path (no url yet) to a bibliography file """ ui = get_ui() @@ -32,17 +76,28 @@ def command(args): copy = config().import_copy rp = repo.Repository(config()) # Extract papers from bib - papers = Paper.many_from_path(bibpath) + papers = many_from_path(bibpath) keys = args.keys or papers.keys() for k in keys: try: p = papers[k] if isinstance(p, Exception): - ui.error('Could not load entry for citekey {}.'.format(k)) + ui.error('could not load entry for citekey {}.'.format(k)) else: - doc_file = extract_doc_path_from_bibdata(p) - if doc_file is None: - ui.warning("No file for %s." % p.citekey) - add_paper_with_docfile(rp, p, docfile=doc_file, copy=copy) + docfile = bibstruct.extract_docfile(p.bibdata) + if docfile is None: + ui.warning("no file for {}.".format(p.citekey)) + else: + copy_doc = args.copy + if copy_doc is None: + copy_doc = config().import_copy + if copy_doc: + docfile = rp.databroker.copy_doc(p.citekey, docfile) + + p.docpath = docfile + rp.push_paper(p) + ui.print_('{} imported'.format(color.dye(p.citekey, color.cyan))) except KeyError: - ui.error('No entry found for citekey {}.'.format(k)) + ui.error('no entry found for citekey {}.'.format(k)) + except IOError, e: + ui.error(e.message) diff --git a/papers/commands/rename_cmd.py b/papers/commands/rename_cmd.py new file mode 100644 index 0000000..0540544 --- /dev/null +++ b/papers/commands/rename_cmd.py @@ -0,0 +1,27 @@ +from ..uis import get_ui +from ..configs import config +from .. import bibstruct +from .. import content +from .. import repo +from .. import paper + +def parser(subparsers): + parser = subparsers.add_parser('rename', help='rename the citekey of a repository') + parser.add_argument('citekey', + help='current citekey') + parser.add_argument('new_citekey', + help='new citekey') + return parser + + +def command(args): + """ + :param bibfile: bibtex file (in .bib, .bibml or .yaml format. + :param docfile: path (no url yet) to a pdf or ps file + """ + + ui = get_ui() + rp = repo.Repository(config()) + + paper = rp.pull_paper(args.citekey) + rp.rename_paper(paper, args.new_citekey) diff --git a/papers/databroker.py b/papers/databroker.py index a69f988..5679d53 100644 --- a/papers/databroker.py +++ b/papers/databroker.py @@ -53,7 +53,7 @@ class DataBroker(object): # docbroker def is_pubsdir_doc(self, docpath): - return self.docbroker.is_pusdir_doc(docpath) + return self.docbroker.is_pubsdir_doc(docpath) def copy_doc(self, citekey, source_path, overwrite=False): return self.docbroker.copy_doc(citekey, source_path, overwrite=overwrite) diff --git a/papers/datacache.py b/papers/datacache.py index 9859699..9d7a1f6 100644 --- a/papers/datacache.py +++ b/papers/datacache.py @@ -63,7 +63,7 @@ class DataCache(object): # docbroker def is_pubsdir_doc(self, docpath): - return self.databroker.is_pusdir_doc(docpath) + return self.databroker.is_pubsdir_doc(docpath) def copy_doc(self, citekey, source_path, overwrite=False): return self.databroker.copy_doc(citekey, source_path, overwrite=overwrite) diff --git a/papers/endecoder.py b/papers/endecoder.py index 2febb27..546141e 100644 --- a/papers/endecoder.py +++ b/papers/endecoder.py @@ -34,10 +34,12 @@ class EnDecoder(object): decode_fmt = {'bibyaml' : pybtex.database.input.bibyaml, 'bibtex' : pybtex.database.input.bibtex, + 'bib' : pybtex.database.input.bibtex, 'bibtexml': pybtex.database.input.bibtexml} encode_fmt = {'bibyaml' : pybtex.database.output.bibyaml, 'bibtex' : pybtex.database.output.bibtex, + 'bib' : pybtex.database.output.bibtex, 'bibtexml': pybtex.database.output.bibtexml} def encode_metadata(self, metadata): diff --git a/papers/papers_cmd.py b/papers/papers_cmd.py index 9135371..a308f9d 100644 --- a/papers/papers_cmd.py +++ b/papers/papers_cmd.py @@ -15,14 +15,18 @@ from .__init__ import __version__ CORE_CMDS = collections.OrderedDict([ ('init', commands.init_cmd), ('add', commands.add_cmd), + ('rename', commands.rename_cmd), + ('remove', commands.remove_cmd), ('list', commands.list_cmd), + + ('attach', commands.attach_cmd), ('open', commands.open_cmd), - ('websearch', commands.websearch_cmd), - ('remove', commands.remove_cmd), ('tag', commands.tag_cmd), - ('attach', commands.attach_cmd), + ('export', commands.export_cmd), - # ('import', commands.import_cmd), + ('import', commands.import_cmd), + + ('websearch', commands.websearch_cmd), # ('edit', commands.edit_cmd), # ('update', commands.update_cmd), ]) diff --git a/papers/repo.py b/papers/repo.py index 557968d..e717433 100644 --- a/papers/repo.py +++ b/papers/repo.py @@ -2,6 +2,8 @@ import shutil import glob import itertools +from pybtex.database import BibliographyData + from . import bibstruct from . import events from . import datacache @@ -65,7 +67,7 @@ class Repository(object): """ bibstruct.check_citekey(paper.citekey) if (not overwrite) and self.databroker.exists(paper.citekey, both = False): - raise IOError('files using this the {} citekey already exists'.format(citekey)) + raise IOError('files using the {} citekey already exists'.format(paper.citekey)) if (not overwrite) and self.citekeys is not None and paper.citekey in self.citekeys: raise CiteKeyCollision('citekey {} already in use'.format(paper.citekey)) @@ -101,8 +103,11 @@ class Repository(object): # check if new_citekey does not exists if self.databroker.exists(new_citekey, both=False): raise IOError("can't rename paper to {}, conflicting files exists".format(new_citekey)) - # modify bibdata - raise NotImplementedError + # modify bibdata (__delitem__ not implementd by pybtex) + new_bibdata = BibliographyData() + new_bibdata.entries[new_citekey] = paper.bibdata.entries[old_citekey] + paper.bibdata = new_bibdata + # move doc file if necessary if self.databroker.is_pubsdir_doc(paper.docpath): new_docpath = self.databroker.copy_doc(new_citekey, paper.docpath) @@ -110,11 +115,12 @@ class Repository(object): paper.docpath = new_docpath # push_paper to new_citekey - self.databroker.push(new_citekey, paper.metadata) + paper.citekey = new_citekey + self.push_paper(paper, event=False) # remove_paper of old_citekey - self.databroker.remove(old_citekey) + self.remove_paper(old_citekey, event=False) # send event - RenameEvent(paper, old_citekey).send() + events.RenameEvent(paper, old_citekey).send() def unique_citekey(self, base_key): """Create a unique citekey for a given basekey.""" diff --git a/tests/fake_env.py b/tests/fake_env.py index 1acfcb3..a8f480c 100644 --- a/tests/fake_env.py +++ b/tests/fake_env.py @@ -140,13 +140,13 @@ class FakeInput(): input() raise IndexError """ - def __init__(self, module_list, inputs=None): + def __init__(self, inputs, module_list=tuple()): self.inputs = list(inputs) or [] self.module_list = module_list self._cursor = 0 def as_global(self): - for md in module_list: + for md in self.module_list: md.input = self md.editor_input = self # if mdname.endswith('files'): diff --git a/tests/str_fixtures.py b/tests/str_fixtures.py index 0730beb..37a0702 100644 --- a/tests/str_fixtures.py +++ b/tests/str_fixtures.py @@ -66,6 +66,22 @@ bibtexml_raw0 = """ """ +bibtex_external0 = """ +@techreport{Page99, + number = {1999-66}, + month = {November}, + author = {Lawrence Page and Sergey Brin and Rajeev Motwani and Terry Winograd}, + note = {Previous number = SIDL-WP-1999-0120}, + title = {The PageRank Citation Ranking: Bringing Order to the Web.}, + type = {Technical Report}, + publisher = {Stanford InfoLab}, + year = {1999}, +institution = {Stanford InfoLab}, + url = {http://ilpubs.stanford.edu:8090/422/}, + abstract = "The importance of a Web page is an inherently subjective matter, which depends on the readers interests, knowledge and attitudes. But there is still much that can be said objectively about the relative importance of Web pages. This paper describes PageRank, a mathod for rating Web pages objectively and mechanically, effectively measuring the human interest and attention devoted to them. We compare PageRank to an idealized random Web surfer. We show how to efficiently compute PageRank for large numbers of pages. And, we show how to apply PageRank to search and to user navigation.", +} +""" + bibtex_raw0 = """ @techreport{ Page99, diff --git a/tests/test_usecase.py b/tests/test_usecase.py index 060665b..34e321e 100644 --- a/tests/test_usecase.py +++ b/tests/test_usecase.py @@ -1,159 +1,31 @@ -import sys -import os -import shutil -import glob import unittest -import pkgutil import re +import os import testenv -import fake_filesystem -import fake_filesystem_shutil -import fake_filesystem_glob +import fake_env from papers import papers_cmd -from papers import color, files -from papers.p3 import io, input +from papers import color, content, filebroker, uis, beets_ui, p3 -import fixtures +import str_fixtures +from papers.commands import init_cmd # code for fake fs -real_os = os -real_open = open -real_shutil = shutil -real_glob = glob - -fake_os, fake_open, fake_shutil, fake_glob = None, None, None, None - - -def _mod_list(): - ml = [] - import papers - for importer, modname, ispkg in pkgutil.walk_packages( - path=papers.__path__, - prefix=papers.__name__ + '.', - onerror=lambda x: None): - # HACK to not load textnote - if not modname.startswith('papers.plugs.texnote'): - ml.append((modname, __import__(modname, fromlist='dummy'))) - return ml - -mod_list = _mod_list() - - -def _create_fake_fs(): - global fake_os, fake_open, fake_shutil, fake_glob - - fake_fs = fake_filesystem.FakeFilesystem() - fake_os = fake_filesystem.FakeOsModule(fake_fs) - fake_open = fake_filesystem.FakeFileOpen(fake_fs) - fake_shutil = fake_filesystem_shutil.FakeShutilModule(fake_fs) - fake_glob = fake_filesystem_glob.FakeGlobModule(fake_fs) - - fake_fs.CreateDirectory(fake_os.path.expanduser('~')) - - try: - __builtins__.open = fake_open - __builtins__.file = fake_open - except AttributeError: - __builtins__['open'] = fake_open - __builtins__['file'] = fake_open - - sys.modules['os'] = fake_os - sys.modules['shutil'] = fake_shutil - sys.modules['glob'] = fake_glob - - for mdname, md in mod_list: - md.os = fake_os - md.shutil = fake_shutil - md.open = fake_open - md.file = fake_open - - return fake_fs - - -def _copy_data(fs): - """Copy all the data directory into the fake fs""" - datadir = real_os.path.join(real_os.path.dirname(__file__), 'data') - for filename in real_os.listdir(datadir): - real_path = real_os.path.join(datadir, filename) - fake_path = fake_os.path.join('data', filename) - if real_os.path.isfile(real_path): - with real_open(real_path, 'r') as f: - fs.CreateFile(fake_path, contents=f.read()) - if real_os.path.isdir(real_path): - fs.CreateDirectory(fake_path) - - - # redirecting output - -def redirect(f): - def newf(*args, **kwargs): - old_stderr, old_stdout = sys.stderr, sys.stdout - stdout = io.StringIO() - stderr = io.StringIO() - sys.stdout, sys.stderr = stdout, stderr - try: - return f(*args, **kwargs), stdout, stderr - finally: - sys.stderr, sys.stdout = old_stderr, old_stdout - return newf - - -# Test helpers - -# automating input - -real_input = input - - -class FakeInput(): - """ Replace the input() command, and mock user input during tests - - Instanciate as : - input = FakeInput(['yes', 'no']) - then replace the input command in every module of the package : - input.as_global() - Then : - input() returns 'yes' - input() returns 'no' - input() raise IndexError - """ - - def __init__(self, inputs=None): - self.inputs = list(inputs) or [] - self._cursor = 0 - - def as_global(self): - for mdname, md in mod_list: - md.input = self - md.editor_input = self - # if mdname.endswith('files'): - # md.editor_input = self - - def add_input(self, inp): - self.inputs.append(inp) - - def __call__(self, *args, **kwargs): - inp = self.inputs[self._cursor] - self._cursor += 1 - return inp - - class TestFakeInput(unittest.TestCase): def test_input(self): - input = FakeInput(['yes', 'no']) + input = fake_env.FakeInput(['yes', 'no']) self.assertEqual(input(), 'yes') self.assertEqual(input(), 'no') with self.assertRaises(IndexError): input() def test_input2(self): - other_input = FakeInput(['yes', 'no']) + other_input = fake_env.FakeInput(['yes', 'no'], module_list=[color]) other_input.as_global() self.assertEqual(color.input(), 'yes') self.assertEqual(color.input(), 'no') @@ -161,10 +33,11 @@ class TestFakeInput(unittest.TestCase): color.input() def test_editor_input(self): - other_input = FakeInput(['yes', 'no']) + other_input = fake_env.FakeInput(['yes', 'no'], + module_list=[content, color]) other_input.as_global() - self.assertEqual(files.editor_input(), 'yes') - self.assertEqual(files.editor_input(), 'no') + self.assertEqual(content.editor_input(), 'yes') + self.assertEqual(content.editor_input(), 'no') with self.assertRaises(IndexError): color.input() @@ -173,7 +46,7 @@ class CommandTestCase(unittest.TestCase): """Abstract TestCase intializing the fake filesystem.""" def setUp(self): - self.fs = _create_fake_fs() + self.fs = fake_env.create_fake_fs([content, filebroker, init_cmd]) def execute_cmds(self, cmds, fs=None): """ Execute a list of commands, and capture their output @@ -189,10 +62,10 @@ class CommandTestCase(unittest.TestCase): for cmd in cmds: if hasattr(cmd, '__iter__'): if len(cmd) == 2: - input = FakeInput(cmd[1]) + input = fake_env.FakeInput(cmd[1], [content, uis, beets_ui, p3]) input.as_global() - _, stdout, stderr = redirect(papers_cmd.execute)(cmd[0].split()) + _, stdout, stderr = fake_env.redirect(papers_cmd.execute)(cmd[0].split()) if len(cmd) == 3: actual_out = color.undye(stdout.getvalue()) correct_out = color.undye(cmd[2]) @@ -200,12 +73,14 @@ class CommandTestCase(unittest.TestCase): else: assert type(cmd) == str - _, stdout, stderr = redirect(papers_cmd.execute)(cmd.split()) + _, stdout, stderr = fake_env.redirect(papers_cmd.execute)(cmd.split()) assert(stderr.getvalue() == '') outs.append(color.undye(stdout.getvalue())) return outs + def tearDown(self): + fake_env.unset_fake_fs([content, filebroker]) class DataCommandTestCase(CommandTestCase): """Abstract TestCase intializing the fake filesystem and @@ -214,7 +89,7 @@ class DataCommandTestCase(CommandTestCase): def setUp(self): CommandTestCase.setUp(self) - _copy_data(self.fs) + fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'data'), 'data') # Actual tests @@ -222,10 +97,16 @@ class DataCommandTestCase(CommandTestCase): class TestInit(CommandTestCase): def test_init(self): - papers_cmd.execute('papers init -p paper_test2'.split()) - self.assertEqual(set(fake_os.listdir('/paper_test2/')), - {'bibdata', 'doc', 'meta', 'papers.yaml'}) + pubsdir = os.path.expanduser('~/papers_test2') + papers_cmd.execute('papers init -p {}'.format(pubsdir).split()) + self.assertEqual(set(self.fs['os'].listdir(pubsdir)), + {'bib', 'doc', 'meta'}) + def test_init2(self): + pubsdir = os.path.expanduser('~/.papers') + papers_cmd.execute('papers init'.split()) + self.assertEqual(set(self.fs['os'].listdir(pubsdir)), + {'bib', 'doc', 'meta'}) class TestAdd(DataCommandTestCase): @@ -240,7 +121,7 @@ class TestAdd(DataCommandTestCase): 'papers add -b /data/pagerank.bib -d /data/pagerank.pdf', ] self.execute_cmds(cmds) - self.assertEqual(set(fake_os.listdir('/not_default/doc')), {'Page99.pdf'}) + self.assertEqual(set(self.fs['os'].listdir('/not_default/doc')), {'Page99.pdf'}) class TestList(DataCommandTestCase): @@ -288,12 +169,12 @@ class TestUsecase(DataCommandTestCase): def test_first(self): correct = ['Initializing papers in /paper_first.\n', '', - '0: [Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) \n', + '[Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) \n', '', '', 'search network\n', - '0: [Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) search network\n', - 'search network\n'] + '[Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) search network\n' + ] cmds = ['papers init -p paper_first/', 'papers add -d data/pagerank.pdf -b data/pagerank.bib', @@ -302,7 +183,6 @@ class TestUsecase(DataCommandTestCase): 'papers tag Page99 network+search', 'papers tag Page99', 'papers tag search', - 'papers tag 0', ] self.assertEqual(correct, self.execute_cmds(cmds)) @@ -343,18 +223,18 @@ class TestUsecase(DataCommandTestCase): def test_editor_success(self): cmds = ['papers init', - ('papers add', [fixtures.pagerankbib]), + ('papers add', [str_fixtures.bibtex_external0]), ('papers remove Page99', ['y']), ] self.execute_cmds(cmds) def test_edit(self): - bib = fixtures.pagerankbib + bib = str_fixtures.bibtex_external0 bib1 = re.sub('year = \{1999\}', 'year = {2007}', bib) bib2 = re.sub('Lawrence Page', 'Lawrence Ridge', bib1) bib3 = re.sub('Page99', 'Ridge07', bib2) - line = '0: [Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) \n' + line = '[Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) \n' line1 = re.sub('1999', '2007', line) line2 = re.sub('L. Page', 'L. Ridge', line1) line3 = re.sub('Page99', 'Ridge07', line2) @@ -374,9 +254,9 @@ class TestUsecase(DataCommandTestCase): def test_export(self): cmds = ['papers init', - ('papers add', [fixtures.pagerankbib]), + ('papers add', [str_fixtures.bibtex_external0]), 'papers export Page99', - ('papers export Page99 -f bibtex', [], fixtures.pagerankbib_generated), + ('papers export Page99 -f bibtex', [], str_fixtures.bibtex_raw0), 'papers export Page99 -f bibyaml', ]