updated import and added rename command

papers/commands/__init__.py

@@ -1,12 +1,18 @@
+# core
 import init_cmd
 import add_cmd
+import rename_cmd
+import remove_cmd
 import list_cmd
+# doc
+import attach_cmd
 import open_cmd
-import websearch_cmd
-import remove_cmd
 import tag_cmd
-import attach_cmd
+# bulk
 import export_cmd
-# import import_cmd
+import import_cmd
+# bonus
+import websearch_cmd
+
 # import edit_cmd
 # import update_cmd

papers/commands/import_cmd.py

@@ -1,10 +1,17 @@
+import os
+
+from pybtex.database import Entry, BibliographyData, FieldDict, Person
+
 from .. import repo
+from .. import endecoder
+from .. import bibstruct
+from .. import color
 from ..paper import Paper
-from .helpers import add_paper_with_docfile, extract_doc_path_from_bibdata
 from ..configs import config
 from ..uis import get_ui
 
 
+
 def parser(subparsers):
     parser = subparsers.add_parser('import',
             help='import paper(s) to the repository')
@@ -19,6 +26,43 @@ def parser(subparsers):
     return parser
 
 
+def many_from_path(bibpath):
+    """Extract list of papers found in bibliographic files in path.
+
+    The behavior is to:
+        - ignore wrong entries,
+        - overwrite duplicated entries.
+    :returns: dictionary of (key, paper | exception)
+        if loading of entry failed, the excpetion is returned in the
+        dictionary in place of the paper
+    """
+    coder = endecoder.EnDecoder()
+
+    bibpath = os.path.expanduser(bibpath)
+    if os.path.isdir(bibpath):
+        all_files = [os.path.join(bibpath, f) for f in os.listdir(bibpath)
+                     if os.path.splitext(f)[-1][1:] in list(coder.decode_fmt.keys())]
+    else:
+        all_files = [bibpath]
+
+    biblist = []
+    for filepath in all_files:
+        with open(filepath, 'r') as f:
+            biblist.append(coder.decode_bibdata(f.read()))
+
+    papers = {}
+    for b in biblist:
+        for k in b.entries:
+            try:
+                bibdata = BibliographyData()
+                bibdata.entries[k] = b.entries[k]
+
+                papers[k] = Paper(bibdata, citekey=k)
+            except ValueError, e:
+                papers[k] = e
+    return papers
+
+
 def command(args):
     """
         :param bibpath: path (no url yet) to a bibliography file
@@ -32,17 +76,28 @@ def command(args):
         copy = config().import_copy
     rp = repo.Repository(config())
     # Extract papers from bib
-    papers = Paper.many_from_path(bibpath)
+    papers = many_from_path(bibpath)
     keys = args.keys or papers.keys()
     for k in keys:
         try:
             p = papers[k]
             if isinstance(p, Exception):
-                ui.error('Could not load entry for citekey {}.'.format(k))
+                ui.error('could not load entry for citekey {}.'.format(k))
+            else:
+                docfile = bibstruct.extract_docfile(p.bibdata)
+                if docfile is None:
+                    ui.warning("no file for {}.".format(p.citekey))
                 else:
-                doc_file = extract_doc_path_from_bibdata(p)
+                    copy_doc = args.copy
-                if doc_file is None:
+                    if copy_doc is None:
-                    ui.warning("No file for %s." % p.citekey)
+                        copy_doc = config().import_copy
-                add_paper_with_docfile(rp, p, docfile=doc_file, copy=copy)
+                    if copy_doc:
+                        docfile = rp.databroker.copy_doc(p.citekey, docfile)
+                    
+                    p.docpath = docfile
+                rp.push_paper(p)
+                ui.print_('{} imported'.format(color.dye(p.citekey, color.cyan)))
         except KeyError:
-            ui.error('No entry found for citekey {}.'.format(k))
+            ui.error('no entry found for citekey {}.'.format(k))
+        except IOError, e:
+            ui.error(e.message)

papers/commands/rename_cmd.py

@@ -0,0 +1,27 @@
+from ..uis import get_ui
+from ..configs import config
+from .. import bibstruct
+from .. import content
+from .. import repo
+from .. import paper
+
+def parser(subparsers):
+    parser = subparsers.add_parser('rename', help='rename the citekey of a repository')
+    parser.add_argument('citekey',
+                        help='current citekey')
+    parser.add_argument('new_citekey',
+                        help='new citekey')
+    return parser
+
+
+def command(args):
+    """
+    :param bibfile: bibtex file (in .bib, .bibml or .yaml format.
+    :param docfile: path (no url yet) to a pdf or ps file
+    """
+
+    ui = get_ui()
+    rp = repo.Repository(config())
+
+    paper = rp.pull_paper(args.citekey)
+    rp.rename_paper(paper, args.new_citekey)

papers/databroker.py

@@ -53,7 +53,7 @@ class DataBroker(object):
     # docbroker
 
     def is_pubsdir_doc(self, docpath):
-        return self.docbroker.is_pusdir_doc(docpath)
+        return self.docbroker.is_pubsdir_doc(docpath)
 
     def copy_doc(self, citekey, source_path, overwrite=False):
         return self.docbroker.copy_doc(citekey, source_path, overwrite=overwrite)

papers/datacache.py

@@ -63,7 +63,7 @@ class DataCache(object):
         # docbroker
 
     def is_pubsdir_doc(self, docpath):
-        return self.databroker.is_pusdir_doc(docpath)
+        return self.databroker.is_pubsdir_doc(docpath)
 
     def copy_doc(self, citekey, source_path, overwrite=False):
         return self.databroker.copy_doc(citekey, source_path, overwrite=overwrite)

papers/endecoder.py

@@ -34,10 +34,12 @@ class EnDecoder(object):
 
     decode_fmt = {'bibyaml' : pybtex.database.input.bibyaml,
                   'bibtex'  : pybtex.database.input.bibtex,
+                  'bib'     : pybtex.database.input.bibtex,
                   'bibtexml': pybtex.database.input.bibtexml}
 
     encode_fmt = {'bibyaml' : pybtex.database.output.bibyaml,
                   'bibtex'  : pybtex.database.output.bibtex,
+                  'bib'     : pybtex.database.output.bibtex,
                   'bibtexml': pybtex.database.output.bibtexml}
 
     def encode_metadata(self, metadata):

papers/papers_cmd.py

@@ -15,14 +15,18 @@ from .__init__ import __version__
 CORE_CMDS = collections.OrderedDict([
         ('init',        commands.init_cmd),
         ('add',         commands.add_cmd),
+        ('rename',      commands.rename_cmd),
+        ('remove',      commands.remove_cmd),
         ('list',        commands.list_cmd),
+
+        ('attach',      commands.attach_cmd),
         ('open',        commands.open_cmd),
-        ('websearch',   commands.websearch_cmd),
-        ('remove',      commands.remove_cmd),
         ('tag',         commands.tag_cmd),
-        ('attach',      commands.attach_cmd),
+
         ('export',      commands.export_cmd),
-        # ('import',      commands.import_cmd),
+        ('import',      commands.import_cmd),
+
+        ('websearch',   commands.websearch_cmd),
         # ('edit',        commands.edit_cmd),
         # ('update',      commands.update_cmd),
         ])

papers/repo.py

@@ -2,6 +2,8 @@ import shutil
 import glob
 import itertools
 
+from pybtex.database import BibliographyData
+
 from . import bibstruct
 from . import events 
 from . import datacache
@@ -65,7 +67,7 @@ class Repository(object):
         """
         bibstruct.check_citekey(paper.citekey)
         if (not overwrite) and self.databroker.exists(paper.citekey, both = False):
-            raise IOError('files using this the {} citekey already exists'.format(citekey))
+            raise IOError('files using the {} citekey already exists'.format(paper.citekey))
         if (not overwrite) and self.citekeys is not None and paper.citekey in self.citekeys:
             raise CiteKeyCollision('citekey {} already in use'.format(paper.citekey))
         
@@ -101,8 +103,11 @@ class Repository(object):
             # check if new_citekey does not exists
             if self.databroker.exists(new_citekey, both=False):
                 raise IOError("can't rename paper to {}, conflicting files exists".format(new_citekey))                
-            # modify bibdata
+            # modify bibdata (__delitem__ not implementd by pybtex)
-            raise NotImplementedError
+            new_bibdata = BibliographyData()
+            new_bibdata.entries[new_citekey] = paper.bibdata.entries[old_citekey]
+            paper.bibdata = new_bibdata
+
             # move doc file if necessary
             if self.databroker.is_pubsdir_doc(paper.docpath):
                 new_docpath = self.databroker.copy_doc(new_citekey, paper.docpath)
@@ -110,11 +115,12 @@ class Repository(object):
                 paper.docpath = new_docpath
 
             # push_paper to new_citekey
-            self.databroker.push(new_citekey, paper.metadata)
+            paper.citekey = new_citekey
+            self.push_paper(paper, event=False)
             # remove_paper of old_citekey
-            self.databroker.remove(old_citekey)
+            self.remove_paper(old_citekey, event=False)
             # send event
-            RenameEvent(paper, old_citekey).send()
+            events.RenameEvent(paper, old_citekey).send()
 
     def unique_citekey(self, base_key):
         """Create a unique citekey for a given basekey."""

tests/fake_env.py

@@ -140,13 +140,13 @@ class FakeInput():
         input() raise IndexError
      """
 
-    def __init__(self, module_list, inputs=None):
+    def __init__(self, inputs, module_list=tuple()):
         self.inputs = list(inputs) or []
         self.module_list = module_list
         self._cursor = 0
 
     def as_global(self):
-        for md in module_list:
+        for md in self.module_list:
             md.input = self
             md.editor_input = self
             # if mdname.endswith('files'):

tests/str_fixtures.py

@@ -66,6 +66,22 @@ bibtexml_raw0 = """<?xml version='1.0' encoding='UTF-8'?>
 </bibtex:file>
 """
 
+bibtex_external0 = """
+@techreport{Page99,
+     number = {1999-66},
+      month = {November},
+     author = {Lawrence Page and Sergey Brin and Rajeev Motwani and Terry Winograd},
+       note = {Previous number = SIDL-WP-1999-0120},
+      title = {The PageRank Citation Ranking: Bringing Order to the Web.},
+       type = {Technical Report},
+  publisher = {Stanford InfoLab},
+       year = {1999},
+institution = {Stanford InfoLab},
+        url = {http://ilpubs.stanford.edu:8090/422/},
+   abstract = "The importance of a Web page is an inherently subjective matter, which depends on the readers interests, knowledge and attitudes. But there is still much that can be said objectively about the relative importance of Web pages. This paper describes PageRank, a mathod for rating Web pages objectively and mechanically, effectively measuring the human interest and attention devoted to them. We compare PageRank to an idealized random Web surfer. We show how to efficiently compute PageRank for large numbers of pages. And, we show how to apply PageRank to search and to user navigation.",
+}
+"""
+
 bibtex_raw0 = """
 @techreport{
     Page99,

tests/test_usecase.py

@@ -1,159 +1,31 @@
-import sys
-import os
-import shutil
-import glob
 import unittest
-import pkgutil
 import re
+import os
 
 import testenv
-import fake_filesystem
+import fake_env
-import fake_filesystem_shutil
-import fake_filesystem_glob
 
 from papers import papers_cmd
-from papers import color, files
+from papers import color, content, filebroker, uis, beets_ui, p3
-from papers.p3 import io, input
 
-import fixtures
+import str_fixtures
 
+from papers.commands import init_cmd
 
     # code for fake fs
 
-real_os = os
-real_open = open
-real_shutil = shutil
-real_glob = glob
-
-fake_os, fake_open, fake_shutil, fake_glob = None, None, None, None
-
-
-def _mod_list():
-    ml = []
-    import papers
-    for importer, modname, ispkg in pkgutil.walk_packages(
-                                        path=papers.__path__,
-                                        prefix=papers.__name__ + '.',
-                                        onerror=lambda x: None):
-        # HACK to not load textnote
-        if not modname.startswith('papers.plugs.texnote'):
-            ml.append((modname, __import__(modname, fromlist='dummy')))
-    return ml
-
-mod_list = _mod_list()
-
-
-def _create_fake_fs():
-    global fake_os, fake_open, fake_shutil, fake_glob
-
-    fake_fs = fake_filesystem.FakeFilesystem()
-    fake_os = fake_filesystem.FakeOsModule(fake_fs)
-    fake_open = fake_filesystem.FakeFileOpen(fake_fs)
-    fake_shutil = fake_filesystem_shutil.FakeShutilModule(fake_fs)
-    fake_glob = fake_filesystem_glob.FakeGlobModule(fake_fs)
-
-    fake_fs.CreateDirectory(fake_os.path.expanduser('~'))
-
-    try:
-        __builtins__.open = fake_open
-        __builtins__.file = fake_open
-    except AttributeError:
-        __builtins__['open'] = fake_open
-        __builtins__['file'] = fake_open
-
-    sys.modules['os']     = fake_os
-    sys.modules['shutil'] = fake_shutil
-    sys.modules['glob']   = fake_glob
-
-    for mdname, md in mod_list:
-        md.os = fake_os
-        md.shutil = fake_shutil
-        md.open = fake_open
-        md.file = fake_open
-
-    return fake_fs
-
-
-def _copy_data(fs):
-    """Copy all the data directory into the fake fs"""
-    datadir = real_os.path.join(real_os.path.dirname(__file__), 'data')
-    for filename in real_os.listdir(datadir):
-        real_path = real_os.path.join(datadir, filename)
-        fake_path = fake_os.path.join('data', filename)
-        if real_os.path.isfile(real_path):
-            with real_open(real_path, 'r') as f:
-                fs.CreateFile(fake_path, contents=f.read())
-        if real_os.path.isdir(real_path):
-            fs.CreateDirectory(fake_path)
-
-
-    # redirecting output
-
-def redirect(f):
-    def newf(*args, **kwargs):
-        old_stderr, old_stdout = sys.stderr, sys.stdout
-        stdout = io.StringIO()
-        stderr = io.StringIO()
-        sys.stdout, sys.stderr = stdout, stderr
-        try:
-            return f(*args, **kwargs), stdout, stderr
-        finally:
-            sys.stderr, sys.stdout = old_stderr, old_stdout
-    return newf
-
-
-# Test helpers
-
-# automating input
-
-real_input = input
-
-
-class FakeInput():
-    """ Replace the input() command, and mock user input during tests
-
-        Instanciate as :
-        input = FakeInput(['yes', 'no'])
-        then replace the input command in every module of the package :
-        input.as_global()
-        Then :
-        input() returns 'yes'
-        input() returns 'no'
-        input() raise IndexError
-     """
-
-    def __init__(self, inputs=None):
-        self.inputs = list(inputs) or []
-        self._cursor = 0
-
-    def as_global(self):
-        for mdname, md in mod_list:
-            md.input = self
-            md.editor_input = self
-            # if mdname.endswith('files'):
-            #     md.editor_input = self
-
-    def add_input(self, inp):
-        self.inputs.append(inp)
-
-    def __call__(self, *args, **kwargs):
-        inp = self.inputs[self._cursor]
-        self._cursor += 1
-        return inp
-
-
 class TestFakeInput(unittest.TestCase):
 
     def test_input(self):
 
-        input = FakeInput(['yes', 'no'])
+        input = fake_env.FakeInput(['yes', 'no'])
         self.assertEqual(input(), 'yes')
         self.assertEqual(input(), 'no')
         with self.assertRaises(IndexError):
             input()
 
     def test_input2(self):
-        other_input = FakeInput(['yes', 'no'])
+        other_input = fake_env.FakeInput(['yes', 'no'], module_list=[color])
         other_input.as_global()
         self.assertEqual(color.input(), 'yes')
         self.assertEqual(color.input(), 'no')
@@ -161,10 +33,11 @@ class TestFakeInput(unittest.TestCase):
             color.input()
 
     def test_editor_input(self):
-        other_input = FakeInput(['yes', 'no'])
+        other_input = fake_env.FakeInput(['yes', 'no'], 
+                                         module_list=[content, color])
         other_input.as_global()
-        self.assertEqual(files.editor_input(), 'yes')
+        self.assertEqual(content.editor_input(), 'yes')
-        self.assertEqual(files.editor_input(), 'no')
+        self.assertEqual(content.editor_input(), 'no')
         with self.assertRaises(IndexError):
             color.input()
 
@@ -173,7 +46,7 @@ class CommandTestCase(unittest.TestCase):
     """Abstract TestCase intializing the fake filesystem."""
 
     def setUp(self):
-        self.fs = _create_fake_fs()
+        self.fs = fake_env.create_fake_fs([content, filebroker, init_cmd])
 
     def execute_cmds(self, cmds, fs=None):
         """ Execute a list of commands, and capture their output
@@ -189,10 +62,10 @@ class CommandTestCase(unittest.TestCase):
         for cmd in cmds:
             if hasattr(cmd, '__iter__'):
                 if len(cmd) == 2:
-                    input = FakeInput(cmd[1])
+                    input = fake_env.FakeInput(cmd[1], [content, uis, beets_ui, p3])
                     input.as_global()
 
-                _, stdout, stderr = redirect(papers_cmd.execute)(cmd[0].split())
+                _, stdout, stderr = fake_env.redirect(papers_cmd.execute)(cmd[0].split())
                 if len(cmd) == 3:
                     actual_out  = color.undye(stdout.getvalue())
                     correct_out = color.undye(cmd[2])
@@ -200,12 +73,14 @@ class CommandTestCase(unittest.TestCase):
 
             else:
                 assert type(cmd) == str
-                _, stdout, stderr = redirect(papers_cmd.execute)(cmd.split())
+                _, stdout, stderr = fake_env.redirect(papers_cmd.execute)(cmd.split())
 
             assert(stderr.getvalue() == '')
             outs.append(color.undye(stdout.getvalue()))
         return outs
 
+    def tearDown(self):
+        fake_env.unset_fake_fs([content, filebroker])
 
 class DataCommandTestCase(CommandTestCase):
     """Abstract TestCase intializing the fake filesystem and
@@ -214,7 +89,7 @@ class DataCommandTestCase(CommandTestCase):
 
     def setUp(self):
         CommandTestCase.setUp(self)
-        _copy_data(self.fs)
+        fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'data'), 'data')
 
 
 # Actual tests
@@ -222,10 +97,16 @@ class DataCommandTestCase(CommandTestCase):
 class TestInit(CommandTestCase):
 
     def test_init(self):
-        papers_cmd.execute('papers init -p paper_test2'.split())
+        pubsdir = os.path.expanduser('~/papers_test2')
-        self.assertEqual(set(fake_os.listdir('/paper_test2/')),
+        papers_cmd.execute('papers init -p {}'.format(pubsdir).split())
-                         {'bibdata', 'doc', 'meta', 'papers.yaml'})
+        self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
+                         {'bib', 'doc', 'meta'})
 
+    def test_init2(self):
+        pubsdir = os.path.expanduser('~/.papers')
+        papers_cmd.execute('papers init'.split())
+        self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
+                         {'bib', 'doc', 'meta'})
 
 class TestAdd(DataCommandTestCase):
 
@@ -240,7 +121,7 @@ class TestAdd(DataCommandTestCase):
                 'papers add -b /data/pagerank.bib -d /data/pagerank.pdf',
                 ]
         self.execute_cmds(cmds)
-        self.assertEqual(set(fake_os.listdir('/not_default/doc')), {'Page99.pdf'})
+        self.assertEqual(set(self.fs['os'].listdir('/not_default/doc')), {'Page99.pdf'})
 
 
 class TestList(DataCommandTestCase):
@@ -288,12 +169,12 @@ class TestUsecase(DataCommandTestCase):
     def test_first(self):
         correct = ['Initializing papers in /paper_first.\n',
                    '',
-                   '0: [Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web"  (1999) \n',
+                   '[Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web"  (1999) \n',
                    '',
                    '',
                    'search network\n',
-                   '0: [Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web"  (1999) search network\n',
+                   '[Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web"  (1999) search network\n'
-                   'search network\n']
+                  ]
 
         cmds = ['papers init -p paper_first/',
                 'papers add -d data/pagerank.pdf -b data/pagerank.bib',
@@ -302,7 +183,6 @@ class TestUsecase(DataCommandTestCase):
                 'papers tag Page99 network+search',
                 'papers tag Page99',
                 'papers tag search',
-                'papers tag 0',
                ]
 
         self.assertEqual(correct, self.execute_cmds(cmds))
@@ -343,18 +223,18 @@ class TestUsecase(DataCommandTestCase):
 
     def test_editor_success(self):
         cmds = ['papers init',
-                ('papers add', [fixtures.pagerankbib]),
+                ('papers add', [str_fixtures.bibtex_external0]),
                 ('papers remove Page99', ['y']),
                ]
         self.execute_cmds(cmds)
 
     def test_edit(self):
-        bib = fixtures.pagerankbib
+        bib = str_fixtures.bibtex_external0
         bib1 = re.sub('year = \{1999\}', 'year = {2007}', bib)
         bib2 = re.sub('Lawrence Page', 'Lawrence Ridge', bib1)
         bib3 = re.sub('Page99', 'Ridge07', bib2)
 
-        line = '0: [Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web"  (1999) \n'
+        line = '[Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web"  (1999) \n'
         line1 = re.sub('1999', '2007', line)
         line2 = re.sub('L. Page', 'L. Ridge', line1)
         line3 = re.sub('Page99', 'Ridge07', line2)
@@ -374,9 +254,9 @@ class TestUsecase(DataCommandTestCase):
 
     def test_export(self):
         cmds = ['papers init',
-                ('papers add', [fixtures.pagerankbib]),
+                ('papers add', [str_fixtures.bibtex_external0]),
                 'papers export Page99',
-                ('papers export Page99 -f bibtex', [], fixtures.pagerankbib_generated),
+                ('papers export Page99 -f bibtex', [], str_fixtures.bibtex_raw0),
                 'papers export Page99 -f bibyaml',
                ]