aselimov/pubs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated import and added rename command

Browse Source
This commit is contained in:
Fabien Benureau 2013-11-10 19:35:21 +01:00
parent bc82d0de8c
commit 8c54b19207
11 changed files with 181 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
papers/commands/__init__.py
View File

@ -1,12 +1,18 @@
# core
import init_cmd
import add_cmd
import list_cmd
import open_cmd
import websearch_cmd
import rename_cmd
import remove_cmd
import tag_cmd
import list_cmd
# doc
import attach_cmd
import open_cmd
import tag_cmd
# bulk
import export_cmd
# import import_cmd
import import_cmd
# bonus
import websearch_cmd
# import edit_cmd
# import update_cmd

73
papers/commands/import_cmd.py
View File

@ -1,10 +1,17 @@
import os
from pybtex.database import Entry, BibliographyData, FieldDict, Person
from .. import repo
from .. import endecoder
from .. import bibstruct
from .. import color
from ..paper import Paper
from .helpers import add_paper_with_docfile, extract_doc_path_from_bibdata
from ..configs import config
from ..uis import get_ui
def parser(subparsers):
parser = subparsers.add_parser('import',
help='import paper(s) to the repository')
@ -19,9 +26,46 @@ def parser(subparsers):
return parser
def many_from_path(bibpath):
"""Extract list of papers found in bibliographic files in path.
The behavior is to:
- ignore wrong entries,
- overwrite duplicated entries.
:returns: dictionary of (key, paper | exception)
if loading of entry failed, the excpetion is returned in the
dictionary in place of the paper
"""
coder = endecoder.EnDecoder()
bibpath = os.path.expanduser(bibpath)
if os.path.isdir(bibpath):
all_files = [os.path.join(bibpath, f) for f in os.listdir(bibpath)
if os.path.splitext(f)[-1][1:] in list(coder.decode_fmt.keys())]
else:
all_files = [bibpath]
biblist = []
for filepath in all_files:
with open(filepath, 'r') as f:
biblist.append(coder.decode_bibdata(f.read()))
papers = {}
for b in biblist:
for k in b.entries:
try:
bibdata = BibliographyData()
bibdata.entries[k] = b.entries[k]
papers[k] = Paper(bibdata, citekey=k)
except ValueError, e:
papers[k] = e
return papers
def command(args):
"""
:param bibpath: path (no url yet) to a bibliography file
:param bibpath: path (no url yet) to a bibliography file
"""
ui = get_ui()
@ -32,17 +76,28 @@ def command(args):
copy = config().import_copy
rp = repo.Repository(config())
# Extract papers from bib
papers = Paper.many_from_path(bibpath)
papers = many_from_path(bibpath)
keys = args.keys or papers.keys()
for k in keys:
try:
p = papers[k]
if isinstance(p, Exception):
ui.error('Could not load entry for citekey {}.'.format(k))
ui.error('could not load entry for citekey {}.'.format(k))
else:
doc_file = extract_doc_path_from_bibdata(p)
if doc_file is None:
ui.warning("No file for %s." % p.citekey)
add_paper_with_docfile(rp, p, docfile=doc_file, copy=copy)
docfile = bibstruct.extract_docfile(p.bibdata)
if docfile is None:
ui.warning("no file for {}.".format(p.citekey))
else:
copy_doc = args.copy
if copy_doc is None:
copy_doc = config().import_copy
if copy_doc:
docfile = rp.databroker.copy_doc(p.citekey, docfile)
p.docpath = docfile
rp.push_paper(p)
ui.print_('{} imported'.format(color.dye(p.citekey, color.cyan)))
except KeyError:
ui.error('No entry found for citekey {}.'.format(k))
ui.error('no entry found for citekey {}.'.format(k))
except IOError, e:
ui.error(e.message)

27
papers/commands/rename_cmd.py Normal file
View File

@ -0,0 +1,27 @@
from ..uis import get_ui
from ..configs import config
from .. import bibstruct
from .. import content
from .. import repo
from .. import paper
def parser(subparsers):
parser = subparsers.add_parser('rename', help='rename the citekey of a repository')
parser.add_argument('citekey',
help='current citekey')
parser.add_argument('new_citekey',
help='new citekey')
return parser
def command(args):
"""
:param bibfile: bibtex file (in .bib, .bibml or .yaml format.
:param docfile: path (no url yet) to a pdf or ps file
"""
ui = get_ui()
rp = repo.Repository(config())
paper = rp.pull_paper(args.citekey)
rp.rename_paper(paper, args.new_citekey)

2
papers/databroker.py
View File

@ -53,7 +53,7 @@ class DataBroker(object):
# docbroker
def is_pubsdir_doc(self, docpath):
return self.docbroker.is_pusdir_doc(docpath)
return self.docbroker.is_pubsdir_doc(docpath)
def copy_doc(self, citekey, source_path, overwrite=False):
return self.docbroker.copy_doc(citekey, source_path, overwrite=overwrite)

2
papers/datacache.py
View File

@ -63,7 +63,7 @@ class DataCache(object):
# docbroker
def is_pubsdir_doc(self, docpath):
return self.databroker.is_pusdir_doc(docpath)
return self.databroker.is_pubsdir_doc(docpath)
def copy_doc(self, citekey, source_path, overwrite=False):
return self.databroker.copy_doc(citekey, source_path, overwrite=overwrite)

2
papers/endecoder.py
View File

@ -34,10 +34,12 @@ class EnDecoder(object):
decode_fmt = {'bibyaml' : pybtex.database.input.bibyaml,
'bibtex' : pybtex.database.input.bibtex,
'bib' : pybtex.database.input.bibtex,
'bibtexml': pybtex.database.input.bibtexml}
encode_fmt = {'bibyaml' : pybtex.database.output.bibyaml,
'bibtex' : pybtex.database.output.bibtex,
'bib' : pybtex.database.output.bibtex,
'bibtexml': pybtex.database.output.bibtexml}
def encode_metadata(self, metadata):

14
papers/papers_cmd.py
View File

@ -15,14 +15,18 @@ from .__init__ import __version__
CORE_CMDS = collections.OrderedDict([
('init', commands.init_cmd),
('add', commands.add_cmd),
('list', commands.list_cmd),
('open', commands.open_cmd),
('websearch', commands.websearch_cmd),
('rename', commands.rename_cmd),
('remove', commands.remove_cmd),
('tag', commands.tag_cmd),
('list', commands.list_cmd),
('attach', commands.attach_cmd),
('open', commands.open_cmd),
('tag', commands.tag_cmd),
('export', commands.export_cmd),
# ('import', commands.import_cmd),
('import', commands.import_cmd),
('websearch', commands.websearch_cmd),
# ('edit', commands.edit_cmd),
# ('update', commands.update_cmd),
])

18
papers/repo.py
View File

@ -2,6 +2,8 @@ import shutil
import glob
import itertools
from pybtex.database import BibliographyData
from . import bibstruct
from . import events
from . import datacache
@ -65,7 +67,7 @@ class Repository(object):
"""
bibstruct.check_citekey(paper.citekey)
if (not overwrite) and self.databroker.exists(paper.citekey, both = False):
raise IOError('files using this the {} citekey already exists'.format(citekey))
raise IOError('files using the {} citekey already exists'.format(paper.citekey))
if (not overwrite) and self.citekeys is not None and paper.citekey in self.citekeys:
raise CiteKeyCollision('citekey {} already in use'.format(paper.citekey))
@ -101,8 +103,11 @@ class Repository(object):
# check if new_citekey does not exists
if self.databroker.exists(new_citekey, both=False):
raise IOError("can't rename paper to {}, conflicting files exists".format(new_citekey))
# modify bibdata
raise NotImplementedError
# modify bibdata (__delitem__ not implementd by pybtex)
new_bibdata = BibliographyData()
new_bibdata.entries[new_citekey] = paper.bibdata.entries[old_citekey]
paper.bibdata = new_bibdata
# move doc file if necessary
if self.databroker.is_pubsdir_doc(paper.docpath):
new_docpath = self.databroker.copy_doc(new_citekey, paper.docpath)
@ -110,11 +115,12 @@ class Repository(object):
paper.docpath = new_docpath
# push_paper to new_citekey
self.databroker.push(new_citekey, paper.metadata)
paper.citekey = new_citekey
self.push_paper(paper, event=False)
# remove_paper of old_citekey
self.databroker.remove(old_citekey)
self.remove_paper(old_citekey, event=False)
# send event
RenameEvent(paper, old_citekey).send()
events.RenameEvent(paper, old_citekey).send()
def unique_citekey(self, base_key):
"""Create a unique citekey for a given basekey."""

4
tests/fake_env.py
View File

@ -140,13 +140,13 @@ class FakeInput():
input() raise IndexError
"""
def __init__(self, module_list, inputs=None):
def __init__(self, inputs, module_list=tuple()):
self.inputs = list(inputs) or []
self.module_list = module_list
self._cursor = 0
def as_global(self):
for md in module_list:
for md in self.module_list:
md.input = self
md.editor_input = self
# if mdname.endswith('files'):

16
tests/str_fixtures.py
View File

@ -66,6 +66,22 @@ bibtexml_raw0 = """<?xml version='1.0' encoding='UTF-8'?>
</bibtex:file>
"""
bibtex_external0 = """
@techreport{Page99,
number = {1999-66},
month = {November},
author = {Lawrence Page and Sergey Brin and Rajeev Motwani and Terry Winograd},
note = {Previous number = SIDL-WP-1999-0120},
title = {The PageRank Citation Ranking: Bringing Order to the Web.},
type = {Technical Report},
publisher = {Stanford InfoLab},
year = {1999},
institution = {Stanford InfoLab},
url = {http://ilpubs.stanford.edu:8090/422/},
abstract = "The importance of a Web page is an inherently subjective matter, which depends on the readers interests, knowledge and attitudes. But there is still much that can be said objectively about the relative importance of Web pages. This paper describes PageRank, a mathod for rating Web pages objectively and mechanically, effectively measuring the human interest and attention devoted to them. We compare PageRank to an idealized random Web surfer. We show how to efficiently compute PageRank for large numbers of pages. And, we show how to apply PageRank to search and to user navigation.",
}
"""
bibtex_raw0 = """
@techreport{
Page99,

192
tests/test_usecase.py
View File

@ -1,159 +1,31 @@
import sys
import os
import shutil
import glob
import unittest
import pkgutil
import re
import os
import testenv
import fake_filesystem
import fake_filesystem_shutil
import fake_filesystem_glob
import fake_env
from papers import papers_cmd
from papers import color, files
from papers.p3 import io, input
from papers import color, content, filebroker, uis, beets_ui, p3
import fixtures
import str_fixtures
from papers.commands import init_cmd
# code for fake fs
real_os = os
real_open = open
real_shutil = shutil
real_glob = glob
fake_os, fake_open, fake_shutil, fake_glob = None, None, None, None
def _mod_list():
ml = []
import papers
for importer, modname, ispkg in pkgutil.walk_packages(
path=papers.__path__,
prefix=papers.__name__ + '.',
onerror=lambda x: None):
# HACK to not load textnote
if not modname.startswith('papers.plugs.texnote'):
ml.append((modname, __import__(modname, fromlist='dummy')))
return ml
mod_list = _mod_list()
def _create_fake_fs():
global fake_os, fake_open, fake_shutil, fake_glob
fake_fs = fake_filesystem.FakeFilesystem()
fake_os = fake_filesystem.FakeOsModule(fake_fs)
fake_open = fake_filesystem.FakeFileOpen(fake_fs)
fake_shutil = fake_filesystem_shutil.FakeShutilModule(fake_fs)
fake_glob = fake_filesystem_glob.FakeGlobModule(fake_fs)
fake_fs.CreateDirectory(fake_os.path.expanduser('~'))
try:
__builtins__.open = fake_open
__builtins__.file = fake_open
except AttributeError:
__builtins__['open'] = fake_open
__builtins__['file'] = fake_open
sys.modules['os'] = fake_os
sys.modules['shutil'] = fake_shutil
sys.modules['glob'] = fake_glob
for mdname, md in mod_list:
md.os = fake_os
md.shutil = fake_shutil
md.open = fake_open
md.file = fake_open
return fake_fs
def _copy_data(fs):
"""Copy all the data directory into the fake fs"""
datadir = real_os.path.join(real_os.path.dirname(__file__), 'data')
for filename in real_os.listdir(datadir):
real_path = real_os.path.join(datadir, filename)
fake_path = fake_os.path.join('data', filename)
if real_os.path.isfile(real_path):
with real_open(real_path, 'r') as f:
fs.CreateFile(fake_path, contents=f.read())
if real_os.path.isdir(real_path):
fs.CreateDirectory(fake_path)
# redirecting output
def redirect(f):
def newf(*args, **kwargs):
old_stderr, old_stdout = sys.stderr, sys.stdout
stdout = io.StringIO()
stderr = io.StringIO()
sys.stdout, sys.stderr = stdout, stderr
try:
return f(*args, **kwargs), stdout, stderr
finally:
sys.stderr, sys.stdout = old_stderr, old_stdout
return newf
# Test helpers
# automating input
real_input = input
class FakeInput():
""" Replace the input() command, and mock user input during tests
Instanciate as :
input = FakeInput(['yes', 'no'])
then replace the input command in every module of the package :
input.as_global()
Then :
input() returns 'yes'
input() returns 'no'
input() raise IndexError
"""
def __init__(self, inputs=None):
self.inputs = list(inputs) or []
self._cursor = 0
def as_global(self):
for mdname, md in mod_list:
md.input = self
md.editor_input = self
# if mdname.endswith('files'):
# md.editor_input = self
def add_input(self, inp):
self.inputs.append(inp)
def __call__(self, *args, **kwargs):
inp = self.inputs[self._cursor]
self._cursor += 1
return inp
class TestFakeInput(unittest.TestCase):
def test_input(self):
input = FakeInput(['yes', 'no'])
input = fake_env.FakeInput(['yes', 'no'])
self.assertEqual(input(), 'yes')
self.assertEqual(input(), 'no')
with self.assertRaises(IndexError):
input()
def test_input2(self):
other_input = FakeInput(['yes', 'no'])
other_input = fake_env.FakeInput(['yes', 'no'], module_list=[color])
other_input.as_global()
self.assertEqual(color.input(), 'yes')
self.assertEqual(color.input(), 'no')
@ -161,10 +33,11 @@ class TestFakeInput(unittest.TestCase):
color.input()
def test_editor_input(self):
other_input = FakeInput(['yes', 'no'])
other_input = fake_env.FakeInput(['yes', 'no'],
module_list=[content, color])
other_input.as_global()
self.assertEqual(files.editor_input(), 'yes')
self.assertEqual(files.editor_input(), 'no')
self.assertEqual(content.editor_input(), 'yes')
self.assertEqual(content.editor_input(), 'no')
with self.assertRaises(IndexError):
color.input()
@ -173,7 +46,7 @@ class CommandTestCase(unittest.TestCase):
"""Abstract TestCase intializing the fake filesystem."""
def setUp(self):
self.fs = _create_fake_fs()
self.fs = fake_env.create_fake_fs([content, filebroker, init_cmd])
def execute_cmds(self, cmds, fs=None):
""" Execute a list of commands, and capture their output
@ -189,10 +62,10 @@ class CommandTestCase(unittest.TestCase):
for cmd in cmds:
if hasattr(cmd, '__iter__'):
if len(cmd) == 2:
input = FakeInput(cmd[1])
input = fake_env.FakeInput(cmd[1], [content, uis, beets_ui, p3])
input.as_global()
_, stdout, stderr = redirect(papers_cmd.execute)(cmd[0].split())
_, stdout, stderr = fake_env.redirect(papers_cmd.execute)(cmd[0].split())
if len(cmd) == 3:
actual_out = color.undye(stdout.getvalue())
correct_out = color.undye(cmd[2])
@ -200,12 +73,14 @@ class CommandTestCase(unittest.TestCase):
else:
assert type(cmd) == str
_, stdout, stderr = redirect(papers_cmd.execute)(cmd.split())
_, stdout, stderr = fake_env.redirect(papers_cmd.execute)(cmd.split())
assert(stderr.getvalue() == '')
outs.append(color.undye(stdout.getvalue()))
return outs
def tearDown(self):
fake_env.unset_fake_fs([content, filebroker])
class DataCommandTestCase(CommandTestCase):
"""Abstract TestCase intializing the fake filesystem and
@ -214,7 +89,7 @@ class DataCommandTestCase(CommandTestCase):
def setUp(self):
CommandTestCase.setUp(self)
_copy_data(self.fs)
fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'data'), 'data')
# Actual tests
@ -222,10 +97,16 @@ class DataCommandTestCase(CommandTestCase):
class TestInit(CommandTestCase):
def test_init(self):
papers_cmd.execute('papers init -p paper_test2'.split())
self.assertEqual(set(fake_os.listdir('/paper_test2/')),
{'bibdata', 'doc', 'meta', 'papers.yaml'})
pubsdir = os.path.expanduser('~/papers_test2')
papers_cmd.execute('papers init -p {}'.format(pubsdir).split())
self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
{'bib', 'doc', 'meta'})
def test_init2(self):
pubsdir = os.path.expanduser('~/.papers')
papers_cmd.execute('papers init'.split())
self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
{'bib', 'doc', 'meta'})
class TestAdd(DataCommandTestCase):
@ -240,7 +121,7 @@ class TestAdd(DataCommandTestCase):
'papers add -b /data/pagerank.bib -d /data/pagerank.pdf',
]
self.execute_cmds(cmds)
self.assertEqual(set(fake_os.listdir('/not_default/doc')), {'Page99.pdf'})
self.assertEqual(set(self.fs['os'].listdir('/not_default/doc')), {'Page99.pdf'})
class TestList(DataCommandTestCase):
@ -288,12 +169,12 @@ class TestUsecase(DataCommandTestCase):
def test_first(self):
correct = ['Initializing papers in /paper_first.\n',
'',
'0: [Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) \n',
'[Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) \n',
'',
'',
'search network\n',
'0: [Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) search network\n',
'search network\n']
'[Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) search network\n'
]
cmds = ['papers init -p paper_first/',
'papers add -d data/pagerank.pdf -b data/pagerank.bib',
@ -302,7 +183,6 @@ class TestUsecase(DataCommandTestCase):
'papers tag Page99 network+search',
'papers tag Page99',
'papers tag search',
'papers tag 0',
]
self.assertEqual(correct, self.execute_cmds(cmds))
@ -343,18 +223,18 @@ class TestUsecase(DataCommandTestCase):
def test_editor_success(self):
cmds = ['papers init',
('papers add', [fixtures.pagerankbib]),
('papers add', [str_fixtures.bibtex_external0]),
('papers remove Page99', ['y']),
]
self.execute_cmds(cmds)
def test_edit(self):
bib = fixtures.pagerankbib
bib = str_fixtures.bibtex_external0
bib1 = re.sub('year = \{1999\}', 'year = {2007}', bib)
bib2 = re.sub('Lawrence Page', 'Lawrence Ridge', bib1)
bib3 = re.sub('Page99', 'Ridge07', bib2)
line = '0: [Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) \n'
line = '[Page99] L. Page et al. "The PageRank Citation Ranking Bringing Order to the Web" (1999) \n'
line1 = re.sub('1999', '2007', line)
line2 = re.sub('L. Page', 'L. Ridge', line1)
line3 = re.sub('Page99', 'Ridge07', line2)
@ -374,9 +254,9 @@ class TestUsecase(DataCommandTestCase):
def test_export(self):
cmds = ['papers init',
('papers add', [fixtures.pagerankbib]),
('papers add', [str_fixtures.bibtex_external0]),
'papers export Page99',
('papers export Page99 -f bibtex', [], fixtures.pagerankbib_generated),
('papers export Page99 -f bibtex', [], str_fixtures.bibtex_raw0),
'papers export Page99 -f bibyaml',
]