Fabien Benureau
11 years ago
commit
7f7c955a68
@ -1 +1,2 @@
|
||||
--ignore-directory=is:build
|
||||
--ignore-directory=is:pubs.egg-info
|
||||
|
||||
@ -1 +0,0 @@
|
||||
__version__ = 3
|
||||
@ -1,70 +0,0 @@
|
||||
from .. import repo
|
||||
from .. import files
|
||||
from ..paper import Paper, NoDocumentFile, get_bibentry_from_string
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
from .helpers import add_paper_with_docfile, extract_doc_path_from_bibdata
|
||||
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('add', help='add a paper to the repository')
|
||||
parser.add_argument('-b', '--bibfile',
|
||||
help='bibtex, bibtexml or bibyaml file', default=None)
|
||||
parser.add_argument('-d', '--docfile', help='pdf or ps file', default=None)
|
||||
parser.add_argument('-t', '--tags', help='tags associated to the paper, separated by commas',
|
||||
default=None)
|
||||
parser.add_argument('-c', '--copy', action='store_true', default=None,
|
||||
help="copy document files into library directory (default)")
|
||||
parser.add_argument('-C', '--nocopy', action='store_false', dest='copy',
|
||||
help="don't copy document files (opposite of -c)")
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
"""
|
||||
:param bibfile: bibtex file (in .bib, .bibml or .yaml format.
|
||||
:param docfile: path (no url yet) to a pdf or ps file
|
||||
"""
|
||||
|
||||
ui = get_ui()
|
||||
bibfile = args.bibfile
|
||||
docfile = args.docfile
|
||||
tags = args.tags
|
||||
copy = args.copy
|
||||
|
||||
if copy is None:
|
||||
copy = config().import_copy
|
||||
rp = repo.Repository(config())
|
||||
if bibfile is None:
|
||||
cont = True
|
||||
bibstr = ''
|
||||
while cont:
|
||||
try:
|
||||
bibstr = files.editor_input(config().edit_cmd, bibstr, suffix='.yaml')
|
||||
key, bib = get_bibentry_from_string(bibstr)
|
||||
cont = False
|
||||
except Exception:
|
||||
cont = ui.input_yn(
|
||||
question='Invalid bibfile. Edit again ?',
|
||||
default='y')
|
||||
if not cont:
|
||||
ui.exit(0)
|
||||
p = Paper(bibentry=bib, citekey=key)
|
||||
else:
|
||||
p = Paper.load(bibfile)
|
||||
if tags is not None:
|
||||
p.tags = set(tags.split(','))
|
||||
# Check if another doc file is specified in bibtex
|
||||
docfile2 = extract_doc_path_from_bibdata(p)
|
||||
if docfile is None:
|
||||
docfile = docfile2
|
||||
elif docfile2 is not None:
|
||||
ui.warning(
|
||||
"Skipping document file from bib file: %s, using %s instead."
|
||||
% (docfile2, docfile))
|
||||
try:
|
||||
add_paper_with_docfile(rp, p, docfile=docfile, copy=copy)
|
||||
except ValueError, v:
|
||||
ui.error(v.message)
|
||||
ui.exit(1)
|
||||
# TODO handle case where citekey exists
|
||||
@ -1,40 +0,0 @@
|
||||
import sys
|
||||
|
||||
from pybtex.database import BibliographyData
|
||||
|
||||
from .. import repo
|
||||
from .. import files
|
||||
from .helpers import parse_references, add_references_argument
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('export',
|
||||
help='export bibliography')
|
||||
parser.add_argument('-f', '--bib-format', default='bibtex',
|
||||
help="export format")
|
||||
add_references_argument(parser)
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
"""
|
||||
:param bib_format (in 'bibtex', 'yaml')
|
||||
"""
|
||||
|
||||
ui = get_ui()
|
||||
bib_format = args.bib_format
|
||||
references = args.references
|
||||
|
||||
rp = repo.Repository(config())
|
||||
papers = [rp.get_paper(c)
|
||||
for c in parse_references(rp, references)]
|
||||
if len(papers) == 0:
|
||||
papers = rp.all_papers()
|
||||
bib = BibliographyData()
|
||||
for p in papers:
|
||||
bib.add_entry(p.citekey, p.bibentry)
|
||||
try:
|
||||
files.write_bibdata(bib, sys.stdout, bib_format)
|
||||
except KeyError:
|
||||
ui.error("Invalid output format: %s." % bib_format)
|
||||
@ -1,70 +0,0 @@
|
||||
from .. import files
|
||||
from .. import color
|
||||
from .. import pretty
|
||||
from ..repo import InvalidReference
|
||||
from ..paper import NoDocumentFile
|
||||
from ..uis import get_ui
|
||||
|
||||
|
||||
def add_references_argument(parser, single=False):
|
||||
if single:
|
||||
parser.add_argument('reference',
|
||||
help='reference to the paper (citekey or number)')
|
||||
else:
|
||||
parser.add_argument('references', nargs='*',
|
||||
help="one or several reference to export (citekeysor numbers)")
|
||||
|
||||
|
||||
def add_docfile_to_paper(repo, paper, docfile, copy=False):
|
||||
if copy:
|
||||
repo.import_document(paper.citekey, docfile)
|
||||
else:
|
||||
paper.set_external_document(docfile)
|
||||
repo.add_or_update(paper)
|
||||
|
||||
|
||||
def add_paper_with_docfile(repo, paper, docfile=None, copy=False):
|
||||
repo.add_paper(paper)
|
||||
if docfile is not None:
|
||||
add_docfile_to_paper(repo, paper, docfile, copy=copy)
|
||||
|
||||
|
||||
def extract_doc_path_from_bibdata(paper):
|
||||
try:
|
||||
file_path = paper.get_document_file_from_bibdata(remove=True)
|
||||
if files.check_file(file_path):
|
||||
return file_path
|
||||
else:
|
||||
ui = get_ui()
|
||||
ui.warning("File does not exist for %s (%s)."
|
||||
% (paper.citekey, file_path))
|
||||
except NoDocumentFile:
|
||||
return None
|
||||
|
||||
|
||||
def parse_reference(rp, ref):
|
||||
try:
|
||||
return rp.ref2citekey(ref)
|
||||
except InvalidReference:
|
||||
ui = get_ui()
|
||||
ui.error("no paper with reference: %s."
|
||||
% color.dye(ref, color.citekey))
|
||||
ui.exit(-1)
|
||||
|
||||
|
||||
def parse_references(rp, refs):
|
||||
citekeys = [parse_reference(rp, ref) for ref in refs]
|
||||
return citekeys
|
||||
|
||||
def paper_oneliner(p, n = 0, citekey_only = False):
|
||||
if citekey_only:
|
||||
return p.citekey
|
||||
else:
|
||||
bibdesc = pretty.bib_oneliner(p.bibentry)
|
||||
return (u'{num:d}: [{citekey}] {descr} {tags}'.format(
|
||||
num=int(n),
|
||||
citekey=color.dye(p.citekey, color.purple),
|
||||
descr=bibdesc,
|
||||
tags=color.dye(' '.join(p.tags),
|
||||
color.purple, bold=True),
|
||||
)).encode('utf-8')
|
||||
@ -1,48 +0,0 @@
|
||||
from .. import repo
|
||||
from ..paper import Paper
|
||||
from .helpers import add_paper_with_docfile, extract_doc_path_from_bibdata
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('import',
|
||||
help='import paper(s) to the repository')
|
||||
parser.add_argument('bibpath',
|
||||
help='path to bibtex, bibtexml or bibyaml file (or directory)')
|
||||
parser.add_argument('-c', '--copy', action='store_true', default=None,
|
||||
help="copy document files into library directory (default)")
|
||||
parser.add_argument('-C', '--nocopy', action='store_false', dest='copy',
|
||||
help="don't copy document files (opposite of -c)")
|
||||
parser.add_argument('keys', nargs='*',
|
||||
help="one or several keys to import from the file")
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
"""
|
||||
:param bibpath: path (no url yet) to a bibliography file
|
||||
"""
|
||||
|
||||
ui = get_ui()
|
||||
bibpath = args.bibpath
|
||||
copy = args.copy
|
||||
|
||||
if copy is None:
|
||||
copy = config().import_copy
|
||||
rp = repo.Repository(config())
|
||||
# Extract papers from bib
|
||||
papers = Paper.many_from_path(bibpath)
|
||||
keys = args.keys or papers.keys()
|
||||
for k in keys:
|
||||
try:
|
||||
p = papers[k]
|
||||
if isinstance(p, Exception):
|
||||
ui.error('Could not load entry for citekey {}.'.format(k))
|
||||
else:
|
||||
doc_file = extract_doc_path_from_bibdata(p)
|
||||
if doc_file is None:
|
||||
ui.warning("No file for %s." % p.citekey)
|
||||
add_paper_with_docfile(rp, p, docfile=doc_file, copy=copy)
|
||||
except KeyError:
|
||||
ui.error('No entry found for citekey {}.'.format(k))
|
||||
@ -1,42 +0,0 @@
|
||||
# init command
|
||||
|
||||
import os
|
||||
|
||||
from ..repo import Repository
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
from .. import color
|
||||
from .. import files
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('init',
|
||||
help="initialize the papers directory")
|
||||
parser.add_argument('-p', '--path', default=None,
|
||||
help='path to papers directory (if none, ~/.papers is used)')
|
||||
parser.add_argument('-d', '--doc-dir', default=None,
|
||||
help=('path to document directory (if none, documents '
|
||||
'are stored in the same directory)'))
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
"""Create a .papers directory"""
|
||||
|
||||
ui = get_ui()
|
||||
path = args.path
|
||||
doc_dir = args.doc_dir
|
||||
|
||||
if path is not None:
|
||||
config().papers_dir = files.clean_path(os.getcwd(), path)
|
||||
ppd = config().papers_dir
|
||||
if os.path.exists(ppd) and len(os.listdir(ppd)) > 0:
|
||||
ui.error('directory {} is not empty.'.format(
|
||||
color.dye(ppd, color.filepath)))
|
||||
ui.exit()
|
||||
|
||||
ui.print_('Initializing papers in {}.'.format(
|
||||
color.dye(ppd, color.filepath)))
|
||||
|
||||
repo = Repository(config(), load = False)
|
||||
repo.save()
|
||||
config().save()
|
||||
@ -1,64 +0,0 @@
|
||||
import sys
|
||||
|
||||
from .. import repo
|
||||
from .. import color
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
from ..__init__ import __version__
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('update', help='update the repository to the lastest format')
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
|
||||
ui = get_ui()
|
||||
|
||||
code_version = __version__
|
||||
repo_version = int(config().version)
|
||||
|
||||
if repo_version == code_version:
|
||||
ui.print_('You papers repository is up-to-date.')
|
||||
sys.exit(0)
|
||||
elif repo_version > code_version:
|
||||
ui.print_('Your repository was generated with an newer version of papers.\n'
|
||||
'You should not use papers until you install the newest version.')
|
||||
sys.exit(0)
|
||||
else:
|
||||
msg = ("You should backup the paper directory {} before continuing."
|
||||
"Continue ?").format(color.dye(config().papers_dir, color.filepath))
|
||||
sure = ui.input_yn(question=msg, default='n')
|
||||
if not sure:
|
||||
sys.exit(0)
|
||||
|
||||
if repo_version == 1:
|
||||
rp = repo.Repository(config())
|
||||
for p in rp.all_papers():
|
||||
tags = set(p.metadata['tags'])
|
||||
tags = tags.union(p.metadata['labels'])
|
||||
p.metadata.pop('labels', [])
|
||||
rp.save_paper(p)
|
||||
repo_version = 2
|
||||
|
||||
|
||||
if repo_version == 2:
|
||||
# update config
|
||||
print 'bla'
|
||||
cfg_update = [('papers-directory', 'papers_dir'),
|
||||
('open-cmd', 'open_cmd'),
|
||||
('edit-cmd', 'edit_cmd'),
|
||||
('import-copy', 'import_copy'),
|
||||
('import-move', 'import_move'),
|
||||
]
|
||||
for old, new in cfg_update:
|
||||
try:
|
||||
config()._cfg.set('papers', new, config()._cfg.get('papers', old))
|
||||
config()._cfg.remove_option('papers', old)
|
||||
except Exception:
|
||||
pass
|
||||
config().save()
|
||||
repo_version = 3
|
||||
|
||||
config().version = repo_version
|
||||
config().save()
|
||||
@ -1,221 +0,0 @@
|
||||
"""
|
||||
This module can't depend on configs.
|
||||
If you feel the need to import configs, you are not in the right place.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from .p3 import io
|
||||
from io import StringIO
|
||||
|
||||
import yaml
|
||||
|
||||
from . import color
|
||||
|
||||
try:
|
||||
import pybtex
|
||||
import pybtex.database
|
||||
import pybtex.database.input
|
||||
import pybtex.database.input.bibtex
|
||||
import pybtex.database.input.bibtexml
|
||||
import pybtex.database.input.bibyaml
|
||||
import pybtex.database.output
|
||||
import pybtex.database.output.bibtex
|
||||
import pybtex.database.output.bibtexml
|
||||
import pybtex.database.output.bibyaml
|
||||
|
||||
except ImportError:
|
||||
print(color.dye('error', color.error) +
|
||||
": you need to install Pybtex; try running 'pip install "
|
||||
"pybtex' or 'easy_install pybtex'")
|
||||
exit(-1)
|
||||
|
||||
_papersdir = None
|
||||
|
||||
BIB_EXTENSIONS = ['.bib', '.bibyaml', '.bibml', '.yaml']
|
||||
FORMATS_INPUT = {'bib' : pybtex.database.input.bibtex,
|
||||
'xml' : pybtex.database.input.bibtexml,
|
||||
'yml' : pybtex.database.input.bibyaml,
|
||||
'yaml' : pybtex.database.input.bibyaml,
|
||||
'bibyaml': pybtex.database.input.bibyaml}
|
||||
FORMATS_OUTPUT = {'bib' : pybtex.database.output.bibtex,
|
||||
'bibtex' : pybtex.database.output.bibtex,
|
||||
'xml' : pybtex.database.output.bibtexml,
|
||||
'yml' : pybtex.database.output.bibyaml,
|
||||
'yaml' : pybtex.database.output.bibyaml,
|
||||
'bibyaml': pybtex.database.output.bibyaml}
|
||||
|
||||
|
||||
def clean_path(*args):
|
||||
return os.path.abspath(os.path.expanduser(os.path.join(*args)))
|
||||
|
||||
|
||||
def name_from_path(fullpdfpath, verbose=False):
|
||||
name, ext = os.path.splitext(os.path.split(fullpdfpath)[1])
|
||||
if verbose:
|
||||
if ext != '.pdf' and ext != '.ps':
|
||||
print('{}: extension {} not recognized'.format(
|
||||
color.dye('warning', color.warning),
|
||||
color.dye(ext, color.cyan)))
|
||||
return name, ext
|
||||
|
||||
|
||||
def check_directory(path, fail=False):
|
||||
if fail:
|
||||
if not os.path.exists(path):
|
||||
raise IOError("File does not exist: {}.".format(path))
|
||||
if not os.path.isdir(path):
|
||||
raise IOError("{} is not a directory.".format(path))
|
||||
return True
|
||||
else:
|
||||
return os.path.exists(path) and os.path.isdir(path)
|
||||
|
||||
|
||||
def check_file(path, fail=False):
|
||||
if fail:
|
||||
if not os.path.exists(path):
|
||||
raise IOError("File does not exist: {}.".format(path))
|
||||
if not os.path.isfile(path):
|
||||
raise IOError("{} is not a file.".format(path))
|
||||
return True
|
||||
else:
|
||||
return os.path.exists(path) and os.path.isfile(path)
|
||||
|
||||
|
||||
# yaml I/O
|
||||
|
||||
def write_yamlfile(filepath, datamap):
|
||||
try:
|
||||
with open(filepath, 'w') as f:
|
||||
yaml.dump(datamap, f)
|
||||
except IOError:
|
||||
print('{}: impossible to read or write on file {}'.format(
|
||||
color.dye('error', color.error),
|
||||
color.dye(filepath, color.filepath)))
|
||||
exit(-1)
|
||||
|
||||
|
||||
def read_yamlfile(filepath):
|
||||
check_file(filepath, fail=True)
|
||||
try:
|
||||
with open(filepath, 'r') as f:
|
||||
return yaml.load(f)
|
||||
except IOError:
|
||||
print('{}: impossible to read file {}'.format(
|
||||
color.dye('error', color.error),
|
||||
color.dye(filepath, color.filepath)))
|
||||
exit(-1)
|
||||
|
||||
|
||||
def load_bibdata(filename, filepath):
|
||||
return load_externalbibfile(filepath)
|
||||
|
||||
|
||||
def write_bibdata(bib_data, file_, format_):
|
||||
writer = FORMATS_OUTPUT[format_].Writer()
|
||||
writer.write_stream(bib_data, file_)
|
||||
|
||||
|
||||
def save_bibdata(bib_data, filepath):
|
||||
with open(filepath, 'w') as f:
|
||||
write_bibdata(bib_data, f, 'yaml')
|
||||
|
||||
|
||||
def save_meta(meta_data, filepath):
|
||||
new_meta = meta_data.copy()
|
||||
# Cannot store sets in yaml
|
||||
new_meta['tags'] = list(new_meta['tags'])
|
||||
write_yamlfile(filepath, new_meta)
|
||||
|
||||
|
||||
# is this function ever used? 08/06/2013
|
||||
def load_meta(filepath):
|
||||
return read_yamlfile(filepath)
|
||||
|
||||
|
||||
# specific to bibliography data
|
||||
|
||||
def load_externalbibfile(fullbibpath):
|
||||
check_file(fullbibpath, fail=True)
|
||||
filename, ext = os.path.splitext(os.path.split(fullbibpath)[1])
|
||||
if ext[1:] in list(FORMATS_INPUT.keys()):
|
||||
with open(fullbibpath) as f:
|
||||
return _parse_bibdata_formated_stream(f, ext[1:])
|
||||
else:
|
||||
print('{}: {} not recognized format for bibliography'.format(
|
||||
color.dye('error', color.error),
|
||||
color.dye(ext, color.cyan)))
|
||||
exit(-1)
|
||||
|
||||
|
||||
def _parse_bibdata_formated_stream(stream, fmt):
|
||||
"""Parse a stream for bibdata, using the supplied format."""
|
||||
try:
|
||||
parser = FORMATS_INPUT[fmt].Parser()
|
||||
data = parser.parse_stream(stream)
|
||||
if len(list(data.entries.keys())) > 0:
|
||||
return data
|
||||
except Exception:
|
||||
pass
|
||||
raise ValueError('content format is not recognized.')
|
||||
|
||||
def parse_bibdata(content, format_=None):
|
||||
"""Parse bib data from string or stream.
|
||||
|
||||
Raise ValueError if no bibdata is present.
|
||||
:content: stream
|
||||
:param format_: (bib|xml|yml) if format is None, tries to recognize the
|
||||
format automatically.
|
||||
"""
|
||||
fmts = [format_]
|
||||
if format_ is None:
|
||||
fmts = FORMATS_INPUT.keys()
|
||||
# we need to reuse the content
|
||||
content = content if type(content) == str else str(content.read())
|
||||
|
||||
# If you use StingIO from io then the content must be unicode
|
||||
# Let call this quick fix a hack but we should think it more carefully
|
||||
content = unicode(content)
|
||||
# This bug was really a pain in the ass to discover because of the (old) except Expection below!
|
||||
# I changed it to the only kind of error that can raise _parse_bibdata_formated_stream, which is a ValueError
|
||||
|
||||
for fmt in fmts:
|
||||
try:
|
||||
return _parse_bibdata_formated_stream(StringIO(content), fmt)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
raise ValueError('content format is not recognized.')
|
||||
|
||||
|
||||
def editor_input(editor, initial="", suffix=None):
|
||||
"""Use an editor to get input"""
|
||||
if suffix is None:
|
||||
suffix = '.tmp'
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file:
|
||||
tfile_name = temp_file.name
|
||||
temp_file.write(initial)
|
||||
temp_file.flush()
|
||||
cmd = editor.split() # this enable editor command with option, e.g. gvim -f
|
||||
cmd.append(tfile_name)
|
||||
subprocess.call(cmd)
|
||||
with open(tfile_name) as temp_file:
|
||||
content = temp_file.read()
|
||||
os.remove(tfile_name)
|
||||
return content
|
||||
|
||||
|
||||
def edit_file(editor, path_to_file, temporary=True):
|
||||
if temporary:
|
||||
check_file(path_to_file, fail=True)
|
||||
with open(path_to_file) as f:
|
||||
content = f.read()
|
||||
content = editor_input(editor, content)
|
||||
with open(path_to_file, 'w') as f:
|
||||
f.write(content)
|
||||
else:
|
||||
cmd = editor.split() # this enable editor command with option, e.g. gvim -f
|
||||
cmd.append(path_to_file)
|
||||
subprocess.call(cmd)
|
||||
@ -1,313 +0,0 @@
|
||||
import os
|
||||
|
||||
import unicodedata
|
||||
import re
|
||||
from cStringIO import StringIO
|
||||
import yaml
|
||||
|
||||
from pybtex.database import Entry, BibliographyData, FieldDict, Person
|
||||
|
||||
import files
|
||||
|
||||
|
||||
DEFAULT_TYPE = 'article'
|
||||
|
||||
CONTROL_CHARS = ''.join(map(unichr, range(0, 32) + range(127, 160)))
|
||||
CITEKEY_FORBIDDEN_CHARS = '@\'\\,#}{~%/' # '/' is OK for bibtex but forbidden
|
||||
# here since we transform citekeys into filenames
|
||||
CITEKEY_EXCLUDE_RE = re.compile('[%s]'
|
||||
% re.escape(CONTROL_CHARS + CITEKEY_FORBIDDEN_CHARS))
|
||||
|
||||
BASE_META = {
|
||||
'external-document': None,
|
||||
'tags': set(),
|
||||
'notes': [],
|
||||
}
|
||||
|
||||
|
||||
def str2citekey(s):
|
||||
key = unicodedata.normalize('NFKD', unicode(s)).encode('ascii', 'ignore')
|
||||
key = CITEKEY_EXCLUDE_RE.sub('', key)
|
||||
# Normalize chars and remove non-ascii
|
||||
return key
|
||||
|
||||
|
||||
def get_bibentry_from_file(bibfile):
|
||||
"""Extract first entry (supposed to be the only one) from given file.
|
||||
"""
|
||||
bib_data = files.load_externalbibfile(bibfile)
|
||||
first_key = list(bib_data.entries.keys())[0]
|
||||
first_entry = bib_data.entries[first_key]
|
||||
return first_key, first_entry
|
||||
|
||||
|
||||
def get_bibentry_from_string(content):
|
||||
"""Extract first entry (supposed to be the only one) from given file.
|
||||
"""
|
||||
bib_data = files.parse_bibdata(StringIO(content))
|
||||
first_key = list(bib_data.entries.keys())[0]
|
||||
first_entry = bib_data.entries[first_key]
|
||||
return first_key, first_entry
|
||||
|
||||
|
||||
def copy_person(p):
|
||||
return Person(first=p.get_part_as_text('first'),
|
||||
middle=p.get_part_as_text('middle'),
|
||||
prelast=p.get_part_as_text('prelast'),
|
||||
last=p.get_part_as_text('last'),
|
||||
lineage=p.get_part_as_text('lineage'))
|
||||
|
||||
|
||||
def copy_bibentry(entry):
|
||||
fd = FieldDict(entry.fields.parent, entry.fields)
|
||||
persons = dict([(k, [copy_person(p) for p in v])
|
||||
for k, v in entry.persons.items()])
|
||||
return Entry(entry.type, fields=fd, persons=persons)
|
||||
|
||||
|
||||
def get_safe_metadata(meta):
|
||||
base_meta = Paper.create_meta()
|
||||
if meta is not None:
|
||||
base_meta.update(meta)
|
||||
base_meta['tags'] = set(base_meta['tags'])
|
||||
return base_meta
|
||||
|
||||
|
||||
def get_safe_metadata_from_content(content):
|
||||
return get_safe_metadata(yaml.load(content))
|
||||
|
||||
|
||||
def get_safe_metadata_from_path(metapath):
|
||||
if metapath is None:
|
||||
content = None
|
||||
else:
|
||||
content = files.read_yamlfile(metapath)
|
||||
return get_safe_metadata(content)
|
||||
|
||||
|
||||
def check_citekey(citekey):
|
||||
# TODO This is not the right way to test that (17/12/2012)
|
||||
if unicode(citekey) != str2citekey(citekey):
|
||||
raise ValueError("Invalid citekey: %s" % citekey)
|
||||
|
||||
|
||||
class NoDocumentFile(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Paper(object):
|
||||
"""Paper class. The object is responsible for the integrity of its own
|
||||
data, and for loading and writing it to disc.
|
||||
|
||||
The object uses a pybtex.database.BibliographyData object to store
|
||||
biblography data and an additional dictionary to store meta data.
|
||||
"""
|
||||
|
||||
def __init__(self, bibentry=None, metadata=None, citekey=None):
|
||||
if bibentry is None:
|
||||
bibentry = Entry(DEFAULT_TYPE)
|
||||
self.bibentry = bibentry
|
||||
if metadata is None:
|
||||
metadata = Paper.create_meta()
|
||||
self.metadata = metadata
|
||||
check_citekey(citekey)
|
||||
self.citekey = citekey
|
||||
|
||||
def __eq__(self, other):
|
||||
return (isinstance(self, Paper) and type(other) is type(self)
|
||||
and self.bibentry == other.bibentry
|
||||
and self.metadata == other.metadata
|
||||
and self.citekey == other.citekey)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Paper(%s, %s, %s)' % (
|
||||
self.citekey, self.bibentry, self.metadata)
|
||||
|
||||
def __str__(self):
|
||||
return self.__repr__()
|
||||
|
||||
# TODO add mechanism to verify keys (15/12/2012)
|
||||
|
||||
def get_external_document_path(self):
|
||||
if self.metadata['external-document'] is not None:
|
||||
return self.metadata['external-document']
|
||||
else:
|
||||
raise NoDocumentFile
|
||||
|
||||
def get_document_path(self):
|
||||
return self.get_external_document_path()
|
||||
|
||||
def set_external_document(self, docpath):
|
||||
fullpdfpath = os.path.abspath(docpath)
|
||||
files.check_file(fullpdfpath, fail=True)
|
||||
self.metadata['external-document'] = fullpdfpath
|
||||
|
||||
def check_document_path(self):
|
||||
return files.check_file(self.get_external_document_path())
|
||||
|
||||
def generate_citekey(self):
|
||||
"""Generate a citekey from bib_data.
|
||||
|
||||
Raises:
|
||||
KeyError if no author nor editor is defined.
|
||||
"""
|
||||
author_key = 'author'
|
||||
if not 'author' in self.bibentry.persons:
|
||||
author_key = 'editor'
|
||||
try:
|
||||
first_author = self.bibentry.persons[author_key][0]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
'No author or editor defined: cannot generate a citekey.')
|
||||
try:
|
||||
year = self.bibentry.fields['year']
|
||||
except KeyError:
|
||||
year = ''
|
||||
citekey = u'{}{}'.format(u''.join(first_author.last()), year)
|
||||
return str2citekey(citekey)
|
||||
|
||||
def save(self, bib_filepath, meta_filepath):
|
||||
"""Creates a BibliographyData object containing a single entry and
|
||||
saves it to disc.
|
||||
"""
|
||||
if self.citekey is None:
|
||||
raise ValueError(
|
||||
'No valid citekey initialized. Cannot save paper')
|
||||
bibdata = BibliographyData(entries={self.citekey: self.bibentry})
|
||||
files.save_bibdata(bibdata, bib_filepath)
|
||||
files.save_meta(self.metadata, meta_filepath)
|
||||
|
||||
def update(self, key=None, bib=None, meta=None):
|
||||
if key is not None:
|
||||
check_citekey(key)
|
||||
self.citekey = key
|
||||
if bib is not None:
|
||||
self.bibentry = bib
|
||||
if meta is not None:
|
||||
self.metadata = meta
|
||||
|
||||
def get_document_file_from_bibdata(self, remove=False):
|
||||
"""Try extracting document file from bib data.
|
||||
Raises NoDocumentFile if not found.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
remove: default: False
|
||||
remove field after extracting information
|
||||
"""
|
||||
try:
|
||||
field = self.bibentry.fields['file']
|
||||
# Check if this is mendeley specific
|
||||
for f in field.split(':'):
|
||||
if len(f) > 0:
|
||||
break
|
||||
if remove:
|
||||
self.bibentry.fields.pop('file')
|
||||
# This is a hck for Mendeley. Make clean
|
||||
if f[0] != '/':
|
||||
f = '/' + f
|
||||
return f
|
||||
except (KeyError, IndexError):
|
||||
raise NoDocumentFile('No file found in bib data.')
|
||||
|
||||
def copy(self):
|
||||
return Paper(bibentry=copy_bibentry(self.bibentry),
|
||||
metadata=self.metadata.copy(),
|
||||
citekey=self.citekey)
|
||||
|
||||
@classmethod
|
||||
def load(cls, bibpath, metapath=None):
|
||||
key, entry = get_bibentry_from_file(bibpath)
|
||||
metadata = get_safe_metadata_from_path(metapath)
|
||||
p = Paper(bibentry=entry, metadata=metadata, citekey=key)
|
||||
return p
|
||||
|
||||
@classmethod
|
||||
def create_meta(cls):
|
||||
return BASE_META.copy()
|
||||
|
||||
@classmethod
|
||||
def many_from_path(cls, bibpath):
|
||||
"""Extract list of papers found in bibliographic files in path.
|
||||
|
||||
The behavior is to:
|
||||
- ignore wrong entries,
|
||||
- overwrite duplicated entries.
|
||||
:returns: dictionary of (key, paper | exception)
|
||||
if loading of entry failed, the excpetion is returned in the
|
||||
dictionary in place of the paper
|
||||
"""
|
||||
bibpath = files.clean_path(bibpath)
|
||||
if os.path.isdir(bibpath):
|
||||
all_files = [os.path.join(bibpath, f) for f in os.listdir(bibpath)
|
||||
if os.path.splitext(f)[-1] in files.BIB_EXTENSIONS]
|
||||
else:
|
||||
all_files = [bibpath]
|
||||
bib_data = [files.load_externalbibfile(f) for f in all_files]
|
||||
papers = {}
|
||||
for b in bib_data:
|
||||
for k in b.entries:
|
||||
try:
|
||||
papers[k] = Paper(bibentry=b.entries[k], citekey=k)
|
||||
except ValueError, e:
|
||||
papers[k] = e
|
||||
return papers
|
||||
|
||||
|
||||
# tags
|
||||
|
||||
@property
|
||||
def tags(self):
|
||||
return self.metadata.setdefault('tags', set())
|
||||
|
||||
@tags.setter
|
||||
def tags(self, value):
|
||||
if not hasattr(value, '__iter__'):
|
||||
raise ValueError('tags must be iterables')
|
||||
self.metadata['tags'] = set(value)
|
||||
|
||||
def add_tag(self, tag):
|
||||
self.tags.add(tag)
|
||||
|
||||
def remove_tag(self, tag):
|
||||
"""Remove a tag from a paper if present."""
|
||||
self.tags.discard(tag)
|
||||
|
||||
|
||||
class PaperInRepo(Paper):
|
||||
"""Extend paper class with command specific to the case where the paper
|
||||
lives in a repository.
|
||||
"""
|
||||
|
||||
def __init__(self, repo, *args, **kwargs):
|
||||
Paper.__init__(self, *args, **kwargs)
|
||||
self.repo = repo
|
||||
|
||||
def get_document_path_in_repo(self):
|
||||
return self.repo.find_document(self.citekey)
|
||||
|
||||
def get_document_path(self):
|
||||
try:
|
||||
return self.get_document_path_in_repo()
|
||||
except NoDocumentFile:
|
||||
return self.get_external_document_path()
|
||||
|
||||
def copy(self):
|
||||
return PaperInRepo.from_paper(self.as_paper().copy(), self.repo)
|
||||
|
||||
def as_paper(self):
|
||||
return Paper(bibentry=self.bibentry,
|
||||
metadata=self.metadata,
|
||||
citekey=self.citekey)
|
||||
|
||||
@classmethod
|
||||
def load(cls, repo, bibpath, metapath=None):
|
||||
key, entry = get_bibentry_from_file(bibpath)
|
||||
metadata = get_safe_metadata_from_path(metapath)
|
||||
p = cls(repo, bibentry=entry, metadata=metadata, citekey=key)
|
||||
return p
|
||||
|
||||
@classmethod
|
||||
def from_paper(cls, paper, repo):
|
||||
return cls(repo, bibentry=paper.bibentry, metadata=paper.metadata,
|
||||
citekey=paper.citekey)
|
||||
@ -1,5 +0,0 @@
|
||||
#!/usr/bin/env python2
|
||||
# -*- coding:utf-8 -*-
|
||||
|
||||
from papers import papers_cmd
|
||||
papers_cmd.execute()
|
||||
@ -1,52 +0,0 @@
|
||||
# display formatting
|
||||
|
||||
from . import color
|
||||
from pybtex.bibtex.utils import bibtex_purify
|
||||
|
||||
|
||||
# A bug in pybtex makes the abbreviation wrong here
|
||||
# (Submitted with racker ID: ID: 3605659)
|
||||
# The purification should also be applied to names but unfortunately
|
||||
# it removes dots which is annoying on abbreviations.
|
||||
def person_repr(p):
|
||||
return ' '.join(s for s in [
|
||||
' '.join(p.first(abbr=True)),
|
||||
' '.join(p.last(abbr=False)),
|
||||
' '.join(p.lineage(abbr=True))] if s)
|
||||
|
||||
|
||||
def short_authors(bibentry):
|
||||
try:
|
||||
authors = [person_repr(p) for p in bibentry.persons['author']]
|
||||
if len(authors) < 3:
|
||||
return ', '.join(authors)
|
||||
else:
|
||||
return authors[0] + (' et al.' if len(authors) > 1 else '')
|
||||
except KeyError: # When no author is defined
|
||||
return ''
|
||||
|
||||
|
||||
def bib_oneliner(bibentry):
|
||||
authors = short_authors(bibentry)
|
||||
title = bibtex_purify(bibentry.fields['title'])
|
||||
year = bibtex_purify(bibentry.fields.get('year', ''))
|
||||
journal = ''
|
||||
field = 'journal'
|
||||
if bibentry.type == 'inproceedings':
|
||||
field = 'booktitle'
|
||||
journal = bibtex_purify(bibentry.fields.get(field, ''))
|
||||
return u'{authors} \"{title}\" {journal} ({year})'.format(
|
||||
authors=color.dye(authors, color.cyan),
|
||||
title=title,
|
||||
journal=color.dye(journal, color.yellow),
|
||||
year=year,
|
||||
)
|
||||
|
||||
|
||||
def bib_desc(bib_data):
|
||||
article = bib_data.entries[list(bib_data.entries.keys())[0]]
|
||||
s = '\n'.join('author: {}'.format(person_repr(p))
|
||||
for p in article.persons['author'])
|
||||
s += '\n'
|
||||
s += '\n'.join('{}: {}'.format(k, v) for k, v in article.fields.items())
|
||||
return s
|
||||
@ -1,221 +0,0 @@
|
||||
import os
|
||||
import shutil
|
||||
import glob
|
||||
import itertools
|
||||
|
||||
from . import files
|
||||
from .paper import PaperInRepo, NoDocumentFile, check_citekey
|
||||
from .events import RemoveEvent, RenameEvent, AddEvent
|
||||
|
||||
BASE_FILE = 'papers.yaml'
|
||||
BIB_DIR = 'bibdata'
|
||||
META_DIR = 'meta'
|
||||
DOC_DIR = 'doc'
|
||||
|
||||
|
||||
class CiteKeyCollision(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidReference(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Repository(object):
|
||||
|
||||
def __init__(self, config, load=True):
|
||||
"""Initialize the repository.
|
||||
|
||||
:param load: if load is True, load the repository from disk,
|
||||
from path config.papers_dir.
|
||||
"""
|
||||
self.config = config
|
||||
self.citekeys = []
|
||||
if load:
|
||||
self.load()
|
||||
|
||||
# @classmethod
|
||||
# def from_directory(cls, config, papersdir=None):
|
||||
# repo = cls(config)
|
||||
# if papersdir is None:
|
||||
# papersdir = config.papers_dir
|
||||
# repo.papersdir = files.clean_path(papersdir)
|
||||
# repo.load()
|
||||
# return repo
|
||||
|
||||
def __contains__(self, citekey):
|
||||
"""Allows to use 'if citekey in repo' pattern"""
|
||||
return citekey in self.citekeys
|
||||
|
||||
def __len__(self):
|
||||
return len(self.citekeys)
|
||||
|
||||
# load, save repo
|
||||
def _init_dirs(self, autodoc=True):
|
||||
"""Create, if necessary, the repository directories.
|
||||
|
||||
Should only be called by load or save.
|
||||
"""
|
||||
self.bib_dir = files.clean_path(self.config.papers_dir, BIB_DIR)
|
||||
self.meta_dir = files.clean_path(self.config.papers_dir, META_DIR)
|
||||
if self.config.doc_dir == 'doc':
|
||||
self.doc_dir = files.clean_path(self.config.papers_dir, DOC_DIR)
|
||||
else:
|
||||
self.doc_dir = files.clean_path(self.config.doc_dir)
|
||||
self.cfg_path = files.clean_path(self.config.papers_dir, 'papers.yaml')
|
||||
|
||||
for d in [self.bib_dir, self.meta_dir, self.doc_dir]:
|
||||
if not os.path.exists(d):
|
||||
os.makedirs(d)
|
||||
|
||||
def load(self):
|
||||
"""Load the repository, creating dirs if necessary"""
|
||||
self._init_dirs()
|
||||
repo_config = files.read_yamlfile(self.cfg_path)
|
||||
self.citekeys = repo_config['citekeys']
|
||||
|
||||
def save(self):
|
||||
"""Save the repo, creating dirs if necessary"""
|
||||
self._init_dirs()
|
||||
repo_cfg = {'citekeys': self.citekeys}
|
||||
files.write_yamlfile(self.cfg_path, repo_cfg)
|
||||
|
||||
# reference
|
||||
def ref2citekey(self, ref):
|
||||
"""Tries to get citekey from given reference.
|
||||
Ref can be a citekey or a number.
|
||||
"""
|
||||
if ref in self.citekeys:
|
||||
return ref
|
||||
else:
|
||||
try:
|
||||
return self.citekeys[int(ref)]
|
||||
except (IndexError, ValueError):
|
||||
raise InvalidReference
|
||||
|
||||
# papers
|
||||
def all_papers(self):
|
||||
for key in self.citekeys:
|
||||
yield self.get_paper(key)
|
||||
|
||||
def get_paper(self, citekey):
|
||||
"""Load a paper by its citekey from disk, if necessary."""
|
||||
if citekey in self.citekeys:
|
||||
return PaperInRepo.load(self, self._bibfile(citekey),
|
||||
self._metafile(citekey))
|
||||
else:
|
||||
raise InvalidReference
|
||||
|
||||
def _add_citekey(self, citekey):
|
||||
if citekey not in self.citekeys:
|
||||
self.citekeys.append(citekey)
|
||||
self.save()
|
||||
|
||||
def _write_paper(self, paper):
|
||||
"""Warning: overwrites the paper without checking if it exists."""
|
||||
paper.save(self._bibfile(paper.citekey),
|
||||
self._metafile(paper.citekey))
|
||||
self._add_citekey(paper.citekey)
|
||||
|
||||
def _remove_paper(self, citekey, remove_doc=True):
|
||||
""" This version of remove is not meant to be accessed from outside.
|
||||
It removes paper without raising the Remove Event"""
|
||||
paper = self.get_paper(citekey)
|
||||
self.citekeys.remove(citekey)
|
||||
os.remove(self._metafile(citekey))
|
||||
os.remove(self._bibfile(citekey))
|
||||
# Eventually remove associated document
|
||||
if remove_doc:
|
||||
try:
|
||||
path = paper.get_document_path_in_repo()
|
||||
os.remove(path)
|
||||
except NoDocumentFile:
|
||||
pass
|
||||
self.save()
|
||||
|
||||
def _move_doc(self, old_citekey, paper):
|
||||
"""Fragile. Make more robust"""
|
||||
try:
|
||||
old_docfile = self.find_document(old_citekey)
|
||||
ext = os.path.splitext(old_docfile)[1]
|
||||
new_docfile = os.path.join(self.doc_dir, paper.citekey + ext)
|
||||
shutil.move(old_docfile, new_docfile)
|
||||
paper.set_external_document(new_docfile)
|
||||
except NoDocumentFile:
|
||||
pass
|
||||
|
||||
def _add_paper(self, paper, overwrite=False):
|
||||
check_citekey(paper.citekey)
|
||||
if not overwrite and paper.citekey in self.citekeys:
|
||||
raise CiteKeyCollision('Citekey {} already in use'.format(
|
||||
paper.citekey))
|
||||
self._write_paper(paper)
|
||||
|
||||
# add, remove papers
|
||||
def add_paper(self, paper):
|
||||
self._add_paper(paper)
|
||||
AddEvent(paper.citekey).send()
|
||||
|
||||
def save_paper(self, paper, old_citekey=None, overwrite=False):
|
||||
if old_citekey is None:
|
||||
old_citekey = paper.citekey
|
||||
if not old_citekey in self.citekeys:
|
||||
raise ValueError('Paper not in repository, first add it.')
|
||||
if old_citekey == paper.citekey:
|
||||
self._write_paper(paper)
|
||||
else:
|
||||
self._add_paper(paper, overwrite=overwrite) # This checks for collisions
|
||||
# We do not want to send the RemoveEvent, associated documents should be moved
|
||||
self._remove_paper(old_citekey, remove_doc=False)
|
||||
self._move_doc(old_citekey, paper)
|
||||
RenameEvent(paper, old_citekey).send()
|
||||
|
||||
def remove_paper(self, citekey, remove_doc=True):
|
||||
RemoveEvent(citekey).send()
|
||||
self._remove_paper(citekey, remove_doc)
|
||||
|
||||
def _bibfile(self, citekey):
|
||||
return os.path.join(self.bib_dir, citekey + '.bibyaml')
|
||||
|
||||
def _metafile(self, citekey):
|
||||
return os.path.join(self.meta_dir, citekey + '.meta')
|
||||
|
||||
def generate_citekey(self, paper, citekey=None):
|
||||
"""Create a unique citekey for the given paper."""
|
||||
if citekey is None:
|
||||
citekey = paper.generate_citekey()
|
||||
for n in itertools.count():
|
||||
if not citekey + _base27(n) in self.citekeys:
|
||||
return citekey + _base27(n)
|
||||
|
||||
def find_document(self, citekey):
|
||||
found = glob.glob('{}/{}.*'.format(self.doc_dir, citekey))
|
||||
if found:
|
||||
return found[0]
|
||||
else:
|
||||
raise NoDocumentFile
|
||||
|
||||
def import_document(self, citekey, doc_file):
|
||||
if citekey not in self.citekeys:
|
||||
raise ValueError("Unknown citekey: {}.".format(citekey))
|
||||
else:
|
||||
if not os.path.isfile(doc_file):
|
||||
raise ValueError("No file {} found.".format(doc_file))
|
||||
ext = os.path.splitext(doc_file)[1]
|
||||
new_doc_file = os.path.join(self.doc_dir, citekey + ext)
|
||||
shutil.copy(doc_file, new_doc_file)
|
||||
|
||||
def get_tags(self):
|
||||
tags = set()
|
||||
for p in self.all_papers():
|
||||
tags = tags.union(p.tags)
|
||||
return tags
|
||||
|
||||
|
||||
def _base27(n):
|
||||
return _base27((n - 1) // 26) + chr(ord('a') + ((n - 1) % 26)) if n else ''
|
||||
|
||||
|
||||
def _base(num, b):
|
||||
q, r = divmod(num - 1, len(b))
|
||||
return _base(q, b) + b[r] if num else ''
|
||||
@ -0,0 +1 @@
|
||||
__version__ = 5
|
||||
@ -0,0 +1,92 @@
|
||||
import unicodedata
|
||||
import re
|
||||
|
||||
# citekey stuff
|
||||
|
||||
CONTROL_CHARS = ''.join(map(unichr, range(0, 32) + range(127, 160)))
|
||||
CITEKEY_FORBIDDEN_CHARS = '@\'\\,#}{~%/' # '/' is OK for bibtex but forbidden
|
||||
# here since we transform citekeys into filenames
|
||||
CITEKEY_EXCLUDE_RE = re.compile('[%s]'
|
||||
% re.escape(CONTROL_CHARS + CITEKEY_FORBIDDEN_CHARS))
|
||||
|
||||
def str2citekey(s):
|
||||
key = unicodedata.normalize('NFKD', unicode(s)).encode('ascii', 'ignore')
|
||||
key = CITEKEY_EXCLUDE_RE.sub('', key)
|
||||
# Normalize chars and remove non-ascii
|
||||
return key
|
||||
|
||||
def check_citekey(citekey):
|
||||
# TODO This is not the right way to test that (17/12/2012)
|
||||
if unicode(citekey) != str2citekey(citekey):
|
||||
raise ValueError("Invalid citekey: %s" % citekey)
|
||||
|
||||
def verify_bibdata(bibdata):
|
||||
if bibdata is None or len(bibdata) == 0:
|
||||
raise ValueError('no valid bibdata')
|
||||
if len(bibdata) > 1:
|
||||
raise ValueError('ambiguous: multiple entries in the bibdata.')
|
||||
|
||||
def get_entry(bibdata):
|
||||
verify_bibdata(bibdata)
|
||||
for e in bibdata.items():
|
||||
return e
|
||||
|
||||
def extract_citekey(bibdata):
|
||||
verify_bibdata(bibdata)
|
||||
citekey, entry = get_entry(bibdata)
|
||||
return citekey
|
||||
|
||||
def author_last(author_str):
|
||||
""" Return the last name of the author """
|
||||
return author_str.split(',')[0]
|
||||
|
||||
def generate_citekey(bibdata):
|
||||
""" Generate a citekey from bib_data.
|
||||
|
||||
:param generate: if False, return the citekey defined in the file,
|
||||
does not generate a new one.
|
||||
:raise ValueError: if no author nor editor is defined.
|
||||
"""
|
||||
citekey, entry = get_entry(bibdata)
|
||||
|
||||
author_key = 'author' if 'author' in entry else 'editor'
|
||||
try:
|
||||
first_author = entry[author_key][0]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
'No author or editor defined: cannot generate a citekey.')
|
||||
try:
|
||||
year = entry['year']
|
||||
except KeyError:
|
||||
year = ''
|
||||
citekey = u'{}{}'.format(u''.join(author_last(first_author)), year)
|
||||
|
||||
return str2citekey(citekey)
|
||||
|
||||
def extract_docfile(bibdata, remove=False):
|
||||
""" Try extracting document file from bib data.
|
||||
Returns None if not found.
|
||||
|
||||
:param remove: remove field after extracting information (default: False)
|
||||
"""
|
||||
citekey, entry = get_entry(bibdata)
|
||||
|
||||
try:
|
||||
if 'file' in entry:
|
||||
field = entry['file']
|
||||
# Check if this is mendeley specific
|
||||
for f in field.split(':'):
|
||||
if len(f) > 0:
|
||||
break
|
||||
if remove:
|
||||
entry.pop('file')
|
||||
# This is a hck for Mendeley. Make clean
|
||||
if f[0] != '/':
|
||||
f = '/' + f
|
||||
return f
|
||||
if 'attachments' in entry:
|
||||
return entry['attachments']
|
||||
if 'pdf' in entry:
|
||||
return entry['pdf']
|
||||
except (KeyError, IndexError):
|
||||
return None
|
||||
@ -1,12 +1,19 @@
|
||||
import add_cmd
|
||||
import import_cmd
|
||||
import export_cmd
|
||||
# core
|
||||
import init_cmd
|
||||
import add_cmd
|
||||
import rename_cmd
|
||||
import remove_cmd
|
||||
import list_cmd
|
||||
# doc
|
||||
import attach_cmd
|
||||
import open_cmd
|
||||
import edit_cmd
|
||||
import remove_cmd
|
||||
import websearch_cmd
|
||||
import tag_cmd
|
||||
import attach_cmd
|
||||
import update_cmd
|
||||
import note_cmd
|
||||
# bulk
|
||||
import export_cmd
|
||||
import import_cmd
|
||||
# bonus
|
||||
import websearch_cmd
|
||||
|
||||
# import edit_cmd
|
||||
# import update_cmd
|
||||
@ -0,0 +1,120 @@
|
||||
import datetime
|
||||
|
||||
from ..uis import get_ui
|
||||
from ..configs import config
|
||||
from .. import bibstruct
|
||||
from .. import content
|
||||
from .. import repo
|
||||
from .. import paper
|
||||
from .. import templates
|
||||
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('add', help='add a paper to the repository')
|
||||
parser.add_argument('bibfile', nargs='?', default = None,
|
||||
help='bibtex, bibtexml or bibyaml file')
|
||||
parser.add_argument('-d', '--docfile', help='pdf or ps file', default=None)
|
||||
parser.add_argument('-t', '--tags', help='tags associated to the paper, separated by commas',
|
||||
default=None)
|
||||
parser.add_argument('-k', '--citekey', help='citekey associated with the paper;\nif not provided, one will be generated automatically.',
|
||||
default=None)
|
||||
parser.add_argument('-c', '--copy', action='store_true', default=None,
|
||||
help="copy document files into library directory (default)")
|
||||
parser.add_argument('-C', '--nocopy', action='store_false', dest='copy',
|
||||
help="don't copy document files (opposite of -c)")
|
||||
return parser
|
||||
|
||||
|
||||
def bibdata_from_editor(ui, rp):
|
||||
again = True
|
||||
bibstr = templates.add_bib
|
||||
while again:
|
||||
try:
|
||||
bibstr = content.editor_input(config().edit_cmd,
|
||||
bibstr,
|
||||
suffix='.bib')
|
||||
if bibstr == templates.add_bib:
|
||||
again = ui.input_yn(
|
||||
question='Bibfile not edited. Edit again ?',
|
||||
default='y')
|
||||
if not again:
|
||||
ui.exit(0)
|
||||
else:
|
||||
bibdata = rp.databroker.verify(bibstr)
|
||||
bibstruct.verify_bibdata(bibdata)
|
||||
# REFACTOR Generate citykey
|
||||
again = False
|
||||
except ValueError:
|
||||
again = ui.input_yn(
|
||||
question='Invalid bibfile. Edit again ?',
|
||||
default='y')
|
||||
if not again:
|
||||
ui.exit(0)
|
||||
|
||||
return bibdata
|
||||
|
||||
def command(args):
|
||||
"""
|
||||
:param bibfile: bibtex file (in .bib, .bibml or .yaml format.
|
||||
:param docfile: path (no url yet) to a pdf or ps file
|
||||
"""
|
||||
|
||||
ui = get_ui()
|
||||
bibfile = args.bibfile
|
||||
docfile = args.docfile
|
||||
tags = args.tags
|
||||
citekey = args.copy
|
||||
|
||||
rp = repo.Repository(config())
|
||||
|
||||
# get bibfile
|
||||
|
||||
if bibfile is None:
|
||||
bibdata = bibdata_from_editor(ui, rp)
|
||||
else:
|
||||
bibdata_raw = content.get_content(bibfile)
|
||||
bibdata = rp.databroker.verify(bibdata_raw)
|
||||
if bibdata is None:
|
||||
ui.error('invalid bibfile {}.'.format(bibfile))
|
||||
|
||||
# citekey
|
||||
|
||||
citekey = args.citekey
|
||||
if citekey is None:
|
||||
base_key = bibstruct.extract_citekey(bibdata)
|
||||
citekey = rp.unique_citekey(base_key)
|
||||
else:
|
||||
rp.databroker.exists(citekey, both=False)
|
||||
|
||||
# tags
|
||||
|
||||
if tags is not None:
|
||||
p.tags = set(tags.split(','))
|
||||
|
||||
p = paper.Paper(bibdata, citekey=citekey)
|
||||
p.added = datetime.datetime.now()
|
||||
|
||||
# document file
|
||||
|
||||
bib_docfile = bibstruct.extract_docfile(bibdata)
|
||||
if docfile is None:
|
||||
docfile = bib_docfile
|
||||
elif bib_docfile is not None:
|
||||
ui.warning(('Skipping document file from bib file '
|
||||
'{}, using {} instead.').format(bib_docfile, docfile))
|
||||
|
||||
if docfile is not None:
|
||||
copy_doc = args.copy
|
||||
if copy_doc is None:
|
||||
copy_doc = config().import_copy
|
||||
if copy_doc:
|
||||
docfile = rp.databroker.add_doc(citekey, docfile)
|
||||
|
||||
# create the paper
|
||||
|
||||
try:
|
||||
p.docpath = docfile
|
||||
rp.push_paper(p)
|
||||
except ValueError, v:
|
||||
ui.error(v.message)
|
||||
ui.exit(1)
|
||||
@ -0,0 +1,44 @@
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
|
||||
from .. import repo
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
from .. import endecoder
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('export',
|
||||
help='export bibliography')
|
||||
# parser.add_argument('-f', '--bib-format', default='bibtex',
|
||||
# help='export format')
|
||||
parser.add_argument('citekeys', nargs='*',
|
||||
help='one or several citekeys')
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
"""
|
||||
"""
|
||||
# :param bib_format (only 'bibtex' now)
|
||||
|
||||
ui = get_ui()
|
||||
|
||||
rp = repo.Repository(config())
|
||||
|
||||
try:
|
||||
papers = [rp.pull_paper(c) for c in args.citekeys]
|
||||
except repo.InvalidReference, v:
|
||||
ui.error(v)
|
||||
ui.exit(1)
|
||||
|
||||
if len(papers) == 0:
|
||||
papers = rp.all_papers()
|
||||
bib = {}
|
||||
for p in papers:
|
||||
bib[p.citekey] = p.bibentry
|
||||
try:
|
||||
exporter = endecoder.EnDecoder()
|
||||
bibdata_raw = exporter.encode_bibdata(bib)
|
||||
print(bibdata_raw, end='')
|
||||
except KeyError:
|
||||
ui.error("Invalid output format: %s." % bib_format)
|
||||
@ -0,0 +1,104 @@
|
||||
import os
|
||||
import datetime
|
||||
|
||||
from .. import repo
|
||||
from .. import endecoder
|
||||
from .. import bibstruct
|
||||
from .. import color
|
||||
from ..paper import Paper
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
|
||||
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('import',
|
||||
help='import paper(s) to the repository')
|
||||
parser.add_argument('bibpath',
|
||||
help='path to bibtex, bibtexml or bibyaml file (or directory)')
|
||||
parser.add_argument('-c', '--copy', action='store_true', default=None,
|
||||
help="copy document files into library directory (default)")
|
||||
parser.add_argument('-C', '--nocopy', action='store_false', dest='copy',
|
||||
help="don't copy document files (opposite of -c)")
|
||||
parser.add_argument('keys', nargs='*',
|
||||
help="one or several keys to import from the file")
|
||||
return parser
|
||||
|
||||
|
||||
def many_from_path(bibpath):
|
||||
"""Extract list of papers found in bibliographic files in path.
|
||||
|
||||
The behavior is to:
|
||||
- ignore wrong entries,
|
||||
- overwrite duplicated entries.
|
||||
:returns: dictionary of (key, paper | exception)
|
||||
if loading of entry failed, the excpetion is returned in the
|
||||
dictionary in place of the paper
|
||||
"""
|
||||
coder = endecoder.EnDecoder()
|
||||
|
||||
bibpath = os.path.expanduser(bibpath)
|
||||
if os.path.isdir(bibpath):
|
||||
print([os.path.splitext(f)[-1][1:] for f in os.listdir(bibpath)])
|
||||
all_files = [os.path.join(bibpath, f) for f in os.listdir(bibpath)
|
||||
if os.path.splitext(f)[-1][1:] == 'bib']
|
||||
else:
|
||||
all_files = [bibpath]
|
||||
|
||||
biblist = []
|
||||
for filepath in all_files:
|
||||
with open(filepath, 'r') as f:
|
||||
biblist.append(coder.decode_bibdata(f.read()))
|
||||
|
||||
papers = {}
|
||||
for b in biblist:
|
||||
for k in b.keys():
|
||||
try:
|
||||
bibdata = {}
|
||||
bibdata[k] = b[k]
|
||||
|
||||
papers[k] = Paper(bibdata, citekey=k)
|
||||
papers[k].added = datetime.datetime.now()
|
||||
except ValueError, e:
|
||||
papers[k] = e
|
||||
return papers
|
||||
|
||||
|
||||
def command(args):
|
||||
"""
|
||||
:param bibpath: path (no url yet) to a bibliography file
|
||||
"""
|
||||
|
||||
ui = get_ui()
|
||||
bibpath = args.bibpath
|
||||
copy = args.copy
|
||||
|
||||
if copy is None:
|
||||
copy = config().import_copy
|
||||
rp = repo.Repository(config())
|
||||
# Extract papers from bib
|
||||
papers = many_from_path(bibpath)
|
||||
keys = args.keys or papers.keys()
|
||||
for k in keys:
|
||||
try:
|
||||
p = papers[k]
|
||||
if isinstance(p, Exception):
|
||||
ui.error('could not load entry for citekey {}.'.format(k))
|
||||
else:
|
||||
docfile = bibstruct.extract_docfile(p.bibdata)
|
||||
if docfile is None:
|
||||
ui.warning("no file for {}.".format(p.citekey))
|
||||
else:
|
||||
copy_doc = args.copy
|
||||
if copy_doc is None:
|
||||
copy_doc = config().import_copy
|
||||
if copy_doc:
|
||||
docfile = rp.databroker.add_doc(p.citekey, docfile)
|
||||
|
||||
p.docpath = docfile
|
||||
rp.push_paper(p)
|
||||
ui.print_('{} imported'.format(color.dye(p.citekey, color.cyan)))
|
||||
except KeyError:
|
||||
ui.error('no entry found for citekey {}.'.format(k))
|
||||
except IOError, e:
|
||||
ui.error(e.message)
|
||||
@ -0,0 +1,46 @@
|
||||
# init command
|
||||
|
||||
import os
|
||||
|
||||
from .. import databroker
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
from .. import color
|
||||
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('init',
|
||||
help="initialize the pubs directory")
|
||||
parser.add_argument('-p', '--pubsdir', default=None,
|
||||
help='path to pubs directory (if none, ~/.ubs is used)')
|
||||
parser.add_argument('-d', '--docsdir', default='docsdir://',
|
||||
help=('path to document directory (if not specified, documents will'
|
||||
'be stored in /path/to/pubsdir/doc/)'))
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
"""Create a .pubs directory"""
|
||||
|
||||
ui = get_ui()
|
||||
pubsdir = args.pubsdir
|
||||
docsdir = args.docsdir
|
||||
|
||||
if pubsdir is None:
|
||||
pubsdir = '~/.pubs'
|
||||
|
||||
pubsdir = os.path.normpath(os.path.abspath(os.path.expanduser(pubsdir)))
|
||||
|
||||
if os.path.exists(pubsdir) and len(os.listdir(pubsdir)) > 0:
|
||||
ui.error('directory {} is not empty.'.format(
|
||||
color.dye(pubsdir, color.filepath)))
|
||||
ui.exit()
|
||||
|
||||
ui.print_('Initializing pubs in {}.'.format(
|
||||
color.dye(pubsdir, color.filepath)))
|
||||
|
||||
config().pubsdir = pubsdir
|
||||
config().docsdir = docsdir
|
||||
config().save()
|
||||
|
||||
databroker.DataBroker(pubsdir, create=True)
|
||||
@ -0,0 +1,27 @@
|
||||
from .. import repo
|
||||
from .. import content
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('note',
|
||||
help='edit the note attached to a paper')
|
||||
parser.add_argument('citekey',
|
||||
help='citekey of the paper')
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
"""
|
||||
"""
|
||||
|
||||
ui = get_ui()
|
||||
|
||||
|
||||
rp = repo.Repository(config())
|
||||
if not rp.databroker.exists(args.citekey):
|
||||
ui.error("citekey {} not found".format(args.citekey))
|
||||
ui.exit(1)
|
||||
|
||||
notepath = rp.databroker.real_notepath(args.citekey)
|
||||
content.edit_file(config().edit_cmd, notepath, temporary=False)
|
||||
@ -0,0 +1,27 @@
|
||||
from ..uis import get_ui
|
||||
from ..configs import config
|
||||
from .. import bibstruct
|
||||
from .. import content
|
||||
from .. import repo
|
||||
from .. import paper
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('rename', help='rename the citekey of a repository')
|
||||
parser.add_argument('citekey',
|
||||
help='current citekey')
|
||||
parser.add_argument('new_citekey',
|
||||
help='new citekey')
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
"""
|
||||
:param bibfile: bibtex file (in .bib, .bibml or .yaml format.
|
||||
:param docfile: path (no url yet) to a pdf or ps file
|
||||
"""
|
||||
|
||||
ui = get_ui()
|
||||
rp = repo.Repository(config())
|
||||
|
||||
paper = rp.pull_paper(args.citekey)
|
||||
rp.rename_paper(paper, args.new_citekey)
|
||||
@ -0,0 +1,37 @@
|
||||
import sys
|
||||
|
||||
from .. import repo
|
||||
from .. import color
|
||||
from ..configs import config
|
||||
from ..uis import get_ui
|
||||
from ..__init__ import __version__
|
||||
|
||||
def parser(subparsers):
|
||||
parser = subparsers.add_parser('update', help='update the repository to the lastest format')
|
||||
return parser
|
||||
|
||||
|
||||
def command(args):
|
||||
|
||||
ui = get_ui()
|
||||
|
||||
code_version = __version__
|
||||
repo_version = int(config().version)
|
||||
|
||||
if repo_version == code_version:
|
||||
ui.print_('Your pubs repository is up-to-date.')
|
||||
sys.exit(0)
|
||||
elif repo_version > code_version:
|
||||
ui.print_('Your repository was generated with an newer version of pubs.\n'
|
||||
'You should not use pubs until you install the newest version.')
|
||||
sys.exit(0)
|
||||
else:
|
||||
msg = ("You should backup the pubs directory {} before continuing."
|
||||
"Continue ?").format(color.dye(config().papers_dir, color.filepath))
|
||||
sure = ui.input_yn(question=msg, default='n')
|
||||
if not sure:
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
# config().version = repo_version
|
||||
# config().save()
|
||||
@ -0,0 +1,124 @@
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
import urlparse
|
||||
import httplib
|
||||
import urllib2
|
||||
|
||||
import uis
|
||||
|
||||
|
||||
# files i/o
|
||||
|
||||
def check_file(path, fail=True):
|
||||
if fail:
|
||||
if not os.path.exists(path):
|
||||
raise IOError("File does not exist: {}.".format(path))
|
||||
if not os.path.isfile(path):
|
||||
raise IOError("{} is not a file.".format(path))
|
||||
return True
|
||||
else:
|
||||
return os.path.exists(path) and os.path.isfile(path)
|
||||
|
||||
def check_directory(path, fail=True):
|
||||
if fail:
|
||||
if not os.path.exists(path):
|
||||
raise IOError("File does not exist: {}.".format(path))
|
||||
if not os.path.isdir(path):
|
||||
raise IOError("{} is not a directory.".format(path))
|
||||
return True
|
||||
else:
|
||||
return os.path.exists(path) and os.path.isdir(path)
|
||||
|
||||
def read_file(filepath):
|
||||
check_file(filepath)
|
||||
with open(filepath, 'r') as f:
|
||||
s = f.read()
|
||||
return s
|
||||
|
||||
def write_file(filepath, data):
|
||||
check_directory(os.path.dirname(filepath))
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(data)
|
||||
|
||||
|
||||
# dealing with formatless content
|
||||
|
||||
def content_type(path):
|
||||
parsed = urlparse.urlparse(path)
|
||||
if parsed.scheme == 'http':
|
||||
return 'url'
|
||||
else:
|
||||
return 'file'
|
||||
|
||||
def url_exists(url):
|
||||
parsed = urlparse.urlparse(url)
|
||||
conn = httplib.HTTPConnection(parsed.netloc)
|
||||
conn.request('HEAD', parsed.path)
|
||||
response = conn.getresponse()
|
||||
conn.close()
|
||||
return response.status == 200
|
||||
|
||||
|
||||
def check_content(path):
|
||||
if content_type(path) == 'url':
|
||||
return url_exists(path)
|
||||
else:
|
||||
return check_file(path)
|
||||
|
||||
def get_content(path):
|
||||
"""Will be useful when we need to get content from url"""
|
||||
if content_type(path) == 'url':
|
||||
uis.get_ui().print_('dowloading {}'.format(path))
|
||||
response = urllib2.urlopen(path)
|
||||
return response.read()
|
||||
else:
|
||||
return read_file(path)
|
||||
|
||||
def move_content(source, target, overwrite = False):
|
||||
if source == target:
|
||||
return
|
||||
if not overwrite and os.path.exists(target):
|
||||
raise IOError('target file exists')
|
||||
shutil.move(source, target)
|
||||
|
||||
def copy_content(source, target, overwrite = False):
|
||||
if source == target:
|
||||
return
|
||||
if not overwrite and os.path.exists(target):
|
||||
raise IOError('target file exists')
|
||||
shutil.copy(source, target)
|
||||
|
||||
|
||||
# editor input
|
||||
|
||||
def editor_input(editor, initial="", suffix=None):
|
||||
"""Use an editor to get input"""
|
||||
if suffix is None:
|
||||
suffix = '.tmp'
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file:
|
||||
tfile_name = temp_file.name
|
||||
temp_file.write(initial)
|
||||
temp_file.flush()
|
||||
cmd = editor.split() # this enable editor command with option, e.g. gvim -f
|
||||
cmd.append(tfile_name)
|
||||
subprocess.call(cmd)
|
||||
with open(tfile_name) as temp_file:
|
||||
content = temp_file.read()
|
||||
os.remove(tfile_name)
|
||||
return content
|
||||
|
||||
def edit_file(editor, path_to_file, temporary=True):
|
||||
if temporary:
|
||||
check_file(path_to_file, fail=True)
|
||||
with open(path_to_file) as f:
|
||||
content = f.read()
|
||||
content = editor_input(editor, content)
|
||||
with open(path_to_file, 'w') as f:
|
||||
f.write(content)
|
||||
else:
|
||||
cmd = editor.split() # this enable editor command with option, e.g. gvim -f
|
||||
cmd.append(path_to_file)
|
||||
subprocess.call(cmd)
|
||||
@ -0,0 +1,84 @@
|
||||
from . import filebroker
|
||||
from . import endecoder
|
||||
|
||||
|
||||
class DataBroker(object):
|
||||
""" DataBroker class
|
||||
|
||||
This is aimed at being a simple, high level interface to the content stored on disk.
|
||||
Requests are optimistically made, and exceptions are raised if something goes wrong.
|
||||
"""
|
||||
|
||||
def __init__(self, directory, create=False):
|
||||
self.filebroker = filebroker.FileBroker(directory, create=create)
|
||||
self.endecoder = endecoder.EnDecoder()
|
||||
self.docbroker = filebroker.DocBroker(directory, scheme='docsdir', subdir='doc')
|
||||
self.notebroker = filebroker.DocBroker(directory, scheme='notesdir', subdir='notes')
|
||||
|
||||
# filebroker+endecoder
|
||||
|
||||
def pull_metadata(self, citekey):
|
||||
metadata_raw = self.filebroker.pull_metafile(citekey)
|
||||
return self.endecoder.decode_metadata(metadata_raw)
|
||||
|
||||
def pull_bibdata(self, citekey):
|
||||
bibdata_raw = self.filebroker.pull_bibfile(citekey)
|
||||
return self.endecoder.decode_bibdata(bibdata_raw)
|
||||
|
||||
def push_metadata(self, citekey, metadata):
|
||||
metadata_raw = self.endecoder.encode_metadata(metadata)
|
||||
self.filebroker.push_metafile(citekey, metadata_raw)
|
||||
|
||||
def push_bibdata(self, citekey, bibdata):
|
||||
bibdata_raw = self.endecoder.encode_bibdata(bibdata)
|
||||
self.filebroker.push_bibfile(citekey, bibdata_raw)
|
||||
|
||||
def push(self, citekey, metadata, bibdata):
|
||||
self.filebroker.push(citekey, metadata, bibdata)
|
||||
|
||||
def remove(self, citekey):
|
||||
self.filebroker.remove(citekey)
|
||||
|
||||
def exists(self, citekey, both = True):
|
||||
return self.filebroker.exists(citekey, both=both)
|
||||
|
||||
def listing(self, filestats=True):
|
||||
return self.filebroker.listing(filestats=filestats)
|
||||
|
||||
def verify(self, bibdata_raw):
|
||||
try:
|
||||
return self.endecoder.decode_bibdata(bibdata_raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
# docbroker
|
||||
|
||||
def in_docsdir(self, docpath):
|
||||
return self.docbroker.in_docsdir(docpath)
|
||||
|
||||
def real_docpath(self, docpath):
|
||||
return self.docbroker.real_docpath(docpath)
|
||||
|
||||
def add_doc(self, citekey, source_path, overwrite=False):
|
||||
return self.docbroker.add_doc(citekey, source_path, overwrite=overwrite)
|
||||
|
||||
def remove_doc(self, docpath, silent=True):
|
||||
return self.docbroker.remove_doc(docpath, silent=silent)
|
||||
|
||||
def rename_doc(self, docpath, new_citekey):
|
||||
return self.docbroker.rename_doc(docpath, new_citekey)
|
||||
|
||||
# notesbroker
|
||||
|
||||
def real_notepath(self, citekey):
|
||||
notepath = 'notesdir://{}.txt'.format(citekey)
|
||||
return self.notebroker.real_docpath(notepath)
|
||||
|
||||
def remove_note(self, citekey, silent=True):
|
||||
notepath = 'notesdir://{}.txt'.format(citekey)
|
||||
return self.notebroker.remove_doc(notepath, silent=silent)
|
||||
|
||||
def rename_note(self, old_citekey, new_citekey):
|
||||
notepath = 'notesdir://{}.txt'.format(old_citekey)
|
||||
return self.notebroker.rename_doc(notepath, new_citekey)
|
||||
|
||||
@ -0,0 +1,100 @@
|
||||
|
||||
from . import databroker
|
||||
|
||||
class DataCache(object):
|
||||
""" DataCache class, provides a very similar interface as DataBroker
|
||||
|
||||
Has two roles :
|
||||
1. Provides a buffer between the commands and the hard drive.
|
||||
Until a command request a hard drive ressource, it does not touch it.
|
||||
2. Keeps a up-to-date, pickled version of the repository, to speed up things
|
||||
when they are a lot of files. Update are also done only when required.
|
||||
Changes are detected using data modification timestamps.
|
||||
|
||||
For the moment, only (1) is implemented.
|
||||
"""
|
||||
def __init__(self, directory, create=False):
|
||||
self.directory = directory
|
||||
self._databroker = None
|
||||
if create:
|
||||
self._create()
|
||||
|
||||
@property
|
||||
def databroker(self):
|
||||
if self._databroker is None:
|
||||
self._databroker = databroker.DataBroker(self.directory, create=False)
|
||||
return self._databroker
|
||||
|
||||
def _create(self):
|
||||
self._databroker = databroker.DataBroker(self.directory, create=True)
|
||||
|
||||
def pull_metadata(self, citekey):
|
||||
return self.databroker.pull_metadata(citekey)
|
||||
|
||||
def pull_bibdata(self, citekey):
|
||||
return self.databroker.pull_bibdata(citekey)
|
||||
|
||||
def push_metadata(self, citekey, metadata):
|
||||
self.databroker.push_metadata(citekey, metadata)
|
||||
|
||||
def push_bibdata(self, citekey, bibdata):
|
||||
self.databroker.push_bibdata(citekey, bibdata)
|
||||
|
||||
def push(self, citekey, metadata, bibdata):
|
||||
self.databroker.push(citekey, metadata, bibdata)
|
||||
|
||||
def remove(self, citekey):
|
||||
self.databroker.remove(citekey)
|
||||
|
||||
def exists(self, citekey, both=True):
|
||||
return self.databroker.exists(citekey, both=both)
|
||||
|
||||
def citekeys(self):
|
||||
listings = self.listing(filestats=False)
|
||||
return set(listings['metafiles']).intersection(listings['bibfiles'])
|
||||
|
||||
def listing(self, filestats=True):
|
||||
return self.databroker.listing(filestats=filestats)
|
||||
|
||||
def verify(self, bibdata_raw):
|
||||
"""Will return None if bibdata_raw can't be decoded"""
|
||||
return self.databroker.verify(bibdata_raw)
|
||||
|
||||
# docbroker
|
||||
|
||||
def in_docsdir(self, docpath):
|
||||
return self.databroker.in_docsdir(docpath)
|
||||
|
||||
def real_docpath(self, docpath):
|
||||
return self.databroker.real_docpath(docpath)
|
||||
|
||||
def add_doc(self, citekey, source_path, overwrite=False):
|
||||
return self.databroker.add_doc(citekey, source_path, overwrite=overwrite)
|
||||
|
||||
def remove_doc(self, docpath, silent=True):
|
||||
return self.databroker.remove_doc(docpath, silent=silent)
|
||||
|
||||
def rename_doc(self, docpath, new_citekey):
|
||||
return self.databroker.rename_doc(docpath, new_citekey)
|
||||
|
||||
# notesbroker
|
||||
|
||||
def real_notepath(self, citekey):
|
||||
return self.databroker.real_notepath(citekey)
|
||||
|
||||
def remove_note(self, citekey, silent=True):
|
||||
return self.databroker.remove_note(citekey, silent=True)
|
||||
|
||||
def rename_note(self, old_citekey, new_citekey):
|
||||
return self.databroker.rename_note(old_citekey, new_citekey)
|
||||
|
||||
|
||||
# class ChangeTracker(object):
|
||||
|
||||
# def __init__(self, cache, directory):
|
||||
# self.cache = cache
|
||||
# self.directory = directory
|
||||
|
||||
# def changes(self):
|
||||
# """ Returns the list of modified files since the last cache was saved to disk"""
|
||||
# pass
|
||||
@ -0,0 +1,111 @@
|
||||
from __future__ import print_function, absolute_import, division, unicode_literals
|
||||
import copy
|
||||
|
||||
try:
|
||||
import cStringIO as StringIO
|
||||
except ImportError:
|
||||
import StringIO
|
||||
|
||||
try:
|
||||
import bibtexparser as bp
|
||||
except ImportError:
|
||||
print(color.dye('error', color.error) +
|
||||
": you need to install bibterxparser; try running 'pip install "
|
||||
"bibtexparser'.")
|
||||
exit(-1)
|
||||
|
||||
import yaml
|
||||
|
||||
from . import color
|
||||
|
||||
|
||||
def sanitize_citekey(record):
|
||||
record['id'] = record['id'].strip('\n')
|
||||
return record
|
||||
|
||||
def customizations(record):
|
||||
""" Use some functions delivered by the library
|
||||
|
||||
:param record: a record
|
||||
:returns: -- customized record
|
||||
"""
|
||||
record = bp.customization.convert_to_unicode(record)
|
||||
record = bp.customization.type(record)
|
||||
record = bp.customization.author(record)
|
||||
record = bp.customization.editor(record)
|
||||
record = bp.customization.journal(record)
|
||||
record = bp.customization.keyword(record)
|
||||
record = bp.customization.link(record)
|
||||
record = bp.customization.page_double_hyphen(record)
|
||||
record = bp.customization.doi(record)
|
||||
|
||||
record = sanitize_citekey(record)
|
||||
|
||||
return record
|
||||
|
||||
bibfield_order = ['author', 'title', 'journal', 'institution', 'publisher', 'year', 'month', 'number', 'pages', 'link', 'doi', 'id', 'note', 'abstract']
|
||||
|
||||
|
||||
class EnDecoder(object):
|
||||
""" Encode and decode content.
|
||||
|
||||
Design choices:
|
||||
* Has no interaction with disk.
|
||||
* Incoming content is not trusted.
|
||||
* Returned content must be correctly formatted (no one else checks).
|
||||
* Failures raise ValueError
|
||||
* encode_bibdata will try to recognize exceptions
|
||||
"""
|
||||
|
||||
def encode_metadata(self, metadata):
|
||||
return yaml.safe_dump(metadata, allow_unicode=True, encoding='UTF-8', indent = 4)
|
||||
|
||||
def decode_metadata(self, metadata_raw):
|
||||
return yaml.safe_load(metadata_raw)
|
||||
|
||||
def encode_bibdata(self, bibdata):
|
||||
"""Encode bibdata """
|
||||
return '\n'.join(self._encode_bibentry(citekey, entry)
|
||||
for citekey, entry in bibdata.items())
|
||||
|
||||
@staticmethod
|
||||
def _encode_field(key, value):
|
||||
if key == 'link':
|
||||
return ', '.join(link['url'] for link in value)
|
||||
elif key == 'author':
|
||||
return ' and '.join(author for author in value)
|
||||
elif key == 'journal':
|
||||
return value['name']
|
||||
else:
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _encode_bibentry(citekey, bibentry):
|
||||
bibraw = '@{}{{{},\n'.format(bibentry['type'], citekey)
|
||||
bibentry = copy.copy(bibentry)
|
||||
for key in bibfield_order:
|
||||
if key in bibentry:
|
||||
value = bibentry.pop(key)
|
||||
bibraw += ' {} = {{{}}},\n'.format(key, EnDecoder._encode_field(key, value))
|
||||
for key, value in bibentry.items():
|
||||
if key != 'type':
|
||||
bibraw += ' {} = {{{}}},\n'.format(key, EnDecoder._encode_field(key, value))
|
||||
bibraw += '}\n'
|
||||
return bibraw
|
||||
|
||||
def decode_bibdata(self, bibdata_raw):
|
||||
""""""
|
||||
bibdata_rawutf8 = bibdata_raw
|
||||
#bibdata_rawutf8 = unicode(bibdata_raw, 'utf8') # FIXME this doesn't work
|
||||
bibdata_stream = StringIO.StringIO(bibdata_rawutf8)
|
||||
return self._decode_bibdata(bibdata_stream)
|
||||
|
||||
def _decode_bibdata(self, bibdata_stream):
|
||||
try:
|
||||
entries = bp.bparser.BibTexParser(bibdata_stream, customization=customizations).get_entry_dict()
|
||||
if len(entries) > 0:
|
||||
return entries
|
||||
except Exception:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise ValueError('could not parse bibdata')
|
||||
@ -0,0 +1,199 @@
|
||||
import os
|
||||
import shutil
|
||||
import re
|
||||
import urlparse
|
||||
|
||||
from .content import check_file, check_directory, read_file, write_file
|
||||
from .content import check_content, content_type, get_content
|
||||
|
||||
def filter_filename(filename, ext):
|
||||
""" Return the filename without the extension if the extension matches ext.
|
||||
Otherwise return None
|
||||
"""
|
||||
pattern ='.*\{}$'.format(ext)
|
||||
if re.match(pattern, filename) is not None:
|
||||
return filename[:-len(ext)]
|
||||
|
||||
class FileBroker(object):
|
||||
""" Handles all access to meta and bib files of the repository.
|
||||
|
||||
* Does *absolutely no* encoding/decoding.
|
||||
* Communicate failure with exceptions.
|
||||
"""
|
||||
|
||||
def __init__(self, directory, create=False):
|
||||
self.directory = directory
|
||||
self.metadir = os.path.join(self.directory, 'meta')
|
||||
self.bibdir = os.path.join(self.directory, 'bib')
|
||||
if create:
|
||||
self._create()
|
||||
check_directory(self.directory)
|
||||
check_directory(self.metadir)
|
||||
check_directory(self.bibdir)
|
||||
|
||||
def _create(self):
|
||||
if not check_directory(self.directory, fail = False):
|
||||
os.mkdir(self.directory)
|
||||
if not check_directory(self.metadir, fail = False):
|
||||
os.mkdir(self.metadir)
|
||||
if not check_directory(self.bibdir, fail = False):
|
||||
os.mkdir(self.bibdir)
|
||||
|
||||
def pull_metafile(self, citekey):
|
||||
filepath = os.path.join(self.metadir, citekey + '.yaml')
|
||||
return read_file(filepath)
|
||||
|
||||
def pull_bibfile(self, citekey):
|
||||
filepath = os.path.join(self.bibdir, citekey + '.bib')
|
||||
return read_file(filepath)
|
||||
|
||||
def push_metafile(self, citekey, metadata):
|
||||
"""Put content to disk. Will gladly override anything standing in its way."""
|
||||
filepath = os.path.join(self.metadir, citekey + '.yaml')
|
||||
write_file(filepath, metadata)
|
||||
|
||||
def push_bibfile(self, citekey, bibdata):
|
||||
"""Put content to disk. Will gladly override anything standing in its way."""
|
||||
filepath = os.path.join(self.bibdir, citekey + '.bib')
|
||||
write_file(filepath, bibdata)
|
||||
|
||||
def push(self, citekey, metadata, bibdata):
|
||||
"""Put content to disk. Will gladly override anything standing in its way."""
|
||||
self.push_metafile(citekey, metadata)
|
||||
self.push_bibfile(citekey, bibdata)
|
||||
|
||||
def remove(self, citekey):
|
||||
metafilepath = os.path.join(self.metadir, citekey + '.yaml')
|
||||
if check_file(metafilepath):
|
||||
os.remove(metafilepath)
|
||||
bibfilepath = os.path.join(self.bibdir, citekey + '.bib')
|
||||
if check_file(bibfilepath):
|
||||
os.remove(bibfilepath)
|
||||
|
||||
def exists(self, citekey, both=True):
|
||||
if both:
|
||||
return (check_file(os.path.join(self.metadir, citekey + '.yaml'), fail=False) and
|
||||
check_file(os.path.join(self.bibdir, citekey + '.bib'), fail=False))
|
||||
else:
|
||||
return (check_file(os.path.join(self.metadir, citekey + '.yaml'), fail=False) or
|
||||
check_file(os.path.join(self.bibdir, citekey + '.bib'), fail=False))
|
||||
|
||||
|
||||
def listing(self, filestats=True):
|
||||
metafiles = []
|
||||
for filename in os.listdir(self.metadir):
|
||||
citekey = filter_filename(filename, '.yaml')
|
||||
if citekey is not None:
|
||||
if filestats:
|
||||
stats = os.stat(os.path.join(path, filename))
|
||||
metafiles.append(citekey, stats)
|
||||
else:
|
||||
metafiles.append(citekey)
|
||||
|
||||
bibfiles = []
|
||||
for filename in os.listdir(self.bibdir):
|
||||
citekey = filter_filename(filename, '.bib')
|
||||
if citekey is not None:
|
||||
if filestats:
|
||||
stats = os.stat(os.path.join(path, filename))
|
||||
bibfiles.append(citekey, stats)
|
||||
else:
|
||||
bibfiles.append(citekey)
|
||||
|
||||
return {'metafiles': metafiles, 'bibfiles': bibfiles}
|
||||
|
||||
|
||||
class DocBroker(object):
|
||||
""" DocBroker manages the document files optionally attached to the papers.
|
||||
|
||||
* only one document can be attached to a paper (might change in the future)
|
||||
* this document can be anything, the content is never processed.
|
||||
* these document have an adress of the type "docsdir://citekey.pdf"
|
||||
* docsdir:// correspond to /path/to/pubsdir/doc (configurable)
|
||||
* document outside of the repository will not be removed.
|
||||
* move_doc only applies from inside to inside the docsdir
|
||||
"""
|
||||
|
||||
def __init__(self, directory, scheme='docsdir', subdir='doc'):
|
||||
self.scheme = scheme
|
||||
self.docdir = os.path.join(directory, subdir)
|
||||
if not check_directory(self.docdir, fail = False):
|
||||
os.mkdir(self.docdir)
|
||||
|
||||
def in_docsdir(self, docpath):
|
||||
try:
|
||||
parsed = urlparse.urlparse(docpath)
|
||||
except Exception:
|
||||
return False
|
||||
return parsed.scheme == self.scheme
|
||||
|
||||
# def doc_exists(self, citekey, ext='.txt'):
|
||||
# return check_file(os.path.join(self.docdir, citekey + ext), fail=False)
|
||||
|
||||
def real_docpath(self, docpath):
|
||||
""" Return the full path
|
||||
Essentially transform pubsdir://doc/{citekey}.{ext} to /path/to/pubsdir/doc/{citekey}.{ext}.
|
||||
Return absoluted paths of regular ones otherwise.
|
||||
"""
|
||||
if self.in_docsdir(docpath):
|
||||
parsed = urlparse.urlparse(docpath)
|
||||
if parsed.path == '':
|
||||
docpath = os.path.join(self.docdir, parsed.netloc)
|
||||
else:
|
||||
docpath = os.path.join(self.docdir, parsed.netloc, parsed.path[1:])
|
||||
elif content_type(docpath) != 'file':
|
||||
return docpath
|
||||
return os.path.normpath(os.path.abspath(docpath))
|
||||
|
||||
def add_doc(self, citekey, source_path, overwrite=False):
|
||||
""" Add a document to the docsdir, and return its location.
|
||||
|
||||
The document will be named {citekey}.{ext}.
|
||||
The location will be docsdir://{citekey}.{ext}.
|
||||
:param overwrite: will overwrite existing file.
|
||||
:return: the above location
|
||||
"""
|
||||
full_source_path = self.real_docpath(source_path)
|
||||
check_content(full_source_path)
|
||||
|
||||
target_path = '{}://{}'.format(self.scheme, citekey + os.path.splitext(source_path)[-1])
|
||||
full_target_path = self.real_docpath(target_path)
|
||||
if not overwrite and check_file(full_target_path, fail=False):
|
||||
raise IOError('{} file exists.'.format(full_target_path))
|
||||
|
||||
doc_content = get_content(full_source_path)
|
||||
with open(full_target_path, 'wb') as f:
|
||||
f.write(doc_content)
|
||||
|
||||
return target_path
|
||||
|
||||
def remove_doc(self, docpath, silent=True):
|
||||
""" Will remove only file hosted in docsdir://
|
||||
|
||||
:raise ValueError: for other paths, unless :param silent: is True
|
||||
"""
|
||||
if not self.in_docsdir(docpath):
|
||||
if not silent:
|
||||
raise ValueError(('the file to be removed {} is set as external. '
|
||||
'you should remove it manually.').format(docpath))
|
||||
return
|
||||
filepath = self.real_docpath(docpath)
|
||||
if check_file(filepath):
|
||||
os.remove(filepath)
|
||||
|
||||
def rename_doc(self, docpath, new_citekey):
|
||||
""" Move a document inside the docsdir
|
||||
|
||||
:raise IOError: if docpath doesn't point to a file
|
||||
if new_citekey doc exists already.
|
||||
:raise ValueError: if docpath is not in docsdir().
|
||||
|
||||
if an exception is raised, the files on disk haven't changed.
|
||||
"""
|
||||
if not self.in_docsdir(docpath):
|
||||
raise ValueError('cannot rename an external file ({}).'.format(docpath))
|
||||
|
||||
new_docpath = self.add_doc(new_citekey, docpath)
|
||||
self.remove_doc(docpath)
|
||||
|
||||
return new_docpath
|
||||
@ -0,0 +1,97 @@
|
||||
import copy
|
||||
import collections
|
||||
import datetime
|
||||
|
||||
from . import bibstruct
|
||||
|
||||
#DEFAULT_META = collections.OrderedDict([('docfile', None), ('tags', set()), ('added', )])
|
||||
DEFAULT_META = {'docfile': None, 'tags': set(), 'added': None}
|
||||
|
||||
class Paper(object):
|
||||
""" Paper class.
|
||||
|
||||
The object is not responsible of any disk I/O.
|
||||
self.bibdata is a dictionary of bibligraphic fields
|
||||
self.metadata is a dictionary
|
||||
|
||||
The paper class provides methods to access the fields for its metadata
|
||||
in a pythonic manner.
|
||||
"""
|
||||
|
||||
def __init__(self, bibdata, citekey=None, metadata=None):
|
||||
self.citekey = citekey
|
||||
self.metadata = metadata
|
||||
self.bibdata = bibdata
|
||||
|
||||
_, self.bibentry = bibstruct.get_entry(self.bibdata)
|
||||
|
||||
if self.metadata is None:
|
||||
self.metadata = copy.deepcopy(DEFAULT_META)
|
||||
if self.citekey is None:
|
||||
self.citekey = bibstruct.extract_citekey(self.bibdata)
|
||||
bibstruct.check_citekey(self.citekey)
|
||||
|
||||
self.metadata['tags'] = set(self.metadata.get('tags', []))
|
||||
|
||||
def __eq__(self, other):
|
||||
return (isinstance(self, Paper) and type(other) is type(self)
|
||||
and self.bibdata == other.bibdata
|
||||
and self.metadata == other.metadata
|
||||
and self.citekey == other.citekey)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Paper(%s, %s, %s)' % (
|
||||
self.citekey, self.bibentry, self.metadata)
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
return Paper(citekey =self.citekey,
|
||||
metadata=copy.deepcopy(self.metadata, memo),
|
||||
bibdata=copy.deepcopy(self.bibdata, memo))
|
||||
|
||||
def __copy__(self):
|
||||
return Paper(citekey =self.citekey,
|
||||
metadata=self.metadata,
|
||||
bibdata=self.bibdata)
|
||||
|
||||
def deepcopy(self):
|
||||
return self.__deepcopy__({})
|
||||
|
||||
# docpath
|
||||
|
||||
@property
|
||||
def docpath(self):
|
||||
return self.metadata.get('docfile', '')
|
||||
|
||||
@docpath.setter
|
||||
def docpath(self, path):
|
||||
"""Does not verify if the path exists."""
|
||||
self.metadata['docfile'] = path
|
||||
|
||||
# tags
|
||||
|
||||
@property
|
||||
def tags(self):
|
||||
return self.metadata['tags']
|
||||
|
||||
@tags.setter
|
||||
def tags(self, value):
|
||||
if not hasattr(value, '__iter__'):
|
||||
raise ValueError('tags must be iterables')
|
||||
self.metadata['tags'] = set(value)
|
||||
|
||||
def add_tag(self, tag):
|
||||
self.tags.add(tag)
|
||||
|
||||
def remove_tag(self, tag):
|
||||
"""Remove a tag from a paper if present."""
|
||||
self.tags.discard(tag)
|
||||
|
||||
# added date
|
||||
|
||||
@property
|
||||
def added(self):
|
||||
datetime.datetime.strptime(self.metadata['added'], '%Y-%m-%d %H:%M:%S')
|
||||
|
||||
@added.setter
|
||||
def added(self, value):
|
||||
self.metadata['added'] = value.strftime('%Y-%m-%d %H:%M:%S')
|
||||
@ -0,0 +1,60 @@
|
||||
# display formatting
|
||||
|
||||
from . import color
|
||||
|
||||
|
||||
# should be adaptated to bibtexparser dicts
|
||||
def person_repr(p):
|
||||
raise NotImplementedError
|
||||
return ' '.join(s for s in [
|
||||
' '.join(p.first(abbr=True)),
|
||||
' '.join(p.last(abbr=False)),
|
||||
' '.join(p.lineage(abbr=True))] if s)
|
||||
|
||||
def short_authors(bibentry):
|
||||
try:
|
||||
authors = [p for p in bibentry['author']]
|
||||
if len(authors) < 3:
|
||||
return ', '.join(authors)
|
||||
else:
|
||||
return authors[0] + (' et al.' if len(authors) > 1 else '')
|
||||
except KeyError: # When no author is defined
|
||||
return ''
|
||||
|
||||
|
||||
def bib_oneliner(bibentry):
|
||||
authors = short_authors(bibentry)
|
||||
journal, journal_field = '', 'journal'
|
||||
if 'journal' in bibentry:
|
||||
journal = bibentry['journal']['name']
|
||||
elif bibentry['type'] == 'inproceedings':
|
||||
journal = bibentry.get('booktitle', '')
|
||||
|
||||
return u'{authors} \"{title}\" {journal} ({year})'.format(
|
||||
authors=color.dye(authors, color.cyan),
|
||||
title=bibentry['title'],
|
||||
journal=color.dye(journal, color.yellow),
|
||||
year=bibentry['year'],
|
||||
)
|
||||
|
||||
|
||||
def bib_desc(bib_data):
|
||||
article = bib_data[list(bib_data.keys())[0]]
|
||||
s = '\n'.join('author: {}'.format(p)
|
||||
for p in article['author'])
|
||||
s += '\n'
|
||||
s += '\n'.join('{}: {}'.format(k, v) for k, v in article.items())
|
||||
return s
|
||||
|
||||
|
||||
def paper_oneliner(p, n = 0, citekey_only = False):
|
||||
if citekey_only:
|
||||
return p.citekey
|
||||
else:
|
||||
bibdesc = bib_oneliner(p.bibentry)
|
||||
return (u'[{citekey}] {descr} {tags}'.format(
|
||||
citekey=color.dye(p.citekey, color.purple),
|
||||
descr=bibdesc,
|
||||
tags=color.dye(' '.join(p.tags),
|
||||
color.tag, bold=False),
|
||||
)).encode('utf-8')
|
||||
@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env python2
|
||||
# -*- coding:utf-8 -*-
|
||||
|
||||
from pubs import pubs_cmd
|
||||
pubs_cmd.execute()
|
||||
@ -0,0 +1,140 @@
|
||||
import shutil
|
||||
import glob
|
||||
import itertools
|
||||
|
||||
from . import bibstruct
|
||||
from . import events
|
||||
from . import datacache
|
||||
from .paper import Paper
|
||||
|
||||
def _base27(n):
|
||||
return _base27((n - 1) // 26) + chr(ord('a') + ((n - 1) % 26)) if n else ''
|
||||
|
||||
|
||||
class CiteKeyCollision(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidReference(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Repository(object):
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self._citekeys = None
|
||||
self.databroker = datacache.DataCache(self.config.pubsdir)
|
||||
|
||||
@property
|
||||
def citekeys(self):
|
||||
if self._citekeys is None:
|
||||
self._citekeys = self.databroker.citekeys()
|
||||
return self._citekeys
|
||||
|
||||
def __contains__(self, citekey):
|
||||
""" Allows to use 'if citekey in repo' pattern
|
||||
|
||||
Warning: costly the first time.
|
||||
"""
|
||||
return citekey in self.citekeys
|
||||
|
||||
def __len__(self):
|
||||
"""Warning: costly the first time."""
|
||||
return len(self.citekeys)
|
||||
|
||||
# papers
|
||||
def all_papers(self):
|
||||
for key in self.citekeys:
|
||||
yield self.pull_paper(key)
|
||||
|
||||
def pull_paper(self, citekey):
|
||||
"""Load a paper by its citekey from disk, if necessary."""
|
||||
if self.databroker.exists(citekey, both = True):
|
||||
return Paper(self.databroker.pull_bibdata(citekey),
|
||||
citekey=citekey,
|
||||
metadata=self.databroker.pull_metadata(citekey))
|
||||
else:
|
||||
raise InvalidReference('{} citekey not found'.format(citekey))
|
||||
|
||||
def push_paper(self, paper, overwrite=False, event=True):
|
||||
""" Push a paper to disk
|
||||
|
||||
:param overwrite: if False, mimick the behavior of adding a paper
|
||||
if True, mimick the behavior of updating a paper
|
||||
"""
|
||||
bibstruct.check_citekey(paper.citekey)
|
||||
if (not overwrite) and self.databroker.exists(paper.citekey, both = False):
|
||||
raise IOError('files using the {} citekey already exists'.format(paper.citekey))
|
||||
if (not overwrite) and self.citekeys is not None and paper.citekey in self.citekeys:
|
||||
raise CiteKeyCollision('citekey {} already in use'.format(paper.citekey))
|
||||
|
||||
self.databroker.push_bibdata(paper.citekey, paper.bibdata)
|
||||
self.databroker.push_metadata(paper.citekey, paper.metadata)
|
||||
self.citekeys.add(paper.citekey)
|
||||
if event:
|
||||
events.AddEvent(paper.citekey).send()
|
||||
|
||||
def remove_paper(self, citekey, remove_doc=True, event=True):
|
||||
""" Remove a paper. Is silent if nothing needs to be done."""
|
||||
|
||||
if event:
|
||||
events.RemoveEvent(citekey).send()
|
||||
if remove_doc:
|
||||
try:
|
||||
metadata = self.databroker.pull_metadata(citekey)
|
||||
docpath = metadata.get('docfile')
|
||||
self.databroker.remove_doc(docpath, silent=True)
|
||||
self.databroker.remove_note(citekey, silent=True)
|
||||
except IOError:
|
||||
pass # FXME: if IOError is about being unable to
|
||||
# remove the file, we need to issue an error.I
|
||||
|
||||
self.citekeys.remove(citekey)
|
||||
self.databroker.remove(citekey)
|
||||
|
||||
def rename_paper(self, paper, new_citekey):
|
||||
old_citekey = paper.citekey
|
||||
# check if new_citekey is not the same as paper.citekey
|
||||
if old_citekey == new_citekey:
|
||||
push_paper(paper, overwrite=True, event=False)
|
||||
else:
|
||||
# check if new_citekey does not exists
|
||||
if self.databroker.exists(new_citekey, both=False):
|
||||
raise IOError("can't rename paper to {}, conflicting files exists".format(new_citekey))
|
||||
|
||||
new_bibdata = {}
|
||||
new_bibdata[new_citekey] = paper.bibdata[old_citekey]
|
||||
paper.bibdata = new_bibdata
|
||||
|
||||
# move doc file if necessary
|
||||
if self.databroker.in_docsdir(paper.docpath):
|
||||
paper.docpath = self.databroker.rename_doc(paper.docpath, new_citekey)
|
||||
|
||||
# move note file if necessary
|
||||
try:
|
||||
self.databroker.rename_note(old_citekey, new_citekey)
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
# push_paper to new_citekey
|
||||
paper.citekey = new_citekey
|
||||
self.push_paper(paper, event=False)
|
||||
# remove_paper of old_citekey
|
||||
self.remove_paper(old_citekey, event=False)
|
||||
# send event
|
||||
events.RenameEvent(paper, old_citekey).send()
|
||||
|
||||
def unique_citekey(self, base_key):
|
||||
"""Create a unique citekey for a given basekey."""
|
||||
for n in itertools.count():
|
||||
if not base_key + _base27(n) in self.citekeys:
|
||||
return base_key + _base27(n)
|
||||
|
||||
def get_tags(self):
|
||||
"""FIXME: bibdata doesn't need to be read."""
|
||||
tags = set()
|
||||
for p in self.all_papers():
|
||||
tags = tags.union(p.tags)
|
||||
return tags
|
||||
|
||||
@ -0,0 +1 @@
|
||||
from str_templates import *
|
||||
@ -0,0 +1,18 @@
|
||||
# Input the bibliographic data for your article
|
||||
# Supported formats are bibtex (template below), bibyaml and bibtexml
|
||||
|
||||
@article{
|
||||
YourCitekey,
|
||||
author = "LastName1, FirstName1 and LastName2, FirstName2",
|
||||
title = "",
|
||||
journal = "",
|
||||
number = "7",
|
||||
pages = "23-31",
|
||||
volume = "13",
|
||||
year = "2013",
|
||||
|
||||
doi = "",
|
||||
issn = "",
|
||||
keywords = "",
|
||||
abstract = ""
|
||||
}
|
||||
@ -0,0 +1,20 @@
|
||||
add_bib = """
|
||||
# Input the bibliographic data for your article
|
||||
# Supported formats are bibtex (template below), bibyaml and bibtexml
|
||||
|
||||
@article{
|
||||
YourCitekey,
|
||||
author = "LastName1, FirstName1 and LastName2, FirstName2",
|
||||
title = "",
|
||||
journal = "",
|
||||
number = "7",
|
||||
pages = "23-31",
|
||||
volume = "13",
|
||||
year = "2013",
|
||||
|
||||
doi = "",
|
||||
issn = "",
|
||||
keywords = "",
|
||||
abstract = ""
|
||||
}
|
||||
"""
|
||||
@ -0,0 +1,14 @@
|
||||
@incollection{
|
||||
year={2007},
|
||||
isbn={978-3-540-74957-8},
|
||||
booktitle={Machine Learning: ECML 2007},
|
||||
volume={4701},
|
||||
series={Lecture Notes in Computer Science},
|
||||
editor={Kok, JoostN. and Koronacki, Jacek and Mantaras, RaomonLopezde and Matwin, Stan and Mladenič, Dunja and Skowron, Andrzej},
|
||||
doi={10.1007/978-3-540-74958-5_70},
|
||||
title={Transfer Learning in Reinforcement Learning Problems Through Partial Policy Recycling},
|
||||
url={http://dx.doi.org/10.1007/978-3-540-74958-5_70},
|
||||
publisher={Springer Berlin Heidelberg},
|
||||
author={Ramon, Jan and Driessens, Kurt and Croonenborghs, Tom},
|
||||
pages={699-707}
|
||||
}
|
||||
@ -0,0 +1,162 @@
|
||||
import sys
|
||||
import os
|
||||
import shutil
|
||||
import glob
|
||||
import unittest
|
||||
import pkgutil
|
||||
import re
|
||||
|
||||
import dotdot
|
||||
import fake_filesystem
|
||||
import fake_filesystem_shutil
|
||||
import fake_filesystem_glob
|
||||
|
||||
from pubs import color
|
||||
from pubs.p3 import io, input
|
||||
|
||||
# code for fake fs
|
||||
|
||||
real_os = os
|
||||
real_open = open
|
||||
real_file = file
|
||||
real_shutil = shutil
|
||||
real_glob = glob
|
||||
|
||||
|
||||
|
||||
# def _mod_list():
|
||||
# ml = []
|
||||
# import pubs
|
||||
# for importer, modname, ispkg in pkgutil.walk_packages(
|
||||
# path=pubs.__path__,
|
||||
# prefix=pubs.__name__ + '.',
|
||||
# onerror=lambda x: None):
|
||||
# # HACK to not load textnote
|
||||
# if not modname.startswith('pubs.plugs.texnote'):
|
||||
# ml.append((modname, __import__(modname, fromlist='dummy')))
|
||||
# return ml
|
||||
|
||||
|
||||
|
||||
def create_fake_fs(module_list):
|
||||
|
||||
fake_fs = fake_filesystem.FakeFilesystem()
|
||||
fake_os = fake_filesystem.FakeOsModule(fake_fs)
|
||||
fake_open = fake_filesystem.FakeFileOpen(fake_fs)
|
||||
fake_shutil = fake_filesystem_shutil.FakeShutilModule(fake_fs)
|
||||
fake_glob = fake_filesystem_glob.FakeGlobModule(fake_fs)
|
||||
|
||||
fake_fs.CreateDirectory(fake_os.path.expanduser('~'))
|
||||
|
||||
try:
|
||||
__builtins__.open = fake_open
|
||||
__builtins__.file = fake_open
|
||||
except AttributeError:
|
||||
__builtins__['open'] = fake_open
|
||||
__builtins__['file'] = fake_open
|
||||
|
||||
sys.modules['os'] = fake_os
|
||||
sys.modules['shutil'] = fake_shutil
|
||||
sys.modules['glob'] = fake_glob
|
||||
|
||||
for md in module_list:
|
||||
md.os = fake_os
|
||||
md.shutil = fake_shutil
|
||||
md.open = fake_open
|
||||
md.file = fake_open
|
||||
|
||||
return {'fs': fake_fs,
|
||||
'os': fake_os,
|
||||
'open': fake_open,
|
||||
'shutil': fake_shutil,
|
||||
'glob': fake_glob}
|
||||
|
||||
def unset_fake_fs(module_list):
|
||||
try:
|
||||
__builtins__.open = real_open
|
||||
__builtins__.file = real_file
|
||||
except AttributeError:
|
||||
__builtins__['open'] = real_open
|
||||
__builtins__['file'] = real_file
|
||||
|
||||
sys.modules['os'] = real_os
|
||||
sys.modules['shutil'] = real_shutil
|
||||
sys.modules['glob'] = real_glob
|
||||
|
||||
for md in module_list:
|
||||
md.os = real_os
|
||||
md.shutil = real_shutil
|
||||
md.open = real_open
|
||||
md.file = real_file
|
||||
|
||||
|
||||
def copy_dir(fs, real_dir, fake_dir = None):
|
||||
"""Copy all the data directory into the fake fs"""
|
||||
if fake_dir is None:
|
||||
fake_dir = real_dir
|
||||
for filename in real_os.listdir(real_dir):
|
||||
real_path = real_os.path.join(real_dir, filename)
|
||||
fake_path = fs['os'].path.join(fake_dir, filename)
|
||||
if real_os.path.isfile(real_path):
|
||||
with real_open(real_path, 'r') as f:
|
||||
fs['fs'].CreateFile(fake_path, contents=f.read())
|
||||
if real_os.path.isdir(real_path):
|
||||
fs['fs'].CreateDirectory(fake_path)
|
||||
copy_dir(fs, real_path, fake_path)
|
||||
|
||||
|
||||
# redirecting output
|
||||
|
||||
def redirect(f):
|
||||
def newf(*args, **kwargs):
|
||||
old_stderr, old_stdout = sys.stderr, sys.stdout
|
||||
stdout = io.StringIO()
|
||||
stderr = io.StringIO()
|
||||
sys.stdout, sys.stderr = stdout, stderr
|
||||
try:
|
||||
return f(*args, **kwargs), stdout, stderr
|
||||
finally:
|
||||
sys.stderr, sys.stdout = old_stderr, old_stdout
|
||||
return newf
|
||||
|
||||
|
||||
# Test helpers
|
||||
|
||||
# automating input
|
||||
|
||||
real_input = input
|
||||
|
||||
|
||||
class FakeInput():
|
||||
""" Replace the input() command, and mock user input during tests
|
||||
|
||||
Instanciate as :
|
||||
input = FakeInput(['yes', 'no'])
|
||||
then replace the input command in every module of the package :
|
||||
input.as_global()
|
||||
Then :
|
||||
input() returns 'yes'
|
||||
input() returns 'no'
|
||||
input() raise IndexError
|
||||
"""
|
||||
|
||||
def __init__(self, inputs, module_list=tuple()):
|
||||
self.inputs = list(inputs) or []
|
||||
self.module_list = module_list
|
||||
self._cursor = 0
|
||||
|
||||
def as_global(self):
|
||||
for md in self.module_list:
|
||||
md.input = self
|
||||
md.editor_input = self
|
||||
# if mdname.endswith('files'):
|
||||
# md.editor_input = self
|
||||
|
||||
def add_input(self, inp):
|
||||
self.inputs.append(inp)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
inp = self.inputs[self._cursor]
|
||||
self._cursor += 1
|
||||
return inp
|
||||
|
||||
@ -1,50 +1,36 @@
|
||||
from pybtex.database import Person
|
||||
|
||||
from papers.paper import Paper, get_bibentry_from_string
|
||||
|
||||
turing1950 = Paper()
|
||||
turing1950.bibentry.fields['title'] = u'Computing machinery and intelligence.'
|
||||
turing1950.bibentry.fields['year'] = u'1950'
|
||||
turing1950.bibentry.persons['author'] = [Person(u'Alan Turing')]
|
||||
turing1950.citekey = turing1950.generate_citekey()
|
||||
turing1950.tags = ['computer', 'AI']
|
||||
|
||||
|
||||
doe2013 = Paper()
|
||||
doe2013.bibentry.fields['title'] = u'Nice title.'
|
||||
doe2013.bibentry.fields['year'] = u'2013'
|
||||
doe2013.bibentry.persons['author'] = [Person(u'John Doe')]
|
||||
doe2013.citekey = doe2013.generate_citekey()
|
||||
|
||||
|
||||
pagerankbib = """
|
||||
@techreport{Page99,
|
||||
number = {1999-66},
|
||||
month = {November},
|
||||
author = {Lawrence Page and Sergey Brin and Rajeev Motwani and Terry Winograd},
|
||||
note = {Previous number = SIDL-WP-1999-0120},
|
||||
title = {The PageRank Citation Ranking: Bringing Order to the Web.},
|
||||
type = {Technical Report},
|
||||
publisher = {Stanford InfoLab},
|
||||
year = {1999},
|
||||
institution = {Stanford InfoLab},
|
||||
url = {http://ilpubs.stanford.edu:8090/422/},
|
||||
}
|
||||
"""
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import dotdot
|
||||
from pubs import endecoder
|
||||
import str_fixtures
|
||||
|
||||
page99 = Paper(bibentry=get_bibentry_from_string(pagerankbib)[1])
|
||||
|
||||
pagerankbib_generated = """@techreport{
|
||||
Page99,
|
||||
author = "Page, Lawrence and Brin, Sergey and Motwani, Rajeev and Winograd, Terry",
|
||||
publisher = "Stanford InfoLab",
|
||||
title = "The PageRank Citation Ranking: Bringing Order to the Web.",
|
||||
url = "http://ilpubs.stanford.edu:8090/422/",
|
||||
number = "1999-66",
|
||||
month = "November",
|
||||
note = "Previous number = SIDL-WP-1999-0120",
|
||||
year = "1999",
|
||||
institution = "Stanford InfoLab"
|
||||
}
|
||||
coder = endecoder.EnDecoder()
|
||||
|
||||
franny_bib = """@article{Franny1961,
|
||||
author = "Salinger, J. D.",
|
||||
title = "Franny and Zooey",
|
||||
year = "1961"}
|
||||
"""
|
||||
|
||||
doe_bib = """
|
||||
@article{Doe2013,
|
||||
author = "Doe, John",
|
||||
title = "Nice Title",
|
||||
year = "2013"}
|
||||
"""
|
||||
|
||||
franny_bibdata = coder.decode_bibdata(franny_bib)
|
||||
franny_bibentry = franny_bibdata['Franny1961']
|
||||
|
||||
doe_bibdata = coder.decode_bibdata(doe_bib)
|
||||
doe_bibentry = doe_bibdata['Doe2013']
|
||||
|
||||
turing_bibdata = coder.decode_bibdata(str_fixtures.turing_bib)
|
||||
turing_bibentry = turing_bibdata['turing1950computing']
|
||||
turing_metadata = coder.decode_metadata(str_fixtures.turing_meta)
|
||||
|
||||
page_bibdata = coder.decode_bibdata(str_fixtures.bibtex_raw0)
|
||||
page_bibentry = page_bibdata['Page99']
|
||||
page_metadata = coder.decode_metadata(str_fixtures.metadata_raw0)
|
||||
|
||||
page_metadata = coder.decode_metadata(str_fixtures.metadata_raw0)
|
||||
|
||||
@ -0,0 +1,55 @@
|
||||
bibtex_external0 = """
|
||||
@techreport{Page99,
|
||||
number = {1999-66},
|
||||
month = {November},
|
||||
author = {Lawrence Page and Sergey Brin and Rajeev Motwani and Terry Winograd},
|
||||
note = {Previous number = SIDL-WP-1999-0120},
|
||||
title = {The PageRank Citation Ranking: Bringing Order to the Web.},
|
||||
type = {Technical Report},
|
||||
publisher = {Stanford InfoLab},
|
||||
year = {1999},
|
||||
institution = {Stanford InfoLab},
|
||||
url = {http://ilpubs.stanford.edu:8090/422/},
|
||||
abstract = "The importance of a Web page is an inherently subjective matter, which depends on the readers interests, knowledge and attitudes. But there is still much that can be said objectively about the relative importance of Web pages. This paper describes PageRank, a mathod for rating Web pages objectively and mechanically, effectively measuring the human interest and attention devoted to them. We compare PageRank to an idealized random Web surfer. We show how to efficiently compute PageRank for large numbers of pages. And, we show how to apply PageRank to search and to user navigation.",
|
||||
}
|
||||
"""
|
||||
|
||||
bibtex_raw0 = """@techreport{
|
||||
Page99,
|
||||
author = "Page, Lawrence and Brin, Sergey and Motwani, Rajeev and Winograd, Terry",
|
||||
publisher = "Stanford InfoLab",
|
||||
title = "The PageRank Citation Ranking: Bringing Order to the Web.",
|
||||
url = "http://ilpubs.stanford.edu:8090/422/",
|
||||
abstract = "The importance of a Web page is an inherently subjective matter, which depends on the readers interests, knowledge and attitudes. But there is still much that can be said objectively about the relative importance of Web pages. This paper describes PageRank, a mathod for rating Web pages objectively and mechanically, effectively measuring the human interest and attention devoted to them. We compare PageRank to an idealized random Web surfer. We show how to efficiently compute PageRank for large numbers of pages. And, we show how to apply PageRank to search and to user navigation.",
|
||||
number = "1999-66",
|
||||
month = "November",
|
||||
note = "Previous number = SIDL-WP-1999-0120",
|
||||
year = "1999",
|
||||
type = "Technical Report",
|
||||
institution = "Stanford InfoLab"
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
metadata_raw0 = """docfile: docsdir://Page99.pdf
|
||||
tags: [search, network]
|
||||
added: '2013-11-14 13:14:20'
|
||||
"""
|
||||
|
||||
turing_bib = """@article{turing1950computing,
|
||||
title={Computing machinery and intelligence},
|
||||
author={Turing, Alan M},
|
||||
journal={Mind},
|
||||
volume={59},
|
||||
number={236},
|
||||
pages={433--460},
|
||||
year={1950},
|
||||
publisher={JSTOR}
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
turing_meta = """\
|
||||
tags: [AI, computer]
|
||||
added: '2013-11-14 13:14:20'
|
||||
"""
|
||||
@ -1,11 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
rm -Rf tmpdir/*;
|
||||
papers init -p tmpdir/;
|
||||
papers add -d data/pagerank.pdf -b data/pagerank.bib;
|
||||
papers list;
|
||||
papers tag;
|
||||
papers tag Page99 network+search;
|
||||
papers tag Page99;
|
||||
papers tag search;
|
||||
papers tag 0;
|
||||
pubs init -p tmpdir/;
|
||||
pubs add -d data/pagerank.pdf -b data/pagerank.bib;
|
||||
pubs list;
|
||||
pubs tag;
|
||||
pubs tag Page99 network+search;
|
||||
pubs tag Page99;
|
||||
pubs tag search;
|
||||
pubs tag 0;
|
||||
#rm -Rf tmpdir/*;
|
||||
|
||||
@ -0,0 +1,35 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import unittest
|
||||
import copy
|
||||
|
||||
import dotdot
|
||||
from pubs import bibstruct
|
||||
|
||||
import fixtures
|
||||
|
||||
|
||||
class TestGenerateCitekey(unittest.TestCase):
|
||||
|
||||
def test_fails_on_empty_paper(self):
|
||||
with self.assertRaises(ValueError):
|
||||
bibstruct.generate_citekey(None)
|
||||
|
||||
def test_escapes_chars(self):
|
||||
doe_bibdata = copy.deepcopy(fixtures.doe_bibdata)
|
||||
citekey, entry = bibstruct.get_entry(doe_bibdata)
|
||||
entry['author'] = [u'Zôu\\@/ , John']
|
||||
key = bibstruct.generate_citekey(doe_bibdata)
|
||||
|
||||
def test_simple(self):
|
||||
bibdata = copy.deepcopy(fixtures.doe_bibdata)
|
||||
key = bibstruct.generate_citekey(bibdata)
|
||||
self.assertEqual(key, 'Doe2013')
|
||||
|
||||
bibdata = copy.deepcopy(fixtures.franny_bibdata)
|
||||
key = bibstruct.generate_citekey(bibdata)
|
||||
self.assertEqual(key, 'Salinger1961')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,85 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import unittest
|
||||
import os
|
||||
|
||||
import dotdot
|
||||
import fake_env
|
||||
|
||||
from pubs import content, filebroker, databroker, datacache
|
||||
|
||||
import str_fixtures
|
||||
from pubs import endecoder
|
||||
|
||||
class TestFakeFs(unittest.TestCase):
|
||||
"""Abstract TestCase intializing the fake filesystem."""
|
||||
|
||||
def setUp(self):
|
||||
self.fs = fake_env.create_fake_fs([content, filebroker])
|
||||
|
||||
def tearDown(self):
|
||||
fake_env.unset_fake_fs([content, filebroker])
|
||||
|
||||
|
||||
class TestDataBroker(unittest.TestCase):
|
||||
|
||||
def test_databroker(self):
|
||||
|
||||
ende = endecoder.EnDecoder()
|
||||
page99_metadata = ende.decode_metadata(str_fixtures.metadata_raw0)
|
||||
page99_bibdata = ende.decode_bibdata(str_fixtures.bibtex_raw0)
|
||||
|
||||
for db_class in [databroker.DataBroker, datacache.DataCache]:
|
||||
self.fs = fake_env.create_fake_fs([content, filebroker])
|
||||
|
||||
db = db_class('tmp', create=True)
|
||||
|
||||
db.push_metadata('citekey1', page99_metadata)
|
||||
self.assertTrue(db.exists('citekey1', both=False))
|
||||
self.assertFalse(db.exists('citekey1', both=True))
|
||||
|
||||
db.push_bibdata('citekey1', page99_bibdata)
|
||||
self.assertTrue(db.exists('citekey1', both=True))
|
||||
|
||||
self.assertEqual(db.pull_metadata('citekey1'), page99_metadata)
|
||||
pulled = db.pull_bibdata('citekey1')['Page99']
|
||||
for key, value in pulled.items():
|
||||
self.assertEqual(pulled[key], page99_bibdata['Page99'][key])
|
||||
self.assertEqual(db.pull_bibdata('citekey1'), page99_bibdata)
|
||||
|
||||
fake_env.unset_fake_fs([content, filebroker])
|
||||
|
||||
def test_existing_data(self):
|
||||
|
||||
ende = endecoder.EnDecoder()
|
||||
page99_bibdata = ende.decode_bibdata(str_fixtures.bibtex_raw0)
|
||||
|
||||
for db_class in [databroker.DataBroker, datacache.DataCache]:
|
||||
self.fs = fake_env.create_fake_fs([content, filebroker])
|
||||
fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'testrepo'), 'repo')
|
||||
|
||||
db = db_class('repo', create=False)
|
||||
|
||||
self.assertEqual(db.pull_bibdata('Page99'), page99_bibdata)
|
||||
|
||||
for citekey in ['10.1371_journal.pone.0038236',
|
||||
'10.1371journal.pone.0063400',
|
||||
'journal0063400']:
|
||||
db.pull_bibdata(citekey)
|
||||
db.pull_metadata(citekey)
|
||||
|
||||
with self.assertRaises(IOError):
|
||||
db.pull_bibdata('citekey')
|
||||
with self.assertRaises(IOError):
|
||||
db.pull_metadata('citekey')
|
||||
|
||||
db.add_doc('Larry99', 'docsdir://Page99.pdf')
|
||||
self.assertTrue(content.check_file('repo/doc/Page99.pdf', fail=False))
|
||||
self.assertTrue(content.check_file('repo/doc/Larry99.pdf', fail=False))
|
||||
|
||||
db.remove_doc('docsdir://Page99.pdf')
|
||||
|
||||
fake_env.unset_fake_fs([content, filebroker])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,50 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import unittest
|
||||
|
||||
import yaml
|
||||
|
||||
import dotdot
|
||||
from pubs import endecoder
|
||||
|
||||
from str_fixtures import bibtex_raw0, metadata_raw0
|
||||
|
||||
def compare_yaml_str(s1, s2):
|
||||
if s1 == s2:
|
||||
return True
|
||||
else:
|
||||
y1 = yaml.safe_load(s1)
|
||||
y2 = yaml.safe_load(s2)
|
||||
return y1 == y2
|
||||
|
||||
|
||||
class TestEnDecode(unittest.TestCase):
|
||||
|
||||
def test_endecode_bibtex(self):
|
||||
|
||||
decoder = endecoder.EnDecoder()
|
||||
entry = decoder.decode_bibdata(bibtex_raw0)
|
||||
|
||||
bibraw1 = decoder.encode_bibdata(entry)
|
||||
entry1 = decoder.decode_bibdata(bibraw1)
|
||||
bibraw2 = decoder.encode_bibdata(entry1)
|
||||
entry2 = decoder.decode_bibdata(bibraw2)
|
||||
|
||||
for citekey in entry1.keys():
|
||||
bibentry1 = entry1[citekey]
|
||||
bibentry2 = entry2[citekey]
|
||||
for key, value in bibentry1.items():
|
||||
self.assertEqual(bibentry1[key], bibentry2[key])
|
||||
|
||||
self.assertEqual(bibraw1, bibraw2)
|
||||
|
||||
def test_endecode_metadata(self):
|
||||
|
||||
decoder = endecoder.EnDecoder()
|
||||
entry = decoder.decode_metadata(metadata_raw0)
|
||||
metadata_output0 = decoder.encode_metadata(entry)
|
||||
|
||||
self.assertEqual(set(metadata_raw0.split('\n')), set(metadata_output0.split('\n')))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,114 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import unittest
|
||||
import os
|
||||
|
||||
import dotdot
|
||||
import fake_env
|
||||
|
||||
from pubs import content, filebroker
|
||||
|
||||
class TestFakeFs(unittest.TestCase):
|
||||
"""Abstract TestCase intializing the fake filesystem."""
|
||||
|
||||
def setUp(self):
|
||||
self.fs = fake_env.create_fake_fs([content, filebroker])
|
||||
|
||||
def tearDown(self):
|
||||
fake_env.unset_fake_fs([content, filebroker])
|
||||
|
||||
|
||||
class TestFileBroker(TestFakeFs):
|
||||
|
||||
def test_pushpull1(self):
|
||||
|
||||
fb = filebroker.FileBroker('bla', create = True)
|
||||
|
||||
fb.push_metafile('citekey1', 'abc')
|
||||
fb.push_bibfile('citekey1', 'cdef')
|
||||
|
||||
self.assertEqual(fb.pull_metafile('citekey1'), 'abc')
|
||||
self.assertEqual(fb.pull_bibfile('citekey1'), 'cdef')
|
||||
|
||||
fb.push_bibfile('citekey1', 'ghi')
|
||||
|
||||
self.assertEqual(fb.pull_bibfile('citekey1'), 'ghi')
|
||||
|
||||
def test_existing_data(self):
|
||||
|
||||
fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'tmpdir'), 'tmpdir')
|
||||
fb = filebroker.FileBroker('tmpdir', create = True)
|
||||
|
||||
with open('tmpdir/bib/Page99.bib', 'r') as f:
|
||||
self.assertEqual(fb.pull_bibfile('Page99'), f.read())
|
||||
|
||||
with open('tmpdir/meta/Page99.yaml', 'r') as f:
|
||||
self.assertEqual(fb.pull_metafile('Page99'), f.read())
|
||||
|
||||
def test_errors(self):
|
||||
|
||||
with self.assertRaises(IOError):
|
||||
filebroker.FileBroker('tmpdir', create = False)
|
||||
|
||||
fb = filebroker.FileBroker('tmpdir', create = True)
|
||||
with self.assertRaises(IOError):
|
||||
fb.pull_bibfile('Page99')
|
||||
with self.assertRaises(IOError):
|
||||
fb.pull_metafile('Page99')
|
||||
|
||||
def test_errors(self):
|
||||
|
||||
with self.assertRaises(IOError):
|
||||
filebroker.FileBroker('tmpdir', create = False)
|
||||
|
||||
fb = filebroker.FileBroker('tmpdir', create = True)
|
||||
|
||||
self.assertFalse(fb.exists('Page99'))
|
||||
with self.assertRaises(IOError):
|
||||
fb.pull_bibfile('Page99')
|
||||
with self.assertRaises(IOError):
|
||||
fb.pull_metafile('Page99')
|
||||
|
||||
def test_remove(self):
|
||||
|
||||
with self.assertRaises(IOError):
|
||||
filebroker.FileBroker('tmpdir', create = False)
|
||||
|
||||
fb = filebroker.FileBroker('tmpdir', create = True)
|
||||
|
||||
fb.push_bibfile('citekey1', 'abc')
|
||||
self.assertEqual(fb.pull_bibfile('citekey1'), 'abc')
|
||||
fb.push_metafile('citekey1', 'defg')
|
||||
self.assertEqual(fb.pull_metafile('citekey1'), 'defg')
|
||||
self.assertTrue(fb.exists('citekey1'))
|
||||
|
||||
fb.remove('citekey1')
|
||||
with self.assertRaises(IOError):
|
||||
self.assertEqual(fb.pull_bibfile('citekey1'), 'abc')
|
||||
with self.assertRaises(IOError):
|
||||
self.assertEqual(fb.pull_metafile('citekey1'), 'defg')
|
||||
self.assertFalse(fb.exists('citekey1'))
|
||||
|
||||
|
||||
class TestDocBroker(TestFakeFs):
|
||||
|
||||
def test_doccopy(self):
|
||||
|
||||
fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'data'), 'data')
|
||||
|
||||
fb = filebroker.FileBroker('tmpdir', create = True)
|
||||
docb = filebroker.DocBroker('tmpdir')
|
||||
|
||||
docpath = docb.add_doc('Page99', 'data/pagerank.pdf')
|
||||
self.assertTrue(content.check_file(os.path.join('tmpdir', 'doc/Page99.pdf')))
|
||||
|
||||
self.assertTrue(docb.in_docsdir(docpath))
|
||||
self.assertEqual(docpath, 'docsdir://Page99.pdf')
|
||||
docb.remove_doc('docsdir://Page99.pdf')
|
||||
|
||||
self.assertFalse(content.check_file(os.path.join('tmpdir', 'doc/Page99.pdf'), fail=False))
|
||||
with self.assertRaises(IOError):
|
||||
self.assertFalse(content.check_file(os.path.join('tmpdir', 'doc/Page99.pdf'), fail=True))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -1,121 +1,47 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import unittest
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
import yaml
|
||||
from pybtex.database import Person
|
||||
|
||||
import testenv
|
||||
import dotdot
|
||||
import fixtures
|
||||
from papers.paper import Paper
|
||||
|
||||
|
||||
BIB = """
|
||||
entries:
|
||||
Turing1950:
|
||||
author:
|
||||
- first: 'Alan'
|
||||
last: 'Turing'
|
||||
title: 'Computing machinery and intelligence.'
|
||||
type: article
|
||||
year: '1950'
|
||||
"""
|
||||
META = """
|
||||
external-document: null
|
||||
notes: []
|
||||
tags: ['AI', 'computer']
|
||||
"""
|
||||
|
||||
|
||||
class TestCreateCitekey(unittest.TestCase):
|
||||
|
||||
def test_fails_on_empty_paper(self):
|
||||
paper = Paper()
|
||||
with self.assertRaises(ValueError):
|
||||
paper.generate_citekey()
|
||||
|
||||
def test_escapes_chars(self):
|
||||
paper = Paper()
|
||||
paper.bibentry.persons['author'] = [
|
||||
Person(last=u'Z ôu\\@/', first='Zde'),
|
||||
Person(string='John Doe')]
|
||||
key = paper.generate_citekey()
|
||||
self.assertEqual(key, 'Zou')
|
||||
|
||||
def test_simple(self):
|
||||
paper = Paper()
|
||||
paper.bibentry.persons['author'] = [Person(string='John Doe')]
|
||||
paper.bibentry.fields['year'] = '2001'
|
||||
key = paper.generate_citekey()
|
||||
self.assertEqual(key, 'Doe2001')
|
||||
|
||||
|
||||
class TestSaveLoad(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
os.makedirs(os.path.join(self.tmpdir, 'bibdata'))
|
||||
os.makedirs(os.path.join(self.tmpdir, 'meta'))
|
||||
self.bibfile = os.path.join(self.tmpdir, 'bib.bibyaml')
|
||||
with open(self.bibfile, 'w') as f:
|
||||
f.write(BIB)
|
||||
self.metafile = os.path.join(self.tmpdir, 'meta.meta')
|
||||
with open(self.metafile, 'w') as f:
|
||||
f.write(META)
|
||||
self.dest_bibfile = os.path.join(self.tmpdir, 'written_bib.yaml')
|
||||
self.dest_metafile = os.path.join(self.tmpdir, 'written_meta.yaml')
|
||||
|
||||
def test_load_valid(self):
|
||||
p = Paper.load(self.bibfile, metapath=self.metafile)
|
||||
self.assertEqual(fixtures.turing1950, p)
|
||||
|
||||
def test_save_fails_with_no_citekey(self):
|
||||
p = Paper()
|
||||
with self.assertRaises(ValueError):
|
||||
p.save(self.dest_bibfile, self.dest_metafile)
|
||||
|
||||
def test_save_creates_bib(self):
|
||||
fixtures.turing1950.save(self.dest_bibfile, self.dest_metafile)
|
||||
self.assertTrue(os.path.exists(self.dest_bibfile))
|
||||
|
||||
def test_save_creates_meta(self):
|
||||
fixtures.turing1950.save(self.dest_bibfile, self.dest_metafile)
|
||||
self.assertTrue(os.path.exists(self.dest_metafile))
|
||||
|
||||
def test_save_right_bib(self):
|
||||
fixtures.turing1950.save(self.dest_bibfile, self.dest_metafile)
|
||||
with open(self.dest_bibfile, 'r') as f:
|
||||
written = yaml.load(f)
|
||||
ok = yaml.load(BIB)
|
||||
self.assertEqual(written, ok)
|
||||
|
||||
def test_save_right_meta(self):
|
||||
fixtures.turing1950.save(self.dest_bibfile, self.dest_metafile)
|
||||
with open(self.dest_metafile, 'r') as f:
|
||||
written = yaml.load(f)
|
||||
ok = yaml.load(META)
|
||||
self.assertEqual(written, ok)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmpdir)
|
||||
|
||||
|
||||
class TestCopy(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.orig = Paper()
|
||||
self.orig.bibentry.fields['title'] = u'Nice title.'
|
||||
self.orig.bibentry.fields['year'] = u'2013'
|
||||
self.orig.bibentry.persons['author'] = [Person(u'John Doe')]
|
||||
self.orig.citekey = self.orig.generate_citekey()
|
||||
|
||||
def test_copy_equal(self):
|
||||
copy = self.orig.copy()
|
||||
self.assertEqual(copy, self.orig)
|
||||
|
||||
def test_copy_can_be_changed(self):
|
||||
copy = self.orig.copy()
|
||||
copy.bibentry.fields['year'] = 2014
|
||||
self.assertEqual(self.orig.bibentry.fields['year'], u'2013')
|
||||
from pubs.paper import Paper
|
||||
|
||||
|
||||
class TestAttributes(unittest.TestCase):
|
||||
|
||||
def test_tags(self):
|
||||
p = Paper(fixtures.page_bibdata, metadata=fixtures.page_metadata).deepcopy()
|
||||
self.assertEqual(p.tags, set(['search', 'network']))
|
||||
|
||||
def test_add_tag(self):
|
||||
p = Paper(fixtures.page_bibdata, metadata=fixtures.page_metadata).deepcopy()
|
||||
p.add_tag('algorithm')
|
||||
self.assertEqual(p.tags, set(['search', 'network', 'algorithm']))
|
||||
p.add_tag('algorithm')
|
||||
self.assertEqual(p.tags, set(['search', 'network', 'algorithm']))
|
||||
|
||||
def test_set_tags(self):
|
||||
p = Paper(fixtures.page_bibdata, metadata=fixtures.page_metadata).deepcopy()
|
||||
p.tags = ['algorithm']
|
||||
self.assertEqual(p.tags, set(['algorithm']))
|
||||
|
||||
def test_remove_tags(self):
|
||||
p = Paper(fixtures.page_bibdata, metadata=fixtures.page_metadata).deepcopy()
|
||||
p.remove_tag('network')
|
||||
self.assertEqual(p.tags, set(['search']))
|
||||
|
||||
def test_mixed_tags(self):
|
||||
p = Paper(fixtures.page_bibdata, metadata=fixtures.page_metadata).deepcopy()
|
||||
p.add_tag('algorithm')
|
||||
self.assertEqual(p.tags, set(['search', 'network', 'algorithm']))
|
||||
p.remove_tag('network')
|
||||
self.assertEqual(p.tags, set(['search', 'algorithm']))
|
||||
p.tags = ['ranking']
|
||||
self.assertEqual(p.tags, set(['ranking']))
|
||||
p.remove_tag('ranking')
|
||||
self.assertEqual(p.tags, set())
|
||||
p.remove_tag('ranking')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import unittest
|
||||
import os
|
||||
|
||||
import dotdot
|
||||
import fake_env
|
||||
|
||||
from pubs import endecoder, pretty
|
||||
|
||||
from str_fixtures import bibtex_raw0
|
||||
|
||||
class TestPretty(unittest.TestCase):
|
||||
|
||||
def test_oneliner(self):
|
||||
|
||||
decoder = endecoder.EnDecoder()
|
||||
bibdata = decoder.decode_bibdata(bibtex_raw0)
|
||||
pretty.bib_oneliner(bibdata['Page99'])
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -1,91 +1,101 @@
|
||||
from unittest import TestCase
|
||||
import unittest
|
||||
|
||||
import testenv
|
||||
import fixtures
|
||||
from papers.commands.list_cmd import (_check_author_match,
|
||||
import dotdot
|
||||
from pubs.commands.list_cmd import (_check_author_match,
|
||||
_check_field_match,
|
||||
_check_query_block,
|
||||
filter_paper,
|
||||
InvalidQuery)
|
||||
|
||||
from pubs.paper import Paper
|
||||
|
||||
import fixtures
|
||||
|
||||
doe_paper = Paper(fixtures.doe_bibdata)
|
||||
page_paper = Paper(fixtures.page_bibdata)
|
||||
turing_paper = Paper(fixtures.turing_bibdata, metadata=fixtures.turing_metadata)
|
||||
|
||||
class TestAuthorFilter(TestCase):
|
||||
class TestAuthorFilter(unittest.TestCase):
|
||||
|
||||
def test_fails_if_no_author(self):
|
||||
no_doe = fixtures.doe2013.copy()
|
||||
no_doe.bibentry.persons = {}
|
||||
no_doe = doe_paper.deepcopy()
|
||||
no_doe.bibentry['author'] = []
|
||||
self.assertTrue(not _check_author_match(no_doe, 'whatever'))
|
||||
|
||||
def test_match_case(self):
|
||||
self.assertTrue(_check_author_match(fixtures.doe2013, 'doe'))
|
||||
self.assertTrue(_check_author_match(fixtures.doe2013, 'doe',
|
||||
self.assertTrue(_check_author_match(doe_paper, 'doe'))
|
||||
self.assertTrue(_check_author_match(doe_paper, 'doe',
|
||||
case_sensitive=False))
|
||||
|
||||
def test_do_not_match_case(self):
|
||||
self.assertFalse(_check_author_match(fixtures.doe2013, 'dOe'))
|
||||
self.assertFalse(_check_author_match(fixtures.doe2013, 'doe',
|
||||
self.assertFalse(_check_author_match(doe_paper, 'dOe'))
|
||||
self.assertFalse(_check_author_match(doe_paper, 'doe',
|
||||
case_sensitive=True))
|
||||
|
||||
def test_match_not_first_author(self):
|
||||
self.assertTrue(_check_author_match(fixtures.page99, 'wani'))
|
||||
self.assertTrue(_check_author_match(page_paper, 'motwani'))
|
||||
|
||||
def test_do_not_match_first_name(self):
|
||||
self.assertTrue(not _check_author_match(fixtures.page99, 'larry'))
|
||||
self.assertTrue(not _check_author_match(page_paper, 'larry'))
|
||||
|
||||
|
||||
class TestCheckTag(TestCase):
|
||||
class TestCheckTag(unittest.TestCase):
|
||||
pass
|
||||
|
||||
|
||||
class TestCheckField(TestCase):
|
||||
class TestCheckField(unittest.TestCase):
|
||||
|
||||
def test_match_case(self):
|
||||
self.assertTrue(_check_field_match(fixtures.doe2013, 'title', 'nice'))
|
||||
self.assertTrue(_check_field_match(fixtures.doe2013, 'title', 'nice',
|
||||
self.assertTrue(_check_field_match(doe_paper, 'title', 'nice'))
|
||||
self.assertTrue(_check_field_match(doe_paper, 'title', 'nice',
|
||||
case_sensitive=False))
|
||||
self.assertTrue(_check_field_match(fixtures.doe2013, 'year', '2013'))
|
||||
self.assertTrue(_check_field_match(doe_paper, 'year', '2013'))
|
||||
|
||||
def test_do_not_match_case(self):
|
||||
self.assertFalse(_check_field_match(fixtures.doe2013, 'title',
|
||||
self.assertTrue(_check_field_match(doe_paper, 'title',
|
||||
'Title', case_sensitive=True))
|
||||
self.assertFalse(_check_field_match(fixtures.doe2013, 'title', 'nice',
|
||||
self.assertFalse(_check_field_match(doe_paper, 'title', 'nice',
|
||||
case_sensitive=True))
|
||||
|
||||
|
||||
class TestCheckQueryBlock(TestCase):
|
||||
class TestCheckQueryBlock(unittest.TestCase):
|
||||
|
||||
def test_raise_invalid_if_no_value(self):
|
||||
with self.assertRaises(InvalidQuery):
|
||||
_check_query_block(fixtures.doe2013, 'title')
|
||||
_check_query_block(doe_paper, 'title')
|
||||
|
||||
def test_raise_invalid_if_too_much(self):
|
||||
with self.assertRaises(InvalidQuery):
|
||||
_check_query_block(fixtures.doe2013, 'whatever:value:too_much')
|
||||
_check_query_block(doe_paper, 'whatever:value:too_much')
|
||||
|
||||
|
||||
class TestFilterPaper(TestCase):
|
||||
class TestFilterPaper(unittest.TestCase):
|
||||
|
||||
def test_case(self):
|
||||
self.assertTrue(filter_paper(fixtures.doe2013, ['title:nice']))
|
||||
self.assertTrue(filter_paper(fixtures.doe2013, ['title:Nice']))
|
||||
self.assertFalse(filter_paper(fixtures.doe2013, ['title:nIce']))
|
||||
self.assertTrue (filter_paper(doe_paper, ['title:nice']))
|
||||
self.assertTrue (filter_paper(doe_paper, ['title:Nice']))
|
||||
self.assertFalse(filter_paper(doe_paper, ['title:nIce']))
|
||||
|
||||
def test_fields(self):
|
||||
self.assertTrue(filter_paper(fixtures.doe2013, ['year:2013']))
|
||||
self.assertFalse(filter_paper(fixtures.doe2013, ['year:2014']))
|
||||
self.assertTrue(filter_paper(fixtures.doe2013, ['author:doe']))
|
||||
self.assertTrue(filter_paper(fixtures.doe2013, ['author:Doe']))
|
||||
self.assertTrue (filter_paper(doe_paper, ['year:2013']))
|
||||
self.assertFalse(filter_paper(doe_paper, ['year:2014']))
|
||||
self.assertTrue (filter_paper(doe_paper, ['author:doe']))
|
||||
self.assertTrue (filter_paper(doe_paper, ['author:Doe']))
|
||||
|
||||
def test_tags(self):
|
||||
self.assertTrue(filter_paper(fixtures.turing1950, ['tag:computer']))
|
||||
self.assertFalse(filter_paper(fixtures.turing1950, ['tag:Ai']))
|
||||
self.assertTrue(filter_paper(fixtures.turing1950, ['tag:AI']))
|
||||
self.assertTrue(filter_paper(fixtures.turing1950, ['tag:ai']))
|
||||
self.assertTrue (filter_paper(turing_paper, ['tag:computer']))
|
||||
self.assertFalse(filter_paper(turing_paper, ['tag:Ai']))
|
||||
self.assertTrue (filter_paper(turing_paper, ['tag:AI']))
|
||||
self.assertTrue (filter_paper(turing_paper, ['tag:ai']))
|
||||
|
||||
def test_multiple(self):
|
||||
self.assertTrue(filter_paper(fixtures.doe2013,
|
||||
self.assertTrue (filter_paper(doe_paper,
|
||||
['author:doe', 'year:2013']))
|
||||
self.assertFalse(filter_paper(fixtures.doe2013,
|
||||
self.assertFalse(filter_paper(doe_paper,
|
||||
['author:doe', 'year:2014']))
|
||||
self.assertFalse(filter_paper(fixtures.doe2013,
|
||||
self.assertFalse(filter_paper(doe_paper,
|
||||
['author:doee', 'year:2014']))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@ -1,18 +1,22 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import unittest
|
||||
|
||||
import testenv
|
||||
from papers.commands.tag_cmd import _parse_tags, _tag_groups
|
||||
import dotdot
|
||||
from pubs.commands.tag_cmd import _parse_tags, _tag_groups
|
||||
|
||||
class TestTag(unittest.TestCase):
|
||||
|
||||
def test_tag_parsing(self):
|
||||
|
||||
self.assertEqual(['+abc', '+def9'], _parse_tags( 'abc+def9'))
|
||||
self.assertEqual(['+abc', '-def9'], _parse_tags( 'abc-def9'))
|
||||
self.assertEqual(['-abc', '-def9'], _parse_tags('-abc-def9'))
|
||||
self.assertEqual(['+abc', '-def9'], _parse_tags('+abc-def9'))
|
||||
self.assertEqual(['+abc', '+def9'], _parse_tags([ 'abc+def9']))
|
||||
self.assertEqual(['+abc', '-def9'], _parse_tags([ 'abc-def9']))
|
||||
self.assertEqual(['-abc', '-def9'], _parse_tags(['-abc-def9']))
|
||||
self.assertEqual(['+abc', '-def9'], _parse_tags(['+abc-def9']))
|
||||
|
||||
self.assertEqual(({'math', 'romance'}, {'war'}), _tag_groups(_parse_tags('-war+math+romance')))
|
||||
self.assertEqual(({'math', 'romance'}, {'war'}), _tag_groups(_parse_tags('+math+romance-war')))
|
||||
self.assertEqual(({'math', 'romance'}, {'war'}), _tag_groups(_parse_tags('math+romance-war')))
|
||||
self.assertEqual(({'math', 'romance'}, {'war'}), _tag_groups(_parse_tags(['-war+math+romance'])))
|
||||
self.assertEqual(({'math', 'romance'}, {'war'}), _tag_groups(_parse_tags(['+math+romance-war'])))
|
||||
self.assertEqual(({'math', 'romance'}, {'war'}), _tag_groups(_parse_tags(['math+romance-war'])))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@ -0,0 +1,15 @@
|
||||
|
||||
@article{10.1371_journal.pone.0038236,
|
||||
author = {Caroline Lyon AND Chrystopher L. Nehaniv AND Joe Saunders},
|
||||
journal = {PLoS ONE},
|
||||
publisher = {Public Library of Science},
|
||||
title = {Interactive Language Learning by Robots: The Transition from Babbling to Word Forms},
|
||||
year = {2012},
|
||||
month = {06},
|
||||
volume = {7},
|
||||
url = {http://dx.doi.org/10.1371%2Fjournal.pone.0038236},
|
||||
pages = {e38236},
|
||||
abstract = {<p>The advent of humanoid robots has enabled a new approach to investigating the acquisition of language, and we report on the development of robots able to acquire rudimentary linguistic skills. Our work focuses on early stages analogous to some characteristics of a human child of about 6 to 14 months, the transition from babbling to first word forms. We investigate one mechanism among many that may contribute to this process, a key factor being the sensitivity of learners to the statistical distribution of linguistic elements. As well as being necessary for learning word meanings, the acquisition of anchor word forms facilitates the segmentation of an acoustic stream through other mechanisms. In our experiments some salient one-syllable word forms are learnt by a humanoid robot in real-time interactions with naive participants. Words emerge from random syllabic babble through a learning process based on a dialogue between the robot and the human participant, whose speech is perceived by the robot as a stream of phonemes. Numerous ways of representing the speech as syllabic segments are possible. Furthermore, the pronunciation of many words in spontaneous speech is variable. However, in line with research elsewhere, we observe that salient content words are more likely than function words to have consistent canonical representations; thus their relative frequency increases, as does their influence on the learner. Variable pronunciation may contribute to early word form acquisition. The importance of contingent interaction in real-time between teacher and learner is reflected by a reinforcement process, with variable success. The examination of individual cases may be more informative than group results. Nevertheless, word forms are usually produced by the robot after a few minutes of dialogue, employing a simple, real-time, frequency dependent mechanism. This work shows the potential of human-robot interaction systems in studies of the dynamics of early language acquisition.</p>},
|
||||
number = {6},
|
||||
doi = {10.1371/journal.pone.0038236}
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
|
||||
@article{10.1371/journal.pone.0063400,
|
||||
author = {Martius, , Georg AND Der, , Ralf AND Ay, , Nihat},
|
||||
journal = {PLoS ONE},
|
||||
publisher = {Public Library of Science},
|
||||
title = {Information Driven Self-Organization of Complex Robotic Behaviors},
|
||||
year = {2013},
|
||||
month = {05},
|
||||
volume = {8},
|
||||
url = {http://dx.doi.org/10.1371%2Fjournal.pone.0063400},
|
||||
pages = {e63400},
|
||||
abstract = {<p>Information theory is a powerful tool to express principles to drive autonomous systems because it is domain invariant and allows for an intuitive interpretation. This paper studies the use of the predictive information (PI), also called excess entropy or effective measure complexity, of the sensorimotor process as a driving force to generate behavior. We study nonlinear and nonstationary systems and introduce the time-local predicting information (TiPI) which allows us to derive exact results together with explicit update rules for the parameters of the controller in the dynamical systems framework. In this way the information principle, formulated at the level of behavior, is translated to the dynamics of the synapses. We underpin our results with a number of case studies with high-dimensional robotic systems. We show the spontaneous cooperativity in a complex physical system with decentralized control. Moreover, a jointly controlled humanoid robot develops a high behavioral variety depending on its physics and the environment it is dynamically embedded into. The behavior can be decomposed into a succession of low-dimensional modes that increasingly explore the behavior space. This is a promising way to avoid the curse of dimensionality which hinders learning systems to scale well.</p>},
|
||||
number = {5},
|
||||
doi = {10.1371/journal.pone.0063400}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
@techreport{Page99,
|
||||
number = {1999-66},
|
||||
month = {November},
|
||||
author = {Lawrence Page and Sergey Brin and Rajeev Motwani and Terry Winograd},
|
||||
note = {Previous number = SIDL-WP-1999-0120},
|
||||
title = {The PageRank Citation Ranking: Bringing Order to the Web.},
|
||||
type = {Technical Report},
|
||||
publisher = {Stanford InfoLab},
|
||||
year = {1999},
|
||||
institution = {Stanford InfoLab},
|
||||
url = {http://ilpubs.stanford.edu:8090/422/},
|
||||
abstract = {The importance of a Web page is an inherently subjective matter, which depends on the readers interests, knowledge and attitudes. But there is still much that can be said objectively about the relative importance of Web pages. This paper describes PageRank, a mathod for rating Web pages objectively and mechanically, effectively measuring the human interest and attention devoted to them. We compare PageRank to an idealized random Web surfer. We show how to efficiently compute PageRank for large numbers of pages. And, we show how to apply PageRank to search and to user navigation.}
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
@article{10.1371/journal.pone.0063400,
|
||||
author = {Martius, , Georg AND Der, , Ralf AND Ay, , Nihat},
|
||||
journal = {PLoS ONE},
|
||||
publisher = {Public Library of Science},
|
||||
title = {Information Driven Self-Organization of Complex Robotic Behaviors},
|
||||
}
|
||||
Binary file not shown.
@ -0,0 +1,3 @@
|
||||
docfile: null
|
||||
notes: []
|
||||
tags: []
|
||||
@ -0,0 +1,3 @@
|
||||
docfile: null
|
||||
notes: []
|
||||
tags: []
|
||||
@ -0,0 +1,3 @@
|
||||
docfile: pubsdir://doc/Page99.pdf
|
||||
notes: []
|
||||
tags: [search, network]
|
||||
@ -0,0 +1,3 @@
|
||||
docfile: null
|
||||
notes: []
|
||||
tags: []
|
||||
Loading…
Reference in new issue