aselimov/pubs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'feat/python3' into develop

Browse Source
This commit is contained in:
Olivier Mangin 2014-09-11 18:50:23 +02:00
commit d37b15dc3d
16 changed files with 136 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
pubs/beets_ui.py
View File

@ -1,57 +0,0 @@
# This file contains functions taken from the user interface of the beet
# tool (http://beets.radbox.org).
#
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
import locale
import sys
from .p3 import input
class UserError(Exception):
"""UI exception. Commands should throw this in order to display
nonrecoverable errors to the user.
"""
pass
def _encoding(config):
"""Tries to guess the encoding used by the terminal."""
# Configured override?
# Determine from locale settings.
try:
default_enc = locale.getdefaultlocale()[1] or 'utf8'
except ValueError:
# Invalid locale environment variable setting. To avoid
# failing entirely for no good reason, assume UTF-8.
default_enc = 'utf8'
return config.get('terminal-encoding', default_enc)
def input_():
"""Get input and decodes the result to a Unicode string.
Raises a UserError if stdin is not available. The prompt is sent to
stdout rather than stderr. A printed between the prompt and the
input cursor.
"""
# raw_input incorrectly sends prompts to stderr, not stdout, so we
# use print() explicitly to display prompts.
# http://bugs.python.org/issue1927
try:
resp = input()
except EOFError:
raise UserError('stdin stream ended while input required')
return resp.decode(sys.stdin.encoding or 'utf8', 'ignore')

4
pubs/commands/tag_cmd.py
View File

@ -83,12 +83,12 @@ def command(args):
rp = Repository(config())
if citekeyOrTag is None:
ui.print_(color.dye(' '.join(rp.get_tags()), color=color.blue))
ui.print_(color.dye(' '.join(sorted(rp.get_tags())), color=color.blue))
else:
if rp.databroker.exists(citekeyOrTag):
p = rp.pull_paper(citekeyOrTag)
if tags == []:
ui.print_(color.dye(' '.join(p.tags),
ui.print_(color.dye(' '.join(sorted(p.tags)),
color=color.blue))
else:
add_tags, remove_tags = _tag_groups(_parse_tags(tags))

15
pubs/configs.py
View File

@ -1,10 +1,9 @@
import os
import collections
from .p3 import configparser
from . import content
from .p3 import configparser, _read_config
from .content import system_path, check_file
from .content import check_file, _open
# constant stuff (DFT = DEFAULT)
@ -64,17 +63,15 @@ class Config(object):
_config = self
def load(self, path=DFT_CONFIG_PATH):
if not content.check_file(path, fail=False):
if not check_file(path, fail=False):
raise IOError(("The configuration file {} does not exist."
" Did you run 'pubs init' ?").format(path))
with open(content.system_path(path), 'r') as f:
read = self._cfg.readfp(f)
# if len(read) == 0:
# raise IOError("Syntax error in {} config file. Aborting.".format(path))
with _open(path, 'r') as f:
_read_config(self._cfg, f)
return self
def save(self, path=DFT_CONFIG_PATH):
with open(content.system_path(path), 'w') as f:
with _open(path, 'w') as f:
self._cfg.write(f)
def __setattr__(self, name, value):

8
pubs/content.py
View File

@ -43,6 +43,10 @@ def system_path(path):
return os.path.abspath(os.path.expanduser(path))
def _open(path, mode):
return io.open(system_path(path), mode, encoding=ENCODING)
def check_file(path, fail=True):
syspath = system_path(path)
return (_check_system_path_exists(syspath, fail=fail)
@ -57,14 +61,14 @@ def check_directory(path, fail=True):
def read_file(filepath):
check_file(filepath)
with io.open(system_path(filepath), 'r', encoding=ENCODING) as f:
with _open(filepath, 'r') as f:
content = f.read()
return content
def write_file(filepath, data):
check_directory(os.path.dirname(filepath))
with io.open(system_path(filepath), 'w', encoding=ENCODING) as f:
with _open(filepath, 'w') as f:
f.write(data)

34
pubs/p3.py
View File

@ -1,21 +1,53 @@
import io
import sys
if sys.version_info[0] == 2:
import ConfigParser as configparser
input = raw_input
_read_config = configparser.SafeConfigParser.readfp
def input():
raw_input().decode(sys.stdin.encoding or 'utf8', 'ignore')
# The following has to be a function so that it can be mocked
# for test_usecase.
def _get_raw_stdout():
return sys.stdout
ustr = unicode
uchr = unichr
from urlparse import urlparse
from urllib2 import urlopen
from httplib import HTTPConnection
file = None
_fake_stdio = io.BytesIO # Only for tests to capture std{out,err}
def _get_fake_stdio_ucontent(stdio):
ustdio = io.TextIOWrapper(stdio)
ustdio.seek(0)
return ustdio.read()
else:
import configparser
_read_config = configparser.SafeConfigParser.read_file
ustr = str
uchr = chr
from urllib.parse import urlparse
from urllib.request import urlopen
from http.client import HTTPConnection
def _fake_stdio():
return io.TextIOWrapper(io.BytesIO()) # Only for tests to capture std{out,err}
def _get_fake_stdio_ucontent(stdio):
stdio.flush()
stdio.seek(0)
return stdio.read()
# The following has to be a function so that it can be mocked
# for test_usecase.
def _get_raw_stdout():
return sys.stdout.buffer
configparser = configparser
input = input

3
pubs/paper.py
View File

@ -2,6 +2,7 @@ import copy
from dateutil.parser import parse as datetime_parse
from . import bibstruct
from .p3 import ustr
DEFAULT_META = {'docfile': None, 'tags': set()}
@ -11,7 +12,7 @@ def _clean_metadata(metadata):
meta = copy.deepcopy(DEFAULT_META)
meta.update(metadata or {}) # handles None metadata
meta['tags'] = set(meta.get('tags', [])) # tags should be a set
if 'added' in meta and isinstance(meta['added'], basestring):
if 'added' in meta and isinstance(meta['added'], ustr):
meta['added'] = datetime_parse(meta['added'])
return meta

6
pubs/pretty.py
View File

@ -53,9 +53,9 @@ def paper_oneliner(p, citekey_only = False):
return p.citekey
else:
bibdesc = bib_oneliner(p.bibentry)
return (u'[{citekey}] {descr} {tags}'.format(
return u'[{citekey}] {descr} {tags}'.format(
citekey=color.dye(p.citekey, color.purple),
descr=bibdesc,
tags=color.dye(' '.join(p.tags),
tags=color.dye(' '.join(sorted(p.tags)),
color.tag, bold=False),
)).encode('utf-8')
)

4
pubs/pubs
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python2
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from pubs import pubs_cmd
pubs_cmd.execute()
pubs_cmd.execute()

2
pubs/pubs_cmd.py
View File

@ -1,5 +1,3 @@
#!/usr/bin/env python2
# -*- coding:utf-8 -*-
import sys
import argparse

2
pubs/templates/__init__.py
View File

@ -1 +1 @@
from str_templates import *
from . str_templates import *

35
pubs/uis.py
View File

@ -1,11 +1,13 @@
from __future__ import print_function
import sys
import locale
import codecs
from .beets_ui import _encoding, input_
from .content import editor_input
from .p3 import ustr
from . import color
from .p3 import _get_raw_stdout
# package-shared ui that can be accessed using :
# from uis import get_ui
@ -14,6 +16,16 @@ from . import color
_ui = None
def _get_encoding(config):
"""Get local terminal encoding or user preference in config."""
enc = None
try:
enc = locale.getdefaultlocale()[1]
except ValueError:
pass # Keep default
return config.get('terminal-encoding', enc or 'utf8')
def get_ui():
if _ui is None:
raise ValueError('ui not instanciated yet')
@ -30,19 +42,26 @@ class UI:
"""
def __init__(self, config):
self.encoding = _encoding(config)
color.setup(config.color)
self.editor = config.edit_cmd
self.encoding = _get_encoding(config)
self._stdout = codecs.getwriter(self.encoding)(_get_raw_stdout(),
errors='replace')
def print_(self, *strings):
"""Like print, but rather than raising an error when a character
is not in the terminal's encoding's character set, just silently
replaces it.
"""
txt = [s.encode(self.encoding, 'replace')
if isinstance(s, ustr) else s
for s in strings]
print(' '.join(txt))
print(' '.join(strings), file=self._stdout)
def input(self):
try:
data = input()
except EOFError:
self.error('Standard input ended while waiting for answer.')
self.exit(1)
return data
def input_choice(self, options, option_chars, default=None, question=''):
"""Ask the user to chose between a set of options. The iser is asked
@ -65,7 +84,7 @@ class UI:
for c, o in zip(displayed_chars, options)])
self.print_(question, option_str)
while True:
answer = input_()
answer = self.input()
if answer is None or answer == '':
if default is not None:
return default

3
setup.py
View File

@ -2,6 +2,7 @@
from setuptools import setup, find_packages
setup(name='pubs',
version='4',
author='Fabien Benureau, Olivier Mangin, Jonathan Grizou',
@ -10,5 +11,5 @@ setup(name='pubs',
description='research papers manager',
requires=['pyyaml', 'bibtexparser', 'dateutil'],
packages=find_packages(),
scripts=['pubs/pubs']
scripts=['pubs/pubs'],
)

37
tests/fake_env.py
View File

@ -10,14 +10,13 @@ import fake_filesystem
import fake_filesystem_shutil
import fake_filesystem_glob
from pubs.p3 import input
from pubs.p3 import input, _fake_stdio, _get_fake_stdio_ucontent
from pubs import content, filebroker
# code for fake fs
real_os = os
real_open = open
real_file = file
real_shutil = shutil
real_glob = glob
real_io = io
@ -41,7 +40,7 @@ ENCODING = 'utf8'
class UnicodeStringIOWrapper(object):
"""This is a hack because fake_filesystem does not provied mock of io.
"""This is a hack because fake_filesystem does not provide mock of io.
"""
override = ['read', 'readline', 'readlines', 'write', 'writelines']
@ -55,6 +54,10 @@ class UnicodeStringIOWrapper(object):
else:
return self._strio.__getattribute__(name)
def __iter__(self):
for l in self.readlines():
yield l
def read(self, *args):
return self._strio.read(*args).decode(ENCODING)
@ -78,13 +81,24 @@ class UnicodeStringIOWrapper(object):
return self._strio.__exit__(*args)
def _force_binary_mode(mode):
if 'b' in mode:
raise ValueError('Open should not happen in binary mode.')
return mode + 'b'
class FakeIO(object):
def __init__(self, fake_open):
self.fake_open = fake_open
def open(self, *args, **kwargs):
# Forces python3 mode for FakeFileOpen
# Forces binary mode for FakeFileOpen
args = list(args)
if len(args) > 1:
args[1] = _force_binary_mode(args[1])
else:
kwargs['mode'] = _force_binary_mode(kwargs.get('mode', 'r'))
fakefs_stringio = self.fake_open.Call(*args, **kwargs)
return UnicodeStringIOWrapper(fakefs_stringio)
@ -100,8 +114,6 @@ def create_fake_fs(module_list):
fake_fs.CreateDirectory(fake_os.path.expanduser('~'))
__builtins__.update({'open': fake_open, 'file': fake_open})
sys.modules['os'] = fake_os
sys.modules['shutil'] = fake_shutil
sys.modules['glob'] = fake_glob
@ -111,7 +123,7 @@ def create_fake_fs(module_list):
md.os = fake_os
md.shutil = fake_shutil
md.open = fake_open
md.file = fake_open
md.file = None
md.io = fake_io
return {'fs': fake_fs,
@ -125,10 +137,8 @@ def create_fake_fs(module_list):
def unset_fake_fs(module_list):
try:
__builtins__.open = real_open
__builtins__.file = real_file
except AttributeError:
__builtins__['open'] = real_open
__builtins__['file'] = real_file
sys.modules['os'] = real_os
sys.modules['shutil'] = real_shutil
@ -139,7 +149,6 @@ def unset_fake_fs(module_list):
md.os = real_os
md.shutil = real_shutil
md.open = real_open
md.file = real_file
md.io = real_io
@ -163,11 +172,11 @@ def copy_dir(fs, real_dir, fake_dir = None):
def redirect(f):
def newf(*args, **kwargs):
old_stderr, old_stdout = sys.stderr, sys.stdout
stdout = io.BytesIO()
stderr = io.BytesIO()
stdout = _fake_stdio()
stderr = _fake_stdio()
sys.stdout, sys.stderr = stdout, stderr
try:
return f(*args, **kwargs), stdout, stderr
return f(*args, **kwargs), _get_fake_stdio_ucontent(stdout), _get_fake_stdio_ucontent(stderr)
finally:
sys.stderr, sys.stdout = old_stderr, old_stdout
return newf
@ -190,7 +199,7 @@ class FakeInput():
Then :
input() returns 'yes'
input() returns 'no'
input() raise IndexError
input() raises IndexError
"""
def __init__(self, inputs, module_list=tuple()):

1
tests/test_endecoder.py
View File

@ -34,7 +34,6 @@ class TestEnDecode(unittest.TestCase):
data = decoder.encode_metadata(dummy_metadata)
self.assertIsInstance(data, ustr)
def test_endecode_bibtex(self):
decoder = endecoder.EnDecoder()
entry = decoder.decode_bibdata(bibtex_raw0)

8
tests/test_filebroker.py
View File

@ -29,11 +29,11 @@ class TestFileBroker(fake_env.TestFakeFs):
fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'testrepo'), 'testrepo')
fb = filebroker.FileBroker('testrepo', create = True)
with open('testrepo/bib/Page99.bib', 'r') as f:
self.assertEqual(fb.pull_bibfile('Page99'), f.read())
bib_content = content.read_file('testrepo/bib/Page99.bib')
self.assertEqual(fb.pull_bibfile('Page99'), bib_content)
with open('testrepo/meta/Page99.yaml', 'r') as f:
self.assertEqual(fb.pull_metafile('Page99'), f.read())
meta_content = content.read_file('testrepo/meta/Page99.yaml')
self.assertEqual(fb.pull_metafile('Page99'), meta_content)
def test_errors(self):

49
tests/test_usecase.py
View File

@ -8,7 +8,7 @@ import dotdot
import fake_env
from pubs import pubs_cmd
from pubs import color, content, filebroker, uis, beets_ui, p3, endecoder, configs
from pubs import color, content, filebroker, uis, p3, endecoder, configs
import str_fixtures
import fixtures
@ -59,27 +59,27 @@ class CommandTestCase(unittest.TestCase):
self.fs = fake_env.create_fake_fs([content, filebroker, configs, init_cmd, import_cmd])
self.default_pubs_dir = self.fs['os'].path.expanduser('~/.pubs')
def execute_cmds(self, cmds, fs=None, capture_output=CAPTURE_OUTPUT):
def execute_cmds(self, cmds, capture_output=CAPTURE_OUTPUT):
""" Execute a list of commands, and capture their output
A command can be a string, or a tuple of size 2 or 3.
In the latter case, the command is :
1. a string reprensenting the command to execute
2. the user inputs to feed to the command during execution
3. the output excpected, verified with assertEqual
3. the output expected, verified with assertEqual
"""
outs = []
for cmd in cmds:
if hasattr(cmd, '__iter__'):
if not isinstance(cmd, p3.ustr):
if len(cmd) == 2:
input = fake_env.FakeInput(cmd[1], [content, uis, beets_ui, p3])
input = fake_env.FakeInput(cmd[1], [content, uis, p3])
input.as_global()
if capture_output:
_, stdout, stderr = fake_env.redirect(pubs_cmd.execute)(cmd[0].split())
if len(cmd) == 3 and capture_output:
actual_out = color.undye(stdout.getvalue())
actual_out = color.undye(stdout)
correct_out = color.undye(cmd[2])
self.assertEqual(actual_out, correct_out)
else:
@ -93,8 +93,8 @@ class CommandTestCase(unittest.TestCase):
pubs_cmd.execute(cmd.split())
if capture_output:
assert(stderr.getvalue() == '')
outs.append(color.undye(stdout.getvalue()))
assert(stderr == '')
outs.append(color.undye(stdout))
if PRINT_OUTPUT:
print(outs)
return outs
@ -119,13 +119,13 @@ class TestInit(CommandTestCase):
def test_init(self):
pubsdir = os.path.expanduser('~/pubs_test2')
pubs_cmd.execute('pubs init -p {}'.format(pubsdir).split())
self.execute_cmds(['pubs init -p {}'.format(pubsdir)])
self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
{'bib', 'doc', 'meta', 'notes'})
def test_init2(self):
pubsdir = os.path.expanduser('~/.pubs')
pubs_cmd.execute('pubs init'.split())
self.execute_cmds(['pubs init'])
self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
{'bib', 'doc', 'meta', 'notes'})
@ -246,13 +246,13 @@ class TestList(DataCommandTestCase):
class TestUsecase(DataCommandTestCase):
def test_first(self):
correct = [b'Initializing pubs in /paper_first\n',
b'',
b'[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) \n',
b'\n',
b'',
b'search network\n',
b'[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) search network\n'
correct = ['Initializing pubs in /paper_first\n',
'',
'[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) \n',
'\n',
'',
'network search\n',
'[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) network search\n'
]
cmds = ['pubs init -p paper_first/',
@ -293,12 +293,13 @@ class TestUsecase(DataCommandTestCase):
print(self.fs['os'].listdir(docdir))
self.assertNotIn('turing-mind-1950.pdf', self.fs['os'].listdir(docdir))
def test_tag_list(self):
correct = [b'Initializing pubs in /paper_first\n',
b'',
b'',
b'',
b'search network\n',
correct = ['Initializing pubs in /paper_first\n',
'',
'',
'',
'search network\n',
]
cmds = ['pubs init -p paper_first/',
@ -362,8 +363,8 @@ class TestUsecase(DataCommandTestCase):
'pubs export Page99',
]
outs = self.execute_cmds(cmds)
out_raw = outs[2].decode()
self.assertEqual(endecoder.EnDecoder().decode_bibdata(out_raw), fixtures.page_bibdata)
self.assertEqual(endecoder.EnDecoder().decode_bibdata(outs[2]),
fixtures.page_bibdata)
def test_import(self):
cmds = ['pubs init',