Merge branch 'feat/python3' into develop

pubs/beets_ui.py

@@ -1,57 +0,0 @@
-# This file contains functions taken from the user interface of the beet
-# tool (http://beets.radbox.org).
-#
-# Copyright 2013, Adrian Sampson.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-
-
-import locale
-import sys
-
-from .p3 import input
-
-
-class UserError(Exception):
-    """UI exception. Commands should throw this in order to display
-    nonrecoverable errors to the user.
-    """
-    pass
-
-
-def _encoding(config):
-    """Tries to guess the encoding used by the terminal."""
-    # Configured override?
-    # Determine from locale settings.
-    try:
-        default_enc = locale.getdefaultlocale()[1] or 'utf8'
-    except ValueError:
-        # Invalid locale environment variable setting. To avoid
-        # failing entirely for no good reason, assume UTF-8.
-        default_enc = 'utf8'
-    return config.get('terminal-encoding', default_enc)
-
-
-def input_():
-    """Get input and decodes the result to a Unicode string.
-    Raises a UserError if stdin is not available. The prompt is sent to
-    stdout rather than stderr. A printed between the prompt and the
-    input cursor.
-    """
-    # raw_input incorrectly sends prompts to stderr, not stdout, so we
-    # use print() explicitly to display prompts.
-    # http://bugs.python.org/issue1927
-    try:
-        resp = input()
-    except EOFError:
-        raise UserError('stdin stream ended while input required')
-    return resp.decode(sys.stdin.encoding or 'utf8', 'ignore')

pubs/commands/tag_cmd.py

@@ -83,12 +83,12 @@ def command(args):
     rp = Repository(config())
 
     if citekeyOrTag is None:
-        ui.print_(color.dye(' '.join(rp.get_tags()), color=color.blue))
+        ui.print_(color.dye(' '.join(sorted(rp.get_tags())), color=color.blue))
     else:
         if rp.databroker.exists(citekeyOrTag):
             p = rp.pull_paper(citekeyOrTag)
             if tags == []:
-                ui.print_(color.dye(' '.join(p.tags),
+                ui.print_(color.dye(' '.join(sorted(p.tags)),
                           color=color.blue))
             else:
                 add_tags, remove_tags = _tag_groups(_parse_tags(tags))

pubs/configs.py

@@ -1,10 +1,9 @@
 import os
 import collections
 
-from .p3 import configparser
+from .p3 import configparser, _read_config
-from . import content
 
-from .content import system_path, check_file
+from .content import check_file, _open
 
 
 # constant stuff (DFT = DEFAULT)
@@ -64,17 +63,15 @@ class Config(object):
         _config = self
 
     def load(self, path=DFT_CONFIG_PATH):
-        if not content.check_file(path, fail=False):
+        if not check_file(path, fail=False):
             raise IOError(("The configuration file {} does not exist."
                            " Did you run 'pubs init' ?").format(path))
-        with open(content.system_path(path), 'r') as f:
+        with _open(path, 'r') as f:
-            read = self._cfg.readfp(f)
+            _read_config(self._cfg, f)
-        # if len(read) == 0:
-        #     raise IOError("Syntax error in {} config file. Aborting.".format(path))
         return self
 
     def save(self, path=DFT_CONFIG_PATH):
-        with open(content.system_path(path), 'w') as f:
+        with _open(path, 'w') as f:
             self._cfg.write(f)
 
     def __setattr__(self, name, value):

pubs/content.py

@@ -43,6 +43,10 @@ def system_path(path):
     return os.path.abspath(os.path.expanduser(path))
 
 
+def _open(path, mode):
+        return io.open(system_path(path), mode, encoding=ENCODING)
+
+
 def check_file(path, fail=True):
     syspath = system_path(path)
     return (_check_system_path_exists(syspath, fail=fail)
@@ -57,14 +61,14 @@ def check_directory(path, fail=True):
 
 def read_file(filepath):
     check_file(filepath)
-    with io.open(system_path(filepath), 'r', encoding=ENCODING) as f:
+    with _open(filepath, 'r') as f:
         content = f.read()
     return content
 
 
 def write_file(filepath, data):
     check_directory(os.path.dirname(filepath))
-    with io.open(system_path(filepath), 'w', encoding=ENCODING) as f:
+    with _open(filepath, 'w') as f:
         f.write(data)
 
 

pubs/p3.py

@@ -1,21 +1,53 @@
+import io
 import sys
 
 if sys.version_info[0] == 2:
     import ConfigParser as configparser
-    input = raw_input
+    _read_config = configparser.SafeConfigParser.readfp
+
+    def input():
+        raw_input().decode(sys.stdin.encoding or 'utf8', 'ignore')
+    
+    # The following has to be a function so that it can be mocked
+    # for test_usecase.
+    def _get_raw_stdout():
+        return sys.stdout
+
     ustr = unicode
     uchr = unichr
     from urlparse import urlparse
     from urllib2 import urlopen
     from httplib import HTTPConnection
+    file = None
+    _fake_stdio = io.BytesIO  # Only for tests to capture std{out,err}
+
+    def _get_fake_stdio_ucontent(stdio):
+        ustdio = io.TextIOWrapper(stdio)
+        ustdio.seek(0)
+        return ustdio.read()
+
 else:
     import configparser
+    _read_config = configparser.SafeConfigParser.read_file
     ustr = str
     uchr = chr
     from urllib.parse import urlparse
     from urllib.request import urlopen
     from http.client import HTTPConnection
 
+    def _fake_stdio():
+        return io.TextIOWrapper(io.BytesIO())  # Only for tests to capture std{out,err}
+
+    def _get_fake_stdio_ucontent(stdio):
+        stdio.flush()
+        stdio.seek(0)
+        return stdio.read()
+
+    # The following has to be a function so that it can be mocked
+    # for test_usecase.
+    def _get_raw_stdout():
+        return sys.stdout.buffer
+
 configparser = configparser
 input = input
 

pubs/paper.py

@@ -2,6 +2,7 @@ import copy
 from dateutil.parser import parse as datetime_parse
 
 from . import bibstruct
+from .p3 import ustr
 
 
 DEFAULT_META = {'docfile': None, 'tags': set()}
@@ -11,7 +12,7 @@ def _clean_metadata(metadata):
     meta = copy.deepcopy(DEFAULT_META)
     meta.update(metadata or {})  # handles None metadata
     meta['tags'] = set(meta.get('tags', []))  # tags should be a set
-    if 'added' in meta and isinstance(meta['added'], basestring):
+    if 'added' in meta and isinstance(meta['added'], ustr):
         meta['added'] = datetime_parse(meta['added'])
     return meta
 

pubs/pretty.py

@@ -53,9 +53,9 @@ def paper_oneliner(p, citekey_only = False):
         return p.citekey
     else:
         bibdesc = bib_oneliner(p.bibentry)
-        return (u'[{citekey}] {descr} {tags}'.format(
+        return u'[{citekey}] {descr} {tags}'.format(
             citekey=color.dye(p.citekey, color.purple),
             descr=bibdesc,
-            tags=color.dye(' '.join(p.tags),
+            tags=color.dye(' '.join(sorted(p.tags)),
                            color.tag, bold=False),
-            )).encode('utf-8')
+            )

pubs/pubs

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding:utf-8 -*-
 
 from pubs import pubs_cmd

pubs/pubs_cmd.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python2
-# -*- coding:utf-8 -*-
 import sys
 
 import argparse

pubs/templates/__init__.py

@@ -1 +1 @@
-from str_templates import *
+from . str_templates import *

pubs/uis.py

@@ -1,11 +1,13 @@
 from __future__ import print_function
 
 import sys
+import locale
+import codecs
 
-from .beets_ui import _encoding, input_
 from .content import editor_input
-from .p3 import ustr
 from . import color
+from .p3 import _get_raw_stdout
+
 
 # package-shared ui that can be accessed using :
 # from uis import get_ui
@@ -14,6 +16,16 @@ from . import color
 _ui = None
 
 
+def _get_encoding(config):
+    """Get local terminal encoding or user preference in config."""
+    enc = None
+    try:
+        enc = locale.getdefaultlocale()[1]
+    except ValueError:
+        pass  # Keep default
+    return config.get('terminal-encoding', enc or 'utf8')
+
+
 def get_ui():
     if _ui is None:
         raise ValueError('ui not instanciated yet')
@@ -30,19 +42,26 @@ class UI:
     """
 
     def __init__(self, config):
-        self.encoding = _encoding(config)
         color.setup(config.color)
         self.editor = config.edit_cmd
+        self.encoding = _get_encoding(config)
+        self._stdout = codecs.getwriter(self.encoding)(_get_raw_stdout(),
+                                                       errors='replace')
 
     def print_(self, *strings):
         """Like print, but rather than raising an error when a character
         is not in the terminal's encoding's character set, just silently
         replaces it.
         """
-        txt = [s.encode(self.encoding, 'replace')
+        print(' '.join(strings), file=self._stdout)
-               if isinstance(s, ustr) else s
+
-               for s in strings]
+    def input(self):
-        print(' '.join(txt))
+        try:
+            data = input()
+        except EOFError:
+            self.error('Standard input ended while waiting for answer.')
+            self.exit(1)
+        return data
 
     def input_choice(self, options, option_chars, default=None, question=''):
         """Ask the user to chose between a set of options. The iser is asked
@@ -65,7 +84,7 @@ class UI:
                                 for c, o in zip(displayed_chars, options)])
         self.print_(question, option_str)
         while True:
-            answer = input_()
+            answer = self.input()
             if answer is None or answer == '':
                 if default is not None:
                     return default

setup.py

@@ -2,6 +2,7 @@
 
 from setuptools import setup, find_packages
 
+
 setup(name='pubs',
       version='4',
       author='Fabien Benureau, Olivier Mangin, Jonathan Grizou',
@@ -10,5 +11,5 @@ setup(name='pubs',
       description='research papers manager',
       requires=['pyyaml', 'bibtexparser', 'dateutil'],
       packages=find_packages(),
-      scripts=['pubs/pubs']
+      scripts=['pubs/pubs'],
       )

tests/fake_env.py

@@ -10,14 +10,13 @@ import fake_filesystem
 import fake_filesystem_shutil
 import fake_filesystem_glob
 
-from pubs.p3 import input
+from pubs.p3 import input, _fake_stdio, _get_fake_stdio_ucontent
 from pubs import content, filebroker
 
     # code for fake fs
 
 real_os = os
 real_open = open
-real_file = file
 real_shutil = shutil
 real_glob = glob
 real_io = io
@@ -41,7 +40,7 @@ ENCODING = 'utf8'
 
 
 class UnicodeStringIOWrapper(object):
-    """This is a hack because fake_filesystem does not provied mock of io.
+    """This is a hack because fake_filesystem does not provide mock of io.
     """
 
     override = ['read', 'readline', 'readlines', 'write', 'writelines']
@@ -55,6 +54,10 @@ class UnicodeStringIOWrapper(object):
         else:
             return self._strio.__getattribute__(name)
 
+    def __iter__(self):
+        for l in self.readlines():
+            yield l
+
     def read(self, *args):
         return self._strio.read(*args).decode(ENCODING)
 
@@ -78,13 +81,24 @@ class UnicodeStringIOWrapper(object):
         return self._strio.__exit__(*args)
 
 
+def _force_binary_mode(mode):
+    if 'b' in mode:
+        raise ValueError('Open should not happen in binary mode.')
+    return mode + 'b'
+
+
 class FakeIO(object):
 
     def __init__(self, fake_open):
         self.fake_open = fake_open
 
     def open(self, *args, **kwargs):
-        # Forces python3 mode for FakeFileOpen
+        # Forces binary mode for FakeFileOpen
+        args = list(args)
+        if len(args) > 1:
+            args[1] = _force_binary_mode(args[1])
+        else:
+            kwargs['mode'] = _force_binary_mode(kwargs.get('mode', 'r'))
         fakefs_stringio = self.fake_open.Call(*args, **kwargs)
         return UnicodeStringIOWrapper(fakefs_stringio)
 
@@ -100,8 +114,6 @@ def create_fake_fs(module_list):
 
     fake_fs.CreateDirectory(fake_os.path.expanduser('~'))
 
-    __builtins__.update({'open': fake_open, 'file': fake_open})
-
     sys.modules['os']     = fake_os
     sys.modules['shutil'] = fake_shutil
     sys.modules['glob']   = fake_glob
@@ -111,7 +123,7 @@ def create_fake_fs(module_list):
         md.os = fake_os
         md.shutil = fake_shutil
         md.open = fake_open
-        md.file = fake_open
+        md.file = None
         md.io = fake_io
 
     return {'fs': fake_fs,
@@ -125,10 +137,8 @@ def create_fake_fs(module_list):
 def unset_fake_fs(module_list):
     try:
         __builtins__.open = real_open
-        __builtins__.file = real_file
     except AttributeError:
         __builtins__['open'] = real_open
-        __builtins__['file'] = real_file
 
     sys.modules['os']     = real_os
     sys.modules['shutil'] = real_shutil
@@ -139,7 +149,6 @@ def unset_fake_fs(module_list):
         md.os = real_os
         md.shutil = real_shutil
         md.open = real_open
-        md.file = real_file
         md.io = real_io
 
 
@@ -163,11 +172,11 @@ def copy_dir(fs, real_dir, fake_dir = None):
 def redirect(f):
     def newf(*args, **kwargs):
         old_stderr, old_stdout = sys.stderr, sys.stdout
-        stdout = io.BytesIO()
+        stdout = _fake_stdio()
-        stderr = io.BytesIO()
+        stderr = _fake_stdio()
         sys.stdout, sys.stderr = stdout, stderr
         try:
-            return f(*args, **kwargs), stdout, stderr
+            return f(*args, **kwargs), _get_fake_stdio_ucontent(stdout), _get_fake_stdio_ucontent(stderr)
         finally:
             sys.stderr, sys.stdout = old_stderr, old_stdout
     return newf
@@ -190,7 +199,7 @@ class FakeInput():
         Then :
         input() returns 'yes'
         input() returns 'no'
-        input() raise IndexError
+        input() raises IndexError
      """
 
     def __init__(self, inputs, module_list=tuple()):

tests/test_endecoder.py

@@ -34,7 +34,6 @@ class TestEnDecode(unittest.TestCase):
         data = decoder.encode_metadata(dummy_metadata)
         self.assertIsInstance(data, ustr)
 
-
     def test_endecode_bibtex(self):
         decoder = endecoder.EnDecoder()
         entry = decoder.decode_bibdata(bibtex_raw0)

tests/test_filebroker.py

@@ -29,11 +29,11 @@ class TestFileBroker(fake_env.TestFakeFs):
         fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'testrepo'), 'testrepo')
         fb = filebroker.FileBroker('testrepo', create = True)
 
-        with open('testrepo/bib/Page99.bib', 'r') as f:
+        bib_content = content.read_file('testrepo/bib/Page99.bib')
-            self.assertEqual(fb.pull_bibfile('Page99'), f.read())
+        self.assertEqual(fb.pull_bibfile('Page99'), bib_content)
 
-        with open('testrepo/meta/Page99.yaml', 'r') as f:
+        meta_content = content.read_file('testrepo/meta/Page99.yaml')
-            self.assertEqual(fb.pull_metafile('Page99'), f.read())
+        self.assertEqual(fb.pull_metafile('Page99'), meta_content)
 
     def test_errors(self):
 

tests/test_usecase.py

@@ -8,7 +8,7 @@ import dotdot
 import fake_env
 
 from pubs import pubs_cmd
-from pubs import color, content, filebroker, uis, beets_ui, p3, endecoder, configs
+from pubs import color, content, filebroker, uis, p3, endecoder, configs
 
 import str_fixtures
 import fixtures
@@ -59,27 +59,27 @@ class CommandTestCase(unittest.TestCase):
         self.fs = fake_env.create_fake_fs([content, filebroker, configs, init_cmd, import_cmd])
         self.default_pubs_dir = self.fs['os'].path.expanduser('~/.pubs')
 
-    def execute_cmds(self, cmds, fs=None, capture_output=CAPTURE_OUTPUT):
+    def execute_cmds(self, cmds, capture_output=CAPTURE_OUTPUT):
         """ Execute a list of commands, and capture their output
 
         A command can be a string, or a tuple of size 2 or 3.
         In the latter case, the command is :
         1. a string reprensenting the command to execute
         2. the user inputs to feed to the command during execution
-        3. the output excpected, verified with assertEqual
+        3. the output expected, verified with assertEqual
 
         """
         outs = []
         for cmd in cmds:
-            if hasattr(cmd, '__iter__'):
+            if not isinstance(cmd, p3.ustr):
                 if len(cmd) == 2:
-                    input = fake_env.FakeInput(cmd[1], [content, uis, beets_ui, p3])
+                    input = fake_env.FakeInput(cmd[1], [content, uis, p3])
                     input.as_global()
 
                 if capture_output:
                     _, stdout, stderr = fake_env.redirect(pubs_cmd.execute)(cmd[0].split())
                     if len(cmd) == 3 and capture_output:
-                        actual_out  = color.undye(stdout.getvalue())
+                        actual_out  = color.undye(stdout)
                         correct_out = color.undye(cmd[2])
                         self.assertEqual(actual_out, correct_out)
                 else:
@@ -93,8 +93,8 @@ class CommandTestCase(unittest.TestCase):
                     pubs_cmd.execute(cmd.split())
 
             if capture_output:
-                assert(stderr.getvalue() == '')
+                assert(stderr == '')
-                outs.append(color.undye(stdout.getvalue()))
+                outs.append(color.undye(stdout))
         if PRINT_OUTPUT:
             print(outs)
         return outs
@@ -119,13 +119,13 @@ class TestInit(CommandTestCase):
 
     def test_init(self):
         pubsdir = os.path.expanduser('~/pubs_test2')
-        pubs_cmd.execute('pubs init -p {}'.format(pubsdir).split())
+        self.execute_cmds(['pubs init -p {}'.format(pubsdir)])
         self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
                          {'bib', 'doc', 'meta', 'notes'})
 
     def test_init2(self):
         pubsdir = os.path.expanduser('~/.pubs')
-        pubs_cmd.execute('pubs init'.split())
+        self.execute_cmds(['pubs init'])
         self.assertEqual(set(self.fs['os'].listdir(pubsdir)),
                          {'bib', 'doc', 'meta', 'notes'})
 
@@ -246,13 +246,13 @@ class TestList(DataCommandTestCase):
 class TestUsecase(DataCommandTestCase):
 
     def test_first(self):
-        correct = [b'Initializing pubs in /paper_first\n',
+        correct = ['Initializing pubs in /paper_first\n',
-                   b'',
+                   '',
-                   b'[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) \n',
+                   '[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) \n',
-                   b'\n',
+                   '\n',
-                   b'',
+                   '',
-                   b'search network\n',
+                   'network search\n',
-                   b'[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) search network\n'
+                   '[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) network search\n'
                   ]
 
         cmds = ['pubs init -p paper_first/',
@@ -293,12 +293,13 @@ class TestUsecase(DataCommandTestCase):
         print(self.fs['os'].listdir(docdir))
         self.assertNotIn('turing-mind-1950.pdf', self.fs['os'].listdir(docdir))
 
+
     def test_tag_list(self):
-        correct = [b'Initializing pubs in /paper_first\n',
+        correct = ['Initializing pubs in /paper_first\n',
-                   b'',
+                   '',
-                   b'',
+                   '',
-                   b'',
+                   '',
-                   b'search network\n',
+                   'search network\n',
                   ]
 
         cmds = ['pubs init -p paper_first/',
@@ -362,8 +363,8 @@ class TestUsecase(DataCommandTestCase):
                 'pubs export Page99',
                ]
         outs = self.execute_cmds(cmds)
-        out_raw = outs[2].decode()
+        self.assertEqual(endecoder.EnDecoder().decode_bibdata(outs[2]),
-        self.assertEqual(endecoder.EnDecoder().decode_bibdata(out_raw), fixtures.page_bibdata)
+                         fixtures.page_bibdata)
 
     def test_import(self):
         cmds = ['pubs init',