Merge pull request #133 from benureau/fix/utf8citekeys

Fix/utf8citekeys

pubs/bibstruct.py

@@ -25,19 +25,14 @@ def str2citekey(s):
 
 def check_citekey(citekey):
     if citekey is None or not citekey.strip():
-        raise ValueError(u"Empty citekeys are not valid")
-    # TODO This is not the right way to test that (17/12/2012)
-    if ustr(citekey) != str2citekey(citekey):
-        raise ValueError(u"Invalid `{}` citekey; ".format(citekey) +
-                         u"utf-8 citekeys are not supported yet.\n"
-                         u"See https://github.com/pubs/pubs/issues/28 for details.")
+        raise ValueError("Empty citekeys are not valid")
 
 
 def verify_bibdata(bibdata):
     if bibdata is None or len(bibdata) == 0:
-        raise ValueError(u"no valid bibdata")
+        raise ValueError("no valid bibdata")
     if len(bibdata) > 1:
-        raise ValueError(u"ambiguous: multiple entries in the bibdata.")
+        raise ValueError("ambiguous: multiple entries in the bibdata.")
 
 
 def get_entry(bibdata):
@@ -69,12 +64,12 @@ def generate_citekey(bibdata):
         first_author = entry[author_key][0]
     except KeyError:
         raise ValueError(
-            u"No author or editor defined: cannot generate a citekey.")
+            "No author or editor defined: cannot generate a citekey.")
     try:
         year = entry['year']
     except KeyError:
         year = ''
-    citekey = u'{}{}'.format(u''.join(author_last(first_author)), year)
+    citekey = '{}{}'.format(''.join(author_last(first_author)), year)
 
     return str2citekey(citekey)
 

pubs/color.py

@@ -1,8 +1,6 @@
 """
 Code to handle colored text
-"""
 
-"""
 Here is a little explanation about bash color code, useful to understand
 the code below. See http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
 for a complete referece.
@@ -25,6 +23,7 @@ by the bright version of the font; some terminals allow the user to decide that.
 display colors, with 0 <= c < 8 corresponding to the 8 above colors, and
 8 <= c < 16 their bright version.
 """
+from __future__ import unicode_literals
 
 import sys
 import re
@@ -32,15 +31,15 @@ import os
 import subprocess
 
 
-COLOR_LIST = {u'black': '0', u'red': '1', u'green': '2', u'yellow': '3', u'blue': '4',
-              u'magenta': '5', u'cyan': '6', u'grey': '7',
-              u'brightblack': '8', u'brightred': '9', u'brightgreen': '10',
-              u'brightyellow': '11', u'brightblue': '12', u'brightmagenta': '13',
-              u'brightcyan': '14', u'brightgrey': '15',
-              u'darkgrey': '8', # == brightblack
-              u'gray': '7', u'darkgray': '8', u'brightgray': '15', # gray/grey spelling
-              u'purple': '5', # for compatibility reasons
-              u'white': '15' # == brightgrey
+COLOR_LIST = {'black': '0', 'red': '1', 'green': '2', 'yellow': '3', 'blue': '4',
+              'magenta': '5', 'cyan': '6', 'grey': '7',
+              'brightblack': '8', 'brightred': '9', 'brightgreen': '10',
+              'brightyellow': '11', 'brightblue': '12', 'brightmagenta': '13',
+              'brightcyan': '14', 'brightgrey': '15',
+              'darkgrey': '8', # == brightblack
+              'gray': '7', 'darkgray': '8', 'brightgray': '15', # gray/grey spelling
+              'purple': '5', # for compatibility reasons
+              'white': '15' # == brightgrey
              }
 for c in range(256):
     COLOR_LIST[str(c)] = str(c)
@@ -74,43 +73,43 @@ def generate_colors(stream, color=True, bold=True, italic=True, force_colors=Fal
                           normal colors.
     :param italic:        generate italic colors
     """
-    colors = {u'bold': u'', u'italic': u'', u'end': u'', u'': u''}
+    colors = {'bold': '', 'italic': '', 'end': '', '': ''}
     for name, code in COLOR_LIST.items():
-        colors[name]       = u''
-        colors[u'b' +name] = u''
-        colors[u'i' +name] = u''
-        colors[u'bi'+name] = u''
+        colors[name]      = ''
+        colors['b' +name] = ''
+        colors['i' +name] = ''
+        colors['bi'+name] = ''
 
     color_support = _color_supported(stream, force=force_colors) >= 8
 
     if (color or bold or italic) and color_support:
         bold_flag, italic_flag = '', ''
         if bold:
-            colors['bold'] = u'\033[1m'
+            colors['bold'] = '\033[1m'
             bold_flag = '1;'
         if italic:
-            colors['italic'] = u'\033[3m'
+            colors['italic'] = '\033[3m'
             italic_flag = '3;'
         if bold and italic:
-            colors['bolditalic'] = u'\033[1;3m'
+            colors['bolditalic'] = '\033[1;3m'
 
         for name, code in COLOR_LIST.items():
             if color:
-                colors[name] = u'\033[38;5;{}m'.format(code)
-                colors[u'b'+name] = u'\033[{}38;5;{}m'.format(bold_flag, code)
-                colors[u'i'+name] = u'\033[{}38;5;{}m'.format(italic_flag, code)
-                colors[u'bi'+name] = u'\033[{}38;5;{}m'.format(bold_flag, italic_flag, code)
+                colors[name] = '\033[38;5;{}m'.format(code)
+                colors['b'+name] = '\033[{}38;5;{}m'.format(bold_flag, code)
+                colors['i'+name] = '\033[{}38;5;{}m'.format(italic_flag, code)
+                colors['bi'+name] = '\033[{}38;5;{}m'.format(bold_flag, italic_flag, code)
 
             else:
                 if bold:
-                    colors.update({u'b'+name: u'\033[1m' for i, name in enumerate(COLOR_LIST)})
+                    colors.update({'b'+name: '\033[1m' for i, name in enumerate(COLOR_LIST)})
                 if italic:
-                    colors.update({u'i'+name: u'\033[3m' for i, name in enumerate(COLOR_LIST)})
+                    colors.update({'i'+name: '\033[3m' for i, name in enumerate(COLOR_LIST)})
                 if bold or italic:
-                    colors.update({u'bi'+name: u'\033[{}{}m'.format(bold_flag, italic_flag) for i, name in enumerate(COLOR_LIST)})
+                    colors.update({'bi'+name: '\033[{}{}m'.format(bold_flag, italic_flag) for i, name in enumerate(COLOR_LIST)})
 
         if color or bold or italic:
-            colors[u'end'] = u'\033[0m'
+            colors['end'] = '\033[0m'
 
     return colors
 
@@ -121,11 +120,11 @@ COLORS_ERR = generate_colors(sys.stderr, color=False, bold=False, italic=False)
 
 def dye_out(s, color='end'):
     """Color a string for output on stdout"""
-    return u'{}{}{}'.format(COLORS_OUT[color], s, COLORS_OUT['end'])
+    return '{}{}{}'.format(COLORS_OUT[color], s, COLORS_OUT['end'])
 
 def dye_err(s, color='end'):
     """Color a string for output on stderr"""
-    return u'{}{}{}'.format(COLORS_ERR[color], s, COLORS_OUT['end'])
+    return '{}{}{}'.format(COLORS_ERR[color], s, COLORS_OUT['end'])
 
 
 def setup(conf, force_colors=False):

pubs/commands/add_cmd.py

@@ -1,5 +1,8 @@
+from __future__ import unicode_literals
+
 import argparse
 from ..uis import get_ui
+from .. import p3
 from .. import bibstruct
 from .. import content
 from .. import repo
@@ -29,7 +32,7 @@ def parser(subparsers, conf):
                         default=None
                         ).completer = CommaSeparatedTagsCompletion(conf)
     parser.add_argument('-k', '--citekey', help='citekey associated with the paper;\nif not provided, one will be generated automatically.',
-                        default=None)
+                        default=None, type=p3.u_maybe)
     parser.add_argument('-L', '--link', action='store_false', dest='copy', default=True,
                         help="don't copy document files, just create a link.")
     parser.add_argument('-M', '--move', action='store_true', dest='move', default=False,

pubs/commands/conf_cmd.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from .. import uis
 from .. import config
 from .. import content

pubs/commands/doc_cmd.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import subprocess
 

pubs/commands/edit_cmd.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from ..paper import Paper
 from .. import repo
 

pubs/commands/export_cmd.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import unicode_literals
 
 import argparse
 

pubs/commands/import_cmd.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import datetime
 
@@ -78,10 +80,10 @@ def command(conf, args):
     for k in keys:
         p = papers[k]
         if isinstance(p, Exception):
-            ui.error(u'Could not load entry for citekey {}.'.format(k))
+            ui.error('Could not load entry for citekey {}.'.format(k))
         else:
             rp.push_paper(p, overwrite=args.overwrite)
-            ui.info(u'{} imported.'.format(color.dye_out(p.citekey, 'citekey')))
+            ui.info('{} imported.'.format(color.dye_out(p.citekey, 'citekey')))
             docfile = bibstruct.extract_docfile(p.bibdata)
             if docfile is None:
                 ui.warning("No file for {}.".format(p.citekey))

pubs/commands/init_cmd.py

@@ -1,4 +1,5 @@
 # init command
+from __future__ import unicode_literals
 
 import os
 

pubs/commands/list_cmd.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from datetime import datetime
 
 from .. import repo

pubs/commands/note_cmd.py

@@ -7,7 +7,7 @@ from ..completion import CiteKeyCompletion
 def parser(subparsers, conf):
     parser = subparsers.add_parser('note',
                                    help='edit the note attached to a paper')
-    parser.add_argument('citekey', help='citekey of the paper'
+    parser.add_argument('citekey', help='citekey of the paper',
                         ).completer = CiteKeyCompletion(conf)
     return parser
 

pubs/commands/remove_cmd.py

@@ -1,8 +1,10 @@
+from __future__ import unicode_literals
+
 from .. import repo
 from .. import color
 from ..uis import get_ui
 from ..utils import resolve_citekey_list
-from ..p3 import ustr
+from ..p3 import ustr, u_maybe
 from ..completion import CiteKeyCompletion
 
 
@@ -10,8 +12,8 @@ def parser(subparsers, conf):
     parser = subparsers.add_parser('remove', help='removes a publication')
     parser.add_argument('-f', '--force', action='store_true', default=None,
                         help="does not prompt for confirmation.")
-    parser.add_argument('citekeys', nargs='+',
-                        help="one or several citekeys"
+    parser.add_argument('citekeys', nargs='+', type=u_maybe,
+                        help="one or several citekeys",
                         ).completer = CiteKeyCompletion(conf)
     return parser
 
@@ -37,10 +39,12 @@ def command(conf, args):
             except Exception as e:
                 ui.error(ustr(e))
                 failed = True
-        ui.message('The publication(s) [{}] were removed'.format(
-            ', '.join([color.dye_out(c, 'citekey') for c in keys])))
         if failed:
             ui.exit()  # Exit with nonzero error code
+        else:
+            ui.message('The publication(s) [{}] were removed'.format(
+                ', '.join([color.dye_out(c, 'citekey') for c in keys])))
+
         # FIXME: print should check that removal proceeded well.
     else:
         ui.message('The publication(s) [{}] were {} removed'.format(

pubs/commands/rename_cmd.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from ..uis import get_ui
 from .. import color
 from .. import repo

pubs/commands/tag_cmd.py

@@ -16,6 +16,7 @@ The different use cases are :
 7. > pubs tag -war+math+romance
     display all papers with the tag 'math', 'romance' but not 'war'
 """
+from __future__ import unicode_literals
 
 import re
 

pubs/commands/websearch_cmd.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import webbrowser
 
 from .. import p3

pubs/content.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import sys
 import os
 import shutil
@@ -28,7 +30,7 @@ class UnableToDecodeTextFile(Exception):
 def _check_system_path_exists(path, fail=True):
     answer = os.path.exists(path)
     if not answer and fail:
-        raise IOError(u'File does not exist: {}'.format(path))
+        raise IOError('File does not exist: {}'.format(path))
     else:
         return answer
 
@@ -37,7 +39,7 @@ def _check_system_path_is(nature, path, fail=True):
     check_fun = getattr(os.path, nature)
     answer = check_fun(path)
     if not answer and fail:
-        raise IOError(u'{} is not a {}.'.format(path, nature))
+        raise IOError('{} is not a {}.'.format(path, nature))
     else:
         return answer
 
@@ -56,13 +58,13 @@ def _open(path, mode):
 def check_file(path, fail=True):
     syspath = system_path(path)
     return (_check_system_path_exists(syspath, fail=fail) and
-            _check_system_path_is(u'isfile', syspath, fail=fail))
+            _check_system_path_is('isfile', syspath, fail=fail))
 
 
 def check_directory(path, fail=True):
     syspath = system_path(path)
     return (_check_system_path_exists(syspath, fail=fail) and
-            _check_system_path_is(u'isdir', syspath, fail=fail))
+            _check_system_path_is('isdir', syspath, fail=fail))
 
 
 def read_text_file(filepath, fail=True):
@@ -112,23 +114,23 @@ def write_file(filepath, data, mode='w'):
 
 def content_type(path):
     parsed = urlparse(path)
-    if parsed.scheme == u'http':
-        return u'url'
+    if parsed.scheme == 'http':
+        return 'url'
     else:
-        return u'file'
+        return 'file'
 
 
 def url_exists(url):
     parsed = urlparse(url)
     conn = HTTPConnection(parsed.netloc)
-    conn.request(u'HEAD', parsed.path)
+    conn.request('HEAD', parsed.path)
     response = conn.getresponse()
     conn.close()
     return response.status == 200
 
 
 def check_content(path):
-    if content_type(path) == u'url':
+    if content_type(path) == 'url':
         return url_exists(path)
     else:
         return check_file(path)
@@ -136,7 +138,7 @@ def check_content(path):
 
 def _get_byte_url_content(path, ui=None):
     if ui is not None:
-        ui.message(u'dowloading {}'.format(path))
+        ui.message('dowloading {}'.format(path))
     response = urlopen(path)
     return response.read()
 
@@ -151,7 +153,7 @@ def _dump_byte_url_content(source, target):
 
 def get_content(path, ui=None):
     """Will be useful when we need to get content from url"""
-    if content_type(path) == u'url':
+    if content_type(path) == 'url':
         return _get_byte_url_content(path, ui=ui).decode(encoding='utf-8')
     else:
         return read_text_file(path)
@@ -163,19 +165,19 @@ def move_content(source, target, overwrite=False):
     if source == target:
         return
     if not overwrite and os.path.exists(target):
-        raise IOError(u'target file exists')
+        raise IOError('target file exists')
     shutil.move(source, target)
 
 
 def copy_content(source, target, overwrite=False):
-    source_is_url = content_type(source) == u'url'
+    source_is_url = content_type(source) == 'url'
     if not source_is_url:
         source = system_path(source)
     target = system_path(target)
     if source == target:
         return
     if not overwrite and os.path.exists(target):
-        raise IOError(u'{} file exists.'.format(target))
+        raise IOError('{} file exists.'.format(target))
     if source_is_url:
         _dump_byte_url_content(source, target)
     else:

pubs/databroker.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from . import filebroker
 from . import endecoder
 from .p3 import pickle
@@ -76,7 +78,7 @@ class DataBroker(object):
     def verify(self, bibdata_raw):
         """Will return None if bibdata_raw can't be decoded"""
         try:
-            if bibdata_raw.startswith(u'\ufeff'):
+            if bibdata_raw.startswith('\ufeff'):
                 # remove BOM, because bibtexparser does not support it.
                 bibdata_raw = bibdata_raw[1:]
             return self.endecoder.decode_bibdata(bibdata_raw)

pubs/endecoder.py

@@ -1,5 +1,4 @@
-from __future__ import (print_function, absolute_import, division,
-                        unicode_literals)
+from __future__ import absolute_import, unicode_literals
 
 import copy
 

pubs/filebroker.py

@@ -1,6 +1,6 @@
 import os
 import re
-from .p3 import urlparse
+from .p3 import urlparse, u_maybe
 
 from .content import (check_file, check_directory, read_text_file, write_file,
                       system_path, check_content, copy_content)
@@ -18,7 +18,7 @@ def filter_filename(filename, ext):
     """
     pattern = '.*\{}$'.format(ext)
     if re.match(pattern, filename) is not None:
-        return filename[:-len(ext)]
+        return u_maybe(filename[:-len(ext)])
 
 
 class FileBroker(object):

pubs/p3.py

@@ -1,5 +1,9 @@
 import io
 import sys
+import argparse
+
+from six import b
+
 
 if sys.version_info[0] == 2:
     import cPickle as pickle
@@ -21,12 +25,42 @@ if sys.version_info[0] == 2:
     from urllib2 import urlopen
     from httplib import HTTPConnection
     file = None
-    _fake_stdio = io.BytesIO  # Only for tests to capture std{out,err}
+
+    def u_maybe(s):
+        """Convert to unicode, but only if necessary"""
+        if isinstance(s, str):
+            s = s.decode('utf-8')
+        return s
+
+    class StdIO(io.BytesIO):
+        """Enable printing the streams received by a BytesIO instance"""
+        def __init__(self, *args, **kwargs):
+            self.additional_out = kwargs.pop('additional_out')
+            super(StdIO, self).__init__(*args, **kwargs)
+
+        def write(self, s):
+            if self.additional_out is not None:
+                self.additional_out.write(s)
+
+            super(StdIO, self).write(b(s))
+
+    _fake_stdio = StdIO  # Only for tests to capture std{out,err}
 
     def _get_fake_stdio_ucontent(stdio):
-        ustdio = io.TextIOWrapper(stdio)
-        ustdio.seek(0)
-        return ustdio.read()
+
+        # ustdio = io.TextIOWrapper(stdio)
+        stdio.seek(0)
+        return stdio.read()
+
+    # for details, see http://bugs.python.org/issue9779
+    class ArgumentParser(argparse.ArgumentParser):
+        def _print_message(self, message, file=None):
+            """Fixes the lack of a buffer interface in unicode object """
+            if message:
+                if file is None:
+                    file = _sys.stderr
+                file.write(message.encode('utf-8'))
+
 
 else:
     ustr = str
@@ -43,8 +77,24 @@ else:
     def _get_raw_stderr():
         return sys.stderr.buffer
 
-    def _fake_stdio():
-        return io.TextIOWrapper(io.BytesIO())  # Only for tests to capture std{out,err}
+    def u_maybe(s):
+        return s
+
+    class StdIO(io.BytesIO):
+        """Enable printing the streams received by a BytesIO instance"""
+        def __init__(self, *args, **kwargs):
+            self.additional_out = kwargs.pop('additional_out')
+            super(StdIO, self).__init__(*args, **kwargs)
+
+        def write(self, s):
+            if self.additional_out is not None:
+                self.additional_out.write(s)
+
+            super(StdIO, self).write(s)
+
+    # Only for tests to capture std{out,err}
+    def _fake_stdio(additional_out=False):
+        return io.TextIOWrapper(StdIO(additional_out=additional_out))
 
     def _get_fake_stdio_ucontent(stdio):
         stdio.flush()
@@ -53,6 +103,8 @@ else:
 
     import pickle
 
+    ArgumentParser = argparse.ArgumentParser
+
 input = input
 
 

pubs/pretty.py

@@ -1,4 +1,4 @@
-# display formatting
+from __future__ import unicode_literals
 
 import re
 
@@ -41,7 +41,7 @@ def bib_oneliner(bibdata):
     elif bibdata[TYPE_KEY] == 'inproceedings':
         journal = ' ' + bibdata.get('booktitle', '')
 
-    return sanitize(u'{authors} \"{title}\"{journal}{year}'.format(
+    return sanitize('{authors} \"{title}\"{journal}{year}'.format(
         authors=color.dye_out(authors, 'author'),
         title=color.dye_out(bibdata.get('title', ''), 'title'),
         journal=color.dye_out(journal, 'publisher'),
@@ -66,6 +66,6 @@ def paper_oneliner(p, citekey_only=False):
         bibdesc = bib_oneliner(p.bibdata)
         tags = '' if len(p.tags) == 0 else '| {}'.format(
             ','.join(color.dye_out(t, 'tag') for t in sorted(p.tags)))
-        return u'[{citekey}] {descr} {tags}'.format(
+        return '[{citekey}] {descr} {tags}'.format(
             citekey=color.dye_out(p.citekey, 'citekey'),
             descr=bibdesc, tags=tags)

pubs/pubs_cmd.py

@@ -1,9 +1,9 @@
 # PYTHON_ARGCOMPLETE_OK
 
 import sys
-import argparse
 import collections
 from . import uis
+from . import p3
 from . import config
 from . import commands
 from . import update
@@ -36,7 +36,7 @@ CORE_CMDS = collections.OrderedDict([
 def execute(raw_args=sys.argv):
 
     try:
-        conf_parser = argparse.ArgumentParser(prog="pubs", add_help=False)
+        conf_parser = p3.ArgumentParser(prog="pubs", add_help=False)
         conf_parser.add_argument("-c", "--config", help="path to config file",
                                  type=str, metavar="FILE")
         conf_parser.add_argument('--force-colors', dest='force_colors',
@@ -67,8 +67,8 @@ def execute(raw_args=sys.argv):
         uis.init_ui(conf, force_colors=top_args.force_colors)
         ui = uis.get_ui()
 
-        parser = argparse.ArgumentParser(description="research papers repository",
-                                         prog="pubs", add_help=True)
+        parser = p3.ArgumentParser(description="research papers repository",
+                                   prog="pubs", add_help=True)
         parser.add_argument('--version', action='version', version=__version__)
         subparsers = parser.add_subparsers(title="valid commands", dest="command")
 

pubs/uis.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 import os
 import sys
@@ -14,7 +14,8 @@ from .p3 import _get_raw_stdout, _get_raw_stderr, input, ustr
 from .content import check_file, read_text_file, write_file, system_path
 
 
-DEBUG = False
+DEBUG = False  # unhandled exceptions traces are printed
+DEBUG_ALL_TRACES = False # handled exceptions traces are printed
 # package-shared ui that can be accessed using :
 # from uis import get_ui
 # ui = get_ui()
@@ -42,7 +43,7 @@ def _get_local_editor():
     return os.environ.get('EDITOR', 'nano')
 
 
-def _editor_input(editor, initial=u'', suffix='.tmp'):
+def _editor_input(editor, initial='', suffix='.tmp'):
     """Use an editor to get input"""
     str_initial = initial.encode('utf-8')  # TODO: make it a configuration item
     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file:
@@ -100,15 +101,19 @@ class PrintUI(object):
 
     def info(self, message, **kwargs):
         kwargs['file'] = self._stdout
-        print(u'{}: {}'.format(color.dye_out('info', 'ok'), message), **kwargs)
+        print('{}: {}'.format(color.dye_out('info', 'ok'), message), **kwargs)
 
     def warning(self, message, **kwargs):
         kwargs['file'] = self._stderr
-        print(u'{}: {}'.format(color.dye_err('warning', 'warning'), message), **kwargs)
+        print('{}: {}'.format(color.dye_err('warning', 'warning'), message), **kwargs)
 
     def error(self, message, **kwargs):
         kwargs['file'] = self._stderr
-        print(u'{}: {}'.format(color.dye_err('error', 'error'), message), **kwargs)
+        print('{}: {}'.format(color.dye_err('error', 'error'), message), **kwargs)
+
+        if DEBUG_ALL_TRACES: # if an exception has been raised, print the trace.
+            if sys.exc_info()[0] is not None:
+                traceback.print_exception(*sys.exc_info)
 
     def exit(self, error_code=1):
         sys.exit(error_code)
@@ -118,11 +123,12 @@ class PrintUI(object):
 
         :returns: True if exception has been handled (currently never happens)
         """
-        if (not DEBUG) and (not self.debug):
-            self.error(ustr(exc))
+        self.error(ustr(exc))
+        if DEBUG or self.debug:
+            raise
+        else:
             self.exit()
-        return False
-
+        return True # never happens
 
 class InputUI(PrintUI):
     """UI class. Stores configuration parameters and system information.
@@ -136,7 +142,7 @@ class InputUI(PrintUI):
         try:
             data = input()
         except EOFError:
-            self.error(u'Standard input ended while waiting for answer.')
+            self.error('Standard input ended while waiting for answer.')
             self.exit(1)
         return ustr(data) #.decode('utf-8')
 
@@ -159,10 +165,10 @@ class InputUI(PrintUI):
         if len(set(option_chars)) != len(option_chars): # duplicate chars, char choices are deactivated. #FIXME: should only deactivate ambiguous chars
             option_chars = []
 
-        option_str = u'/'.join(["{}{}".format(color.dye_out(c, 'bold'), s[1:])
+        option_str = '/'.join(["{}{}".format(color.dye_out(c, 'bold'), s[1:])
                                for c, s in zip(displayed_chars, options)])
 
-        self.message(u'{}: {} {}: '.format(color.dye_err('prompt', 'warning'), question, option_str), end='')
+        self.message('{}: {} {}: '.format(color.dye_err('prompt', 'warning'), question, option_str), end='')
         while True:
             answer = self.input()
             if answer is None or answer == '':
@@ -176,7 +182,7 @@ class InputUI(PrintUI):
                         return option_chars.index(answer.lower())
                     except ValueError:
                         pass
-            self.message(u'Incorrect option.', option_str)
+            self.message('Incorrect option.', option_str)
 
 
     def input_choice(self, options, option_chars, default=None, question=''):
@@ -209,7 +215,7 @@ class InputUI(PrintUI):
                     return option_chars.index(answer.lower())
                 except ValueError:
                     pass
-            self.message(u'Incorrect option.', option_str)
+            self.message('Incorrect option.', option_str)
 
     def input_yn(self, question='', default='y'):
         d = 0 if default in (True, 'y', 'yes') else 1

pubs/utils.py

@@ -1,4 +1,6 @@
-# Function here may belong somewhere else. In the mean time...
+# Functions here may belong somewhere else. In the mean time...
+from __future__ import unicode_literals
+
 import re
 
 from . import color
@@ -31,7 +33,7 @@ def resolve_citekey(repo, citekey, ui=None, exit_on_fail=True):
                      "citekeys:".format(citekey))
             for c in citekeys:
                 p = repo.pull_paper(c)
-                ui.message(u'    {}'.format(pretty.paper_oneliner(p)))
+                ui.message('    {}'.format(pretty.paper_oneliner(p)))
             if exit_on_fail:
                 ui.exit()
     return citekey

pubs/version.py

@@ -1 +1 @@
-__version__ = '0.8.dev1'
+__version__ = '0.8.dev2'

tests/fake_env.py

@@ -21,16 +21,24 @@ real_glob    = glob
 real_io      = io
 
 
-# redirecting output
+# capture output
 
-def redirect(f):
+def capture(f, verbose=False):
+    """Capture the stdout and stderr output.
+
+    Useful for comparing the output with the expected one during tests.
+
+    :param f:        The function to capture output from.
+    :param verbose:  If True, print call will still display their outputs.
+                     If False, they will be silenced.
+
+    """
     def newf(*args, **kwargs):
         old_stderr, old_stdout = sys.stderr, sys.stdout
-        stdout = _fake_stdio()
-        stderr = _fake_stdio()
-        sys.stdout, sys.stderr = stdout, stderr
+        sys.stdout = _fake_stdio(additional_out=old_stderr if verbose else None)
+        sys.stderr = _fake_stdio(additional_out=old_stderr if False else None)
         try:
-            return f(*args, **kwargs), _get_fake_stdio_ucontent(stdout), _get_fake_stdio_ucontent(stderr)
+            return f(*args, **kwargs), _get_fake_stdio_ucontent(sys.stdout), _get_fake_stdio_ucontent(sys.stderr)
         finally:
             sys.stderr, sys.stdout = old_stderr, old_stdout
     return newf
@@ -39,7 +47,6 @@ def redirect(f):
 # Test helpers
 
 # automating input
-
 real_input = input
 
 
@@ -92,6 +99,7 @@ class TestFakeFs(fake_filesystem_unittest.TestCase):
     def setUp(self):
         self.rootpath = os.path.abspath(os.path.dirname(__file__))
         self.setUpPyfakefs()
+        self.fs.CreateDirectory(os.path.expanduser('~'))
         self.fs.CreateDirectory(self.rootpath)
         os.chdir(self.rootpath)
 

tests/test_bibstruct.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 import os
 import unittest
 import copy
@@ -18,7 +20,7 @@ class TestGenerateCitekey(unittest.TestCase):
     def test_escapes_chars(self):
         doe_bibentry = copy.deepcopy(fixtures.doe_bibentry)
         citekey, bibdata = bibstruct.get_entry(doe_bibentry)
-        bibdata['author'] = [u'Zôu\\@/ , John']
+        bibdata['author'] = ['Zôu\\@/ , John']
         key = bibstruct.generate_citekey(doe_bibentry)
         self.assertEqual(key, 'Zou2013')
 

tests/test_endecoder.py

@@ -1,5 +1,5 @@
-# -*- coding: utf-8 -*-
-from __future__ import print_function
+from __future__ import unicode_literals
+
 import unittest
 
 import yaml
@@ -89,16 +89,16 @@ class TestEnDecode(unittest.TestCase):
 
     def test_endecode_keyword_as_keywords(self):
         decoder = endecoder.EnDecoder()
-        keywords = [u'artificial intelligence', u'Turing test']
+        keywords = ['artificial intelligence', 'Turing test']
         # Add keywords to bibraw
         keyword_str = 'keywords = {artificial intelligence, Turing test},\n'
         biblines = turing_bib.splitlines()
         biblines.insert(-3, keyword_str)
         bibsrc = '\n'.join(biblines)
         entry = decoder.decode_bibdata(bibsrc)['turing1950computing']
-        self.assertNotIn(u'keywords', entry)
-        self.assertIn(u'keyword', entry)
-        self.assertEqual(set(keywords), set(entry[u'keyword']))
+        self.assertNotIn('keywords', entry)
+        self.assertIn('keyword', entry)
+        self.assertEqual(set(keywords), set(entry['keyword']))
 
     def test_endecode_metadata(self):
         decoder = endecoder.EnDecoder()
@@ -110,16 +110,16 @@ class TestEnDecode(unittest.TestCase):
         decoder = endecoder.EnDecoder()
         entry = decoder.decode_bibdata(bibtex_raw0)
         lines = decoder.encode_bibdata(entry).splitlines()
-        self.assertEqual(lines[1].split('=')[0].strip(), u'author')
-        self.assertEqual(lines[2].split('=')[0].strip(), u'title')
-        self.assertEqual(lines[3].split('=')[0].strip(), u'institution')
-        self.assertEqual(lines[4].split('=')[0].strip(), u'publisher')
-        self.assertEqual(lines[5].split('=')[0].strip(), u'year')
-        self.assertEqual(lines[6].split('=')[0].strip(), u'month')
-        self.assertEqual(lines[7].split('=')[0].strip(), u'number')
-        self.assertEqual(lines[8].split('=')[0].strip(), u'url')
-        self.assertEqual(lines[9].split('=')[0].strip(), u'note')
-        self.assertEqual(lines[10].split('=')[0].strip(), u'abstract')
+        self.assertEqual(lines[1].split('=')[0].strip(), 'author')
+        self.assertEqual(lines[2].split('=')[0].strip(), 'title')
+        self.assertEqual(lines[3].split('=')[0].strip(), 'institution')
+        self.assertEqual(lines[4].split('=')[0].strip(), 'publisher')
+        self.assertEqual(lines[5].split('=')[0].strip(), 'year')
+        self.assertEqual(lines[6].split('=')[0].strip(), 'month')
+        self.assertEqual(lines[7].split('=')[0].strip(), 'number')
+        self.assertEqual(lines[8].split('=')[0].strip(), 'url')
+        self.assertEqual(lines[9].split('=')[0].strip(), 'note')
+        self.assertEqual(lines[10].split('=')[0].strip(), 'abstract')
 
     def test_endecode_link_as_url(self):
         decoder = endecoder.EnDecoder()
@@ -129,9 +129,9 @@ class TestEnDecode(unittest.TestCase):
         raw_with_link = bibtex_raw0.replace('url = ', 'link = ')
         entry = decoder.decode_bibdata(raw_with_link)
         lines = decoder.encode_bibdata(entry).splitlines()
-        self.assertEqual(lines[8].split('=')[0].strip(), u'url')
+        self.assertEqual(lines[8].split('=')[0].strip(), 'url')
         self.assertEqual(lines[8].split('=')[1].strip(),
-                         u'{http://ilpubs.stanford.edu:8090/422/},')
+                         '{http://ilpubs.stanford.edu:8090/422/},')
 
     def test_endecode_bibtex_ignores_fields(self):
         decoder = endecoder.EnDecoder()

tests/test_pretty.py

@@ -1,4 +1,5 @@
-# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 import unittest
 import os
 
@@ -19,14 +20,14 @@ class TestPretty(unittest.TestCase):
     def test_oneliner(self):
         decoder = endecoder.EnDecoder()
         bibdata = decoder.decode_bibdata(bibtex_raw0)
-        line = u'Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999)'
+        line = 'Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999)'
         self.assertEqual(color.undye(pretty.bib_oneliner(bibdata['Page99'])), line)
 
     def test_oneliner_no_year(self):
         decoder = endecoder.EnDecoder()
         bibdata = decoder.decode_bibdata(bibtex_raw0)
         bibdata['Page99'].pop('year')
-        line = u'Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web."'
+        line = 'Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web."'
         self.assertEqual(color.undye(pretty.bib_oneliner(bibdata['Page99'])), line)
 
 if __name__ == '__main__':

tests/test_usecase.py

@@ -116,22 +116,21 @@ class CommandTestCase(fake_env.TestFakeFs):
                 input.as_global()
                 try:
                     if capture_output:
-                        _, stdout, stderr = fake_env.redirect(pubs_cmd.execute)(
-                            actual_cmd.split())
+                        capture_wrap = fake_env.capture(pubs_cmd.execute,
+                                                        verbose=PRINT_OUTPUT)
+                        _, stdout, stderr = capture_wrap(actual_cmd.split())
                         actual_out = color.undye(stdout)
                         actual_err = color.undye(stderr)
                         if expected_out is not None:
-                            self.assertEqual(actual_out, expected_out)
+                            self.assertEqual(p3.u_maybe(actual_out), p3.u_maybe(expected_out))
                         if expected_err is not None:
-                            self.assertEqual(actual_err, expected_err)
+                            self.assertEqual(p3.u_maybe(actual_err), p3.u_maybe(expected_err))
                         outs.append(color.undye(actual_out))
                     else:
                         pubs_cmd.execute(actual_cmd.split())
-                except fake_env.FakeInput.UnexpectedInput:
+                except fake_env.FakeInput.UnexpectedInput as e:
                     self.fail('Unexpected input asked by command: {}.'.format(
                         actual_cmd))
-            if PRINT_OUTPUT:
-                print(outs)
             return outs
         except SystemExit as exc:
             exc_class, exc, tb = sys.exc_info()
@@ -200,7 +199,6 @@ class TestAlone(CommandTestCase):
             self.execute_cmds(['pubs'])
             self.assertEqual(cm.exception.code, 2)
 
-
     def test_alone_prints_help(self):
         # capturing the output of `pubs --help` is difficult because argparse
         # raises as SystemExit(0) after calling `print_help`, and this gets
@@ -276,14 +274,18 @@ class TestAdd(URLContentTestCase):
         self.assertEqual(set(os.listdir(bib_dir)), {'CustomCitekey.bib'})
 
     def test_add_utf8_citekey(self):
-        err = ("error: Invalid `hausdorff1949grundzüge` citekey; "
-               "utf-8 citekeys are not supported yet.\n"
-               "See https://github.com/pubs/pubs/issues/28 for details.") # actually not checked
+        correct = ["",
+                   ("added to pubs:\n"
+                    "[hausdorff1949grundzüge] Hausdorff, Felix \"Grundzüge der Mengenlehre\" (1949) \n"),
+                   "The 'hausdorff1949grundzüge' citekey has been renamed into 'アスキー'\n",
+                   "The 'アスキー' citekey has been renamed into 'Ḽơᶉëᶆ_ȋṕšᶙṁ'\n"
+                  ]
         cmds = ['pubs init',
-                ('pubs add bibexamples/utf8.bib', [], '', err),
+                ('pubs add bibexamples/utf8.bib', [], correct[1]),
+                ('pubs rename hausdorff1949grundzüge アスキー', [], correct[2]),
+                ('pubs rename アスキー Ḽơᶉëᶆ_ȋṕšᶙṁ', [], correct[3]),
                ]
-        with self.assertRaises(FakeSystemExit):
-            self.execute_cmds(cmds)
+        self.execute_cmds(cmds)
 
     def test_add_doc_nocopy_does_not_copy(self):
         cmds = ['pubs init',
@@ -350,10 +352,11 @@ class TestList(DataCommandTestCase):
         self.assertEqual(0, len(outs[1].splitlines()))
         self.assertEqual(1, len(outs[3].splitlines()))
 
-    @unittest.expectedFailure #FIXME pyfakefs's shutil.rmtree seems to have problems: submit an issue.
     def test_list_several_no_date(self):
         self.execute_cmds(['pubs init -p testrepo'])
-        shutil.rmtree('testrepo')
+        os.chdir('/') # weird fix for shutil.rmtree invocation.
+        shutil.rmtree(self.rootpath + '/testrepo')
+        os.chdir(self.rootpath)
         self.fs.add_real_directory(os.path.join(self.rootpath, 'testrepo'), read_only=False)
 
         #fake_env.copy_dir(self.fs, testrepo, 'testrepo')