Merge branch 'master' into git-plugin

.gitignore

@@ -1,7 +1,7 @@
 .python-version
 *~
 .DS_Store
-
+.eggs
 
 
 *.py[cod]

.travis.yml

@@ -1,49 +1,105 @@
 # list of environments to test
 matrix:
   include:
+
+    # Full tests (with online API)
     - os: linux
       language: python
       python: 2.7
-    - os: linux
+      env:
-      language: python
+        - TO_TEST=TEST_FULL
-      python: 3.3
-    - os: linux
-      language: python
-      python: 3.4
-    - os: linux
-      language: python
-      python: 3.5
-    - os: linux
-      language: python
-      python: 3.6
     - os: linux
       language: python
       python: 3.7
       dist: xenial
       sudo: true
+      env:
+        - TO_TEST=TEST_FULL
     - os: osx
       language: generic
       python: 2.7
-      before_install:
+      env:
-        - python2 --version
+        - TO_TEST=TEST_FULL
-        - pip2 install -U virtualenv
+      # before_install:
-        - virtualenv env -p python2
+      #   - python2 --version
-        - source env/bin/activate
+      #   - pip2 install -U virtualenv
+      #   - virtualenv env -p python2
+      #   - source env/bin/activate
     - os: osx
       language: generic
       python: ">=3.6"
+      env:
+        - TO_TEST=TEST_FULL
       before_install:
-        - brew update
         - brew outdated python3 || brew install python3 || brew upgrade python3
         - python3 -m venv env
         - source env/bin/activate
 
-# command to install dependencies
+    # Mock tests (with mock API)
-install:
+    - os: linux
-    - python --version
+      language: python
-    - export PUBS_TESTS_MODE=ONLINE
+      python: 3.3
+      env:
+        - TO_TEST=TEST_MOCK
+    - os: linux
+      language: python
+      python: 3.4
+      env:
+        - TO_TEST=TEST_MOCK
+    - os: linux
+      language: python
+      python: 3.5
+      env:
+        - TO_TEST=TEST_MOCK
+    - os: linux
+      language: python
+      python: 3.6
+      env:
+        - TO_TEST=TEST_MOCK
+    - os: linux
+      language: python
+      dist: xenial
+      python: 3.7
+      sudo: true
+      env:
+        - TO_TEST=TEST_MOCK
+
+
+    # Install tests
+    - os: linux
+      language: python
+      python: 2.7
+      env:
+        - TO_TEST=INSTALL
+      if: type = cron
+    - os: linux
+      language: python
+      dist: xenial
+      sudo: true
+      python: 3.7
+      env:
+        - TO_TEST=INSTALL
+      if: type = cron
+    - os: osx
+      language: generic
+      python: 2.7
+      env:
+        - TO_TEST=INSTALL
+      if: type = cron
+    - os: osx
+      language: generic
+      python: ">=3.6"
+      env:
+        - TO_TEST=INSTALL
+      if: type = cron
+
+  allow_failures:
+    - python: 3.3
 
 # command to run tests
 script:
-  - PUBS_TESTS_MODE=MOCK python setup.py test
+  - python --version
-  - PUBS_TESTS_MODE=COLLECT python setup.py test
+  - if [ "$TO_TEST" = "TEST_MOCK" ] ||
+       [ "$TO_TEST" = "TEST_FULL" ]; then PUBS_TESTS_MODE=MOCK    python setup.py test; fi
+  - if [ "$TO_TEST" = "TEST_FULL" ]; then PUBS_TESTS_MODE=COLLECT python setup.py test; fi
+  - if [ "$TO_TEST" = "INSTALL" ]; then pip install -U pip && pip install pubs && pubs --help && pip uninstall -y pubs; fi

MANIFEST.in

@@ -0,0 +1 @@
+include readme.md

changelog.md

@@ -3,16 +3,30 @@
 
 ## Current master
 
-[Full Changelog](https://github.com/pubs/pubs/compare/v0.8.1...master)
+[Full Changelog](https://github.com/pubs/pubs/compare/v0.8.2...master)
+
+### Implemented enhancements
+
+- Add `citekey` filter to `query` ([#193](https://github.com/pubs/pubs/pull/193) by [Shane Stone](https://github.com/shanewstone))
+
+
+### Fixed bugs
+
+
+## [v0.8.2](https://github.com/pubs/pubs/compare/v0.8.1...v0.8.2) (2019-01-06)
+
+Fixes install on python2, and adding old-style arXiv references.
+
+### Fixed bugs
+
+- Fixes adding papers with slashes in their citekeys. [(#179)](https://github.com/pubs/pubs/pull/179) (thanks [Amlesh Sivanantham](https://github.com/zamlz) for reporting.)
+- Fix missing readme.md for python2 pip install. [(#174)](https://github.com/pubs/pubs/pull/174)
 
 ### Implemented enhancements
 
 - [(#45)](https://github.com/pubs/pubs/issues/45) Doc extension visible in pubs list ([#168](https://github.com/pubs/pubs/pull/168))
 
 
-### Fixed bugs
-
-
 ## [v0.8.1](https://github.com/pubs/pubs/compare/v0.8.0...v0.8.1) (2018-08-28)
 
 A hotfix release. All users of 0.8.0 are urged to upgrade.

dev_requirements.txt

@@ -19,6 +19,7 @@ six
 
 # those are the additional packages required to run the tests
 pyfakefs
+certifi
 ddt
 mock
 pytest # optional (python setup.py test works without it), but possible nonetheless

pubs/apis.py

@@ -15,7 +15,7 @@ class ReferenceNotFoundError(Exception):
     pass
 
 
-def get_bibentry_from_api(id_str, id_type, try_doi=True, ui=None):
+def get_bibentry_from_api(id_str, id_type, try_doi=True, ui=None, raw=False):
     """Return a bibtex string from various ID methods.
 
     This is a wrapper around functions that will return a bibtex string given
@@ -50,6 +50,9 @@ def get_bibentry_from_api(id_str, id_type, try_doi=True, ui=None):
         raise ValueError('id_type must be one of `doi`, `isbn`, or `arxiv`.')
 
     bibentry_raw = id_fns[id_type](id_str, try_doi=try_doi, ui=ui)
+    if raw:
+        return bibentry_raw
+
     bibentry = endecoder.EnDecoder().decode_bibdata(bibentry_raw)
     if bibentry is None:
         raise ReferenceNotFoundError(

pubs/bibstruct.py

@@ -51,11 +51,15 @@ def author_last(author_str):
     return author_str.split(',')[0]
 
 
+def valid_citekey(citekey):
+    """Return if a citekey is a valid filename or not"""
+    # FIXME: a bit crude, but efficient for now (and allows unicode citekeys)
+    return not '/' in citekey
+
+
 def generate_citekey(bibdata):
     """ Generate a citekey from bib_data.
 
-        :param generate:  if False, return the citekey defined in the file,
-                          does not generate a new one.
         :raise ValueError:  if no author nor editor is defined.
     """
     citekey, entry = get_entry(bibdata)

pubs/commands/add_cmd.py

@@ -72,6 +72,22 @@ def bibentry_from_editor(conf, ui):
     return bibentry
 
 
+def bibentry_from_api(args, ui, raw=False):
+    try:
+        if args.doi is not None:
+            return apis.get_bibentry_from_api(args.doi, 'doi', ui=ui, raw=raw)
+        elif args.isbn is not None:
+            return apis.get_bibentry_from_api(args.isbn, 'isbn', ui=ui, raw=raw)
+            # TODO distinguish between cases, offer to open the error page in a webbrowser.
+            # TODO offer to confirm/change citekey
+        elif args.arxiv is not None:
+            return apis.get_bibentry_from_api(args.arxiv, 'arxiv', ui=ui, raw=raw)
+    except apis.ReferenceNotFoundError as e:
+        ui.error(str(e))
+        ui.exit(1)
+
+
+
 def command(conf, args):
     """
     :param bibfile: bibtex file (in .bib, .bibml or .yaml format.
@@ -92,19 +108,7 @@ def command(conf, args):
         if args.doi is None and args.isbn is None and args.arxiv is None:
             bibentry = bibentry_from_editor(conf, ui)
         else:
-            bibentry = None
+            bibentry = bibentry_from_api(args, ui)
-            try:
-                if args.doi is not None:
-                    bibentry = apis.get_bibentry_from_api(args.doi, 'doi', ui=ui)
-                elif args.isbn is not None:
-                    bibentry = apis.get_bibentry_from_api(args.isbn, 'isbn', ui=ui)
-                    # TODO distinguish between cases, offer to open the error page in a webbrowser.
-                    # TODO offer to confirm/change citekey
-                elif args.arxiv is not None:
-                    bibentry = apis.get_bibentry_from_api(args.arxiv, 'arxiv', ui=ui)
-            except apis.ReferenceNotFoundError as e:
-                ui.error(str(e))
-                ui.exit(1)
     else:
         bibentry_raw = content.get_content(bibfile, ui=ui)
         bibentry = decoder.decode_bibdata(bibentry_raw)
@@ -116,7 +120,7 @@ def command(conf, args):
     citekey = args.citekey
     if citekey is None:
         base_key = bibstruct.extract_citekey(bibentry)
-        citekey = rp.unique_citekey(base_key)
+        citekey = rp.unique_citekey(base_key, bibentry)
     elif citekey in rp:
         ui.error('citekey already exist {}.'.format(citekey))
         ui.exit(1)

pubs/commands/conf_cmd.py

@@ -17,9 +17,9 @@ def command(conf, args):
 
     while True:
         # get modif from user
-        ui.edit_file(config.get_confpath(), temporary=False)
+        ui.edit_file(conf.filename, temporary=False)
 
-        new_conf = config.load_conf()
+        new_conf = config.load_conf(path=conf.filename)
         try:
             config.check_conf(new_conf)
             ui.message('The configuration file was updated.')

pubs/commands/import_cmd.py

@@ -14,26 +14,28 @@ from ..content import system_path, read_text_file
 from ..command_utils import add_doc_copy_arguments
 
 
-_ABORT_USE_IGNORE_MSG = "Aborting import. Use --ignore-malformed to ignore."
+_ABORT_USE_IGNORE_MSG = " Aborting import. Use --ignore-malformed to ignore."
 _IGNORING_MSG = " Ignoring it."
 
 
 def parser(subparsers, conf):
     parser = subparsers.add_parser(
         'import',
-        help='import paper(s) to the repository')
+        help='import paper(s) to the repository.')
     parser.add_argument(
         'bibpath',
-        help='path to bibtex, bibtexml or bibyaml file (or directory)')
+        help=("path to bibtex, bibtexml or bibyaml file, or a directory "
+              "containing such files; will not recurse into subdirectories."))
     parser.add_argument(
         'keys', nargs='*',
-        help="one or several keys to import from the file")
+        help=("one or several keys to import from the file; if not provided,"
+              " all entries will be imported."))
     parser.add_argument(
         '-O', '--overwrite', action='store_true', default=False,
-        help="Overwrite keys already in the database")
+        help="overwrite keys already in the database.")
     parser.add_argument(
         '-i', '--ignore-malformed', action='store_true', default=False,
-        help="Ignore malformed and unreadable files and entries")
+        help="ignore malformed and unreadable files and entries.")
     add_doc_copy_arguments(parser, copy=False)
     return parser
 
@@ -52,7 +54,6 @@ def many_from_path(ui, bibpath, ignore=False):
 
     bibpath = system_path(bibpath)
     if os.path.isdir(bibpath):
-        print([os.path.splitext(f)[-1][1:] for f in os.listdir(bibpath)])
         all_files = [os.path.join(bibpath, f) for f in os.listdir(bibpath)
                      if os.path.splitext(f)[-1][1:] == 'bib']
     else:

pubs/endecoder.py

@@ -113,7 +113,7 @@ class EnDecoder(object):
                 author for author in entry['author'])
         if 'editor' in entry:
             entry['editor'] = ' and '.join(
-                editor['name'] for editor in entry['editor'])
+                editor for editor in entry['editor'])
         if 'keyword' in entry:
             entry['keyword'] = ', '.join(
                 keyword for keyword in entry['keyword'])
@@ -122,7 +122,7 @@ class EnDecoder(object):
     def decode_bibdata(self, bibdata):
         """Decodes bibdata from string.
 
-        If the decoding fails, returns a BibParseError.
+        If the decoding fails, returns a BibDecodingError.
         """
         if len(bibdata) == 0:
             error_msg = 'parsing error: the provided string has length zero.'
@@ -131,15 +131,23 @@ class EnDecoder(object):
             entries = bp.bparser.BibTexParser(
                 bibdata, common_strings=True, customization=customizations,
                 homogenize_fields=True).get_entry_dict()
-
             # Remove id from bibtexparser attribute which is stored as citekey
             for e in entries:
                 entries[e].pop(BP_ID_KEY)
                 # Convert bibtexparser entrytype key to internal 'type'
                 t = entries[e].pop(BP_ENTRYTYPE_KEY)
                 entries[e][TYPE_KEY] = t
+                # Temporary fix to #188 (to be fully fixed when the upstream
+                # issue: sciunto-org/python-bibtexparser/#229 is fixed too)
+                if 'editor' in entries[e]:
+                    entries[e]['editor'] = [
+                        editor['name'] if isinstance(editor, dict) else editor
+                        for editor in entries[e]['editor']]
             if len(entries) > 0:
                 return entries
+            else:
+                raise self.BibDecodingError(('no valid entry found in the provided data: '
+                                            ' {}').format(bibdata), bibdata)
         except (pyparsing.ParseException, pyparsing.ParseSyntaxException) as e:
             error_msg = self._format_parsing_error(e)
             raise self.BibDecodingError(error_msg, bibdata)
@@ -147,7 +155,6 @@ class EnDecoder(object):
             error_msg = 'parsing error: undefined string in provided data: {}'.format(e)
             raise self.BibDecodingError(error_msg, bibdata)
 
-
     @classmethod
     def _format_parsing_error(cls, e):
         """Transform a pyparsing exception into an error message

pubs/p3.py

@@ -87,10 +87,14 @@ else:
             super(StdIO, self).__init__(*args, **kwargs)
 
         def write(self, s):
+            super(StdIO, self).write(s)
             if self.additional_out is not None:
+                try:
+                    s = s.decode()
+                except AttributeError:
+                    pass
                 self.additional_out.write(s)
 
-            super(StdIO, self).write(s)
 
     # Only for tests to capture std{out,err}
     def _fake_stdio(additional_out=False):

pubs/paper.py

@@ -1,6 +1,8 @@
 import copy
 from dateutil.parser import parse as datetime_parse
 
+from bibtexparser.customization import convert_to_unicode
+
 from . import bibstruct
 from .p3 import ustr
 
@@ -102,6 +104,10 @@ class Paper(object):
     def added(self, value):
         self.metadata['added'] = value
 
+    def get_unicode_bibdata(self):
+        """Converts latex in bibdata fields to unicode."""
+        return convert_to_unicode(self.bibdata)
+
     @staticmethod
     def from_bibentry(bibentry, citekey=None, metadata=None):
         bibentry_key, bibdata = bibstruct.get_entry(bibentry)

pubs/pretty.py

@@ -64,7 +64,7 @@ def paper_oneliner(p, citekey_only=False):
     if citekey_only:
         return p.citekey
     else:
-        bibdesc = bib_oneliner(p.bibdata)
+        bibdesc = bib_oneliner(p.get_unicode_bibdata())
         doc_str = ''
         if p.docpath is not None:
             doc_extension = os.path.splitext(p.docpath)[1]

pubs/pubs_cmd.py

@@ -60,7 +60,8 @@ def execute(raw_args=sys.argv):
                 # an update happened, reload conf.
                 conf = config.load_conf(path=conf_path)
         except config.ConfigurationNotFound:
-            if len(remaining_args) == 0 or remaining_args[0] == 'init':
+            if (len(remaining_args) == 0 or remaining_args[0] == 'init'
+                or all(arg[0] == '-' for arg in remaining_args)):  # only optional arguments
                 conf = config.load_default_conf()
                 conf.filename = conf_path
             else:

pubs/query.py

@@ -6,7 +6,8 @@ from . import bibstruct
 
 
 QUERY_HELP = ('Paper query ("author:Einstein", "title:learning",'
-              '"year:2000", "year:2000-2010", or "tags:math")')
+              ' "year:2000", "year:2000-2010", "citekey:Einstein_1935",'
+              ' or "tags:math")')
 
 
 FIELD_ALIASES = {
@@ -15,6 +16,7 @@ FIELD_ALIASES = {
     't': 'title',
     'tags': 'tag',
     'y': 'year',
+    'key': 'citekey',
 }
 
 
@@ -78,6 +80,12 @@ class AuthorFilter(QueryFilter):
                         for author in paper.bibdata['author']])
 
 
+class CitekeyFilter(QueryFilter):
+
+    def __call__(self, paper):
+        return self._is_query_in(paper.citekey)
+
+
 class TagFilter(QueryFilter):
 
     def __call__(self, paper):
@@ -137,6 +145,9 @@ def _query_block_to_filter(query_block, case_sensitive=None, strict=False):
     field, value = _get_field_value(query_block)
     if field == 'tag':
         return TagFilter(value, case_sensitive=case_sensitive, strict=strict)
+    elif field == 'citekey':
+        return CitekeyFilter(value, case_sensitive=case_sensitive,
+                             strict=strict)
     elif field == 'author':
         return AuthorFilter(value, case_sensitive=case_sensitive,
                             strict=strict)

pubs/version.py

@@ -1 +1 @@
-__version__ = '0.8.1'
+__version__ = '0.8.2-r1'

readme.md

@@ -16,20 +16,23 @@ Pubs is built with the following principles in mind:
 ## Installation
 
 You can install the latest stable version of `pubs` through Pypi, with:
-
+  ```
-    pip install pubs
+  pip install pubs
+  ```
 
 Alternatively, you can:
 
   - install the latest development version with pip:
-
+    ```
     pip install --upgrade git+https://github.com/pubs/pubs
+    ```
 
   - clone the repository and install it manually:
-
+    ```
     git clone https://github.com/pubs/pubs
     cd pubs
     python setup.py install [--user]
+    ```
 
 Arch Linux users can also use the [pubs-git](https://aur.archlinux.org/packages/pubs-git/) AUR package.
 
@@ -37,28 +40,34 @@ Arch Linux users can also use the [pubs-git](https://aur.archlinux.org/packages/
 ## Getting started
 
 Create your library (by default, goes to `~/.pubs/`).
-
+  ```
-    pubs init
+  pubs init
+  ```
 
 Import existing data from bibtex (pubs will try to automatically copy documents defined as 'file' in bibtex):
-
+  ```
-    pubs import path/to/collection.bib
+  pubs import path/to/collection.bib
+  ```
 
 or for a .bib file containing a single reference:
-
+  ```
-    pubs add reference.bib -d article.pdf
+  pubs add reference.bib -d article.pdf
+  ```
 
 pubs can also automatically retrieve the bibtex from a doi:
-
+  ```
-    pubs add -D 10.1007/s00422-012-0514-6 -d article.pdf
+  pubs add -D 10.1007/s00422-012-0514-6 -d article.pdf
+  ```
 
 or an ISBN (dashes are ignored):
-
+  ```
-    pubs add -I 978-0822324669 -d article.pdf
+  pubs add -I 978-0822324669 -d article.pdf
+  ```
 
 or an arXiv id (automatically downloading arXiv article is in the works):
-
+  ```
-    pubs add -X math/9501234 -d article.pdf
+  pubs add -X math/9501234 -d article.pdf
+  ```
 
 
 ## References always up-to-date
@@ -66,8 +75,9 @@ or an arXiv id (automatically downloading arXiv article is in the works):
 If you use latex, you can automatize references, by running `pubs export > references.bib` each time you update your library, which also fits well as a `makefile` rule.
 
 This ensures that your reference file is always up-to-date; you can cite a paper in your manuscript a soon as you add it in pubs. This means that if you have, for instance, a doi on a webpage, you only need to do:
-
+  ```
-    pubs add -D 10.1007/s00422-012-0514-6
+  pubs add -D 10.1007/s00422-012-0514-6
+  ```
 
 and then add `\cite{Loeb_2012}` in your manuscript. After exporting the bibliography, the citation will correctly appear in your compiled pdf.
 
@@ -75,23 +85,25 @@ and then add `\cite{Loeb_2012}` in your manuscript. After exporting the bibliogr
 ## Document management
 
 You can attach a document to a reference:
-
+  ```
-    pubs add Loeb2012_downloaded.pdf Loeb_2012
+  pubs doc add Loeb2012_downloaded.pdf Loeb_2012
+  ```
 
 And open your documents automatically from the command line:
-
+  ```
-    pubs doc open Loeb_2012
+  pubs doc open Loeb_2012
-    pubs doc open --with lp Loeb_2012  # Opens the document with `lp` to actually print it.
+  pubs doc open --with lp Loeb_2012  # Opens the document with `lp` to actually print it.
-
+  ```
 
 ## Customization
 
 Pubs is designed to interact well with your command line tool chain.
 You can add custom commands to pubs by defining aliases in your configuration file (make sure that the alias plugin is activated in your configuration by using `pubs conf`).
-
+  ```ini
-    [[alias]]
+  [[alias]]
-    evince = open --with evince
+  evince = open --with evince
-    count = !pubs list -k "$@" | wc -l
+  count = !pubs list -k "$@" | wc -l
+  ```
 
 The first command defines a new subcommand: `pubs open --with evince` will be executed when `pubs evince` is typed.
 The second starts with a bang: `!`, and is treated as a shell command. If other arguments are provided they are passed to the shell command as in a script. In the example above the `count` alias can take arguments that are be passed to the `pubs list -k` command, hence enabling filters like `pubs count year:2012`.
@@ -102,14 +114,14 @@ The second starts with a bang: `!`, and is treated as a shell command. If other
 For autocompletion to work, you need the [argcomplete](https://argcomplete.readthedocs.io) Python package, and Bash 4.2 or newer. For activating *bash* or *tsch* completion, consult the [argcomplete documentation](https://argcomplete.readthedocs.io/en/latest/#global-completion).
 
 For *zsh* completion, the global activation is not supported but bash completion compatibility can be used for pubs. For that, add the following to your `.zshrc`:
-
+  ```shell
-    # Enable and load bashcompinit
+  # Enable and load bashcompinit
-    autoload -Uz compinit bashcompinit
+  autoload -Uz compinit bashcompinit
-    compinit
+  compinit
-    bashcompinit
+  bashcompinit
-    # Argcomplete explicit registration for pubs
+  # Argcomplete explicit registration for pubs
-    eval "$(register-python-argcomplete pubs)"
+  eval "$(register-python-argcomplete pubs)"
-
+  ```
 
 ## Need more help ?
 
@@ -130,4 +142,5 @@ You can access the self-documented configuration by using `pubs conf`, and all t
 - [Dennis Wilson](https://github.com/d9w)
 - [Bill Flynn](https://github.com/wflynny)
 - [ksunden](https://github.com/ksunden)
+- [Shane Stone](https://github.com/shanewstone)
 - [Amlesh Sivanantham](http://github.com/zamlz)

setup.py

@@ -4,6 +4,7 @@ import unittest
 
 from setuptools import setup
 
+
 with open('pubs/version.py') as f:
     exec(f.read())  # defines __version__
 
@@ -11,11 +12,13 @@ here = os.path.abspath(os.path.dirname(__file__))
 with open(os.path.join(here, 'readme.md'), 'r') as fd:
     long_description = fd.read()
 
+
 def pubs_test_suite():
     test_loader = unittest.TestLoader()
     test_suite = test_loader.discover('tests', pattern='test_*.py')
     return test_suite
 
+
 setup(
     name='pubs',
     version=__version__,
@@ -41,6 +44,8 @@ setup(
             ],
         },
 
+    include_package_data=True,
+
     install_requires=['pyyaml', 'bibtexparser>=1.0', 'python-dateutil', 'six',
                       'requests', 'configobj', 'beautifulsoup4', 'feedparser'],
     extras_require={'autocompletion': ['argcomplete'],
@@ -55,8 +60,8 @@ setup(
         'Intended Audience :: Science/Research',
     ],
 
-    test_suite= 'tests',
+    test_suite='tests',
-    tests_require=['pyfakefs>=3.4', 'mock', 'ddt'],
+    tests_require=['pyfakefs>=3.4', 'mock', 'ddt', 'certifi'],
 
     # in order to avoid 'zipimport.ZipImportError: bad local file header'
     zip_safe=False,

tests/data/collection.bib

@@ -0,0 +1,41 @@
+@article{Einstein_1935,
+	doi = {10.1103/physrev.47.777},
+	url = {https://doi.org/10.1103%2Fphysrev.47.777},
+	year = 1935,
+	month = {may},
+	publisher = {American Physical Society ({APS})},
+	volume = {47},
+	number = {10},
+	pages = {777--780},
+	author = {A. Einstein and B. Podolsky and N. Rosen},
+	title = {Can Quantum-Mechanical Description of Physical Reality Be Considered Complete?},
+	journal = {Physical Review}
+}
+
+@article{Schrodinger_1935,
+	doi = {10.1017/s0305004100013554},
+	url = {https://doi.org/10.1017%2Fs0305004100013554},
+	year = 1935,
+	month = {oct},
+	publisher = {Cambridge University Press ({CUP})},
+	volume = {31},
+	number = {04},
+	pages = {555},
+	author = {E. Schrödinger and M. Born},
+	title = {Discussion of Probability Relations between Separated Systems},
+	journal = {Mathematical Proceedings of the Cambridge Philosophical Society}
+}
+
+@article{Bell_1964,
+	doi = {10.1103/physicsphysiquefizika.1.195},
+	url = {https://doi.org/10.1103%2Fphysicsphysiquefizika.1.195},
+	year = 1964,
+	month = {nov},
+	publisher = {American Physical Society ({APS})},
+	volume = {1},
+	number = {3},
+	pages = {195--200},
+	author = {J. S. Bell},
+	title = {On the Einstein Podolsky Rosen paradox},
+	journal = {Physics Physique {\cyrchar\cyrf}{\cyrchar\cyri}{\cyrchar\cyrz}{\cyrchar\cyri}{\cyrchar\cyrk}{\cyrchar\cyra}}
+}

tests/data/many_fields.bib

@@ -0,0 +1,16 @@
+@ARTICLE{Cesar2013,
+  authors = {Jean César},
+  title = {An amazing title},
+  year = {2013},
+  month = "jan",
+  volume = {12},
+  pages = {12-23},
+  journal = {Nice Journal},
+  abstract = {This is an abstract. This line should be long enough to test
+multilines... and with a french érudit word},
+  comments = {A comment},
+  editors = {Edith Or and Anne Other},
+  keywords = {keyword1, keyword2},
+  links = {http://my.link/to-content},
+  subjects = "Some topic of interest",
+}

tests/readme.md

@@ -1,7 +1,11 @@
 1. Install the dependencies using:
-> pip install -r requirements.txt
+```
+pip install -r ../dev_requirements.txt
+```
 
 2. Run the tests using:
-> python -m unittest discover
+```
+python setup.py test
+```
 
 If you use nosetest, it will complain about addExpectedFailure, which you can safely disregard.

tests/str_fixtures.py

@@ -84,6 +84,18 @@ not_bibtex = """@misc{this looks,
 
 """
 
+bibtex_with_latex = """@article{kjaer2018large,
+  title={A large impact crater beneath Hiawatha Glacier in northwest Greenland},
+  author={Kj{\\ae}r, Kurt H and Larsen, Nicolaj K and Binder, Tobias and Bj{\\o}rk, Anders A and Eisen, Olaf and Fahnestock, Mark A and Funder, Svend and Garde, Adam A and Haack, Henning and Helm, Veit and others},
+  journal={Science advances},
+  volume={4},
+  number={11},
+  pages={eaar8173},
+  year={2018},
+  publisher={American Association for the Advancement of Science}
+}
+"""
+
 sample_conf = """
 [main]
 

tests/test_apis.py

@@ -16,8 +16,12 @@ import mock_requests
 
 
 class APITests(unittest.TestCase):
-    pass
 
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_readme(self, reqget):
+        apis.doi2bibtex('10.1007/s00422-012-0514-6')
+        apis.isbn2bibtex('978-0822324669')
+        apis.arxiv2bibtex('math/9501234')
 
 class TestDOI2Bibtex(APITests):
 
@@ -45,11 +49,12 @@ class TestDOI2Bibtex(APITests):
 
 class TestISBN2Bibtex(APITests):
 
-    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    # try to avoid triggering 403 status during tests.
-    def test_unicode(self, reqget):
+    # @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
-        bib = apis.isbn2bibtex('9782081336742')
+    # def test_unicode(self, reqget):
-        self.assertIsInstance(bib, ustr)
+    #     bib = apis.isbn2bibtex('9782081336742')
-        self.assertIn('Poincaré, Henri', bib)
+    #     self.assertIsInstance(bib, ustr)
+    #     self.assertIn('Poincaré, Henri', bib)
 
     @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
     def test_parses_to_bibtex(self, reqget):

tests/test_endecoder.py

@@ -26,7 +26,7 @@ class TestEnDecode(unittest.TestCase):
     def test_decode_emptystring(self):
         decoder = endecoder.EnDecoder()
         with self.assertRaises(decoder.BibDecodingError):
-            entry = decoder.decode_bibdata('')
+            decoder.decode_bibdata('')
 
     def test_encode_bibtex_is_unicode(self):
         decoder = endecoder.EnDecoder()
@@ -117,11 +117,21 @@ class TestEnDecode(unittest.TestCase):
         self.assertIn('keyword', entry)
         self.assertEqual(set(keywords), set(entry['keyword']))
 
+    def test_decode_metadata(self):
+        decoder = endecoder.EnDecoder()
+        entry = decoder.decode_metadata(metadata_raw0)
+        expected = {'docfile': 'docsdir://Page99.pdf',
+                    'tags': ['search', 'network'],
+                    'added': '2013-11-14 13:14:20',
+                    }
+        self.assertEqual(entry, expected)
+
     def test_endecode_metadata(self):
         decoder = endecoder.EnDecoder()
         entry = decoder.decode_metadata(metadata_raw0)
         metadata_output0 = decoder.encode_metadata(entry)
-        self.assertEqual(set(metadata_raw0.split('\n')), set(metadata_output0.split('\n')))
+        entry_from_encode = decoder.decode_metadata(metadata_output0)
+        self.assertEqual(entry, entry_from_encode)
 
     def test_endecode_bibtex_field_order(self):
         decoder = endecoder.EnDecoder()

tests/test_paper.py

@@ -4,7 +4,9 @@ import unittest
 
 import dotdot
 import fixtures
+import str_fixtures
 from pubs.paper import Paper
+from pubs.endecoder import EnDecoder
 
 
 class TestAttributes(unittest.TestCase):
@@ -47,5 +49,20 @@ class TestAttributes(unittest.TestCase):
             Paper(" ", fixtures.doe_bibdata)
 
 
+class TestPaperUnicodeBibdata(unittest.TestCase):
+
+    def test_no_latex(self):
+        p = Paper.from_bibentry(fixtures.page_bibentry,
+                                metadata=fixtures.page_metadata).deepcopy()
+        self.assertEqual(p.bibdata, p.get_unicode_bibdata())
+
+    def test_latex_converted(self):
+        bib = EnDecoder().decode_bibdata(str_fixtures.bibtex_with_latex)
+        p = Paper.from_bibentry(bib)
+        ubib = p.get_unicode_bibdata()
+        self.assertEqual(ubib['author'][0], u"Kjær, Kurt H")
+        self.assertEqual(ubib['author'][3], u"Bjørk, Anders A")
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_queries.py

@@ -4,9 +4,9 @@ from __future__ import unicode_literals
 import unittest
 
 import dotdot
-from pubs.query import (AuthorFilter, FieldFilter, YearFilter,
+from pubs.query import (AuthorFilter, CitekeyFilter, FieldFilter,
-                        _query_block_to_filter, get_paper_filter,
+                        YearFilter, _query_block_to_filter,
-                        InvalidQuery)
+                        get_paper_filter, InvalidQuery)
 
 from pubs.paper import Paper
 
@@ -43,6 +43,49 @@ class TestAuthorFilter(unittest.TestCase):
         self.assertFalse(AuthorFilter('lawrence')(page_paper))
 
 
+class TestCitekeyFilter(unittest.TestCase):
+
+    def test_fails_if_no_citekey(self):
+        no_citekey = doe_paper.deepcopy()
+        no_citekey.citekey = ''
+        self.assertFalse(CitekeyFilter('whatever')(no_citekey))
+
+    def test_match_case(self):
+        self.assertTrue(CitekeyFilter('doe201')(doe_paper))
+        self.assertTrue(CitekeyFilter('doe201', case_sensitive=False)(doe_paper))
+        self.assertTrue(CitekeyFilter('Doe201')(doe_paper))
+
+    def test_do_not_match_case(self):
+        self.assertFalse(CitekeyFilter('dOe201')(doe_paper))
+        self.assertFalse(CitekeyFilter('dOe201', case_sensitive=True)(doe_paper))
+        self.assertFalse(CitekeyFilter('doe201', case_sensitive=True)(doe_paper))
+        self.assertTrue(CitekeyFilter('dOe201', case_sensitive=False)(doe_paper))
+
+    def test_latex_enc(self):
+        latexenc_paper = doe_paper.deepcopy()
+        latexenc_paper.citekey = "{G}r{\\\"u}n2013"
+        self.assertTrue(CitekeyFilter('Grün')(latexenc_paper))
+        self.assertTrue(CitekeyFilter('Gr{\\\"u}n')(latexenc_paper))
+
+    def test_normalize_unicode(self):
+        latexenc_paper = doe_paper.deepcopy()
+        latexenc_paper.citekey = "Jalape\u00f1o2013"
+        self.assertTrue(CitekeyFilter("Jalapen\u0303o")(latexenc_paper))
+
+    def test_strict(self):
+        latexenc_paper = doe_paper.deepcopy()
+        latexenc_paper.citekey = "Jalape\u00f1o2013"
+        self.assertFalse(CitekeyFilter("Jalapen\u0303o", strict=True)(latexenc_paper))
+        latexenc_paper.citekey = "{G}ros2013"
+        self.assertFalse(CitekeyFilter("Gros", strict=True)(latexenc_paper))
+
+    def test_strict_implies_case(self):
+        latexenc_paper = doe_paper.deepcopy()
+        latexenc_paper.citekey = "Gros2013"
+        self.assertFalse(
+            CitekeyFilter("gros", case_sensitive=False, strict=True)(latexenc_paper))
+
+
 class TestCheckTag(unittest.TestCase):
     pass
 

tests/test_repo.py

@@ -29,10 +29,12 @@ class TestCitekeyGeneration(TestRepo):
 
     def test_generated_key_is_unique(self):
         self.repo.push_paper(Paper.from_bibentry(fixtures.doe_bibentry))
-        c = self.repo.unique_citekey('Doe2013')
+        c = self.repo.unique_citekey('Doe2013', fixtures.doe_bibentry)
         self.repo.push_paper(Paper.from_bibentry(fixtures.doe_bibentry,
                                                  citekey='Doe2013a'))
-        c = self.repo.unique_citekey('Doe2013')
+        c = self.repo.unique_citekey('Doe2013', fixtures.doe_bibentry)
+        self.assertEqual(c, 'Doe2013b')
+        c = self.repo.unique_citekey('bla/bla', fixtures.doe_bibentry)
         self.assertEqual(c, 'Doe2013b')
 
 

tests/test_usecase.py

@@ -6,14 +6,16 @@ import os
 import re
 import sys
 import shutil
-import mock
 
 import six
 import ddt
+import certifi
+import mock
 from pyfakefs.fake_filesystem import FakeFileOpen
 
 import dotdot
 import fake_env
+import mock_requests
 
 from pubs import pubs_cmd, color, content, uis, p3, endecoder
 from pubs.config import conf
@@ -95,6 +97,14 @@ class CommandTestCase(fake_env.TestFakeFs):
         3. the expected output on stdout, verified with assertEqual.
         4. the expected output on stderr, verified with assertEqual.
         """
+        def normalize(s):
+            s = color.undye(s)
+            try:
+                s = s.decode('utf-8')
+            except AttributeError:
+                pass
+            return s
+
         try:
             outs = []
             for cmd in cmds:
@@ -119,8 +129,8 @@ class CommandTestCase(fake_env.TestFakeFs):
                         capture_wrap = fake_env.capture(pubs_cmd.execute,
                                                         verbose=PRINT_OUTPUT)
                         _, stdout, stderr = capture_wrap(actual_cmd.split())
-                        actual_out = color.undye(stdout)
+                        actual_out = normalize(stdout)
-                        actual_err = color.undye(stderr)
+                        actual_err = normalize(stderr)
                         if expected_out is not None:
                             self.assertEqual(p3.u_maybe(actual_out), p3.u_maybe(expected_out))
                         if expected_err is not None:
@@ -153,9 +163,10 @@ class DataCommandTestCase(CommandTestCase):
         super(DataCommandTestCase, self).setUp(nsec_stat=nsec_stat)
         self.fs.add_real_directory(os.path.join(self.rootpath, 'data'), read_only=False)
         self.fs.add_real_directory(os.path.join(self.rootpath, 'bibexamples'), read_only=False)
+        # add certificate for web querries
+        self.fs.add_real_file(certifi.where(), read_only=True)
+        self.fs.add_real_file(mock_requests._data_filepath, read_only=False)
 
-        # fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'data'), 'data')
-        # fake_env.copy_dir(self.fs, os.path.join(os.path.dirname(__file__), 'bibexamples'), 'bibexamples')
 
     def assertFileContentEqual(self, path, expected_content):
         self.assertTrue(os.path.isfile(path))
@@ -276,7 +287,7 @@ class TestAdd(URLContentTestCase):
     def test_add_utf8_citekey(self):
         correct = ["",
                    ("added to pubs:\n"
-                    "[hausdorff1949grundzüge] Hausdorff, Felix \"Grundzüge der Mengenlehre\" (1949) \n"),
+                    "[hausdorff1949grundzüge] Hausdorff, Felix \"Grundzüge der Mengenlehre\" (1949) \n"),
                    "The 'hausdorff1949grundzüge' citekey has been renamed into 'アスキー'\n",
                    "The 'アスキー' citekey has been renamed into 'Ḽơᶉëᶆ_ȋṕšᶙṁ'\n"
                   ]
@@ -493,6 +504,18 @@ class TestList(DataCommandTestCase):
         outs = self.execute_cmds(cmds)
         self.assertEqual(1 + 1, len(outs[-1].split('\n')))
 
+    def test_list_with_citekey_query(self):
+        cmds = ['pubs init',
+                'pubs import data/',
+                'pubs list citekey:Page99',
+                'pubs list key:eiNstein_1935',
+                'pubs list --ignore-case key:eiNstein_1935',
+                ]
+        outs = self.execute_cmds(cmds)
+        self.assertEqual(1, len(outs[2].splitlines()))
+        self.assertEqual(0, len(outs[3].splitlines()))
+        self.assertEqual(1, len(outs[4].splitlines()))
+
 
 class TestTag(DataCommandTestCase):
 
@@ -849,7 +872,7 @@ class TestUsecase(DataCommandTestCase):
                ]
 
         outs = self.execute_cmds(cmds)
-        self.assertEqual(4 + 1, len(outs[-1].split('\n')))
+        self.assertEqual(9, len(outs[-1].split('\n')))
 
     def test_import_one(self):
         cmds = ['pubs init',
@@ -973,7 +996,7 @@ class TestUsecase(DataCommandTestCase):
         alt_conf = os.path.expanduser('~/.alt_conf')
         cmds = ['pubs -c ' + alt_conf + ' init',
                 'pubs --config ' + alt_conf + ' import data/ Page99',
-                'pubs list -c ' + alt_conf
+                'pubs list -c ' + alt_conf,
                ]
         outs = self.execute_cmds(cmds)
         # check if pubs works as expected
@@ -982,6 +1005,11 @@ class TestUsecase(DataCommandTestCase):
         self.assertFalse(os.path.isfile(self.default_conf_path))
         self.assertTrue(os.path.isfile(alt_conf))
 
+        with open(alt_conf, 'r') as fd:
+            conf_text = fd.read()
+        outs = self.execute_cmds([('pubs conf -c ' + alt_conf, conf_text)])
+
+
     def test_statistics(self):
         cmds = ['pubs init',
                 'pubs statistics',
@@ -1002,10 +1030,10 @@ class TestUsecase(DataCommandTestCase):
         self.assertEqual(lines[2], 'Total tags: 3, 2 (50%) of papers have at least one tag')
 
     def test_add_no_extension(self):
-        # This tests checks that a paper which document has no
+        """This tests checks that a paper which document has no extension does
-        # extension does not raise issues when listing. This test might
+        not raise issues when listing. This test might be removed if decided to
-        # be removed if decided to prevent such documents. It would then need
+        prevent such documents. It would then need to be replaced by a check
-        # to be replaced by a check that this is prevented.
+        that this is prevented."""
         self.fs.add_real_file(os.path.join(self.rootpath, 'data', 'pagerank.pdf'),
                               target_path=os.path.join('data', 'no-ext'))
         correct = ['Initializing pubs in /pubs\n',
@@ -1019,6 +1047,26 @@ class TestUsecase(DataCommandTestCase):
                ]
         self.assertEqual(correct, self.execute_cmds(cmds, capture_output=True))
 
+    @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get)
+    def test_readme(self, reqget):
+        """Test that the readme example work."""
+        self.fs.add_real_file(os.path.join(self.rootpath, 'data/pagerank.pdf'), target_path='data/Loeb_2012.pdf')
+        self.fs.add_real_file(os.path.join(self.rootpath, 'data/pagerank.pdf'), target_path='data/oyama2000the.pdf')
+        self.fs.add_real_file(os.path.join(self.rootpath, 'data/pagerank.pdf'), target_path='data/Knuth1995.pdf')
+
+        cmds = ['pubs init',
+                'pubs import data/collection.bib',
+                'pubs add data/pagerank.bib -d data/pagerank.pdf',
+                #'pubs add -D 10.1007/s00422-012-0514-6 -d data/pagerank.pdf',
+                'pubs add -I 978-0822324669 -d data/oyama2000the.pdf',
+                'pubs add -X math/9501234 -d data/Knuth1995.pdf',
+                'pubs add -D 10.1007/s00422-012-0514-6',
+                'pubs doc add data/Loeb_2012.pdf Loeb_2012',
+               ]
+        self.execute_cmds(cmds, capture_output=True)
+#        self.assertEqual(correct, self.execute_cmds(cmds, capture_output=True))
+
+
 
 @ddt.ddt
 class TestCache(DataCommandTestCase):