diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..af0312e --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,60 @@ +name: Pubs tests + +on: + push: + pull_request: + schedule: + - cron: '0 8 * * *' + + +jobs: + unit-test: + name: Run unit tests + strategy: + matrix: + os: [macos-latest, ubuntu-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r dev_requirements.txt + - name: Configure git author (fix issue with environment variable) + run: | + # Manually sets some git user and email to avoid failure of the test + # (For some reason the environment variables set in the test are not + # taken into account by git on the runner.) + git config --global user.name "Pubs test" + git config --global user.email "unittest@pubs.org" + - name: Test with pytest (mock API mode) + env: + PUBS_TESTS_MODE: MOCK + run: pytest + - name: Test with pytest (online API mode) + env: + PUBS_TESTS_MODE: COLLECT + run: pytest + + install-test: + name: Test installation + strategy: + matrix: + os: [macos-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} + if: github.event_name == 'schedule' + + steps: + - uses: actions/setup-python@v2 + - name: install test + run: | + pip install -U pip + pip install pubs + pubs --help + pip uninstall -y pubs diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cbd0ad2..0000000 --- a/.travis.yml +++ /dev/null @@ -1,105 +0,0 @@ -# list of environments to test -matrix: - include: - - # Full tests (with online API) - - os: linux - language: python - python: 2.7 - env: - - TO_TEST=TEST_FULL - - os: linux - language: python - python: 3.7 - dist: xenial - sudo: true - env: - - TO_TEST=TEST_FULL - - os: osx - language: generic - python: 2.7 - env: - - TO_TEST=TEST_FULL - # before_install: - # - python2 --version - # - pip2 install -U virtualenv - # - virtualenv env -p python2 - # - source env/bin/activate - - os: osx - language: generic - python: ">=3.6" - env: - - TO_TEST=TEST_FULL - before_install: - - brew outdated python3 || brew install python3 || brew upgrade python3 - - python3 -m venv env - - source env/bin/activate - - # Mock tests (with mock API) - - os: linux - language: python - python: 3.3 - env: - - TO_TEST=TEST_MOCK - - os: linux - language: python - python: 3.4 - env: - - TO_TEST=TEST_MOCK - - os: linux - language: python - python: 3.5 - env: - - TO_TEST=TEST_MOCK - - os: linux - language: python - python: 3.6 - env: - - TO_TEST=TEST_MOCK - - os: linux - language: python - dist: xenial - python: 3.7 - sudo: true - env: - - TO_TEST=TEST_MOCK - - - # Install tests - - os: linux - language: python - python: 2.7 - env: - - TO_TEST=INSTALL - if: type = cron - - os: linux - language: python - dist: xenial - sudo: true - python: 3.7 - env: - - TO_TEST=INSTALL - if: type = cron - - os: osx - language: generic - python: 2.7 - env: - - TO_TEST=INSTALL - if: type = cron - - os: osx - language: generic - python: ">=3.6" - env: - - TO_TEST=INSTALL - if: type = cron - - allow_failures: - - python: 3.3 - -# command to run tests -script: - - python --version - - if [ "$TO_TEST" = "TEST_MOCK" ] || - [ "$TO_TEST" = "TEST_FULL" ]; then PUBS_TESTS_MODE=MOCK python setup.py test; fi - - if [ "$TO_TEST" = "TEST_FULL" ]; then PUBS_TESTS_MODE=COLLECT python setup.py test; fi - - if [ "$TO_TEST" = "INSTALL" ]; then pip install -U pip && pip install pubs && pubs --help && pip uninstall -y pubs; fi diff --git a/changelog.md b/changelog.md index 52dadd9..ca05610 100644 --- a/changelog.md +++ b/changelog.md @@ -5,8 +5,24 @@ [Full Changelog](https://github.com/pubs/pubs/compare/v0.8.3...master) +### Implemented enhancements + +- Migration from Travis CI to Github actions ([#260](https://github.com/pubs/pubs/pull/260)) +- Allow passing named arguments to custom commands ([#241](https://github.com/pubs/pubs/pull/241) by [jkulhanek](https://github.com/jkulhanek)) +- Added support for non-standard bibtex types, e.g. @collection, @software, etc. ([#226](https://github.com/pubs/pubs/pull/226)) +- The number of displayed authors in listings is now configurable, as the `max_authors` value in the `main` section of the configuration. ([#225](https://github.com/pubs/pubs/pull/225)) +- More explicit add command dialogs when copying and moving documents. +- Empty tags are not added to papers anymore. + ### Fixed bugs +- Fixed collision when entry uses `type` field ([#252](https://github.com/pubs/pubs/pull/252)) +- Note on comma in alias descriptions ([#240](https://github.com/pubs/pubs/pull/240) [StanczakDominik](https://github.com/StanczakDominik)) +- Note path correctly expand user '~' ([#250](https://github.com/pubs/pubs/pull/250)) +- Tests don't run on python 2.7 or <=3.4. They may still work, but support will not be tested and will eventually be dropped. ([#223](https://github.com/pubs/pubs/pull/223)) +- Fixed the reported number of paper with a tag in the statistic command ([#232](https://github.com/pubs/pubs/pull/232) by [beuerle](https://github.com/beuerle)) +- Fixed a crash when resolving citekeys introduced by [#225](https://github.com/pubs/pubs/pull/225) ([#233](https://github.com/pubs/pubs/pull/233) by [beuerle](https://github.com/beuerle)) + ## [v0.8.3](https://github.com/pubs/pubs/compare/v0.8.2...v0.8.3) (2019-08-12) @@ -38,7 +54,6 @@ A hotfix release. All users of 0.8.0 are urged to upgrade. ### Fixed bugs - Fix adding paper with DOIs, ISBNs or arXiv references. [(#165)](https://github.com/pubs/pubs/pull/165) - - Fix statistics command when there is not yet any paper in the repository. [(#164)](https://github.com/pubs/pubs/pull/164) @@ -49,69 +64,39 @@ A long overdue feature release. Add supports for arXiv bibtex fetching, and many ### Implemented enhancements - Adds `move`, and `link` options for handling of documents during `import` (copy being the default). Makes `copy` the default for document handling during `add`. [(#159)](https://github.com/pubs/pubs/pull/159) - - Support for downloading arXiv reference from their ID ([#146](https://github.com/pubs/pubs/issues/146) by [joe-antognini](https://github.com/joe-antognini)) - - Better feedback when an error is encountered while adding a reference from a DOI, ISBN or arXiv ID [#155](https://github.com/pubs/pubs/issues/155) - - Better dialog after editing paper [(#142)](https://github.com/pubs/pubs/issues/142) - - Add a command to open urls ([#139](https://github.com/pubs/pubs/issues/139) by [ksunden](https://github.com/ksunden)) - - More robust cache on version change [(#138)](https://github.com/pubs/pubs/issues/138) - - Allow utf8 citekeys [(#133)](https://github.com/pubs/pubs/issues/133) - - Adds tag list completion in `pubs add -t ` [(#130)](https://github.com/pubs/pubs/issues/130) - - Wider Travis coverage ([#107](https://github.com/pubs/pubs/issues/107) and [#108](https://github.com/pubs/pubs/issues/108)) - - Uses bibtexparser bwriter instead of internal encoder and adds `--ignore-fields` option to export. [(#106)](https://github.com/pubs/pubs/issues/106) - - Configurable alias descriptions ([#104](https://github.com/pubs/pubs/issues/104) by [wflynny](https://github.com/wflynny)) - - Support year ranges in query [(#102)](https://github.com/pubs/pubs/issues/102) - - Tests can now be run with `python setup.py test` [#155](https://github.com/pubs/pubs/issues/155) ### Fixed bugs - [[#144]](https://github.com/pubs/pubs/issues/144) More robust handling of the `doc_add` options [(#159)](https://github.com/pubs/pubs/pull/159) - - [[#149]](https://github.com/pubs/pubs/issues/149) More robust handling of parsing and citekey errors [(#87)](https://github.com/pubs/pubs/pull/87) - - [[#148]](https://github.com/pubs/pubs/issues/148) Fix compatibility with Pyfakefs 3.7 [(#151)](https://github.com/pubs/pubs/pull/151) - - [[#95]](https://github.com/pubs/pubs/issues/95) Error message when editor is missing [(#141)](https://github.com/pubs/pubs/issues/141) - - Fixes tests for printing help on `--help` and without argument. [(#137)](https://github.com/pubs/pubs/issues/137) - - [[#126]](https://github.com/pubs/pubs/issues/126) Removes journal customization [(#127)](https://github.com/pubs/pubs/issues/127) - - Fixes Travis failure on installing python3 for OSX [(#125)](https://github.com/pubs/pubs/issues/125) - - [[#119]](https://github.com/pubs/pubs/issues/119) Removes link and DOI customization. [(#124)](https://github.com/pubs/pubs/issues/124) - - [[#122]](https://github.com/pubs/pubs/issues/122) Fixes common strings [(#123)](https://github.com/pubs/pubs/issues/123) - - [[#28]](https://github.com/pubs/pubs/issues/28) allow utf8 in citekeys [(#120)](https://github.com/pubs/pubs/issues/120) - - Fixes field orders to use 'url' and fixes broken test. [(#118)](https://github.com/pubs/pubs/issues/118) - - [[#25]](https://github.com/pubs/pubs/issues/25) Fix bibtex testcase [(#117)](https://github.com/pubs/pubs/issues/117) - - [[#103]](https://github.com/pubs/pubs/issues/103) Fixes unicode comparison [(#116)](https://github.com/pubs/pubs/issues/116) - - [[#95]](https://github.com/pubs/pubs/issues/95) robust handling of DOIs ([#105](https://github.com/pubs/pubs/issues/105) by [wflynny](https://github.com/wflynny)) - - [[#99]](https://github.com/pubs/pubs/issues/99) Print help when no subcommand is provided ([#100](https://github.com/pubs/pubs/issues/100) by [wflynny](https://github.com/wflynny)) - - Fix defaults not used in config. [(#97)](https://github.com/pubs/pubs/issues/97) - - Fixes content not read from urls because of call to `os.abspath` [(#96)](https://github.com/pubs/pubs/issues/96) - - [[#93]](https://github.com/pubs/pubs/issues/93) actually save the modifications on `edit -m`. [(#94)](https://github.com/pubs/pubs/issues/94) - - [[#88]](https://github.com/pubs/pubs/issues/88) Adds proper escaping for arguments in alias plugin. [(#91)](https://github.com/pubs/pubs/issues/91) diff --git a/dev_requirements.txt b/dev_requirements.txt index 1891f81..0b107fb 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -20,6 +20,6 @@ six # those are the additional packages required to run the tests pyfakefs certifi -ddt +ddt>=1.4.1 mock -pytest # optional (python setup.py test works without it), but possible nonetheless +pytest diff --git a/pubs/bibstruct.py b/pubs/bibstruct.py index 5df6a98..23c3f6a 100644 --- a/pubs/bibstruct.py +++ b/pubs/bibstruct.py @@ -16,7 +16,7 @@ from .p3 import ustr, uchr # Citekey stuff -TYPE_KEY = 'type' +TYPE_KEY = 'ENTRYTYPE' CONTROL_CHARS = ''.join(map(uchr, list(range(0, 32)) + list(range(127, 160)))) CITEKEY_FORBIDDEN_CHARS = '@\'\\,#}{~%/ ' # '/' is OK for bibtex but forbidden diff --git a/pubs/color.py b/pubs/color.py index 86e8622..7a3ad64 100644 --- a/pubs/color.py +++ b/pubs/color.py @@ -144,7 +144,7 @@ def setup(conf, force_colors=False): # undye -undye_re = re.compile('\x1b\[[;\d]*[A-Za-z]') +undye_re = re.compile('\x1b\\[[;\\d]*[A-Za-z]') def undye(s): """Purge string s of color""" diff --git a/pubs/commands/add_cmd.py b/pubs/commands/add_cmd.py index cf67fa9..3499629 100644 --- a/pubs/commands/add_cmd.py +++ b/pubs/commands/add_cmd.py @@ -6,6 +6,7 @@ from .. import p3 from .. import bibstruct from .. import content from .. import repo +from .. import color from .. import paper from .. import templates from .. import apis @@ -150,15 +151,17 @@ def command(conf, args): doc_add = conf['main']['doc_add'] rp.push_paper(p) - ui.message('added to pubs:\n{}'.format(pretty.paper_oneliner(p))) + ui.message('added to pubs:\n{}'.format(pretty.paper_oneliner(p, max_authors=conf['main']['max_authors']))) if docfile is not None: - rp.push_doc(p.citekey, docfile, copy=(doc_add in ('copy', 'move'))) - if doc_add == 'move' and content.content_type(docfile) != 'url': - content.remove_file(docfile) - - if doc_add == 'move': - ui.message('{} was moved to the pubs repository.'.format(docfile)) - elif doc_add == 'copy': - ui.message('{} was copied to the pubs repository.'.format(docfile)) + rp.push_doc_paper(p, docfile, copy=(doc_add in ('copy', 'move'))) + + if doc_add in ('move', 'copy'): + if doc_add == 'move' and content.content_type(docfile) != 'url': + content.remove_file(docfile) + + docpath = content.system_path(rp.databroker.real_docpath(p.docpath)) + verb = 'moved' if doc_add == 'move' else 'copied' + ui.message('{} was {} to {} inside the pubs repository.'.format(color.dye_out(docfile, 'filepath'), verb, + color.dye_out(docpath, 'filepath'))) rp.close() diff --git a/pubs/commands/doc_cmd.py b/pubs/commands/doc_cmd.py index 277c006..95e314e 100644 --- a/pubs/commands/doc_cmd.py +++ b/pubs/commands/doc_cmd.py @@ -34,7 +34,7 @@ def parser(subparsers, conf): ).completer = CiteKeyCompletion(conf) add_exclusives = add_parser.add_mutually_exclusive_group() add_exclusives.add_argument( - '-L', '--link', action='store_false', dest='link', default=False, + '-L', '--link', action='store_true', dest='link', default=False, help='do not copy document files, just create a link') add_exclusives.add_argument( '-M', '--move', action='store_true', dest='move', default=False, @@ -71,7 +71,7 @@ def command(conf, args): # ui.exit() if args.action == 'add': - citekey = resolve_citekey(rp, args.citekey[0], ui=ui, exit_on_fail=True) + citekey = resolve_citekey(rp, conf, args.citekey[0], ui=ui, exit_on_fail=True) paper = rp.pull_paper(citekey) if paper.docpath is not None and not args.force: @@ -92,13 +92,14 @@ def command(conf, args): if not args.link and args.move: content.remove_file(document) + # FIXME: coherence with add command, the destination location should be given when copying/moving. ui.message('{} added to {}'.format( color.dye_out(document, 'filepath'), color.dye_out(paper.citekey, 'citekey'))) elif args.action == 'remove': - for key in resolve_citekey_list(rp, args.citekeys, ui=ui, exit_on_fail=True): + for key in resolve_citekey_list(rp, conf, args.citekeys, ui=ui, exit_on_fail=True): paper = rp.pull_paper(key) # if there is no document (and the user cares) -> inform + continue @@ -126,7 +127,7 @@ def command(conf, args): color.dye_err(args.path[0], 'filepath'))) ui.exit(1) - for key in resolve_citekey_list(rp, args.citekeys, ui=ui, exit_on_fail=True): + for key in resolve_citekey_list(rp, conf, args.citekeys, ui=ui, exit_on_fail=True): try: paper = rp.pull_paper(key) doc = paper.docpath @@ -143,7 +144,7 @@ def command(conf, args): elif args.action == 'open': with_command = args.cmd - citekey = resolve_citekey(rp, args.citekey[0], ui=ui, exit_on_fail=True) + citekey = resolve_citekey(rp, conf, args.citekey[0], ui=ui, exit_on_fail=True) paper = rp.pull_paper(citekey) if paper.docpath is None: diff --git a/pubs/commands/edit_cmd.py b/pubs/commands/edit_cmd.py index 431d19c..5ea7679 100644 --- a/pubs/commands/edit_cmd.py +++ b/pubs/commands/edit_cmd.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from ..paper import Paper from .. import repo +from .. import color from ..uis import get_ui from ..endecoder import EnDecoder @@ -29,7 +30,7 @@ def command(conf, args): meta = args.meta rp = repo.Repository(conf) - citekey = resolve_citekey(rp, args.citekey, ui=ui, exit_on_fail=True) + citekey = resolve_citekey(rp, conf, args.citekey, ui=ui, exit_on_fail=True) paper = rp.pull_paper(citekey) coder = EnDecoder() @@ -55,17 +56,18 @@ def command(conf, args): new_paper = Paper(paper.citekey, paper.bibdata, metadata=content) rp.push_paper(new_paper, overwrite=True, event=False) - ui.info(('The metadata of paper `{}` was successfully ' - 'edited.'.format(citekey))) + ui.info(("The metadata of paper '{}' was successfully " + "edited.".format(color.dye_out(citekey, 'citekey')))) else: new_paper = Paper.from_bibentry(content, metadata=paper.metadata) if rp.rename_paper(new_paper, old_citekey=paper.citekey): - ui.info(('Paper `{}` was successfully edited and renamed ' - 'as `{}`.'.format(citekey, new_paper.citekey))) + ui.info(("Paper '{}' was successfully edited and renamed " + "as '{}'.".format(color.dye_out(citekey, 'citekey'), + color.dye_out(new_paper.citekey, 'citekey')))) else: - ui.info(('Paper `{}` was successfully edited.'.format( - citekey))) + ui.info(("Paper '{}' was successfully edited.".format( + color.dye_out(citekey, 'citekey')))) break except coder.BibDecodingError: @@ -84,7 +86,7 @@ def command(conf, args): break elif choice == 'overwrite': paper = rp.push_paper(paper, overwrite=True) - ui.info(('Paper `{}` was overwritten.'.format(citekey))) + ui.info(('Paper `{}` was overwritten.'.format(color.dye_out(citekey, 'citekey')))) break # else edit again # Also handle malformed bibtex and metadata diff --git a/pubs/commands/export_cmd.py b/pubs/commands/export_cmd.py index 2bbe01d..b316efc 100644 --- a/pubs/commands/export_cmd.py +++ b/pubs/commands/export_cmd.py @@ -46,7 +46,7 @@ def command(conf, args): if len(args.citekeys) < 1: papers = rp.all_papers() else: - for key in resolve_citekey_list(repo=rp, citekeys=args.citekeys, ui=ui, exit_on_fail=True): + for key in resolve_citekey_list(rp, conf, args.citekeys, ui=ui, exit_on_fail=True): papers.append(rp.pull_paper(key)) bib = {} diff --git a/pubs/commands/list_cmd.py b/pubs/commands/list_cmd.py index b615e54..24e533b 100644 --- a/pubs/commands/list_cmd.py +++ b/pubs/commands/list_cmd.py @@ -54,7 +54,7 @@ def command(conf, args): papers = sorted(papers, key=date_added) if len(papers) > 0: ui.message('\n'.join( - pretty.paper_oneliner(p, citekey_only=args.citekeys) + pretty.paper_oneliner(p, citekey_only=args.citekeys, max_authors=conf['main']['max_authors']) for p in papers)) rp.close() diff --git a/pubs/commands/note_cmd.py b/pubs/commands/note_cmd.py index 9466566..04c7eba 100644 --- a/pubs/commands/note_cmd.py +++ b/pubs/commands/note_cmd.py @@ -23,7 +23,7 @@ def command(conf, args): ui = get_ui() rp = repo.Repository(conf) - citekey = resolve_citekey(rp, args.citekey, ui=ui, exit_on_fail=True) + citekey = resolve_citekey(rp, conf, args.citekey, ui=ui, exit_on_fail=True) notepath = rp.databroker.real_notepath(citekey, rp.conf['main']['note_extension']) if args.append is None: ui.edit_file(notepath, temporary=False) diff --git a/pubs/commands/remove_cmd.py b/pubs/commands/remove_cmd.py index 0b7f9e0..e19211c 100644 --- a/pubs/commands/remove_cmd.py +++ b/pubs/commands/remove_cmd.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from .. import repo from .. import color +from .. import pretty from ..uis import get_ui from ..utils import resolve_citekey_list from ..p3 import ustr, u_maybe @@ -24,12 +25,16 @@ def command(conf, args): force = args.force rp = repo.Repository(conf) - keys = resolve_citekey_list(repo=rp, citekeys=args.citekeys, ui=ui, exit_on_fail=True) + keys = resolve_citekey_list(rp, conf, args.citekeys, ui=ui, exit_on_fail=True) + plural = 's' if len(keys) > 1 else '' if force is None: - are_you_sure = (("Are you sure you want to delete the publication(s) [{}]" - " (this will also delete associated documents)?") - .format(', '.join([color.dye_out(c, 'citekey') for c in args.citekeys]))) + to_remove_str = '\n'.join(pretty.paper_oneliner(rp.pull_paper(key), + max_authors=conf['main']['max_authors']) + for key in keys) + are_you_sure = (("Are you sure you want to delete the following publication{}" + " (this will also delete associated documents)?:\n{}\n") + .format(plural, to_remove_str)) sure = ui.input_yn(question=are_you_sure, default='n') if force or sure: failed = False # Whether something failed @@ -42,11 +47,12 @@ def command(conf, args): if failed: ui.exit() # Exit with nonzero error code else: - ui.message('The publication(s) [{}] were removed'.format( + + ui.message('The publication{} {} were removed'.format(plural, ', '.join([color.dye_out(c, 'citekey') for c in keys]))) # FIXME: print should check that removal proceeded well. else: - ui.message('The publication(s) [{}] were {} removed'.format( + ui.message('The publication{} {} were {} removed'.format(plural, ', '.join([color.dye_out(c, 'citekey') for c in keys]), color.dye_out('not','bold'))) diff --git a/pubs/commands/rename_cmd.py b/pubs/commands/rename_cmd.py index af14c4f..3461250 100644 --- a/pubs/commands/rename_cmd.py +++ b/pubs/commands/rename_cmd.py @@ -26,7 +26,7 @@ def command(conf, args): rp = repo.Repository(conf) # TODO: here should be a test whether the new citekey is valid - key = resolve_citekey(repo=rp, citekey=args.citekey, ui=ui, exit_on_fail=True) + key = resolve_citekey(rp, conf, args.citekey, ui=ui, exit_on_fail=True) paper = rp.pull_paper(key) rp.rename_paper(paper, args.new_citekey) ui.message("The '{}' citekey has been renamed into '{}'".format( diff --git a/pubs/commands/statistics_cmd.py b/pubs/commands/statistics_cmd.py index e3b6583..a606fa0 100644 --- a/pubs/commands/statistics_cmd.py +++ b/pubs/commands/statistics_cmd.py @@ -22,7 +22,7 @@ def command(conf, args): else: doc_count = sum([0 if p.docpath is None else 1 for p in papers]) tag_count = len(list(rp.get_tags())) - papers_with_tags = sum([0 if p.tags else 1 for p in papers]) + papers_with_tags = sum([1 if p.tags else 0 for p in papers]) ui.message(color.dye_out('Repository statistics:', 'bold')) ui.message('Total papers: {}, {} ({}) have a document attached'.format( diff --git a/pubs/commands/tag_cmd.py b/pubs/commands/tag_cmd.py index da656b4..c12bf8f 100644 --- a/pubs/commands/tag_cmd.py +++ b/pubs/commands/tag_cmd.py @@ -55,7 +55,9 @@ def _parse_tag_seq(s): if last != 0: raise ValueError('could not match tag expression') else: - tags.append(s[last:(m.start())]) + tag = s[last:(m.start())] + if len(tag) > 0: + tags.append(s[last:(m.start())]) last = m.start() if last == len(s): raise ValueError('could not match tag expression') @@ -89,7 +91,7 @@ def command(conf, args): else: not_citekey = False try: - citekeyOrTag = resolve_citekey(repo=rp, citekey=citekeyOrTag, ui=ui, exit_on_fail=True) + citekeyOrTag = resolve_citekey(rp, conf, citekeyOrTag, ui=ui, exit_on_fail=True) except SystemExit: not_citekey = True if not not_citekey: @@ -117,7 +119,7 @@ def command(conf, args): len(p.tags.intersection(excluded)) == 0): papers_list.append(p) - ui.message('\n'.join(pretty.paper_oneliner(p) + ui.message('\n'.join(pretty.paper_oneliner(p, max_authors=conf['main']['max_authors']) for p in papers_list)) rp.close() diff --git a/pubs/commands/url_cmd.py b/pubs/commands/url_cmd.py index 7ac7b7f..224da7f 100644 --- a/pubs/commands/url_cmd.py +++ b/pubs/commands/url_cmd.py @@ -22,7 +22,7 @@ def command(conf, args): ui = get_ui() rp = repo.Repository(conf) - for key in resolve_citekey_list(rp, args.citekey, ui=ui, exit_on_fail=False): + for key in resolve_citekey_list(rp, conf, args.citekey, ui=ui, exit_on_fail=False): try: paper = rp.pull_paper(key) url = paper.bibdata['url'] diff --git a/pubs/config/spec.py b/pubs/config/spec.py index bfbb79f..92a8747 100644 --- a/pubs/config/spec.py +++ b/pubs/config/spec.py @@ -27,6 +27,10 @@ edit_cmd = string(default='') # Which default extension to use when creating a note file. note_extension = string(default='txt') +# How many authors to display when displaying a citation. If there are more +# authors, only the first author is diplayed followed by 'et al.'. +max_authors = integer(default=3) + # If true debug mode is on which means exceptions are not catched and # the full python stack is printed. debug = boolean(default=False) @@ -115,6 +119,9 @@ active = force_list(default=list('alias')) # command = !pubs list -k | wc -l # description = lists number of pubs in repo +# To use commas in the description, wrap them in a "" string. For example: +# description = "lists number of pubs in repo, greets the user afterward" + [[git]] # The git plugin will commit changes to the repository in a git repository # created at the root of the pubs directory. All detected changes will be diff --git a/pubs/content.py b/pubs/content.py index b1e403b..e06b72e 100644 --- a/pubs/content.py +++ b/pubs/content.py @@ -114,7 +114,7 @@ def write_file(filepath, data, mode='w'): def content_type(path): parsed = urlparse(path) - if parsed.scheme == 'http': + if parsed.scheme in ('http', 'https'): return 'url' else: return 'file' diff --git a/pubs/endecoder.py b/pubs/endecoder.py index 346af3a..6c88bd7 100644 --- a/pubs/endecoder.py +++ b/pubs/endecoder.py @@ -51,7 +51,6 @@ def customizations(record): """ # record = bp.customization.convert_to_unicode(record) # transform \& into & ones, messing-up latex - record = bp.customization.type(record) record = bp.customization.author(record) record = bp.customization.editor(record) record = bp.customization.keyword(record) @@ -119,18 +118,18 @@ class EnDecoder(object): keyword for keyword in entry['keyword']) return entry - def decode_bibdata(self, bibdata): + def decode_bibdata(self, bibstr): """Decodes bibdata from string. If the decoding fails, returns a BibDecodingError. """ - if len(bibdata) == 0: + if len(bibstr) == 0: error_msg = 'parsing error: the provided string has length zero.' - raise self.BibDecodingError(error_msg, bibdata) + raise self.BibDecodingError(error_msg, bibstr) try: entries = bp.bparser.BibTexParser( - bibdata, common_strings=True, customization=customizations, - homogenize_fields=True).get_entry_dict() + bibstr, common_strings=True, customization=customizations, + homogenize_fields=True, ignore_nonstandard_types=False).get_entry_dict() # Remove id from bibtexparser attribute which is stored as citekey for e in entries: entries[e].pop(BP_ID_KEY) @@ -147,13 +146,13 @@ class EnDecoder(object): return entries else: raise self.BibDecodingError(('no valid entry found in the provided data: ' - ' {}').format(bibdata), bibdata) + ' {}').format(bibstr), bibstr) except (pyparsing.ParseException, pyparsing.ParseSyntaxException) as e: error_msg = self._format_parsing_error(e) - raise self.BibDecodingError(error_msg, bibdata) + raise self.BibDecodingError(error_msg, bibstr) except bibtexparser.bibdatabase.UndefinedString as e: error_msg = 'parsing error: undefined string in provided data: {}'.format(e) - raise self.BibDecodingError(error_msg, bibdata) + raise self.BibDecodingError(error_msg, bibstr) @classmethod def _format_parsing_error(cls, e): diff --git a/pubs/filebroker.py b/pubs/filebroker.py index f48ba93..7c052b3 100644 --- a/pubs/filebroker.py +++ b/pubs/filebroker.py @@ -16,7 +16,7 @@ def filter_filename(filename, ext): """ Return the filename without the extension if the extension matches ext. Otherwise return None """ - pattern = '.*\{}$'.format(ext) + pattern = '.*\\{}$'.format(ext) if re.match(pattern, filename) is not None: return u_maybe(filename[:-len(ext)]) @@ -154,7 +154,7 @@ class DocBroker(object): def __init__(self, directory, scheme='docsdir', subdir='doc'): self.scheme = scheme - self.docdir = os.path.join(directory, subdir) + self.docdir = os.path.expanduser(os.path.join(directory, subdir)) if not check_directory(self.docdir, fail=False): os.mkdir(system_path(self.docdir)) diff --git a/pubs/plugs/git/git.py b/pubs/plugs/git/git.py index 5be9221..3bff3d6 100644 --- a/pubs/plugs/git/git.py +++ b/pubs/plugs/git/git.py @@ -16,11 +16,13 @@ GITIGNORE = """# files or directories for the git plugin to ignore class GitPlugin(PapersPlugin): - """The git plugin creates a git repository in the pubs directory and commit the changes - to the pubs repository everytime a paper is modified. + """Make the pubs repository also a git repository. - It also add the `pubs git` subcommand, so git commands can be executed in the git repository - from the command line. + The git plugin creates a git repository in the pubs directory + and commit the changes to the pubs repository. + + It also add the `pubs git` subcommand, so git commands can be executed + in the git repository from the command line. """ name = 'git' @@ -28,10 +30,10 @@ class GitPlugin(PapersPlugin): def __init__(self, conf, ui): self.ui = ui - self.pubsdir = os.path.expanduser(conf['main']['pubsdir']) - self.manual = conf['plugins'].get('git', {}).get('manual', False) + self.pubsdir = os.path.expanduser(conf['main']['pubsdir']) + self.manual = conf['plugins'].get('git', {}).get('manual', False) self.force_color = conf['plugins'].get('git', {}).get('force_color', True) - self.quiet = conf['plugins'].get('git', {}).get('quiet', True) + self.quiet = conf['plugins'].get('git', {}).get('quiet', True) self.list_of_changes = [] self._gitinit() @@ -72,17 +74,18 @@ class GitPlugin(PapersPlugin): """ colorize = ' -c color.ui=always' if self.force_color else '' git_cmd = 'git -C {}{} {}'.format(self.pubsdir, colorize, cmd) - #print(git_cmd) p = Popen(git_cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=True) output, err = p.communicate(input_stdin) p.wait() if p.returncode != 0: - raise RuntimeError('The git plugin encountered an error when running the git command:\n' + - '{}\n\nReturned output:\n{}\n'.format(git_cmd, output.decode('utf-8')) + - 'If needed, you may fix the state of the {} git repository '.format(self.pubsdir) + - 'manually.\nIf relevant, you may submit a bug report at ' + - 'https://github.com/pubs/pubs/issues') + raise RuntimeError(( + 'The git plugin encountered an error when running the git command:\n' + '{}\n\n' + 'Returned output:\n{}\n' + 'If needed, you may fix the state of the {} git repository manually.\n' + 'If relevant, you may submit a bug report at https://github.com/pubs/pubs/issues' + ).format(git_cmd, output.decode('utf-8'), self.pubsdir)) elif command: self.ui.message(output.decode('utf-8'), end='') elif not self.quiet: @@ -97,10 +100,11 @@ def paper_change_event(event): git = GitPlugin.get_instance() if not git.manual: event_desc = event.description - for a, b in [('\\','\\\\'), ('"','\\"'), ('$','\\$'), ('`','\\`')]: + for a, b in [('\\', '\\\\'), ('"', '\\"'), ('$', '\\$'), ('`', '\\`')]: event_desc = event_desc.replace(a, b) git.list_of_changes.append(event_desc) + @PostCommandEvent.listen() def git_commit(event): if GitPlugin.is_loaded(): diff --git a/pubs/pretty.py b/pubs/pretty.py index 0c951e6..6b12fb6 100644 --- a/pubs/pretty.py +++ b/pubs/pretty.py @@ -23,19 +23,24 @@ def person_repr(p): ' '.join(p.lineage(abbr=True))] if s) -def short_authors(bibdata): +def short_authors(bibdata, max_authors=3): + """ + :param max_authors: number of authors to display completely. Additional authors will be + represented by 'et al.'. + """ try: authors = [p for p in bibdata['author']] - if len(authors) < 3: - return ' and '.join(authors) + if 0 < max_authors < len(authors): + authors_str = '{} et al.'.format(authors[0]) else: - return authors[0] + (' et al.' if len(authors) > 1 else '') + authors_str = ' and '.join(authors) + return authors_str except KeyError: # When no author is defined return '' -def bib_oneliner(bibdata): - authors = short_authors(bibdata) +def bib_oneliner(bibdata, max_authors=3): + authors = short_authors(bibdata, max_authors=max_authors) journal = '' if 'journal' in bibdata: journal = ' ' + bibdata['journal'] @@ -60,11 +65,11 @@ def bib_desc(bib_data): return s -def paper_oneliner(p, citekey_only=False): +def paper_oneliner(p, citekey_only=False, max_authors=3): if citekey_only: return p.citekey else: - bibdesc = bib_oneliner(p.get_unicode_bibdata()) + bibdesc = bib_oneliner(p.get_unicode_bibdata(), max_authors=max_authors) doc_str = '' if p.docpath is not None: doc_extension = os.path.splitext(p.docpath)[1] diff --git a/pubs/repo.py b/pubs/repo.py index 4b341a4..e97e022 100644 --- a/pubs/repo.py +++ b/pubs/repo.py @@ -182,17 +182,22 @@ class Repository(object): events.RenameEvent(paper, old_citekey).send() return True + def push_doc(self, citekey, docfile, copy=None): p = self.pull_paper(citekey) + return self.push_doc_paper(p, docfile, copy=copy) + + def push_doc_paper(self, paper, docfile, copy=None): + """Same as push_doc, only the Paper instance is provided rather than the citekey""" if copy is None: copy = self.conf['main']['doc_add'] in ('copy', 'move') if copy: - docfile = self.databroker.add_doc(citekey, docfile) + docfile = self.databroker.add_doc(paper.citekey, docfile) else: docfile = system_path(docfile) - p.docpath = docfile - self.push_paper(p, overwrite=True, event=False) - events.DocAddEvent(citekey).send() + paper.docpath = docfile + self.push_paper(paper, overwrite=True, event=False) + events.DocAddEvent(paper.citekey).send() def unique_citekey(self, base_key, bibentry): """Create a unique citekey for a given base key. diff --git a/pubs/uis.py b/pubs/uis.py index 46258ce..fc96aed 100644 --- a/pubs/uis.py +++ b/pubs/uis.py @@ -3,6 +3,7 @@ from __future__ import print_function, unicode_literals import os import sys import shlex +import errno import locale import codecs import tempfile @@ -237,7 +238,7 @@ class InputUI(PrintUI): try: subprocess.call(cmd) except OSError as e: - if e.errno == os.errno.ENOENT: + if e.errno == errno.ENOENT: self.error(("Error while calling editor '{}'. The editor may " "not be present. You can change the text editor " "that pubs uses by setting the $EDITOR environment " diff --git a/pubs/utils.py b/pubs/utils.py index eda7480..120edfd 100644 --- a/pubs/utils.py +++ b/pubs/utils.py @@ -7,7 +7,7 @@ from . import color from . import pretty -def resolve_citekey(repo, citekey, ui=None, exit_on_fail=True): +def resolve_citekey(repo, conf, citekey, ui=None, exit_on_fail=True): """Check that a citekey exists, or autocompletes it if not ambiguous. :returns found citekey """ @@ -29,22 +29,23 @@ def resolve_citekey(repo, citekey, ui=None, exit_on_fail=True): elif citekey not in citekeys: if ui is not None: citekeys = sorted(citekeys) - ui.error("Be more specific; '{}' matches multiples " - "citekeys:".format(citekey)) + msg = ["Be more specific; '{}' matches multiples citekeys:".format(citekey)] for c in citekeys: p = repo.pull_paper(c) - ui.message(' {}'.format(pretty.paper_oneliner(p))) + paper_str = pretty.paper_oneliner(p, max_authors=conf['main']['max_authors']) + msg.append(' {}'.format(paper_str)) + ui.error('\n'.join(msg)) if exit_on_fail: ui.exit() return citekey -def resolve_citekey_list(repo, citekeys, ui=None, exit_on_fail=True): +def resolve_citekey_list(repo, conf, citekeys, ui=None, exit_on_fail=True): shutdown = False keys = [] for key in citekeys: try: - keys.append(resolve_citekey(repo, key, ui, exit_on_fail)) + keys.append(resolve_citekey(repo, conf, key, ui=ui, exit_on_fail=exit_on_fail)) except SystemExit: shutdown = exit_on_fail @@ -73,11 +74,11 @@ def standardize_doi(doi): """ doi_regexes = ( - '(10\.\d{4,9}/[-._;()/:A-z0-9\>\<]+)', - '(10.1002/[^\s]+)', - '(10\.\d{4}/\d+-\d+X?(\d+)\d+<[\d\w]+:[\d\w]*>\d+.\d+.\w+;\d)', - '(10\.1021/\w\w\d+\+)', - '(10\.1207/[\w\d]+\&\d+_\d+)') + r'(10\.\d{4,9}/[-._;()/:A-z0-9\>\<]+)', + r'(10.1002/[^\s]+)', + r'(10\.\d{4}/\d+-\d+X?(\d+)\d+<[\d\w]+:[\d\w]*>\d+.\d+.\w+;\d)', + r'(10\.1021/\w\w\d+\+)', + r'(10\.1207/[\w\d]+\&\d+_\d+)') doi_pattern = re.compile('|'.join(doi_regexes)) match = doi_pattern.search(doi) diff --git a/readme.md b/readme.md index d596f18..446a8ad 100644 --- a/readme.md +++ b/readme.md @@ -15,24 +15,24 @@ correct bugs, but have no short-term plans to add major features to it. Pubs doe is supposed to do: help us do science, so now we are mostly doing that. **Notice:** pubs is relatively stable but comes with no warranty; do keep backups of your data. -**Notice:** pubs currently works with Python 2.7, but support will be dropped as soon as maintaining it becomes tedious. +**Notice:** pubs currently works with Python 2.7, but support is being dropped (tests are not run anymore). ## Installation You can install the latest stable version of `pubs` through Pypi, with: - ``` + ```shell pip install pubs ``` Alternatively, you can: - install the latest development version with pip: - ``` + ```shell pip install --upgrade git+https://github.com/pubs/pubs ``` - clone the repository and install it manually: - ``` + ```shell git clone https://github.com/pubs/pubs cd pubs python setup.py install [--user] @@ -44,32 +44,32 @@ Arch Linux users can also use the [pubs-git](https://aur.archlinux.org/packages/ ## Getting started Create your library (by default, goes to `~/.pubs/`). - ``` + ```shell pubs init ``` Import existing data from bibtex (pubs will try to automatically copy documents defined as 'file' in bibtex): - ``` + ```shell pubs import path/to/collection.bib ``` or for a .bib file containing a single reference: - ``` + ```shell pubs add reference.bib -d article.pdf ``` pubs can also automatically retrieve the bibtex from a doi: - ``` + ```shell pubs add -D 10.1007/s00422-012-0514-6 -d article.pdf ``` or an ISBN (dashes are ignored): - ``` + ```shell pubs add -I 978-0822324669 -d article.pdf ``` or an arXiv id (automatically downloading arXiv article is in the works): - ``` + ```shell pubs add -X math/9501234 -d article.pdf ``` @@ -79,7 +79,7 @@ or an arXiv id (automatically downloading arXiv article is in the works): If you use latex, you can automatize references, by running `pubs export > references.bib` each time you update your library, which also fits well as a `makefile` rule. This ensures that your reference file is always up-to-date; you can cite a paper in your manuscript a soon as you add it in pubs. This means that if you have, for instance, a DOI on a webpage, you only need to do: - ``` + ```shell pubs add -D 10.1007/s00422-012-0514-6 ``` @@ -89,12 +89,12 @@ and then add `\cite{Loeb_2012}` in your manuscript. After exporting the bibliogr ## Document management You can attach a document to a reference: - ``` + ```shell pubs doc add Loeb2012_downloaded.pdf Loeb_2012 ``` And open your documents automatically from the command line: - ``` + ```shell pubs doc open Loeb_2012 pubs doc open --with lp Loeb_2012 # Opens the document with `lp` to actually print it. ``` @@ -111,21 +111,21 @@ Pubs comes with a git plugin that automatically commits your changes. You only n You can then also conveniently interact with the git repository by using `pubs git `. -## Multiple pubs Repository +## Multiple pubs Repositories You may want to have different pubs repositories, for different projects. To create an alternate repository: - ``` + ```shell pubs --config /path/to/config init --pubsdir /path/to/desired_repository_directory ``` The configuration file and repository will be automatically created. Then you can add papers to the new repository: - ``` + ```shell pubs --config /path/to/config add -D 10.1007/s00422-012-0514-6 ``` A useful thing might be to define an alias in your shell: - ``` + ```shell alias pubs2="pubs --config /path/to/config" ``` and then use `pubs2` as you would use `pubs` directly. Note that you cannot use the alias plugin below to do this. @@ -141,13 +141,13 @@ You can add custom commands to pubs by defining aliases in your configuration fi count = !pubs list -k "$@" | wc -l ``` -The first command defines a new subcommand: `pubs open --with evince` will be executed when `pubs evince` is typed. -The second starts with a bang: `!`, and is treated as a shell command. If other arguments are provided they are passed to the shell command as in a script. In the example above the `count` alias can take arguments that are be passed to the `pubs list -k` command, hence enabling filters like `pubs count year:2012`. +The first configuration line defines a new subcommand: `pubs open --with evince` will be executed when `pubs evince` is typed. +The second starts with a bang: `!`, which means that it is treated as a shell command. If other arguments are provided they are passed to the shell command as in a script. In the example above the `count` alias can take arguments that are passed over to the `pubs list -k` command, hence enabling filters like `pubs count year:2012`. ## Autocompletion -For autocompletion to work, you need the [argcomplete](https://argcomplete.readthedocs.io) Python package, and Bash 4.2 or newer. For activating *bash* or *tsch* completion, consult the [argcomplete documentation](https://argcomplete.readthedocs.io/en/latest/#global-completion). +For autocompletion to work, you need the [argcomplete](https://kislyuk.github.io/argcomplete) Python package, and Bash 4.2 or newer. For activating *bash* or *tsch* completion, consult the [argcomplete documentation](https://kislyuk.github.io/argcomplete/#global-completion). For *zsh* completion, the global activation is not supported but bash completion compatibility can be used for pubs. For that, add the following to your `.zshrc`: ```shell @@ -183,3 +183,7 @@ You can access the self-documented configuration by using `pubs conf`, and all t - [Shane Stone](https://github.com/shanewstone) - [Amlesh Sivanantham](http://github.com/zamlz) - [DV Klopfenstein](http://github.com/dvklopfenstein) +- [beuerle](https://github.com/beuerle) +- [Jonáš Kulhánek](https://github.com/jkulhanek) +- [Dominik Stańczak](https://github.com/StanczakDominik) +- [Gustavo José de Sousa](https://github.com/guludo) diff --git a/setup.py b/setup.py index d7fe2dd..011ad68 100644 --- a/setup.py +++ b/setup.py @@ -45,23 +45,22 @@ setup( }, include_package_data=True, - install_requires=['pyyaml', 'bibtexparser>=1.0', 'python-dateutil', 'six', 'requests', 'configobj', 'beautifulsoup4', 'feedparser'], extras_require={'autocompletion': ['argcomplete'], }, + python_requires='>=3.6', classifiers=[ 'Development Status :: 4 - Beta', 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Intended Audience :: Developers', 'Intended Audience :: Science/Research', ], test_suite='tests', - tests_require=['pyfakefs>=3.4', 'mock', 'ddt', 'certifi'], + tests_require=['pyfakefs>=3.4', 'mock', 'ddt>=1.4.1', 'certifi', 'pytest'], # in order to avoid 'zipimport.ZipImportError: bad local file header' zip_safe=False, diff --git a/tests/data/collection.bib b/tests/data/three_articles.bib similarity index 100% rename from tests/data/collection.bib rename to tests/data/three_articles.bib diff --git a/tests/data_non_standard/non_standard_collection.bib b/tests/data_non_standard/non_standard_collection.bib new file mode 100644 index 0000000..0dbd1f2 --- /dev/null +++ b/tests/data_non_standard/non_standard_collection.bib @@ -0,0 +1,9 @@ +@collection{Geometric_phases, + title = {Geometric phases in physics}, + editor = {Shapere, Alfred and Wilczek, Frank}, + year = {1989}, + series = {Advanced Series in Mathematical Physics}, + volume = {5}, + publisher = {World Scientific}, + isbn = {9789971506216} +} diff --git a/tests/data_non_standard/non_standard_software.bib b/tests/data_non_standard/non_standard_software.bib new file mode 100644 index 0000000..87ecd7b --- /dev/null +++ b/tests/data_non_standard/non_standard_software.bib @@ -0,0 +1,7 @@ +@software{hadoop, + author = {{Apache Software Foundation}}, + title = {Hadoop}, + url = {https://hadoop.apache.org}, + version = {0.20.2}, + date = {2010-02-19}, +} diff --git a/tests/fake_env.py b/tests/fake_env.py index 6d790a3..1d1865f 100644 --- a/tests/fake_env.py +++ b/tests/fake_env.py @@ -9,7 +9,7 @@ import dotdot from pyfakefs import fake_filesystem, fake_filesystem_unittest -from pubs.p3 import input, _fake_stdio, _get_fake_stdio_ucontent +from pubs.p3 import input from pubs import content, filebroker, uis # code for fake fs @@ -29,29 +29,6 @@ original_exception_handler = uis.InputUI.handle_exception locale.setlocale(locale.LC_ALL, '') -# capture output - -def capture(f, verbose=False): - """Capture the stdout and stderr output. - - Useful for comparing the output with the expected one during tests. - - :param f: The function to capture output from. - :param verbose: If True, print call will still display their outputs. - If False, they will be silenced. - - """ - def newf(*args, **kwargs): - old_stderr, old_stdout = sys.stderr, sys.stdout - sys.stdout = _fake_stdio(additional_out=old_stderr if verbose else None) - sys.stderr = _fake_stdio(additional_out=old_stderr if False else None) - try: - return f(*args, **kwargs), _get_fake_stdio_ucontent(sys.stdout), _get_fake_stdio_ucontent(sys.stderr) - finally: - sys.stderr, sys.stdout = old_stderr, old_stdout - return newf - - # Test helpers # automating input diff --git a/tests/str_fixtures.py b/tests/str_fixtures.py index 1372a80..d2fbfec 100644 --- a/tests/str_fixtures.py +++ b/tests/str_fixtures.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals -bibtex_external0 = """ +bibtex_external0 = r""" @techreport{Page99, number = {1999-66}, month = {November}, @@ -17,7 +17,7 @@ institution = {Stanford InfoLab}, } """ -bibtex_external_alt = """ +bibtex_external_alt = r""" @techreport{Page99, number = {1999-66}, month = {November}, @@ -33,7 +33,7 @@ institution = {Stanford InfoLab}, } """ -bibtex_raw0 = """@techreport{ +bibtex_raw0 = r"""@techreport{ Page99, author = "Page, Lawrence and Brin, Sergey and Motwani, Rajeev and Winograd, Terry", publisher = "Stanford InfoLab", @@ -50,12 +50,12 @@ bibtex_raw0 = """@techreport{ """ -metadata_raw0 = """docfile: docsdir://Page99.pdf +metadata_raw0 = r"""docfile: docsdir://Page99.pdf tags: [search, network] added: '2013-11-14 13:14:20' """ -turing_bib = """@article{turing1950computing, +turing_bib = r"""@article{turing1950computing, title={Computing machinery and intelligence}, author={Turing, Alan M}, journal={Mind}, @@ -75,7 +75,7 @@ added: '2013-11-14 13:14:20' """ # Should not parse (see #113) -bibtex_no_citekey = """@Manual{, +bibtex_no_citekey = r"""@Manual{, title = {R: A Language and Environment for Statistical Computing}, author = {{R Core Team}}, organization = {R Foundation for Statistical Computing}, @@ -85,7 +85,7 @@ bibtex_no_citekey = """@Manual{, } """ -bibtex_month = """@inproceedings{Goyal2017, +bibtex_month = r"""@inproceedings{Goyal2017, author = {Goyal, Anirudh and Sordoni, Alessandro and C{\^{o}}t{\'{e}}, Marc-Alexandre and Ke, Nan Rosemary and Bengio, Yoshua}, title = {Z-Forcing: Training Stochastic Recurrent Networks}, year = {2017}, @@ -94,15 +94,15 @@ bibtex_month = """@inproceedings{Goyal2017, } """ -not_bibtex = """@misc{this looks, +not_bibtex = r"""@misc{this looks, like = a = bibtex file but , is not a real one! """ -bibtex_with_latex = """@article{kjaer2018large, +bibtex_with_latex = r"""@article{kjaer2018large, title={A large impact crater beneath Hiawatha Glacier in northwest Greenland}, - author={Kj{\\ae}r, Kurt H and Larsen, Nicolaj K and Binder, Tobias and Bj{\\o}rk, Anders A and Eisen, Olaf and Fahnestock, Mark A and Funder, Svend and Garde, Adam A and Haack, Henning and Helm, Veit and others}, + author={Kj{\ae}r, Kurt H and Larsen, Nicolaj K and Binder, Tobias and Bj{\o}rk, Anders A and Eisen, Olaf and Fahnestock, Mark A and Funder, Svend and Garde, Adam A and Haack, Henning and Helm, Veit and others}, journal={Science advances}, volume={4}, number={11}, diff --git a/tests/test_endecoder.py b/tests/test_endecoder.py index d9ade7f..3e81e8e 100644 --- a/tests/test_endecoder.py +++ b/tests/test_endecoder.py @@ -57,6 +57,12 @@ class TestEnDecode(unittest.TestCase): self.assertEqual(bibraw1, bibraw2) + def test_decode_bibtex_preserves_type_field(self): + """Test that multiple encode/decode step preserve data""" + decoder = endecoder.EnDecoder() + entry = decoder.decode_bibdata(bibtex_raw0) + self.assertEqual(entry['Page99']['type'], "Technical Report") + def test_endecode_bibtex_BOM(self): """Test that bibtexparser if fine with BOM-prefixed data""" decoder = endecoder.EnDecoder() diff --git a/tests/test_filebroker.py b/tests/test_filebroker.py index f8560a9..1f09008 100644 --- a/tests/test_filebroker.py +++ b/tests/test_filebroker.py @@ -71,6 +71,16 @@ class TestFileBroker(fake_env.TestFakeFs): class TestDocBroker(fake_env.TestFakeFs): + def test_expanduser(self): + """Test that real_docpath expand the user ~""" + + self.fs.add_real_directory(os.path.join(self.rootpath, 'data'), read_only=False) + fb = filebroker.FileBroker('~/testrepo', create = True) + docb = filebroker.DocBroker('~/testrepo') + self.assertTrue(os.path.isabs(docb.docdir)) + self.assertTrue(os.path.isabs(docb.real_docpath('docsdir://abc'))) + + def test_doccopy(self): self.fs.add_real_directory(os.path.join(self.rootpath, 'data'), read_only=False) diff --git a/tests/test_git.py b/tests/test_git.py index 9eb60f8..d25e6e7 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -1,5 +1,6 @@ -import unittest +import os import subprocess +import unittest import sand_env @@ -16,11 +17,20 @@ class TestGitPlugin(sand_env.SandboxedCommandTestCase): def setUp(self, nsec_stat=True): super(TestGitPlugin, self).setUp() + # Backup environment variables and set git author + self.env_backup = os.environ.copy() + os.environ['GIT_AUTHOR_NAME'] = "Pubs test" + os.environ['GIT_AUTHOR_EMAIL'] = "unittest@pubs.org" + # Setup pubs repository self.execute_cmds([('pubs init',)]) conf = config.load_conf(path=self.default_conf_path) conf['plugins']['active'] = ['git'] config.save_conf(conf, path=self.default_conf_path) + def tearDown(self): + super().tearDown() + os.environ = self.env_backup + def test_git(self): self.execute_cmds([('pubs add data/pagerank.bib',)]) hash_a = git_hash(self.default_pubs_dir) @@ -31,7 +41,7 @@ class TestGitPlugin(sand_env.SandboxedCommandTestCase): self.execute_cmds([('pubs rename Page99a ABC',)]) hash_c = git_hash(self.default_pubs_dir) - self.execute_cmds([('pubs remove ABC', ['y']),]) + self.execute_cmds([('pubs remove ABC', ['y'])]) hash_d = git_hash(self.default_pubs_dir) self.execute_cmds([('pubs doc add testrepo/doc/Page99.pdf Page99',)]) @@ -72,6 +82,7 @@ class TestGitPlugin(sand_env.SandboxedCommandTestCase): # self.assertEqual(hash_i, hash_j) def test_manual(self): + print(self.default_pubs_dir) conf = config.load_conf(path=self.default_conf_path) conf['plugins']['active'] = ['git'] conf['plugins']['git']['manual'] = True @@ -101,6 +112,5 @@ class TestGitPlugin(sand_env.SandboxedCommandTestCase): self.assertNotEqual(hash_l, hash_m) - if __name__ == '__main__': unittest.main() diff --git a/tests/test_note_append.py b/tests/test_note_append.py index c3009f2..32eebee 100644 --- a/tests/test_note_append.py +++ b/tests/test_note_append.py @@ -46,7 +46,7 @@ class TestNoteAppend(DataCommandTestCase): # * Pass the command split into a command and its args to # execute_cmdsplit, which is called by execute_cmds: cmd_split = ['pubs', 'note', 'Page99', '-a', 'xxx yyy'] - self.execute_cmdsplit(cmd_split, expected_out=None, expected_err=None) + self.execute_cmd_capture(cmd_split, expected_out=None, expected_err=None) note_lines.append('xxx yyy') self.assertFileContentEqual(fin_notes, self._get_note_content(note_lines)) diff --git a/tests/test_plug_alias.py b/tests/test_plug_alias.py index 63086f7..d91f677 100644 --- a/tests/test_plug_alias.py +++ b/tests/test_plug_alias.py @@ -1,7 +1,7 @@ import shlex import unittest - import dotdot +import argparse import pubs from pubs import config @@ -61,6 +61,19 @@ class AliasTestCase(unittest.TestCase): shlex.split(self.subprocess.called.splitlines()[-1])[1:], args) + def testShellAliasNamedArguments(self): + parser = argparse.ArgumentParser() + parser.add_argument('--test2') + subparsers = parser.add_subparsers(title='commands', dest='command') + + alias = Alias.create_alias('test', '!echo "$@"') + alias.parser(subparsers) + args = ['test', '2', '--option', '3'] + args = parser.parse_args(args) + + self.assertEqual(args.command, 'test') + self.assertListEqual(args.arguments, ['2', '--option', '3']) + class AliasPluginTestCase(unittest.TestCase): diff --git a/tests/test_pretty.py b/tests/test_pretty.py index 0c09211..b73edfd 100644 --- a/tests/test_pretty.py +++ b/tests/test_pretty.py @@ -28,5 +28,15 @@ class TestPretty(unittest.TestCase): line = 'Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web."' self.assertEqual(color.undye(pretty.bib_oneliner(bibdata['Page99'])), line) + def test_oneliner_max_authors(self): + decoder = endecoder.EnDecoder() + bibdata = decoder.decode_bibdata(bibtex_raw0) + for max_authors in [1, 2, 3]: + line = 'Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999)' + self.assertEqual(color.undye(pretty.bib_oneliner(bibdata['Page99'], max_authors=max_authors)), line) + for max_authors in [-1, 0, 4, 5, 10]: + line = 'Page, Lawrence and Brin, Sergey and Motwani, Rajeev and Winograd, Terry "The PageRank Citation Ranking: Bringing Order to the Web." (1999)' + self.assertEqual(color.undye(pretty.bib_oneliner(bibdata['Page99'], max_authors=max_authors)), line) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_queries.py b/tests/test_queries.py index 207c4fc..a0ca6c7 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -200,8 +200,8 @@ class TestFilterPaper(unittest.TestCase): def test_latex_enc(self): latexenc_paper = doe_paper.deepcopy() - latexenc_paper.bibentry['Doe2013']['title'] = "{E}l Ni{\~n}o" - latexenc_paper.bibentry['Doe2013']['author'][0] = "Erd\H{o}s, Paul" + latexenc_paper.bibentry['Doe2013']['title'] = r"{E}l Ni{\~n}o" + latexenc_paper.bibentry['Doe2013']['author'][0] = r"Erd\H{o}s, Paul" self.assertTrue(get_paper_filter(['title:El'])(latexenc_paper)) self.assertTrue(get_paper_filter(['title:Niño'])(latexenc_paper)) self.assertTrue(get_paper_filter(['author:erdős'])(latexenc_paper)) @@ -209,12 +209,12 @@ class TestFilterPaper(unittest.TestCase): def test_normalize_unicode(self): latexenc_paper = doe_paper.deepcopy() - latexenc_paper.bibentry['Doe2013']['title'] = "{E}l Ni{\~n}o" + latexenc_paper.bibentry['Doe2013']['title'] = r"{E}l Ni{\~n}o" self.assertTrue(get_paper_filter(['title:Nin\u0303o'])(latexenc_paper)) def test_strict(self): latexenc_paper = doe_paper.deepcopy() - latexenc_paper.bibentry['Doe2013']['title'] = "El Ni{\~n}o" + latexenc_paper.bibentry['Doe2013']['title'] = r"El Ni{\~n}o" self.assertFalse(get_paper_filter( ['title:Nin\u0303o'], strict=True)(latexenc_paper)) diff --git a/tests/test_usecase.py b/tests/test_usecase.py index 348366e..9219657 100644 --- a/tests/test_usecase.py +++ b/tests/test_usecase.py @@ -12,11 +12,13 @@ import ddt import certifi import mock from pyfakefs.fake_filesystem import FakeFileOpen +import pytest import dotdot import fake_env import mock_requests + from pubs import pubs_cmd, color, content, uis, p3, endecoder from pubs.config import conf @@ -25,7 +27,7 @@ import fixtures # makes the tests very noisy -PRINT_OUTPUT = False +PRINT_OUTPUT = True #False CAPTURE_OUTPUT = True @@ -118,14 +120,12 @@ class CommandTestCase(fake_env.TestFakeFs): input.as_global() try: if capture_output: - actual_out = self.execute_cmdsplit( - actual_cmd.split(), expected_out, expected_err) + actual_out = self.execute_cmd_capture(actual_cmd.split(), expected_out, expected_err) outs.append(color.undye(actual_out)) else: pubs_cmd.execute(actual_cmd.split()) except fake_env.FakeInput.UnexpectedInput: - self.fail('Unexpected input asked by command: {}.'.format( - actual_cmd)) + self.fail('Unexpected input asked by command: {}.'.format(actual_cmd)) return outs except SystemExit as exc: exc_class, exc, tb = sys.exc_info() @@ -145,27 +145,39 @@ class CommandTestCase(fake_env.TestFakeFs): pass return s - def execute_cmdsplit(self, actual_cmdlist, expected_out, expected_err): - """Run a single command, which has been split into a list containing cmd and args""" - capture_wrap = fake_env.capture(pubs_cmd.execute, - verbose=PRINT_OUTPUT) - _, stdout, stderr = capture_wrap(actual_cmdlist) - actual_out = self.normalize(stdout) - actual_err = self.normalize(stderr) + def execute_cmd_capture(self, cmd, expected_out, expected_err): + """Run a single command, captures the output and and stderr and compare it to the expected ones""" + sys_stdout, sys_stderr = sys.stdout, sys.stderr + sys.stdout = p3._fake_stdio(additional_out=sys_stdout if PRINT_OUTPUT else None) + sys.stderr = p3._fake_stdio(additional_out=sys_stderr if PRINT_OUTPUT else None) + + try: + pubs_cmd.execute(cmd) + finally: + # capturing output even if exception was raised. + self.captured_stdout = self.normalize(p3._get_fake_stdio_ucontent(sys.stdout)) + self.captured_stderr = self.normalize(p3._get_fake_stdio_ucontent(sys.stderr)) + sys.stderr, sys.stdout = sys_stderr, sys_stdout + if expected_out is not None: - self.assertEqual(p3.u_maybe(actual_out), p3.u_maybe(expected_out)) + self.assertEqual(p3.u_maybe(self.captured_stdout), p3.u_maybe(expected_out)) if expected_err is not None: - self.assertEqual(p3.u_maybe(actual_err), p3.u_maybe(expected_err)) - return actual_out + self.assertEqual(p3.u_maybe(self.captured_stderr), p3.u_maybe(expected_err)) + return self.captured_stdout + + def update_config(self, config_update, path=None): + """Allow to set the config parameters. Must have done a `pubs init` beforehand.""" + if path is None: + path = self.default_conf_path + cfg = conf.load_conf(path=path) + for section, section_update in config_update.items(): + cfg[section].update(section_update) + conf.save_conf(cfg, path=path) - def tearDown(self): - pass class DataCommandTestCase(CommandTestCase): - """Abstract TestCase intializing the fake filesystem and - copying fake data. - """ + """Abstract TestCase intializing the fake filesystem and copying fake data.""" def setUp(self, nsec_stat=True): super(DataCommandTestCase, self).setUp(nsec_stat=nsec_stat) @@ -182,8 +194,7 @@ class DataCommandTestCase(CommandTestCase): class URLContentTestCase(DataCommandTestCase): - """Mocks access to online files by using files in data directory. - """ + """Mocks access to online files by using files in data directory.""" def setUp(self): super(URLContentTestCase, self).setUp() @@ -209,6 +220,8 @@ class URLContentTestCase(DataCommandTestCase): content.url_exists = self._original_url_exist + + # Actual tests class TestAlone(CommandTestCase): @@ -615,6 +628,7 @@ class TestTag(DataCommandTestCase): with self.assertRaises(FakeSystemExit): self.execute_cmds(cmds) + class TestURL(DataCommandTestCase): def setUp(self): @@ -720,7 +734,7 @@ class TestUsecase(DataCommandTestCase): def test_first(self): correct = ['Initializing pubs in /paper_first\n', 'added to pubs:\n[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) \n' - 'data/pagerank.pdf was copied to the pubs repository.\n', + 'data/pagerank.pdf was copied to /paper_first/doc/Page99.pdf inside the pubs repository.\n', '[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) [pdf] \n', '\n', '', @@ -1072,14 +1086,31 @@ class TestUsecase(DataCommandTestCase): target_path=os.path.join('data', 'no-ext')) correct = ['Initializing pubs in /pubs\n', 'added to pubs:\n[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) \n' - 'data/no-ext was copied to the pubs repository.\n', + 'data/no-ext was copied to /pubs/doc/Page99 inside the pubs repository.\n', '[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) [NOEXT] \n', ] cmds = ['pubs init -p /pubs', 'pubs add -d data/no-ext data/pagerank.bib', 'pubs list', ] - self.assertEqual(correct, self.execute_cmds(cmds, capture_output=True)) + actual = self.execute_cmds(cmds, capture_output=True) + self.assertEqual(correct, actual) + + def test_add_non_standard(self): + """Test that non-standard bibtex are correctly added""" + self.fs.add_real_directory(os.path.join(self.rootpath, 'data_non_standard'), read_only=False) + correct = ['Initializing pubs in /pubs\n', + 'added to pubs:\n[Geometric_phases] "Geometric phases in physics" (1989) \n', + 'added to pubs:\n[hadoop] Foundation, Apache Software "Hadoop" \n', + ] + cmds = ['pubs init -p /pubs', + 'pubs add data_non_standard/non_standard_collection.bib', + 'pubs add data_non_standard/non_standard_software.bib', + # 'pubs list', + ] + actual = self.execute_cmds(cmds, capture_output=True) + self.assertEqual(correct, actual) + @mock.patch('pubs.apis.requests.get', side_effect=mock_requests.mock_requests_get) def test_readme(self, reqget): @@ -1089,7 +1120,7 @@ class TestUsecase(DataCommandTestCase): self.fs.add_real_file(os.path.join(self.rootpath, 'data/pagerank.pdf'), target_path='data/Knuth1995.pdf') cmds = ['pubs init', - 'pubs import data/collection.bib', + 'pubs import data/three_articles.bib', 'pubs add data/pagerank.bib -d data/pagerank.pdf', #'pubs add -D 10.1007/s00422-012-0514-6 -d data/pagerank.pdf', 'pubs add -I 978-0822324669 -d data/oyama2000the.pdf', @@ -1100,6 +1131,21 @@ class TestUsecase(DataCommandTestCase): self.execute_cmds(cmds, capture_output=True) # self.assertEqual(correct, self.execute_cmds(cmds, capture_output=True)) + def test_ambiguous_citekey(self): + cmds = ['pubs init', + 'pubs add data/pagerank.bib', + 'pubs add data/pagerank.bib', # now we have Page99 and Page99a + 'pubs edit Page', + ] + output = '\n'.join(["error: Be more specific; 'Page' matches multiples citekeys:", + " [Page99] Page, Lawrence et al. \"The PageRank Citation Ranking: Bringing Order to the Web.\" (1999) ", + " [Page99a] Page, Lawrence et al. \"The PageRank Citation Ranking: Bringing Order to the Web.\" (1999) \n"]) + + with self.assertRaises(FakeSystemExit): + self.execute_cmds(cmds) + + self.assertEqual(self.captured_stderr, output) + @ddt.ddt @@ -1145,5 +1191,23 @@ class TestCache(DataCommandTestCase): self.assertEqual(line1, out[4]) +class TestConfigChange(DataCommandTestCase): + + def test_max_authors_default(self): + line_al = '[Page99] Page, Lawrence et al. "The PageRank Citation Ranking: Bringing Order to the Web." (1999) \n' + line_full = '[Page99] Page, Lawrence and Brin, Sergey and Motwani, Rajeev and Winograd, Terry "The PageRank Citation Ranking: Bringing Order to the Web." (1999) \n' + + self.execute_cmds(['pubs init', 'pubs add data/pagerank.bib']) + + for max_authors in [1, 2, 3]: + self.update_config({'main': {'max_authors': max_authors}}) + self.execute_cmds([('pubs list', None, line_al, None)]) + + for max_authors in [-1, 0, 4, 5, 10]: + self.update_config({'main': {'max_authors': max_authors}}) + self.execute_cmds([('pubs list', None, line_full, None)]) + + + if __name__ == '__main__': unittest.main(verbosity=2)