all: new data, fixes, invenio 3.2 and idref

* Fixes viaf-pid in mef records. * Closes rero#30. * Closes rero/rero-ils#555. * Closes rero/rero-ils#657. * Tests for duplicated pids. * Updates viaf, rero, gnd data. * Adapts format transformation (024 $0). * Updates invenio to version 3.2.1 * Implements idRef records. * Implements idRef OAI harvesting. * Implements GND OAI harvesting. * Uses seperat tables for authorities. Co-Authored-by: Peter Weber <[email protected]>
rerowep · Apr 1, 2020 · 96af5d2 · 96af5d2
1 parent d0630ae
commit 96af5d2
Show file tree

Hide file tree

Showing 130 changed files with 9,282 additions and 2,301 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -39,6 +39,8 @@ known_first_party = rero_mef
 multi_line_output = 2
 default_section = THIRDPARTY
 skip = .eggs
+# Needed for isort since version 4.3.10
+reverse_relative = true
 
 # RST files (used by sphinx)
 [*.rst]

diff --git a/.travis.yml b/.travis.yml
@@ -29,7 +29,8 @@ addons:
 # See https://github.com/travis-ci/travis-ci/issues/8836
 sudo: true
 
-dist: trusty # Chrome driver fails if not trusty dist
+# Chrome driver fails if not trusty dist
+dist: trusty
 
 notifications:
   email: # sent to the committer and the author
@@ -54,39 +55,42 @@ env:
     # Enable end-to-end tests
     - E2E=no
   matrix:
-    - REQUIREMENTS=prod
-    # - REQUIREMENTS=prod E2E=yes
-    - REQUIREMENTS=qa
+    - REQUIREMENTS="--deploy" E2E=yes
+    - REQUIREMENTS=""
 
 python:
   - "3.6"
 
 before_install:
   # Stop default travis services
-  - "sudo service mysql stop"
-  - "sudo service postgresql stop"
+  - sudo service mysql stop
+  - sudo service postgresql stop
+  # start chromedriver
+  - PATH=$PATH:$HOME/webdrivers
+  - if [ ${E2E} == 'yes' ] && [ ! -f $HOME/webdrivers/chromedriver ]; then wget https://chromedriver.storage.googleapis.com/2.31/chromedriver_linux64.zip -P $HOME/webdrivers; unzip -d $HOME/webdrivers $HOME/webdrivers/chromedriver_linux64.zip; fi  # noqa
   # Start docker services
-  - "docker-compose up -d"
-  # Chrome webdriver for Selenium
-  - "PATH=$PATH:$HOME/webdrivers"
-  - "if [ ${E2E} == "yes" ] && [ ! -f $HOME/webdrivers/chromedriver ]; then wget https://chromedriver.storage.googleapis.com/2.31/chromedriver_linux64.zip -P $HOME/webdrivers; unzip -d $HOME/webdrivers $HOME/webdrivers/chromedriver_linux64.zip; fi"
-  - "nvm install 6; nvm use 6"
-  - "travis_retry pip install --upgrade setuptools py pipenv==2018.10.9"
-  - "travis_retry pip install twine wheel coveralls"
+  - docker-compose up -d
+  # - LATEST_CHROMEDRIVER_VERSION=`curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE"`
+  # - curl "https://chromedriver.storage.googleapis.com/${LATEST_CHROMEDRIVER_VERSION}/chromedriver_linux64.zip" -O
+  # - unzip chromedriver_linux64.zip -d ~/bin
+  - nvm install 8; nvm use 8
+  - travis_retry pip install --upgrade setuptools py pipenv==2018.10.9
+  - travis_retry pip install twine wheel coveralls
   # patch default version of travis which is mark as unsave by pipenv check
-  - "travis_retry pip install --upgrade numpy"
+  - travis_retry pip install --upgrade numpy six
+  - pipenv run pip list
 
 install:
-  - "./scripts/bootstrap --ci"
+  - ./scripts/bootstrap --ci $REQUIREMENTS
 
 before_script:
   # https://docs.travis-ci.com/user/gui-and-headless-browsers/
-  - "export DISPLAY=:99.0"
-  - "if [ ${E2E} == 'yes' ]; then sh -e /etc/init.d/xvfb start; sleep 3; fi"
   # Allow services running inside docker to start
-  - "./docker/wait-for-services.sh"
+  - ./docker/wait-for-services.sh
 
 script:
+  - export DISPLAY=:99.0
+  - if [ ${E2E} == 'yes' ]; then sh -e /etc/init.d/xvfb start; sleep 3; fi
   - ./run-tests.sh # E2E tests are skipped if E2E env is set to 'no'
 
 after_success:

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -56,6 +56,7 @@ include Pipfile
 include babel.ini
 include pytest.ini
 recursive-include data *.json
+recursive-include data *.yml
 recursive-include docs *.bat
 recursive-include docs *.py
 recursive-include docs *.rst
@@ -70,4 +71,3 @@ recursive-include tests *.xml
 recursive-include rero_mef *.html
 recursive-include rero_mef *.png
 recursive-include rero_mef *.scss
-
diff --git a/Pipfile b/Pipfile
@@ -4,47 +4,65 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-rero-mef = {editable = true, path = "."}
+# Default from Invenio
 Babel = ">=2.4.0"
 Flask-BabelEx = ">=0.9.3"
-ijson = ">=2.3"
-invenio = {version = "~=3.1.0",extras = ["base", "postgresql", "auth", "metadata", "elasticsearch6"]}
+## Third party invenio modules used by RERO ILS
+invenio-oaiharvester = {editable = true, ref = "v1.0.0a4", git = "https://github.com/inveniosoftware/invenio-oaiharvester.git"}
+## Invenio base modules used by RERO ILS
+# same as invenio metadata extras without invenio-search-ui
+invenio-indexer = ">=1.1.1,<1.2.0"
+invenio-jsonschemas = ">=1.0.1,<1.1.0"
+invenio-oaiserver = ">=1.1.1,<1.2.0"
+invenio-pidstore = ">=1.1.0,<1.2.0"
+invenio-records-rest = ">=1.6.4,<1.7.0"
+invenio-records-ui = ">=1.0.1,<1.1.0"
+invenio-records = ">=1.3.0,<1.4.0"
+## Default from Invenio
+invenio = {version = "==3.2.1", extras = ["base", "postgresql", "auth", "elasticsearch6"]}
 uwsgi = ">=2.0"
-pymarc = "*"
-prompt-toolkit = "*"
-pipenv = "*"
+uwsgitop = ">=0.11"
+uwsgi-tools = ">=1.1.1"
+## RERO ILS specific python modules
 psutil = "*"
-invenio-admin = "*"
-idna = "==2.7"
-lxml = "==4.2.5"
-amqp = "*"
-# TODO: causes datetime serialize error
-celery = "<4.3.0"
+ijson = ">=2.5.1"
+pymarc = ">=3.1.10"
+PyYAML = ">=5.3"
+## Additionnal constraints on python modules
+# TODO: remove when https://github.com/inveniosoftware/invenio-accounts/issues/306 will be solved
+invenio-accounts = "<=1.1.2"
+# self install
+rero-mef = {editable = true, path = "."}
+## Deployment
+raven = "*"
+lxml = ">=3.5.0,<4.2.6"
 
 [dev-packages]
-rero-mef = {editable = true, path = "."}
+## Python packages development dependencies (order matters)
+#----------------------------------------------------------
+## Default from Invenio
 Flask-Debugtoolbar = ">=0.10.1"
 Sphinx = ">=1.5.1"
 check-manifest = ">=0.35"
-coverage = ">=4.4.1"
-isort = ">=4.3.5,<4.3.10"
+coverage = ">=4.5.3"
+isort = ">=4.3"
 mock = ">=2.0.0"
 marshmallow = ">=2.15.1,<3.0.0"
-pydocstyle = ">=2.0.0"
-pytest = ">=3.3.1"
-pytest-cov = ">=2.5.1"
-pytest-invenio = ">=1.1.0"
+pydocstyle = ">=3.0.0"
+pytest = ">=4.6.4"
+pytest-cov = ">=2.7.1"
+pytest-invenio = ">=1.2.1,<1.3.0"
 pytest-mock = ">=1.6.0"
 pytest-pep8 = ">=1.0.6"
 pytest-random-order = ">=0.5.4"
 pytest-runner = ">=3.0.0,<5"
-pymarc = ">=3.1.10"
-urllib3 = "<1.25,>=1.21.1"
-responses = "*"
+## RERO ILS specific python packages
+autoflake = ">=1.3.1"
 
 [requires]
 python_version = "3.6"
 
 [scripts]
 test = "python setup.py test"
 build_sphinx = "python setup.py build_sphinx"
+dev = "pytest --no-cov -vs tests"