diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 6dd79e1c1c..b5b170e407 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -35,20 +35,21 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.10"] - session: ["doctest", "gallery", "linkcheck"] + python-version: ["310"] + session: ["docs-tests", "docs-linkcheck", "gallery_tests"] include: - os: "ubuntu-latest" - python-version: "3.10" + python-version: "310" session: "tests" - coverage: "--coverage" + posargs: "--cov=lib/iris --cov-report=xml" - os: "ubuntu-latest" - python-version: "3.9" + python-version: "39" session: "tests" - os: "ubuntu-latest" - python-version: "3.8" + python-version: "38" session: "tests" + env: IRIS_TEST_DATA_VERSION: "2.19" ENV_NAME: "ci-tests" @@ -63,7 +64,7 @@ jobs: CACHE_WEEKS: 2 run: | echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV} - echo "LOCK_FILE=requirements/ci/nox.lock/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV} + echo "LOCK_FILE=requirements/ci/locks/py$(echo ${{ matrix.python-version }})-linux-64.lock" >> ${GITHUB_ENV} - name: "data cache" uses: ./.github/workflows/composite/iris-data-cache @@ -91,10 +92,10 @@ jobs: - name: "conda environment cache" uses: ./.github/workflows/composite/conda-env-cache with: - cache_build: 0 + cache_build: 1 cache_period: ${{ env.CACHE_PERIOD }} env_name: ${{ env.ENV_NAME }} - install_packages: "cartopy nox pip" + install_packages: "cartopy tox'<4'" - name: "conda info" run: | @@ -108,8 +109,8 @@ jobs: cache_period: ${{ env.CACHE_PERIOD }} env_name: ${{ env.ENV_NAME }} - - name: "nox cache" - uses: ./.github/workflows/composite/nox-cache + - name: "tox cache" + uses: ./.github/workflows/composite/tox-cache with: cache_build: 1 env_name: ${{ env.ENV_NAME }} @@ -134,11 +135,9 @@ jobs: cat ${MPL_RC} - name: "iris ${{ matrix.session }}" - env: - PY_VER: ${{ matrix.python-version }} run: | - nox --session ${{ matrix.session }} -- --verbose ${{ matrix.coverage }} + tox -e py${{ matrix.python-version }}-${{ matrix.session }} -- ${{ matrix.posargs }} - name: Upload coverage report uses: codecov/codecov-action@v3 - if: ${{ matrix.coverage }} + if: contains(${{ matrix.posargs }}, "--cov=lib/iris --cov-report=xml") diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index 835930e065..7a54d9f822 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -54,7 +54,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["38", "39", "310"] session: ["wheel"] env: ENV_NAME: "ci-wheels" @@ -74,7 +74,7 @@ jobs: CACHE_WEEKS: 2 run: | echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV} - echo "LOCK_FILE=requirements/ci/nox.lock/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV} + echo "LOCK_FILE=requirements/ci/locks/py$(echo ${{ matrix.python-version }})-linux-64.lock" >> ${GITHUB_ENV} - name: "conda package cache" uses: ./.github/workflows/composite/conda-pkg-cache @@ -95,23 +95,21 @@ jobs: - name: "conda environment cache" uses: ./.github/workflows/composite/conda-env-cache with: - cache_build: 0 + cache_build: 1 cache_period: ${{ env.CACHE_PERIOD }} env_name: ${{ env.ENV_NAME }} - install_packages: "nox pip" + install_packages: "tox'<4' pip" - - name: "nox cache" - uses: ./.github/workflows/composite/nox-cache + - name: "tox cache" + uses: ./.github/workflows/composite/tox-cache with: cache_build: 0 env_name: ${{ env.ENV_NAME }} lock_file: ${{ env.LOCK_FILE }} - - name: "nox install and test wheel" - env: - PY_VER: ${{ matrix.python-version }} + - name: "tox install and test wheel" run: | - nox --session ${{ matrix.session }} -- --verbose + tox -e py${{ matrix.python-version }}-${{ matrix.session }} show-artifacts: needs: build diff --git a/.github/workflows/composite/nox-cache/action.yml b/.github/workflows/composite/tox-cache/action.yml similarity index 65% rename from .github/workflows/composite/nox-cache/action.yml rename to .github/workflows/composite/tox-cache/action.yml index 468dd22d81..48962f9d51 100644 --- a/.github/workflows/composite/nox-cache/action.yml +++ b/.github/workflows/composite/tox-cache/action.yml @@ -1,9 +1,9 @@ -name: "nox cache" -description: "cache the nox test environments" +name: "tox cache" +description: "cache the tox test environments" inputs: cache_build: - description: "nox cache build number" + description: "tox cache build number" required: false default: "0" env_name: @@ -18,5 +18,5 @@ runs: steps: - uses: actions/cache@v3 with: - path: ${{ github.workspace }}/.nox - key: ${{ runner.os }}-nox-${{ inputs.env_name }}-s${{ matrix.session }}-py${{ matrix.python-version }}-b${{ inputs.cache_build }}-${{ hashFiles(inputs.lock_file) }} + path: ${{ github.workspace }}/.tox + key: ${{ runner.os }}-tox-${{ inputs.env_name }}-s${{ matrix.session }}-py${{ matrix.python-version }}-b${{ inputs.cache_build }}-${{ hashFiles(inputs.lock_file) }} diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index 86f6bd37c7..a2f9038d66 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -48,27 +48,29 @@ def _prep_data_gen_env() -> None: """ root_dir = Path(__file__).parents[1] - python_version = "3.10" + python_version = "310" data_gen_var = "DATA_GEN_PYTHON" if data_gen_var in environ: print("Using existing data generation environment.") else: print("Setting up the data generation environment ...") - # Get Nox to build an environment for the `tests` session, but don't - # run the session. Will re-use a cached environment if appropriate. + # Get tox to build an environment. It will re-use a cached environment + # if appropriate. subprocess.run( [ - "nox", - f"--noxfile={root_dir / 'noxfile.py'}", - "--session=tests", - "--install-only", - f"--python={python_version}", + "tox", + "-c", + root_dir, + "-e", + f"py{python_version}", ] ) # Find the environment built above, set it to be the data generation # environment. data_gen_python = next( - (root_dir / ".nox").rglob(f"tests*/bin/python{python_version}") + (root_dir / ".tox").rglob( + f"py{python_version}/bin/python{python_version[:1]}.{python_version[1:]}" + ) ).resolve() environ[data_gen_var] = str(data_gen_python) @@ -98,7 +100,7 @@ def _prep_data_gen_env() -> None: def _setup_common() -> None: _check_requirements("asv") - _check_requirements("nox") + _check_requirements("tox") _prep_data_gen_env() @@ -140,7 +142,7 @@ def _asv_compare(*commits: str, overnight_mode: bool = False) -> None: class _SubParserGenerator(ABC): - """Convenience for holding all the necessary argparse info in 1 place.""" + """Convenience for holding all the necessary argparse info in one place.""" name: str = NotImplemented description: str = NotImplemented diff --git a/docs/src/common_links.inc b/docs/src/common_links.inc index 55461369ff..1f588e3ec0 100644 --- a/docs/src/common_links.inc +++ b/docs/src/common_links.inc @@ -35,6 +35,7 @@ .. _scitools-iris: https://pypi.org/project/scitools-iris/ .. _sphinx: https://www.sphinx-doc.org/en/master/ .. _test-iris-imagehash: https://github.com/SciTools/test-iris-imagehash +.. _tox: https://tox.readthedocs.io/en/latest/ .. _using git: https://docs.github.com/en/github/using-git .. _requirements: https://github.com/SciTools/iris/tree/main/requirements .. _CF-UGRID: https://ugrid-conventions.github.io/ugrid-conventions/ diff --git a/docs/src/developers_guide/contributing_running_tests.rst b/docs/src/developers_guide/contributing_running_tests.rst index f60cedba05..58fd342f05 100644 --- a/docs/src/developers_guide/contributing_running_tests.rst +++ b/docs/src/developers_guide/contributing_running_tests.rst @@ -12,8 +12,8 @@ There are two options for running the tests: the tests or use ``python`` interactively to investigate any issues. See :ref:`test manual env`. -* Use ``nox``. This will automatically generate an environment and run test - sessions consistent with our GitHub continuous integration. See :ref:`using nox`. +* Use ``tox``. This will automatically generate an environment and run test + sessions consistent with our GitHub continuous integration. See :ref:`using tox`. .. _test manual env: @@ -101,97 +101,84 @@ using the commands ``python test_mapping.py -h`` or ``python test_mapping.py --help``. .. tip:: A useful command line option to use is ``-d``. This will display - matplotlib_ figures as the tests are run. For example:: + `matplotlib_` figures as the tests are run. For example:: python test_mapping.py -d -.. _using nox: +.. _using tox: -Using Nox for Testing Iris +Using tox for Testing Iris ========================== -The `nox`_ tool has for adopted for automated testing on `Iris GitHub Actions`_ +The `tox`_ tool has been adopted for automated testing on `Iris GitHub Actions`_ and also locally on the command-line for developers. -`nox`_ is similar to `tox`_, but instead leverages the expressiveness and power of a Python -configuration file rather than an `.ini` style file. As with `tox`_, `nox`_ can use `virtualenv`_ -to create isolated Python environments, but in addition also supports `conda`_ as a testing -environment backend. +`tox`_ uses `virtualenv`_ to create isolated Python environments and is +configured within a tox.ini file. - -Where is Nox Used? +Where is tox Used? ------------------ -Iris uses `nox`_ as a convenience to fully automate the process of executing the Iris tests, but also -automates the process of: +Iris uses `tox`_ as a convenience to fully automate the process of: -* building the documentation and executing the doc-tests -* building the documentation gallery +* executing the Iris tests, +* building the documentation, +* executing the doc-tests, +* executing the gallery tests, and * running the documentation URL link check -You can perform all of these tasks manually yourself, however the onus is on you to first ensure -that all of the required package dependencies are installed and available in the testing environment. +You can perform all of these tasks manually yourself, however the onus is on you +to first ensure that all of the required package dependencies are installed and +available in the testing environment. -`Nox`_ has been configured to automatically do this for you, and provides a means to easily replicate -the remote testing behaviour of `Iris GitHub Actions`_ locally for the developer. +`tox`_ has been configured to automatically do this for you, and provides a +means to easily replicate the remote testing behaviour of `Iris GitHub Actions`_ +locally for the developer. -Installing Nox +Installing tox -------------- -We recommend installing `nox`_ using `conda`_. To install `nox`_ in a separate `conda`_ environment:: +We recommend installing `tox`_ using `conda`_. To install `tox`_ in a separate +`conda`_ environment:: - conda create -n nox -c conda-forge nox - conda activate nox + conda create -n tox -c conda-forge tox + conda activate tox -To install `nox`_ in an existing active `conda`_ environment:: +To install `tox`_ in an existing active `conda`_ environment:: - conda install -c conda-forge nox + conda install -c conda-forge tox -The `nox`_ package is also available on PyPI, however `nox`_ has been configured to use the `conda`_ -backend for Iris, so an installation of `conda`_ must always be available. +The `tox`_ package is also available on PyPI, however `tox`_ has been configured +to use `conda`_ to create the testing environments and so an installation of +`conda`_ must always be available. -Testing with Nox +Testing with tox ---------------- -The `nox`_ configuration file `noxfile.py` is available in the root ``iris`` project directory, and -defines all the `nox`_ sessions (i.e., tasks) that may be performed. `nox`_ must always be executed -from the ``iris`` root directory. - -To list the configured `nox`_ sessions for Iris:: - - nox --list - -To run the Iris tests for all configured versions of Python:: +The `tox`_ configuration file `tox.ini` is available in the root ``iris`` +project directory, and defines all the `tox`_ test environments (i.e., tasks) +that may be performed. `tox`_ must always be executed from the ``iris`` root +directory. - nox --session tests +To list the configured `tox`_ sessions for Iris:: -To build the Iris documentation specifically for Python 3.7:: + tox --listenvs-all - nox --session doctest-3.7 +To run the Iris tests for Python 3.10:: -To run all the Iris `nox`_ sessions:: + tox -e py310-tests - nox +To build the Iris documentation specifically for Python 3.10:: -For further `nox`_ command-line options:: - - nox --help - -.. tip:: - For `nox`_ sessions that use the `conda`_ backend, you can use the ``-v`` or ``--verbose`` - flag to display the `nox`_ `conda`_ environment package details and environment info. - For example:: + tox -e py310-docs - nox --session tests -- --verbose +For further `tox`_ command-line options:: + tox --help -.. note:: `nox`_ will cache its testing environments in the `.nox` root ``iris`` project directory. +.. note:: `tox`_ will cache its testing environments in the `.tox` root ``iris`` project directory. -.. _setuptools: https://setuptools.readthedocs.io/en/latest/ -.. _tox: https://tox.readthedocs.io/en/latest/ -.. _virtualenv: https://virtualenv.pypa.io/en/latest/ -.. _PyPI: https://pypi.org/project/nox/ -.. _v41.5.0: https://setuptools.readthedocs.io/en/latest/history.html#v41-5-0 +.. _virtualenv: https://virtualenv.pypa.io/en/latest/ \ No newline at end of file diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 904cb106ed..ade199708e 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -49,6 +49,10 @@ This document explains the changes made to Iris for this release #. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) added :func:`iris.plot.hist` and :func:`iris.quickplot.hist`. (:pull:`5189`) +#. `@tinyendian`_ edited :func:`~iris.analysis.cartography.rotate_winds` to + enable lazy computation of rotated wind vector components (:issue:`4934`, + :pull:`4972`) + 🐛 Bugs Fixed ============= @@ -132,8 +136,8 @@ This document explains the changes made to Iris for this release (:pull:`5101`) #. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) added coverage testing. This - can be enabled by using the "--coverage" flag when running the tests with - nox i.e. ``nox --session tests -- --coverage``. (:pull:`4765`) + can be enabled when running with tox i.e. ``tox -e py310-tests -- --cov``. + (:pull:`4765`) #. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) removed the ``--coding-tests`` option from Iris' test runner. (:pull:`4765`) @@ -147,6 +151,13 @@ This document explains the changes made to Iris for this release #. `@trexfeathers`_ moved the benchmark runner conveniences from ``noxfile.py`` to a dedicated ``benchmarks/bm_runner.py``. (:pull:`5215`) +#. `@bjlittle`_ follow-up to :pull:`4972`, enforced ``dask>=2022.09.0`` minimum + pin for first use of `dask.array.ma.empty_like`_ and replaced `@tinyendian`_ + workaround. (:pull:`5225`) + +# `@lbdreyer`_ replaced nox with tox as the automated test runner. + (:pull:`5184`) + .. comment Whatsnew author names (@github name) in alphabetical order. Note that, @@ -156,9 +167,12 @@ This document explains the changes made to Iris for this release .. _@ed-hawkins: https://github.com/ed-hawkins .. _@scottrobinson02: https://github.com/scottrobinson02 .. _@agriyakhetarpal: https://github.com/agriyakhetarpal +.. _@tinyendian: https://github.com/tinyendian + .. comment Whatsnew resources in alphabetical order: .. _#ShowYourStripes: https://showyourstripes.info/s/globe/ .. _README.md: https://github.com/SciTools/iris#----- +.. _dask.array.ma.empty_like: https://docs.dask.org/en/stable/generated/dask.array.ma.empty_like.html diff --git a/lib/iris/analysis/cartography.py b/lib/iris/analysis/cartography.py index a8e90a63ad..5b11495d5a 100644 --- a/lib/iris/analysis/cartography.py +++ b/lib/iris/analysis/cartography.py @@ -15,6 +15,7 @@ import cartopy.crs as ccrs import cartopy.img_transform import cf_units +import dask.array as da import numpy as np import numpy.ma as ma @@ -1206,9 +1207,15 @@ def rotate_winds(u_cube, v_cube, target_cs): x = x.transpose() y = y.transpose() - # Create resulting cubes. - ut_cube = u_cube.copy() - vt_cube = v_cube.copy() + # Create resulting cubes - produce lazy output data if at least + # one input cube has lazy data + lazy_output = u_cube.has_lazy_data() or v_cube.has_lazy_data() + if lazy_output: + ut_cube = u_cube.copy(data=da.empty_like(u_cube.lazy_data())) + vt_cube = v_cube.copy(data=da.empty_like(v_cube.lazy_data())) + else: + ut_cube = u_cube.copy() + vt_cube = v_cube.copy() ut_cube.rename("transformed_{}".format(u_cube.name())) vt_cube.rename("transformed_{}".format(v_cube.name())) @@ -1236,8 +1243,12 @@ def rotate_winds(u_cube, v_cube, target_cs): apply_mask = mask.any() if apply_mask: # Make masked arrays to accept masking. - ut_cube.data = ma.asanyarray(ut_cube.data) - vt_cube.data = ma.asanyarray(vt_cube.data) + if lazy_output: + ut_cube = ut_cube.copy(data=da.ma.empty_like(ut_cube.core_data())) + vt_cube = vt_cube.copy(data=da.ma.empty_like(vt_cube.core_data())) + else: + ut_cube.data = ma.asanyarray(ut_cube.data) + vt_cube.data = ma.asanyarray(vt_cube.data) # Project vectors with u, v components one horiz slice at a time and # insert into the resulting cubes. @@ -1250,16 +1261,20 @@ def rotate_winds(u_cube, v_cube, target_cs): for dim in dims: index[dim] = slice(None, None) index = tuple(index) - u = u_cube.data[index] - v = v_cube.data[index] + u = u_cube.core_data()[index] + v = v_cube.core_data()[index] ut, vt = _transform_distance_vectors(u, v, ds, dx2, dy2) if apply_mask: - ut = ma.asanyarray(ut) - ut[mask] = ma.masked - vt = ma.asanyarray(vt) - vt[mask] = ma.masked - ut_cube.data[index] = ut - vt_cube.data[index] = vt + if lazy_output: + ut = da.ma.masked_array(ut, mask=mask) + vt = da.ma.masked_array(vt, mask=mask) + else: + ut = ma.asanyarray(ut) + ut[mask] = ma.masked + vt = ma.asanyarray(vt) + vt[mask] = ma.masked + ut_cube.core_data()[index] = ut + vt_cube.core_data()[index] = vt # Calculate new coords of locations in target coordinate system. xyz_tran = target_crs.transform_points(src_crs, x, y) diff --git a/lib/iris/tests/test_coding_standards.py b/lib/iris/tests/test_coding_standards.py index e3786f5cd5..662dd7f352 100644 --- a/lib/iris/tests/test_coding_standards.py +++ b/lib/iris/tests/test_coding_standards.py @@ -71,8 +71,8 @@ def test_python_versions(): This test is designed to fail whenever Iris' supported Python versions are updated, insisting that versions are updated EVERYWHERE in-sync. """ - latest_supported = "3.10" - all_supported = ["3.8", "3.9", latest_supported] + latest_supported = "310" + all_supported = ["38", "39", latest_supported] root_dir = Path(__file__).parents[3] workflows_dir = root_dir / ".github" / "workflows" @@ -81,7 +81,7 @@ def test_python_versions(): # Places that are checked: setup_cfg_file = root_dir / "setup.cfg" requirements_dir = root_dir / "requirements" - nox_file = root_dir / "noxfile.py" + tox_config_file = root_dir / "tox.ini" ci_wheels_file = workflows_dir / "ci-wheels.yml" ci_tests_file = workflows_dir / "ci-tests.yml" asv_config_file = benchmarks_dir / "asv.conf.json" @@ -92,16 +92,11 @@ def test_python_versions(): setup_cfg_file, "\n ".join( [ - "Programming Language :: Python :: " + ver + f"Programming Language :: Python :: {ver[:1]}.{ver[1:]}" for ver in all_supported ] ), ), - ( - nox_file, - "_PY_VERSIONS_ALL = [" - + ", ".join([f'"{ver}"' for ver in all_supported]), - ), ( ci_wheels_file, "python-version: [" @@ -111,16 +106,19 @@ def test_python_versions(): ci_tests_file, ( f'python-version: ["{latest_supported}"]\n' - f'{" " * 8}session: ["doctest", "gallery", "linkcheck"]' + f'{" " * 8}session: ["docs-tests", "docs-linkcheck", "gallery_tests"]' ), ), - (asv_config_file, f"PY_VER={latest_supported}"), + ( + asv_config_file, + f"PY_VER={latest_supported[:1]}.{latest_supported[1:]}", + ), (benchmark_runner_file, f'python_version = "{latest_supported}"'), ] for ver in all_supported: - req_yaml = requirements_dir / f"py{ver.replace('.', '')}.yml" - text_searches.append((req_yaml, f"- python ={ver}")) + req_yaml = requirements_dir / f"py{ver}.yml" + text_searches.append((req_yaml, f"- python ={ver[:1]}.{ver[1:]}")) text_searches.append( ( @@ -129,6 +127,13 @@ def test_python_versions(): ) ) + text_searches.append( + ( + tox_config_file, + f'{" " * 4}py{ver}: {{toxinidir}}{{/}}requirements', + ) + ) + for path, search in text_searches: assert search in path.read_text() @@ -194,7 +199,6 @@ def last_change_by_fname(): def test_license_headers(self): exclude_patterns = ( "setup.py", - "noxfile.py", "build/*", "dist/*", "docs/gallery_code/*/*.py", diff --git a/lib/iris/tests/unit/analysis/cartography/test_rotate_winds.py b/lib/iris/tests/unit/analysis/cartography/test_rotate_winds.py index 7952b3bb46..e522e4f077 100644 --- a/lib/iris/tests/unit/analysis/cartography/test_rotate_winds.py +++ b/lib/iris/tests/unit/analysis/cartography/test_rotate_winds.py @@ -511,5 +511,62 @@ def test_non_earth_semimajor_axis(self): rotate_winds(u, v, other_cs) +class TestLazyRotateWinds(tests.IrisTest): + def _compare_lazy_rotate_winds(self, masked): + # Compute wind rotation with lazy data and compare results + + # Choose target coord system that will (not) lead to masked results + if masked: + coord_sys = iris.coord_systems.OSGB() + else: + coord_sys = iris.coord_systems.GeogCS(6371229) + + u, v = uv_cubes() + + # Create deep copy of the cubes with rechunked lazy data to check if + # input data is modified, and if Dask metadata is preserved + u_lazy = u.copy(data=u.copy().lazy_data().rechunk([2, 1])) + v_lazy = v.copy(data=v.copy().lazy_data().rechunk([1, 2])) + + ut_ref, vt_ref = rotate_winds(u, v, coord_sys) + self.assertFalse(ut_ref.has_lazy_data()) + self.assertFalse(vt_ref.has_lazy_data()) + # Ensure that choice of target coordinates leads to (no) masking + self.assertTrue(ma.isMaskedArray(ut_ref.data) == masked) + + # Results are lazy if at least one component is lazy + ut, vt = rotate_winds(u_lazy, v, coord_sys) + self.assertTrue(ut.has_lazy_data()) + self.assertTrue(vt.has_lazy_data()) + self.assertTrue(ut.core_data().chunksize == (2, 1)) + self.assertArrayAllClose(ut.data, ut_ref.data, rtol=1e-5) + self.assertArrayAllClose(vt.data, vt_ref.data, rtol=1e-5) + + ut, vt = rotate_winds(u, v_lazy, coord_sys) + self.assertTrue(ut.has_lazy_data()) + self.assertTrue(vt.has_lazy_data()) + self.assertTrue(vt.core_data().chunksize == (1, 2)) + self.assertArrayAllClose(ut.data, ut_ref.data, rtol=1e-5) + self.assertArrayAllClose(vt.data, vt_ref.data, rtol=1e-5) + + ut, vt = rotate_winds(u_lazy, v_lazy, coord_sys) + self.assertTrue(ut.has_lazy_data()) + self.assertTrue(vt.has_lazy_data()) + self.assertTrue(ut.core_data().chunksize == (2, 1)) + self.assertTrue(vt.core_data().chunksize == (1, 2)) + self.assertArrayAllClose(ut.data, ut_ref.data, rtol=1e-5) + self.assertArrayAllClose(vt.data, vt_ref.data, rtol=1e-5) + + # Ensure that input data has not been modified + self.assertArrayAllClose(u.data, u_lazy.data, rtol=1e-5) + self.assertArrayAllClose(v.data, v_lazy.data, rtol=1e-5) + + def test_lazy_rotate_winds_masked(self): + self._compare_lazy_rotate_winds(True) + + def test_lazy_rotate_winds_notmasked(self): + self._compare_lazy_rotate_winds(False) + + if __name__ == "__main__": tests.main() diff --git a/noxfile.py b/noxfile.py deleted file mode 100755 index d34155ecaf..0000000000 --- a/noxfile.py +++ /dev/null @@ -1,301 +0,0 @@ -""" -Perform test automation with nox. - -For further details, see https://nox.thea.codes/en/stable/# - -""" - -import hashlib -import os -from pathlib import Path - -import nox -from nox.logger import logger - -#: Default to reusing any pre-existing nox environments. -nox.options.reuse_existing_virtualenvs = True - -#: Python versions we can run sessions under -_PY_VERSIONS_ALL = ["3.8", "3.9", "3.10"] -_PY_VERSION_LATEST = _PY_VERSIONS_ALL[-1] - -#: One specific python version for docs builds -_PY_VERSION_DOCSBUILD = _PY_VERSION_LATEST - -#: Cirrus-CI environment variable hook. -PY_VER = os.environ.get("PY_VER", _PY_VERSIONS_ALL) - -#: Default cartopy cache directory. -CARTOPY_CACHE_DIR = os.environ.get("HOME") / Path(".local/share/cartopy") - -# https://github.com/numpy/numpy/pull/19478 -# https://github.com/matplotlib/matplotlib/pull/22099 -#: Common session environment variables. -ENV = dict(NPY_DISABLE_CPU_FEATURES="AVX512F,AVX512CD,AVX512_SKX") - - -def session_lockfile(session: nox.sessions.Session) -> Path: - """Return the path of the session lockfile.""" - return Path( - f"requirements/locks/py{session.python.replace('.', '')}-linux-64.lock" - ) - - -def session_cachefile(session: nox.sessions.Session) -> Path: - """Returns the path of the session lockfile cache.""" - lockfile = session_lockfile(session) - tmp_dir = Path(session.create_tmp()) - cache = tmp_dir / lockfile.name - return cache - - -def venv_populated(session: nox.sessions.Session) -> bool: - """Returns True if the conda venv has been created - and the list of packages in the lockfile installed.""" - return session_cachefile(session).is_file() - - -def venv_changed(session: nox.sessions.Session) -> bool: - """Returns True if the installed session is different to that specified - in the lockfile.""" - changed = False - cache = session_cachefile(session) - lockfile = session_lockfile(session) - if cache.is_file(): - with open(lockfile, "rb") as fi: - expected = hashlib.sha256(fi.read()).hexdigest() - with open(cache, "r") as fi: - actual = fi.read() - changed = actual != expected - return changed - - -def cache_venv(session: nox.sessions.Session) -> None: - """ - Cache the nox session environment. - - This consists of saving a hexdigest (sha256) of the associated - conda lock file. - - Parameters - ---------- - session: object - A `nox.sessions.Session` object. - - """ - lockfile = session_lockfile(session) - cache = session_cachefile(session) - with open(lockfile, "rb") as fi: - hexdigest = hashlib.sha256(fi.read()).hexdigest() - with open(cache, "w") as fo: - fo.write(hexdigest) - - -def cache_cartopy(session: nox.sessions.Session) -> None: - """ - Determine whether to cache the cartopy natural earth shapefiles. - - Parameters - ---------- - session: object - A `nox.sessions.Session` object. - - """ - if not CARTOPY_CACHE_DIR.is_dir(): - session.run_always( - "python", - "-c", - "import cartopy; cartopy.io.shapereader.natural_earth()", - ) - - -def prepare_venv(session: nox.sessions.Session) -> None: - """ - Create and cache the nox session conda environment, and additionally - provide conda environment package details and info. - - Note that, iris is installed into the environment using pip. - - Parameters - ---------- - session: object - A `nox.sessions.Session` object. - - Notes - ----- - See - - https://github.com/theacodes/nox/issues/346 - - https://github.com/theacodes/nox/issues/260 - - """ - lockfile = session_lockfile(session) - venv_dir = session.virtualenv.location_name - - if not venv_populated(session): - # environment has been created but packages not yet installed - # populate the environment from the lockfile - logger.debug(f"Populating conda env at {venv_dir}") - session.conda_install("--file", str(lockfile)) - cache_venv(session) - - elif venv_changed(session): - # destroy the environment and rebuild it - logger.debug(f"Lockfile changed. Re-creating conda env at {venv_dir}") - _re_orig = session.virtualenv.reuse_existing - session.virtualenv.reuse_existing = False - session.virtualenv.create() - session.conda_install("--file", str(lockfile)) - session.virtualenv.reuse_existing = _re_orig - cache_venv(session) - - logger.debug(f"Environment {venv_dir} is up to date") - - cache_cartopy(session) - - # Determine whether verbose diagnostics have been requested - # from the command line. - verbose = "-v" in session.posargs or "--verbose" in session.posargs - - if verbose: - session.run_always("conda", "info") - session.run_always("conda", "list", f"--prefix={venv_dir}") - session.run_always( - "conda", - "list", - f"--prefix={venv_dir}", - "--explicit", - ) - - -@nox.session(python=PY_VER, venv_backend="conda") -def tests(session: nox.sessions.Session): - """ - Perform iris system, integration and unit tests. - - Coverage testing is enabled if the "--coverage" or "-c" flag is used. - - Parameters - ---------- - session: object - A `nox.sessions.Session` object. - - """ - prepare_venv(session) - session.install("--no-deps", "--editable", ".") - session.env.update(ENV) - run_args = [ - "pytest", - "-n", - "auto", - "lib/iris/tests", - ] - if "-c" in session.posargs or "--coverage" in session.posargs: - run_args[-1:-1] = ["--cov=lib/iris", "--cov-report=xml"] - session.run(*run_args) - - -@nox.session(python=_PY_VERSION_DOCSBUILD, venv_backend="conda") -def doctest(session: nox.sessions.Session): - """ - Perform iris doctests and gallery. - - Parameters - ---------- - session: object - A `nox.sessions.Session` object. - - """ - prepare_venv(session) - session.install("--no-deps", "--editable", ".") - session.env.update(ENV) - session.cd("docs") - session.run( - "make", - "clean", - "html", - external=True, - ) - session.run( - "make", - "doctest", - external=True, - ) - - -@nox.session(python=_PY_VERSION_DOCSBUILD, venv_backend="conda") -def gallery(session: nox.sessions.Session): - """ - Perform iris gallery doc-tests. - - Parameters - ---------- - session: object - A `nox.sessions.Session` object. - - """ - prepare_venv(session) - session.install("--no-deps", "--editable", ".") - session.env.update(ENV) - session.run( - "pytest", - "-n", - "auto", - "docs/gallery_tests", - ) - - -@nox.session(python=_PY_VERSION_DOCSBUILD, venv_backend="conda") -def linkcheck(session: nox.sessions.Session): - """ - Perform iris doc link check. - - Parameters - ---------- - session: object - A `nox.sessions.Session` object. - - """ - prepare_venv(session) - session.install("--no-deps", "--editable", ".") - session.cd("docs") - session.run( - "make", - "clean", - "html", - external=True, - ) - session.run( - "make", - "linkcheck", - external=True, - ) - - -@nox.session(python=PY_VER, venv_backend="conda") -def wheel(session: nox.sessions.Session): - """ - Perform iris local wheel install and import test. - - Parameters - ---------- - session: object - A `nox.sessions.Session` object. - - """ - prepare_venv(session) - session.cd("dist") - fname = list(Path(".").glob("scitools_iris-*.whl")) - if len(fname) == 0: - raise ValueError("Cannot find wheel to install.") - if len(fname) > 1: - emsg = ( - f"Expected to find 1 wheel to install, found {len(fname)} instead." - ) - raise ValueError(emsg) - session.install(fname[0].name) - session.run( - "python", - "-c", - "import iris; print(f'{iris.__version__=}')", - external=True, - ) diff --git a/requirements/py310.yml b/requirements/py310.yml index e3bada6596..49b8ae78ab 100644 --- a/requirements/py310.yml +++ b/requirements/py310.yml @@ -14,7 +14,7 @@ dependencies: - cartopy >=0.21 - cf-units >=3.1 - cftime >=1.5 - - dask-core >=2.26 + - dask-core >=2022.9.0 - matplotlib >=3.5 - netcdf4 - numpy >=1.19 diff --git a/requirements/py38.yml b/requirements/py38.yml index 9393060113..1e3c61f369 100644 --- a/requirements/py38.yml +++ b/requirements/py38.yml @@ -14,7 +14,7 @@ dependencies: - cartopy >=0.21 - cf-units >=3.1 - cftime >=1.5 - - dask-core >=2.26 + - dask-core >=2022.9.0 - matplotlib >=3.5 - netcdf4 - numpy >=1.19 diff --git a/requirements/py39.yml b/requirements/py39.yml index 349784ec46..6551eee495 100644 --- a/requirements/py39.yml +++ b/requirements/py39.yml @@ -14,7 +14,7 @@ dependencies: - cartopy >=0.21 - cf-units >=3.1 - cftime >=1.5 - - dask-core >=2.26 + - dask-core >=2022.9.0 - matplotlib >=3.5 - netcdf4 - numpy >=1.19 diff --git a/setup.cfg b/setup.cfg index ba9844f5d8..aac726776c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,7 +50,7 @@ install_requires = cartopy>=0.21 cf-units>=3.1 cftime>=1.5.0 - dask[array]>=2.26 + dask[array]>=2022.9.0 matplotlib>=3.5 netcdf4 numpy>=1.19 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000..97699bd5d3 --- /dev/null +++ b/tox.ini @@ -0,0 +1,40 @@ +[tox] +requires = + tox-conda + +[testenv] +conda_spec = + py38: {toxinidir}{/}requirements{/}locks{/}py38-linux-64.lock + py39: {toxinidir}{/}requirements{/}locks{/}py39-linux-64.lock + py310: {toxinidir}{/}requirements{/}locks{/}py310-linux-64.lock +usedevelop = + true + +[testenv:py{38,39,310}-tests] +description = Perform Iris unit, integration and system tests. +commands = + pytest -n auto {posargs} {toxinidir}{/}lib{/}iris{/}tests + +[testenv:py{38,39,310}-gallery_tests] +description = Perform Iris gallery tests. +commands = + pytest -n auto {posargs} docs/gallery_tests + +[testenv:py{38,39,310}-docs{,-linkcheck,-tests}] +description = Build, and optionally linkcheck or test, the Iris documentation +allowlist_externals = + make +changedir = + {toxinidir}{/}docs +commands = + make clean html + linkcheck: make linkcheck + tests: make doctest + +[testenv:py{38,39,310}-wheel] +description = Install wheel and test the Iris import +skip_install = true +deps = + {toxinidir}{/}dist{/}scitools_iris*.whl +commands = + python -c "import iris; print(f'\{iris.__version__=\}')" \ No newline at end of file