Merge branch 'master' into initial-final-graph

hjoliver · Oct 25, 2022 · 9024065 · 9024065
2 parents 858a8c1 b0ff549
commit 9024065
Show file tree

Hide file tree

Showing 103 changed files with 3,093 additions and 5,180 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -15,4 15,4 @@ Thanks for your contribution:
 - [ ] Tests are included (or explain why tests are not needed).
 - [ ] `CHANGES.md` entry included if this is a change that can affect users
 - [ ] [Cylc-Doc](https://github.com/cylc/cylc-doc) pull request opened if required at cylc/cylc-doc/pull/XXXX.
-- [ ] If this is a bug fix, PRs raised to both master and the relevant maintenance branch.
 - [ ] If this is a bug fix, PR should be raised against the relevant `?.?.x` branch.
diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
@@ -70,3 70,10 @@ jobs:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  with:
  release-url: ${{ steps.create-release.outputs.html_url }}
 
  - name: Bump dev version
  uses: cylc/release-actions/stage-2/bump-dev-version@v1
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  with:
  init-file: 'cylc/flow/__init__.py'
diff --git a/.github/workflows/bash.yml b/.github/workflows/bash.yml
@@ -14,7 14,9 @@ on:
  - '**.md'
  - '**/README*/**'
  push:
- branches: [master]
  branches:
  - master
  - '8.*.x'
  paths-ignore:
  - '.github/workflows/*.ya?ml'
  - '!.github/workflows/bash.yml'

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
@@ -4,7 4,9 @@ on:
  pull_request:
  workflow_dispatch:
  push:
- branches: [master]
  branches:
  - master
  - '8.*.x'
 
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}

diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
@@ -13,7 13,9 @@ on:
  - '**.md'
  - '**/README*/**'
  push:
- branches: [master]
  branches:
  - master
  - '8.*.x'
  paths-ignore:
  - '.github/workflows/*.ya?ml'
  - '!.github/workflows/test_functional.yml'
@@ -191,7 193,7 @@ jobs:
  timeout-minutes: 35
  continue-on-error: true
  run: |
- echo "::set-output name=finished::false"
  echo "finished=false" >> $GITHUB_OUTPUT
  if [[ '${{ matrix.test-base }}' == 'tests/k' ]]; then
  NPROC=4
  else
@@ -202,7 204,7 @@ jobs:
  -j "${NPROC}" \
  --state=save \
  $(cat test-file) \
- || (echo "::set-output name=finished::true" && false)
  || (echo "finished=true" >> $GITHUB_OUTPUT && false)
 
  - name: Time Out
  if: steps.test.outcome == 'failure' && steps.test.outputs.finished != 'true'
@@ -251,7 253,7 @@ jobs:
  run: |
  # artifact name cannot contain '/' characters
  CID="$(sed 's|/|-|g' <<< "${{ matrix.name || matrix.chunk }}")"
- echo "::set-output name=uploadname::$CID"
  echo "uploadname=$CID" >> $GITHUB_OUTPUT
 
  - name: Upload artifact
  if: failure() && steps.test.outcome == 'failure'

diff --git a/.github/workflows/test_tutorial_workflow.yml b/.github/workflows/test_tutorial_workflow.yml
@@ -2,7 2,9 @@ name: test-tutorial-workflow
 
 on:
  push:
- branches: [master]
  branches:
  - master
  - '8.*.x'
  pull_request:
  paths-ignore:
  - '.github/workflows/*.ya?ml'

diff --git a/CHANGES.md b/CHANGES.md
@@ -9,29 9,71 @@ creating a new release entry be sure to copy & paste the span tag with the
 `actions:bind` attribute, which is used by a regex to find the text to be
 updated. Only the first match gets replaced, so it's fine to leave the old
 ones in. -->
--------------------------------------------------------------------------------
 
 -------------------------------------------------------------------------------
 ## __cylc-8.1.0 (<span actions:bind='release-date'>Upcoming</span>)__
 
 ### Enhancements
 
 [#5184](https://github.com/cylc/cylc-flow/pull/5184) - scan for active
 runs of the same workflow at install time.
 
 [#5032](https://github.com/cylc/cylc-flow/pull/5032) - set a default limit of
 100 for the "default" queue.
 
-[#5055](https://github.com/cylc/cylc-flow/pull/5055) - Hard-code the serial
-numbers of Cylc Lint's style issues and allow users to ignore Cylc Lint issues
-using `--ignore <Issue Code>`.
 [#5055](https://github.com/cylc/cylc-flow/pull/5055) and
 [#5086](https://github.com/cylc/cylc-flow/pull/5086) - Upgrades to `cylc lint`
 - Allow users to ignore Cylc Lint issues using `--ignore <Issue Code>`.
 - Allow settings for `cylc lint` to be recorded in a pyproject.toml file.
 - Allow files to be excluded from `cylc lint` checks.
 
 [#5081](https://github.com/cylc/cylc-flow/pull/5081) - Reduced amount that
 gets logged at "INFO" level in scheduler logs.
 
 -------------------------------------------------------------------------------
 ## __cylc-8.0.4 (<span actions:bind='release-date'>Pending YYYY-MM-DD</span>)__
 
 Maintenance release.
 
 ### Fixes
 
 [#5196](https://github.com/cylc/cylc-flow/pull/5196) - Replace traceback
 with warning, for scan errors where workflow is stopped.
 
 -------------------------------------------------------------------------------
-## __cylc-8.0.3 (<span actions:bind='release-date'>Upcoming</span>)__
 ## __cylc-8.0.3 (<span actions:bind='release-date'>Released 2022-10-17</span>)__
 
 Maintenance release.
 
 ### Fixes
 
 [#5192](https://github.com/cylc/cylc-flow/pull/5192) -
 Recompute runahead limit after use of `cylc remove`.
 
 [#5188](https://github.com/cylc/cylc-flow/pull/5188) -
 Fix task state selectors in `cylc trigger` and other commands.
 
 [#5125](https://github.com/cylc/cylc-flow/pull/5125) - Allow rose-suite.conf
 changes to be considered by ``cylc reinstall``.
 
 [#5023](https://github.com/cylc/cylc-flow/pull/5023),
 [#5187](https://github.com/cylc/cylc-flow/pull/5187) -
 tasks force-triggered
 after a shutdown was ordered should submit to run immediately on restart.
 
 [#5137](https://github.com/cylc/cylc-flow/pull/5137) -
 Install the `ana/` directory to remote platforms by default.
 
 [#5146](https://github.com/cylc/cylc-flow/pull/5146) - no-flow tasks should not
 retrigger incomplete children.
 
 [#5104](https://github.com/cylc/cylc-flow/pull/5104) - Fix retriggering of
 failed tasks after a reload.
 
 [#5139](https://github.com/cylc/cylc-flow/pull/5139) - Fix bug where
 `cylc install` could hang if there was a large uncommitted diff in the
 source dir (for git/svn repos).
 
 [#5131](https://github.com/cylc/cylc-flow/pull/5131) - Infer workflow run number
 for `workflow_state` xtrigger.
 

diff --git a/conda-environment.yml b/conda-environment.yml
@@ -12,12 12,14 @@ dependencies:
  - jinja2 >=3.0,<3.1
  - metomi-isodatetime >=1!3.0.0, <1!3.1.0
  # Constrain protobuf version for compatible Scheduler-UIS comms across hosts
- - protobuf >=3.19.0,<3.20.0
  - protobuf >=4.21.2,<4.22.0
  - psutil >=5.6.0
  - python
  - pyzmq >=22,<23
  - setuptools >=49
  - urwid >=2,<3
  # Add # [py<3.11] for tomli once Python 3.11 Released
  - tomli >=2.0.0
 
 # optional dependencies
  #- empy >=3.3,<3.4

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
@@ -1614,18 1614,26 @@ def default_for(
  with Conf('localhost', meta=Platform, desc='''
  A default platform for running jobs on the the scheduler host.
 
- .. attention::
  This platform configures the host on which
  :term:`schedulers <scheduler>` run. By default this is the
  host where ``cylc play`` is run, however, we often configure
  Cylc to start schedulers on dedicated hosts by configuring
  :cylc:conf:`global.cylc[scheduler][run hosts]available`.
 
- It is common practice to start Cylc schedulers on dedicated
- hosts, in which case **"localhost" is the scheduler host and
- not necessarily where you ran "cylc play"**.
  This platform affects connections made to the scheduler host and
  any jobs run on it.
 
  .. versionadded:: 8.0.0
  '''):
  Conf('hosts', VDR.V_STRING_LIST, ['localhost'], desc='''
  List of hosts for the localhost platform. You are unlikely to
  need to change this.
 
  The scheduler hosts are configured by
  :cylc:conf:`global.cylc[scheduler][run hosts]available`.
  See :ref:`Submitting Workflows To a Pool Of Hosts` for
  more information.
 
  .. seealso::
 
  :cylc:conf:`global.cylc[platforms][<platform name>]hosts`

diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py
@@ -222,10 222,17 @@ def get_script_common_text(this: str, example: Optional[str] = None):
 
  The following directories already get installed by default:
 
- * ``app/``
- * ``bin/``
- * ``etc/``
- * ``lib/``
  ``ana/``
  Rose ana analysis modules
  ``app/``
  Rose applications
  ``bin/``
  Cylc bin directory (added to ``PATH``)
  ``etc/``
  Miscellaneous resources
  ``lib/``
  Cylc lib directory (``lib/python`` added to ``PYTHONPATH``
  for workflow config)
 
  These should be located in the top level of your Cylc workflow,
  i.e. the directory that contains your ``flow.cylc`` file.

diff --git a/cylc/flow/data_messages_pb2.py b/cylc/flow/data_messages_pb2.py
diff --git a/cylc/flow/exceptions.py b/cylc/flow/exceptions.py
@@ -16,6 16,7 @@
 """Exceptions for "expected" errors."""
 
 import errno
 from textwrap import wrap
 from typing import (
  Callable,
  Iterable,
@@ -115,6 116,13 @@ class WorkflowEventError(CylcError):
 
 class CommandFailedError(CylcError):
  """Exception for when scheduler commands fail."""
  def __init__(self, value: Union[str, Exception]):
  self.value = value
 
  def __str__(self) -> str:
  if isinstance(self.value, Exception):
  return f"{type(self.value).__name__}: {self.value}"
  return self.value
 
 
 class ServiceFileError(CylcError):
@@ -365,11 373,58 @@ def __init__(self, data):
  def __str__(self):
  ret = 'Could not select host from:'
  for host, data in sorted(self.data.items()):
- ret  = f'\n {host}:'
- for key, value in data.items():
- ret  = f'\n {key}: {value}'
  if host != 'ranking':
  ret  = f'\n {host}:'
  for key, value in data.items():
  ret  = f'\n {key}: {value}'
  hint = self.get_hint()
  if hint:
  ret  = f'\n\n{hint}'
  return ret
 
  def get_hint(self):
  """Return a hint to explain this error for certain cases."""
  if all(
  # all procs came back with special SSH error code 255
  datum.get('returncode') == 255
  for key, datum in self.data.items()
  if key != 'ranking'
  ):
  # likely SSH issues
  return (
  'Cylc could not establish SSH connection to the run hosts.'
  '\nEnsure you can SSH to these hosts without having to'
  ' answer any prompts.'
  )
 
  if (
  # a ranking expression was used
  self.data.get('ranking')
  # and all procs came back with special 'cylc psutil' error code 2
  # (which is used for errors relating to the extraction of metrics)
  and all(
  datum.get('returncode') == 2
  for key, datum in self.data.items()
  if key != 'ranking'
  )
  ):
  # likely an issue with the ranking expression
  ranking = "\n".join(
  wrap(
  self.data.get("ranking"),
  initial_indent=' ',
  subsequent_indent=' ',
  )
  )
  return (
  'This is likely an error in the ranking expression:'
  f'\n{ranking}'
  '\n\nConfigured by:'
  '\n global.cylc[scheduler][run hosts]ranking'
  )
 
  return None
 
 
 class NoHostsError(CylcError):
  """None of the hosts of a given platform were reachable."""