Merge pull request #6168 from wxtim/feat.symlink_log.job

Allow Symlinking for log/job
cylc · Jul 31, 2024 · 2143d66 · 2143d66
2 parents 9e991dd 152e794
commit 2143d66
Show file tree

Hide file tree

Showing 9 changed files with 86 additions and 62 deletions.
diff --git a/changes.d/feat.6168.md b/changes.d/feat.6168.md
@@ -0,0 1 @@
 Allow symlinking log/job separately from log
diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
@@ -30,6 30,7 @@
 from cylc.flow.exceptions import GlobalConfigError
 from cylc.flow.hostuserutil import get_user_home
 from cylc.flow.network.client_factory import CommsMeth
 from cylc.flow.pathutil import SYMLINKABLE_LOCATIONS
 from cylc.flow.parsec.config import (
  ConfigNode as Conf,
  ParsecConfig,
@@ -1148,55 1149,21 @@ def default_for(
 
  .. versionadded:: 8.0.0
  """)
- Conf('log', VDR.V_STRING, None, desc="""
- Alternative location for the log dir.
-
- If specified the workflow log directory will be created in
- ``<this-path>/cylc-run/<workflow-id>/log`` and a
- symbolic link will be created from
- ``$HOME/cylc-run/<workflow-id>/log``. If not specified
- the workflow log directory will be created in
- ``$HOME/cylc-run/<workflow-id>/log``.
-
- .. versionadded:: 8.0.0
- """)
- Conf('share', VDR.V_STRING, None, desc="""
- Alternative location for the share dir.
-
- If specified the workflow share directory will be
- created in ``<this-path>/cylc-run/<workflow-id>/share``
- and a symbolic link will be created from
- ``<$HOME/cylc-run/<workflow-id>/share``. If not specified
- the workflow share directory will be created in
- ``$HOME/cylc-run/<workflow-id>/share``.
-
- .. versionadded:: 8.0.0
- """)
- Conf('share/cycle', VDR.V_STRING, None, desc="""
- Alternative directory for the share/cycle dir.
-
- If specified the workflow share/cycle directory
- will be created in
- ``<this-path>/cylc-run/<workflow-id>/share/cycle``
- and a symbolic link will be created from
- ``$HOME/cylc-run/<workflow-id>/share/cycle``. If not
- specified the workflow share/cycle directory will be
- created in ``$HOME/cylc-run/<workflow-id>/share/cycle``.
-
- .. versionadded:: 8.0.0
- """)
- Conf('work', VDR.V_STRING, None, desc="""
- Alternative directory for the work dir.
-
- If specified the workflow work directory will be created in
- ``<this-path>/cylc-run/<workflow-id>/work`` and a
- symbolic link will be created from
- ``$HOME/cylc-run/<workflow-id>/work``. If not specified
- the workflow work directory will be created in
- ``$HOME/cylc-run/<workflow-id>/work``.
-
- .. versionadded:: 8.0.0
- """)
  for folder, versionadded in SYMLINKABLE_LOCATIONS.items():
  Conf(folder, VDR.V_STRING, None, desc=f"""
  Alternative location for the log dir.
 
  If specified the workflow {folder} directory will
  be created in
  ``<this-path>/cylc-run/<workflow-id>/{folder}``
  and a symbolic link will be created from
  ``$HOME/cylc-run/<workflow-id>/{folder}``.
  If not specified the workflow log directory will
  be created in
  ``$HOME/cylc-run/<workflow-id>/{folder}``.
 
  .. versionadded:: {versionadded}
  """)
  with Conf('platforms', desc='''
  Platforms allow you to define compute resources available at your
  site.

diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
@@ -45,6 45,14 @@
 """Matches relative paths that are explicit (starts with ./)"""
 
 SHELL_ENV_VARS = re.compile(r'\$[^$/]*')
 SYMLINKABLE_LOCATIONS: Dict[str, str] = {
  # Location: Version Added
  'share': '8.0.0',
  'share/cycle': '8.0.0',
  'log': '8.0.0',
  'log/job': '8.4.0',
  'work': '8.0.0',
 }
 
 
 def expand_path(*args: Union[Path, str]) -> str:
@@ -234,7 242,7 @@ def get_dirs_to_symlink(
  if base_dir:
  dirs_to_symlink['run'] = os.path.join(
  base_dir, 'cylc-run', workflow_id)
- for dir_ in ['log', 'share', 'share/cycle', 'work']:
  for dir_ in SYMLINKABLE_LOCATIONS:
  link = symlink_conf[install_target].get(dir_, None)
  if (not link) or link == base_dir:
  continue

diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
@@ -59,6 59,7 @@
  is_remote_host,
 )
 from cylc.flow.pathutil import (
  SYMLINKABLE_LOCATIONS,
  expand_path,
  get_cylc_run_dir,
  get_workflow_run_dir,
@@ -203,6 204,11 @@ class LogDir:
  DB = 'db'
  """The public database"""
 
  JOB = 'job'
  """The job log directory."""
 
  LOG_JOB_DIR = os.path.join(LogDir.DIRNAME, LogDir.JOB)
 
  SHARE_DIR = 'share'
  """Workflow share directory."""
 
@@ -258,9 264,7 @@ class Install:
  RESERVED_NAMES = frozenset([FLOW_FILE, SUITE_RC, *RESERVED_DIRNAMES])
  """Reserved filenames that cannot be used as run names."""
 
- SYMLINK_DIRS = frozenset([
- SHARE_CYCLE_DIR, SHARE_DIR, LogDir.DIRNAME, WORK_DIR, ''
- ])
  SYMLINK_DIRS = frozenset(list(SYMLINKABLE_LOCATIONS)   [''])
  """The paths of the symlink dirs that may be set in
  global.cylc[install][symlink dirs], relative to the run dir
  ('' represents the run dir)."""

diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t
@@ -33,6 33,7 @@ create_test_global_config "" "
  [[[localhost]]]
  run = ${TEST_DIR}/${SYM_NAME}/run
  log = ${TEST_DIR}/${SYM_NAME}/log
  log/job = ${TEST_DIR}/${SYM_NAME}/job
  share = ${TEST_DIR}/${SYM_NAME}/share
  share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle
  work = ${TEST_DIR}/${SYM_NAME}/work
@@ -63,13 64,20 @@ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}
  \`-- ${TEST_NAME_BASE}
  \`-- share
  \`-- cycle
 ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}
  \`-- cylc-clean
  \`-- ${TEST_NAME_BASE}
  \`-- log
  \`-- job
 ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE}
 |-- ${FUNCTIONAL_DIR}
 | \`-- cylc-clean
 | \`-- ${TEST_NAME_BASE}
 | \`-- log
-| \`-- install
-| \`-- ${INSTALL_LOG_FILE}
 | |-- install
 | | \`-- ${INSTALL_LOG_FILE}
 | \`-- job -> ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${WORKFLOW_NAME}/log/job
 \`-- leave-me-alone
 ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}

diff --git a/tests/functional/cylc-clean/01-remote.t b/tests/functional/cylc-clean/01-remote.t
@@ -37,6 37,7 @@ create_test_global_config "" "
  [[[${CYLC_TEST_INSTALL_TARGET}]]]
  run = ${TEST_DIR}/${SYM_NAME}/run
  log = ${TEST_DIR}/${SYM_NAME}/other
  log/job = ${TEST_DIR}/${SYM_NAME}/job
  share = ${TEST_DIR}/${SYM_NAME}/other
  share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle
  work = ${TEST_DIR}/${SYM_NAME}/other
@@ -81,16 82,22 @@ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}
 | \`-- share
 | \`-- cycle
 \`-- leave-me-alone
 ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}
  \`-- cylc-clean
  \`-- ${TEST_NAME_BASE}
  \`-- log
  \`-- job
  \`-- 1
  \`-- santa
  |-- 01
  \`-- NN -> 01
 ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}
  \`-- cylc-clean
  \`-- ${TEST_NAME_BASE}
  |-- log
- | \`-- job
- | \`-- 1
- | \`-- santa
- | |-- 01
- | \`-- NN -> 01
  | \`-- job -> ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${WORKFLOW_NAME}/log/job
  |-- share
  | \`-- cycle -> ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${WORKFLOW_NAME}/share/cycle
  \`-- work

diff --git a/tests/functional/cylc-install/01-symlinks.t b/tests/functional/cylc-install/01-symlinks.t
@@ -24,7 24,7 @@ if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then
  skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"'
 fi
 
-set_test_number 30
 set_test_number 32
 
 create_test_global_config "" "
 [install]
@@ -33,6 33,7 @@ create_test_global_config "" "
  run = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_run_dir
  share = \$TMPDIR/\$USER/test_cylc_symlink/
  log = \$TMPDIR/\$USER/test_cylc_symlink/
  log/job = \$TMPDIR/\$USER/test_cylc_symlink/job_log_dir
  share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_share_dir
  work = \$TMPDIR/\$USER/test_cylc_symlink/
 "
@@ -53,6 54,10 @@ TEST_SYM="${TEST_NAME_BASE}-share-cycle-glblcfg"
 run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \
  = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle"
 
 TEST_SYM="${TEST_NAME_BASE}-log-job-glblcfg"
 run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \
  = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job"
 
 for DIR in 'work' 'share' 'log'; do
  TEST_SYM="${TEST_NAME_BASE}-${DIR}-glbcfg"
  run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \
@@ -137,6 142,10 @@ TEST_SYM="${TEST_NAME}-share-cycle"
 run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \
  = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle"
 
 TEST_SYM="${TEST_NAME}-log-job"
 run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \
  = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job"
 
 for DIR in 'work' 'share' 'log'; do
  TEST_SYM="${TEST_NAME}-${DIR}"
  run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \

diff --git a/tests/functional/remote/04-symlink-dirs.t b/tests/functional/remote/04-symlink-dirs.t
@@ -24,7 24,7 @@ if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then
  skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"'
 fi
 
-set_test_number 12
 set_test_number 14
 
 create_test_global_config "" "
 [install]
@@ -33,12 33,14 @@ create_test_global_config "" "
  run = \$TMPDIR/\$USER/cylctb_tmp_run_dir
  share = \$TMPDIR/\$USER
  log = \$TMPDIR/\$USER
  log/job = \$TMPDIR/\$USER/cylctb_tmp_log_job_dir
  share/cycle = \$TMPDIR/\$USER/cylctb_tmp_share_dir
  work = \$TMPDIR/\$USER
  [[[$CYLC_TEST_INSTALL_TARGET]]]
  run = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_run_dir
  share = \$TMPDIR/\$USER/test_cylc_symlink/
  log = \$TMPDIR/\$USER/test_cylc_symlink/
  log/job = \$TMPDIR/\$USER/cylctb_tmp_log_job_dir
  share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_share_dir
  work = \$TMPDIR/\$USER/test_cylc_symlink/
 "
@@ -66,6 68,14 @@ else
  fail "$TEST_SYM.localhost"
 fi
 
 TEST_SYM="${TEST_NAME_BASE}-log/job-symlink-exists-ok"
 if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}/log/job") == \
 "$TMPDIR/$USER/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
  ok "$TEST_SYM.localhost"
 else
  fail "$TEST_SYM.localhost"
 fi
 
 for DIR in 'work' 'share' 'log'; do
  TEST_SYM="${TEST_NAME_BASE}-${DIR}-symlink-exists-ok"
  if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}/${DIR}") == \
@@ -94,6 104,14 @@ else
  fail "${TEST_NAME_BASE}-share/cycle-symlink-exists-ok.remotehost"
 fi
 
 # shellcheck disable=SC2016
 LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/log/job'"')"
 if [[ "$LINK" == *"/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
  ok "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"
 else
  fail "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"
 fi
 
 for DIR in 'work' 'share' 'log'; do
 # shellcheck disable=SC2016
  LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/$DIR'"')"

diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py
@@ -203,13 203,15 @@ def test_make_workflow_run_tree(
  run = $DEE
  work = $DAH
  log = $DUH
  log/job =$RAY
  share = $DOH
  share/cycle = $DAH
  ''',
  {
  'run': '$DEE/cylc-run/morpheus',
  'work': '$DAH/cylc-run/morpheus/work',
  'log': '$DUH/cylc-run/morpheus/log',
  'log/job': '$RAY/cylc-run/morpheus/log/job',
  'share': '$DOH/cylc-run/morpheus/share',
  'share/cycle': '$DAH/cylc-run/morpheus/share/cycle'
  },