Skip to content

Commit c252734

Browse files
MetRonnieMetRonnie
authored
Merge pull request #6655 from oliver-sanders/actions.micromamba
actions: provision python from conda-forge
2 parents 207984e + 768dd36 commit c252734

29 files changed

+105
-104
lines changed

‎.github/workflows/test_fast.yml

+35-26
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ concurrency:
1212
group: ${{ github.workflow }}-${{ github.ref }}
1313
cancel-in-progress: true
1414

15+
defaults:
16+
run:
17+
shell: bash -c "exec $CONDA_PREFIX/bin/bash -elo pipefail {0}"
18+
1519
jobs:
1620
test:
1721
runs-on: ${{ matrix.os }}
@@ -20,12 +24,10 @@ jobs:
2024
fail-fast: false # don't stop on first failure
2125
matrix:
2226
os: ['ubuntu-latest']
23-
python-version: ['3.8', '3.10', '3.11', '3']
27+
python-version: ['3.7', '3.8', '3.10', '3.11', '3.12', '3']
2428
include:
25-
- os: 'ubuntu-22.04'
26-
python-version: '3.7'
2729
- os: 'macos-latest'
28-
python-version: '3.9' # oldest supported version
30+
python-version: '3.8' # oldest version with arm64 builds
2931
# non-utc timezone test
3032
- os: 'ubuntu-latest'
3133
python-version: '3.9' # not the oldest, not the most recent version
@@ -39,24 +41,26 @@ jobs:
3941
- name: Checkout
4042
uses: actions/checkout@v4
4143

42-
- name: Configure Python
43-
uses: actions/setup-python@v5
44+
- name: Install System Dependencies
45+
uses: mamba-org/setup-micromamba@v2
4446
with:
45-
python-version: ${{ matrix.python-version }}
46-
47-
- name: Apt-Get Install
48-
if: startsWith(matrix.os, 'ubuntu')
47+
cache-environment: true
48+
post-cleanup: 'all'
49+
environment-name: cylc-fast-test
50+
create-args: >-
51+
python=${{ matrix.python-version }}
52+
pip
53+
bash
54+
subversion
55+
sqlite
56+
57+
- name: Install Python Dependencies
4958
run: |
50-
sudo apt-get update
51-
sudo apt-get install -y sqlite3
59+
pip install -e ."[all]"
5260
5361
- name: Patch DNS
5462
uses: cylc/release-actions/patch-dns@v1
5563

56-
- name: Install
57-
run: |
58-
pip install -e ."[all]"
59-
6064
- name: Configure git # Needed by the odd test
6165
uses: cylc/release-actions/configure-git@v1
6266

@@ -99,21 +103,26 @@ jobs:
99103
lint:
100104
runs-on: 'ubuntu-latest'
101105
timeout-minutes: 10
106+
strategy:
107+
matrix:
108+
python-version: ['3']
102109
steps:
103-
- name: Apt-Get Install
104-
run: |
105-
sudo apt-get update
106-
sudo apt-get install -y shellcheck
107-
108110
- name: Checkout
109111
uses: actions/checkout@v4
110112

111-
- name: Configure Python
112-
uses: actions/setup-python@v5
113+
- name: Install System Dependencies
114+
uses: mamba-org/setup-micromamba@v2
113115
with:
114-
python-version: 3.x
115-
116-
- name: Install
116+
cache-environment: true
117+
post-cleanup: 'all'
118+
environment-name: cylc-lint-test
119+
create-args: >-
120+
python=${{ matrix.python-version }}
121+
pip
122+
bash
123+
shellcheck
124+
125+
- name: Install Python Dependencies
117126
run: |
118127
pip install -e ."[tests]"
119128

‎.github/workflows/test_functional.yml

+28-35
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ concurrency:
3232
group: ${{ github.workflow }}-${{ github.ref }}
3333
cancel-in-progress: true
3434

35+
defaults:
36+
run:
37+
shell: bash -c "exec $CONDA_PREFIX/bin/bash -elo pipefail {0}"
38+
3539
jobs:
3640
test:
3741
runs-on: ${{ matrix.os }}
@@ -56,34 +60,34 @@ jobs:
5660
platform: '_local_background*'
5761
# tests/k
5862
- name: 'flaky'
59-
os: 'ubuntu-22.04'
63+
os: 'ubuntu-latest'
6064
python-version: '3.7'
6165
test-base: 'tests/k'
6266
chunk: '1/1'
6367
platform: '_local_background* _local_at*'
6468
# remote platforms
6569
- name: '_remote_background_indep_poll'
66-
os: 'ubuntu-22.04'
70+
os: 'ubuntu-latest'
6771
python-version: '3.7'
6872
test-base: 'tests/f tests/k'
6973
chunk: '1/1'
7074
platform: '_remote_background_indep_poll _remote_at_indep_poll'
7175
- name: '_remote_background_indep_tcp'
72-
os: 'ubuntu-22.04'
76+
os: 'ubuntu-latest'
7377
test-base: 'tests/f tests/k'
7478
python-version: '3.7'
7579
chunk: '1/1'
7680
platform: '_remote_background_indep_tcp _remote_at_indep_tcp'
7781
# macos
7882
- name: 'macos 1/5'
7983
os: 'macos-latest'
80-
python-version: '3.9'
84+
python-version: '3.8' # oldest available
8185
test-base: 'tests/f'
8286
chunk: '1/5'
8387
platform: '_local_background*'
8488
- name: 'macos 2/5'
8589
os: 'macos-latest'
86-
python-version: '3.9'
90+
python-version: '3' # newest available
8791
test-base: 'tests/f'
8892
chunk: '2/5'
8993
platform: '_local_background*'
@@ -101,9 +105,21 @@ jobs:
101105
uses: actions/checkout@v4
102106

103107
- name: Configure Python
104-
uses: actions/setup-python@v5
108+
uses: mamba-org/setup-micromamba@v2
105109
with:
106-
python-version: ${{ matrix.python-version }}
110+
cache-environment: true
111+
post-cleanup: 'all'
112+
environment-name: cylc-functional-test
113+
create-args: >-
114+
python=${{ matrix.python-version }}
115+
pip
116+
bash
117+
coreutils
118+
grep
119+
sed
120+
sqlite
121+
subversion
122+
tree
107123
108124
- name: Create global config
109125
run: |
@@ -128,36 +144,11 @@ jobs:
128144
__HERE__
129145
cat "$GLOBAL_CFG_PATH"
130146
131-
- name: Brew Install
132-
if: startsWith(matrix.os, 'macos')
133-
run: |
134-
# install system deps
135-
brew update
136-
brew install bash coreutils gnu-sed grep
137-
138-
# add GNU coreutils and sed to the user PATH
139-
# (see instructions in brew install output)
140-
echo "$(brew --prefix)/opt/coreutils/libexec/gnubin" >> "${GITHUB_PATH}"
141-
echo "$(brew --prefix)/opt/grep/libexec/gnubin" >> "${GITHUB_PATH}"
142-
echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> "${GITHUB_PATH}"
143-
144-
# add coreutils to the bashrc too (for jobs)
145-
cat >> "${HOME}/.bashrc" <<__HERE__
146-
PATH="$(brew --prefix)/opt/coreutils/libexec/gnubin:$PATH"
147-
PATH="$(brew --prefix)/opt/grep/libexec/gnubin:$PATH"
148-
PATH="$(brew --prefix)/opt/gnu-sed/libexec/gnubin:$PATH"
149-
export PATH
150-
__HERE__
151-
152-
- name: Apt-Get Install
153-
if: startsWith(matrix.os, 'ubuntu')
154-
run: |
155-
sudo apt-get update
156-
sudo apt-get install -y sqlite3 tree at
157-
158147
- name: Add .github/bin/ to PATH
159148
# Sets up mocked mail command & any other custom executables
160-
run: echo "${{ github.workspace }}/.github/bin" >> $GITHUB_PATH
149+
# Adding to $GITHUB_PATH does not work when using setup-micromamba and/or login shell
150+
run: |
151+
echo "export PATH=\"${{ github.workspace }}/.github/bin:$PATH\"" >> ~/.bash_profile
161152
162153
- name: Install
163154
run: |
@@ -167,6 +158,8 @@ jobs:
167158
- name: Configure Atrun
168159
if: contains(matrix.platform, '_local_at')
169160
run: |
161+
sudo apt-get update
162+
sudo apt-get install -y at
170163
cat >> "$GLOBAL_CFG_PATH" << __HERE__
171164
[platforms]
172165
[[_local_at_indep_tcp]]

‎tests/flakyfunctional/database/00-simple.t

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ cmp_ok "${NAME}" "${SORTED_ORIG}"
3838

3939
NAME='select-workflow-params.out'
4040
sqlite3 "${DB_FILE}" \
41-
'SELECT key, value FROM workflow_params
42-
WHERE key != "uuid_str" AND key != "cycle_point_tz" ORDER BY key' \
41+
"SELECT key, value FROM workflow_params
42+
WHERE key != 'uuid_str' AND key != 'cycle_point_tz' ORDER BY key" \
4343
>"${NAME}"
4444
cmp_ok "${NAME}" << __EOF__
4545
UTC_mode|0

‎tests/flakyfunctional/hold-release/15-hold-after.t

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}"
2727
workflow_run_ok "${TEST_NAME_BASE}-1" \
2828
cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}"
2929
sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \
30-
'SELECT cycle, name, status FROM task_pool WHERE cycle=="20140102T0000Z" ORDER BY name' \
30+
"SELECT cycle, name, status FROM task_pool WHERE cycle=='20140102T0000Z' ORDER BY name" \
3131
>'taskpool.out'
3232
cmp_ok 'taskpool.out' <<'__OUT__'
3333
20140102T0000Z|foo|waiting

‎tests/flakyfunctional/job-submission/19-chatty.t

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ done
7070
# Task pool in database contains the correct states
7171
TEST_NAME="${TEST_NAME_BASE}-db-task-pool"
7272
DB_FILE="${WORKFLOW_RUN_DIR}/log/db"
73-
QUERY='SELECT cycle, name, status FROM task_states WHERE name LIKE "nh%"'
73+
QUERY="SELECT cycle, name, status FROM task_states WHERE name LIKE 'nh%'"
7474
run_ok "$TEST_NAME" sqlite3 "$DB_FILE" "$QUERY"
7575
sort "${TEST_NAME}.stdout" > "${TEST_NAME}.stdout.sorted"
7676
cmp_ok "${TEST_NAME}.stdout.sorted" << '__OUT__'

‎tests/flakyfunctional/restart/46-stop-clock-time.t

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
dumpdbtables() {
2323
sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \
24-
'SELECT * FROM workflow_params WHERE key=="stop_clock_time";' \
24+
"SELECT * FROM workflow_params WHERE key=='stop_clock_time';" \
2525
>'stopclocktime.out'
2626
}
2727

‎tests/functional/cylc-cat-log/05-remote-tail.t

+1-2
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,7 @@ TEST_NAME="${TEST_NAME_BASE}-validate"
3535
run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}"
3636
#-------------------------------------------------------------------------------
3737
$SCP "${PWD}/bin/my-tailer.sh" \
38-
"${CYLC_TEST_HOST}:cylc-run/.bin/my-tailer.sh
39-
"
38+
"${CYLC_TEST_HOST}:cylc-run/.bin/my-tailer.sh"
4039
#-------------------------------------------------------------------------------
4140
# Run detached.
4241
workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}"

‎tests/functional/cylc-poll/11-event-time.t

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ workflow_run_ok "${TEST_NAME_BASE}" \
2727

2828
RUND="$RUN_DIR/${WORKFLOW_NAME}"
2929
sed -n 's/CYLC_JOB_EXIT_TIME=//p' "${RUND}/log/job/1/w1/NN/job.status" >'st-time.txt'
30-
sqlite3 "${RUND}/log/db" '
30+
sqlite3 "${RUND}/log/db" "
3131
SELECT time_run_exit FROM task_jobs
32-
WHERE cycle=="1" AND name=="w1" AND submit_num=="1"' >'db-time.txt'
32+
WHERE cycle=='1' AND name=='w1' AND submit_num=='1'" >'db-time.txt'
3333
run_ok "${TEST_NAME_BASE}-time-run-exit" diff -u 'st-time.txt' 'db-time.txt'
3434

3535
purge

‎tests/functional/cylc-set/00-set-succeeded.t

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,12 @@ reftest_run
2828
for TASK in foo bar
2929
do
3030
sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
31-
"SELECT status FROM task_states WHERE name is \"$TASK\"" > "${TASK}.1"
31+
"SELECT status FROM task_states WHERE name is '$TASK'" > "${TASK}.1"
3232

3333
cmp_ok ${TASK}.1 - <<<succeeded
3434

3535
sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
36-
"SELECT outputs FROM task_outputs WHERE name is \"$TASK\"" > "${TASK}.2"
36+
"SELECT outputs FROM task_outputs WHERE name is '$TASK'" > "${TASK}.2"
3737

3838
cmp_json \
3939
"check-${TASK}-outputs" \

‎tests/functional/cylc-set/03-set-failed.t

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ cylc stop --now --now --interval=2 --max-polls=5 "${WORKFLOW_NAME}"
4242

4343
# Check the DB records all the outputs.
4444
sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
45-
"SELECT outputs FROM task_outputs WHERE name is \"foo\"" > db-foo.1
45+
"SELECT outputs FROM task_outputs WHERE name is 'foo'" > db-foo.1
4646

4747
# Json string list of outputs from the db may not be ordered correctly.
4848
python3 - << __END__ > db-foo.2

‎tests/functional/cylc-set/04-switch.t

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ reftest_run
2828
# The branch-point task foo should be recorded as succeeded.
2929

3030
sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
31-
"SELECT status FROM task_states WHERE name is \"foo\"" > db-foo.2
31+
"SELECT status FROM task_states WHERE name is 'foo'" > db-foo.2
3232

3333
cmp_ok "db-foo.2" - << __OUT__
3434
succeeded
@@ -40,7 +40,7 @@ __OUT__
4040
# submitted, started
4141

4242
sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
43-
"SELECT outputs FROM task_outputs WHERE name is \"foo\"" > db-foo.1
43+
"SELECT outputs FROM task_outputs WHERE name is 'foo'" > db-foo.1
4444

4545
# Json string list of outputs from the db may not be ordered correctly.
4646
python3 - << __END__ > db-foo.2

‎tests/functional/cylc-set/05-expire.t

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,14 @@ install_and_validate
2626
reftest_run
2727

2828
sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
29-
"SELECT status FROM task_states WHERE name is \"bar\"" > db-bar.1
29+
"SELECT status FROM task_states WHERE name is 'bar'" > db-bar.1
3030

3131
cmp_ok "db-bar.1" - << __OUT__
3232
expired
3333
__OUT__
3434

3535
sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
36-
"SELECT outputs FROM task_outputs WHERE name is \"bar\"" > db-bar.2
36+
"SELECT outputs FROM task_outputs WHERE name is 'bar'" > db-bar.2
3737

3838
cmp_ok "db-bar.2" - << __OUT__
3939
{"expired": "(manually completed)"}

‎tests/functional/database/06-task-message.t

+3-3
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,16 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
2727
DB_FILE="$RUN_DIR/${WORKFLOW_NAME}/log/db"
2828

2929
NAME='select-task-events.out'
30-
sqlite3 "${DB_FILE}" '
30+
sqlite3 "${DB_FILE}" "
3131
SELECT
3232
cycle, name, event, message
3333
FROM
3434
task_events
3535
WHERE
36-
event GLOB "message *"
36+
event GLOB 'message *'
3737
ORDER BY
3838
event
39-
' >"${NAME}"
39+
" >"${NAME}"
4040
cmp_ok "${NAME}" <<'__SELECT__'
4141
1|t1|message critical|You are being critical
4242
1|t1|message info|You are normal

‎tests/functional/events/20-workflow-event-handlers.t

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
3737
cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
3838

3939
LOGD="$RUN_DIR/${WORKFLOW_NAME}/log"
40-
WORKFLOW_UUID="$(sqlite3 "${LOGD}/db" 'SELECT value FROM workflow_params WHERE key=="uuid_str"')"
40+
WORKFLOW_UUID="$(sqlite3 "${LOGD}/db" "SELECT value FROM workflow_params WHERE key=='uuid_str'")"
4141
LOG_FILE="${LOGD}/scheduler/log"
4242
grep_ok "\\[('workflow-event-handler-00', 'startup') ret_code\\] 0" "${LOG_FILE}"
4343
grep_ok "\\[('workflow-event-handler-00', 'startup') out\\] Your ${WORKFLOW_NAME} workflow has a startup event and URL http://myworkflows.com/${WORKFLOW_NAME}.html and workflow-priority as HIGH and workflow-UUID as ${WORKFLOW_UUID}." "${LOG_FILE}"

‎tests/functional/job-file-trap/00-sigusr1.t

+2-2
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ run_tests() {
4444
poll_grep_workflow_log 'vacated/USR1'
4545
sleep 1 # a bit of extra time for workflow db update to complete
4646
sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \
47-
'SELECT status FROM task_states WHERE name=="t1";' \
47+
"SELECT status FROM task_states WHERE name=='t1';" \
4848
>"${TEST_NAME}-db-t1" 2>'/dev/null'
4949
grep_ok "^\(submitted\|running\)$" "${TEST_NAME}-db-t1"
5050
# Start the job again and see what happens
@@ -55,7 +55,7 @@ run_tests() {
5555
poll_workflow_stopped
5656
# Test t1 status in DB
5757
sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \
58-
'SELECT status FROM task_states WHERE name=="t1";' >"${TEST_NAME}-db-t1"
58+
"SELECT status FROM task_states WHERE name=='t1';" >"${TEST_NAME}-db-t1"
5959
cmp_ok "${TEST_NAME}-db-t1" - <<<'succeeded'
6060
# Test reference
6161
grep_ok 'WORKFLOW REFERENCE TEST PASSED' "${WORKFLOW_RUN_DIR}/log/scheduler/log"

0 commit comments

Comments
 (0)