Skip to content

Commit 003ca45

Browse files
bedrogebedroge
authored
Merge pull request #153 from ocaisa/better_init
Make all Lmod-based shell initialisations more robust, add additional features to facilitate site integration
2 parents c2ab5c1 + 0d8e06e commit 003ca45

File tree

11 files changed

+514
-82
lines changed

11 files changed

+514
-82
lines changed

‎.github/workflows/modules/append_module.lua‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
setenv("INSIDE_GITHUB_ACTIONS", "true")
2+
-- Interfere with PATH so Lmod keeps a record
3+
prepend_path("PATH", "/snap/bin")

‎.github/workflows/modules/prepend_module.lua‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
setenv("INSIDE_GITHUB_ACTIONS", "true")
2+
-- Interfere with PATH so Lmod keeps a record
3+
prepend_path("PATH", "/snap/bin")

‎.github/workflows/scripts/test_init_scripts.sh‎

Lines changed: 91 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,17 @@ if [ -z ${EXPECTED_EASYBUILD_VERSION} ]; then
1111
exit 1
1212
fi
1313

14+
if [ -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
15+
echo "\$EESSI_SOFTWARE_SUBDIR_OVERRIDE has to be set (e.g., x86_64/intel/haswell) so we can do well defined string comparison for the architecture."
16+
exit 1
17+
fi
18+
1419
# initialize assert framework
1520
if [ ! -d assert.sh ]; then
1621
echo "assert.sh not cloned."
1722
echo ""
1823
echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`"
24+
echo "(see workflow file that calls this script for how to only clone specific commit if you are worried about security)"
1925
exit 1
2026
fi
2127
. assert.sh/assert.sh
@@ -28,12 +34,16 @@ for shell in ${SHELLS[@]}; do
2834
echo RUNNING TESTS FOR SHELL: $shell
2935
echo = | awk 'NF += (OFS = $_) + 100'
3036
if [[ ! " ${TEST_SHELLS[*]} " =~ [[:space:]]${shell}[[:space:]] ]]; then
31-
### EXCEPTION FOR CSH ###
3237
echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m"
3338
else
39+
if [ "$shell" = "csh" ]; then
40+
# make sure our .cshrc is empty before we begin as we will clobber it
41+
[ -f ~/.cshrc ] && mv ~/.cshrc ~/.cshrc_orig
42+
fi
43+
3444
# TEST 1: Source Script and check Module Output
35-
expected="Module for EESSI/$EESSI_VERSION loaded successfully"
36-
assert "$shell -c 'source init/lmod/$shell' 2>&1 " "${expected}"
45+
expected_pattern=".*EESSI has selected $EESSI_SOFTWARE_SUBDIR_OVERRIDE as the compatible CPU target for EESSI/$EESSI_VERSION.*"
46+
assert_raises "$shell -c 'source init/lmod/$shell' 2>&1 | grep -E \"${expected_pattern}\""
3747

3848
# TEST 2: Check if module overviews first section is the loaded EESSI module
3949
if [ "$shell" = "csh" ]; then
@@ -43,21 +53,22 @@ for shell in ${SHELLS[@]}; do
4353
echo "source init/lmod/$shell" > ~/.cshrc
4454
MODULE_SECTIONS=($($shell -c "module ov" 2>&1 | grep -e '---'))
4555
else
46-
MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
56+
MODULE_SECTIONS=($($shell -c "source init/lmod/$shell >/dev/null 2>&1; module ov 2>&1 | grep -e '---'"))
4757
fi
48-
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"
58+
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR_OVERRIDE/modules/all"
4959
assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"'
60+
# echo "${MODULE_SECTIONS[1]}" "$PATTERN"
5061

5162
# TEST 3: Check if module overviews second section is the EESSI init module
52-
assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules"
63+
assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/init/modules"
5364

5465
# TEST 4: Load EasyBuild module and check version
5566
# eb --version outputs: "This is EasyBuild 5.1.1 (framework: 5.1.1, easyblocks: 5.1.1) on host ..."
5667
if [ "$shell" = "csh" ]; then
5768
echo "source init/lmod/$shell" > ~/.cshrc
58-
command="$shell -c 'module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version' | cut -d \" \" -f4"
69+
command="$shell -c 'module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version' | tail -n 1 | awk '{print \$4}'"
5970
else
60-
command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version | cut -d \" \" -f4'"
71+
command="$shell -c 'source init/lmod/$shell >/dev/null 2>&1; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version' | tail -n 1 | awk '{print \$4}'"
6172
fi
6273
assert "$command" "$EXPECTED_EASYBUILD_VERSION"
6374

@@ -69,12 +80,82 @@ for shell in ${SHELLS[@]}; do
6980
EASYBUILD_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; which eb")
7081
fi
7182
# escape the dots in ${EASYBUILD_VERSION}
72-
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/EasyBuild/${EXPECTED_EASYBUILD_VERSION//./\\.}/bin/eb"
73-
echo "$EASYBUILD_PATH" | grep -E "$PATTERN"
83+
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR_OVERRIDE/software/EasyBuild/${EXPECTED_EASYBUILD_VERSION//./\\.}/bin/eb"
84+
# echo "$EASYBUILD_PATH" | grep -E "$PATTERN"
7485
assert_raises 'echo "$EASYBUILD_PATH" | grep -E "$PATTERN"'
86+
# echo "$EASYBUILD_PATH" "$PATTERN"
87+
88+
# TEST 6 and 7: Check the various options (EESSI_DEFAULT_MODULES_APPEND, EESSI_DEFAULT_MODULES_APPEND, EESSI_EXTRA_MODULEPATH) all work
89+
if [ "$shell" = "csh" ]; then
90+
echo "setenv EESSI_DEFAULT_MODULES_APPEND append_module" > ~/.cshrc
91+
echo "setenv EESSI_DEFAULT_MODULES_PREPEND prepend_module" >> ~/.cshrc
92+
echo "setenv EESSI_EXTRA_MODULEPATH .github/workflows/modules" >> ~/.cshrc
93+
echo "source init/lmod/$shell" >> ~/.cshrc
94+
TEST_LMOD_SYSTEM_DEFAULT_MODULES=$($shell -c 'echo $LMOD_SYSTEM_DEFAULT_MODULES')
95+
TEST_MODULEPATH=$($shell -c 'echo $MODULEPATH')
96+
elif [ "$shell" = "fish" ]; then
97+
TEST_LMOD_SYSTEM_DEFAULT_MODULES=$($shell -c 'set -x EESSI_DEFAULT_MODULES_APPEND append_module ; set -x EESSI_DEFAULT_MODULES_PREPEND prepend_module ; set -x EESSI_EXTRA_MODULEPATH .github/workflows/modules ; source init/lmod/'"$shell"' 2>/dev/null; echo $LMOD_SYSTEM_DEFAULT_MODULES')
98+
TEST_MODULEPATH=$($shell -c 'set -x EESSI_DEFAULT_MODULES_APPEND append_module ; set -x EESSI_DEFAULT_MODULES_PREPEND prepend_module ; set -x EESSI_EXTRA_MODULEPATH .github/workflows/modules ; source init/lmod/'"$shell"' 2>/dev/null; echo $MODULEPATH')
99+
else
100+
TEST_LMOD_SYSTEM_DEFAULT_MODULES=$($shell -c 'export EESSI_DEFAULT_MODULES_APPEND=append_module ; export EESSI_DEFAULT_MODULES_PREPEND=prepend_module ; export EESSI_EXTRA_MODULEPATH=.github/workflows/modules ; source init/lmod/'"$shell"' ; echo $LMOD_SYSTEM_DEFAULT_MODULES')
101+
TEST_MODULEPATH=$($shell -c 'export EESSI_DEFAULT_MODULES_APPEND=append_module ; export EESSI_DEFAULT_MODULES_PREPEND=prepend_module ; export EESSI_EXTRA_MODULEPATH=.github/workflows/modules ; source init/lmod/'"$shell"' 2>/dev/null; echo $MODULEPATH')
102+
fi
103+
LMOD_SYSTEM_DEFAULT_MODULES_PATTERN='^prepend_module:.*:append_module$'
104+
# echo "$TEST_LMOD_SYSTEM_DEFAULT_MODULES" AND "$LMOD_SYSTEM_DEFAULT_MODULES_PATTERN"
105+
assert_raises 'echo "$TEST_LMOD_SYSTEM_DEFAULT_MODULES" | grep -E "$LMOD_SYSTEM_DEFAULT_MODULES_PATTERN"'
106+
if [ "$shell" = "fish" ]; then
107+
MODULEPATH_PATTERN='\.github/workflows/modules$'
108+
else
109+
MODULEPATH_PATTERN=':\.github/workflows/modules$'
110+
fi
111+
# echo "$TEST_MODULEPATH" AND "$MODULEPATH_PATTERN"
112+
assert_raises 'echo "$TEST_MODULEPATH" | grep -E "$MODULEPATH_PATTERN"'
113+
114+
# TEST 8 and 9: Add a conditional test depending on whether we have the Lmod command is available locally or not (Ubuntu-based location for CI)
115+
if [ -d "$LMOD_PKG/init" ]; then
116+
echo "Running check for locally available Lmod with purge"
117+
if [ "$shell" = "csh" ]; then
118+
echo "source $LMOD_PKG/init/$shell" > ~/.cshrc
119+
echo "source init/lmod/$shell" >> ~/.cshrc
120+
TEST_EESSI_WITH_PURGE=$($shell -c 'echo')
121+
echo "source $LMOD_PKG/init/$shell" > ~/.cshrc
122+
echo "setenv EESSI_NO_MODULE_PURGE_ON_INIT 1" >> ~/.cshrc
123+
echo "source init/lmod/$shell" >> ~/.cshrc
124+
TEST_EESSI_WITHOUT_PURGE=$($shell -c 'echo $EESSI_NO_MODULE_PURGE_ON_INIT')
125+
elif [ "$shell" = "fish" ]; then
126+
TEST_EESSI_WITH_PURGE=$($shell -c "source $LMOD_PKG/init/$shell 2>/dev/null ; source init/lmod/$shell 2>/dev/null")
127+
TEST_EESSI_WITHOUT_PURGE=$($shell -c "set -x EESSI_NO_MODULE_PURGE_ON_INIT 1 ; source $LMOD_PKG/init/$shell 2>/dev/null ; source init/lmod/$shell 2>/dev/null")
128+
else
129+
TEST_EESSI_WITH_PURGE=$($shell -c "source $LMOD_PKG/init/$shell 2>/dev/null ; source init/lmod/$shell 2>/dev/null")
130+
TEST_EESSI_WITHOUT_PURGE=$($shell -c "export EESSI_NO_MODULE_PURGE_ON_INIT=1 ; source $LMOD_PKG/init/$shell 2>/dev/null ; source init/lmod/$shell 2>/dev/null")
131+
fi
132+
# In the first case we should have the test and in the second case we shouldn't
133+
pattern="Modules purged before initialising EESSI"
134+
echo $TEST_EESSI_WITH_PURGE
135+
assert_raises 'echo "$TEST_EESSI_WITH_PURGE" | grep "$pattern"'
136+
# this case should raise 1
137+
echo $TEST_EESSI_WITHOUT_PURGE
138+
assert_raises 'echo "$TEST_EESSI_WITHOUT_PURGE" | grep "$pattern"' 1
139+
fi
140+
141+
# Optional test 10, check if the prompt has been updated
142+
if [ "$shell" = "bash" ] || [ "$shell" = "ksh" ] || [ "$shell" = "zsh" ]; then
143+
# Typically this is a non-interactive shell, so manually unset PS1 and reset to a non-exported variable when testing
144+
TEST_EESSI_PS1_UPDATE=$($shell -c "unset PS1 ; PS1='$ ' ; source init/lmod/$shell 2>/dev/null ; echo \"\$PS1\"")
145+
TEST_EESSI_NO_PS1_UPDATE=$($shell -c "unset PS1 ; source init/lmod/$shell 2>/dev/null ; echo \"\$PS1\"")
146+
pattern="{EESSI/${EESSI_VERSION}} "
147+
assert_raises 'echo "$TEST_EESSI_PS1_UPDATE" | grep "$pattern"'
148+
assert_raises 'echo "$TEST_EESSI_NO_PS1_UPDATE" | grep "$pattern"' 1
149+
fi
75150

76151
# End Test Suite
77152
assert_end "source_eessi_$shell"
153+
154+
if [ "$shell" = "csh" ]; then
155+
# Restore our .cshrc
156+
[ -f ~/.cshrc_orig ] && mv ~/.cshrc_orig ~/.cshrc
157+
fi
158+
78159
fi
79160
done
80161

‎.github/workflows/tests_eessi_module.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -109,11 +109,11 @@ jobs:
109109
# First do (and undo) the Lmod initialisation
110110
export MODULEPATH=init/modules
111111
# Turn on debug output in case we want to take a look
112-
export EESSI_DEBUG_INIT=true
112+
export EESSI_MODULE_DEBUG_INIT=true
113113
CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath)
114114
module load EESSI/${{matrix.EESSI_VERSION}}
115-
# EESSI_DEBUG_INIT/EESSI_ARCHDETECT_OPTIONS only relevant for Lmod init
116-
unset EESSI_DEBUG_INIT
115+
# EESSI_MODULE_DEBUG_INIT/EESSI_ARCHDETECT_OPTIONS only relevant for Lmod init
116+
unset EESSI_MODULE_DEBUG_INIT
117117
# Store all relevant environment variables
118118
env | grep -E '(^EESSI_|^LMOD_RC|^LMOD_PACKAGE_PATH|^MODULEPATH)' | grep -v EESSI_ARCHDETECT_OPTIONS | sort > "${moduleoutfile}"
119119
module unload EESSI/${{matrix.EESSI_VERSION}}
@@ -200,7 +200,7 @@ jobs:
200200
fi
201201
202202
# Turn on debug output in case we want to take a look
203-
export EESSI_DEBUG_INIT=true
203+
export EESSI_MODULE_DEBUG_INIT=true
204204
205205
initial_env_file="initial_env.txt"
206206
module_cycled_file="load_unload_cycle.txt"

‎.github/workflows/tests_init_module.yml‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,18 +46,34 @@ jobs:
4646
sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/lmod/${shell}
4747
done
4848
49-
- name: Clone assert.sh script
50-
run: git clone https://github.com/lehmannro/assert.sh.git
49+
- name: Clone assert.sh at pinned commit
50+
run: |
51+
ASSERT_COMMIT_SHA=fe359e341670f1e8e86a3804ca00e5a3ebc30fa4
52+
git clone --no-checkout https://github.com/lehmannro/assert.sh.git
53+
cd assert.sh
54+
git fetch --depth 1 origin $ASSERT_COMMIT_SHA
55+
git checkout $ASSERT_COMMIT_SHA
5156
5257
- name: Install missing shells
5358
run: |
5459
sudo apt update
5560
sudo apt install zsh ksh fish tcsh
5661
echo "# INIT ZSH" > ~/.zshrc
5762
58-
- name: Run tests for available shells
63+
- name: Run tests for available shells without system Lmod
5964
run: |
65+
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
66+
export EESSI_VERSION=${{matrix.EESSI_VERSION}}
67+
export EXPECTED_EASYBUILD_VERSION=${{matrix.EXPECTED_EASYBUILD_VERSION}}
68+
.github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" "csh"
69+
- name: Run tests for available shells with system Lmod
70+
run: |
71+
# We also want to perform the same test when there is an Lmod version available on the system
72+
sudo apt install lmod
73+
source /usr/share/lmod/lmod/init/bash
74+
export MODULEPATH=/usr/share/lmod/lmod/modulefiles
75+
module avail
76+
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
6077
export EESSI_VERSION=${{matrix.EESSI_VERSION}}
6178
export EXPECTED_EASYBUILD_VERSION=${{matrix.EXPECTED_EASYBUILD_VERSION}}
6279
.github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" "csh"
63-

0 commit comments

Comments
 (0)