diff --git a/.github/pseudo-cluster/reframe/docker-entrypoint.sh b/.github/pseudo-cluster/reframe/docker-entrypoint.sh
index 71eb08fc03..a01dba35f2 100755
--- a/.github/pseudo-cluster/reframe/docker-entrypoint.sh
+++ b/.github/pseudo-cluster/reframe/docker-entrypoint.sh
@@ -8,11 +8,13 @@ sudo service munge start
cp -r /usr/local/share/reframe .
cd reframe
./bootstrap.sh
-pip install pytest-cov
+pip install coverage
+source $HOME/.profile
echo "Running unittests with backend scheduler: ${BACKEND}"
tempdir=$(mktemp -d -p /scratch)
-TMPDIR=$tempdir ./test_reframe.py --cov=reframe --cov-report=xml \
+TMPDIR=$tempdir coverage run --source=reframe ./test_reframe.py \
--rfm-user-config=ci-scripts/configs/ci-cluster.py \
--rfm-user-system=pseudo-cluster:compute-${BACKEND:-squeue}
+coverage xml -o coverage.xml
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3831e4257a..5bda327307 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -7,7 +7,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+ python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
@@ -19,25 +19,9 @@ jobs:
./bootstrap.sh
- name: Generic Unittests
run: |
- pip install pytest-cov
- ./test_reframe.py --cov=reframe --cov-report=xml
- - name: Upload coverage reports
- uses: codecov/codecov-action@v4.2.0
-
- unittest-py-eol:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python-version: ['3.6', '3.7']
- steps:
- - uses: actions/checkout@v4
- - name: Build Image for Python ${{ matrix.python-version }}
- run: |
- docker build --build-arg PYTHON_VERSION=${{ matrix.python-version }} -f ci-scripts/dockerfiles/reframe-python.dockerfile -t reframe-python${{ matrix.python-version }}:latest .
- - name: Run Unittests
- run: |
- docker run --name reframe-python${{ matrix.python-version }} reframe-python${{ matrix.python-version }}:latest
- docker cp reframe-python${{ matrix.python-version }}:/home/rfmuser/reframe/coverage.xml .
+ pip install coverage
+ coverage run --source=reframe ./test_reframe.py
+ coverage xml -o coverage.xml
- name: Upload coverage reports
uses: codecov/codecov-action@v4.2.0
@@ -45,7 +29,7 @@ jobs:
runs-on: macos-latest
strategy:
matrix:
- python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+ python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
@@ -57,8 +41,9 @@ jobs:
./bootstrap.sh
- name: Generic Unittests
run: |
- pip install pytest-cov
- ./test_reframe.py --cov=reframe --cov-report=xml
+ pip install coverage
+ coverage run --source=reframe ./test_reframe.py
+ coverage xml -o coverage.xml
- name: Upload coverage reports
uses: codecov/codecov-action@v4.2.0
@@ -66,7 +51,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- modules-version: [lmod, lmod77, tmod32, tmod4]
+ modules-version: [envmodules, lmod, spack]
steps:
- uses: actions/checkout@v4
- name: Login to GitHub Container Registry
@@ -121,7 +106,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+ python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
steps:
- uses: actions/checkout@v4
- name: Setup up Python ${{ matrix.python-version }}
@@ -144,7 +129,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+ python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
diff --git a/.github/workflows/test-flux.yaml b/.github/workflows/test-flux.yaml
index cd10473543..fbe6b0dd4d 100644
--- a/.github/workflows/test-flux.yaml
+++ b/.github/workflows/test-flux.yaml
@@ -10,7 +10,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- container: ['fluxrm/flux-sched:focal']
+ container: ['fluxrm/flux-sched:noble']
container:
image: ${{ matrix.container }}
@@ -30,7 +30,6 @@ jobs:
run: |
apt-get update && apt-get install -y python3-pip
./bootstrap.sh
- pip install pytest-cov
export PATH=$PWD/bin:$PATH
which reframe
@@ -41,6 +40,7 @@ jobs:
which reframe
flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -l
flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -r
- flux start python3 ./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=examples/howto/flux/settings.py
+ flux start coverage run --source=reframe ./test_reframe.py --rfm-user-config=examples/howto/flux/settings.py
+ coverage xml -o coverage.xml
- name: Upload coverage reports
uses: codecov/codecov-action@v4.2.0
diff --git a/ci-scripts/configs/cscs-ci.py b/ci-scripts/configs/cscs-ci.py
deleted file mode 100644
index 221b58a1c9..0000000000
--- a/ci-scripts/configs/cscs-ci.py
+++ /dev/null
@@ -1,153 +0,0 @@
-# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
-# ReFrame Project Developers. See the top-level LICENSE file for details.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-#
-# CSCS CI settings
-#
-
-import reframe.utility.osext as osext
-
-
-site_configuration = {
- 'systems': [
- {
- 'name': 'daint',
- 'descr': 'Piz Daint CI nodes',
- 'hostnames': [
- 'daint'
- ],
- 'modules_system': 'tmod',
- 'resourcesdir': '/apps/common/UES/reframe/resources',
- 'partitions': [
- {
- 'name': 'gpu',
- 'scheduler': 'slurm',
- 'time_limit': '10m',
- 'access': [
- '--constraint=gpu',
- '--partition=cscsci',
- f'--account={osext.osgroup()}'
- ],
- 'environs': [
- 'builtin'
- ],
- 'descr': 'Hybrid nodes (Haswell/P100)',
- 'max_jobs': 100,
- 'resources': [
- {
- 'name': 'switches',
- 'options': [
- '--switches={num_switches}'
- ]
- }
- ],
- 'launcher': 'srun'
- }
- ]
- },
- {
- 'name': 'dom',
- 'descr': 'Dom TDS',
- 'hostnames': [
- 'dom'
- ],
- 'modules_system': 'tmod',
- 'resourcesdir': '/apps/common/UES/reframe/resources',
- 'partitions': [
- {
- 'name': 'slurm',
- 'scheduler': 'slurm',
- 'time_limit': '10m',
- 'access': [
- '--constraint=gpu',
- f'--account={osext.osgroup()}'
- ],
- 'environs': [
- 'builtin'
- ],
- 'descr': 'Hybrid nodes (Haswell/P100)',
- 'max_jobs': 100,
- 'resources': [
- {
- 'name': 'switches',
- 'options': [
- '--switches={num_switches}'
- ]
- }
- ],
- 'launcher': 'srun'
- },
- {
- 'name': 'pbs',
- 'scheduler': 'pbs',
- 'time_limit': '10m',
- 'access': [
- 'proc=gpu',
- f'-A {osext.osgroup()}'
- ],
- 'environs': [
- 'builtin'
- ],
- 'descr': 'Hybrid nodes (Haswell/P100)',
- 'max_jobs': 100,
- 'launcher': 'mpiexec'
- },
- {
- 'name': 'torque',
- 'scheduler': 'torque',
- 'time_limit': '10m',
- 'access': [
- '-l proc=gpu',
- f'-A {osext.osgroup()}'
- ],
- 'environs': [
- 'builtin'
- ],
- 'descr': 'Hybrid nodes (Haswell/P100)',
- 'max_jobs': 100,
- 'launcher': 'mpiexec'
- }
- ]
- },
- {
- 'name': 'tsa',
- 'descr': 'Tsa MCH',
- 'hostnames': [
- r'tsa-\w+\d+'
- ],
- 'modules_system': 'tmod',
- 'resourcesdir': '/apps/common/UES/reframe/resources',
- 'partitions': [
- {
- 'name': 'cn',
- 'scheduler': 'slurm',
- 'access': [
- '--partition=cn-regression'
- ],
- 'environs': [
- 'builtin'
- ],
- 'descr': 'Tsa compute nodes',
- 'max_jobs': 20,
- 'resources': [
- {
- 'name': '_rfm_gpu',
- 'options': [
- '--gres=gpu:{num_gpus_per_node}'
- ]
- }
- ],
- 'launcher': 'srun'
- }
- ]
- },
- ],
- 'general': [
- {
- 'check_search_path': ['checks/'],
- 'check_search_recursive': True
- }
- ]
-}
diff --git a/ci-scripts/configs/envmod.py b/ci-scripts/configs/envmod.py
new file mode 100644
index 0000000000..6d31563ca0
--- /dev/null
+++ b/ci-scripts/configs/envmod.py
@@ -0,0 +1,23 @@
+# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+site_configuration = {
+ 'systems': [
+ {
+ 'name': 'envmodsys',
+ 'descr': 'Generic system using Environment Modules',
+ 'hostnames': ['.*'],
+ 'modules_system': 'envmod',
+ 'partitions': [
+ {
+ 'name': 'default',
+ 'scheduler': 'local',
+ 'launcher': 'local',
+ 'environs': ['builtin']
+ }
+ ]
+ }
+ ]
+}
diff --git a/ci-scripts/configs/lmod.py b/ci-scripts/configs/lmod.py
index e5fe1983ee..0ac569b197 100644
--- a/ci-scripts/configs/lmod.py
+++ b/ci-scripts/configs/lmod.py
@@ -3,15 +3,11 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-#
-# Generic fallback configuration
-#
-
site_configuration = {
'systems': [
{
- 'name': 'generic',
- 'descr': 'Generic example system',
+ 'name': 'lmodsys',
+ 'descr': 'Generic system using Lmod',
'hostnames': ['.*'],
'modules_system': 'lmod',
'partitions': [
@@ -22,49 +18,6 @@
'environs': ['builtin']
}
]
- },
- ],
- 'environments': [
- {
- 'name': 'builtin',
- 'cc': 'cc',
- 'cxx': '',
- 'ftn': ''
- },
- ],
- 'logging': [
- {
- 'handlers': [
- {
- 'type': 'stream',
- 'name': 'stdout',
- 'level': 'info',
- 'format': '%(message)s'
- },
- {
- 'type': 'file',
- 'level': 'debug',
- 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501
- 'append': False
- }
- ],
- 'handlers_perflog': [
- {
- 'type': 'filelog',
- 'prefix': '%(check_system)s/%(check_partition)s',
- 'level': 'info',
- 'format': (
- '%(check_job_completion_time)s|reframe %(version)s|'
- '%(check_info)s|jobid=%(check_jobid)s|'
- '%(check_perf_var)s=%(check_perf_value)s|'
- 'ref=%(check_perf_ref)s '
- '(l=%(check_perf_lower_thres)s, '
- 'u=%(check_perf_upper_thres)s)|'
- '%(check_perf_unit)s'
- ),
- 'append': True
- }
- ]
}
- ],
+ ]
}
diff --git a/ci-scripts/configs/spack.py b/ci-scripts/configs/spack.py
index 7f2862bbac..d95d8c885a 100644
--- a/ci-scripts/configs/spack.py
+++ b/ci-scripts/configs/spack.py
@@ -3,15 +3,11 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-#
-# Generic fallback configuration
-#
-
site_configuration = {
'systems': [
{
- 'name': 'generic',
- 'descr': 'Generic example system',
+ 'name': 'spacksys',
+ 'descr': 'Generic system using Spack',
'hostnames': ['.*'],
'modules_system': 'spack',
'partitions': [
@@ -22,49 +18,6 @@
'environs': ['builtin']
}
]
- },
- ],
- 'environments': [
- {
- 'name': 'builtin',
- 'cc': 'cc',
- 'cxx': '',
- 'ftn': ''
- },
- ],
- 'logging': [
- {
- 'handlers': [
- {
- 'type': 'stream',
- 'name': 'stdout',
- 'level': 'info',
- 'format': '%(message)s'
- },
- {
- 'type': 'file',
- 'level': 'debug',
- 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501
- 'append': False
- }
- ],
- 'handlers_perflog': [
- {
- 'type': 'filelog',
- 'prefix': '%(check_system)s/%(check_partition)s',
- 'level': 'info',
- 'format': (
- '%(check_job_completion_time)s|reframe %(version)s|'
- '%(check_info)s|jobid=%(check_jobid)s|'
- '%(check_perf_var)s=%(check_perf_value)s|'
- 'ref=%(check_perf_ref)s '
- '(l=%(check_perf_lower_thres)s, '
- 'u=%(check_perf_upper_thres)s)|'
- '%(check_perf_unit)s'
- ),
- 'append': True
- }
- ]
}
- ],
+ ]
}
diff --git a/ci-scripts/configs/tmod32.py b/ci-scripts/configs/tmod32.py
deleted file mode 100644
index c82594db06..0000000000
--- a/ci-scripts/configs/tmod32.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
-# ReFrame Project Developers. See the top-level LICENSE file for details.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-#
-# Generic fallback configuration
-#
-
-site_configuration = {
- 'systems': [
- {
- 'name': 'generic',
- 'descr': 'Generic example system',
- 'hostnames': ['.*'],
- 'modules_system': 'tmod32',
- 'partitions': [
- {
- 'name': 'default',
- 'scheduler': 'local',
- 'launcher': 'local',
- 'environs': ['builtin']
- }
- ]
- },
- ],
- 'environments': [
- {
- 'name': 'builtin',
- 'cc': 'cc',
- 'cxx': '',
- 'ftn': ''
- },
- ],
- 'logging': [
- {
- 'handlers': [
- {
- 'type': 'stream',
- 'name': 'stdout',
- 'level': 'info',
- 'format': '%(message)s'
- },
- {
- 'type': 'file',
- 'level': 'debug',
- 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501
- 'append': False
- }
- ],
- 'handlers_perflog': [
- {
- 'type': 'filelog',
- 'prefix': '%(check_system)s/%(check_partition)s',
- 'level': 'info',
- 'format': (
- '%(check_job_completion_time)s|reframe %(version)s|'
- '%(check_info)s|jobid=%(check_jobid)s|'
- '%(check_perf_var)s=%(check_perf_value)s|'
- 'ref=%(check_perf_ref)s '
- '(l=%(check_perf_lower_thres)s, '
- 'u=%(check_perf_upper_thres)s)|'
- '%(check_perf_unit)s'
- ),
- 'append': True
- }
- ]
- }
- ],
-}
diff --git a/ci-scripts/configs/tmod4.py b/ci-scripts/configs/tmod4.py
deleted file mode 100644
index 1b9d4c6531..0000000000
--- a/ci-scripts/configs/tmod4.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
-# ReFrame Project Developers. See the top-level LICENSE file for details.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-#
-# Generic fallback configuration
-#
-
-site_configuration = {
- 'systems': [
- {
- 'name': 'generic',
- 'descr': 'Generic example system',
- 'hostnames': ['.*'],
- 'modules_system': 'tmod4',
- 'partitions': [
- {
- 'name': 'default',
- 'scheduler': 'local',
- 'launcher': 'local',
- 'environs': ['builtin']
- }
- ]
- },
- ],
- 'environments': [
- {
- 'name': 'builtin',
- 'cc': 'cc',
- 'cxx': '',
- 'ftn': ''
- },
- ],
- 'logging': [
- {
- 'handlers': [
- {
- 'type': 'stream',
- 'name': 'stdout',
- 'level': 'info',
- 'format': '%(message)s'
- },
- {
- 'type': 'file',
- 'level': 'debug',
- 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501
- 'append': False
- }
- ],
- 'handlers_perflog': [
- {
- 'type': 'filelog',
- 'prefix': '%(check_system)s/%(check_partition)s',
- 'level': 'info',
- 'format': (
- '%(check_job_completion_time)s|reframe %(version)s|'
- '%(check_info)s|jobid=%(check_jobid)s|'
- '%(check_perf_var)s=%(check_perf_value)s|'
- 'ref=%(check_perf_ref)s '
- '(l=%(check_perf_lower_thres)s, '
- 'u=%(check_perf_upper_thres)s)|'
- '%(check_perf_unit)s'
- ),
- 'append': True
- }
- ]
- }
- ],
-}
diff --git a/ci-scripts/dockerfiles/Lmod.dockerfile b/ci-scripts/dockerfiles/Lmod.dockerfile
index b908d76022..4240d9eb5a 100644
--- a/ci-scripts/dockerfiles/Lmod.dockerfile
+++ b/ci-scripts/dockerfiles/Lmod.dockerfile
@@ -1,8 +1,8 @@
-FROM ubuntu:20.04
+FROM ubuntu:24.04
ENV TZ=Europe/Zurich
ENV DEBIAN_FRONTEND=noninteractive
-ENV _LMOD_VER=8.4.12
+ENV _LMOD_VER=9.0.4
# Setup apt
RUN \
@@ -11,7 +11,7 @@ RUN \
update-ca-certificates
# Required utilities
-RUN apt-get -y install wget
+RUN apt-get -y install bc wget
# Install Lmod
RUN \
diff --git a/ci-scripts/dockerfiles/eb-spack-howto.dockerfile b/ci-scripts/dockerfiles/eb-spack-howto.dockerfile
index f308189495..c0a92e2962 100644
--- a/ci-scripts/dockerfiles/eb-spack-howto.dockerfile
+++ b/ci-scripts/dockerfiles/eb-spack-howto.dockerfile
@@ -3,10 +3,10 @@
#
-FROM ghcr.io/reframe-hpc/lmod:8.4.12
+FROM ghcr.io/reframe-hpc/lmod:9.0.4
-ENV _SPACK_VER=0.22.2
-ENV _EB_VER=4.9.4
+ENV _SPACK_VER=1.1.0
+ENV _EB_VER=5.1.2
# Install ReFrame unit test requirements
@@ -19,10 +19,8 @@ RUN useradd -ms /bin/bash rfmuser
USER rfmuser
# Install Spack
-RUN git clone --branch v${_SPACK_VER} https://github.com/spack/spack ~/spack && \
- cd ~/spack
-
-RUN pip3 install easybuild==${_EB_VER}
+RUN git clone --branch v${_SPACK_VER} --depth 1 https://github.com/spack/spack ~/spack
+RUN pip3 install --break-system-packages easybuild==${_EB_VER}
ENV PATH="/home/rfmuser/.local/bin:${PATH}"
@@ -35,6 +33,6 @@ RUN ./bootstrap.sh
RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/rfmuser/spack/share/spack/setup-env.sh' > /home/rfmuser/setup.sh
-ENV BASH_ENV /home/rfmuser/setup.sh
+ENV BASH_ENV=/home/rfmuser/setup.sh
-CMD ["/bin/bash", "-c", "./bin/reframe --system=tutorialsys -r -C examples/tutorial/config/baseline_modules.py -R -c examples/tutorial/easybuild/eb_test.py -c examples/tutorial/spack/spack_test.py"]
+CMD ["/bin/bash", "-c", "./bin/reframe --system=tutorialsys --exec-policy=serial -r -C examples/tutorial/config/baseline_modules.py -R -c examples/tutorial/easybuild/eb_test.py -c examples/tutorial/spack/spack_test.py"]
diff --git a/ci-scripts/dockerfiles/Tmod4.dockerfile b/ci-scripts/dockerfiles/envmodules.dockerfile
similarity index 62%
rename from ci-scripts/dockerfiles/Tmod4.dockerfile
rename to ci-scripts/dockerfiles/envmodules.dockerfile
index d8f7fb66aa..e9360e38c0 100644
--- a/ci-scripts/dockerfiles/Tmod4.dockerfile
+++ b/ci-scripts/dockerfiles/envmodules.dockerfile
@@ -1,8 +1,8 @@
-FROM ubuntu:20.04
+FROM ubuntu:24.04
ENV TZ=Europe/Zurich
ENV DEBIAN_FRONTEND=noninteractive
-ENV _TMOD_VER=4.6.0
+ENV _ENVMOD_VER=5.6.1
# Setup apt
RUN \
@@ -13,14 +13,14 @@ RUN \
# Required utilities
RUN apt-get -y install wget less
-# Install Tmod4
+# Install Environment Modules
RUN \
apt-get -y install autoconf tcl-dev && \
- wget -q https://github.com/cea-hpc/modules/archive/v${_TMOD_VER}.tar.gz -O tmod.tar.gz && \
+ wget -q https://github.com/cea-hpc/modules/archive/v${_ENVMOD_VER}.tar.gz -O tmod.tar.gz && \
tar xzf tmod.tar.gz && \
- cd modules-${_TMOD_VER} && \
+ cd modules-${_ENVMOD_VER} && \
./configure && make install && \
- cd .. && rm -rf tmod.tar.gz modules-${_TMOD_VER} && \
+ cd .. && rm -rf tmod.tar.gz modules-${_ENVMOD_VER} && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
diff --git a/ci-scripts/dockerfiles/reframe-tmod4.dockerfile b/ci-scripts/dockerfiles/reframe-envmodules.dockerfile
similarity index 58%
rename from ci-scripts/dockerfiles/reframe-tmod4.dockerfile
rename to ci-scripts/dockerfiles/reframe-envmodules.dockerfile
index 360a448991..6ab319c745 100644
--- a/ci-scripts/dockerfiles/reframe-tmod4.dockerfile
+++ b/ci-scripts/dockerfiles/reframe-envmodules.dockerfile
@@ -2,7 +2,7 @@
# Execute this from the top-level ReFrame source directory
#
-FROM ghcr.io/reframe-hpc/tmod:4.6.0
+FROM ghcr.io/reframe-hpc/envmodules:5.6.1
# ReFrame requirements
@@ -21,6 +21,7 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/
WORKDIR /home/rfmuser/reframe
RUN ./bootstrap.sh
-RUN pip install pytest-cov
+RUN pip install --break-system-packages coverage
+ENV BASH_ENV=/home/rfmuser/.profile
-CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/tmod4.py"]
+CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py --rfm-user-config=ci-scripts/configs/envmod.py; coverage xml -o coverage.xml"]
diff --git a/ci-scripts/dockerfiles/reframe-lmod.dockerfile b/ci-scripts/dockerfiles/reframe-lmod.dockerfile
index f95dae4b67..2bd098d4a9 100644
--- a/ci-scripts/dockerfiles/reframe-lmod.dockerfile
+++ b/ci-scripts/dockerfiles/reframe-lmod.dockerfile
@@ -3,7 +3,7 @@
#
-FROM ghcr.io/reframe-hpc/lmod:8.4.12
+FROM ghcr.io/reframe-hpc/lmod:9.0.4
# Install ReFrame unit test requirements
RUN apt-get -y update && \
@@ -20,6 +20,8 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/
WORKDIR /home/rfmuser/reframe
RUN ./bootstrap.sh
-RUN pip install pytest-cov
+RUN pip install --break-system-packages coverage
+RUN echo '. /usr/local/lmod/lmod/init/profile' >> /home/rfmuser/.profile
+ENV BASH_ENV=/home/rfmuser/.profile
-CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/lmod.py"]
+CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py -v --rfm-user-config=ci-scripts/configs/lmod.py; coverage xml -o coverage.xml"]
diff --git a/ci-scripts/dockerfiles/reframe-lmod77.dockerfile b/ci-scripts/dockerfiles/reframe-lmod77.dockerfile
deleted file mode 100644
index 5ee2500394..0000000000
--- a/ci-scripts/dockerfiles/reframe-lmod77.dockerfile
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Execute this from the top-level ReFrame source directory
-#
-
-
-FROM ghcr.io/reframe-hpc/lmod:7.7
-
-# Install ReFrame unit test requirements
-RUN apt-get -y update && \
- apt-get -y install gcc git make python3 python3-pip
-
-# ReFrame user
-RUN useradd -ms /bin/bash rfmuser
-
-USER rfmuser
-
-# Install ReFrame from the current directory
-COPY --chown=rfmuser . /home/rfmuser/reframe/
-
-WORKDIR /home/rfmuser/reframe
-
-RUN ./bootstrap.sh
-RUN pip install pytest-cov
-
-CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/lmod.py"]
diff --git a/ci-scripts/dockerfiles/reframe-python.dockerfile b/ci-scripts/dockerfiles/reframe-python.dockerfile
index d80d09f542..13156b8ed8 100644
--- a/ci-scripts/dockerfiles/reframe-python.dockerfile
+++ b/ci-scripts/dockerfiles/reframe-python.dockerfile
@@ -3,7 +3,7 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-ARG PYTHON_VERSION=3.6
+ARG PYTHON_VERSION=3.9
FROM docker.io/python:${PYTHON_VERSION}
@@ -17,7 +17,8 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/
WORKDIR /home/rfmuser/reframe
-RUN ./bootstrap.sh +docs
-RUN pip install pytest-cov
+RUN ./bootstrap.sh
+RUN pip install --break-system-packages coverage
+ENV BASH_ENV=/home/rfmuser/.profile
-CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml"]
+CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py; coverage xml -o coverage.xml"]
diff --git a/ci-scripts/dockerfiles/reframe-spack.dockerfile b/ci-scripts/dockerfiles/reframe-spack.dockerfile
new file mode 100644
index 0000000000..c8c2bebd97
--- /dev/null
+++ b/ci-scripts/dockerfiles/reframe-spack.dockerfile
@@ -0,0 +1,33 @@
+#
+# Execute this from the top-level ReFrame source directory
+#
+
+
+FROM ubuntu:24.04
+
+ENV _SPACK_VER=1.1.0
+
+# Install ReFrame unit test requirements
+RUN apt-get -y update && \
+ apt-get -y install gcc git make python3 python3-pip
+
+# ReFrame user
+RUN useradd -ms /bin/bash rfmuser
+
+USER rfmuser
+
+# Install Spack
+RUN git clone --branch v${_SPACK_VER} https://github.com/spack/spack ~/spack
+
+# Install ReFrame from the current directory
+COPY --chown=rfmuser . /home/rfmuser/reframe/
+
+WORKDIR /home/rfmuser/reframe
+
+RUN ./bootstrap.sh
+RUN pip install --break-system-packages coverage
+
+RUN echo '. /home/rfmuser/spack/share/spack/setup-env.sh' >> /home/rfmuser/.profile
+ENV BASH_ENV=/home/rfmuser/.profile
+
+CMD ["/bin/bash", "-c", "coverage run --source=reframe ./test_reframe.py -v --rfm-user-config=ci-scripts/configs/spack.py; coverage xml -o coverage.xml"]
diff --git a/ci-scripts/dockerfiles/reframe-tmod32.dockerfile b/ci-scripts/dockerfiles/reframe-tmod32.dockerfile
deleted file mode 100644
index df9418589f..0000000000
--- a/ci-scripts/dockerfiles/reframe-tmod32.dockerfile
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Execute this from the top-level ReFrame source directory
-#
-
-FROM ghcr.io/reframe-hpc/tmod:3.2.10
-
-# ReFrame requirements
-RUN yum -y install gcc make git python3 python3-pip
-
-# ReFrame user
-RUN useradd -ms /bin/bash rfmuser
-RUN pip3 install pytest-cov
-
-USER rfmuser
-
-# Install ReFrame from the current directory
-COPY --chown=rfmuser . /home/rfmuser/reframe/
-
-WORKDIR /home/rfmuser/reframe
-
-RUN ./bootstrap.sh
-
-CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/tmod32.py"]
diff --git a/docs/config_reference.rst b/docs/config_reference.rst
index f8e8723df1..2aaa042242 100644
--- a/docs/config_reference.rst
+++ b/docs/config_reference.rst
@@ -161,25 +161,28 @@ System Configuration
The modules system that should be used for loading environment modules on this system.
Available values are the following:
- - ``tmod``: The classic Tcl implementation of the `environment modules `__ (version 3.2).
- - ``tmod31``: The classic Tcl implementation of the `environment modules `__ (version 3.1).
- A separate backend is required for Tmod 3.1, because Python bindings are different from Tmod 3.2.
- - ``tmod32``: A synonym of ``tmod``.
- - ``tmod4``: The `new environment modules `__ implementation (versions older than 4.1 are not supported).
+ - ``envmod``: The `new environment modules `__ implementation (versions older than 4.1 are not supported).
- ``lmod``: The `Lua implementation `__ of the environment modules.
- ``spack``: `Spack `__'s built-in mechanism for managing modules.
+ - ``tmod4``: (deprecated) Synonym of ``envmod``.
- ``nomod``: This is to denote that no modules system is used by this system.
Normally, upon loading the configuration of the system ReFrame checks that a sane installation exists for the modules system requested and will issue an error if it fails to find one.
The modules system sanity check is skipped when the :attr:`~config.general.resolve_module_conflicts` is set to :obj:`False`.
This is useful in cases where the current system does not have a modules system but the remote partitions have one and you would like ReFrame to generate the module commands.
- .. versionadded:: 3.4
+ .. versionadded:: 3.4
The ``spack`` backend is added.
- .. versionchanged:: 4.5.0
+ .. versionchanged:: 4.5.0
The modules system sanity check is skipped when the :attr:`config.general.resolve_module_conflicts` is not set.
+ .. versionchanged:: 4.10
+ The ``tmod``, ``tmod31``, ``tmod32`` backends are no more supported.
+
+ .. deprecated:: 4.10
+ The ``tmod4`` backend is deprecated; please use ``envmod`` instead.
+
.. py:attribute:: systems.modules
diff --git a/docs/howto.rst b/docs/howto.rst
index 97c72bf55b..9e3acece64 100644
--- a/docs/howto.rst
+++ b/docs/howto.rst
@@ -97,7 +97,7 @@ Integrating with EasyBuild
ReFrame integrates with the `EasyBuild `__ build automation framework, which allows you to use EasyBuild for building the source code of your test.
-Let's consider a simple ReFrame test that installs ``bzip2-1.0.6`` given the easyconfig `bzip2-1.0.6.eb `__ and checks that the installed version is correct.
+Let's consider a simple ReFrame test that installs ``zlib-1.3.1`` given the easyconfig `zlib-1.3.1.eb `__ and checks that the installed version is correct.
The following code block shows the check, highlighting the lines specific to this tutorial:
.. literalinclude:: ../examples/tutorial/easybuild/eb_test.py
@@ -138,7 +138,7 @@ ReFrame generates the following commands to build and install the easyconfig:
.. code-block:: bash
:caption: Run in the EasyBuild+Spack container.
- cat output/tutorialsys/default/builtin/BZip2EBCheck/rfm_build.sh
+ cat output/tutorialsys/default/builtin/ZlibEBCheck/rfm_build.sh
.. code-block:: bash
@@ -147,7 +147,7 @@ ReFrame generates the following commands to build and install the easyconfig:
export EASYBUILD_INSTALLPATH=${stagedir}/easybuild
export EASYBUILD_PREFIX=${stagedir}/easybuild
export EASYBUILD_SOURCEPATH=${stagedir}/easybuild
- eb bzip2-1.0.6.eb -f
+ eb zlib-1.3.1.eb -f
All the files generated by EasyBuild (sources, temporary files, installed software and the corresponding modules) are kept under the test's stage directory, thus the relevant EasyBuild environment variables are set.
@@ -169,13 +169,13 @@ This generated final run script is the following:
.. code-block:: bash
:caption: Run in the EasyBuild+Spack container.
- cat output/tutorialsys/default/builtin/BZip2EBCheck/rfm_job.sh
+ cat output/tutorialsys/default/builtin/ZlibEBCheck/rfm_job.sh
.. code-block:: bash
module use ${stagedir}/easybuild/modules/all
- module load bzip/1.0.6
- bzip2 --help
+ module load zlib/1.3.1
+ ls $LD_LIBRARY_PATH/libz.so.1.3.1
Packaging the installation
@@ -204,7 +204,7 @@ Integrating with Spack
ReFrame can also use `Spack `__ to build a software package and test it.
-The example shown here is the equivalent to the `EasyBuild <#integrating-with-easybuild>`__ one that built ``bzip2``.
+The example shown here is the equivalent to the `EasyBuild <#integrating-with-easybuild>`__ one that built ``zlib``.
Here is the test code:
.. literalinclude:: ../examples/tutorial/spack/spack_test.py
@@ -244,7 +244,7 @@ Here is what ReFrame generates as a build script for this example:
spack env create -d rfm_spack_env
spack -e rfm_spack_env config add "config:install_tree:root:opt/spack"
- spack -e rfm_spack_env add bzip2@1.0.6
+ spack -e rfm_spack_env add zlib@1.3.1
spack -e rfm_spack_env install
As you might have noticed ReFrame expects that Spack is already installed on the system.
@@ -262,12 +262,12 @@ Here is the stage directory structure:
│ │ └── darwin-catalina-skylake
│ ├── spack.lock
│ └── spack.yaml
- ├── rfm_BZip2SpackCheck_build.err
- ├── rfm_BZip2SpackCheck_build.out
- ├── rfm_BZip2SpackCheck_build.sh
- ├── rfm_BZip2SpackCheck_job.err
- ├── rfm_BZip2SpackCheck_job.out
- └── rfm_BZip2SpackCheck_job.sh
+ ├── rfm_ZlibSpackCheck_build.err
+ ├── rfm_ZlibSpackCheck_build.out
+ ├── rfm_ZlibSpackCheck_build.sh
+ ├── rfm_ZlibSpackCheck_job.err
+ ├── rfm_ZlibSpackCheck_job.out
+ └── rfm_ZlibSpackCheck_job.sh
Finally, here is the generated run script that ReFrame uses to run the test, once its build has succeeded:
@@ -276,8 +276,8 @@ Finally, here is the generated run script that ReFrame uses to run the test, onc
#!/bin/bash
spack env create -d rfm_spack_env
- eval `spack -e rfm_spack_env load --sh bzip2@1.0.6`
- bzip2 --help
+ eval `spack -e rfm_spack_env load --sh zlib@1.3.1`
+ pkg-config --libs zlib
From this point on, sanity and performance checking are exactly identical to any other ReFrame test.
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 7447db2b40..a24877df97 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,20 +1,13 @@
archspec==0.2.5
ClusterShell==1.9.3
-docutils==0.18.1; python_version < '3.9'
docutils==0.21.2; python_version >= '3.9'
fasteners==0.19; python_version < '3.10'
fasteners==0.20; python_version >= '3.10'
-jinja2==3.0.3; python_version == '3.6'
-jinja2==3.1.6; python_version >= '3.7'
+jinja2==3.1.6
jsonschema==3.2.0
-PyYAML==6.0.1; python_version < '3.8'
-PyYAML==6.0.3; python_version >= '3.8'
-semver==2.13.0; python_version == '3.6'
-semver==3.0.4; python_version >= '3.7'
-Sphinx==5.3.0; python_version < '3.8'
-Sphinx==7.1.2; python_version == '3.8'
+PyYAML==6.0.3
+semver==3.0.4
Sphinx==7.4.7; python_version == '3.9'
Sphinx==8.1.3; python_version == '3.10'
Sphinx==8.2.3; python_version >= '3.11'
-sphinx-rtd-theme==2.0.0; python_version < '3.9'
-sphinx-rtd-theme==3.0.2; python_version >= '3.9'
+sphinx-rtd-theme==3.0.2
diff --git a/docs/started.rst b/docs/started.rst
index 3ab07c0295..fa6bd84aa5 100644
--- a/docs/started.rst
+++ b/docs/started.rst
@@ -5,7 +5,7 @@ Getting Started
Requirements
------------
-* Python 3.6 or higher.
+* Python 3.9 or higher.
Python 2 is not supported.
* The required Python packages are the following:
@@ -16,12 +16,11 @@ Requirements
.. note::
.. versionchanged:: 3.0
- Support for Python 3.5 has been dropped.
+ Support for Python 3.5 is dropped.
+ .. versionchanged:: 4.10
-.. warning::
- Although ReFrame supports Python 3.6 and 3.7, you should note that these Python versions have reached end-of-life and you are strongly advised to use a newer version.
- ReFrame installations on these Python versions may use out-of-date dependencies due to incompatibilities.
+ Support for Python < 3.9 is dropped.
Getting the Framework
diff --git a/examples/tutorial/dockerfiles/eb-spack.dockerfile b/examples/tutorial/dockerfiles/eb-spack.dockerfile
index 645507a887..34a6fe1217 100644
--- a/examples/tutorial/dockerfiles/eb-spack.dockerfile
+++ b/examples/tutorial/dockerfiles/eb-spack.dockerfile
@@ -3,10 +3,10 @@
#
-FROM ghcr.io/reframe-hpc/lmod:8.4.12
+FROM ghcr.io/reframe-hpc/lmod:9.0.4
-ENV _SPACK_VER=0.16
-ENV _EB_VER=4.4.1
+ENV _SPACK_VER=1.1.0
+ENV _EB_VER=5.1.2
RUN apt-get -y update && \
apt-get -y install curl && \
@@ -22,7 +22,7 @@ RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/reframe-hpc/ref
ENV PATH=/usr/local/share/reframe/bin:$PATH
# Install EasyBuild
-RUN pip3 install easybuild==${_EB_VER}
+RUN pip3 install --break-system-packages easybuild==${_EB_VER}
# Add tutorial user
RUN useradd -ms /bin/bash -G sudo user && \
@@ -33,7 +33,7 @@ WORKDIR /home/user
# Install Spack
RUN mkdir .local && cd .local && \
- git clone --branch releases/v${_SPACK_VER} --depth 1 https://github.com/spack/spack
+ git clone --branch v${_SPACK_VER} --depth 1 https://github.com/spack/spack
-RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/user/.local/spack/share/spack/setup-env.sh' > /home/user/.profile
-ENV BASH_ENV /home/user/.profile
+RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/user/.local/spack/share/spack/setup-env.sh' >> /home/user/.profile
+ENV BASH_ENV=/home/user/.profile
diff --git a/examples/tutorial/easybuild/eb_test.py b/examples/tutorial/easybuild/eb_test.py
index 8e50d915f8..d859ab394a 100644
--- a/examples/tutorial/easybuild/eb_test.py
+++ b/examples/tutorial/easybuild/eb_test.py
@@ -8,17 +8,17 @@
@rfm.simple_test
-class BZip2EBCheck(rfm.RegressionTest):
+class ZlibEBCheck(rfm.RegressionTest):
descr = 'Demo test using EasyBuild to build the test code'
valid_systems = ['*']
valid_prog_environs = ['builtin']
- executable = 'bzip2'
- executable_opts = ['--help']
+ executable = 'ls'
+ executable_opts = ['$LD_LIBRARY_PATH/libz.so.1.3.1']
build_system = 'EasyBuild'
@run_before('compile')
def setup_build_system(self):
- self.build_system.easyconfigs = ['bzip2-1.0.6.eb']
+ self.build_system.easyconfigs = ['zlib-1.3.1.eb']
self.build_system.options = ['-f']
@run_before('run')
@@ -26,5 +26,5 @@ def prepare_run(self):
self.modules = self.build_system.generated_modules
@sanity_function
- def assert_version(self):
- return sn.assert_found(r'Version 1.0.6', self.stderr)
+ def assert_exists(self):
+ return sn.assert_eq(self.job.exitcode, 0)
diff --git a/examples/tutorial/spack/spack_test.py b/examples/tutorial/spack/spack_test.py
index a095b11d99..e7ae86b1e4 100644
--- a/examples/tutorial/spack/spack_test.py
+++ b/examples/tutorial/spack/spack_test.py
@@ -8,18 +8,20 @@
@rfm.simple_test
-class BZip2SpackCheck(rfm.RegressionTest):
+class ZlibSpackCheck(rfm.RegressionTest):
descr = 'Demo test using Spack to build the test code'
valid_systems = ['*']
valid_prog_environs = ['builtin']
- executable = 'bzip2'
- executable_opts = ['--help']
+ executable = 'pkg-config'
+ executable_opts = ['--libs', 'zlib']
build_system = 'Spack'
@run_before('compile')
def setup_build_system(self):
- self.build_system.specs = ['bzip2@1.0.6']
+ self.build_system.specs = ['zlib@1.3.1']
@sanity_function
def assert_version(self):
- return sn.assert_found(r'Version 1.0.6', self.stderr)
+ return sn.assert_found(
+ r'-L.*/spack/linux-.*/zlib-1.3.1-.*/lib -lz', self.stdout
+ )
diff --git a/reframe/__init__.py b/reframe/__init__.py
index a661177c12..aeff4e1f58 100644
--- a/reframe/__init__.py
+++ b/reframe/__init__.py
@@ -10,7 +10,7 @@
INSTALL_PREFIX = os.path.normpath(
os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
)
-MIN_PYTHON_VERSION = (3, 6, 0)
+MIN_PYTHON_VERSION = (3, 9, 0)
# Check python version
if sys.version_info[:3] < MIN_PYTHON_VERSION:
diff --git a/reframe/core/logging.py b/reframe/core/logging.py
index ec810e061a..bf1e1d6c48 100644
--- a/reframe/core/logging.py
+++ b/reframe/core/logging.py
@@ -337,12 +337,8 @@ class CheckFieldFormatter(logging.Formatter):
# NOTE: This formatter will work only for the '%' style
def __init__(self, fmt=None, datefmt=None, perffmt=None,
ignore_keys=None, style='%'):
- if sys.version_info[:2] <= (3, 7):
- super().__init__(fmt, datefmt, style)
- else:
- super().__init__(fmt, datefmt, style,
- validate=(fmt != '%(check_#ALL)s'))
-
+ super().__init__(fmt, datefmt, style,
+ validate=(fmt != '%(check_#ALL)s'))
self.__fmt = fmt
self.__fmtperf = perffmt[:-1] if perffmt else ''
self.__specs = re.findall(r'\%\((\S+?)\)s', fmt)
@@ -809,11 +805,10 @@ def __init__(self, name, level=logging.NOTSET):
def setLevel(self, level):
self.level = _check_level(level)
- if sys.version_info[:2] >= (3, 7):
- # Clear the internal cache of the base logger, otherwise the
- # logger will remain disabled if its level is raised and then
- # lowered again
- self._cache.clear()
+ # Clear the internal cache of the base logger, otherwise the
+ # logger will remain disabled if its level is raised and then
+ # lowered again
+ self._cache.clear()
def makeRecord(self, name, level, fn, lno, msg, args, exc_info,
func=None, extra=None, sinfo=None):
diff --git a/reframe/core/modules.py b/reframe/core/modules.py
index e8c9c504b1..292acd8843 100644
--- a/reframe/core/modules.py
+++ b/reframe/core/modules.py
@@ -112,13 +112,15 @@ def create(cls, modules_kind=None, validate=True):
modules_impl = {
None: NoModImpl,
'nomod': NoModImpl,
- 'tmod31': TMod31Impl,
- 'tmod': TModImpl,
- 'tmod32': TModImpl,
- 'tmod4': TMod4Impl,
+ 'tmod4': EnvModulesImpl,
+ 'envmod': EnvModulesImpl,
'lmod': LModImpl,
'spack': SpackImpl
}
+ if modules_kind == 'tmod4':
+ getlogger().warning("'tmod4' backend is deprecated; "
+ "please use 'envmod' instead")
+
try:
impl_cls = modules_impl[modules_kind]
except KeyError:
@@ -583,244 +585,8 @@ def __str__(self):
return self.name() + ' ' + self.version()
-class TModImpl(ModulesSystemImpl):
- '''Base class for TMod Module system (Tcl).'''
-
- MIN_VERSION = (3, 2)
-
- def __init__(self):
- self._version = None
- self._validated = False
- if self.validate:
- self._do_validate()
-
- def _do_validate(self):
- # Try to figure out if we are indeed using the TCL version
- try:
- completed = osext.run_command('modulecmd -V')
- except OSError as e:
- raise ConfigError(
- 'could not find a sane TMod installation') from e
-
- version_match = re.search(r'^VERSION=(\S+)', completed.stdout,
- re.MULTILINE)
- tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout,
- re.MULTILINE)
-
- if version_match is None or tcl_version_match is None:
- raise ConfigError('could not find a sane TMod installation')
-
- version = version_match.group(1)
- try:
- ver_major, ver_minor = [int(v) for v in version.split('.')[:2]]
- except ValueError:
- raise ConfigError(
- 'could not parse TMod version string: ' + version) from None
-
- if (ver_major, ver_minor) < self.MIN_VERSION:
- raise ConfigError(
- f'unsupported TMod version: '
- f'{version} (required >= {self.MIN_VERSION})'
- )
-
- self._version = version
- try:
- # Try the Python bindings now
- completed = osext.run_command(self.modulecmd())
- except OSError as e:
- raise ConfigError(
- f'could not get the Python bindings for TMod: {e}'
- ) from e
-
- if re.search(r'Unknown shell type', completed.stderr):
- raise ConfigError(
- 'Python is not supported by this TMod installation'
- )
-
- self._validated = True
-
- def name(self):
- return 'tmod'
-
- def version(self):
- return self._version
-
- def modulecmd(self, *args):
- return ' '.join(['modulecmd', 'python', *args])
-
- def _execute(self, cmd, *args):
- if not self._validated:
- self._do_validate()
-
- modulecmd = self.modulecmd(cmd, *args)
- completed = osext.run_command(modulecmd)
- if re.search(r'\bERROR\b', completed.stderr) is not None:
- raise SpawnedProcessError(modulecmd,
- completed.stdout,
- completed.stderr,
- completed.returncode)
-
- exec(self.process(completed.stdout))
- return completed.stderr
-
- def available_modules(self, substr):
- output = self.execute('avail', '-t', substr)
- ret = []
- for line in output.split('\n'):
- if not line or line[-1] == ':':
- # Ignore empty lines and path entries
- continue
-
- module = re.sub(r'\(default\)', '', line)
- ret.append(Module(module))
-
- return ret
-
- def loaded_modules(self):
- try:
- # LOADEDMODULES may be defined but empty
- return [Module(m)
- for m in os.environ['LOADEDMODULES'].split(':') if m]
- except KeyError:
- return []
-
- def conflicted_modules(self, module):
- output = self.execute_with_path('show', str(module), path=module.path)
- return [Module(m.group(1))
- for m in re.finditer(r'^conflict\s+(\S+)',
- output, re.MULTILINE)]
-
- def is_module_loaded(self, module):
- return module in self.loaded_modules()
-
- def load_module(self, module):
- self.execute_with_path('load', str(module), path=module.path)
-
- def unload_module(self, module):
- self.execute('unload', str(module))
-
- def unload_all(self):
- self.execute('purge')
-
- def searchpath(self):
- path = os.getenv('MODULEPATH', '')
- return path.split(':')
-
- def searchpath_add(self, *dirs):
- if dirs:
- self.execute('use', *dirs)
-
- def searchpath_remove(self, *dirs):
- if dirs:
- self.execute('unuse', *dirs)
-
- def emit_load_instr(self, module):
- commands = []
- if module.path:
- commands.append(f'module use {module.path}')
-
- commands.append(f'module load {module.fullname}')
- if module.path:
- commands.append(f'module unuse {module.path}')
-
- return commands
-
- def emit_unload_instr(self, module):
- return [f'module unload {module}']
-
-
-class TMod31Impl(TModImpl):
- '''Module system for TMod (Tcl).'''
-
- MIN_VERSION = (3, 1)
-
- def __init__(self):
- self._version = None
- self._command = None
- self._validated = False
- if self.validate:
- self._do_validate()
-
- def _do_validate(self):
- # Try to figure out if we are indeed using the TCL version
- try:
- modulecmd = os.getenv('MODULESHOME')
- modulecmd = os.path.join(modulecmd, 'modulecmd.tcl')
- completed = osext.run_command(modulecmd)
- except OSError as e:
- raise ConfigError(
- f'could not find a sane TMod31 installation: {e}'
- ) from e
-
- version_match = re.search(r'Release Tcl (\S+)', completed.stderr,
- re.MULTILINE)
- tcl_version_match = version_match
-
- if version_match is None or tcl_version_match is None:
- raise ConfigError('could not find a sane TMod31 installation')
-
- version = version_match.group(1)
- try:
- ver_major, ver_minor = [int(v) for v in version.split('.')[:2]]
- except ValueError:
- raise ConfigError(
- 'could not parse TMod31 version string: ' + version) from None
-
- if (ver_major, ver_minor) < self.MIN_VERSION:
- raise ConfigError(
- f'unsupported TMod version: {version} '
- f'(required >= {self.MIN_VERSION})'
- )
-
- self._version = version
- self._command = f'{modulecmd} python'
- try:
- # Try the Python bindings now
- completed = osext.run_command(self._command)
- except OSError as e:
- raise ConfigError(
- f'could not get the Python bindings for TMod31: {e}'
- )
-
- if re.search(r'Unknown shell type', completed.stderr):
- raise ConfigError(
- 'Python is not supported by this TMod installation'
- )
-
- self._validated = True
-
- def name(self):
- return 'tmod31'
-
- def modulecmd(self, *args):
- return ' '.join([self._command, *args])
-
- def _execute(self, cmd, *args):
- if not self._validated:
- self._do_validate()
-
- modulecmd = self.modulecmd(cmd, *args)
- completed = osext.run_command(modulecmd)
- if re.search(r'\bERROR\b', completed.stderr) is not None:
- raise SpawnedProcessError(modulecmd,
- completed.stdout,
- completed.stderr,
- completed.returncode)
-
- exec_match = re.search(r"^exec\s'(\S+)'", completed.stdout,
- re.MULTILINE)
- if exec_match is None:
- raise ConfigError('could not use the python bindings')
-
- with open(exec_match.group(1), 'r') as content_file:
- cmd = content_file.read()
-
- exec(self.process(cmd))
- return completed.stderr
-
-
-class TMod4Impl(TModImpl):
- '''Module system for TMod 4.'''
+class EnvModulesImpl(ModulesSystemImpl):
+ '''Module system for Environment Modules.'''
MIN_VERSION = (4, 1)
@@ -867,7 +633,10 @@ def _do_validate(self):
self._validated = True
def name(self):
- return 'tmod4'
+ return 'envmod'
+
+ def version(self):
+ return self._version
def modulecmd(self, *args):
return ' '.join(['modulecmd', 'python', *args])
@@ -899,20 +668,34 @@ def load_module(self, module):
# 'restore' discards previous module path manipulations
for op, mp in self._extra_module_paths:
if op == '+':
- super().searchpath_add(mp)
+ self.execute('use', mp)
else:
- super().searchpath_remove(mp)
+ self.execute('unuse', mp)
return []
else:
- return super().load_module(module)
+ self.execute_with_path('load', str(module), path=module.path)
def unload_module(self, module):
if module.collection:
- # Module collection are not unloaded
+ # Module collections are not unloaded
return
- super().unload_module(module)
+ self.execute('unload', str(module))
+
+ def loaded_modules(self):
+ try:
+ # LOADEDMODULES may be defined but empty
+ return [Module(m)
+ for m in os.environ['LOADEDMODULES'].split(':') if m]
+ except KeyError:
+ return []
+
+ def is_module_loaded(self, module):
+ return module in self.loaded_modules()
+
+ def unload_all(self):
+ self.execute('purge')
def conflicted_modules(self, module):
if module.collection:
@@ -921,7 +704,10 @@ def conflicted_modules(self, module):
# collection
return []
- return super().conflicted_modules(module)
+ output = self.execute_with_path('show', str(module), path=module.path)
+ return [Module(m.group(1))
+ for m in re.finditer(r'^conflict\s+(\S+)',
+ output, re.MULTILINE)]
def _emit_restore_instr(self, module):
cmds = [f'module restore {module}']
@@ -938,28 +724,51 @@ def emit_load_instr(self, module):
if module.collection:
return self._emit_restore_instr(module)
- return super().emit_load_instr(module)
+ commands = []
+ if module.path:
+ commands.append(f'module use {module.path}')
+
+ commands.append(f'module load {module.fullname}')
+ if module.path:
+ commands.append(f'module unuse {module.path}')
+
+ return commands
def emit_unload_instr(self, module):
if module.collection:
return []
- return super().emit_unload_instr(module)
+ return [f'module unload {module}']
+
+ def searchpath(self):
+ path = os.getenv('MODULEPATH', '')
+ return path.split(':')
def searchpath_add(self, *dirs):
if dirs:
self._extra_module_paths += [('+', mp) for mp in dirs]
-
- super().searchpath_add(*dirs)
+ self.execute('use', *dirs)
def searchpath_remove(self, *dirs):
if dirs:
self._extra_module_paths += [('-', mp) for mp in dirs]
+ self.execute('unuse', *dirs)
+
+ def available_modules(self, substr):
+ output = self.execute('avail', '-t', substr)
+ ret = []
+ for line in output.split('\n'):
+ if not line or line[-1] == ':':
+ # Ignore empty lines and path entries
+ continue
- super().searchpath_remove(*dirs)
+ module = re.sub(r'\(default\)', '', line)
+ ret.append(Module(module))
+
+ return ret
-class LModImpl(TMod4Impl):
+class LModImpl(EnvModulesImpl):
'''Module system for Lmod (Tcl/Lua).'''
def __init__(self):
diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index 09f03f0470..76790c4306 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -2616,7 +2616,7 @@ def _copy_to_outputdir(self):
dst = os.path.join(
self.outputdir, os.path.relpath(f, self.stagedir)
)
- osext.copytree(f, dst, dirs_exist_ok=True)
+ shutil.copytree(f, dst, dirs_exist_ok=True)
else:
shutil.copy2(f, self.outputdir)
diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py
index 3ef7a0b9d1..a84dbd72fd 100644
--- a/reframe/frontend/autodetect.py
+++ b/reframe/frontend/autodetect.py
@@ -59,7 +59,7 @@ def __enter__(self):
src = os.path.join(rfm.INSTALL_PREFIX, p)
if os.path.isdir(src):
dst = os.path.join(self._workdir, p)
- osext.copytree(src, dst, dirs_exist_ok=True)
+ shutil.copytree(src, dst, dirs_exist_ok=True)
else:
shutil.copy2(src, self._workdir)
except FileNotFoundError:
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index ac03ea0fda..255acf948a 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -270,7 +270,7 @@
"max_local_jobs": {"type": "number"},
"modules_system": {
"type": "string",
- "enum": ["tmod", "tmod31", "tmod32", "tmod4",
+ "enum": ["tmod4", "envmod",
"lmod", "nomod", "spack"]
},
"modules": {"$ref": "#/defs/modules_list"},
diff --git a/reframe/utility/osext.py b/reframe/utility/osext.py
index 544c47900b..1b58972f4c 100644
--- a/reframe/utility/osext.py
+++ b/reframe/utility/osext.py
@@ -27,6 +27,7 @@
import reframe.utility as util
from reframe.core.exceptions import (ReframeError, SpawnedProcessError,
SpawnedProcessTimeout)
+from reframe.core.warnings import user_deprecation_warning
from . import OrderedSet
@@ -409,52 +410,20 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=shutil.copy2,
This function will automatically delegate to :py:func:`shutil.copytree`
for Python versions >= 3.8.
+
+ .. deprecated:: 4.10
+
+ Please use :py:func:`shutil.copytree` directly.
'''
+ user_deprecation_warning('`osext.copytree()` is deprecated; '
+ 'please use `shutil.copytree()` directly')
+
if src == os.path.commonpath([src, dst]):
raise ValueError("cannot copy recursively the parent directory "
"`%s' into one of its descendants `%s'" % (src, dst))
- if sys.version_info[1] >= 8:
- return shutil.copytree(src, dst, symlinks, ignore, copy_function,
- ignore_dangling_symlinks, dirs_exist_ok)
-
- if not dirs_exist_ok:
- return shutil.copytree(src, dst, symlinks, ignore, copy_function,
- ignore_dangling_symlinks)
-
- # dirs_exist_ok=True and Python < 3.8
- if not os.path.exists(dst):
- return shutil.copytree(src, dst, symlinks, ignore, copy_function,
- ignore_dangling_symlinks)
-
- # dst exists; manually descend into the subdirectories, but do some sanity
- # checking first
-
- # We raise the following errors to comply with the copytree()'s behaviour
-
- if not os.path.isdir(dst):
- raise FileExistsError(errno.EEXIST, 'File exists', dst)
-
- if not os.path.exists(src):
- raise FileNotFoundError(errno.ENOENT, 'No such file or directory', src)
-
- if not os.path.isdir(src):
- raise NotADirectoryError(errno.ENOTDIR, 'Not a directory', src)
-
- _, subdirs, files = list(os.walk(src))[0]
- ignore_paths = ignore(src, os.listdir(src)) if ignore else {}
- for f in files:
- if f not in ignore_paths:
- copy_function(os.path.join(src, f), os.path.join(dst, f),
- follow_symlinks=not symlinks)
-
- for d in subdirs:
- if d not in ignore_paths:
- copytree(os.path.join(src, d), os.path.join(dst, d),
- symlinks, ignore, copy_function,
- ignore_dangling_symlinks, dirs_exist_ok)
-
- return dst
+ return shutil.copytree(src, dst, symlinks, ignore, copy_function,
+ ignore_dangling_symlinks, dirs_exist_ok)
def copytree_virtual(src, dst, file_links=None,
@@ -464,10 +433,10 @@ def copytree_virtual(src, dst, file_links=None,
``file_links``.
If ``file_links`` is empty or :class:`None`, this is equivalent to
- :func:`copytree()`. The rest of the arguments are passed as-is to
- :func:`copytree()`. Paths in ``file_links`` must be relative to ``src``.
- If you try to pass ``'.'`` in ``file_links``, an :py:class:`OSError` will
- be raised.
+ :py:func:`shutil.copytree()`. The rest of the arguments are passed as-is
+ to :py:func:`shutil.copytree()`. Paths in ``file_links`` must be relative
+ to ``src``. If you try to pass ``'.'`` in ``file_links``, an
+ :py:class:`OSError` will be raised.
'''
@@ -510,8 +479,8 @@ def ignore(dir, contents):
if os.path.join(dir, c) in link_targets}
# Copy to dst ignoring the file_links
- copytree(src, dst, symlinks, ignore,
- copy_function, ignore_dangling_symlinks, dirs_exist_ok)
+ shutil.copytree(src, dst, symlinks, ignore,
+ copy_function, ignore_dangling_symlinks, dirs_exist_ok)
# Now create the symlinks
for f in link_targets:
diff --git a/reframe/utility/profile.py b/reframe/utility/profile.py
index 144970df0d..c6b94d52eb 100644
--- a/reframe/utility/profile.py
+++ b/reframe/utility/profile.py
@@ -6,9 +6,6 @@
# A lightweight time profiler
import time
-import sys
-
-from collections import OrderedDict
class ProfilerError(Exception):
@@ -33,10 +30,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
class TimeProfiler:
def __init__(self):
self._region_stack = ['root']
- if sys.version_info[:2] < (3, 8):
- self._region_times = OrderedDict()
- else:
- self._region_times = {}
+ self._region_times = {}
@property
def current_region(self):
diff --git a/requirements.txt b/requirements.txt
index 4b5a2c45a9..0edca43782 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,40 +1,21 @@
archspec==0.2.5
-argcomplete==3.1.2; python_version < '3.8'
argcomplete==3.6.3; python_version >= '3.8'
ClusterShell==1.9.3
fasteners==0.19; python_version < '3.10'
fasteners==0.20; python_version >= '3.10'
-importlib_metadata==4.0.1; python_version < '3.8'
-jinja2==3.0.3; python_version == '3.6'
-jinja2==3.1.6; python_version >= '3.7'
+jinja2==3.1.6
jsonschema==3.2.0
-lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
-lxml==5.4.0; python_version < '3.8' and platform_machine != 'aarch64'
-lxml==6.0.2; python_version >= '3.8'
-pytest==7.0.1; python_version < '3.8'
-pytest==8.3.5; python_version == '3.8'
+lxml==6.0.2
pytest==8.4.2; python_version == '3.9'
pytest==9.0.1; python_version >= '3.10'
-pytest-forked==1.4.0; python_version == '3.6'
-pytest-forked==1.6.0; python_version >= '3.7'
+pytest-forked==1.6.0
pytest-parallel==0.1.1
-pytest-rerunfailures==10.3; python_version == '3.6'
-pytest-rerunfailures==13.0; python_version == '3.7'
-pytest-rerunfailures==14.0; python_version == '3.8'
pytest-rerunfailures==16.0.1; python_version == '3.9'
pytest-rerunfailures==16.1; python_version >= '3.10'
-PyYAML==6.0.1; python_version < '3.8'
-PyYAML==6.0.3; python_version >= '3.8'
-requests==2.27.1; python_version == '3.6'
-requests==2.31.0; python_version == '3.7'
-requests==2.32.4; python_version >= '3.8'
-semver==2.13.0; python_version == '3.6'
-semver==3.0.4; python_version >= '3.7'
-setuptools==59.6.0; python_version == '3.6'
-setuptools==68.0.0; python_version == '3.7'
-setuptools==75.3.0; python_version == '3.8'
-setuptools==80.9.0; python_version >= '3.9'
-tabulate==0.8.10; python_version == '3.6'
-tabulate==0.9.0; python_version >= '3.7'
+PyYAML==6.0.3
+requests==2.32.4
+semver==3.0.4
+setuptools==80.9.0
+tabulate==0.9.0
wcwidth==0.2.14
-#+pygelf%pygelf==0.4.0
+#+pygelf%pygelf==0.4.3
diff --git a/setup.cfg b/setup.cfg
index a3db8c3f05..d63c748f29 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -9,9 +9,6 @@ long_description = file: README_minimal.md
long_description_content_type = text/markdown
classifiers =
Development Status :: 5 - Production/Stable
- Programming Language :: Python :: 3.6
- Programming Language :: Python :: 3.7
- Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
@@ -24,29 +21,21 @@ classifiers =
[options]
packages = find_namespace:
-python_requires = >=3.6
+python_requires = >=3.9
scripts = bin/reframe
install_requires =
archspec >= 0.2.4
argcomplete
- argcomplete <= 3.1.2; python_version < '3.8'
ClusterShell
fasteners==0.19; python_version < '3.10'
fasteners
- jinja2==3.0.3; python_version == '3.6'
jinja2
jsonschema
- lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
- lxml==5.4.0; python_version < '3.8' and platform_machine != 'aarch64'
lxml
- PyYAML==6.0.1; python_version < '3.8'
PyYAML
requests
- requests <= 2.27.1; python_version == '3.6'
semver
- semver <= 2.13.0; python_version == '3.6'
tabulate
- tabulate <= 0.8.10; python_version == '3.6'
[options.packages.find]
include = reframe,reframe.*,hpctestlib.*
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 8742dc173b..640be0b94d 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -634,7 +634,7 @@ def test_timestamp_option_default(run_reframe):
assert returncode == 0
matches = re.findall(
- r'(stage|output) directory: .*\/(\d{8}T\d{6}\+\d{4})', stdout
+ r'(stage|output) directory: .*\/(\d{8}T\d{6}(\+|-)\d{4})', stdout
)
assert len(matches) == 2
diff --git a/unittests/test_loader.py b/unittests/test_loader.py
index 729708bde7..ba32e5bc31 100644
--- a/unittests/test_loader.py
+++ b/unittests/test_loader.py
@@ -8,7 +8,6 @@
import shutil
import reframe as rfm
-import reframe.utility.osext as osext
from reframe.core.exceptions import ReframeSyntaxError
from reframe.frontend.loader import RegressionCheckLoader
@@ -148,7 +147,7 @@ def test_relative_import_outside_rfm_prefix(loader, tmp_path):
# imported as a hierarchical module. If not, we want to make sure that
# reframe will still load its parent modules
- osext.copytree(
+ shutil.copytree(
os.path.abspath('unittests/resources/checks_unlisted/testlib'),
tmp_path / 'testlib', dirs_exist_ok=True
)
diff --git a/unittests/test_modules.py b/unittests/test_modules.py
index feffdc5eb3..5f09e7dd7d 100644
--- a/unittests/test_modules.py
+++ b/unittests/test_modules.py
@@ -9,10 +9,11 @@
import reframe.core.environments as env
import reframe.core.modules as modules
import unittests.utility as test_util
+from reframe.utility.versioning import parse as parse_version
from reframe.core.exceptions import ConfigError, EnvironError
-@pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'spack', 'nomod'])
+@pytest.fixture(params=['envmod', 'lmod', 'spack', 'nomod'])
def modules_system_nopath(request, monkeypatch):
# Always pretend to be on a clean modules environment
monkeypatch.setenv('MODULEPATH', '')
@@ -77,7 +78,11 @@ def module_collection(modules_system, tmp_path, monkeypatch):
# Remove the temporary collection
if modules_system.name == 'lmod':
- prefix = os.path.join(os.environ['HOME'], '.lmod.d')
+ lmod_version = parse_version(modules_system.version)
+ if lmod_version < (9,):
+ prefix = os.path.join(os.environ['HOME'], '.lmod.d')
+ else:
+ prefix = os.path.join(os.environ['HOME'], '.config', 'lmod')
else:
prefix = os.path.join(os.environ['HOME'], '.module')
diff --git a/unittests/test_utility.py b/unittests/test_utility.py
index ca0939bd97..fb0a204ebb 100644
--- a/unittests/test_utility.py
+++ b/unittests/test_utility.py
@@ -22,6 +22,7 @@
from reframe.core.exceptions import (ConfigError,
SpawnedProcessError,
SpawnedProcessTimeout)
+from reframe.core.warnings import ReframeDeprecationWarning
def test_command_success():
@@ -281,7 +282,8 @@ def test_copytree(tmp_path):
dir_src.mkdir()
dir_dst = tmp_path / 'dst'
dir_dst.mkdir()
- osext.copytree(str(dir_src), str(dir_dst), dirs_exist_ok=True)
+ with pytest.warns(ReframeDeprecationWarning):
+ osext.copytree(str(dir_src), str(dir_dst), dirs_exist_ok=True)
def test_copytree_src_parent_of_dst(tmp_path):
@@ -289,7 +291,8 @@ def test_copytree_src_parent_of_dst(tmp_path):
src_path = (dst_path / '..').resolve()
with pytest.raises(ValueError):
- osext.copytree(str(src_path), str(dst_path))
+ with pytest.warns(ReframeDeprecationWarning):
+ osext.copytree(str(src_path), str(dst_path))
@pytest.fixture(params=['dirs_exist_ok=True', 'dirs_exist_ok=False'])
@@ -303,7 +306,8 @@ def test_copytree_dst_notdir(tmp_path, dirs_exist_ok):
dst = tmp_path / 'dst'
dst.touch()
with pytest.raises(FileExistsError, match=fr'{dst}'):
- osext.copytree(str(dir_src), str(dst), dirs_exist_ok=dirs_exist_ok)
+ with pytest.warns(ReframeDeprecationWarning):
+ osext.copytree(str(dir_src), str(dst), dirs_exist_ok=dirs_exist_ok)
def test_copytree_src_notdir(tmp_path, dirs_exist_ok):
@@ -312,7 +316,8 @@ def test_copytree_src_notdir(tmp_path, dirs_exist_ok):
dst = tmp_path / 'dst'
dst.mkdir()
with pytest.raises(NotADirectoryError, match=fr'{src}'):
- osext.copytree(str(src), str(dst), dirs_exist_ok=dirs_exist_ok)
+ with pytest.warns(ReframeDeprecationWarning):
+ osext.copytree(str(src), str(dst), dirs_exist_ok=dirs_exist_ok)
def test_copytree_src_does_not_exist(tmp_path, dirs_exist_ok):
@@ -320,7 +325,8 @@ def test_copytree_src_does_not_exist(tmp_path, dirs_exist_ok):
dst = tmp_path / 'dst'
dst.mkdir()
with pytest.raises(FileNotFoundError, match=fr'{src}'):
- osext.copytree(str(src), str(dst), dirs_exist_ok=dirs_exist_ok)
+ with pytest.warns(ReframeDeprecationWarning):
+ osext.copytree(str(src), str(dst), dirs_exist_ok=dirs_exist_ok)
@pytest.fixture