From 0a37e47176c1fb0fd5c444fff2b90f6e7b7215a9 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 19:30:50 +0200 Subject: [PATCH 01/19] Expect env.txt in conda-meta --- tests/test_examples.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_examples.py b/tests/test_examples.py index 5a64c867a..7adf8d4b8 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -466,6 +466,9 @@ def test_example_extra_envs(tmp_path, request): input_path = _example_path("extra_envs") for installer, install_dir in create_installer(input_path, tmp_path): _run_installer(input_path, installer, install_dir, request=request) + assert "@EXPLICIT" in (install_dir / "conda-meta" / "env.txt").read_text() + for envtxt in install_dir.glob("envs/*/conda-meta/env.txt"): + assert "@EXPLICIT" in envtxt.read_text() def test_example_extra_files(tmp_path, request): From 26a468d3f64641938804ad0b49c7c83c769ef88c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 19:40:50 +0200 Subject: [PATCH 02/19] Add implementation --- constructor/nsis/main.nsi.tmpl | 4 ++++ constructor/osx/run_installation.sh | 5 ++++- constructor/shar.py | 7 +++++++ constructor/winexe.py | 5 ++--- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index 6dec989ef..9f134565e 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1262,6 +1262,8 @@ Section "Install" # A conda-meta\history file is required for a valid conda prefix SetOutPath "$INSTDIR\conda-meta" File {{ conda_history }} + # Place copy of env.txt as a lockfile that can be used to restore the env + File {{ env_txt_abspath }} SetOutPath "$INSTDIR" File {{ conda_exe }} @@ -1382,6 +1384,8 @@ Section "Install" # A conda-meta\history file is required for a valid conda prefix SetOutPath "{{ env.conda_meta }}" File "{{ env.history_abspath }}" + # We also place a copy of the env.txt file here, as a lockfile + File "{{ env.env_txt_abspath }}" # Set channels System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_CHANNELS", "{{ channels }}").r0' diff --git a/constructor/osx/run_installation.sh b/constructor/osx/run_installation.sh index 4225ebac2..91882c61b 100644 --- a/constructor/osx/run_installation.sh +++ b/constructor/osx/run_installation.sh @@ -60,6 +60,8 @@ fi # Move the prepackaged history file into place mv "$PREFIX/pkgs/conda-meta/history" "$PREFIX/conda-meta/history" +# Place a copy of the lockfile in conda-meta for future, potential restoring +cp "$PREFIX/pkgs/env.txt" "$PREFIX/conda-meta/env.txt" rm -f "$PREFIX/env.txt" # Same, but for the extra environments @@ -102,7 +104,8 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 # Move the prepackaged history file into place mv "${env_pkgs}/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history" - rm -f "${env_pkgs}env.txt" + # Move the input lockfile in conda-meta for future, potential restoring + mv "${env_pkgs}env.txt" "$PREFIX/envs/$env_name/conda-meta/env.txt" done # Cleanup! diff --git a/constructor/shar.py b/constructor/shar.py index 727da01ca..1d945827d 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -170,6 +170,8 @@ def create(info, verbose=False): pre_t.add(record_file_src, record_file_dest) pre_t.addfile(tarinfo=tarfile.TarInfo("conda-meta/history")) post_t.add(join(tmp_dir, "conda-meta", "history"), "conda-meta/history") + # Place a copy of the lockfile in conda-meta for future, potential restoring + post_t.add(join(tmp_dir, "env.txt"), "conda-meta/env.txt") for env_name in info.get("_extra_envs_info", {}): pre_t.addfile(tarinfo=tarfile.TarInfo(f"envs/{env_name}/conda-meta/history")) @@ -177,6 +179,11 @@ def create(info, verbose=False): join(tmp_dir, "envs", env_name, "conda-meta", "history"), f"envs/{env_name}/conda-meta/history", ) + # Place a copy of the lockfile in conda-meta for future, potential restoring + post_t.add( + join(tmp_dir, "envs", env_name, "env.txt"), + f"envs/{env_name}/conda-meta/env.txt", + ) extra_files = copy_extra_files(info.get("extra_files", []), tmp_dir) for path in extra_files: diff --git a/constructor/winexe.py b/constructor/winexe.py index 913b07b17..0dd0e8255 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -83,9 +83,8 @@ def setup_envs_commands(info, dir_path): "prefix": r"$INSTDIR", "env_txt": r"$INSTDIR\pkgs\env.txt", # env.txt as seen by the running installer "env_txt_dir": r"$INSTDIR\pkgs", # env.txt location in the installer filesystem - "env_txt_abspath": join( - dir_path, "env.txt" - ), # env.txt path while building the installer + # env.txt path while building the installer + "env_txt_abspath": join(dir_path, "env.txt"), "conda_meta": r"$INSTDIR\conda-meta", "history_abspath": join(dir_path, "conda-meta", "history"), "final_channels": get_final_channels(info), From a593880b090bf3aa9ae14a977f81b54464553f96 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 19:42:55 +0200 Subject: [PATCH 03/19] add news --- news/1059-lockfiles | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 news/1059-lockfiles diff --git a/news/1059-lockfiles b/news/1059-lockfiles new file mode 100644 index 000000000..46bc4ae39 --- /dev/null +++ b/news/1059-lockfiles @@ -0,0 +1,19 @@ +### Enhancements + +* Ship `conda-meta/env.txt` as a lockfile that provisions the initial state of the environment. (#1052 via #1059) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* From f9b3d0e711395764fd1f8e9eb3d2585d341e3660 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 20:12:05 +0200 Subject: [PATCH 04/19] this one is not needed --- constructor/nsis/main.nsi.tmpl | 2 -- 1 file changed, 2 deletions(-) diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index 9f134565e..e3b052d7d 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1262,8 +1262,6 @@ Section "Install" # A conda-meta\history file is required for a valid conda prefix SetOutPath "$INSTDIR\conda-meta" File {{ conda_history }} - # Place copy of env.txt as a lockfile that can be used to restore the env - File {{ env_txt_abspath }} SetOutPath "$INSTDIR" File {{ conda_exe }} From 1ece8283700e9a3320a6dac0b7b51cd7a4a3cee2 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 20:56:45 +0200 Subject: [PATCH 05/19] run extra_envs example only --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b30c6e09..661ea11ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -136,11 +136,13 @@ jobs: - name: conda config run: conda config --show-sources - name: Run unit tests + if: false run: | pytest -vv --cov=constructor --cov-branch tests/ -m "not examples" coverage run --branch --append -m constructor -V coverage json - uses: codecov/codecov-action@fdcc8476540edceab3de004e990f80d881c6cc00 # v5.5.0 + if: false with: token: ${{ secrets.CODECOV_TOKEN }} flags: unit @@ -155,7 +157,7 @@ jobs: CONSTRUCTOR_SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.17763.0/x86/signtool.exe" run: | rm -rf coverage.json - pytest -vv --cov=constructor --cov-branch tests/test_examples.py + pytest -vv --cov=constructor --cov-branch tests/test_examples.py -k extra_envs coverage run --branch --append -m constructor -V coverage json - uses: codecov/codecov-action@fdcc8476540edceab3de004e990f80d881c6cc00 # v5.5.0 From e178c1a6a8f6f5530d62997757c53d6f431576d7 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 21:13:35 +0200 Subject: [PATCH 06/19] rename target location for lockfile --- constructor/nsis/main.nsi.tmpl | 2 +- constructor/osx/run_installation.sh | 6 +++--- constructor/shar.py | 4 ++-- tests/test_examples.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index e3b052d7d..be8611f9c 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1383,7 +1383,7 @@ Section "Install" SetOutPath "{{ env.conda_meta }}" File "{{ env.history_abspath }}" # We also place a copy of the env.txt file here, as a lockfile - File "{{ env.env_txt_abspath }}" + File /oname=initial-state.lockfile.txt "{{ env.env_txt_abspath }}" # Set channels System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_CHANNELS", "{{ channels }}").r0' diff --git a/constructor/osx/run_installation.sh b/constructor/osx/run_installation.sh index 91882c61b..65ffad3a4 100644 --- a/constructor/osx/run_installation.sh +++ b/constructor/osx/run_installation.sh @@ -60,8 +60,8 @@ fi # Move the prepackaged history file into place mv "$PREFIX/pkgs/conda-meta/history" "$PREFIX/conda-meta/history" -# Place a copy of the lockfile in conda-meta for future, potential restoring -cp "$PREFIX/pkgs/env.txt" "$PREFIX/conda-meta/env.txt" +# Place a copy of the input lockfile in conda-meta for future, potential restoring +cp "$PREFIX/pkgs/env.txt" "$PREFIX/conda-meta/initial-state.lockfile.txt" rm -f "$PREFIX/env.txt" # Same, but for the extra environments @@ -105,7 +105,7 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do # Move the prepackaged history file into place mv "${env_pkgs}/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history" # Move the input lockfile in conda-meta for future, potential restoring - mv "${env_pkgs}env.txt" "$PREFIX/envs/$env_name/conda-meta/env.txt" + mv "${env_pkgs}env.txt" "$PREFIX/envs/$env_name/conda-meta/initial-state.lockfile.txt" done # Cleanup! diff --git a/constructor/shar.py b/constructor/shar.py index 1d945827d..f7768f39d 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -171,7 +171,7 @@ def create(info, verbose=False): pre_t.addfile(tarinfo=tarfile.TarInfo("conda-meta/history")) post_t.add(join(tmp_dir, "conda-meta", "history"), "conda-meta/history") # Place a copy of the lockfile in conda-meta for future, potential restoring - post_t.add(join(tmp_dir, "env.txt"), "conda-meta/env.txt") + post_t.add(join(tmp_dir, "env.txt"), "conda-meta/initial-state.lockfile.txt") for env_name in info.get("_extra_envs_info", {}): pre_t.addfile(tarinfo=tarfile.TarInfo(f"envs/{env_name}/conda-meta/history")) @@ -182,7 +182,7 @@ def create(info, verbose=False): # Place a copy of the lockfile in conda-meta for future, potential restoring post_t.add( join(tmp_dir, "envs", env_name, "env.txt"), - f"envs/{env_name}/conda-meta/env.txt", + f"envs/{env_name}/conda-meta/initial-state.lockfile.txt", ) extra_files = copy_extra_files(info.get("extra_files", []), tmp_dir) diff --git a/tests/test_examples.py b/tests/test_examples.py index 7adf8d4b8..cb8ac89b4 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -466,8 +466,8 @@ def test_example_extra_envs(tmp_path, request): input_path = _example_path("extra_envs") for installer, install_dir in create_installer(input_path, tmp_path): _run_installer(input_path, installer, install_dir, request=request) - assert "@EXPLICIT" in (install_dir / "conda-meta" / "env.txt").read_text() - for envtxt in install_dir.glob("envs/*/conda-meta/env.txt"): + assert "@EXPLICIT" in (install_dir / "conda-meta" / "initial-state.lockfile.txt").read_text() + for envtxt in install_dir.glob("envs/*/conda-meta/initial-state.lockfile.txt"): assert "@EXPLICIT" in envtxt.read_text() From ead76c91f452f3d6cc3a9da9ff5f4c5fad21990b Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 21:13:54 +0200 Subject: [PATCH 07/19] pre-commit --- tests/test_examples.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index cb8ac89b4..712a4a474 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -466,7 +466,9 @@ def test_example_extra_envs(tmp_path, request): input_path = _example_path("extra_envs") for installer, install_dir in create_installer(input_path, tmp_path): _run_installer(input_path, installer, install_dir, request=request) - assert "@EXPLICIT" in (install_dir / "conda-meta" / "initial-state.lockfile.txt").read_text() + assert ( + "@EXPLICIT" in (install_dir / "conda-meta" / "initial-state.lockfile.txt").read_text() + ) for envtxt in install_dir.glob("envs/*/conda-meta/initial-state.lockfile.txt"): assert "@EXPLICIT" in envtxt.read_text() From 5201e8983735834f9195d758fcb2aef3807e5e3c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 21:14:32 +0200 Subject: [PATCH 08/19] amend news --- news/1059-lockfiles | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/1059-lockfiles b/news/1059-lockfiles index 46bc4ae39..af5a0d12a 100644 --- a/news/1059-lockfiles +++ b/news/1059-lockfiles @@ -1,6 +1,6 @@ ### Enhancements -* Ship `conda-meta/env.txt` as a lockfile that provisions the initial state of the environment. (#1052 via #1059) +* Ship `conda-meta/initial-state.lockfile.txt` as a copy of the lockfile that provisions the initial state of each environment. (#1052 via #1059) ### Bug fixes From a08a0aae1923a00793630a966ca14e8d2fe6c132 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 21:38:01 +0200 Subject: [PATCH 09/19] run uninstall _after_ assertions --- tests/test_examples.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 712a4a474..500369215 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -465,13 +465,16 @@ def test_example_extra_pages_win(tmp_path, request, extra_pages, monkeypatch): def test_example_extra_envs(tmp_path, request): input_path = _example_path("extra_envs") for installer, install_dir in create_installer(input_path, tmp_path): - _run_installer(input_path, installer, install_dir, request=request) + _run_installer(input_path, installer, install_dir, request=request, uninstall=False) assert ( "@EXPLICIT" in (install_dir / "conda-meta" / "initial-state.lockfile.txt").read_text() ) for envtxt in install_dir.glob("envs/*/conda-meta/initial-state.lockfile.txt"): assert "@EXPLICIT" in envtxt.read_text() + if sys.platform.startswith("win"): + _run_uninstaller_exe(install_dir=install_dir) + def test_example_extra_files(tmp_path, request): input_path = _example_path("extra_files") From 3c9f35770e2cb7a77ce1f245a2381106392e7eb1 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 22:26:42 +0200 Subject: [PATCH 10/19] revert --- .github/workflows/main.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 661ea11ad..2b30c6e09 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -136,13 +136,11 @@ jobs: - name: conda config run: conda config --show-sources - name: Run unit tests - if: false run: | pytest -vv --cov=constructor --cov-branch tests/ -m "not examples" coverage run --branch --append -m constructor -V coverage json - uses: codecov/codecov-action@fdcc8476540edceab3de004e990f80d881c6cc00 # v5.5.0 - if: false with: token: ${{ secrets.CODECOV_TOKEN }} flags: unit @@ -157,7 +155,7 @@ jobs: CONSTRUCTOR_SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.17763.0/x86/signtool.exe" run: | rm -rf coverage.json - pytest -vv --cov=constructor --cov-branch tests/test_examples.py -k extra_envs + pytest -vv --cov=constructor --cov-branch tests/test_examples.py coverage run --branch --append -m constructor -V coverage json - uses: codecov/codecov-action@fdcc8476540edceab3de004e990f80d881c6cc00 # v5.5.0 From 2661f027721504d6331338dea8c6d5c0597ad134 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 23:31:00 +0200 Subject: [PATCH 11/19] Update tests/test_examples.py Co-authored-by: Marco Esters --- tests/test_examples.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 500369215..00518fe9a 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -469,7 +469,9 @@ def test_example_extra_envs(tmp_path, request): assert ( "@EXPLICIT" in (install_dir / "conda-meta" / "initial-state.lockfile.txt").read_text() ) - for envtxt in install_dir.glob("envs/*/conda-meta/initial-state.lockfile.txt"): + for env in install_dir.glob("envs/*/conda-meta/"): + envtxt / "initial-state.lockfile.txt" + assert envtxt.exists() assert "@EXPLICIT" in envtxt.read_text() if sys.platform.startswith("win"): From 95848545e06c8e20187d20b5b9cea342451d6fea Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 23:53:05 +0200 Subject: [PATCH 12/19] Just use initial-state.explicit.txt as the input file for environments directly --- constructor/header.sh | 6 ++---- constructor/nsis/main.nsi.tmpl | 15 ++++----------- constructor/osx/run_installation.sh | 9 ++------- constructor/preconda.py | 10 +++++----- constructor/shar.py | 17 ++++++++--------- constructor/winexe.py | 18 ++++++++++-------- news/1059-lockfiles | 2 +- tests/test_examples.py | 4 ++-- 8 files changed, 34 insertions(+), 47 deletions(-) diff --git a/constructor/header.sh b/constructor/header.sh index 8dfd30c32..e11d5cfd9 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -598,8 +598,7 @@ CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS="{{ channels }}" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ CONDA_QUIET="$BATCH" \ -"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }} || exit 1 -rm -f "$PREFIX/pkgs/env.txt" +"$CONDA_EXEC" install --offline --file "$PREFIX/conda-meta/initial-state.explicit.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }} || exit 1 {%- if has_conda %} mkdir -p "$PREFIX/envs" @@ -638,8 +637,7 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do CONDA_CHANNELS="$env_channels" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ CONDA_QUIET="$BATCH" \ - "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 - rm -f "${env_pkgs}env.txt" + "$CONDA_EXEC" install --offline --file "$PREFIX/envs/$env_name/conda-meta/initial-state.explicit.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 done {%- endif %} diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index be8611f9c..c14c6fd19 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1375,15 +1375,12 @@ Section "Install" ${Print} "Setting up the {{ env.name }} environment..." SetDetailsPrint listonly - # List of packages to install - SetOutPath "{{ env.env_txt_dir }}" - File "{{ env.env_txt_abspath }}" # A conda-meta\history file is required for a valid conda prefix SetOutPath "{{ env.conda_meta }}" File "{{ env.history_abspath }}" - # We also place a copy of the env.txt file here, as a lockfile - File /oname=initial-state.lockfile.txt "{{ env.env_txt_abspath }}" + # List of packages to install, as a lockfile + File "{{ env.lockfile_txt_abspath }}" # Set channels System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_CHANNELS", "{{ channels }}").r0' @@ -1393,10 +1390,10 @@ Section "Install" # Run conda install ${If} $Ana_CreateShortcuts_State = ${BST_CHECKED} ${Print} "Installing packages for {{ env.name }}, creating shortcuts if necessary..." - push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.env_txt }}" {{ env.shortcuts }} {{ env.no_rcs_arg }}' + push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.lockfile_txt }}" {{ env.shortcuts }} {{ env.no_rcs_arg }}' ${Else} ${Print} "Installing packages for {{ env.name }}..." - push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.env_txt }}" --no-shortcuts {{ env.no_rcs_arg }}' + push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.lockfile_txt }}" --no-shortcuts {{ env.no_rcs_arg }}' ${EndIf} push 'Failed to link extracted packages to {{ env.prefix }}!' push 'WithLog' @@ -1404,10 +1401,6 @@ Section "Install" call AbortRetryNSExecWait SetDetailsPrint both - # Cleanup {{ env.name }} env.txt - SetOutPath "$INSTDIR" - Delete "{{ env.env_txt }}" - # Restore shipped conda-meta\history for remapped # channels and retain only the first transaction SetOutPath "{{ env.conda_meta }}" diff --git a/constructor/osx/run_installation.sh b/constructor/osx/run_installation.sh index 65ffad3a4..84a3fdc53 100644 --- a/constructor/osx/run_installation.sh +++ b/constructor/osx/run_installation.sh @@ -53,16 +53,13 @@ CONDA_SAFETY_CHECKS=disabled \ CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS={{ channels }} \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ -"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }}; then +"$CONDA_EXEC" install --offline --file "$PREFIX/conda-meta/initial-state.explicit.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }}; then echo "ERROR: could not complete the conda install" exit 1 fi # Move the prepackaged history file into place mv "$PREFIX/pkgs/conda-meta/history" "$PREFIX/conda-meta/history" -# Place a copy of the input lockfile in conda-meta for future, potential restoring -cp "$PREFIX/pkgs/env.txt" "$PREFIX/conda-meta/initial-state.lockfile.txt" -rm -f "$PREFIX/env.txt" # Same, but for the extra environments @@ -101,11 +98,9 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS="$env_channels" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ - "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 + "$CONDA_EXEC" install --offline --file "$PREFIX/envs/$env_name/conda-meta/initial-state.explicit.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 # Move the prepackaged history file into place mv "${env_pkgs}/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history" - # Move the input lockfile in conda-meta for future, potential restoring - mv "${env_pkgs}env.txt" "$PREFIX/envs/$env_name/conda-meta/initial-state.lockfile.txt" done # Cleanup! diff --git a/constructor/preconda.py b/constructor/preconda.py index 6509cf925..a064d9ab4 100644 --- a/constructor/preconda.py +++ b/constructor/preconda.py @@ -48,7 +48,7 @@ except ImportError: import ruamel_json as json -files = ".constructor-build.info", "urls", "urls.txt", "env.txt" +files = ".constructor-build.info", "urls", "urls.txt", "initial-state.explicit.txt" def write_index_cache(info, dst_dir, used_packages): @@ -172,7 +172,7 @@ def write_files(info, dst_dir): # base environment file used with conda install --file # (list of specs/dists to install) - write_env_txt(info, dst_dir, final_urls_md5s) + write_initial_state_lockfile_txt(info, dst_dir, final_urls_md5s) for fn in files: os.chmod(join(dst_dir, fn), 0o664) @@ -185,7 +185,7 @@ def write_files(info, dst_dir): user_requested_specs = env_config.get("user_requested_specs", env_config.get("specs", ())) write_conda_meta(info, env_dst_dir, env_urls_md5, user_requested_specs) # environment installation list - write_env_txt(info, env_dst_dir, env_urls_md5) + write_initial_state_lockfile_txt(info, env_dst_dir, env_urls_md5) # channels write_channels_txt(info, env_dst_dir, env_config) # shortcuts @@ -245,7 +245,7 @@ def write_repodata_record(info, dst_dir): json.dump(rr_json, rf, indent=2, sort_keys=True) -def write_env_txt(info, dst_dir, urls): +def write_initial_state_lockfile_txt(info, dst_dir, urls): """ urls is an iterable of tuples with url and md5 values """ @@ -257,7 +257,7 @@ def write_env_txt(info, dst_dir, urls): @EXPLICIT """ ).lstrip() - with open(join(dst_dir, "env.txt"), "w") as envf: + with open(join(dst_dir, "initial-state.explicit.txt"), "w") as envf: envf.write(header) for url, md5 in urls: maybe_different_url = ensure_transmuted_ext(info, url) diff --git a/constructor/shar.py b/constructor/shar.py index f7768f39d..deaf80cd2 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -133,10 +133,16 @@ def create(info, verbose=False): post_t = tarfile.open(postconda_tarball, "w:bz2") for dist in preconda_files: fn = filename_dist(dist) - pre_t.add(join(tmp_dir, fn), "pkgs/" + fn) + pre_t.add( + join(tmp_dir, fn), + f"conda-meta/{fn}" if dist == "initial-state.explicit.txt" else f"pkgs/{fn}", + ) for env_name in info.get("_extra_envs_info", ()): - pre_t.add(join(tmp_dir, "envs", env_name, "env.txt"), f"pkgs/envs/{env_name}/env.txt") + pre_t.add( + join(tmp_dir, "envs", env_name, "initial-state.explicit.txt"), + f"envs/{env_name}/conda-meta/initial-state.explicit.txt", + ) pre_t.add( join(tmp_dir, "envs", env_name, "shortcuts.txt"), f"pkgs/envs/{env_name}/shortcuts.txt" ) @@ -170,8 +176,6 @@ def create(info, verbose=False): pre_t.add(record_file_src, record_file_dest) pre_t.addfile(tarinfo=tarfile.TarInfo("conda-meta/history")) post_t.add(join(tmp_dir, "conda-meta", "history"), "conda-meta/history") - # Place a copy of the lockfile in conda-meta for future, potential restoring - post_t.add(join(tmp_dir, "env.txt"), "conda-meta/initial-state.lockfile.txt") for env_name in info.get("_extra_envs_info", {}): pre_t.addfile(tarinfo=tarfile.TarInfo(f"envs/{env_name}/conda-meta/history")) @@ -179,11 +183,6 @@ def create(info, verbose=False): join(tmp_dir, "envs", env_name, "conda-meta", "history"), f"envs/{env_name}/conda-meta/history", ) - # Place a copy of the lockfile in conda-meta for future, potential restoring - post_t.add( - join(tmp_dir, "envs", env_name, "env.txt"), - f"envs/{env_name}/conda-meta/initial-state.lockfile.txt", - ) extra_files = copy_extra_files(info.get("extra_files", []), tmp_dir) for path in extra_files: diff --git a/constructor/winexe.py b/constructor/winexe.py index 0dd0e8255..8706de4e5 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -81,10 +81,10 @@ def setup_envs_commands(info, dir_path): { "name": "base", "prefix": r"$INSTDIR", - "env_txt": r"$INSTDIR\pkgs\env.txt", # env.txt as seen by the running installer - "env_txt_dir": r"$INSTDIR\pkgs", # env.txt location in the installer filesystem - # env.txt path while building the installer - "env_txt_abspath": join(dir_path, "env.txt"), + # initial-state.explicit.txt as seen by the running installer + "lockfile_txt": r"$INSTDIR\pkgs\initial-state.explicit.txt", + # initial-state.explicit.txt path while building the installer + "lockfile_txt_abspath": join(dir_path, "initial-state.explicit.txt"), "conda_meta": r"$INSTDIR\conda-meta", "history_abspath": join(dir_path, "conda-meta", "history"), "final_channels": get_final_channels(info), @@ -107,9 +107,12 @@ def setup_envs_commands(info, dir_path): { "name": env_name, "prefix": join("$INSTDIR", "envs", env_name), - "env_txt": join("$INSTDIR", "pkgs", "envs", env_name, "env.txt"), - "env_txt_dir": join("$INSTDIR", "pkgs", "envs", env_name), - "env_txt_abspath": join(dir_path, "envs", env_name, "env.txt"), + "lockfile_txt": join( + "$INSTDIR", "envs", env_name, "conda-meta", "initial-state.explicit.txt" + ), + "lockfile_txt_abspath": join( + dir_path, "envs", env_name, "initial-state.explicit.txt" + ), "conda_meta": join("$INSTDIR", "envs", env_name, "conda-meta"), "history_abspath": join(dir_path, "envs", env_name, "conda-meta", "history"), "final_channels": get_final_channels(channel_info), @@ -174,7 +177,6 @@ def make_nsi( "licensefile": abspath(info.get("license_file", join(NSIS_DIR, "placeholder_license.txt"))), "conda_history": "@" + join("conda-meta", "history"), "conda_exe": "@_conda.exe", - "env_txt": "@env.txt", "urls_file": "@urls", "urls_txt_file": "@urls.txt", "pre_install": "@pre_install.bat", diff --git a/news/1059-lockfiles b/news/1059-lockfiles index af5a0d12a..e4586e8ad 100644 --- a/news/1059-lockfiles +++ b/news/1059-lockfiles @@ -1,6 +1,6 @@ ### Enhancements -* Ship `conda-meta/initial-state.lockfile.txt` as a copy of the lockfile that provisions the initial state of each environment. (#1052 via #1059) +* Ship `conda-meta/initial-state.explicit.txt` as a copy of the lockfile that provisions the initial state of each environment. (#1052 via #1059) ### Bug fixes diff --git a/tests/test_examples.py b/tests/test_examples.py index 500369215..143f5ebf8 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -467,9 +467,9 @@ def test_example_extra_envs(tmp_path, request): for installer, install_dir in create_installer(input_path, tmp_path): _run_installer(input_path, installer, install_dir, request=request, uninstall=False) assert ( - "@EXPLICIT" in (install_dir / "conda-meta" / "initial-state.lockfile.txt").read_text() + "@EXPLICIT" in (install_dir / "conda-meta" / "initial-state.explicit.txt").read_text() ) - for envtxt in install_dir.glob("envs/*/conda-meta/initial-state.lockfile.txt"): + for envtxt in install_dir.glob("envs/*/conda-meta/initial-state.explicit.txt"): assert "@EXPLICIT" in envtxt.read_text() if sys.platform.startswith("win"): From 550a98ba6155d5274b0e6d2fea700ae4f2212fcf Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 11 Sep 2025 23:56:10 +0200 Subject: [PATCH 13/19] rename function name --- constructor/preconda.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/constructor/preconda.py b/constructor/preconda.py index a064d9ab4..3d105955e 100644 --- a/constructor/preconda.py +++ b/constructor/preconda.py @@ -172,7 +172,7 @@ def write_files(info, dst_dir): # base environment file used with conda install --file # (list of specs/dists to install) - write_initial_state_lockfile_txt(info, dst_dir, final_urls_md5s) + write_initial_state_explicit_txt(info, dst_dir, final_urls_md5s) for fn in files: os.chmod(join(dst_dir, fn), 0o664) @@ -185,7 +185,7 @@ def write_files(info, dst_dir): user_requested_specs = env_config.get("user_requested_specs", env_config.get("specs", ())) write_conda_meta(info, env_dst_dir, env_urls_md5, user_requested_specs) # environment installation list - write_initial_state_lockfile_txt(info, env_dst_dir, env_urls_md5) + write_initial_state_explicit_txt(info, env_dst_dir, env_urls_md5) # channels write_channels_txt(info, env_dst_dir, env_config) # shortcuts @@ -245,7 +245,7 @@ def write_repodata_record(info, dst_dir): json.dump(rr_json, rf, indent=2, sort_keys=True) -def write_initial_state_lockfile_txt(info, dst_dir, urls): +def write_initial_state_explicit_txt(info, dst_dir, urls): """ urls is an iterable of tuples with url and md5 values """ From d194aee43a7b12075115a90ae325d49895ba8356 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 12 Sep 2025 01:04:57 +0200 Subject: [PATCH 14/19] Patch locations in PKG --- constructor/osxpkg.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index bc6d7a7ef..b9668049a 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -562,8 +562,18 @@ def create(info, verbose=False): fresh_dir(PACKAGE_ROOT) fresh_dir(SCRIPTS_DIR) pkgs_dir = join(prefix, "pkgs") + conda_meta = join(prefix, "conda-meta") os.makedirs(pkgs_dir) + os.makedirs(conda_meta) preconda.write_files(info, pkgs_dir) + # We need to move the initial-state.explicit.txt files from pkgs/ to their conda-meta targets + shutil.move(join(pkgs_dir, "initial-state.explicit.txt"), conda_meta) + if isdir(join(pkgs_dir, "envs")): + for envname in os.listdir(join(pkgs_dir, "envs")): + lockfile = join(pkgs_dir, "envs", envname, "initial-state.explicit.txt") + env_conda_meta = join(prefix, "envs", envname, "conda-meta") + os.makedirs(env_conda_meta) + shutil.move(lockfile, env_conda_meta) preconda.copy_extra_files(info.get("extra_files", []), prefix) # These are the user-provided scripts, maybe patched to have a shebang # They will be called by a wrapping script added later, if present From 26a464e9755229d77ab0d40090713d963c99aaa9 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 12 Sep 2025 01:15:41 +0200 Subject: [PATCH 15/19] fix path in windows --- constructor/winexe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constructor/winexe.py b/constructor/winexe.py index 8706de4e5..010c1b73a 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -82,7 +82,7 @@ def setup_envs_commands(info, dir_path): "name": "base", "prefix": r"$INSTDIR", # initial-state.explicit.txt as seen by the running installer - "lockfile_txt": r"$INSTDIR\pkgs\initial-state.explicit.txt", + "lockfile_txt": r"$INSTDIR\conda-meta\initial-state.explicit.txt", # initial-state.explicit.txt path while building the installer "lockfile_txt_abspath": join(dir_path, "initial-state.explicit.txt"), "conda_meta": r"$INSTDIR\conda-meta", From fe1e040a7152f8cb0214a2c48d366740645749fc Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 13 Sep 2025 14:13:30 +0200 Subject: [PATCH 16/19] Refactor write_files() --- .gitignore | 3 ++ constructor/osx/run_installation.sh | 19 +++++++--- constructor/osxpkg.py | 12 +----- constructor/preconda.py | 58 +++++++++++++++++++---------- constructor/shar.py | 22 ++++------- constructor/winexe.py | 5 ++- 6 files changed, 66 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index 3e0406724..22609e17e 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,6 @@ build/ # setuptools-scm constructor/_version.py + +# Temporary workspaces +tmp/ diff --git a/constructor/osx/run_installation.sh b/constructor/osx/run_installation.sh index 84a3fdc53..ec69a9b1c 100644 --- a/constructor/osx/run_installation.sh +++ b/constructor/osx/run_installation.sh @@ -45,6 +45,11 @@ fi # Perform the conda install notify "Installing packages. This might take a few minutes." + +# 'install' below will modify the history file in a way we don't want; +# keep a copy to restore later +cp "$PREFIX/conda-meta/history" "$PREFIX/conda-meta/history.bak" + # shellcheck disable=SC2086 if ! \ CONDA_REGISTER_ENVS="{{ register_envs }}" \ @@ -58,8 +63,8 @@ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ exit 1 fi -# Move the prepackaged history file into place -mv "$PREFIX/pkgs/conda-meta/history" "$PREFIX/conda-meta/history" +# Restore history file as provided by installer +mv "$PREFIX/conda-meta/history.bak" "$PREFIX/conda-meta/history" # Same, but for the extra environments @@ -72,8 +77,9 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do fi notify "Installing ${env_name} packages..." - mkdir -p "$PREFIX/envs/$env_name/conda-meta" - touch "$PREFIX/envs/$env_name/conda-meta/history" + # 'install' below will modify the history file in a way we don't want; + # keep a copy to restore later + cp "$PREFIX/envs/$env_name/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history.bak" if [[ -f "${env_pkgs}channels.txt" ]]; then env_channels="$(cat "${env_pkgs}channels.txt")" @@ -99,8 +105,9 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do CONDA_CHANNELS="$env_channels" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ "$CONDA_EXEC" install --offline --file "$PREFIX/envs/$env_name/conda-meta/initial-state.explicit.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 - # Move the prepackaged history file into place - mv "${env_pkgs}/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history" + + # Restore history file as provided by installer + mv "$PREFIX/envs/$env_name/conda-meta/history.bak" "$PREFIX/envs/$env_name/conda-meta/history" done # Cleanup! diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index b9668049a..3785ac617 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -562,18 +562,8 @@ def create(info, verbose=False): fresh_dir(PACKAGE_ROOT) fresh_dir(SCRIPTS_DIR) pkgs_dir = join(prefix, "pkgs") - conda_meta = join(prefix, "conda-meta") os.makedirs(pkgs_dir) - os.makedirs(conda_meta) - preconda.write_files(info, pkgs_dir) - # We need to move the initial-state.explicit.txt files from pkgs/ to their conda-meta targets - shutil.move(join(pkgs_dir, "initial-state.explicit.txt"), conda_meta) - if isdir(join(pkgs_dir, "envs")): - for envname in os.listdir(join(pkgs_dir, "envs")): - lockfile = join(pkgs_dir, "envs", envname, "initial-state.explicit.txt") - env_conda_meta = join(prefix, "envs", envname, "conda-meta") - os.makedirs(env_conda_meta) - shutil.move(lockfile, env_conda_meta) + preconda.write_files(info, prefix) preconda.copy_extra_files(info.get("extra_files", []), prefix) # These are the user-provided scripts, maybe patched to have a shebang # They will be called by a wrapping script added later, if present diff --git a/constructor/preconda.py b/constructor/preconda.py index 3d105955e..9f24de701 100644 --- a/constructor/preconda.py +++ b/constructor/preconda.py @@ -48,7 +48,12 @@ except ImportError: import ruamel_json as json -files = ".constructor-build.info", "urls", "urls.txt", "initial-state.explicit.txt" +files = ( + "pkgs/.constructor-build.info", + "pkgs/urls", + "pkgs/urls.txt", + "conda-meta/initial-state.explicit.txt", +) def write_index_cache(info, dst_dir, used_packages): @@ -135,8 +140,19 @@ def system_info(): return out -def write_files(info, dst_dir): - with open(join(dst_dir, ".constructor-build.info"), "w") as fo: +def write_files(info: dict, workspace: str): + """ + Prepare files on disk to be shipped as part of the pre-conda payload. + + Contents: + + - `conda-meta/initial-state.explicit.txt` + - `pkgs/`: + """ + os.makedirs(join(workspace, "conda-meta"), exist_ok=True) + pkgs_dir = join(workspace, "pkgs") + os.makedirs(pkgs_dir, exist_ok=True) + with open(join(pkgs_dir, ".constructor-build.info"), "w") as fo: json.dump(system_info(), fo) all_urls = info["_urls"].copy() @@ -146,7 +162,7 @@ def write_files(info, dst_dir): final_urls_md5s = tuple((get_final_url(info, url), md5) for url, md5 in info["_urls"]) all_final_urls_md5s = tuple((get_final_url(info, url), md5) for url, md5 in all_urls) - with open(join(dst_dir, "urls"), "w") as fo: + with open(join(pkgs_dir, "urls"), "w") as fo: for url, md5 in all_final_urls_md5s: maybe_different_url = ensure_transmuted_ext(info, url) if maybe_different_url != url: # transmuted, no md5 @@ -154,7 +170,7 @@ def write_files(info, dst_dir): else: fo.write(f"{url}#{md5}\n") - with open(join(dst_dir, "urls.txt"), "w") as fo: + with open(join(pkgs_dir, "urls.txt"), "w") as fo: for url, _ in all_final_urls_md5s: fo.write("%s\n" % url) @@ -163,33 +179,37 @@ def write_files(info, dst_dir): all_dists += env_info["_dists"] all_dists = list({dist: None for dist in all_dists}) # de-duplicate - write_index_cache(info, dst_dir, all_dists) + write_index_cache(info, pkgs_dir, all_dists) # base environment conda-meta - write_conda_meta(info, dst_dir, final_urls_md5s) + write_conda_meta(info, join(workspace, "conda-meta"), final_urls_md5s) - write_repodata_record(info, dst_dir) + write_repodata_record(info, pkgs_dir) # base environment file used with conda install --file # (list of specs/dists to install) - write_initial_state_explicit_txt(info, dst_dir, final_urls_md5s) - - for fn in files: - os.chmod(join(dst_dir, fn), 0o664) + write_initial_state_explicit_txt(info, join(workspace, "conda-meta"), final_urls_md5s) for env_name, env_info in info.get("_extra_envs_info", {}).items(): env_config = info["extra_envs"][env_name] - env_dst_dir = os.path.join(dst_dir, "envs", env_name) + env_pkgs = os.path.join(workspace, "pkgs", "envs", env_name) + env_conda_meta = os.path.join(workspace, "envs", env_name, "conda-meta") + os.makedirs(env_pkgs, exist_ok=True) + os.makedirs(env_conda_meta, exist_ok=True) # environment conda-meta env_urls_md5 = tuple((get_final_url(info, url), md5) for url, md5 in env_info["_urls"]) user_requested_specs = env_config.get("user_requested_specs", env_config.get("specs", ())) - write_conda_meta(info, env_dst_dir, env_urls_md5, user_requested_specs) + write_conda_meta(info, env_conda_meta, env_urls_md5, user_requested_specs) # environment installation list - write_initial_state_explicit_txt(info, env_dst_dir, env_urls_md5) + write_initial_state_explicit_txt(info, env_conda_meta, env_urls_md5) # channels - write_channels_txt(info, env_dst_dir, env_config) + write_channels_txt(info, env_pkgs, env_config) # shortcuts - write_shortcuts_txt(info, env_dst_dir, env_config) + write_shortcuts_txt(info, env_pkgs, env_config) + + for dirpath, _, filenames in os.walk(workspace): + for filename in filenames: + os.chmod(join(dirpath, filename), 0o664) def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None): @@ -212,9 +232,7 @@ def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None): builder.append("# update specs: %s" % update_specs) builder.append("\n") - if not isdir(join(dst_dir, "conda-meta")): - os.makedirs(join(dst_dir, "conda-meta")) - with open(join(dst_dir, "conda-meta", "history"), "w") as fh: + with open(join(dst_dir, "history"), "w") as fh: fh.write("\n".join(builder)) diff --git a/constructor/shar.py b/constructor/shar.py index deaf80cd2..f6361a0f9 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -131,21 +131,15 @@ def create(info, verbose=False): postconda_tarball = join(tmp_dir, "postconda.tar.bz2") pre_t = tarfile.open(preconda_tarball, "w:bz2") post_t = tarfile.open(postconda_tarball, "w:bz2") - for dist in preconda_files: - fn = filename_dist(dist) - pre_t.add( - join(tmp_dir, fn), - f"conda-meta/{fn}" if dist == "initial-state.explicit.txt" else f"pkgs/{fn}", - ) + for rel_path in preconda_files: + pre_t.add(join(tmp_dir, rel_path), rel_path) for env_name in info.get("_extra_envs_info", ()): - pre_t.add( - join(tmp_dir, "envs", env_name, "initial-state.explicit.txt"), + for rel_path in ( + f"pkgs/envs/{env_name}/shortcuts.txt", f"envs/{env_name}/conda-meta/initial-state.explicit.txt", - ) - pre_t.add( - join(tmp_dir, "envs", env_name, "shortcuts.txt"), f"pkgs/envs/{env_name}/shortcuts.txt" - ) + ): + pre_t.add(join(tmp_dir, rel_path), rel_path) for key in "pre_install", "post_install": if key in info: @@ -171,7 +165,7 @@ def create(info, verbose=False): elif filename_dist(dist).endswith(".tar.bz2"): _dist = filename_dist(dist)[:-8] record_file = join(_dist, "info", "repodata_record.json") - record_file_src = join(tmp_dir, record_file) + record_file_src = join(tmp_dir, "pkgs", record_file) record_file_dest = join("pkgs", record_file) pre_t.add(record_file_src, record_file_dest) pre_t.addfile(tarinfo=tarfile.TarInfo("conda-meta/history")) @@ -191,7 +185,7 @@ def create(info, verbose=False): pre_t.close() post_t.close() - tarball = join(tmp_dir, "tmp.tar") + tarball = join(tmp_dir, "pkgs", "tmp.tar") t = tarfile.open(tarball, "w") t.add(preconda_tarball, basename(preconda_tarball)) t.add(postconda_tarball, basename(postconda_tarball)) diff --git a/constructor/winexe.py b/constructor/winexe.py index 010c1b73a..ca523e201 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -84,7 +84,7 @@ def setup_envs_commands(info, dir_path): # initial-state.explicit.txt as seen by the running installer "lockfile_txt": r"$INSTDIR\conda-meta\initial-state.explicit.txt", # initial-state.explicit.txt path while building the installer - "lockfile_txt_abspath": join(dir_path, "initial-state.explicit.txt"), + "lockfile_txt_abspath": join(dir_path, "conda-meta", "initial-state.explicit.txt"), "conda_meta": r"$INSTDIR\conda-meta", "history_abspath": join(dir_path, "conda-meta", "history"), "final_channels": get_final_channels(info), @@ -111,7 +111,7 @@ def setup_envs_commands(info, dir_path): "$INSTDIR", "envs", env_name, "conda-meta", "initial-state.explicit.txt" ), "lockfile_txt_abspath": join( - dir_path, "envs", env_name, "initial-state.explicit.txt" + dir_path, "envs", env_name, "conda-meta", "initial-state.explicit.txt" ), "conda_meta": join("$INSTDIR", "envs", env_name, "conda-meta"), "history_abspath": join(dir_path, "envs", env_name, "conda-meta", "history"), @@ -171,6 +171,7 @@ def make_nsi( "outfile": info["_outpath"], "vipv": make_VIProductVersion(info["version"]), "constructor_version": info["CONSTRUCTOR_VERSION"], + # @-prefixed paths point to {dir_path} "iconfile": "@icon.ico", "headerimage": "@header.bmp", "welcomeimage": "@welcome.bmp", From edce6e7d20e965dd13899546f8235474446cd033 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 13 Sep 2025 14:30:54 +0200 Subject: [PATCH 17/19] Fix some paths in windows --- constructor/winexe.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/constructor/winexe.py b/constructor/winexe.py index ca523e201..bf0f91a22 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -178,13 +178,13 @@ def make_nsi( "licensefile": abspath(info.get("license_file", join(NSIS_DIR, "placeholder_license.txt"))), "conda_history": "@" + join("conda-meta", "history"), "conda_exe": "@_conda.exe", - "urls_file": "@urls", - "urls_txt_file": "@urls.txt", + "urls_file": "@" + join("pkgs", "urls"), + "urls_txt_file": "@" + join("pkgs", "urls.txt"), "pre_install": "@pre_install.bat", "post_install": "@post_install.bat", "pre_uninstall": "@pre_uninstall.bat", - "index_cache": "@cache", - "repodata_record": "@repodata_record.json", + "index_cache": "@" + join("pkgs", "cache"), + "repodata_record": "@" + join("pkgs", "repodata_record.json"), } conclusion_text = info.get("conclusion_text", "") From 9dda4cd344dd1fa91b04c98ece2053cf2da89a38 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 13 Sep 2025 15:02:26 +0200 Subject: [PATCH 18/19] finish docstring --- constructor/preconda.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/constructor/preconda.py b/constructor/preconda.py index 9f24de701..d90152844 100644 --- a/constructor/preconda.py +++ b/constructor/preconda.py @@ -142,12 +142,21 @@ def system_info(): def write_files(info: dict, workspace: str): """ - Prepare files on disk to be shipped as part of the pre-conda payload. + Prepare files on disk to be shipped as part of the pre-conda payload, mostly + configuration and metadata files: - Contents: + - `conda-meta/initial-state.explicit.txt`: Lockfile to provision the base environment. + - `conda-meta/history`: Prepared history file with the right requested specs in input file. + - `pkgs/urls` and `pkgs/urls.txt`: Direct URLs of packages used, with and without MD5 hashes. + - `pkgs/cache/*.json`: Trimmed repodata to mock offline channels in use. + - `pkgs/channels.txt`: Channels in use. + - `pkgs/shortcuts.txt`: Which packages should have their shortcuts created, if any. - - `conda-meta/initial-state.explicit.txt` - - `pkgs/`: + If extra envs are requested, this will also write: + + - Their corresponding `envs//conda-meta/` files. + - Their corresponding `pkgs/channels.txt` and `pkgs/shortcuts.txt` under + `pkgs/envs/`. """ os.makedirs(join(workspace, "conda-meta"), exist_ok=True) pkgs_dir = join(workspace, "pkgs") @@ -190,6 +199,9 @@ def write_files(info: dict, workspace: str): # (list of specs/dists to install) write_initial_state_explicit_txt(info, join(workspace, "conda-meta"), final_urls_md5s) + for fn in files: + os.chmod(join(workspace, fn), 0o664) + for env_name, env_info in info.get("_extra_envs_info", {}).items(): env_config = info["extra_envs"][env_name] env_pkgs = os.path.join(workspace, "pkgs", "envs", env_name) @@ -207,10 +219,6 @@ def write_files(info: dict, workspace: str): # shortcuts write_shortcuts_txt(info, env_pkgs, env_config) - for dirpath, _, filenames in os.walk(workspace): - for filename in filenames: - os.chmod(join(dirpath, filename), 0o664) - def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None): if user_requested_specs is None: From 2e3f3ffb533bf25ad1bd7003b99cb8ac860722eb Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 13 Sep 2025 15:04:27 +0200 Subject: [PATCH 19/19] pre-commit --- constructor/preconda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constructor/preconda.py b/constructor/preconda.py index d90152844..2beeb3dc6 100644 --- a/constructor/preconda.py +++ b/constructor/preconda.py @@ -155,7 +155,7 @@ def write_files(info: dict, workspace: str): If extra envs are requested, this will also write: - Their corresponding `envs//conda-meta/` files. - - Their corresponding `pkgs/channels.txt` and `pkgs/shortcuts.txt` under + - Their corresponding `pkgs/channels.txt` and `pkgs/shortcuts.txt` under `pkgs/envs/`. """ os.makedirs(join(workspace, "conda-meta"), exist_ok=True)