Skip to content

Commit 06f6749

Browse files
vmoensvmoens
committed
[CI] Use uv instead of conda
1 parent 80bfa6e commit 06f6749

File tree

101 files changed

+963
-1267
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

101 files changed

+963
-1267
lines changed

‎.github/unittest/linux/scripts/post_process.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22

33
set -e
44

5-
eval "$(./conda/bin/conda shell.bash hook)"
6-
conda activate ./env
5+
root_dir="$(git rev-parse --show-toplevel)"
6+
source "${root_dir}/.venv/bin/activate"

‎.github/unittest/linux/scripts/run_all.sh‎

Lines changed: 37 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ set -v
99

1010
if [[ $OSTYPE != 'darwin'* ]]; then
1111
apt-get update && apt-get upgrade -y
12-
apt-get install -y vim git wget cmake
12+
apt-get install -y vim git wget curl cmake
1313

1414
# Enable universe repository
1515
# apt-get install -y software-properties-common
@@ -45,37 +45,23 @@ fi
4545
# Avoid error: "fatal: unsafe repository"
4646
git config --global --add safe.directory '*'
4747
root_dir="$(git rev-parse --show-toplevel)"
48-
conda_dir="${root_dir}/conda"
49-
env_dir="${root_dir}/env"
50-
lib_dir="${env_dir}/lib"
48+
env_dir="${root_dir}/.venv"
5149

5250
cd "${root_dir}"
5351

54-
case "$(uname -s)" in
55-
Darwin*) os=MacOSX;;
56-
*) os=Linux
57-
esac
52+
# 1. Install uv
53+
printf "* Installing uv\n"
54+
curl -LsSf https://astral.sh/uv/install.sh | sh
55+
export PATH="$HOME/.local/bin:$PATH"
5856

59-
# 1. Install conda at ./conda
60-
if [ ! -d "${conda_dir}" ]; then
61-
printf "* Installing conda\n"
62-
wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh"
63-
bash ./miniconda.sh -b -f -p "${conda_dir}"
64-
fi
65-
eval "$(${conda_dir}/bin/conda shell.bash hook)"
66-
67-
# 2. Create test environment at ./env
57+
# 2. Create test environment at ./.venv
6858
printf "python: ${PYTHON_VERSION}\n"
69-
if [ ! -d "${env_dir}" ]; then
70-
printf "* Creating a test environment\n"
71-
conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION"
72-
fi
73-
conda activate "${env_dir}"
59+
printf "* Creating a test environment with uv\n"
60+
uv venv "${env_dir}" --python="${PYTHON_VERSION}"
61+
source "${env_dir}/bin/activate"
7462

75-
# 3. Install Conda dependencies
63+
# 3. Install dependencies (except PyTorch)
7664
printf "* Installing dependencies (except PyTorch)\n"
77-
echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml"
78-
cat "${this_dir}/environment.yml"
7965

8066
if [ "${CU_VERSION:-}" == cpu ] ; then
8167
export MUJOCO_GL=glfw
@@ -85,27 +71,31 @@ fi
8571

8672
export SDL_VIDEODRIVER=dummy
8773

88-
# legacy from bash scripts: remove?
89-
conda env config vars set \
90-
MAX_IDLE_COUNT=1000 \
91-
MUJOCO_GL=$MUJOCO_GL PYOPENGL_PLATFORM=$MUJOCO_GL DISPLAY=:99 SDL_VIDEODRIVER=dummy LAZY_LEGACY_OP=False RL_LOGGING_LEVEL=DEBUG TOKENIZERS_PARALLELISM=true
74+
# Set environment variables
75+
export MAX_IDLE_COUNT=1000
76+
export PYOPENGL_PLATFORM=$MUJOCO_GL
77+
export DISPLAY=:99
78+
export LAZY_LEGACY_OP=False
79+
export RL_LOGGING_LEVEL=DEBUG
80+
export TOKENIZERS_PARALLELISM=true
9281

93-
pip3 install pip --upgrade
94-
pip install virtualenv
82+
# Install dependencies from environment.yml using uv
83+
uv pip install hypothesis future cloudpickle pygame "moviepy<2.0.0" tqdm \
84+
pytest pytest-cov pytest-mock pytest-instafail pytest-rerunfailures \
85+
pytest-timeout pytest-asyncio expecttest "pybind11[global]" pyyaml scipy \
86+
hydra-core tensorboard "imageio==2.26.0" wandb dm_control "mujoco<3.3.6" \
87+
mlflow av coverage ray transformers ninja timm protobuf
9588

96-
conda env update --file "${this_dir}/environment.yml" --prune
97-
98-
# Reset conda env variables
99-
conda deactivate
100-
conda activate "${env_dir}"
89+
# Install pip for compatibility with packages that expect it
90+
uv pip install pip
10191

10292
echo "installing gymnasium"
10393
if [[ "$PYTHON_VERSION" == "3.12" ]]; then
104-
pip3 install ale-py
105-
pip3 install sympy
106-
pip3 install "gymnasium[mujoco]>=1.1" mo-gymnasium[mujoco]
94+
uv pip install ale-py
95+
uv pip install sympy
96+
uv pip install "gymnasium[mujoco]>=1.1" mo-gymnasium[mujoco]
10797
else
108-
pip3 install "gymnasium[atari,mujoco]>=1.1" mo-gymnasium[mujoco]
98+
uv pip install "gymnasium[atari,mujoco]>=1.1" mo-gymnasium[mujoco]
10999
fi
110100

111101
# sanity check: remove?
@@ -140,15 +130,15 @@ git submodule sync && git submodule update --init --recursive
140130
printf "Installing PyTorch with %s\n" "${CU_VERSION}"
141131
if [[ "$TORCH_VERSION" == "nightly" ]]; then
142132
if [ "${CU_VERSION:-}" == cpu ] ; then
143-
pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
133+
uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
144134
else
145-
pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U
135+
uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U
146136
fi
147137
elif [[ "$TORCH_VERSION" == "stable" ]]; then
148138
if [ "${CU_VERSION:-}" == cpu ] ; then
149-
pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U
139+
uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U
150140
else
151-
pip3 install torch torchvision --index-url https://download.pytorch.org/whl/$CU_VERSION -U
141+
uv pip install torch torchvision --index-url https://download.pytorch.org/whl/$CU_VERSION -U
152142
fi
153143
else
154144
printf "Failed to install pytorch"
@@ -167,14 +157,13 @@ python -c "import functorch"
167157

168158
# install tensordict
169159
if [[ "$RELEASE" == 0 ]]; then
170-
pip3 install git+https://github.com/pytorch/tensordict.git
160+
uv pip install git+https://github.com/pytorch/tensordict.git
171161
else
172-
pip3 install tensordict
162+
uv pip install tensordict
173163
fi
174164

175165
printf "* Installing torchrl\n"
176-
python setup.py develop
177-
166+
uv pip install -e . --no-build-isolation
178167

179168
if [ "${CU_VERSION:-}" != cpu ] ; then
180169
printf "* Installing VC1\n"

‎.github/unittest/linux_distributed/scripts/install.sh‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ unset PYTORCH_VERSION
77

88
set -e
99

10-
eval "$(./conda/bin/conda shell.bash hook)"
11-
conda activate ./env
10+
root_dir="$(git rev-parse --show-toplevel)"
11+
source "${root_dir}/.venv/bin/activate"
1212

1313
if [ "${CU_VERSION:-}" == cpu ] ; then
1414
version="cpu"
@@ -28,15 +28,15 @@ git submodule sync && git submodule update --init --recursive
2828

2929
if [[ "$TORCH_VERSION" == "nightly" ]]; then
3030
if [ "${CU_VERSION:-}" == cpu ] ; then
31-
pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
31+
uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
3232
else
33-
pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U
33+
uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U
3434
fi
3535
elif [[ "$TORCH_VERSION" == "stable" ]]; then
3636
if [ "${CU_VERSION:-}" == cpu ] ; then
37-
pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U
37+
uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U
3838
else
39-
pip3 install torch torchvision --index-url https://download.pytorch.org/whl/$CU_VERSION -U
39+
uv pip install torch torchvision --index-url https://download.pytorch.org/whl/$CU_VERSION -U
4040
fi
4141
else
4242
printf "Failed to install pytorch"
@@ -47,14 +47,14 @@ fi
4747
python -c "import functorch"
4848

4949
## install snapshot
50-
#pip install git+https://github.com/pytorch/torchsnapshot
50+
#uv pip install git+https://github.com/pytorch/torchsnapshot
5151

5252
# install tensordict
5353
if [[ "$RELEASE" == 0 ]]; then
54-
pip3 install git+https://github.com/pytorch/tensordict.git
54+
uv pip install git+https://github.com/pytorch/tensordict.git
5555
else
56-
pip3 install tensordict
56+
uv pip install tensordict
5757
fi
5858

5959
printf "* Installing torchrl\n"
60-
python setup.py develop
60+
uv pip install -e . --no-build-isolation

‎.github/unittest/linux_distributed/scripts/post_process.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22

33
set -e
44

5-
eval "$(./conda/bin/conda shell.bash hook)"
6-
conda activate ./env
5+
root_dir="$(git rev-parse --show-toplevel)"
6+
source "${root_dir}/.venv/bin/activate"

‎.github/unittest/linux_distributed/scripts/run_test.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
set -e
44

5-
eval "$(./conda/bin/conda shell.bash hook)"
6-
conda activate ./env
5+
root_dir="$(git rev-parse --show-toplevel)"
6+
source "${root_dir}/.venv/bin/activate"
77

88
export PYTORCH_TEST_WITH_SLOW='1'
99
python -m torch.utils.collect_env

‎.github/unittest/linux_distributed/scripts/setup_env.sh‎

Lines changed: 27 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -9,34 +9,28 @@ set -e
99

1010
this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
1111
# Avoid error: "fatal: unsafe repository"
12+
apt-get update && apt-get install -y git curl wget gcc g++
1213
git config --global --add safe.directory '*'
1314
root_dir="$(git rev-parse --show-toplevel)"
14-
conda_dir="${root_dir}/conda"
15-
env_dir="${root_dir}/env"
16-
lib_dir="${env_dir}/lib"
15+
env_dir="${root_dir}/.venv"
1716

1817
cd "${root_dir}"
1918

20-
case "$(uname -s)" in
21-
Darwin*) os=MacOSX;;
22-
*) os=Linux
23-
esac
2419

25-
# 1. Install conda at ./conda
26-
if [ ! -d "${conda_dir}" ]; then
27-
printf "* Installing conda\n"
28-
wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh"
29-
bash ./miniconda.sh -b -f -p "${conda_dir}"
30-
fi
31-
eval "$(${conda_dir}/bin/conda shell.bash hook)"
20+
# 1. Install uv
21+
printf "* Installing uv\n"
22+
curl -LsSf https://astral.sh/uv/install.sh | sh
23+
export PATH="$HOME/.local/bin:$PATH"
24+
25+
26+
# 2. Create test environment at ./.venv
27+
printf "python: ${PYTHON_VERSION}
28+
"
29+
printf "* Creating a test environment with uv
30+
"
31+
uv venv "${env_dir}" --python="${PYTHON_VERSION}"
32+
source "${env_dir}/bin/activate"
3233

33-
# 2. Create test environment at ./env
34-
printf "python: ${PYTHON_VERSION}\n"
35-
if [ ! -d "${env_dir}" ]; then
36-
printf "* Creating a test environment\n"
37-
conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION"
38-
fi
39-
conda activate "${env_dir}"
4034

4135
## 3. Install mujoco
4236
#printf "* Installing mujoco and related\n"
@@ -63,15 +57,14 @@ else
6357
fi
6458

6559
export MUJOCO_GL=$PRIVATE_MUJOCO_GL
66-
conda env config vars set MUJOCO_PY_MUJOCO_PATH=$root_dir/.mujoco/mujoco210 \
67-
MAX_IDLE_COUNT=1000 \
68-
DISPLAY=:99 \
69-
MJLIB_PATH=$root_dir/.mujoco/mujoco-2.1.1/lib/libmujoco.so.2.1.1 \
70-
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$root_dir/.mujoco/mujoco210/bin \
71-
SDL_VIDEODRIVER=dummy \
72-
MUJOCO_GL=$PRIVATE_MUJOCO_GL \
73-
PYOPENGL_PLATFORM=$PRIVATE_MUJOCO_GL \
74-
TOKENIZERS_PARALLELISM=true
60+
export MUJOCO_PY_MUJOCO_PATH=$root_dir/.mujoco/mujoco210
61+
export MAX_IDLE_COUNT=1000
62+
export DISPLAY=:99
63+
export MJLIB_PATH=$root_dir/.mujoco/mujoco-2.1.1/lib/libmujoco.so.2.1.1
64+
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$root_dir/.mujoco/mujoco210/bin
65+
export SDL_VIDEODRIVER=dummy
66+
export PYOPENGL_PLATFORM=$PRIVATE_MUJOCO_GL
67+
export TOKENIZERS_PARALLELISM=true
7568

7669
# Software rendering requires GLX and OSMesa.
7770
if [ $PRIVATE_MUJOCO_GL == 'egl' ] || [ $PRIVATE_MUJOCO_GL == 'osmesa' ] ; then
@@ -85,19 +78,17 @@ if [ $PRIVATE_MUJOCO_GL == 'egl' ] || [ $PRIVATE_MUJOCO_GL == 'osmesa' ] ; then
8578
yum -y install freeglut
8679
fi
8780

88-
pip install pip --upgrade
81+
uv pip install pip --upgrade
8982

90-
conda env update --file "${this_dir}/environment.yml" --prune
83+
# Dependencies installed via uv pip (see converted script)
9184

92-
conda deactivate
93-
conda activate "${env_dir}"
9485

9586
if [[ $OSTYPE != 'darwin'* ]]; then
9687
# install ale-py: manylinux names are broken for CentOS so we need to manually download and
9788
# rename them
9889
PY_VERSION=$(python --version)
9990
echo "installing gymnasium"
100-
pip install "gymnasium[atari]>=1.1"
91+
uv pip install "gymnasium[atari]>=1.1"
10192
else
102-
pip install "gymnasium[atari]>=1.1"
93+
uv pip install "gymnasium[atari]>=1.1"
10394
fi

‎.github/unittest/linux_libs/scripts_ataridqn/install.sh‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ unset PYTORCH_VERSION
77

88
set -e
99

10-
eval "$(./conda/bin/conda shell.bash hook)"
11-
conda activate ./env
10+
root_dir="$(git rev-parse --show-toplevel)"
11+
source "${root_dir}/.venv/bin/activate"
1212

1313
if [ "${CU_VERSION:-}" == cpu ] ; then
1414
version="cpu"
@@ -29,15 +29,15 @@ git submodule sync && git submodule update --init --recursive
2929
printf "Installing PyTorch with cu128"
3030
if [[ "$TORCH_VERSION" == "nightly" ]]; then
3131
if [ "${CU_VERSION:-}" == cpu ] ; then
32-
pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
32+
uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
3333
else
34-
pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U
34+
uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U
3535
fi
3636
elif [[ "$TORCH_VERSION" == "stable" ]]; then
3737
if [ "${CU_VERSION:-}" == cpu ] ; then
38-
pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu
38+
uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
3939
else
40-
pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128
40+
uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128
4141
fi
4242
else
4343
printf "Failed to install pytorch"
@@ -46,16 +46,16 @@ fi
4646

4747
# install tensordict
4848
if [[ "$RELEASE" == 0 ]]; then
49-
pip3 install git+https://github.com/pytorch/tensordict.git
49+
uv pip install git+https://github.com/pytorch/tensordict.git
5050
else
51-
pip3 install tensordict
51+
uv pip install tensordict
5252
fi
5353

5454
# smoke test
5555
python -c "import functorch;import tensordict"
5656

5757
printf "* Installing torchrl\n"
58-
python setup.py develop
58+
uv pip install -e . --no-build-isolation
5959

6060
# smoke test
6161
python -c "import torchrl"

‎.github/unittest/linux_libs/scripts_ataridqn/post_process.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22

33
set -e
44

5-
eval "$(./conda/bin/conda shell.bash hook)"
6-
conda activate ./env
5+
root_dir="$(git rev-parse --show-toplevel)"
6+
source "${root_dir}/.venv/bin/activate"

‎.github/unittest/linux_libs/scripts_ataridqn/run_test.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
set -e
44

5-
eval "$(./conda/bin/conda shell.bash hook)"
6-
conda activate ./env
5+
root_dir="$(git rev-parse --show-toplevel)"
6+
source "${root_dir}/.venv/bin/activate"
77

88
apt-get update && apt-get remove swig -y && apt-get install -y git gcc patchelf libosmesa6-dev libgl1-mesa-glx libglfw3 swig3.0 cmake
99
ln -s /usr/bin/swig3.0 /usr/bin/swig
@@ -18,7 +18,7 @@ root_dir="$(git rev-parse --show-toplevel)"
1818
env_dir="${root_dir}/env"
1919
lib_dir="${env_dir}/lib"
2020

21-
conda deactivate && conda activate ./env
21+
# conda deactivate (not needed with uv) && source ./.venv/bin/activate
2222

2323
python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 200 --capture no -k TestAtariDQN --error-for-skips --runslow
2424
coverage combine

0 commit comments

Comments
 (0)