From 71dff6d8fde8257f993f40f560eaccaf53835adc Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 6 Mar 2021 17:37:43 -0800 Subject: [PATCH 1/7] build ray 2.2.0 dev --- .staroid/build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.staroid/build.sh b/.staroid/build.sh index c459f74..2e13e66 100755 --- a/.staroid/build.sh +++ b/.staroid/build.sh @@ -8,7 +8,7 @@ set -e pwd RAY_REPO=https://github.com/ray-project/ray.git -RAY_CHECKOUT=260b07cf0cf2c10c091711cc3d598663133c2dc5 +RAY_CHECKOUT=5fc761c562006432ed6fd43066ba3d3c7b9c6ac5 PYTHON_VERSION=$1 SHORT_VER=`echo $PYTHON_VERSION | sed "s/\([0-9]*\)[.]\([0-9]*\)[.][0-9]*/\1\2/g"` @@ -61,12 +61,12 @@ if [ "$BUILD_WHEEL" == "true" ]; then WHEEL=`ls .whl/*-cp$SHORT_VER-*` else - if [ "$SHORT_VER" == "36" ]; then - WHEEL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl" + if [ "$SHORT_VER" == "36" ]; then + WHEEL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl" elif [ "$SHORT_VER" == "37" ]; then - WHEEL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl" + WHEEL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl" elif [ "$SHORT_VER" == "38" ]; then - WHEEL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux1_x86_64.whl" + WHEEL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl" fi fi From 1e23809c003b953ba6371df7eccdfaa9c3e8e8db Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 6 Mar 2021 17:43:27 -0800 Subject: [PATCH 2/7] copy all requirements.txt --- .staroid/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.staroid/build.sh b/.staroid/build.sh index 2e13e66..620d762 100755 --- a/.staroid/build.sh +++ b/.staroid/build.sh @@ -86,7 +86,7 @@ cat docker/ray/Dockerfile cat docker/ray-ml/Dockerfile # cp requirements to ray-ml dir -cp python/requirements* docker/ray-ml +cp python/requirements.txt python/requirements_* python/requirements/* docker/ray-ml # build docker image ./build-docker.sh --no-cache-build --gpu --python-version $PYTHON_VERSION From e848252c81b84e5e3c26ee9093a9237609f9bcfa Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 6 Mar 2021 17:45:55 -0800 Subject: [PATCH 3/7] update path requirements_tun.txt --- .staroid/ray_patch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.staroid/ray_patch.sh b/.staroid/ray_patch.sh index 100ab34..2eed0ce 100755 --- a/.staroid/ray_patch.sh +++ b/.staroid/ray_patch.sh @@ -61,7 +61,7 @@ if [ "$OP" == "patch" ]; then $SED_INPLACE "/tensorflow/d" ${RAY_HOME}/python/requirements.txt $SED_INPLACE "/tensorflow/d" ${RAY_HOME}/python/requirements_ml_docker.txt $SED_INPLACE "/tensorflow/d" ${RAY_HOME}/python/requirements_rllib.txt - $SED_INPLACE "/tensorflow/d" ${RAY_HOME}/python/requirements_tune.txt + $SED_INPLACE "/tensorflow/d" ${RAY_HOME}/python/requirements/requirements_tune.txt elif [ "$OP" == "reset" ]; then git checkout ${RAY_HOME}/docker/ray/Dockerfile @@ -73,7 +73,7 @@ elif [ "$OP" == "reset" ]; then git checkout ${RAY_HOME}/python/requirements.txt git checkout ${RAY_HOME}/python/requirements_ml_docker.txt git checkout ${RAY_HOME}/python/requirements_rllib.txt - git checkout ${RAY_HOME}/python/requirements_tune.txt + git checkout ${RAY_HOME}/python/requirements/requirements_tune.txt else echo "Invalid operation $OP" exit 1 From f54fc2ec4988b34a34f7254732bf04382df9d24b Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 7 Mar 2021 15:07:05 -0800 Subject: [PATCH 4/7] run test on branch ray-2.0.0 --- .github/workflows/build-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 199db30..56fe0b0 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -5,7 +5,7 @@ on: tags-ignore: - '*.*' branches: - - master + - ray-2.0.0 pull_request: jobs: From 0fb7bf294e787c012bcbaab08e28a36fc8c6887a Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 7 Mar 2021 15:07:28 -0800 Subject: [PATCH 5/7] try skip install tensorflow-gpu on CI when python version is 3.6.x to avoid no space left error --- .staroid/build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.staroid/build.sh b/.staroid/build.sh index 620d762..88ecd85 100755 --- a/.staroid/build.sh +++ b/.staroid/build.sh @@ -75,8 +75,15 @@ fi ../.staroid/ray_patch.sh patch . $WHEEL # apply additional docker file commands +if [ "$SHORT_VER" == "36" ] && [ "$GITHUB_ACTIONS" == "true" ]; then + # when github action runs python36 build, it stops with no space left error during installing conda install tensorflow-gpu + # so in this case, we skip install tensorflow-gpu package install + $SED_INPLACE "/tensorflow/d" ../.staroid/Dockerfile_staroid +fi + cat ../.staroid/Dockerfile_staroid >> docker/ray-ml/Dockerfile + # print patched files git diff From be435fa92bc98f3b37d4ee72e06f3b6a990c3aa2 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 7 Mar 2021 15:09:48 -0800 Subject: [PATCH 6/7] define SED_INPLACE --- .staroid/build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.staroid/build.sh b/.staroid/build.sh index 88ecd85..ba3c2db 100755 --- a/.staroid/build.sh +++ b/.staroid/build.sh @@ -14,6 +14,13 @@ SHORT_VER=`echo $PYTHON_VERSION | sed "s/\([0-9]*\)[.]\([0-9]*\)[.][0-9]*/\1\2/g echo "PYTHON_VERSION=$PYTHON_VERSION, SHORT_VER=$SHORT_VER" +# sed +SED_INPLACE="sed -i" +uname | grep Darwin > /dev/null +if [ $? -eq 0 ]; then + SED_INPLACE="sed -i .bak" +fi + # Checkout ray source git clone $RAY_REPO ray-py$SHORT_VER cd ray-py$SHORT_VER From 081fbe95e370187778da48ebfb001d185e87224f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 7 Mar 2021 15:13:10 -0800 Subject: [PATCH 7/7] define SED_INPLACE in the beginning of the file --- .github/workflows/build-docker-images.yml | 2 +- .staroid/build.sh | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 56fe0b0..ae6dfeb 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -5,7 +5,7 @@ on: tags-ignore: - '*.*' branches: - - ray-2.0.0 + - main pull_request: jobs: diff --git a/.staroid/build.sh b/.staroid/build.sh index ba3c2db..51178f8 100755 --- a/.staroid/build.sh +++ b/.staroid/build.sh @@ -3,6 +3,13 @@ # https://skaffold.dev/docs/pipeline-stages/builders/custom/ # +# sed +SED_INPLACE="sed -i" +uname | grep Darwin > /dev/null +if [ $? -eq 0 ]; then + SED_INPLACE="sed -i .bak" +fi + set -x set -e pwd @@ -14,13 +21,6 @@ SHORT_VER=`echo $PYTHON_VERSION | sed "s/\([0-9]*\)[.]\([0-9]*\)[.][0-9]*/\1\2/g echo "PYTHON_VERSION=$PYTHON_VERSION, SHORT_VER=$SHORT_VER" -# sed -SED_INPLACE="sed -i" -uname | grep Darwin > /dev/null -if [ $? -eq 0 ]; then - SED_INPLACE="sed -i .bak" -fi - # Checkout ray source git clone $RAY_REPO ray-py$SHORT_VER cd ray-py$SHORT_VER