From d75ec1c778c54daf768157b4a98f1ad0e7612eb7 Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Mon, 10 Feb 2025 02:48:16 +0800 Subject: [PATCH 01/10] fix test --- .github/actions/dependence/action.yml | 3 + .../service/freeThreadPython/action.yml | 31 +++++++ .github/workflows/standard-test-suite.yml | 90 +------------------ 3 files changed, 35 insertions(+), 89 deletions(-) diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 57a55cad42..3299c16ab4 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -116,6 +116,9 @@ runs: - if: inputs.free-thread-python-required=='true' name: Install free-thread python uses: ./.github/actions/service/freeThreadPython + with: + iginx-conda-flag: "true" + iginx-conda-env: "iginxEnv" - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v4 diff --git a/.github/actions/service/freeThreadPython/action.yml b/.github/actions/service/freeThreadPython/action.yml index 6e473ce15f..d01de4b7ac 100644 --- a/.github/actions/service/freeThreadPython/action.yml +++ b/.github/actions/service/freeThreadPython/action.yml @@ -1,9 +1,40 @@ name: "Install_Free-thread_Python" description: "Install Free-thread Python" +inputs: + iginx-conda-flag: + description: "whether to use conda" + required: false + default: "false" + iginx-conda-env: + description: "conda env name" + required: false runs: using: "composite" steps: + # this step creates + - if: inputs.iginx-conda-flag == 'true' + uses: conda-incubator/setup-miniconda@v3 + with: + channels: conda-forge + channel-priority: flexible + conda-remove-defaults: "true" + run-post: "false" + + - if: inputs.iginx-conda-flag == 'true' + name: Env List & install python3.13 + shell: bash -el {0} + run: | + conda config --show channels + conda list + conda env list + conda create -n ${{ inputs.iginx-conda-env }} python=3.13 python-freethreading -c conda-forge + conda activate ${{ inputs.iginx-conda-env }} + python -VV + python -c "import sys;print(sys._is_gil_enabled())" + conda env list + + - name: Set Python variables id: set-python-vars shell: bash diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 89e2c8f639..13ad9ab056 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -11,95 +11,7 @@ concurrency: cancel-in-progress: true jobs: - unit-test: - uses: ./.github/workflows/unit-test.yml - unit-mds: - uses: ./.github/workflows/unit-mds.yml - case-regression: - uses: ./.github/workflows/case-regression.yml - with: - metadata-matrix: '["zookeeper"]' - standalone-test: - uses: ./.github/workflows/standalone-test.yml - with: - metadata-matrix: '["zookeeper"]' - standalone-test-no-optimizer: - uses: ./.github/workflows/standalone-test-no-optimizer.yml - with: - metadata-matrix: '["zookeeper"]' - db-ce: - uses: ./.github/workflows/DB-CE.yml - with: - metadata-matrix: '["zookeeper"]' - db-ce-no-optimizer: - uses: ./.github/workflows/DB-CE.yml - with: - metadata-matrix: '["zookeeper"]' - close-optimizer: "true" - standalone-test-vectordb: - uses: ./.github/workflows/standalone-test.yml - with: - metadata-matrix: '["zookeeper"]' - os-matrix: '["ubuntu-latest", "macos-latest", "windows-latest"]' - db-matrix: '["VectorDB"]' - timeout-minutes: 300 - standalone-test-no-optimizer-vectordb: - uses: ./.github/workflows/standalone-test-no-optimizer.yml - with: - metadata-matrix: '["zookeeper"]' - os-matrix: '["ubuntu-latest", "macos-latest", "windows-latest"]' - db-matrix: '["VectorDB"]' - timeout-minutes: 300 - db-ce-vectordb: - uses: ./.github/workflows/DB-CE.yml - with: - metadata-matrix: '["zookeeper"]' - os-matrix: '["ubuntu-latest", "macos-latest", "windows-latest"]' - db-matrix: '["VectorDB"]' - functest: "NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT" - timeout-minutes: 360 - db-ce-no-optimizer-vectordb: - uses: ./.github/workflows/DB-CE.yml - with: - metadata-matrix: '["zookeeper"]' - os-matrix: '["ubuntu-latest", "macos-latest", "windows-latest"]' - db-matrix: '["VectorDB"]' - functest: "NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT" - timeout-minutes: 360 - close-optimizer: "true" - remote-test: - uses: ./.github/workflows/remote-test.yml - with: - metadata-matrix: '["zookeeper"]' - assembly-test: - uses: ./.github/workflows/assembly-test.yml free-thread-test: uses: ./.github/workflows/free-thread-test.yml with: - metadata-matrix: '["zookeeper"]' - tpc-h-regression-test: - uses: ./.github/workflows/tpc-h.yml - with: - os-matrix: '["ubuntu-latest"]' - metadata-matrix: '["zookeeper"]' - tpc-h-regression-test-no-optimizer: - uses: ./.github/workflows/tpc-h.yml - with: - os-matrix: '["ubuntu-latest"]' - metadata-matrix: '["zookeeper"]' - close-optimizer: "true" - tpc-h-regression-test-vectordb: - uses: ./.github/workflows/tpc-h.yml - with: - os-matrix: '["ubuntu-latest"]' - metadata-matrix: '["zookeeper"]' - db-matrix: '["VectorDB"]' - timeout-minutes: 360 - tpc-h-regression-test-no-optimizer-vectordb: - uses: ./.github/workflows/tpc-h.yml - with: - os-matrix: '["ubuntu-latest"]' - metadata-matrix: '["zookeeper"]' - db-matrix: '["VectorDB"]' - close-optimizer: "true" - timeout-minutes: 360 + metadata-matrix: '["zookeeper"]' \ No newline at end of file From fd8306cd97242482a8a01d69d198d186d86b60f4 Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Mon, 10 Feb 2025 04:04:25 +0800 Subject: [PATCH 02/10] use in test --- .github/actions/dependence/action.yml | 3 +- .github/actions/iginxRunner/action.yml | 41 ++++++++++++++----- .../service/freeThreadConda/action.yml | 35 ++++++++++++++++ .../service/freeThreadPython/action.yml | 23 ----------- .github/scripts/iginx/iginx.sh | 7 +++- .github/scripts/iginx/iginx_macos.sh | 7 +++- .github/scripts/iginx/iginx_windows.sh | 5 ++- .github/scripts/test/test_union.sh | 2 +- .github/workflows/free-thread-test.yml | 11 ++++- 9 files changed, 92 insertions(+), 42 deletions(-) create mode 100644 .github/actions/service/freeThreadConda/action.yml diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 3299c16ab4..b2a0c6145c 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -115,9 +115,8 @@ runs: - if: inputs.free-thread-python-required=='true' name: Install free-thread python - uses: ./.github/actions/service/freeThreadPython + uses: ./.github/actions/service/freeThreadConda with: - iginx-conda-flag: "true" iginx-conda-env: "iginxEnv" - name: Set up JDK ${{ inputs.java }} diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml index c0b60767ca..d3251b0e3a 100644 --- a/.github/actions/iginxRunner/action.yml +++ b/.github/actions/iginxRunner/action.yml @@ -9,6 +9,14 @@ inputs: description: "to test UDF path detection" required: false default: "false" + activate-conda-flag: + description: "whether to manually activate conda env before start IGinX" + required: false + default: "false" + iginx-conda-env: + description: "required conda env name" + required: false + default: "false" runs: using: "composite" # Mandatory parameter @@ -36,17 +44,28 @@ runs: name: Start IGinX shell: bash -el {0} run: | - which python - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 - fi + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" + if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" + else + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 + fi + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" + if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" + else + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 + fi + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" + if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" + else + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 + fi + fi - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' name: Stop IGinX diff --git a/.github/actions/service/freeThreadConda/action.yml b/.github/actions/service/freeThreadConda/action.yml new file mode 100644 index 0000000000..a6c89c6b71 --- /dev/null +++ b/.github/actions/service/freeThreadConda/action.yml @@ -0,0 +1,35 @@ +name: "Install_Free-thread_Python in Conda" +description: "Install Free-thread Python in Conda" +inputs: + iginx-conda-env: + description: "conda env name" + required: false + +runs: + using: "composite" + steps: + # this step creates env test(default python version) which will not be used + - uses: conda-incubator/setup-miniconda@v3 + with: + channels: conda-forge + channel-priority: flexible + conda-remove-defaults: "true" + run-post: "false" + + # currently we can only install free-thread python 3.13 by conda-forge. latest miniconda only supports 3.12 + - name: Check channels & install python3.13 + shell: bash -el {0} + run: | + conda config --show channels + conda create -n ${{ inputs.iginx-conda-env }} python=3.13 python-freethreading -c conda-forge + conda activate ${{ inputs.iginx-conda-env }} + python -VV + python -c "import sys;print(sys._is_gil_enabled())" + curl -L -O https://github.com/IGinX-THU/IGinX-resources/raw/refs/heads/main/resources/python/pandas-3.0.0.dev0+1654.g32a97a969a-cp313-cp313t-win_amd64.whl + ls -l + if [ "$RUNNER_OS" == "Windows" ]; then + python -m pip install numpy thrift pemjax + python -m pip install pandas*.whl + else + python -m pip install pandas==2.2.3 numpy thrift pemjax + fi \ No newline at end of file diff --git a/.github/actions/service/freeThreadPython/action.yml b/.github/actions/service/freeThreadPython/action.yml index d01de4b7ac..67cc16f56c 100644 --- a/.github/actions/service/freeThreadPython/action.yml +++ b/.github/actions/service/freeThreadPython/action.yml @@ -12,29 +12,6 @@ inputs: runs: using: "composite" steps: - # this step creates - - if: inputs.iginx-conda-flag == 'true' - uses: conda-incubator/setup-miniconda@v3 - with: - channels: conda-forge - channel-priority: flexible - conda-remove-defaults: "true" - run-post: "false" - - - if: inputs.iginx-conda-flag == 'true' - name: Env List & install python3.13 - shell: bash -el {0} - run: | - conda config --show channels - conda list - conda env list - conda create -n ${{ inputs.iginx-conda-env }} python=3.13 python-freethreading -c conda-forge - conda activate ${{ inputs.iginx-conda-env }} - python -VV - python -c "import sys;print(sys._is_gil_enabled())" - conda env list - - - name: Set Python variables id: set-python-vars shell: bash diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index ee9cf74cb8..6df54ebfff 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -28,7 +28,12 @@ sed -i "s/restPort=[0-9]\+/restPort=$2/g" core/target/iginx-core-*/conf/config.p sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" -sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" +if [ -n "$3" ]; then + # 使用 bash, 第一个脚本参数指定了需要激活的环境名 + bash -c "conda activate $3 && nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" +else + sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" +fi sh -c "sleep 3" diff --git a/.github/scripts/iginx/iginx_macos.sh b/.github/scripts/iginx/iginx_macos.sh index 0362088572..7d30f15dde 100644 --- a/.github/scripts/iginx/iginx_macos.sh +++ b/.github/scripts/iginx/iginx_macos.sh @@ -32,7 +32,12 @@ echo "JAVA_HOME is set to $JAVA_HOME" sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" -sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" +if [ -n "$3" ]; then + # 使用 bash, 第一个脚本参数指定了需要激活的环境名 + bash -c "conda activate $3 && nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" +else + sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" +fi sh -c "sleep 3" diff --git a/.github/scripts/iginx/iginx_windows.sh b/.github/scripts/iginx/iginx_windows.sh index 3162fc1027..db7d57e147 100644 --- a/.github/scripts/iginx/iginx_windows.sh +++ b/.github/scripts/iginx/iginx_windows.sh @@ -32,8 +32,9 @@ sed -i 's/-Xmx%MAX_HEAP_SIZE% -Xms%MAX_HEAP_SIZE%/-Xmx4g -Xms4g -XX:MaxMetaspace echo "starting iginx on windows..." -if [[ "$IGINX_CONDA_FLAG" == "true" ]]; then - powershell -Command "conda activate $IGINX_CONDA_ENV;Start-Process -FilePath '$batPath' -NoNewWindow -RedirectStandardOutput 'iginx-$1.log' -RedirectStandardError 'iginx-$1-error.log'" +if [ -n "$3" ]; then + # 第三个参数如果存在，就是需要激活的conda环境名 + powershell -Command "conda activate $3;Start-Process -FilePath '$batPath' -NoNewWindow -RedirectStandardOutput 'iginx-$1.log' -RedirectStandardError 'iginx-$1-error.log'" else powershell -Command "Start-Process -FilePath '$batPath' -NoNewWindow -RedirectStandardOutput 'iginx-$1.log' -RedirectStandardError 'iginx-$1-error.log'" fi diff --git a/.github/scripts/test/test_union.sh b/.github/scripts/test/test_union.sh index c320ee66ea..a97639fbc7 100644 --- a/.github/scripts/test/test_union.sh +++ b/.github/scripts/test/test_union.sh @@ -17,7 +17,7 @@ # along with this program. If not, see . # - +python -VV pwd cd .. diff --git a/.github/workflows/free-thread-test.yml b/.github/workflows/free-thread-test.yml index 48f91b12c6..afc15124a2 100644 --- a/.github/workflows/free-thread-test.yml +++ b/.github/workflows/free-thread-test.yml @@ -35,6 +35,9 @@ jobs: metadata: ${{ fromJSON(inputs.metadata-matrix) }} DB-name: ${{ fromJSON(inputs.db-matrix) }} runs-on: ${{ matrix.os }} + env: + # only used in this workflow file + IGINX_CONDA_ENV: "iginxEnv" steps: - uses: actions/checkout@v4 # use free-threading python instead of stable version @@ -44,6 +47,8 @@ jobs: java: ${{ matrix.java }} free-thread-python-required: "true" scope: "only-java" + iginx-conda-flag: "true" + iginx-conda-env: ${{ env.IGINX_CONDA_ENV }} - name: Run Metadata uses: ./.github/actions/metadataRunner @@ -70,13 +75,17 @@ jobs: - name: Start IGinX uses: ./.github/actions/iginxRunner + with: + activate-conda-flag: "true" + iginx-conda-env: ${{ env.IGINX_CONDA_ENV }} - name: TestController IT if: always() - shell: bash + shell: bash -el {0} env: METADATA_STORAGE: ${{ matrix.metadata }} run: | + conda activate ${{ env.IGINX_CONDA_ENV }} chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format From 28e1a21ad9a374c4bbefbbdab203688d63c3cb26 Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Mon, 10 Feb 2025 04:20:10 +0800 Subject: [PATCH 03/10] fix test --- .github/actions/dependence/action.yml | 2 +- .github/actions/iginxRunner/action.yml | 47 ++++++++------- .../service/freeThreadConda/action.yml | 58 +++++++++---------- .../service/freeThreadPython/action.yml | 14 ++--- .github/scripts/iginx/iginx.sh | 3 +- .github/scripts/iginx/iginx_macos.sh | 3 +- .github/scripts/iginx/iginx_windows.sh | 1 + .github/workflows/free-thread-test.yml | 8 +-- .github/workflows/standard-test-suite.yml | 2 +- 9 files changed, 72 insertions(+), 66 deletions(-) diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index b2a0c6145c..10abe6b846 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -117,7 +117,7 @@ runs: name: Install free-thread python uses: ./.github/actions/service/freeThreadConda with: - iginx-conda-env: "iginxEnv" + iginx-conda-env: "iginxEnv" - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v4 diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml index d3251b0e3a..fea6e81c6b 100644 --- a/.github/actions/iginxRunner/action.yml +++ b/.github/actions/iginxRunner/action.yml @@ -44,28 +44,31 @@ runs: name: Start IGinX shell: bash -el {0} run: | - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" - else - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 - fi - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" - if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" - else - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 - fi - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" - if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" - else - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 - fi - fi + if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then + conda activate ${{ inputs.iginx-conda-env }} + fi + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" + if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" + else + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 + fi + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" + if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" + else + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 + fi + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" + if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" + else + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 + fi + fi - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' name: Stop IGinX diff --git a/.github/actions/service/freeThreadConda/action.yml b/.github/actions/service/freeThreadConda/action.yml index a6c89c6b71..a32355acc2 100644 --- a/.github/actions/service/freeThreadConda/action.yml +++ b/.github/actions/service/freeThreadConda/action.yml @@ -1,35 +1,35 @@ name: "Install_Free-thread_Python in Conda" description: "Install Free-thread Python in Conda" inputs: - iginx-conda-env: - description: "conda env name" - required: false + iginx-conda-env: + description: "conda env name" + required: false runs: - using: "composite" - steps: - # this step creates env test(default python version) which will not be used - - uses: conda-incubator/setup-miniconda@v3 - with: - channels: conda-forge - channel-priority: flexible - conda-remove-defaults: "true" - run-post: "false" + using: "composite" + steps: + # this step creates env test(default python version) which will not be used + - uses: conda-incubator/setup-miniconda@v3 + with: + channels: conda-forge + channel-priority: flexible + conda-remove-defaults: "true" + run-post: "false" - # currently we can only install free-thread python 3.13 by conda-forge. latest miniconda only supports 3.12 - - name: Check channels & install python3.13 - shell: bash -el {0} - run: | - conda config --show channels - conda create -n ${{ inputs.iginx-conda-env }} python=3.13 python-freethreading -c conda-forge - conda activate ${{ inputs.iginx-conda-env }} - python -VV - python -c "import sys;print(sys._is_gil_enabled())" - curl -L -O https://github.com/IGinX-THU/IGinX-resources/raw/refs/heads/main/resources/python/pandas-3.0.0.dev0+1654.g32a97a969a-cp313-cp313t-win_amd64.whl - ls -l - if [ "$RUNNER_OS" == "Windows" ]; then - python -m pip install numpy thrift pemjax - python -m pip install pandas*.whl - else - python -m pip install pandas==2.2.3 numpy thrift pemjax - fi \ No newline at end of file + # currently we can only install free-thread python 3.13 by conda-forge. latest miniconda only supports 3.12 + - name: Check channels & install python3.13 + shell: bash -el {0} + run: | + conda config --show channels + conda create -n ${{ inputs.iginx-conda-env }} python=3.13 python-freethreading -c conda-forge + conda activate ${{ inputs.iginx-conda-env }} + python -VV + python -c "import sys;print(sys._is_gil_enabled())" + curl -L -O https://github.com/IGinX-THU/IGinX-resources/raw/refs/heads/main/resources/python/pandas-3.0.0.dev0+1654.g32a97a969a-cp313-cp313t-win_amd64.whl + ls -l + if [ "$RUNNER_OS" == "Windows" ]; then + python -m pip install numpy thrift pemjax + python -m pip install pandas*.whl + else + python -m pip install pandas==2.2.3 numpy thrift pemjax + fi diff --git a/.github/actions/service/freeThreadPython/action.yml b/.github/actions/service/freeThreadPython/action.yml index 67cc16f56c..d25bee9bd5 100644 --- a/.github/actions/service/freeThreadPython/action.yml +++ b/.github/actions/service/freeThreadPython/action.yml @@ -1,13 +1,13 @@ name: "Install_Free-thread_Python" description: "Install Free-thread Python" inputs: - iginx-conda-flag: - description: "whether to use conda" - required: false - default: "false" - iginx-conda-env: - description: "conda env name" - required: false + iginx-conda-flag: + description: "whether to use conda" + required: false + default: "false" + iginx-conda-env: + description: "conda env name" + required: false runs: using: "composite" diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index 6df54ebfff..4e90dc6245 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -28,9 +28,10 @@ sed -i "s/restPort=[0-9]\+/restPort=$2/g" core/target/iginx-core-*/conf/config.p sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" +python -VV if [ -n "$3" ]; then # 使用 bash, 第一个脚本参数指定了需要激活的环境名 - bash -c "conda activate $3 && nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" + bash -c "source $CONDA/etc/profile.d/conda.sh && conda activate $3 && nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" else sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" fi diff --git a/.github/scripts/iginx/iginx_macos.sh b/.github/scripts/iginx/iginx_macos.sh index 7d30f15dde..6466847681 100644 --- a/.github/scripts/iginx/iginx_macos.sh +++ b/.github/scripts/iginx/iginx_macos.sh @@ -32,9 +32,10 @@ echo "JAVA_HOME is set to $JAVA_HOME" sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" +python -VV if [ -n "$3" ]; then # 使用 bash, 第一个脚本参数指定了需要激活的环境名 - bash -c "conda activate $3 && nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" + bash -c "source $CONDA/etc/profile.d/conda.sh && conda activate $3 && nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" else sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" fi diff --git a/.github/scripts/iginx/iginx_windows.sh b/.github/scripts/iginx/iginx_windows.sh index db7d57e147..f0ce4638d0 100644 --- a/.github/scripts/iginx/iginx_windows.sh +++ b/.github/scripts/iginx/iginx_windows.sh @@ -32,6 +32,7 @@ sed -i 's/-Xmx%MAX_HEAP_SIZE% -Xms%MAX_HEAP_SIZE%/-Xmx4g -Xms4g -XX:MaxMetaspace echo "starting iginx on windows..." +python -VV if [ -n "$3" ]; then # 第三个参数如果存在，就是需要激活的conda环境名 powershell -Command "conda activate $3;Start-Process -FilePath '$batPath' -NoNewWindow -RedirectStandardOutput 'iginx-$1.log' -RedirectStandardError 'iginx-$1-error.log'" diff --git a/.github/workflows/free-thread-test.yml b/.github/workflows/free-thread-test.yml index afc15124a2..56776cafa4 100644 --- a/.github/workflows/free-thread-test.yml +++ b/.github/workflows/free-thread-test.yml @@ -36,8 +36,8 @@ jobs: DB-name: ${{ fromJSON(inputs.db-matrix) }} runs-on: ${{ matrix.os }} env: - # only used in this workflow file - IGINX_CONDA_ENV: "iginxEnv" + # only used in this workflow file + IGINX_CONDA_ENV: "iginxEnv" steps: - uses: actions/checkout@v4 # use free-threading python instead of stable version @@ -76,8 +76,8 @@ jobs: - name: Start IGinX uses: ./.github/actions/iginxRunner with: - activate-conda-flag: "true" - iginx-conda-env: ${{ env.IGINX_CONDA_ENV }} + activate-conda-flag: "true" + iginx-conda-env: ${{ env.IGINX_CONDA_ENV }} - name: TestController IT if: always() diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 13ad9ab056..0cf5ed8d17 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -14,4 +14,4 @@ jobs: free-thread-test: uses: ./.github/workflows/free-thread-test.yml with: - metadata-matrix: '["zookeeper"]' \ No newline at end of file + metadata-matrix: '["zookeeper"]' From 96a474997573791c488e165517d8f7687904463a Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Mon, 10 Feb 2025 05:00:24 +0800 Subject: [PATCH 04/10] test transform --- .github/actions/confWriter/action.yml | 18 ++++-------------- .../actions/service/freeThreadConda/action.yml | 3 ++- .../integration/func/session/PySessionIT.java | 2 +- .../integration/func/udf/TransformIT.java | 9 ++++++++- .../iginx/integration/func/udf/UDFIT.java | 6 +++--- test/src/test/resources/testConfig.properties | 2 +- 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 2d5fbaec88..5451a7cf24 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -75,23 +75,13 @@ runs: id: find_python_cmd shell: bash -el {0} run: | - if [[ "${{ inputs.use-free-thread-python }}" == "true" ]]; then - if [ "$RUNNER_OS" == "Windows" ]; then - pythonPath=$(echo "$FT_PYTHON_PATH" | sed 's/\\/\//g') - pythonCMD="${pythonPath}/python3.13t.exe" - echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV - else - pythonCMD='python3.13t' - echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV - fi + if [ "$RUNNER_OS" == "Windows" ]; then + pythonCMD=python else - if [ "$RUNNER_OS" == "Windows" ]; then - pythonCMD=python - else - pythonCMD=python3 - fi + pythonCMD=python3 fi # have to set env to make it work in ut tests + ${pythonCMD} -VV echo "pythonCMD=${pythonCMD}" >> $GITHUB_ENV echo "pythonCMD=${pythonCMD}" >> $GITHUB_OUTPUT echo ${pythonCMD} diff --git a/.github/actions/service/freeThreadConda/action.yml b/.github/actions/service/freeThreadConda/action.yml index a32355acc2..ee85ebef9c 100644 --- a/.github/actions/service/freeThreadConda/action.yml +++ b/.github/actions/service/freeThreadConda/action.yml @@ -17,11 +17,12 @@ runs: run-post: "false" # currently we can only install free-thread python 3.13 by conda-forge. latest miniconda only supports 3.12 + # thus in action steps afterward, remember to 'conda activate $ENV_NAME' at start if conda should be used - name: Check channels & install python3.13 shell: bash -el {0} run: | conda config --show channels - conda create -n ${{ inputs.iginx-conda-env }} python=3.13 python-freethreading -c conda-forge + conda create -n ${{ inputs.iginx-conda-env }} python=3.13 python-freethreading -c conda-forge -q conda activate ${{ inputs.iginx-conda-env }} python -VV python -c "import sys;print(sys._is_gil_enabled())" diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java index adfbf60a7f..ffbdf6d117 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java @@ -143,7 +143,7 @@ public void insertBaseData() { } } - private boolean pythonNewerThan313() { + public static boolean pythonNewerThan313() { interpreter.exec("import sys; tooNew = sys.version_info >= (3, 13);"); return (boolean) interpreter.get("tooNew"); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java index 09b940a092..e09bdb7690 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java @@ -250,13 +250,20 @@ private void verifyJobState(long jobId, JobState expectedState) throws SessionEx private void verifyJobFinishedBlocked(long jobId) throws SessionException, InterruptedException { LOGGER.info("job is {}", jobId); JobState jobState = JobState.JOB_CREATED; + int timeout = 60000; // 60s while (!jobState.equals(JobState.JOB_CLOSED) && !jobState.equals(JobState.JOB_FAILED) - && !jobState.equals(JobState.JOB_FINISHED)) { + && !jobState.equals(JobState.JOB_FINISHED) + && timeout > 0) { Thread.sleep(500); + timeout -= 500; jobState = session.queryTransformJobStatus(jobId); } LOGGER.info("job {} state is {}", jobId, jobState.toString()); + if (timeout <= 0) { + LOGGER.error("job is stuck, check IGinX server log..."); + fail(); + } if (!needCompareResult) { return; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java index b47a90084a..e836a92ffa 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java @@ -21,6 +21,7 @@ import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; +import static cn.edu.tsinghua.iginx.integration.func.session.PySessionIT.pythonNewerThan313; import static org.junit.Assert.*; import cn.edu.tsinghua.iginx.exception.SessionException; @@ -1417,9 +1418,8 @@ public void testModuleInstallFail() { @Test public void tensorUDFTest() { - boolean torchSupported = System.getenv().getOrDefault("TORCH_SUPPORTED", "true").equals("true"); - Assume.assumeTrue( - "tensorUDFTest is skipped because pytorch is not supported(python>3.12).", torchSupported); + Assume.assumeFalse( + "tensorUDFTest is skipped because pytorch is not supported(python>3.12).", pythonNewerThan313()); String name = "tensorTest"; String filePath = String.join( diff --git a/test/src/test/resources/testConfig.properties b/test/src/test/resources/testConfig.properties index 672f44b3cd..d7560aff33 100644 --- a/test/src/test/resources/testConfig.properties +++ b/test/src/test/resources/testConfig.properties @@ -4,7 +4,7 @@ storageEngineList=IoTDB12,InfluxDB,FileSystem,Relational,MongoDB,Redis,VectorDB relationalStorageEngineList=PostgreSQL,MySQL # the test for every engine -test-list=UserPermissionIT,DataSourceIT,SQLSessionIT,OptimizerIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT,HostUtilsTest +test-list=TransformIT vectordb-test-list=UserPermissionIT,DataSourceIT,SQLSessionIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT,HostUtilsTest # the DB config From cd2e5bad3e24a10cc55d1b75616e1edf66a12bdb Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Mon, 10 Feb 2025 05:31:53 +0800 Subject: [PATCH 05/10] delete useless code && restore tests --- .github/actions/iginxRunner/action.yml | 18 +-- .../service/freeThreadPython/action.yml | 123 ------------------ .github/scripts/iginx/iginx.sh | 9 +- .github/scripts/iginx/iginx_macos.sh | 9 +- .github/scripts/iginx/iginx_windows.sh | 6 +- .github/workflows/standard-test-suite.yml | 88 +++++++++++++ .../iginx/integration/func/udf/UDFIT.java | 3 +- test/src/test/resources/testConfig.properties | 2 +- 8 files changed, 100 insertions(+), 158 deletions(-) delete mode 100644 .github/actions/service/freeThreadPython/action.yml diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml index fea6e81c6b..aa56d24afa 100644 --- a/.github/actions/iginxRunner/action.yml +++ b/.github/actions/iginxRunner/action.yml @@ -49,25 +49,13 @@ runs: fi if [ "$RUNNER_OS" == "Linux" ]; then chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" - else - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 - fi + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 elif [ "$RUNNER_OS" == "Windows" ]; then chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" - if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" - else - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 - fi + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 elif [ "$RUNNER_OS" == "macOS" ]; then chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" - if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 "${{ inputs.iginx-conda-env }}" - else - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 - fi + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 fi - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' diff --git a/.github/actions/service/freeThreadPython/action.yml b/.github/actions/service/freeThreadPython/action.yml deleted file mode 100644 index d25bee9bd5..0000000000 --- a/.github/actions/service/freeThreadPython/action.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: "Install_Free-thread_Python" -description: "Install Free-thread Python" -inputs: - iginx-conda-flag: - description: "whether to use conda" - required: false - default: "false" - iginx-conda-env: - description: "conda env name" - required: false - -runs: - using: "composite" - steps: - - name: Set Python variables - id: set-python-vars - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - echo "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.13.0/python-3.13.0-amd64.exe" >> $GITHUB_ENV - echo "PYTHON_INSTALLER_PATH=python-installer.exe" >> $GITHUB_ENV - echo "CACHE_KEY=python-windows-3.13.0" >> $GITHUB_ENV - elif [ "$RUNNER_OS" == "macOS" ]; then - echo "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.13.0/python-3.13.0-macos11.pkg" >> $GITHUB_ENV - echo "PYTHON_INSTALLER_PATH=python-installer.pkg" >> $GITHUB_ENV - echo "CACHE_KEY=python-macos-3.13.0" >> $GITHUB_ENV - fi - - - name: Restore Python installer - if: runner.os!='Linux' - id: restore-cache-python - uses: actions/cache/restore@v3 - with: - path: ${{ env.PYTHON_INSTALLER_PATH }} - key: ${{ env.CACHE_KEY }} - - - name: Download Python installer - if: steps.restore-cache-python.outputs.cache-hit != 'true' && runner.os=='Windows' - shell: powershell - run: | - Invoke-WebRequest -Uri $env:PYTHON_INSTALLER_URL -OutFile $env:PYTHON_INSTALLER_PATH - - # have to wait until installation finishes - - if: runner.os=='Windows' - name: Setup Python3.13 on Windows - shell: powershell - run: | - Start-Process -FilePath "$(Join-Path -Path $PWD -ChildPath $env:PYTHON_INSTALLER_PATH)" -ArgumentList "/quiet","Include_freethreaded=1" -Wait - $pythonPath = "$env:LocalAppData\Programs\Python\Python313" - Get-ChildItem $pythonPath - $env:Path += ";$pythonPath" - echo "$pythonPath" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "FT_PYTHON_PATH=$pythonPath" | Out-File -FilePath $env:GITHUB_ENV -Append - python3.13t -VV - - - if: runner.os=='Linux' - name: Setup Python3.13 on Linux - shell: bash - run: | - sudo add-apt-repository ppa:deadsnakes - sudo apt-get update - sudo apt-get install python3.13-nogil python3.13-dev - python3.13t -VV - - - name: Download Python installer - if: steps.restore-cache-python.outputs.cache-hit != 'true' && runner.os=='macOS' - shell: bash - run: | - curl -o $PYTHON_INSTALLER_PATH $PYTHON_INSTALLER_URL - - - if: runner.os=='macOS' - name: Setup Python3.13 on MacOS - shell: bash - run: | - cat > ./choicechanges.plist < - - - - - attributeSetting - 1 - choiceAttribute - selected - choiceIdentifier - org.python.Python.PythonTFramework-3.13 - - - - EOF - sudo installer -pkg ./$PYTHON_INSTALLER_PATH \ - -applyChoiceChangesXML ./choicechanges.plist \ - -target / - python3.13t -VV - export PATH="$PATH:/Library/Frameworks/PythonT.framework/Versions/3.13/bin" - - - name: Cache Python installer - if: runner.os!='Linux' && steps.restore-cache-python.outputs.cache-hit != 'true' - id: cache-python - uses: actions/cache/save@v3 - with: - path: ${{ env.PYTHON_INSTALLER_PATH }} - key: ${{ env.CACHE_KEY }} - - - name: Install pip - if: runner.os=='Linux' - shell: bash - run: | - curl -O https://bootstrap.pypa.io/get-pip.py - sudo python3.13t get-pip.py - - # pandas 2.2.3 cannot be directly installed on windows yet. use dev wheel file - - name: Install dependencies for tests - shell: bash - run: | - curl -L -O https://github.com/IGinX-THU/IGinX-resources/raw/refs/heads/main/resources/python/pandas-3.0.0.dev0+1654.g32a97a969a-cp313-cp313t-win_amd64.whl - ls -l - if [ "$RUNNER_OS" == "Windows" ]; then - python3.13t -m pip install numpy thrift pemjax - python3.13t -m pip install pandas*.whl - else - sudo python3.13t -m pip install pandas==2.2.3 numpy thrift pemjax - fi diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index 4e90dc6245..53269480f5 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -28,13 +28,8 @@ sed -i "s/restPort=[0-9]\+/restPort=$2/g" core/target/iginx-core-*/conf/config.p sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" -python -VV -if [ -n "$3" ]; then - # 使用 bash, 第一个脚本参数指定了需要激活的环境名 - bash -c "source $CONDA/etc/profile.d/conda.sh && conda activate $3 && nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" -else - sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" -fi +python3 -VV +bash -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" sh -c "sleep 3" diff --git a/.github/scripts/iginx/iginx_macos.sh b/.github/scripts/iginx/iginx_macos.sh index 6466847681..735ab6fbd7 100644 --- a/.github/scripts/iginx/iginx_macos.sh +++ b/.github/scripts/iginx/iginx_macos.sh @@ -32,13 +32,8 @@ echo "JAVA_HOME is set to $JAVA_HOME" sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" -python -VV -if [ -n "$3" ]; then - # 使用 bash, 第一个脚本参数指定了需要激活的环境名 - bash -c "source $CONDA/etc/profile.d/conda.sh && conda activate $3 && nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" -else - sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" -fi +python3 -VV +bash -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" sh -c "sleep 3" diff --git a/.github/scripts/iginx/iginx_windows.sh b/.github/scripts/iginx/iginx_windows.sh index f0ce4638d0..3162fc1027 100644 --- a/.github/scripts/iginx/iginx_windows.sh +++ b/.github/scripts/iginx/iginx_windows.sh @@ -32,10 +32,8 @@ sed -i 's/-Xmx%MAX_HEAP_SIZE% -Xms%MAX_HEAP_SIZE%/-Xmx4g -Xms4g -XX:MaxMetaspace echo "starting iginx on windows..." -python -VV -if [ -n "$3" ]; then - # 第三个参数如果存在，就是需要激活的conda环境名 - powershell -Command "conda activate $3;Start-Process -FilePath '$batPath' -NoNewWindow -RedirectStandardOutput 'iginx-$1.log' -RedirectStandardError 'iginx-$1-error.log'" +if [[ "$IGINX_CONDA_FLAG" == "true" ]]; then + powershell -Command "conda activate $IGINX_CONDA_ENV;Start-Process -FilePath '$batPath' -NoNewWindow -RedirectStandardOutput 'iginx-$1.log' -RedirectStandardError 'iginx-$1-error.log'" else powershell -Command "Start-Process -FilePath '$batPath' -NoNewWindow -RedirectStandardOutput 'iginx-$1.log' -RedirectStandardError 'iginx-$1-error.log'" fi diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 0cf5ed8d17..89e2c8f639 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -11,7 +11,95 @@ concurrency: cancel-in-progress: true jobs: + unit-test: + uses: ./.github/workflows/unit-test.yml + unit-mds: + uses: ./.github/workflows/unit-mds.yml + case-regression: + uses: ./.github/workflows/case-regression.yml + with: + metadata-matrix: '["zookeeper"]' + standalone-test: + uses: ./.github/workflows/standalone-test.yml + with: + metadata-matrix: '["zookeeper"]' + standalone-test-no-optimizer: + uses: ./.github/workflows/standalone-test-no-optimizer.yml + with: + metadata-matrix: '["zookeeper"]' + db-ce: + uses: ./.github/workflows/DB-CE.yml + with: + metadata-matrix: '["zookeeper"]' + db-ce-no-optimizer: + uses: ./.github/workflows/DB-CE.yml + with: + metadata-matrix: '["zookeeper"]' + close-optimizer: "true" + standalone-test-vectordb: + uses: ./.github/workflows/standalone-test.yml + with: + metadata-matrix: '["zookeeper"]' + os-matrix: '["ubuntu-latest", "macos-latest", "windows-latest"]' + db-matrix: '["VectorDB"]' + timeout-minutes: 300 + standalone-test-no-optimizer-vectordb: + uses: ./.github/workflows/standalone-test-no-optimizer.yml + with: + metadata-matrix: '["zookeeper"]' + os-matrix: '["ubuntu-latest", "macos-latest", "windows-latest"]' + db-matrix: '["VectorDB"]' + timeout-minutes: 300 + db-ce-vectordb: + uses: ./.github/workflows/DB-CE.yml + with: + metadata-matrix: '["zookeeper"]' + os-matrix: '["ubuntu-latest", "macos-latest", "windows-latest"]' + db-matrix: '["VectorDB"]' + functest: "NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT" + timeout-minutes: 360 + db-ce-no-optimizer-vectordb: + uses: ./.github/workflows/DB-CE.yml + with: + metadata-matrix: '["zookeeper"]' + os-matrix: '["ubuntu-latest", "macos-latest", "windows-latest"]' + db-matrix: '["VectorDB"]' + functest: "NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT" + timeout-minutes: 360 + close-optimizer: "true" + remote-test: + uses: ./.github/workflows/remote-test.yml + with: + metadata-matrix: '["zookeeper"]' + assembly-test: + uses: ./.github/workflows/assembly-test.yml free-thread-test: uses: ./.github/workflows/free-thread-test.yml with: metadata-matrix: '["zookeeper"]' + tpc-h-regression-test: + uses: ./.github/workflows/tpc-h.yml + with: + os-matrix: '["ubuntu-latest"]' + metadata-matrix: '["zookeeper"]' + tpc-h-regression-test-no-optimizer: + uses: ./.github/workflows/tpc-h.yml + with: + os-matrix: '["ubuntu-latest"]' + metadata-matrix: '["zookeeper"]' + close-optimizer: "true" + tpc-h-regression-test-vectordb: + uses: ./.github/workflows/tpc-h.yml + with: + os-matrix: '["ubuntu-latest"]' + metadata-matrix: '["zookeeper"]' + db-matrix: '["VectorDB"]' + timeout-minutes: 360 + tpc-h-regression-test-no-optimizer-vectordb: + uses: ./.github/workflows/tpc-h.yml + with: + os-matrix: '["ubuntu-latest"]' + metadata-matrix: '["zookeeper"]' + db-matrix: '["VectorDB"]' + close-optimizer: "true" + timeout-minutes: 360 diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java index e836a92ffa..45fee10fa8 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java @@ -1419,7 +1419,8 @@ public void testModuleInstallFail() { @Test public void tensorUDFTest() { Assume.assumeFalse( - "tensorUDFTest is skipped because pytorch is not supported(python>3.12).", pythonNewerThan313()); + "tensorUDFTest is skipped because pytorch is not supported(python>3.12).", + pythonNewerThan313()); String name = "tensorTest"; String filePath = String.join( diff --git a/test/src/test/resources/testConfig.properties b/test/src/test/resources/testConfig.properties index d7560aff33..672f44b3cd 100644 --- a/test/src/test/resources/testConfig.properties +++ b/test/src/test/resources/testConfig.properties @@ -4,7 +4,7 @@ storageEngineList=IoTDB12,InfluxDB,FileSystem,Relational,MongoDB,Redis,VectorDB relationalStorageEngineList=PostgreSQL,MySQL # the test for every engine -test-list=TransformIT +test-list=UserPermissionIT,DataSourceIT,SQLSessionIT,OptimizerIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT,HostUtilsTest vectordb-test-list=UserPermissionIT,DataSourceIT,SQLSessionIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT,HostUtilsTest # the DB config From b82d60600c5d5e198f7376ad19ec1fccdc8dd7ee Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Mon, 10 Feb 2025 06:15:54 +0800 Subject: [PATCH 06/10] fix test --- .github/actions/confWriter/action.yml | 3 +++ .../cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java | 7 +++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 5451a7cf24..5597de82e2 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -75,6 +75,9 @@ runs: id: find_python_cmd shell: bash -el {0} run: | + if [[ "${{ inputs.use-free-thread-python }}" == "true" ]]; then + echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV + else if [ "$RUNNER_OS" == "Windows" ]; then pythonCMD=python else diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java index 45fee10fa8..b47a90084a 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java @@ -21,7 +21,6 @@ import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; -import static cn.edu.tsinghua.iginx.integration.func.session.PySessionIT.pythonNewerThan313; import static org.junit.Assert.*; import cn.edu.tsinghua.iginx.exception.SessionException; @@ -1418,9 +1417,9 @@ public void testModuleInstallFail() { @Test public void tensorUDFTest() { - Assume.assumeFalse( - "tensorUDFTest is skipped because pytorch is not supported(python>3.12).", - pythonNewerThan313()); + boolean torchSupported = System.getenv().getOrDefault("TORCH_SUPPORTED", "true").equals("true"); + Assume.assumeTrue( + "tensorUDFTest is skipped because pytorch is not supported(python>3.12).", torchSupported); String name = "tensorTest"; String filePath = String.join( From 579567be232645bb01e7141645518bc6cfe5245c Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Mon, 10 Feb 2025 06:25:18 +0800 Subject: [PATCH 07/10] restore tests --- .github/actions/confWriter/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 5597de82e2..157e12a5dc 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -75,9 +75,9 @@ runs: id: find_python_cmd shell: bash -el {0} run: | - if [[ "${{ inputs.use-free-thread-python }}" == "true" ]]; then + if [ "${{ inputs.use-free-thread-python }}" == "true" ]; then echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV - else + fi if [ "$RUNNER_OS" == "Windows" ]; then pythonCMD=python else From 862e61e86acc6b9e7ed97db1c5600661d3dd0f49 Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Thu, 13 Feb 2025 00:14:24 +0800 Subject: [PATCH 08/10] add test --- .../iginx/integration/func/udf/UDFIT.java | 64 +++++++++++++++++++ test/src/test/resources/udf/no_mod_udf.py | 41 ++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 test/src/test/resources/udf/no_mod_udf.py diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java index b47a90084a..8e89eefa30 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java @@ -845,6 +845,70 @@ public void testUDFWithArgs() { compareResult(expected, ret.getResultInString(false, "")); } + @Test + public void testUDFOnlyArgs() { + String filePath = + String.join( + File.separator, + System.getProperty("user.dir"), + "src", + "test", + "resources", + "udf", + "no_mod_udf.py"); + String udfName = "no_mod"; + tool.executeReg(String.format(SINGLE_UDF_REGISTER_SQL, "UDTF", udfName, "NoModUDF", filePath)); + taskToBeRemoved.add(udfName); + + String query = "SELECT no_mod(1);"; + SessionExecuteSqlResult ret = tool.execute(query); + String expected = + "ResultSets:\n" + + "+---------+\n" + + "|no_mod(1)|\n" + + "+---------+\n" + + "| 1|\n" + + "+---------+\n" + + "Total line number = 1\n"; + compareResult(expected, ret.getResultInString(false, "")); + + query = "SELECT no_mod(1.98);"; + ret = tool.execute(query); + expected = + "ResultSets:\n" + + "+---------------+\n" + + "|no_mod(1.98000)|\n" + + "+---------------+\n" + + "| 1.98|\n" + + "+---------------+\n" + + "Total line number = 1\n"; + compareResult(expected, ret.getResultInString(false, "")); + + query = "SELECT no_mod(false), no_mod(TRUE);"; // false/FALSE/False + ret = tool.execute(query); + expected = + "ResultSets:\n" + + "+-------------+------------+\n" + + "|no_mod(false)|no_mod(true)|\n" + + "+-------------+------------+\n" + + "| false| true|\n" + + "+-------------+------------+\n" + + "Total line number = 1\n"; + compareResult(expected, ret.getResultInString(false, "")); + + query = "SELECT no_mod('test_string');"; + ret = tool.execute(query); + expected = + "ResultSets:\n" + + "+---------------------+\n" + + "|no_mod('test_string')|\n" + + "+---------------------+\n" + + "| test_string|\n" + + "+---------------------+\n" + + "Total line number = 1\n"; + compareResult(expected, ret.getResultInString(false, "")); + } + @Test public void testUDFWithKvargs() { String insert = diff --git a/test/src/test/resources/udf/no_mod_udf.py b/test/src/test/resources/udf/no_mod_udf.py new file mode 100644 index 0000000000..01f24729a9 --- /dev/null +++ b/test/src/test/resources/udf/no_mod_udf.py @@ -0,0 +1,41 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +class NoModUDF(): + """ + :return: return the exact same table after replacing column names with "no_mod(${COL_NAME})" + """ + def __init__(self): + pass + + def transform(self, data, args, kvargs): + for i, element in enumerate(data[0]): + if i == 0: + continue # skip key + data[0][i] = f"no_mod({element})" + return data + + +if __name__ == '__main__': + t = NoModUDF() + res = t.transform([ + ["key", "str"], + ["LONG", "BINARY"], + [0, "this"] + ], [], {}) + print(res) \ No newline at end of file From 7dee45bac096f849deb68a1d7f64fde92b23434c Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Thu, 13 Feb 2025 00:23:38 +0800 Subject: [PATCH 09/10] small change --- .github/actions/dependence/action.yml | 2 +- .github/actions/service/freeThreadConda/action.yml | 2 +- .../iginx/integration/func/session/PySessionIT.java | 2 +- test/src/test/resources/udf/no_mod_udf.py | 10 ---------- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 10abe6b846..5541f66065 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -117,7 +117,7 @@ runs: name: Install free-thread python uses: ./.github/actions/service/freeThreadConda with: - iginx-conda-env: "iginxEnv" + iginx-conda-env: ${{ inputs.iginx-conda-env }} - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v4 diff --git a/.github/actions/service/freeThreadConda/action.yml b/.github/actions/service/freeThreadConda/action.yml index ee85ebef9c..6398f995ae 100644 --- a/.github/actions/service/freeThreadConda/action.yml +++ b/.github/actions/service/freeThreadConda/action.yml @@ -17,7 +17,7 @@ runs: run-post: "false" # currently we can only install free-thread python 3.13 by conda-forge. latest miniconda only supports 3.12 - # thus in action steps afterward, remember to 'conda activate $ENV_NAME' at start if conda should be used + # thus in action steps ahead, remember to 'conda activate $ENV_NAME' at start if conda should be used - name: Check channels & install python3.13 shell: bash -el {0} run: | diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java index ffbdf6d117..adfbf60a7f 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java @@ -143,7 +143,7 @@ public void insertBaseData() { } } - public static boolean pythonNewerThan313() { + private boolean pythonNewerThan313() { interpreter.exec("import sys; tooNew = sys.version_info >= (3, 13);"); return (boolean) interpreter.get("tooNew"); } diff --git a/test/src/test/resources/udf/no_mod_udf.py b/test/src/test/resources/udf/no_mod_udf.py index 01f24729a9..1d0323ec64 100644 --- a/test/src/test/resources/udf/no_mod_udf.py +++ b/test/src/test/resources/udf/no_mod_udf.py @@ -29,13 +29,3 @@ def transform(self, data, args, kvargs): continue # skip key data[0][i] = f"no_mod({element})" return data - - -if __name__ == '__main__': - t = NoModUDF() - res = t.transform([ - ["key", "str"], - ["LONG", "BINARY"], - [0, "this"] - ], [], {}) - print(res) \ No newline at end of file From 718ef0ad641a7f591bd0431a6e413842849fae73 Mon Sep 17 00:00:00 2001 From: shinyano <12236590+shinyano@users.noreply.github.com> Date: Sun, 2 Mar 2025 15:24:04 +0800 Subject: [PATCH 10/10] improve action yaml --- .github/actions/iginxRunner/action.yml | 7 +++++-- .github/workflows/free-thread-test.yml | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml index aa56d24afa..18a5183c46 100644 --- a/.github/actions/iginxRunner/action.yml +++ b/.github/actions/iginxRunner/action.yml @@ -9,7 +9,10 @@ inputs: description: "to test UDF path detection" required: false default: "false" - activate-conda-flag: + iginx-conda-flag: + # this step is only needed when using python3.13 by conda because latest conda supports py3.12, + # and we use conda-forge to support 3.13. Thus, the required env would not be activated automatically + # even when we use login mode of bash description: "whether to manually activate conda env before start IGinX" required: false default: "false" @@ -44,7 +47,7 @@ runs: name: Start IGinX shell: bash -el {0} run: | - if [ "${{ inputs.activate-conda-flag }}" == "true" ]; then + if [ "${{ inputs.iginx-conda-flag }}" == "true" ]; then conda activate ${{ inputs.iginx-conda-env }} fi if [ "$RUNNER_OS" == "Linux" ]; then diff --git a/.github/workflows/free-thread-test.yml b/.github/workflows/free-thread-test.yml index 56776cafa4..38cbfccdef 100644 --- a/.github/workflows/free-thread-test.yml +++ b/.github/workflows/free-thread-test.yml @@ -37,6 +37,7 @@ jobs: runs-on: ${{ matrix.os }} env: # only used in this workflow file + IGINX_CONDA_FLAG: "true" IGINX_CONDA_ENV: "iginxEnv" steps: - uses: actions/checkout@v4 @@ -47,7 +48,7 @@ jobs: java: ${{ matrix.java }} free-thread-python-required: "true" scope: "only-java" - iginx-conda-flag: "true" + iginx-conda-flag: ${{ env.IGINX_CONDA_FLAG }} iginx-conda-env: ${{ env.IGINX_CONDA_ENV }} - name: Run Metadata @@ -76,7 +77,7 @@ jobs: - name: Start IGinX uses: ./.github/actions/iginxRunner with: - activate-conda-flag: "true" + iginx-conda-flag: ${{ env.IGINX_CONDA_FLAG }} iginx-conda-env: ${{ env.IGINX_CONDA_ENV }} - name: TestController IT