diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index e8540622b0..b166c8a57a 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -79,23 +79,16 @@ runs: id: find_python_cmd shell: bash -el {0} run: | - if [[ "${{ inputs.use-free-thread-python }}" == "true" ]]; then - if [ "$RUNNER_OS" == "Windows" ]; then - pythonPath=$(echo "$FT_PYTHON_PATH" | sed 's/\\/\//g') - pythonCMD="${pythonPath}/python3.13t.exe" - echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV - else - pythonCMD='python3.13t' - echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV - fi + if [ "${{ inputs.use-free-thread-python }}" == "true" ]; then + echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV + fi + if [ "$RUNNER_OS" == "Windows" ]; then + pythonCMD=python else - if [ "$RUNNER_OS" == "Windows" ]; then - pythonCMD=python - else - pythonCMD=python3 - fi + pythonCMD=python3 fi # have to set env to make it work in ut tests + ${pythonCMD} -VV echo "pythonCMD=${pythonCMD}" >> $GITHUB_ENV echo "pythonCMD=${pythonCMD}" >> $GITHUB_OUTPUT echo ${pythonCMD} diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 57a55cad42..5541f66065 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -115,7 +115,9 @@ runs: - if: inputs.free-thread-python-required=='true' name: Install free-thread python - uses: ./.github/actions/service/freeThreadPython + uses: ./.github/actions/service/freeThreadConda + with: + iginx-conda-env: ${{ inputs.iginx-conda-env }} - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v4 diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml index c0b60767ca..18a5183c46 100644 --- a/.github/actions/iginxRunner/action.yml +++ b/.github/actions/iginxRunner/action.yml @@ -9,6 +9,17 @@ inputs: description: "to test UDF path detection" required: false default: "false" + iginx-conda-flag: + # this step is only needed when using python3.13 by conda because latest conda supports py3.12, + # and we use conda-forge to support 3.13. Thus, the required env would not be activated automatically + # even when we use login mode of bash + description: "whether to manually activate conda env before start IGinX" + required: false + default: "false" + iginx-conda-env: + description: "required conda env name" + required: false + default: "false" runs: using: "composite" # Mandatory parameter @@ -36,16 +47,18 @@ runs: name: Start IGinX shell: bash -el {0} run: | - which python + if [ "${{ inputs.iginx-conda-flag }}" == "true" ]; then + conda activate ${{ inputs.iginx-conda-env }} + fi if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888 elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888 elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888 fi - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' diff --git a/.github/actions/service/freeThreadConda/action.yml b/.github/actions/service/freeThreadConda/action.yml new file mode 100644 index 0000000000..6398f995ae --- /dev/null +++ b/.github/actions/service/freeThreadConda/action.yml @@ -0,0 +1,36 @@ +name: "Install_Free-thread_Python in Conda" +description: "Install Free-thread Python in Conda" +inputs: + iginx-conda-env: + description: "conda env name" + required: false + +runs: + using: "composite" + steps: + # this step creates env test(default python version) which will not be used + - uses: conda-incubator/setup-miniconda@v3 + with: + channels: conda-forge + channel-priority: flexible + conda-remove-defaults: "true" + run-post: "false" + + # currently we can only install free-thread python 3.13 by conda-forge. latest miniconda only supports 3.12 + # thus in action steps ahead, remember to 'conda activate $ENV_NAME' at start if conda should be used + - name: Check channels & install python3.13 + shell: bash -el {0} + run: | + conda config --show channels + conda create -n ${{ inputs.iginx-conda-env }} python=3.13 python-freethreading -c conda-forge -q + conda activate ${{ inputs.iginx-conda-env }} + python -VV + python -c "import sys;print(sys._is_gil_enabled())" + curl -L -O https://github.com/IGinX-THU/IGinX-resources/raw/refs/heads/main/resources/python/pandas-3.0.0.dev0+1654.g32a97a969a-cp313-cp313t-win_amd64.whl + ls -l + if [ "$RUNNER_OS" == "Windows" ]; then + python -m pip install numpy thrift pemjax + python -m pip install pandas*.whl + else + python -m pip install pandas==2.2.3 numpy thrift pemjax + fi diff --git a/.github/actions/service/freeThreadPython/action.yml b/.github/actions/service/freeThreadPython/action.yml deleted file mode 100644 index 6e473ce15f..0000000000 --- a/.github/actions/service/freeThreadPython/action.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: "Install_Free-thread_Python" -description: "Install Free-thread Python" - -runs: - using: "composite" - steps: - - name: Set Python variables - id: set-python-vars - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - echo "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.13.0/python-3.13.0-amd64.exe" >> $GITHUB_ENV - echo "PYTHON_INSTALLER_PATH=python-installer.exe" >> $GITHUB_ENV - echo "CACHE_KEY=python-windows-3.13.0" >> $GITHUB_ENV - elif [ "$RUNNER_OS" == "macOS" ]; then - echo "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.13.0/python-3.13.0-macos11.pkg" >> $GITHUB_ENV - echo "PYTHON_INSTALLER_PATH=python-installer.pkg" >> $GITHUB_ENV - echo "CACHE_KEY=python-macos-3.13.0" >> $GITHUB_ENV - fi - - - name: Restore Python installer - if: runner.os!='Linux' - id: restore-cache-python - uses: actions/cache/restore@v3 - with: - path: ${{ env.PYTHON_INSTALLER_PATH }} - key: ${{ env.CACHE_KEY }} - - - name: Download Python installer - if: steps.restore-cache-python.outputs.cache-hit != 'true' && runner.os=='Windows' - shell: powershell - run: | - Invoke-WebRequest -Uri $env:PYTHON_INSTALLER_URL -OutFile $env:PYTHON_INSTALLER_PATH - - # have to wait until installation finishes - - if: runner.os=='Windows' - name: Setup Python3.13 on Windows - shell: powershell - run: | - Start-Process -FilePath "$(Join-Path -Path $PWD -ChildPath $env:PYTHON_INSTALLER_PATH)" -ArgumentList "/quiet","Include_freethreaded=1" -Wait - $pythonPath = "$env:LocalAppData\Programs\Python\Python313" - Get-ChildItem $pythonPath - $env:Path += ";$pythonPath" - echo "$pythonPath" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "FT_PYTHON_PATH=$pythonPath" | Out-File -FilePath $env:GITHUB_ENV -Append - python3.13t -VV - - - if: runner.os=='Linux' - name: Setup Python3.13 on Linux - shell: bash - run: | - sudo add-apt-repository ppa:deadsnakes - sudo apt-get update - sudo apt-get install python3.13-nogil python3.13-dev - python3.13t -VV - - - name: Download Python installer - if: steps.restore-cache-python.outputs.cache-hit != 'true' && runner.os=='macOS' - shell: bash - run: | - curl -o $PYTHON_INSTALLER_PATH $PYTHON_INSTALLER_URL - - - if: runner.os=='macOS' - name: Setup Python3.13 on MacOS - shell: bash - run: | - cat > ./choicechanges.plist < - - - - - attributeSetting - 1 - choiceAttribute - selected - choiceIdentifier - org.python.Python.PythonTFramework-3.13 - - - - EOF - sudo installer -pkg ./$PYTHON_INSTALLER_PATH \ - -applyChoiceChangesXML ./choicechanges.plist \ - -target / - python3.13t -VV - export PATH="$PATH:/Library/Frameworks/PythonT.framework/Versions/3.13/bin" - - - name: Cache Python installer - if: runner.os!='Linux' && steps.restore-cache-python.outputs.cache-hit != 'true' - id: cache-python - uses: actions/cache/save@v3 - with: - path: ${{ env.PYTHON_INSTALLER_PATH }} - key: ${{ env.CACHE_KEY }} - - - name: Install pip - if: runner.os=='Linux' - shell: bash - run: | - curl -O https://bootstrap.pypa.io/get-pip.py - sudo python3.13t get-pip.py - - # pandas 2.2.3 cannot be directly installed on windows yet. use dev wheel file - - name: Install dependencies for tests - shell: bash - run: | - curl -L -O https://github.com/IGinX-THU/IGinX-resources/raw/refs/heads/main/resources/python/pandas-3.0.0.dev0+1654.g32a97a969a-cp313-cp313t-win_amd64.whl - ls -l - if [ "$RUNNER_OS" == "Windows" ]; then - python3.13t -m pip install numpy thrift pemjax - python3.13t -m pip install pandas*.whl - else - sudo python3.13t -m pip install pandas==2.2.3 numpy thrift pemjax - fi diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index ee9cf74cb8..53269480f5 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -28,7 +28,8 @@ sed -i "s/restPort=[0-9]\+/restPort=$2/g" core/target/iginx-core-*/conf/config.p sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" -sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" +python3 -VV +bash -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" sh -c "sleep 3" diff --git a/.github/scripts/iginx/iginx_macos.sh b/.github/scripts/iginx/iginx_macos.sh index 0362088572..735ab6fbd7 100644 --- a/.github/scripts/iginx/iginx_macos.sh +++ b/.github/scripts/iginx/iginx_macos.sh @@ -32,7 +32,8 @@ echo "JAVA_HOME is set to $JAVA_HOME" sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" -sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" +python3 -VV +bash -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" sh -c "sleep 3" diff --git a/.github/scripts/test/test_union.sh b/.github/scripts/test/test_union.sh index c320ee66ea..a97639fbc7 100644 --- a/.github/scripts/test/test_union.sh +++ b/.github/scripts/test/test_union.sh @@ -17,7 +17,7 @@ # along with this program. If not, see . # - +python -VV pwd cd .. diff --git a/.github/workflows/free-thread-test.yml b/.github/workflows/free-thread-test.yml index 48f91b12c6..38cbfccdef 100644 --- a/.github/workflows/free-thread-test.yml +++ b/.github/workflows/free-thread-test.yml @@ -35,6 +35,10 @@ jobs: metadata: ${{ fromJSON(inputs.metadata-matrix) }} DB-name: ${{ fromJSON(inputs.db-matrix) }} runs-on: ${{ matrix.os }} + env: + # only used in this workflow file + IGINX_CONDA_FLAG: "true" + IGINX_CONDA_ENV: "iginxEnv" steps: - uses: actions/checkout@v4 # use free-threading python instead of stable version @@ -44,6 +48,8 @@ jobs: java: ${{ matrix.java }} free-thread-python-required: "true" scope: "only-java" + iginx-conda-flag: ${{ env.IGINX_CONDA_FLAG }} + iginx-conda-env: ${{ env.IGINX_CONDA_ENV }} - name: Run Metadata uses: ./.github/actions/metadataRunner @@ -70,13 +76,17 @@ jobs: - name: Start IGinX uses: ./.github/actions/iginxRunner + with: + iginx-conda-flag: ${{ env.IGINX_CONDA_FLAG }} + iginx-conda-env: ${{ env.IGINX_CONDA_ENV }} - name: TestController IT if: always() - shell: bash + shell: bash -el {0} env: METADATA_STORAGE: ${{ matrix.metadata }} run: | + conda activate ${{ env.IGINX_CONDA_ENV }} chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java index 09b940a092..e09bdb7690 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java @@ -250,13 +250,20 @@ private void verifyJobState(long jobId, JobState expectedState) throws SessionEx private void verifyJobFinishedBlocked(long jobId) throws SessionException, InterruptedException { LOGGER.info("job is {}", jobId); JobState jobState = JobState.JOB_CREATED; + int timeout = 60000; // 60s while (!jobState.equals(JobState.JOB_CLOSED) && !jobState.equals(JobState.JOB_FAILED) - && !jobState.equals(JobState.JOB_FINISHED)) { + && !jobState.equals(JobState.JOB_FINISHED) + && timeout > 0) { Thread.sleep(500); + timeout -= 500; jobState = session.queryTransformJobStatus(jobId); } LOGGER.info("job {} state is {}", jobId, jobState.toString()); + if (timeout <= 0) { + LOGGER.error("job is stuck, check IGinX server log..."); + fail(); + } if (!needCompareResult) { return; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java index b47a90084a..8e89eefa30 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java @@ -845,6 +845,70 @@ public void testUDFWithArgs() { compareResult(expected, ret.getResultInString(false, "")); } + @Test + public void testUDFOnlyArgs() { + String filePath = + String.join( + File.separator, + System.getProperty("user.dir"), + "src", + "test", + "resources", + "udf", + "no_mod_udf.py"); + String udfName = "no_mod"; + tool.executeReg(String.format(SINGLE_UDF_REGISTER_SQL, "UDTF", udfName, "NoModUDF", filePath)); + taskToBeRemoved.add(udfName); + + String query = "SELECT no_mod(1);"; + SessionExecuteSqlResult ret = tool.execute(query); + String expected = + "ResultSets:\n" + + "+---------+\n" + + "|no_mod(1)|\n" + + "+---------+\n" + + "| 1|\n" + + "+---------+\n" + + "Total line number = 1\n"; + compareResult(expected, ret.getResultInString(false, "")); + + query = "SELECT no_mod(1.98);"; + ret = tool.execute(query); + expected = + "ResultSets:\n" + + "+---------------+\n" + + "|no_mod(1.98000)|\n" + + "+---------------+\n" + + "| 1.98|\n" + + "+---------------+\n" + + "Total line number = 1\n"; + compareResult(expected, ret.getResultInString(false, "")); + + query = "SELECT no_mod(false), no_mod(TRUE);"; // false/FALSE/False + ret = tool.execute(query); + expected = + "ResultSets:\n" + + "+-------------+------------+\n" + + "|no_mod(false)|no_mod(true)|\n" + + "+-------------+------------+\n" + + "| false| true|\n" + + "+-------------+------------+\n" + + "Total line number = 1\n"; + compareResult(expected, ret.getResultInString(false, "")); + + query = "SELECT no_mod('test_string');"; + ret = tool.execute(query); + expected = + "ResultSets:\n" + + "+---------------------+\n" + + "|no_mod('test_string')|\n" + + "+---------------------+\n" + + "| test_string|\n" + + "+---------------------+\n" + + "Total line number = 1\n"; + compareResult(expected, ret.getResultInString(false, "")); + } + @Test public void testUDFWithKvargs() { String insert = diff --git a/test/src/test/resources/udf/no_mod_udf.py b/test/src/test/resources/udf/no_mod_udf.py new file mode 100644 index 0000000000..1d0323ec64 --- /dev/null +++ b/test/src/test/resources/udf/no_mod_udf.py @@ -0,0 +1,31 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +class NoModUDF(): + """ + :return: return the exact same table after replacing column names with "no_mod(${COL_NAME})" + """ + def __init__(self): + pass + + def transform(self, data, args, kvargs): + for i, element in enumerate(data[0]): + if i == 0: + continue # skip key + data[0][i] = f"no_mod({element})" + return data