diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml new file mode 100644 index 0000000..22ac213 --- /dev/null +++ b/.github/workflows/regression.yml @@ -0,0 +1,157 @@ +name: Integration Testing and Analysis + +on: + pull_request: + branches: + - main + - develop + - version* + push: + branches: + - main + - develop + - version* + tags: + - robot* + - regression* + - integration* + +env: + GO_VERSION: '^1.22' + STACKQL_CORE_REPOSITORY: ${{ vars.STACKQL_CORE_REPOSITORY != '' && vars.STACKQL_CORE_REPOSITORY || 'stackql/stackql' }} + STACKQL_CORE_REF: ${{ vars.STACKQL_CORE_REF != '' && vars.STACKQL_CORE_REF || 'main' }} + STACKQL_ANY_SDK_REPOSITORY: ${{ vars.STACKQL_ANY_SDK_REPOSITORY != '' && vars.STACKQL_ANY_SDK_REPOSITORY || 'stackql/any-sdk' }} + STACKQL_ANY_SDK_REF: ${{ vars.STACKQL_ANY_SDK_REF != '' && vars.STACKQL_ANY_SDK_REF || 'main' }} + +jobs: + build-and-deploy: + name: build-and-deploy + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + env: + AWS_DEFAULT_REGION: us-west-1 + REG_MAX_VERSIONS: 3 + REG_MAX_AGE_MONTHS: 6 + REG_WEBSITE_DIR: _deno_website + REG_PROVIDER_PATH: providers/dist + REG_ARTIFACT_REPO_BUCKET: stackql-registry-artifacts + REG_DENO_DEPLOY_ASSET_REPO: deno-deploy-registry + REG_DENO_DEPLOY_API_DEV: stackql-dev-registry + REG_DENO_DEPLOY_API_PROD: stackql-registry + + steps: + - uses: actions/checkout@v4.1.1 + name: "[SETUP] checkout repo" + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + id: go + + - name: Build rust release artifact + if: success() + run: | + cargo build --release --bin client_test_harness + + - name: Download core + uses: actions/checkout@v4.1.1 + with: + repository: ${{ env.STACKQL_CORE_REPOSITORY }} + ref: ${{ env.STACKQL_CORE_REF }} + token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }} + path: stackql-core + + - name: Download any-sdk + uses: actions/checkout@v4.1.1 + with: + repository: ${{ env.STACKQL_ANY_SDK_REPOSITORY }} + ref: ${{ env.STACKQL_ANY_SDK_REF }} + token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }} + path: stackql-any-sdk + + - name: Setup Python + uses: actions/setup-python@v5.0.0 + with: + python-version: '3.11' + + - name: Add dependencies + working-directory: stackql-core + run: | + sudo apt-get install -y jq + pip3 install -r cicd/requirements.txt + + - name: Build stackql from core source + working-directory: stackql-core + run: | + go get ./... + python3 cicd/python/build.py --build + + - name: Generate rewritten registry for simulations + working-directory: stackql-core + run: | + python3 test/python/registry-rewrite.py + + + - name: Prepare load balancing materials + working-directory: stackql-core + run: | + sudo cp /etc/hosts /etc/hosts.bak + python3 test/python/tcp_lb.py --generate-hosts-entries | sudo tee -a /etc/hosts + python3 test/python/tcp_lb.py --generate-nginx-lb > test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf + + + - name: Install and run nginx load balancer + working-directory: stackql-core + run: | + sudo apt-get install -y curl gnupg2 ca-certificates lsb-release ubuntu-keyring + curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor \ + | sudo tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null + gpg --dry-run --quiet --no-keyring --import --import-options import-show /usr/share/keyrings/nginx-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \ + http://nginx.org/packages/ubuntu $(lsb_release -cs) nginx" \ + | sudo tee /etc/apt/sources.list.d/nginx.list + sudo apt-get update + sudo apt-get install nginx + sudo nginx -c "$(pwd)/test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf" + + - name: Create materials for core tests + working-directory: stackql-core + run: | + openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365 + openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365 + openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 + + - name: Start Core Test Mocks + working-directory: stackql-core + run: | + pgrep -f flask | xargs kill -9 || true + flask --app=./test/python/flask/github/app run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1093 & + + - name: Run core robot functional tests + if: success() + run: | + providerRoot="$(realpath $(pwd)/stackql-core/test/registry-mocked)" + sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}' + robot \ + --variable "${sundryCfg}" \ + --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \ + -d test/robot/reports \ + test/robot/functional + + - name: Output from functional tests + if: always() + run: | + cat test/robot/reports/output.xml + + - name: Post core test cleanup + if: always() + run: | + pgrep -f flask | xargs kill -9 || true + \ No newline at end of file diff --git a/.gitignore b/.gitignore index d07fda9..f5de2ed 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,8 @@ stackql*.zip stackql*.pkg stackql_history.txt stackql.log +stackql-core/ +stackql-any-sdk/ +stackql-stackql-provider-registry/ .env nohup.out diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..115d6c7 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,36 @@ +{ + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug Rust", + "cargo": { + "args": ["build", "--manifest-path", "${workspaceFolder}/Cargo.toml"] + }, + "args": [ + "${input:queryStr}", + "${input:dsnStr}" + ], + "cwd": "${workspaceFolder}" + } + ], + "inputs": [ + { + "type": "pickString", + "id": "queryStr", + "description": "Enter the name of the Rust executable", + "options": [ + "SELECT repo, count(*) as has_starred FROM github.activity.repo_stargazers WHERE owner = 'stackql' and repo in ('stackql', 'stackql-deploy') and login = 'generalkroll0' GROUP BY repo;" + ] + }, + + { + "type": "pickString", + "id": "dsnStr", + "description": "Enter the name of the Rust executable", + "options": [ + "host=localhost port=5444" + ] + } + ] + } \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index f2b74cd..6f1a675 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,7 +84,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -114,6 +114,34 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bindgen" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 1.0.109", + "which", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.9.0" @@ -147,19 +175,42 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "client_test_harness" version = "0.1.0" dependencies = [ "env_logger", "futures", + "libpq", + "libpq-sys", "log", + "postgres", "tokio", "tokio-postgres", ] @@ -200,6 +251,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "env_filter" version = "0.1.3" @@ -223,6 +280,16 @@ dependencies = [ "log", ] +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -285,7 +352,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -346,6 +413,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "hmac" version = "0.12.1" @@ -355,6 +428,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -382,7 +464,7 @@ checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -395,12 +477,64 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "libpq" +version = "5.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "457febd48c79e1c69729a1a706144943c3ae0185cb7317efcb90a1a8f843994d" +dependencies = [ + "bitflags 2.9.0", + "libc", + "libpq-sys", + "log", + "thiserror", +] + +[[package]] +name = "libpq-sys" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef060ac05c207c85da15f4eb629100c8782e0db4c06a3c91c86be9c18ae8a23" +dependencies = [ + "bindgen", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "lock_api" version = "0.4.12" @@ -433,6 +567,12 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.5" @@ -453,6 +593,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "object" version = "0.36.7" @@ -491,6 +641,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -527,6 +683,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "portable-atomic" version = "1.11.0" @@ -542,6 +704,20 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postgres" +version = "0.19.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "363e6dfbdd780d3aa3597b6eb430db76bb315fa9bad7fae595bb8def808b8470" +dependencies = [ + "bytes", + "fallible-iterator", + "futures-util", + "log", + "tokio", + "tokio-postgres", +] + [[package]] name = "postgres-protocol" version = "0.6.8" @@ -640,7 +816,7 @@ version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ - "bitflags", + "bitflags 2.9.0", ] [[package]] @@ -678,6 +854,25 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.9.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -701,7 +896,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -715,6 +910,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -772,6 +973,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.100" @@ -783,6 +995,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "tinyvec" version = "1.9.0" @@ -824,7 +1056,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -905,6 +1137,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -953,7 +1191,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.100", "wasm-bindgen-shared", ] @@ -975,7 +1213,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -999,6 +1237,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "whoami" version = "1.6.0" @@ -1098,7 +1348,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags", + "bitflags 2.9.0", ] [[package]] @@ -1118,5 +1368,5 @@ checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] diff --git a/cargo.toml b/Cargo.toml similarity index 70% rename from cargo.toml rename to Cargo.toml index 1de2358..2d393b5 100644 --- a/cargo.toml +++ b/Cargo.toml @@ -1,11 +1,14 @@ -[package] -name = "client_test_harness" -version = "0.1.0" -edition = "2021" - -[dependencies] -tokio = { version = "1.44", features = ["full"] } -tokio-postgres = "0.7" -futures = "0.3" -env_logger = "0.11" -log = "0.4" \ No newline at end of file +[package] +name = "client_test_harness" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokio = { version = "1.44", features = ["full"] } +postgres = { version = "0.19.10" } +libpq = "5.0.2" +libpq-sys = "0.8.0" +tokio-postgres = "0.7" +futures = "0.3" +env_logger = "0.11" +log = "0.4" diff --git a/README.md b/README.md index 6cfe3e7..a083e30 100644 --- a/README.md +++ b/README.md @@ -1 +1,30 @@ # pgwire-lite-rs + +## Testing + +## Robot testing + +Per [`.github/workflows/regression.yml`](/.github/workflows/regression.yml). + +### vscode debug + +- Config in `launch.json` is dependent on [the `CodeLLDB` extension](https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb). +- Also installed [the `rust-analyzer` extension](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer). + +### Manual Dream Run + +Build with `cargo build --release --bin client_test_harness`. + +Then, presuming you have an appropriate `stackql` server running, the output of running `target/release/client_test_harness "SELECT repo, count(*) as has_starred FROM github.activity.repo_stargazers WHERE owner = 'stackql' and repo in ('stackql', 'stackql-deploy') and login = 'generalkroll0' GROUP BY repo;" "host=localhost port=5444"`, once github reaches rate limit: + +```log +Query did some non-notify thing. +--- Notice 1 --- +sqlstate: 01000 +detail: http response status code: 403, response body: {"message":"API rate limit exceeded for 110.144.44.79. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.) If you reach out to GitHub Support for help, please include the request ID E19A:36FB4A:0813:0A62:67F1D9DD and timestamp 2025-04-06 01:33:17 UTC.","documentation_url":"https://docs.github.com/rest/overview/rate-limits-for-the-rest-api","status":"403"} +http response status code: 403, response body: {"message":"API rate limit exceeded for 110.144.44.79. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.) If you reach out to GitHub Support for help, please include the request ID E19B:5D7C:12EAF10:17BA794:67F1D9DD and timestamp 2025-04-06 01:33:17 UTC.","documentation_url":"https://docs.github.com/rest/overview/rate-limits-for-the-rest-api","status":"403"} + +message: a notice level event has occurred +severity: NOTICE +Query executed successfully. +``` \ No newline at end of file diff --git a/cicd/tmp/.gitignore b/cicd/tmp/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/cicd/tmp/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/src/main.rs b/src/main.rs index f1f8e25..5425de7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,312 +1,151 @@ -// use tokio_postgres::{NoTls, Error, AsyncMessage}; -// use tokio::time::{sleep, Duration}; -// use futures::future::poll_fn; -// use std::env; +use postgres::{Client, NoTls}; +use postgres::SimpleQueryMessage::{Row, CommandComplete}; -// #[tokio::main] -// async fn main() -> Result<(), Error> { -// // Initialize logger for debugging -// env::set_var("RUST_LOG", "tokio_postgres=debug"); -// env_logger::init(); +use libpq; +use libpq_sys; -// // 1. Connect to the PostgreSQL-compatible server -// let (client, mut connection) = tokio_postgres::connect( -// "host=localhost port=5444 user=stackql dbname=stackql", -// NoTls, -// ).await?; -// println!("✅ Connected to the server"); +use std::env; -// // 2. Spawn a task to continuously poll the connection for async messages -// tokio::spawn(async move { -// loop { -// match poll_fn(|cx| connection.poll_message(cx)).await { -// Some(Ok(message)) => { -// match message { -// AsyncMessage::Notice(notice) => { -// println!("⚠️ NOTICE from server: {}", notice.message()); -// if let Some(detail) = notice.detail() { -// println!("📝 Detail: {}", detail); -// } -// if let Some(hint) = notice.hint() { -// println!("💡 Hint: {}", hint); -// } -// } -// AsyncMessage::Notification(notification) => { -// println!( -// "🔔 Notification received - Channel: {}, Payload: {}", -// notification.channel(), -// notification.payload() -// ); -// } -// _ => { -// println!("📥 Received unhandled message: {:?}", message); -// } -// } -// } -// Some(Err(e)) => { -// eprintln!("❌ Connection error: {}", e); -// break; -// } -// _none => break, -// } -// } -// }); -// // 3. Run your query and capture all possible messages -// let query = " -// SELECT repo, count(*) as has_starred -// FROM github.activity.repo_stargazers -// WHERE owner = 'fred' -// AND repo in ('stackql', 'stackql-deploy') -// AND login = 'generalkroll0' -// GROUP BY repo; -// "; +use std::collections::HashMap; +use std::ffi::{CStr, c_void}; +use std::os::raw::c_char; +use std::process::exit; +use std::sync::{Arc, Mutex}; -// println!("📥 Executing query..."); -// match client.query(query, &[]).await { -// Ok(rows) => { -// if rows.is_empty() { -// println!("Query returned no data"); -// } else { -// println!("🎉 Query returned {} rows", rows.len()); +use libpq_sys::{ + PGconn, PGresult, PQresultErrorField, + PG_DIAG_SEVERITY, PG_DIAG_SQLSTATE, PG_DIAG_MESSAGE_PRIMARY, + PG_DIAG_MESSAGE_DETAIL, PG_DIAG_MESSAGE_HINT, PG_DIAG_STATEMENT_POSITION, + PG_DIAG_INTERNAL_POSITION, PG_DIAG_INTERNAL_QUERY, + PG_DIAG_CONTEXT, PG_DIAG_SOURCE_FILE, + PG_DIAG_SOURCE_LINE, PG_DIAG_SOURCE_FUNCTION, +}; -// // Print column names -// let columns: Vec = rows[0] -// .columns() -// .iter() -// .map(|col| col.name().to_string()) -// .collect(); -// println!("Columns: {:?}", columns); -// // Print row data -// for row in rows { -// let mut row_data = Vec::new(); -// for (i, col) in row.columns().iter().enumerate() { -// let value: Result = row.try_get(i); -// row_data.push(format!("{}: {:?}", col.name(), value)); -// } -// println!("Row: {:?}", row_data); -// } -// } -// } -// Err(e) => { -// eprintln!("❌ Query failed: {}", e); -// } -// } -// // Give time for asynchronous messages to be processed (e.g., NOTICES) -// println!("⌛ Waiting for possible NOTICES from server..."); -// sleep(Duration::from_secs(5)).await; - -// println!("✅ Test harness completed successfully"); -// Ok(()) -// } - -// use tokio_postgres::{NoTls, Error, AsyncMessage}; -// use tokio::time::{sleep, Duration}; -// use futures::future::poll_fn; -// use tokio::spawn; -// use std::env; - -// #[tokio::main] -// async fn main() -> Result<(), Error> { -// // Set up logging for debugging purposes -// env::set_var("RUST_LOG", "tokio_postgres=debug"); -// env_logger::init(); - -// // 1. Connect to the PostgreSQL server -// let (client, mut connection) = tokio_postgres::connect( -// "host=localhost port=5444 user=stackql dbname=stackql", -// NoTls, -// ).await?; -// println!("✅ Connected to the server"); - -// // 2. Spawn a task to capture asynchronous messages like `NOTICE` -// let notice_task = spawn(async move { -// loop { -// match poll_fn(|cx| connection.poll_message(cx)).await { -// Some(Ok(message)) => match message { -// AsyncMessage::Notice(notice) => { -// println!("⚠️ NOTICE: {}", notice.message()); -// if let Some(detail) = notice.detail() { -// println!("📝 Detail: {}", detail); -// } -// if let Some(hint) = notice.hint() { -// println!("💡 Hint: {}", hint); -// } -// } -// AsyncMessage::Notification(notification) => { -// println!("🔔 Notification: Channel: {}, Payload: {}", notification.channel(), notification.payload()); -// } -// _ => {} // Ignore other message types for now -// }, -// Some(Err(e)) => { -// eprintln!("❌ Connection error while capturing notices: {}", e); -// break; -// } -// _none => break, // Connection closed -// } -// } -// }); - -// // 3. Execute your query -// let query = " -// SELECT repo, count(*) as has_starred -// FROM github.activity.repo_stargazers -// WHERE owner = 'fred' -// AND repo in ('stackql', 'stackql-deploy') -// AND login = 'generalkroll0' -// GROUP BY repo; -// "; - -// println!("📥 Executing query..."); -// match client.query(query, &[]).await { -// Ok(rows) => { -// if rows.is_empty() { -// println!("Query returned no data"); -// } else { -// println!("🎉 Query returned {} rows", rows.len()); - -// for row in rows { -// let repo: &str = row.get("repo"); -// let has_starred: i64 = row.get("has_starred"); -// println!("repo: {}, has_starred: {}", repo, has_starred); -// } -// } -// } -// Err(e) => { -// eprintln!("❌ Query failed: {}", e); -// } -// } - -// // 4. Give some time for asynchronous messages to be processed -// println!("⌛ Waiting for possible NOTICES from server..."); -// sleep(Duration::from_secs(5)).await; - -// // 5. Ensure notice task completes -// if let Err(e) = notice_task.await { -// eprintln!("❌ Notice task failed: {:?}", e); -// } - -// println!("✅ Test harness completed successfully"); -// Ok(()) -// } - -use tokio::net::TcpStream; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use log::debug; -use env_logger; - -#[tokio::main] -async fn main() { - env_logger::init(); - - // Connect to the server (no SSL, no authentication) - let mut stream = TcpStream::connect("127.0.0.1:5444").await.unwrap(); - println!("✅ Connected to server"); - - // Send StartupMessage - let startup_message = create_startup_message("stackql", "stackql"); - stream.write_all(&startup_message).await.unwrap(); - println!("📤 Sent StartupMessage"); +#[derive(Debug)] +pub struct Notice { + pub fields: HashMap<&'static str, String>, +} - let mut buffer = [0; 4096]; - let n = stream.read(&mut buffer).await.unwrap(); - debug!("📥 Received response: {:?}", &buffer[..n]); +type SharedNotices = Arc>>; - // Send query - let query = "SELECT repo, count(*) as has_starred FROM github.activity.repo_stargazers WHERE owner = 'fred' AND repo in ('stackql', 'stackql-deploy') AND login = 'generalkroll0' GROUP BY repo;"; - let query_message = create_query_message(query); - stream.write_all(&query_message).await.unwrap(); - println!("📤 Sent QueryMessage"); +/// Custom notice receiver function. +extern "C" fn notice_receiver(arg: *mut c_void, result: *const PGresult) { + if result.is_null() || arg.is_null() { + return; + } - loop { - let n = stream.read(&mut buffer).await.unwrap(); - if n == 0 { - break; + let field_kinds = [ + (PG_DIAG_SEVERITY, "severity"), + (PG_DIAG_SQLSTATE, "sqlstate"), + (PG_DIAG_MESSAGE_PRIMARY, "message"), + (PG_DIAG_MESSAGE_DETAIL, "detail"), + (PG_DIAG_MESSAGE_HINT, "hint"), + (PG_DIAG_STATEMENT_POSITION, "statement_position"), + (PG_DIAG_INTERNAL_POSITION, "internal_position"), + (PG_DIAG_INTERNAL_QUERY, "internal_query"), + (PG_DIAG_CONTEXT, "context"), + (PG_DIAG_SOURCE_FILE, "source_file"), + (PG_DIAG_SOURCE_LINE, "source_line"), + (PG_DIAG_SOURCE_FUNCTION, "source_function"), + ]; + + let mut notice = Notice { + fields: HashMap::new(), + }; + + for (code, label) in field_kinds.iter() { + let val_ptr = unsafe { PQresultErrorField(result, *code as i32) }; + if !val_ptr.is_null() { + let val = unsafe { CStr::from_ptr(val_ptr) }.to_string_lossy().into_owned(); + notice.fields.insert(*label, val); } - - process_message(&buffer[..n]); } - println!("✅ Done"); -} - -fn create_startup_message(user: &str, database: &str) -> Vec { - let user_string = format!("user\0{}\0", user); - let database_string = format!("database\0{}\0", database); - - let protocol_version = [0x00, 0x03, 0x00, 0x00]; // Protocol version 3.0 - let payload = [protocol_version.as_ref(), user_string.as_bytes(), database_string.as_bytes(), &[0]].concat(); - - let mut message = Vec::new(); - message.extend(&(payload.len() as u32 + 4).to_be_bytes()); - message.extend(&payload); - - message -} - -fn create_query_message(query: &str) -> Vec { - let mut message = Vec::new(); - message.push(b'Q'); - message.extend(&(query.len() as u32 + 5).to_be_bytes()); - message.extend(query.as_bytes()); - message.push(0); // Null terminator - - message + let shared = unsafe { &*(arg as *const Mutex>) }; + if let Ok(mut vec) = shared.lock() { + vec.push(notice); + } } -fn process_message(data: &[u8]) { - let mut offset = 0; - - while offset < data.len() { - let message_type = data[offset] as char; - offset += 1; +fn pq_query(conn: &libpq::Connection, query: &str, notices: SharedNotices) -> Result<(), Box> { + // Capture notice count before the query + let old_len = { + let lock = notices.lock().unwrap(); + lock.len() + }; - let message_length = u32::from_be_bytes(data[offset..offset+4].try_into().unwrap()) as usize; - offset += 4; + // Run the query + let res = conn.exec(query); + let res_status = res.status(); - let content = &data[offset..offset + message_length - 4]; - offset += message_length - 4; + if res_status == libpq::Status::NonFatalError { + println!("Query notify in effect: {:?}", conn.error_message()); + } else { + println!("Query did some non-notify thing."); + } - match message_type { - 'R' => handle_authentication(content), - 'S' => handle_parameter_status(content), - 'T' => handle_row_description(content), - 'D' => handle_data_row(content), - 'C' => println!("✅ Command Complete: {}", String::from_utf8_lossy(content)), - 'E' => println!("❌ Error: {}", String::from_utf8_lossy(content)), - 'N' => println!("⚠️ NOTICE: {}", String::from_utf8_lossy(content)), - 'Z' => println!("🟢 Ready for Query"), - _ => debug!("📥 Unhandled message type: {} Content: {:?}", message_type, content), + // Extract and print only the new notices + let mut locked = notices.lock().unwrap(); + let new_notices = locked.split_off(old_len); + if new_notices.is_empty() { + println!("No notices captured."); + } else { + for (i, notice) in new_notices.iter().enumerate() { + println!("--- Notice {} ---", i + 1); + for (k, v) in ¬ice.fields { + println!("{}: {}", k, v); + } } } + + Ok(()) } -fn handle_authentication(content: &[u8]) { - let auth_type = u32::from_be_bytes(content[0..4].try_into().unwrap()); - if auth_type == 0 { - println!("✅ Authentication successful"); - } else { - println!("❌ Authentication required, unsupported by this client"); +fn pq_main(query_str: &str, dsn: &str) -> Result<(), Box> { + let conninfo = dsn.to_string(); + // let query_str = "\ + // SELECT repo, count(*) as has_starred \ + // FROM github.activity.repo_stargazers \ + // WHERE owner = 'stackql' and repo in ('stackql', 'stackql-deploy') \ + // and login = 'generalkroll0' \ + // GROUP BY repo;\ + // "; + + // Create shared notice storage + let notices: SharedNotices = Arc::new(Mutex::new(Vec::new())); + let notices_raw_ptr = Arc::into_raw(notices.clone()) as *mut c_void; + + let conn = libpq::Connection::new(&conninfo)?; + unsafe { + libpq_sys::PQsetNoticeReceiver((&conn).into(), Some(notice_receiver), notices_raw_ptr); } -} -fn handle_parameter_status(content: &[u8]) { - if let Ok(text) = std::str::from_utf8(content) { - let parts: Vec<&str> = text.split('\0').collect(); - if parts.len() >= 2 { - println!("🔧 Parameter Status: {} = {}", parts[0], parts[1]); - } + // Execute the query + pq_query(&conn, query_str, notices)?; + + // Manually drop libpq's reference to avoid leak + unsafe { + let _ = Arc::from_raw(notices_raw_ptr as *const Mutex>); } -} -fn handle_row_description(content: &[u8]) { - let column_count = u16::from_be_bytes(content[0..2].try_into().unwrap()); - println!("📥 Row Description: {} columns", column_count); + Ok(()) } -fn handle_data_row(content: &[u8]) { - let column_count = u16::from_be_bytes(content[0..2].try_into().unwrap()); - println!("📥 Data Row: {} columns", column_count); +fn main() { + let all_args = env::args().collect::>(); + if all_args.len() < 3 { + println!("Need to at least supply query argument and dsn."); + exit(1); + } + let query = &all_args[1]; + let dsn = &all_args[2]; + if let Err(e) = pq_main(&query, &dsn) { + eprintln!("Error: {}", e); + exit(1); + } + println!("Query executed successfully."); + exit(0); } diff --git a/stackql_server.log b/stackql_server.log deleted file mode 100644 index 48ef9d5..0000000 --- a/stackql_server.log +++ /dev/null @@ -1 +0,0 @@ -nohup: ignoring input diff --git a/test/robot/functional/__init__.robot b/test/robot/functional/__init__.robot new file mode 100644 index 0000000..cb2865f --- /dev/null +++ b/test/robot/functional/__init__.robot @@ -0,0 +1,5 @@ +*** Settings *** +Resource ${CURDIR}/stackql.resource +Suite Setup Prepare StackQL Environment +Suite Teardown Terminate All Processes kill=True + diff --git a/test/robot/functional/stackql.resource b/test/robot/functional/stackql.resource new file mode 100644 index 0000000..f66888b --- /dev/null +++ b/test/robot/functional/stackql.resource @@ -0,0 +1,188 @@ +*** Variables *** +${REPOSITORY_ROOT} ${CURDIR}${/}..${/}..${/}.. +${LOCAL_LIB_HOME} ${REPOSITORY_ROOT}${/}stackql-core${/}test${/}robot${/}lib +${RUST_TESTING_EXE} ${REPOSITORY_ROOT}${/}target${/}release${/}client_test_harness +${EXECUTION_PLATFORM} native # to be overridden from command line, eg "docker" +${SQL_BACKEND} sqlite_embedded # to be overridden from command line, eg "postgres_tcp" +${IS_WSL} false # to be overridden from command line, with string "true" +${SHOULD_RUN_DOCKER_EXTERNAL_TESTS} false # to be overridden from command line, with string "true" +${CONCURRENCY_LIMIT} 1 # to be overridden from command line, with integer value, -1 for no limit +${USE_STACKQL_PREINSTALLED} false # to be overridden from command line, with string "true" +${SUNDRY_CONFIG} {} # to be overridden from command line, with string value + +*** Settings *** +Library Process +Library OperatingSystem +Variables ${LOCAL_LIB_HOME}/stackql_context.py ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${USE_STACKQL_PREINSTALLED} +... ${SUNDRY_CONFIG} +Library Process +Library OperatingSystem +Library String +Library ${LOCAL_LIB_HOME}/StackQLInterfaces.py ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${CONCURRENCY_LIMIT} +Library ${LOCAL_LIB_HOME}/CloudIntegration.py +Library ${LOCAL_LIB_HOME}/web_service_keywords.py + +*** Keywords *** + +Start All Mock Servers + ${port_dict} = Create Dictionary + ... oauth_client_credentials_token=${MOCKSERVER_PORT_OAUTH_CLIENT_CREDENTIALS_TOKEN} + ... github=${MOCKSERVER_PORT_GITHUB} + ... google=${MOCKSERVER_PORT_GOOGLE} + ... okta=${MOCKSERVER_PORT_OKTA} + ... aws=${MOCKSERVER_PORT_AWS} + ... stackql_auth_testing=${MOCKSERVER_PORT_STACKQL_AUTH_TESTING} + ... googleadmin=${MOCKSERVER_PORT_GOOGLEADMIN} + ... k8s=${MOCKSERVER_PORT_K8S} + ... registry=${MOCKSERVER_PORT_REGISTRY} + ... azure=${MOCKSERVER_PORT_AZURE} + ... sumologic=${MOCKSERVER_PORT_SUMOLOGIC} + ... digitalocean=${MOCKSERVER_PORT_DIGITALOCEAN} + Start All Webservers port_dict=${port_dict} + + +Prepare StackQL Environment + Set Environment Variable OKTA_SECRET_KEY ${OKTA_SECRET_STR} + Set Environment Variable GITHUB_SECRET_KEY ${GITHUB_SECRET_STR} + Set Environment Variable K8S_SECRET_KEY ${K8S_SECRET_STR} + Set Environment Variable AZ_ACCESS_TOKEN ${AZURE_SECRET_STR} + Set Environment Variable SUMO_CREDS ${SUMOLOGIC_SECRET_STR} + Set Environment Variable DIGITALOCEAN_TOKEN ${DIGITALOCEAN_SECRET_STR} + Set Environment Variable DUMMY_DIGITALOCEAN_USERNAME ${DUMMY_DIGITALOCEAN_USERNAME_STR} + Set Environment Variable DUMMY_DIGITALOCEAN_PASSWORD ${DUMMY_DIGITALOCEAN_PASSWORD_STR} + Set Environment Variable DB_SETUP_SRC ${DB_SETUP_SRC} + Set Environment Variable GOOGLE_APPLICATION_CREDENTIALS ${GOOGLE_APPLICATION_CREDENTIALS} + Set Environment Variable DD_API_KEY %{DD_API_KEY=myusername} + Set Environment Variable DD_APPLICATION_KEY %{DD_APPLICATION_KEY=mypassword} + # Start All Mock Servers + Generate Container Credentials for StackQL PG Server mTLS + Start StackQL PG Server mTLS ${PG_SRV_PORT_MTLS} ${PG_SRV_MTLS_CFG_STR} {} {} ${SQL_BACKEND_CFG_STR_CANONICAL} ${PG_SRV_PORT_DOCKER_MTLS} + Start StackQL PG Server mTLS ${PG_SRV_PORT_MTLS_WITH_NAMESPACES} ${PG_SRV_MTLS_CFG_STR} ${NAMESPACES_TTL_SPECIALCASE_TRANSPARENT} {} ${SQL_BACKEND_CFG_STR_CANONICAL} ${PG_SRV_PORT_DOCKER_MTLS_WITH_NAMESPACES} + Start StackQL PG Server mTLS ${PG_SRV_PORT_MTLS_WITH_EAGER_GC} ${PG_SRV_MTLS_CFG_STR} {} ${GC_CFG_EAGER} ${SQL_BACKEND_CFG_STR_CANONICAL} ${PG_SRV_PORT_DOCKER_MTLS_WITH_EAGER_GC} + Start StackQL PG Server unencrypted ${PG_SRV_PORT_UNENCRYPTED} {} ${SQL_BACKEND_CFG_STR_CANONICAL} + Start StackQL PG Server mTLS Expose Backing DB ${PG_SRV_PORT_MTLS_EXPORT} ${PG_SRV_MTLS_CFG_STR} {} {} ${SQL_BACKEND_CFG_STR_CANONICAL} ${PG_SRV_PORT_DOCKER_MTLS_EXPORT} + Start Postgres External Source If Viable + Sleep 50s + +Generate Container Credentials for StackQL PG Server mTLS + IF "${EXECUTION_PLATFORM}" == "docker" + ${res} = Run Process docker compose \-f docker\-compose\-credentials.yml + ... run \-\-rm credentialsgen + Log Credentials gen completed + Should Be Equal As Integers ${res.rc} 0 + END + +Start StackQL PG Server mTLS + [Arguments] ${_SRV_PORT_MTLS} ${_MTLS_CFG_STR} ${_NAMESPACES_CFG} ${_GC_CFG} ${_SQL_BACKEND_CFG} ${_DOCKER_PORT} + IF "${EXECUTION_PLATFORM}" == "native" + ${process} = Start Process ${STACKQL_EXE} + ... srv \-\-registry\=${REGISTRY_NO_VERIFY_CFG_STR.get_config_str('native')} + ... \-\-auth\=${AUTH_CFG_STR} + ... \-\-tls\.allowInsecure\=true + ... \-\-pgsrv\.address\=0.0.0.0 + ... \-\-pgsrv\.port\=${_SRV_PORT_MTLS} + ... \-\-pgsrv\.debug\.enable\=true + ... \-\-pgsrv\.tls ${_MTLS_CFG_STR} + ... \-\-namespaces\=${_NAMESPACES_CFG} + ... \-\-gc\=${_GC_CFG} + ... \-\-execution\.concurrency\.limit\=${CONCURRENCY_LIMIT} + ... \-\-sqlBackend\=${_SQL_BACKEND_CFG} + ... \-\-dbInternal\=${DB_INTERNAL_CFG_LAX} + ... stderr=${CURDIR}/tmp/stdout-stackql-srv-mtls-${_SRV_PORT_MTLS}.txt + ... stdout=${CURDIR}/tmp/stderr-stackql-srv-mtls-${_SRV_PORT_MTLS}.txt + ELSE IF "${EXECUTION_PLATFORM}" == "docker" + ${process} = Start Process docker compose + ... \-p stackqlpgsrv\-mtls\-${_DOCKER_PORT} + ... run + ... \-\-rm \-p${_SRV_PORT_MTLS}:${_DOCKER_PORT}/tcp + ... stackqlsrv + ... bash + ... \-c + ... sleep 2 && stackql srv \-\-execution\.concurrency\.limit\=${CONCURRENCY_LIMIT} \-\-registry\='${REGISTRY_NO_VERIFY_CFG_STR.get_config_str('docker')}' \-\-auth\='${AUTH_CFG_STR}' \-\-namespaces\='${_NAMESPACES_CFG}' \-\-gc\='${_GC_CFG}' \-\-sqlBackend\='${_SQL_BACKEND_CFG}' \-\-dbInternal\='${DB_INTERNAL_CFG_LAX}' \-\-tls\.allowInsecure\=true \-\-pgsrv\.address\='0.0.0.0' \-\-pgsrv\.port\=${_DOCKER_PORT} \-\-pgsrv\.debug\.enable\=true \-\-pgsrv\.tls\='{\"keyFilePath\": \"/opt/stackql/srv/credentials/pg_server_key.pem\", \"certFilePath\": \"/opt/stackql/srv/credentials/pg_server_cert.pem\", \"clientCAs\": [\"'$(base64 -w 0 /opt/stackql/srv/credentials/pg_client_cert.pem)'\"]}' + ... stderr=${CURDIR}/tmp/stdout-stackql-srv-mtls-${_SRV_PORT_MTLS}.txt + ... stdout=${CURDIR}/tmp/stderr-stackql-srv-mtls-${_SRV_PORT_MTLS}.txt + END + RETURN ${process} + +Start StackQL PG Server mTLS Expose Backing DB + [Arguments] ${_SRV_PORT_MTLS} ${_MTLS_CFG_STR} ${_NAMESPACES_CFG} ${_GC_CFG} ${_SQL_BACKEND_CFG} ${_DOCKER_PORT} + IF "${EXECUTION_PLATFORM}" == "native" + ${process} = Start Process ${STACKQL_EXE} + ... srv \-\-registry\=${REGISTRY_NO_VERIFY_CFG_STR.get_config_str('native')} + ... \-\-auth\=${AUTH_CFG_STR} + ... \-\-tls\.allowInsecure\=true + ... \-\-pgsrv\.address\=0.0.0.0 + ... \-\-pgsrv\.port\=${_SRV_PORT_MTLS} + ... \-\-pgsrv\.debug\.enable\=true + ... \-\-pgsrv\.tls ${_MTLS_CFG_STR} + ... \-\-namespaces\=${_NAMESPACES_CFG} + ... \-\-gc\=${_GC_CFG} + ... \-\-execution\.concurrency\.limit\=${CONCURRENCY_LIMIT} + ... \-\-sqlBackend\=${_SQL_BACKEND_CFG} + ... \-\-dbInternal\=${DB_INTERNAL_CFG_LAX} + ... \-\-export.alias\=stackql_export + ... stderr=${CURDIR}/tmp/stdout-stackql-srv-mtls-backing-${_SRV_PORT_MTLS}.txt + ... stdout=${CURDIR}/tmp/stderr-stackql-srv-mtls-backing-${_SRV_PORT_MTLS}.txt + ELSE IF "${EXECUTION_PLATFORM}" == "docker" + ${process} = Start Process docker compose + ... \-p stackqlpgsrv\-mtls\-backing\-${_DOCKER_PORT} + ... \-f ./docker\-compose\-persist\-postgres.yml + ... run + ... \-\-rm \-p${_SRV_PORT_MTLS}:${_DOCKER_PORT}/tcp + ... stackqlsrv + ... bash + ... \-c + ... sleep 2 && stackql srv \-\-execution\.concurrency\.limit\=${CONCURRENCY_LIMIT} \-\-registry\='${REGISTRY_NO_VERIFY_CFG_STR.get_config_str('docker')}' \-\-auth\='${AUTH_CFG_STR}' \-\-namespaces\='${_NAMESPACES_CFG}' \-\-gc\='${_GC_CFG}' \-\-sqlBackend\='${_SQL_BACKEND_CFG}' \-\-export.alias\='stackql_export' \-\-dbInternal\='${DB_INTERNAL_CFG_LAX}' \-\-tls\.allowInsecure\=true \-\-pgsrv\.address\='0.0.0.0' \-\-pgsrv\.port\=${_DOCKER_PORT} \-\-pgsrv\.debug\.enable\=true \-\-pgsrv\.tls\='{\"keyFilePath\": \"/opt/stackql/srv/credentials/pg_server_key.pem\", \"certFilePath\": \"/opt/stackql/srv/credentials/pg_server_cert.pem\", \"clientCAs\": [\"'$(base64 -w 0 /opt/stackql/srv/credentials/pg_client_cert.pem)'\"]}' + ... stderr=${CURDIR}/tmp/stdout-stackql-srv-mtls-backing-${_SRV_PORT_MTLS}.txt + ... stdout=${CURDIR}/tmp/stderr-stackql-srv-mtls-backing-${_SRV_PORT_MTLS}.txt + END + RETURN ${process} + +Start StackQL PG Server unencrypted + [Arguments] ${_SRV_PORT_UNENCRYPTED} ${_NAMESPACES_CFG} ${_SQL_BACKEND_CFG} + IF "${EXECUTION_PLATFORM}" == "native" + ${process} = Start Process ${STACKQL_EXE} + ... srv \-\-registry\=${REGISTRY_NO_VERIFY_CFG_STR.get_config_str('native')} + ... \-\-auth\=${AUTH_CFG_STR} + ... \-\-tls\.allowInsecure\=true + ... \-\-pgsrv\.address\=0.0.0.0 + ... \-\-pgsrv\.port\=${_SRV_PORT_UNENCRYPTED} + ... \-\-namespaces\=${_NAMESPACES_CFG} + ... \-\-pgsrv\.debug\.enable\=true + ... \-\-sqlBackend\=${_SQL_BACKEND_CFG} + ... \-\-dbInternal\=${DB_INTERNAL_CFG_LAX} + ... \-\-execution\.concurrency\.limit\=${CONCURRENCY_LIMIT} + ... stderr=${CURDIR}/tmp/stdout-stackql-srv-unencrypted-${_SRV_PORT_UNENCRYPTED}.txt + ... stdout=${CURDIR}/tmp/stderr-stackql-srv-unencrypted-${_SRV_PORT_UNENCRYPTED}.txt + ELSE IF "${EXECUTION_PLATFORM}" == "docker" + ${process} = Start Process docker compose + ... \-p stackqlpgsrv\-unencrypted + ... run + ... \-\-rm \-p${_SRV_PORT_UNENCRYPTED}:${PG_SRV_PORT_DOCKER_UNENCRYPTED}/tcp + ... stackqlsrv + ... bash + ... \-c + ... sleep 2 && stackql srv \-\-execution\.concurrency\.limit\=${CONCURRENCY_LIMIT} \-\-registry\='${REGISTRY_NO_VERIFY_CFG_STR.get_config_str('docker')}' \-\-pgsrv\.debug\.enable\=true \-\-auth\='${AUTH_CFG_STR}' \-\-namespaces\='${_NAMESPACES_CFG}' \-\-sqlBackend\='${_SQL_BACKEND_CFG}' \-\-dbInternal\='${DB_INTERNAL_CFG_LAX}' \-\-tls\.allowInsecure\=true \-\-pgsrv\.address\=0.0.0.0 \-\-pgsrv\.port\=${PG_SRV_PORT_DOCKER_UNENCRYPTED} + ... stderr=${CURDIR}/tmp/stdout-stackql-srv-unencrypted-${_SRV_PORT_UNENCRYPTED}.txt + ... stdout=${CURDIR}/tmp/stderr-stackql-srv-unencrypted-${_SRV_PORT_UNENCRYPTED}.txt + END + RETURN ${process} + +Start Postgres External Source If Viable + IF "${SHOULD_RUN_DOCKER_EXTERNAL_TESTS}" == "true" + ${process} = Start Process docker compose + ... \-p pg\-data\-external + ... \-f docker\-compose\-externals.yml + ... up + ... postgres_external_data_source + END + +Stackql Per Test Teardown + IF "${EXECUTION_PLATFORM}" == "docker" and "${SQL_BACKEND}" == "postgres_tcp" + ${res} = Run Process bash \-c docker kill $(docker ps \-\-filter name\=execrun \-q) + Log Container killed + # Should Be Equal As Integers ${res.rc} 0 + ${restwo} = Run Process bash \-c docker rm $(docker ps \-\-filter status\=exited \-q) + Log Container removed + # Should Be Equal As Integers ${restwo.rc} 0 + END diff --git a/test/robot/functional/stackql_sessions.robot b/test/robot/functional/stackql_sessions.robot new file mode 100644 index 0000000..3db7b8f --- /dev/null +++ b/test/robot/functional/stackql_sessions.robot @@ -0,0 +1,118 @@ +*** Settings *** +Resource ${CURDIR}/stackql.resource + +*** Test Cases *** + +PG Session Debug Behaviour Canonical + [Documentation] The mTLS servers have debug on + Set Environment Variable PGHOST ${PSQL_CLIENT_HOST} + Set Environment Variable PGPORT ${PG_SRV_PORT_MTLS} + Set Environment Variable PGSSLMODE verify\-full + Set Environment Variable PGSSLCERT ${STACKQL_PG_CLIENT_CERT_PATH} + Set Environment Variable PGSSLKEY ${STACKQL_PG_CLIENT_KEY_PATH} + Set Environment Variable PGSSLROOTCERT ${STACKQL_PG_SERVER_CERT_PATH} + ${inputStr} = Catenate + ... SELECT repo, count(1) as has_starred + ... FROM github.activity.repo_stargazers + ... WHERE owner = 'sillyorg' and repo in ('silly', 'silly-but-more') and login = 'sillylogin' + ... GROUP BY repo; + ${outputStr} = Catenate SEPARATOR=\n + ... repo | has_starred + ... ------+------------- + ... (0 rows) + ${outputErrStr} = Catenate SEPARATOR=\n + ... NOTICE: a notice level event has occurred + ... DETAIL: http response status code: 403, response body: "{\\"message\\":\\"API rate limit exceeded for 111.1111.111.11. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.) If you reach out to GitHub Support for help, please include the request ID AAAA:AAAA:BBBBBB:BBBBBB:BBBBBBBB and timestamp 2025-01-01 01:00:00 UTC.\\",\\"documentation_url\\":\\"https://docs.github.com/rest/overview/rate-limits-for-the-rest-api\\",\\"status\\":\\"403\\"}" + ... + ... http response status code: 403, response body: "{\\"message\\":\\"API rate limit exceeded for 111.1111.111.11. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.) If you reach out to GitHub Support for help, please include the request ID AAAA:AAAA:BBBBBB:BBBBBB:BBBBBBBB and timestamp 2025-01-01 01:00:00 UTC.\\",\\"documentation_url\\":\\"https://docs.github.com/rest/overview/rate-limits-for-the-rest-api\\",\\"status\\":\\"403\\"}" + ${posixInput} = Catenate + ... "${PSQL_EXE}" -c "${inputStr}" + ${windowsInput} = Catenate + ... & ${posixInput} + ${input} = Set Variable If "${IS_WINDOWS}" == "1" ${windowsInput} ${posixInput} + ${shellExe} = Set Variable If "${IS_WINDOWS}" == "1" powershell sh + ${result} = Run Process + ... ${shellExe} \-c ${input} + ... stdout=${CURDIR}/tmp/PG-Session-Debug-Behaviour-Canonical.tmp + ... stderr=${CURDIR}/tmp/PG-Session-Debug-Behaviour-Canonical-stderr.tmp + Log STDOUT = "${result.stdout}" + Log STDERR = "${result.stderr}" + # Should Contain ${result.stdout} ${outputStr} collapse_spaces=True + Should Contain ${result.stderr} ${outputErrStr} collapse_spaces=True + [Teardown] Run Keywords Remove Environment Variable PGHOST + ... AND Remove Environment Variable PGPORT + ... AND Remove Environment Variable PGSSLMODE + ... AND Remove Environment Variable PGSSLCERT + ... AND Remove Environment Variable PGSSLKEY + ... AND Remove Environment Variable PGSSLROOTCERT + +PG Session Debug On Unencryopted Server Behaviour Canonical + [Documentation] The unencrypted servers also have debug on + Set Environment Variable PGHOST ${PSQL_CLIENT_HOST} + Set Environment Variable PGPORT ${PG_SRV_PORT_UNENCRYPTED} + Set Environment Variable PGUSER stackql + Set Environment Variable PGPASSWORD ${PSQL_PASSWORD} + ${inputStr} = Catenate + ... SELECT repo, count(1) as has_starred + ... FROM github.activity.repo_stargazers + ... WHERE owner = 'sillyorg' and repo in ('silly', 'silly-but-more') and login = 'sillylogin' + ... GROUP BY repo; + ${outputStr} = Catenate SEPARATOR=\n + ... repo | has_starred + ... ------+------------- + ... (0 rows) + ${outputErrStr} = Catenate SEPARATOR=\n + ... NOTICE: a notice level event has occurred + ... DETAIL: http response status code: 403, response body: "{\\"message\\":\\"API rate limit exceeded for 111.1111.111.11. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.) If you reach out to GitHub Support for help, please include the request ID AAAA:AAAA:BBBBBB:BBBBBB:BBBBBBBB and timestamp 2025-01-01 01:00:00 UTC.\\",\\"documentation_url\\":\\"https://docs.github.com/rest/overview/rate-limits-for-the-rest-api\\",\\"status\\":\\"403\\"}" + ... + ... http response status code: 403, response body: "{\\"message\\":\\"API rate limit exceeded for 111.1111.111.11. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.) If you reach out to GitHub Support for help, please include the request ID AAAA:AAAA:BBBBBB:BBBBBB:BBBBBBBB and timestamp 2025-01-01 01:00:00 UTC.\\",\\"documentation_url\\":\\"https://docs.github.com/rest/overview/rate-limits-for-the-rest-api\\",\\"status\\":\\"403\\"}" + ${posixInput} = Catenate + ... "${PSQL_EXE}" -c "${inputStr}" + ${windowsInput} = Catenate + ... & ${posixInput} + ${input} = Set Variable If "${IS_WINDOWS}" == "1" ${windowsInput} ${posixInput} + ${shellExe} = Set Variable If "${IS_WINDOWS}" == "1" powershell sh + ${result} = Run Process + ... ${shellExe} \-c ${input} + ... stdout=${CURDIR}/tmp/PG-Session-Debug-On-Unencrypted-Server-Behaviour-Canonical.tmp + ... stderr=${CURDIR}/tmp/PG-Session-Debug-On-Unencrypted-Server-Behaviour-Canonical-stderr.tmp + Log STDOUT = "${result.stdout}" + Log STDERR = "${result.stderr}" + # Should Contain ${result.stdout} ${outputStr} collapse_spaces=True + Should Contain ${result.stderr} ${outputErrStr} collapse_spaces=True + [Teardown] Run Keywords Remove Environment Variable PGHOST + ... AND Remove Environment Variable PGPORT + ... AND Remove Environment Variable PGUSER + ... AND Remove Environment Variable PGPASSWORD + +Rust Testing Client Positive Control Notice Messages from GitHub + [Documentation] The unencrypted servers also have debug on + Set Environment Variable PGHOST ${PSQL_CLIENT_HOST} + Set Environment Variable PGPORT ${PG_SRV_PORT_UNENCRYPTED} + Set Environment Variable PGUSER stackql + Set Environment Variable PGPASSWORD ${PSQL_PASSWORD} + ${inputStr} = Catenate + ... SELECT repo, count(1) as has_starred + ... FROM github.activity.repo_stargazers + ... WHERE owner = 'sillyorg' and repo in ('silly', 'silly-but-more') and login = 'sillylogin' + ... GROUP BY repo; + ${posixInput} = Catenate + ... "${RUST_TESTING_EXE}" "${inputStr}" "host=localhost port=${PG_SRV_PORT_UNENCRYPTED}" + ${windowsInput} = Catenate + ... & ${posixInput} + ${input} = Set Variable If "${IS_WINDOWS}" == "1" ${windowsInput} ${posixInput} + ${shellExe} = Set Variable If "${IS_WINDOWS}" == "1" powershell sh + ${outputErrStrFragment} = Catenate SEPARATOR=\n + ... http response status code: 403 + ${result} = Run Process + ... ${shellExe} \-c ${input} + ... stdout=${CURDIR}/tmp/Rust-Testing-Client-Positive-Control-Notice-Messages-from-GitHub.tmp + ... stderr=${CURDIR}/tmp/Rust-Testing-Client-Positive-Control-Notice-Messages-from-GitHub-stderr.tmp + Log STDOUT = "${result.stdout}" + Log STDERR = "${result.stderr}" + Should Be Equal as Strings ${result.rc} 0 + # Should Contain ${result.stderr} ${outputErrStrFragment} collapse_spaces=True + [Teardown] Run Keywords Remove Environment Variable PGHOST + ... AND Remove Environment Variable PGPORT + ... AND Remove Environment Variable PGUSER + ... AND Remove Environment Variable PGPASSWORD diff --git a/test/robot/functional/tmp/.gitignore b/test/robot/functional/tmp/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/test/robot/functional/tmp/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/test/robot/reports/.gitignore b/test/robot/reports/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/test/robot/reports/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore