diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..c4ebe008d --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.x86_64-pc-windows-msvc] +linker = "rust-lld.exe" diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index 59db1864a..9d9deeb84 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -3,16 +3,24 @@ name: 🐂📝 Continuous integration - Test Suite on: workflow_call: +env: + AWS_REGION: us-west-1 + AWS_ROLE: arn:aws:iam::213020545630:role/ci-test-integration-role + +permissions: + id-token: write + contents: read + jobs: - test: - name: Test Suite + rust-tests: + name: Rust Tests runs-on: ${{ matrix.os }} defaults: run: shell: ${{ matrix.shell }} strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-x64-8cpu-32ram-300ssd] include: - os: ubuntu-latest platform: linux @@ -20,19 +28,11 @@ jobs: - os: macos-latest platform: macos shell: bash -leo pipefail {0} - - os: windows-latest + - os: windows-x64-8cpu-32ram-300ssd platform: windows shell: powershell fail-fast: false - env: - AWS_REGION: us-west-1 - AWS_ROLE: arn:aws:iam::213020545630:role/ci-test-integration-role - - permissions: - id-token: write - contents: read - steps: - name: Free up disk space if: matrix.platform == 'linux' @@ -55,10 +55,10 @@ jobs: df -h - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@v6 with: role-to-assume: ${{ env.AWS_ROLE }} role-duration-seconds: 3600 @@ -68,20 +68,15 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@v1 with: rustflags: ${{ matrix.platform == 'windows' && '-D warnings -C link-arg=/DEBUG:NONE' || '-D warnings' }} + cache: true # This is the default--making it explicit. This uses Swatinem/rust-cache under the hood. + cache-all-crates: true + cache-workspace-crates: true - name: Install uv - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: python-version: "3.11" # Test against the earliest supported Python version - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - # Only cache builds from main. Otherwise our caches will be too big and start to churn. (10 GB limit in GitHub Actions) - # Other branches will still use the cache but won't save it. - save-if: ${{ github.ref == 'refs/heads/main' }} - workspaces: "." - - name: Install dependencies (macOS) if: matrix.platform == 'macos' run: | @@ -89,47 +84,10 @@ jobs: brew install redis pkg-config brew services start redis - # NOTE: This adds a lot of bloat and takes a long time to build. - # we have ffmpeg disabled by default in the CI pipeline - # Set PKG_CONFIG_PATH so pkg-config can find FFmpeg libraries - # brew install ffmpeg@7 imagemagick - # echo "PKG_CONFIG_PATH=$(brew --prefix ffmpeg@7)/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV - - # NOTE: This adds a lot of bloat to the Windows image and takes a long time to build. - # we have ffmpeg disabled by default in the CI pipeline - # - name: Install dependencies (Linux) - # if: matrix.platform == 'linux' - # run: | - # sudo apt-get update - # sudo apt install -y clang libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev pkg-config - - # - name: Install dependencies (Windows) - # if: matrix.platform == 'windows' - # run: | - # # Install clang (via llvm) and pkg-config via Chocolatey - # choco install -y llvm pkgconfiglite - - # # Install vcpkg for FFmpeg static development libraries (libavcodec, libavformat, libavutil, libavfilter, libavdevice) - # # Using x64-windows-static-md for static libraries with dynamic MSVC runtime (better compatibility) - # git clone https://github.com/microsoft/vcpkg.git C:\vcpkg - # cd C:\vcpkg - # .\bootstrap-vcpkg.bat - - # # Install FFmpeg (includes libavutil, libavcodec, libavformat, libavfilter, libavdevice) - # # FFmpeg installation provides libavutil.pc and other .pc files needed by pkg-config - # .\vcpkg.exe install ffmpeg:x64-windows-static-md - - # # Set VCPKG_ROOT for cargo to find FFmpeg libraries - # echo "VCPKG_ROOT=C:\vcpkg" >> $env:GITHUB_ENV - - # # Set PKG_CONFIG_PATH so pkg-config can find libavutil.pc and other FFmpeg .pc files - # # vcpkg installs pkg-config files in the installed//lib/pkgconfig directory - # # This must be set before cargo build runs, as ffmpeg-sys-next uses pkg-config during build - # $pkgConfigPath = "C:\vcpkg\installed\x64-windows-static-md\lib\pkgconfig" - # echo "PKG_CONFIG_PATH=$pkgConfigPath" >> $env:GITHUB_ENV - - name: Install nextest runner - uses: taiki-e/install-action@nextest + uses: taiki-e/install-action@v2 + with: + tool: nextest@0.9.114 # Oxen Rust Tests - name: Setup test directories (Linux/macOS) @@ -144,11 +102,7 @@ jobs: mkdir .\data\test\runs - name: Build oxen rust project - run: | - cargo build --workspace - env: - TEMP: ${{ matrix.platform == 'windows' && 'D:\a\_temp' || runner.temp }} - TMP: ${{ matrix.platform == 'windows' && 'D:\a\_temp' || runner.temp }} + run: cargo build --workspace - name: Setup oxen-server user (Linux/macOS) if: matrix.platform != 'windows' @@ -166,13 +120,13 @@ jobs: if: matrix.platform != 'windows' run: | ./target/debug/oxen-server start & - cargo nextest run --workspace --profile ci + cargo nextest run --workspace --profile ci --cargo-profile dev - name: Run oxen rust tests (Windows) if: matrix.platform == 'windows' run: | - Start-Process -FilePath "${{ github.workspace }}\target\debug\oxen-server.exe" -WindowStyle Hidden -ArgumentList "start" - cargo nextest run --workspace --profile ci + Start-Process ".\target\debug\oxen-server.exe" -ArgumentList "start" + cargo nextest run --workspace --profile ci --cargo-profile dev # CLI Tests - name: Run CLI tests (Linux/macOS) @@ -187,31 +141,131 @@ jobs: - name: Run CLI tests (Windows) if: matrix.platform == 'windows' env: - PATH: ${{ env.PATH }};${{ github.workspace }}/target/debug + PATH: ${{ env.PATH }};${{ github.workspace }}\target\debug run: | cd ${{ github.workspace }}\cli-test uv sync uv run pytest tests -v - # Oxen Python Tests - - name: Copy binaries for Python tests (Linux/macOS) + python-tests: + name: Python Tests + runs-on: ${{ matrix.os }} + defaults: + run: + shell: ${{ matrix.shell }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-x64-8cpu-32ram-300ssd] + include: + - os: ubuntu-latest + platform: linux + shell: bash -leo pipefail {0} + - os: macos-latest + platform: macos + shell: bash -leo pipefail {0} + - os: windows-x64-8cpu-32ram-300ssd + platform: windows + shell: powershell + fail-fast: false + + steps: + - name: Free up disk space + if: matrix.platform == 'linux' + run: | + echo "=== Before cleanup ===" + df -h + + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo apt-get clean + + echo "=== After cleanup ===" + df -h + + - name: Checkout + uses: actions/checkout@v6 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_ROLE }} + role-duration-seconds: 3600 + aws-region: ${{ env.AWS_REGION }} + + - name: Setup Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: ${{ matrix.platform == 'windows' && '-D warnings -C link-arg=/DEBUG:NONE' || '-D warnings' }} + cache: true + cache-all-crates: true + cache-workspace-crates: true + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.11" + + - name: Install dependencies (macOS) + if: matrix.platform == 'macos' + run: | + brew update + brew install redis pkg-config + brew services start redis + + - name: Build oxen rust project + run: cargo build --workspace + + - name: Setup test directories (Linux/macOS) + if: matrix.platform != 'windows' + run: | + mkdir -p data/test/runs + mkdir -p /tmp/oxen_sync/ + + - name: Setup test directories (Windows) + if: matrix.platform == 'windows' + run: | + mkdir .\data\test\runs + + - name: Setup oxen-server user (Linux/macOS) + if: matrix.platform != 'windows' + run: | + ./target/debug/oxen-server add-user --email ox@oxen.ai --name Ox --output user_config.toml + cp user_config.toml data/test/config/user_config.toml + + - name: Setup oxen-server user (Windows) + if: matrix.platform == 'windows' + run: | + .\target\debug\oxen-server.exe add-user --email ox@oxen.ai --name Ox --output user_config.toml + copy user_config.toml data\test\config\user_config.toml + + - name: Start oxen-server (Linux/macOS) + if: matrix.platform != 'windows' + run: ./target/debug/oxen-server start & + + - name: Start oxen-server (Windows) + if: matrix.platform == 'windows' + run: Start-Process ".\target\debug\oxen-server.exe" -ArgumentList "start" + + - name: Configure oxen user (Linux/macOS) if: matrix.platform != 'windows' run: | cp ${{ github.workspace }}/target/debug/oxen ~/oxen - chmod +x ~/oxen ~/oxen + chmod +x ~/oxen + ~/oxen config --name "Bessie Testington" --email "bessie@yourcompany.com" - - name: Copy binaries for Python tests (Windows) + - name: Configure oxen user (Windows) if: matrix.platform == 'windows' run: | copy target\debug\oxen.exe ${{ github.workspace }}\oxen.exe + ${{ github.workspace }}\oxen.exe config --name "Bessie Testington" --email "bessie@yourcompany.com" - name: Run oxen-python tests (Linux/macOS) if: matrix.platform != 'windows' run: | cd ${{ github.workspace }}/oxen-python - - ~/oxen config --name "Bessie Testington" --email "bessie@yourcompany.com" - uv sync --no-install-project source .venv/bin/activate maturin develop @@ -221,9 +275,6 @@ jobs: if: matrix.platform == 'windows' run: | cd ${{ github.workspace }}\oxen-python - - ${{ github.workspace }}\oxen.exe config --name "Bessie Testington" --email "bessie@yourcompany.com" - uv sync --no-install-project dir .venv .venv\Scripts\Activate.ps1 diff --git a/bin/test-rust b/bin/test-rust index 915151b07..46c693bc9 100755 --- a/bin/test-rust +++ b/bin/test-rust @@ -53,6 +53,9 @@ cleanup() { echo "==> Removing test data..." rm -rf ./data/ox rm -rf ./data/test/runs + if [ -n "${OXEN_TEST_RUN_DIR:-}" ]; then + rm -rf "$OXEN_TEST_RUN_DIR" + fi else echo "==> Keeping test data in data/ox and data/test/runs" fi diff --git a/crates/lib/src/core/v_latest/workspaces/files.rs b/crates/lib/src/core/v_latest/workspaces/files.rs index e1d0cc37c..6e9f9b778 100644 --- a/crates/lib/src/core/v_latest/workspaces/files.rs +++ b/crates/lib/src/core/v_latest/workspaces/files.rs @@ -727,7 +727,7 @@ pub fn decompress_zip(zip_filepath: &PathBuf) -> Result, OxenError> // Get the canonical (absolute) path of the parent directory let parent = match zip_filepath.parent() { - Some(p) => p.canonicalize()?, + Some(p) => crate::util::fs::canonicalize(p)?, None => std::env::current_dir()?, }; diff --git a/crates/lib/src/test.rs b/crates/lib/src/test.rs index a6a1dc9ec..84b3fe767 100644 --- a/crates/lib/src/test.rs +++ b/crates/lib/src/test.rs @@ -45,7 +45,10 @@ pub static REPO_ROOT: LazyLock = LazyLock::new(|| { pub static TEST_DATA_DIR: LazyLock = LazyLock::new(|| REPO_ROOT.join("data")); pub fn test_run_dir() -> PathBuf { - TEST_DATA_DIR.join("test").join("runs") + match std::env::var("OXEN_TEST_RUN_DIR") { + Ok(dir) => PathBuf::from(dir), + Err(_) => TEST_DATA_DIR.join("test").join("runs"), + } } pub fn test_host() -> String { diff --git a/crates/oxen-py/Cargo.toml b/crates/oxen-py/Cargo.toml index 05ddd7c51..8df78eeca 100644 --- a/crates/oxen-py/Cargo.toml +++ b/crates/oxen-py/Cargo.toml @@ -28,5 +28,5 @@ uuid = { workspace = true } [build-dependencies] bindgen = { version = "0.71.1", default-features = false, features = ["runtime"] } cc = { version = "1.0", features = ["parallel"] } -glob = "0.3" +glob = { workspace = true } pkg-config = { version = "0.3", optional = true } diff --git a/crates/server/src/test.rs b/crates/server/src/test.rs index 9f8f9ccff..094228776 100644 --- a/crates/server/src/test.rs +++ b/crates/server/src/test.rs @@ -16,8 +16,7 @@ use liboxen::test::init_test_env; pub fn get_sync_dir() -> Result { init_test_env(); - let sync_dir = - liboxen::test::REPO_ROOT.join(format!("data/test/runs/{}", uuid::Uuid::new_v4())); + let sync_dir = liboxen::test::test_run_dir().join(uuid::Uuid::new_v4().to_string()); util::fs::create_dir_all(&sync_dir)?; Ok(sync_dir) }