diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index bd97bc42c..bc561b0d1 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -4,7 +4,9 @@ # Nightly E2E tests: # # cloud-e2e Cloud inference (NVIDIA Endpoint API) on ubuntu-latest. -# cloud-experimental-e2e Experimental cloud inference test. +# cloud-experimental-e2e Experimental cloud inference test (main script skips embedded +# check-docs + final cleanup; follow-up steps run check-docs, +# skip/05-network-policy.sh, then cleanup.sh --verify with if: always()). # gpu-e2e Local Ollama inference on a GPU self-hosted runner. # Controlled by the GPU_E2E_ENABLED repository variable. # Set vars.GPU_E2E_ENABLED to "true" in repo settings to enable. @@ -13,7 +15,11 @@ # Runs directly on the runner (not inside Docker) because OpenShell bootstraps # a K3s cluster inside a privileged Docker container — nesting would break networking. # -# Requires NVIDIA_API_KEY repository secret (for cloud-e2e and cloud-experimental-e2e). +# NVIDIA_API_KEY for cloud-e2e and cloud-experimental-e2e: +# - Repository secret: Settings → Secrets and variables → Actions → Repository secrets. +# - Environment secret: only available if the job sets `environment: `. +# (Storing the key under Environments / NVIDIA_API_KEY without `environment:` here leaves the +# variable empty in the job — repository secrets and environment secrets are separate.) # Only runs on schedule and manual dispatch — never on PRs (secret protection). name: nightly-e2e @@ -59,12 +65,14 @@ jobs: cloud-experimental-e2e: if: github.repository == 'NVIDIA/NemoClaw' runs-on: ubuntu-latest - environment: NVIDIA_API_KEY - timeout-minutes: 45 + # Main suite + check-docs + network-policy skip script can exceed 45m on cold runners. + timeout-minutes: 90 steps: - name: Checkout uses: actions/checkout@v6 + # Split Phase 5f (check-docs) and Phase 6 (cleanup) out of the main script so CI shows + # failures in dedicated steps; tear-down always runs last (if: always()). - name: Run cloud-experimental E2E test env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} @@ -75,14 +83,80 @@ jobs: NEMOCLAW_RECREATE_SANDBOX: "1" NEMOCLAW_POLICY_MODE: "custom" NEMOCLAW_POLICY_PRESETS: "npm,pypi" + RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_FINAL_CLEANUP: "1" + RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_CHECK_DOCS: "1" run: bash test/e2e/test-e2e-cloud-experimental.sh + - name: Documentation checks (check-docs.sh) + if: always() + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi + bash test/e2e/e2e-cloud-experimental/check-docs.sh + + - name: Network policy checks (skip/05-network-policy.sh) + if: always() + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + GITHUB_TOKEN: ${{ github.token }} + SANDBOX_NAME: e2e-cloud-experimental + NEMOCLAW_SANDBOX_NAME: e2e-cloud-experimental + run: | + set -euo pipefail + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi + bash test/e2e/e2e-cloud-experimental/skip/05-network-policy.sh + + - name: Tear down cloud-experimental sandbox (always) + if: always() + env: + SANDBOX_NAME: e2e-cloud-experimental + NEMOCLAW_SANDBOX_NAME: e2e-cloud-experimental + run: | + set -euo pipefail + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi + bash test/e2e/e2e-cloud-experimental/cleanup.sh --verify + - name: Upload install log on failure if: failure() uses: actions/upload-artifact@v4 with: name: install-log-cloud-experimental - path: /tmp/nemoclaw-e2e-install.log + path: /tmp/nemoclaw-e2e-cloud-experimental-install.log if-no-files-found: ignore # ── GPU E2E (Ollama local inference) ────────────────────────── diff --git a/test/e2e/e2e-cloud-experimental/check-docs.sh b/test/e2e/e2e-cloud-experimental/check-docs.sh index 67888c573..973476a44 100755 --- a/test/e2e/e2e-cloud-experimental/check-docs.sh +++ b/test/e2e/e2e-cloud-experimental/check-docs.sh @@ -17,6 +17,8 @@ # Environment: # CHECK_DOC_LINKS_REMOTE If 0, skip http(s) probes for links check. # CHECK_DOC_LINKS_VERBOSE If 1, log each URL during curl (same as --verbose). +# CHECK_DOC_LINKS_IGNORE_EXTRA Comma-separated extra http(s) URLs to skip curling (exact match, #fragment ignored). +# CHECK_DOC_LINKS_IGNORE_URL_REGEX If set, skip curl when the whole URL matches this ERE (bash [[ =~ ]]). # NODE Node for CLI check (default: node). # CURL curl binary (default: curl). @@ -51,7 +53,8 @@ Options: --verbose Log each URL while curling (link check). -h, --help Show this help. -Environment: CHECK_DOC_LINKS_REMOTE, CHECK_DOC_LINKS_VERBOSE, NODE, CURL. +Environment: CHECK_DOC_LINKS_REMOTE, CHECK_DOC_LINKS_VERBOSE, CHECK_DOC_LINKS_IGNORE_EXTRA, + CHECK_DOC_LINKS_IGNORE_URL_REGEX, NODE, CURL. EOF } @@ -268,6 +271,54 @@ check_remote_url() { return 0 } +# Normalized form: strip #fragment and trailing slash for ignore-list comparison. +normalize_url_for_ignore_match() { + local u="$1" + u="${u%%\#*}" + u="${u%/}" + printf '%s' "$u" +} + +# Built-in skip list: pages that often fail in CI (bot wall, redirects, or flaky) but are non-critical for doc correctness. +check_docs_default_ignored_urls() { + printf '%s\n' \ + 'https://github.com/NVIDIA/NemoClaw/commits/main' \ + 'https://github.com/NVIDIA/NemoClaw/pulls?q=is%3Apr+is%3Amerged' \ + 'https://github.com/NVIDIA/NemoClaw/pulls?q=is:pr+is:merged' \ + 'https://github.com/openclaw/openclaw/issues/49950' +} + +url_should_skip_remote_probe() { + local url="$1" + local nu ign _re + nu="$(normalize_url_for_ignore_match "$url")" + + while IFS= read -r ign || [[ -n "${ign:-}" ]]; do + [[ -z "${ign:-}" ]] && continue + [[ "$(normalize_url_for_ignore_match "$ign")" == "$nu" ]] && return 0 + done < <(check_docs_default_ignored_urls) + + if [[ -n "${CHECK_DOC_LINKS_IGNORE_EXTRA:-}" ]]; then + local -a _extra_parts=() + local IFS=',' + read -ra _extra_parts <<<"${CHECK_DOC_LINKS_IGNORE_EXTRA}" + unset IFS + for ign in "${_extra_parts[@]}"; do + ign="${ign#"${ign%%[![:space:]]*}"}" + ign="${ign%"${ign##*[![:space:]]}"}" + [[ -z "$ign" ]] && continue + [[ "$(normalize_url_for_ignore_match "$ign")" == "$nu" ]] && return 0 + done + fi + + if [[ -n "${CHECK_DOC_LINKS_IGNORE_URL_REGEX:-}" ]]; then + _re="${CHECK_DOC_LINKS_IGNORE_URL_REGEX}" + [[ "$url" =~ $_re ]] && return 0 + fi + + return 1 +} + run_links_check() { local -a DOC_FILES if [[ ${#EXTRA_FILES[@]} -gt 0 ]]; then @@ -293,6 +344,7 @@ run_links_check() { fi if [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]]; then log "[links] remote: curl unique http(s) targets (disable: CHECK_DOC_LINKS_REMOTE=0 or --local-only)" + log "[links] remote: built-in skip list for flaky/GitHub pages (override: CHECK_DOC_LINKS_IGNORE_EXTRA, CHECK_DOC_LINKS_IGNORE_URL_REGEX)" else log "[links] remote: skipped (local paths only)" fi @@ -356,18 +408,33 @@ run_links_check() { if [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]]; then if [[ -n "$_deduped" ]]; then - log "[links] phase 2/2: curl ${_unique} URL(s) (GET, -L, fail 4xx/5xx)" + local _probe_list="" _skip_count=0 _probe_n=0 + while IFS= read -r url || [[ -n "${url:-}" ]]; do + [[ -z "${url:-}" ]] && continue + if url_should_skip_remote_probe "$url"; then + log "[links] skipped (ignore list): ${url}" + _skip_count=$((_skip_count + 1)) + else + _probe_list+="${url}"$'\n' + fi + done <<<"$_deduped" + _probe_n="$(printf '%s\n' "$_probe_list" | grep -c . || true)" + if [[ "$_skip_count" -gt 0 ]]; then + log "[links] phase 2/2: curl ${_probe_n} URL(s), ${_skip_count} skipped (GET, -L, fail 4xx/5xx)" + else + log "[links] phase 2/2: curl ${_probe_n} URL(s) (GET, -L, fail 4xx/5xx)" + fi _i=0 - while IFS= read -r url || [[ -n "$url" ]]; do - [[ -z "$url" ]] && continue + while IFS= read -r url || [[ -n "${url:-}" ]]; do + [[ -z "${url:-}" ]] && continue _i=$((_i + 1)) if [[ "$VERBOSE" -eq 1 ]]; then - log "[links] [${_i}/${_unique}] ${url}" + log "[links] [${_i}/${_probe_n}] ${url}" fi if ! check_remote_url "$url"; then failures=1 fi - done <<<"$_deduped" + done <<<"$_probe_list" else log "[links] phase 2/2: no http(s) links" fi @@ -384,7 +451,7 @@ run_links_check() { return 1 fi if [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]] && [[ ${_unique:-0} -gt 0 ]]; then - log "[links] phase 2 OK (${_unique} URL(s))" + log "[links] phase 2 OK (${_unique} unique http(s); probed those not in ignore list)" fi log "[links] summary: ${#DOC_FILES[@]} file(s), local OK$( [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]] && [[ ${_unique:-0} -gt 0 ]] && printf ', %s remote OK' "${_unique}" diff --git a/test/e2e/e2e-cloud-experimental/cleanup.sh b/test/e2e/e2e-cloud-experimental/cleanup.sh new file mode 100755 index 000000000..59348e377 --- /dev/null +++ b/test/e2e/e2e-cloud-experimental/cleanup.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared teardown for e2e-cloud-experimental (extracted from test-e2e-cloud-experimental.sh Phase 0 + Phase 6). +# +# Destroys nemoclaw sandbox, OpenShell sandbox, port 18789 forward, and nemoclaw gateway. +# +# Usage: +# SANDBOX_NAME=my-sbx bash test/e2e/e2e-cloud-experimental/cleanup.sh +# SANDBOX_NAME=my-sbx bash test/e2e/e2e-cloud-experimental/cleanup.sh --verify +# +# Environment: +# SANDBOX_NAME or NEMOCLAW_SANDBOX_NAME — default: e2e-cloud-experimental +# +# Modes: +# (default) — destroy only (best-effort; always exits 0) +# --verify — destroy then assert sandbox is gone from openshell get + nemoclaw list (exits 1 on failure) + +set -uo pipefail + +pass() { printf '\033[32m PASS: %s\033[0m\n' "$1"; } +fail() { printf '\033[31m FAIL: %s\033[0m\n' "$1"; } +skip() { printf '\033[33m SKIP: %s\033[0m\n' "$1"; } +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-${SANDBOX_NAME:-e2e-cloud-experimental}}" +VERIFY=0 +if [ "${1:-}" = "--verify" ]; then + VERIFY=1 +fi + +info "e2e-cloud-experimental cleanup: sandbox='${SANDBOX_NAME}' (verify=${VERIFY})" + +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell forward stop 18789 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi + +if [ "$VERIFY" != "1" ]; then + pass "Cleanup destroy complete (no --verify)" + exit 0 +fi + +# ── Post-teardown checks (Phase 6 parity) ── +if command -v openshell >/dev/null 2>&1; then + if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + fail "openshell sandbox get '${SANDBOX_NAME}' still succeeds after cleanup" + exit 1 + fi + pass "openshell: sandbox '${SANDBOX_NAME}' no longer visible to sandbox get" +else + skip "openshell not on PATH — skipped sandbox get check after cleanup" +fi + +if command -v nemoclaw >/dev/null 2>&1; then + set +e + list_out=$(nemoclaw list 2>&1) + list_rc=$? + set -uo pipefail + if [ "$list_rc" -eq 0 ]; then + if echo "$list_out" | grep -Fq " ${SANDBOX_NAME}"; then + fail "nemoclaw list still lists '${SANDBOX_NAME}' after destroy" + exit 1 + fi + pass "nemoclaw list: '${SANDBOX_NAME}' removed from registry" + else + skip "nemoclaw list failed after cleanup — could not verify registry (exit $list_rc)" + fi +else + skip "nemoclaw not on PATH — skipped list check after cleanup" +fi + +pass "Cleanup + verify complete" +exit 0 diff --git a/test/e2e/e2e-cloud-experimental/skip/01-onboard-completion.sh b/test/e2e/e2e-cloud-experimental/skip/01-onboard-completion.sh index 3947d4ded..9aab4bf17 100755 --- a/test/e2e/e2e-cloud-experimental/skip/01-onboard-completion.sh +++ b/test/e2e/e2e-cloud-experimental/skip/01-onboard-completion.sh @@ -16,7 +16,7 @@ # 5) OpenShell sees the sandbox: `openshell sandbox get ` succeeds. # 6) OpenShell list contains the sandbox name. # 7) `openclaw --help`, `openclaw agent --help`, and `openclaw skills list` succeed inside sandbox. -# 8) `openshell inference get` shows provider `nvidia-nim` and the expected model (VDR3 #12). +# 8) `openshell inference get` shows the expected provider (default nvidia-nim; VDR3 #12) and model. # # Requires: # nemoclaw, openshell, openclaw on PATH. @@ -24,15 +24,18 @@ # Env (optional — defaults match test-e2e-cloud-experimental.sh): # SANDBOX_NAME or NEMOCLAW_SANDBOX_NAME (default: e2e-cloud-experimental) # CLOUD_EXPERIMENTAL_MODEL (legacy: SCENARIO_A_MODEL, NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL, NEMOCLAW_SCENARIO_A_MODEL) +# CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER — substring matched in `openshell inference get` (default: nvidia-nim; e.g. ollama-local for local gateways) # # Example: -# bash test/e2e/e2e-cloud-experimental/checks/01-onboard-completion.sh +# bash test/e2e/e2e-cloud-experimental/skip/01-onboard-completion.sh # SANDBOX_NAME=my-box CLOUD_EXPERIMENTAL_MODEL=nvidia/nemotron-3-super-120b-a12b bash ... +# SANDBOX_NAME=test01 CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER=ollama-local CLOUD_EXPERIMENTAL_MODEL=nemotron-3-nano:30b bash ... set -euo pipefail SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-experimental}}" CLOUD_EXPERIMENTAL_MODEL="${CLOUD_EXPERIMENTAL_MODEL:-${SCENARIO_A_MODEL:-${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-${NEMOCLAW_SCENARIO_A_MODEL:-moonshotai/kimi-k2.5}}}}" +CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER="${CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER:-nvidia-nim}" die() { printf '%s\n' "01-onboard-completion: FAIL: $*" >&2 exit 1 @@ -142,8 +145,8 @@ inf_check=$(openshell inference get 2>&1) ig=$? set -e [ "$ig" -eq 0 ] || die "openshell inference get failed: ${inf_check:0:200}" -echo "$inf_check" | grep -qi "nvidia-nim" \ - || die "openshell inference get missing nvidia-nim provider. Output (first 500 chars): ${inf_check:0:500}" +echo "$inf_check" | grep -Fqi "$CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER" \ + || die "openshell inference get missing provider '${CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER}' (set CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER to match Gateway). Output (first 500 chars): ${inf_check:0:500}" if ! echo "$inf_check" | grep -Fq "$CLOUD_EXPERIMENTAL_MODEL"; then die "inference model mismatch: expected substring '${CLOUD_EXPERIMENTAL_MODEL}' (from CLOUD_EXPERIMENTAL_MODEL / NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL) inside 'openshell inference get', but it was not found. If the sandbox was onboarded with another model, export the same id for this check (e.g. NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL=nvidia/nemotron-3-super-120b-a12b). --- openshell inference get (first 800 chars) --- ${inf_check:0:800}" fi diff --git a/test/e2e/e2e-cloud-experimental/skip/04-nemoclaw-openshell-status-parity.sh b/test/e2e/e2e-cloud-experimental/skip/04-nemoclaw-openshell-status-parity.sh index 902e88968..488a8f515 100755 --- a/test/e2e/e2e-cloud-experimental/skip/04-nemoclaw-openshell-status-parity.sh +++ b/test/e2e/e2e-cloud-experimental/skip/04-nemoclaw-openshell-status-parity.sh @@ -8,7 +8,7 @@ # 1) openshell sandbox status --json → .state == running (nemoclaw plugin status path) # 2) else openshell sandbox list → row for name contains Ready (bin/lib/onboard.js isSandboxReady) # Inference model: prefer openshell inference get --json .model; else plain inference get -# (text) must contain nvidia-nim + CLOUD_EXPERIMENTAL_MODEL (same idea as 01-onboard-completion.sh). +# (text) must contain CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER + CLOUD_EXPERIMENTAL_MODEL (same idea as 01-onboard-completion.sh). # nemoclaw list model must match openshell model (JSON or CLOUD_EXPERIMENTAL_MODEL text path). # # Requires: node on PATH (for JSON + list parsing; same shell as post-install suite). @@ -17,6 +17,7 @@ set -euo pipefail SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-experimental}}" CLOUD_EXPERIMENTAL_MODEL="${CLOUD_EXPERIMENTAL_MODEL:-${SCENARIO_A_MODEL:-${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-${NEMOCLAW_SCENARIO_A_MODEL:-moonshotai/kimi-k2.5}}}}" +CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER="${CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER:-nvidia-nim}" export SANDBOX_NAME die() { @@ -81,8 +82,8 @@ if [ -z "$os_model" ]; then inf_rc=$? set -e [ "$inf_rc" -eq 0 ] || die "openshell inference get failed (exit $inf_rc): ${inf_raw:0:240}" - echo "$inf_raw" | grep -qi "nvidia-nim" \ - || die "openshell inference get (text) missing nvidia-nim. Output (first 500 chars): ${inf_raw:0:500}" + echo "$inf_raw" | grep -Fqi "$CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER" \ + || die "openshell inference get (text) missing provider '${CLOUD_EXPERIMENTAL_INFERENCE_PROVIDER}'. Output (first 500 chars): ${inf_raw:0:500}" if ! echo "$inf_raw" | grep -Fq "$CLOUD_EXPERIMENTAL_MODEL"; then die "inference model (text path): expected substring '${CLOUD_EXPERIMENTAL_MODEL}' in 'openshell inference get' (set NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL to match onboarded model). --- output (first 800 chars) --- ${inf_raw:0:800}" fi diff --git a/test/e2e/e2e-cloud-experimental/skip/05-network-policy.sh b/test/e2e/e2e-cloud-experimental/skip/05-network-policy.sh index 10031cd4a..f3e4965d2 100755 --- a/test/e2e/e2e-cloud-experimental/skip/05-network-policy.sh +++ b/test/e2e/e2e-cloud-experimental/skip/05-network-policy.sh @@ -6,13 +6,24 @@ # # A) Host: openshell policy get --full — Version header, network_policies, npm/pypi hosts # (expects NEMOCLAW_POLICY_MODE=custom + npm,pypi presets from suite defaults). -# B) Sandbox over SSH: whitelist HTTPS 2xx/3xx for github / pypi / npm registry; -# blocked probe on E2E_CLOUD_EXPERIMENTAL_EGRESS_BLOCKED_URL (legacy: SCENARIO_A_EGRESS_BLOCKED_URL). +# B) Sandbox over SSH: outlook / Docker Hub (optional curl, commented by default); pypi: venv + pip download; +# npm: npm ping + npm view; huggingface: venv + pip install huggingface_hub + hf|huggingface-cli download +# tiny public config.json (hub + CDN as allowed by preset). Then blocked URL probe. +# Default: curl uses sandbox HTTPS_PROXY / env (matches pip/npm when traffic goes via proxy). +# NEMOCLAW_E2E_CURL_NOPROXY=1: add curl --noproxy '*' (direct TLS; use if CONNECT via proxy returns 403). +# NEMOCLAW_E2E_SKIP_NETWORK_POLICY_HUGGINGFACE=1: skip venv + huggingface_hub + hf download (~5m); still runs pypi + npm + blocked probe. +# +# Vitest (same checks): NEMOCLAW_E2E_NETWORK_POLICY=1 npx vitest run --project network-policy-cli +# +# run_whitelist_egress / curl_exit_hint are optional (outlook/docker curl cases commented out below). +# shellcheck disable=SC2329 set -euo pipefail SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-experimental}}" BLOCKED_URL="${E2E_CLOUD_EXPERIMENTAL_EGRESS_BLOCKED_URL:-${SCENARIO_A_EGRESS_BLOCKED_URL:-https://example.com/}}" +USE_NOPROXY="${NEMOCLAW_E2E_CURL_NOPROXY:-0}" +SKIP_HUGGINGFACE="${NEMOCLAW_E2E_SKIP_NETWORK_POLICY_HUGGINGFACE:-1}" die() { printf '%s\n' "05-network-policy: FAIL: $*" >&2 @@ -25,7 +36,7 @@ curl_exit_hint() { 7) printf '%s' "curl 7 = failed to connect (blocked by policy, down, or wrong port)." ;; 28) printf '%s' "curl 28 = operation timed out (often policy drop or slow path)." ;; 35) printf '%s' "curl 35 = SSL connect error." ;; - 56) printf '%s' "curl 56 = network receive error (TLS reset, proxy/gateway closed connection, etc.)." ;; + 56) printf '%s' "curl 56 = network receive error (TLS reset, proxy CONNECT rejected, etc.)." ;; 60) printf '%s' "curl 60 = peer certificate cannot be authenticated." ;; *) printf '%s' "curl exit $1 — see \`man curl\` EXIT CODES." ;; esac @@ -59,16 +70,25 @@ printf '%s\n' "05-network-policy: policy-yaml OK" # ── B) Egress inside sandbox (SSH) ──────────────────────────────────── ssh_config="$(mktemp)" -wl_log="$(mktemp)" bl_log="$(mktemp)" -trap 'rm -f "$ssh_config" "$wl_log" "$bl_log"' EXIT +trap 'rm -f "$ssh_config" "$bl_log"' EXIT openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null \ || die "egress: openshell sandbox ssh-config failed for '${SANDBOX_NAME}'" TIMEOUT_CMD="" -command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 120" -command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 120" +TIMEOUT_CMD_LONG="" +if command -v timeout >/dev/null 2>&1; then + TIMEOUT_CMD="timeout 180" + TIMEOUT_CMD_LONG="timeout 300" +elif command -v gtimeout >/dev/null 2>&1; then + TIMEOUT_CMD="gtimeout 180" + TIMEOUT_CMD_LONG="gtimeout 300" +fi +if [[ -z "$TIMEOUT_CMD" ]]; then + printf '%s\n' "05-network-policy: WARN: no timeout/gtimeout on PATH — each SSH egress step may hang indefinitely (brew install coreutils for gtimeout)." >&2 + TIMEOUT_CMD_LONG="" +fi ssh_host="openshell-${SANDBOX_NAME}" ssh_base=(ssh -F "$ssh_config" @@ -78,55 +98,278 @@ ssh_base=(ssh -F "$ssh_config" -o LogLevel=ERROR ) -set +e -$TIMEOUT_CMD "${ssh_base[@]}" "$ssh_host" bash -s <<'REMOTE' >"$wl_log" 2>&1 +run_whitelist_egress() { + local case_name=$1 + local url=$2 + local wl_log + printf '%s\n' "05-network-policy: egress running: ${case_name} (curl ${url})" + wl_log=$(mktemp) + set +e + $TIMEOUT_CMD "${ssh_base[@]}" "$ssh_host" bash -s -- "$url" "$USE_NOPROXY" <<'REMOTE' >"$wl_log" 2>&1 set -uo pipefail -for url in https://github.com/ https://pypi.org/ https://registry.npmjs.org/; do - efile=$(mktemp) +url=$1 +np=$2 +efile=$(mktemp) +if [ "$np" = "1" ]; then + code=$(curl --noproxy '*' -sS -o /dev/null -w "%{http_code}" --max-time 60 "$url" 2>"$efile") +else code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 60 "$url" 2>"$efile") - cr=$? - err=$(head -c 800 "$efile" | tr '\n' ' ') - rm -f "$efile" - code=$(printf '%s' "$code" | tr -d '\r' | tail -n 1) - if [ "$cr" -ne 0 ]; then - echo "whitelist: curl transport error for ${url}" - echo " curl_exit=${cr}" - echo " http_code_written=${code:-}" - echo " curl_stderr=${err}" - exit "$cr" - fi - case "$code" in - 2??|3??) ;; - *) - echo "whitelist: unexpected HTTP status for ${url}" - echo " http_code=${code}" - exit 1 - ;; - esac -done +fi +cr=$? +err=$(head -c 800 "$efile" | tr '\n' ' ') +rm -f "$efile" +code=$(printf '%s' "$code" | tr -d '\r' | tail -n 1) +if [ "$cr" -ne 0 ]; then + echo "whitelist: curl transport error for ${url}" + echo " curl_exit=${cr}" + echo " http_code_written=${code:-}" + echo " curl_stderr=${err}" + exit "$cr" +fi +case "$code" in + 2??|3??) ;; + *) + echo "whitelist: unexpected HTTP status for ${url}" + echo " http_code=${code}" + exit 1 + ;; +esac exit 0 REMOTE -wl_rc=$? -set -e -if [ "$wl_rc" -ne 0 ]; then - hint=$(curl_exit_hint "$wl_rc") - die "egress whitelist (github / pypi / npm registry) failed. + local wl_rc=$? + set -e + if [ "$wl_rc" -ne 0 ]; then + hint=$(curl_exit_hint "$wl_rc") + tail_out=$(sed 's/^/ /' "$wl_log" | tail -n 60) + rm -f "$wl_log" + die "egress whitelist case '${case_name}' (${url}) failed. ssh/remote exit: ${wl_rc} hint: ${hint} --- output from sandbox (last 60 lines) --- -$(sed 's/^/ /' "$wl_log" | tail -n 60) +${tail_out} ---" + fi + rm -f "$wl_log" + printf '%s\n' "05-network-policy: egress whitelist OK (${case_name})" +} + +run_whitelist_pypi_via_venv() { + local case_name="pypi" + local wl_log + printf '%s\n' "05-network-policy: egress running: ${case_name} (venv + pip download)" + wl_log=$(mktemp) + set +e + $TIMEOUT_CMD "${ssh_base[@]}" "$ssh_host" bash -s -- "$USE_NOPROXY" <<'REMOTE' >"$wl_log" 2>&1 +set -uo pipefail +np=$1 +VENVD=$(mktemp -d) +PROBE_DL=$(mktemp -d) +cleanup() { rm -rf "$VENVD" "$PROBE_DL"; } +trap cleanup EXIT +if ! command -v python3 >/dev/null 2>&1; then + echo "pypi whitelist: python3 not on PATH" + exit 1 +fi +if ! python3 -m venv "$VENVD" 2>/dev/null; then + echo "pypi whitelist: python3 -m venv failed (need python3-venv / ensure-virtualenv package?)" + exit 1 +fi +# shellcheck disable=SC1091 +. "$VENVD/bin/activate" +if [ "$np" = "1" ]; then + export NO_PROXY='*' + unset HTTPS_PROXY https_proxy HTTP_PROXY http_proxy ALL_PROXY all_proxy || true fi +if ! python -m pip download --no-deps --disable-pip-version-check -d "$PROBE_DL" --timeout 90 idna==3.7; then + echo "pypi whitelist: pip download idna==3.7 from PyPI failed (egress / proxy / policy)" + exit 1 +fi +exit 0 +REMOTE + local wl_rc=$? + set -e + if [ "$wl_rc" -ne 0 ]; then + tail_out=$(sed 's/^/ /' "$wl_log" | tail -n 60) + rm -f "$wl_log" + die "egress whitelist case '${case_name}' (venv + pip download from PyPI) failed. + + ssh/remote exit: ${wl_rc} + + --- output from sandbox (last 60 lines) --- +${tail_out} + ---" + fi + rm -f "$wl_log" + printf '%s\n' "05-network-policy: egress whitelist OK (${case_name})" +} +run_whitelist_npm_via_cli() { + local case_name="npm registry" + local wl_log + printf '%s\n' "05-network-policy: egress running: ${case_name} (npm ping + npm view — lighter than pack/install)" + wl_log=$(mktemp) + set +e + $TIMEOUT_CMD "${ssh_base[@]}" "$ssh_host" bash -s -- "$USE_NOPROXY" <<'REMOTE' >"$wl_log" 2>&1 +set -uo pipefail +np=$1 +WORK=$(mktemp -d) +cleanup() { rm -rf "$WORK"; } +trap cleanup EXIT +cd "$WORK" +export CI=true +export NODE_NO_WARNINGS=1 +export npm_config_progress=false +export npm_config_loglevel=error +export npm_config_fetch_timeout=120000 +export npm_config_fetch_retries=2 +if [ "$np" = "1" ]; then + export NO_PROXY='*' + unset HTTPS_PROXY https_proxy HTTP_PROXY http_proxy ALL_PROXY all_proxy || true +fi +if ! command -v npm >/dev/null 2>&1; then + echo "npm whitelist: npm not on PATH" + exit 1 +fi +# npm ping: minimal registry round-trip (avoids tarball download / long hangs vs npm pack). +echo "npm whitelist: npm ping..." +if ! npm ping --silent 2>/dev/null; then + if ! npm ping; then + echo "npm whitelist: npm ping failed (egress / proxy / policy)" + exit 1 + fi +fi +echo "npm whitelist: npm view is-odd@3.0.1 (metadata)..." +if ! npm view is-odd@3.0.1 version --silent 2>/dev/null; then + if ! npm view is-odd@3.0.1 version; then + echo "npm whitelist: npm view is-odd@3.0.1 failed" + exit 1 + fi +fi +echo "npm whitelist: OK" +exit 0 +REMOTE + local wl_rc=$? + set -e + if [ "$wl_rc" -ne 0 ]; then + tail_out=$(sed 's/^/ /' "$wl_log" | tail -n 60) + rm -f "$wl_log" + die "egress whitelist case '${case_name}' (npm ping / npm view) failed. + + ssh/remote exit: ${wl_rc} + + --- output from sandbox (last 60 lines) --- +${tail_out} + ---" + fi + rm -f "$wl_log" + printf '%s\n' "05-network-policy: egress whitelist OK (${case_name})" +} + +run_whitelist_huggingface_via_cli() { + local case_name="huggingface" + local wl_log + local tcmd="${TIMEOUT_CMD_LONG:-$TIMEOUT_CMD}" + printf '%s\n' "05-network-policy: egress running: ${case_name} (venv + pip huggingface_hub + hf download tiny config.json — up to ~5m)" + wl_log=$(mktemp) + set +e + $tcmd "${ssh_base[@]}" "$ssh_host" bash -s -- "$USE_NOPROXY" <<'REMOTE' >"$wl_log" 2>&1 +set -uo pipefail +np=$1 +VENVD=$(mktemp -d) +DL=$(mktemp -d) +cleanup() { rm -rf "$VENVD" "$DL"; } +trap cleanup EXIT +export HF_HUB_DISABLE_PROGRESS_BARS=1 +export HF_HUB_DISABLE_TELEMETRY=1 +if ! command -v python3 >/dev/null 2>&1; then + echo "huggingface whitelist: python3 not on PATH" + exit 1 +fi +if ! python3 -m venv "$VENVD" 2>/dev/null; then + echo "huggingface whitelist: python3 -m venv failed (need python3-venv?)" + exit 1 +fi +# shellcheck disable=SC1091 +. "$VENVD/bin/activate" +if [ "$np" = "1" ]; then + export NO_PROXY='*' + unset HTTPS_PROXY https_proxy HTTP_PROXY http_proxy ALL_PROXY all_proxy || true +fi +echo "huggingface whitelist: pip install huggingface_hub..." +if ! python -m pip install --disable-pip-version-check --timeout 120 "huggingface_hub>=0.23.0,<1"; then + echo "huggingface whitelist: pip install huggingface_hub failed (PyPI / proxy / policy)" + exit 1 +fi +REPO="hf-internal-testing/tiny-random-bert" +echo "huggingface whitelist: download ${REPO} config.json..." +if command -v hf >/dev/null 2>&1; then + if ! hf download "$REPO" config.json --local-dir "$DL"; then + echo "huggingface whitelist: hf download failed" + exit 1 + fi +elif command -v huggingface-cli >/dev/null 2>&1; then + if ! huggingface-cli download "$REPO" config.json --local-dir "$DL"; then + echo "huggingface whitelist: huggingface-cli download failed" + exit 1 + fi +else + if ! python -c "from huggingface_hub import hf_hub_download; hf_hub_download(repo_id=\"${REPO}\", filename=\"config.json\", local_dir=\"${DL}\")"; then + echo "huggingface whitelist: hf_hub_download (python) failed" + exit 1 + fi +fi +if [ ! -f "$DL/config.json" ]; then + echo "huggingface whitelist: config.json not present under ${DL}" + exit 1 +fi +echo "huggingface whitelist: OK" +exit 0 +REMOTE + local wl_rc=$? + set -e + if [ "$wl_rc" -ne 0 ]; then + tail_out=$(sed 's/^/ /' "$wl_log" | tail -n 60) + rm -f "$wl_log" + die "egress whitelist case '${case_name}' (venv + huggingface_hub + hub download) failed. + + ssh/remote exit: ${wl_rc} + + --- output from sandbox (last 60 lines) --- +${tail_out} + ---" + fi + rm -f "$wl_log" + printf '%s\n' "05-network-policy: egress whitelist OK (${case_name})" +} + +# run_whitelist_egress "outlook" "https://outlook.com/" +# run_whitelist_egress "docker hub" "https://hub.docker.com/" +run_whitelist_pypi_via_venv +run_whitelist_npm_via_cli +if [[ "$SKIP_HUGGINGFACE" == "1" ]]; then + printf '%s\n' "05-network-policy: SKIP huggingface whitelist (NEMOCLAW_E2E_SKIP_NETWORK_POLICY_HUGGINGFACE=1)" +else + run_whitelist_huggingface_via_cli +fi + +printf '%s\n' "05-network-policy: egress running: blocked URL probe (${BLOCKED_URL})" set +e -$TIMEOUT_CMD "${ssh_base[@]}" "$ssh_host" bash -s -- "$BLOCKED_URL" <<'REMOTE' >"$bl_log" 2>&1 +$TIMEOUT_CMD "${ssh_base[@]}" "$ssh_host" bash -s -- "$BLOCKED_URL" "$USE_NOPROXY" <<'REMOTE' >"$bl_log" 2>&1 set -uo pipefail url=$1 -if curl -f -sS -o /dev/null --max-time 30 "$url"; then - echo "expected blocked URL to fail curl, but it succeeded" - exit 1 +np=$2 +if [ "$np" = "1" ]; then + if curl --noproxy '*' -f -sS -o /dev/null --max-time 30 "$url"; then + echo "expected blocked URL to fail curl, but it succeeded" + exit 1 + fi +else + if curl -f -sS -o /dev/null --max-time 30 "$url"; then + echo "expected blocked URL to fail curl, but it succeeded" + exit 1 + fi fi exit 0 REMOTE @@ -140,5 +383,9 @@ $(sed 's/^/ /' "$bl_log" | tail -n 40) ---" fi -printf '%s\n' "05-network-policy: OK (policy-yaml + whitelist + blocked URL)" +if [[ "$SKIP_HUGGINGFACE" == "1" ]]; then + printf '%s\n' "05-network-policy: OK (policy-yaml + pypi + npm + blocked URL; huggingface skipped)" +else + printf '%s\n' "05-network-policy: OK (policy-yaml + pypi + npm + huggingface whitelist + blocked URL)" +fi exit 0 diff --git a/test/e2e/test-e2e-cloud-experimental.sh b/test/e2e/test-e2e-cloud-experimental.sh index dd5fba943..2322b5b64 100755 --- a/test/e2e/test-e2e-cloud-experimental.sh +++ b/test/e2e/test-e2e-cloud-experimental.sh @@ -17,7 +17,16 @@ # Phase 5e: nemoclaw connect → openclaw tui → send message → repeated Ctrl+C → exit (requires `expect`; skipped if missing or RUN_E2E_CLOUD_EXPERIMENTAL_TUI=0). # Phase 5f: check-docs.sh (Markdown links + nemoclaw --help vs commands.md) before Phase 6; skip with RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_CHECK_DOCS=1. # Inherits CHECK_DOC_LINKS_REMOTE (check-docs.sh defaults to 1 — curl unique http(s) links); set CHECK_DOC_LINKS_REMOTE=0 to skip remote probes only. -# Phase 6: cleanup (skipped when RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5=1 or RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_FINAL_CLEANUP=1). +# Phase 6: cleanup via e2e-cloud-experimental/cleanup.sh --verify (skipped when RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5=1 or RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_FINAL_CLEANUP=1). +# Phase 0 pre-cleanup uses the same script without --verify. +# +# Phase tags (optional — slice the suite without editing the script): +# E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS — comma-separated whitelist; unset or "all" = run every phase (still subject to FROM_PHASE5 / SKIP_FINAL_CLEANUP / etc.). +# Example: phase3,phase5b,phase5f +# E2E_CLOUD_EXPERIMENTAL_SKIP_TAGS — comma-separated blacklist applied after ONLY (or alone when ONLY is unset). +# Example: phase0,phase6,phase5e +# Tag names: phase0 phase1 phase2 phase3 phase5 phase5b phase5c phase5d phase5e phase5f phase6 +# Sub-phases (5b–5f) are independent; skipping phase5 does not skip 5b. You must list each phase you want when using ONLY_TAGS. # VDR3 #14 (re-onboard / volume audit) not automated here. # # Optional (not run here): port-8080 onboard conflict — see test/e2e/test-port8080-conflict.sh @@ -59,6 +68,8 @@ # E2E_CLOUD_EXPERIMENTAL_INSTALL_LOG — Phase 3 install log path (default: /tmp/nemoclaw-e2e-cloud-experimental-install.log) # RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_CHECK_DOCS=1 — skip Phase 5f (check-docs.sh) # CHECK_DOC_LINKS_REMOTE=0 — Phase 5f: skip curling http(s) doc links only (default in check-docs.sh: remote checks on) +# E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS — see header: whitelist phases (e.g. phase5b,phase5c) +# E2E_CLOUD_EXPERIMENTAL_SKIP_TAGS — see header: blacklist phases (e.g. phase6,phase5e) # # Usage (Phases 0–1, 3 + cases + Phase 5b–5f + Phase 6 cleanup; Phase 2 skipped): # NVIDIA_API_KEY=nvapi-... bash test/e2e/test-e2e-cloud-experimental.sh @@ -104,6 +115,36 @@ section() { } info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +# Tag filter: ONLY_TAGS whitelist (unset or "all" = allow all); SKIP_TAGS always removes listed phases. +e2e_cloud_experimental_phase_enabled() { + local phase="$1" + local only="${E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS:-}" + local sk="${E2E_CLOUD_EXPERIMENTAL_SKIP_TAGS:-}" + local tok + local IFS=, + + for tok in $sk; do + tok="${tok// /}" + [ -z "$tok" ] && continue + if [ "$tok" = "$phase" ]; then + return 1 + fi + done + + if [ -z "$only" ] || [ "$only" = "all" ]; then + return 0 + fi + + for tok in $only; do + tok="${tok// /}" + [ -z "$tok" ] && continue + if [ "$tok" = "$phase" ]; then + return 0 + fi + done + return 1 +} + # Parse chat completion JSON — content, reasoning_content, or reasoning (e.g. moonshot/kimi via gateway) parse_chat_content() { python3 -c " @@ -190,20 +231,13 @@ fi # nemoclaw destroy clears ~/.nemoclaw/sandboxes.json; align with test-double-onboard.sh. section "Phase 0: Pre-cleanup" -if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ]; then +if ! e2e_cloud_experimental_phase_enabled phase0; then + skip "Phase 0: skipped by tag (phase0 — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ]; then skip "Phase 0: pre-cleanup skipped (RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5=1 — preserving sandbox '${SANDBOX_NAME}')" else info "Destroying leftover sandbox, forwards, and gateway for '${SANDBOX_NAME}'..." - - if command -v nemoclaw >/dev/null 2>&1; then - nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true - fi - if command -v openshell >/dev/null 2>&1; then - openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true - openshell forward stop 18789 2>/dev/null || true - openshell gateway destroy -g nemoclaw 2>/dev/null || true - fi - + SANDBOX_NAME="$SANDBOX_NAME" bash "${E2E_DIR}/e2e-cloud-experimental/cleanup.sh" pass "Pre-cleanup complete" fi @@ -214,49 +248,51 @@ fi # NEMOCLAW_NON_INTERACTIVE=1 for automated path; optional: assert Linux + Docker CE. section "Phase 1: Prerequisites" -if docker info >/dev/null 2>&1; then +if ! e2e_cloud_experimental_phase_enabled phase1; then + skip "Phase 1: skipped by tag (phase1 — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +elif docker info >/dev/null 2>&1; then pass "Docker is running" -else - fail "Docker is not running — cannot continue" - exit 1 -fi -if [ -n "${NVIDIA_API_KEY:-}" ] && [[ "${NVIDIA_API_KEY}" == nvapi-* ]]; then - pass "NVIDIA_API_KEY is set (starts with nvapi-)" -else - fail "NVIDIA_API_KEY not set or invalid — required for e2e-cloud-experimental (Cloud API)" - exit 1 -fi + if [ -n "${NVIDIA_API_KEY:-}" ] && [[ "${NVIDIA_API_KEY}" == nvapi-* ]]; then + pass "NVIDIA_API_KEY is set (starts with nvapi-)" + else + fail "NVIDIA_API_KEY not set or invalid — required for e2e-cloud-experimental (Cloud API)" + exit 1 + fi -if curl -sf --max-time 10 https://integrate.api.nvidia.com/v1/models >/dev/null 2>&1; then - pass "Network access to integrate.api.nvidia.com" -else - fail "Cannot reach integrate.api.nvidia.com" - exit 1 -fi + if curl -sf --max-time 10 https://integrate.api.nvidia.com/v1/models >/dev/null 2>&1; then + pass "Network access to integrate.api.nvidia.com" + else + fail "Cannot reach integrate.api.nvidia.com" + exit 1 + fi -if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ]; then - pass "Phase 1: FROM_PHASE5 mode (NEMOCLAW_NON_INTERACTIVE not required)" -elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL:-1}" = "1" ]; then - pass "Phase 1: interactive install mode (NEMOCLAW_NON_INTERACTIVE not required on host)" -elif [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then - fail "NEMOCLAW_NON_INTERACTIVE=1 is required when RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL=0 (or use default interactive install, or RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5=1)" - exit 1 -else - pass "NEMOCLAW_NON_INTERACTIVE=1" -fi + if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ]; then + pass "Phase 1: FROM_PHASE5 mode (NEMOCLAW_NON_INTERACTIVE not required)" + elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL:-1}" = "1" ]; then + pass "Phase 1: interactive install mode (NEMOCLAW_NON_INTERACTIVE not required on host)" + elif [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required when RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL=0 (or use default interactive install, or RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5=1)" + exit 1 + else + pass "NEMOCLAW_NON_INTERACTIVE=1" + fi -# Nominal scenario: Ubuntu + Docker (Linux + Docker in README). Others may still run; do not hard-fail on macOS. -if [[ "$(uname -s)" == "Linux" ]]; then - pass "Host OS is Linux (nominal for e2e-cloud-experimental / README)" -else - skip "Host is not Linux — e2e-cloud-experimental nominally targets Ubuntu (continuing)" -fi + # Nominal scenario: Ubuntu + Docker (Linux + Docker in README). Others may still run; do not hard-fail on macOS. + if [[ "$(uname -s)" == "Linux" ]]; then + pass "Host OS is Linux (nominal for e2e-cloud-experimental / README)" + else + skip "Host is not Linux — e2e-cloud-experimental nominally targets Ubuntu (continuing)" + fi -if srv_ver=$(docker version -f '{{.Server.Version}}' 2>/dev/null) && [ -n "$srv_ver" ]; then - pass "Docker server version reported (${srv_ver})" + if srv_ver=$(docker version -f '{{.Server.Version}}' 2>/dev/null) && [ -n "$srv_ver" ]; then + pass "Docker server version reported (${srv_ver})" + else + skip "Could not read docker server version from docker version" + fi else - skip "Could not read docker server version from docker version" + fail "Docker is not running — cannot continue" + exit 1 fi # ══════════════════════════════════════════════════════════════════════ @@ -264,7 +300,12 @@ fi # ══════════════════════════════════════════════════════════════════════ # Deferred by request — not part of e2e-cloud-experimental for now. section "Phase 2: Doc review (README prerequisites) — skipped" -skip "Phase 2: doc review (VDR3 #11) — not required for now" + +if ! e2e_cloud_experimental_phase_enabled phase2; then + skip "Phase 2: skipped by tag (phase2 — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +else + skip "Phase 2: doc review (VDR3 #11) — not required for now" +fi # ══════════════════════════════════════════════════════════════════════ # Phase 3: Install + PATH (VDR3 #7, #10) @@ -274,44 +315,45 @@ skip "Phase 2: doc review (VDR3 #11) — not required for now" # nemoclaw onboard — no second onboard pass needed. section "Phase 3: Install and PATH" -cd "$REPO" || { +if ! e2e_cloud_experimental_phase_enabled phase3; then + skip "Phase 3: skipped by tag (phase3 — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +elif ! cd "$REPO"; then fail "Could not cd to repo root: $REPO" exit 1 -} - -export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" -export NEMOCLAW_EXPERIMENTAL=1 -export NEMOCLAW_PROVIDER=cloud -export NEMOCLAW_MODEL="$CLOUD_EXPERIMENTAL_MODEL" -export NEMOCLAW_POLICY_MODE="${NEMOCLAW_POLICY_MODE:-custom}" -export NEMOCLAW_POLICY_PRESETS="${NEMOCLAW_POLICY_PRESETS:-npm,pypi}" +else + export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" + export NEMOCLAW_EXPERIMENTAL=1 + export NEMOCLAW_PROVIDER=cloud + export NEMOCLAW_MODEL="$CLOUD_EXPERIMENTAL_MODEL" + export NEMOCLAW_POLICY_MODE="${NEMOCLAW_POLICY_MODE:-custom}" + export NEMOCLAW_POLICY_PRESETS="${NEMOCLAW_POLICY_PRESETS:-npm,pypi}" -NEMOCLAW_INSTALL_SCRIPT_URL="${NEMOCLAW_INSTALL_SCRIPT_URL:-https://www.nvidia.com/nemoclaw.sh}" -export NEMOCLAW_INSTALL_SCRIPT_URL + NEMOCLAW_INSTALL_SCRIPT_URL="${NEMOCLAW_INSTALL_SCRIPT_URL:-https://www.nvidia.com/nemoclaw.sh}" + export NEMOCLAW_INSTALL_SCRIPT_URL -# Override when running in Docker CI with a host-mounted log dir (see test/e2e/Dockerfile.cloud-experimental). -INSTALL_LOG="${E2E_CLOUD_EXPERIMENTAL_INSTALL_LOG:-/tmp/nemoclaw-e2e-cloud-experimental-install.log}" + # Override when running in Docker CI with a host-mounted log dir (see test/e2e/Dockerfile.cloud-experimental). + INSTALL_LOG="${E2E_CLOUD_EXPERIMENTAL_INSTALL_LOG:-/tmp/nemoclaw-e2e-cloud-experimental-install.log}" -if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ]; then - info "Phase 3: skipping curl|bash install (RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5=1)" - install_exit=0 -elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL:-1}" = "1" ]; then - if ! command -v expect >/dev/null 2>&1; then - fail "Phase 3: expect not on PATH (install expect, or set RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL=0 for non-interactive install)" - exit 1 - fi - export INTERACTIVE_SANDBOX_NAME="${INTERACTIVE_SANDBOX_NAME:-$SANDBOX_NAME}" - export INTERACTIVE_RECREATE_ANSWER="${INTERACTIVE_RECREATE_ANSWER:-n}" - export INTERACTIVE_INFERENCE_SEND="${INTERACTIVE_INFERENCE_SEND:-}" - export INTERACTIVE_MODEL_SEND="${INTERACTIVE_MODEL_SEND:-}" - export INTERACTIVE_PRESETS_SEND="${INTERACTIVE_PRESETS_SEND:-y}" - info "Phase 3: expect-driven interactive curl|bash (URL=${NEMOCLAW_INSTALL_SCRIPT_URL}, sandbox=${INTERACTIVE_SANDBOX_NAME})" - info "Output streams to this terminal AND ${INSTALL_LOG} (via tee) — first prompts may take several minutes after curl/Node install." - if [[ -z "${NVIDIA_API_KEY:-}" ]]; then - info "WARN: NVIDIA_API_KEY unset; expect will fail at API key prompt unless credentials exist on disk." - fi - set +e - expect <<'EXPECT' 2>&1 | tee "$INSTALL_LOG" + if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ]; then + info "Phase 3: skipping curl|bash install (RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5=1)" + install_exit=0 + elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL:-1}" = "1" ]; then + if ! command -v expect >/dev/null 2>&1; then + fail "Phase 3: expect not on PATH (install expect, or set RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL=0 for non-interactive install)" + exit 1 + fi + export INTERACTIVE_SANDBOX_NAME="${INTERACTIVE_SANDBOX_NAME:-$SANDBOX_NAME}" + export INTERACTIVE_RECREATE_ANSWER="${INTERACTIVE_RECREATE_ANSWER:-n}" + export INTERACTIVE_INFERENCE_SEND="${INTERACTIVE_INFERENCE_SEND:-}" + export INTERACTIVE_MODEL_SEND="${INTERACTIVE_MODEL_SEND:-}" + export INTERACTIVE_PRESETS_SEND="${INTERACTIVE_PRESETS_SEND:-y}" + info "Phase 3: expect-driven interactive curl|bash (URL=${NEMOCLAW_INSTALL_SCRIPT_URL}, sandbox=${INTERACTIVE_SANDBOX_NAME})" + info "Output streams to this terminal AND ${INSTALL_LOG} (via tee) — first prompts may take several minutes after curl/Node install." + if [[ -z "${NVIDIA_API_KEY:-}" ]]; then + info "WARN: NVIDIA_API_KEY unset; expect will fail at API key prompt unless credentials exist on disk." + fi + set +e + expect <<'EXPECT' 2>&1 | tee "$INSTALL_LOG" set timeout -1 if {![info exists env(NEMOCLAW_INSTALL_SCRIPT_URL)]} { @@ -384,75 +426,77 @@ expect { } } EXPECT - install_exit=${PIPESTATUS[0]} - set -uo pipefail -else - info "Running: curl -fsSL ${NEMOCLAW_INSTALL_SCRIPT_URL} | bash" - info "Onboard uses EXPERIMENTAL=1, PROVIDER=cloud, MODEL=${CLOUD_EXPERIMENTAL_MODEL} (override: NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL or legacy NEMOCLAW_SCENARIO_A_MODEL)." - info "Policy: NEMOCLAW_POLICY_MODE=${NEMOCLAW_POLICY_MODE} NEMOCLAW_POLICY_PRESETS=${NEMOCLAW_POLICY_PRESETS} (override env to change)." - info "Installs Node.js, openshell, NemoClaw, and runs onboard — may take several minutes." - - curl -fsSL "$NEMOCLAW_INSTALL_SCRIPT_URL" | bash >"$INSTALL_LOG" 2>&1 & - install_pid=$! - tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & - tail_pid=$! - wait "$install_pid" - install_exit=$? - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true -fi + install_exit=${PIPESTATUS[0]} + set -uo pipefail + else + info "Running: curl -fsSL ${NEMOCLAW_INSTALL_SCRIPT_URL} | bash" + info "Onboard uses EXPERIMENTAL=1, PROVIDER=cloud, MODEL=${CLOUD_EXPERIMENTAL_MODEL} (override: NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL or legacy NEMOCLAW_SCENARIO_A_MODEL)." + info "Policy: NEMOCLAW_POLICY_MODE=${NEMOCLAW_POLICY_MODE} NEMOCLAW_POLICY_PRESETS=${NEMOCLAW_POLICY_PRESETS} (override env to change)." + info "Installs Node.js, openshell, NemoClaw, and runs onboard — may take several minutes." + + curl -fsSL "$NEMOCLAW_INSTALL_SCRIPT_URL" | bash >"$INSTALL_LOG" 2>&1 & + install_pid=$! + tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & + tail_pid=$! + wait "$install_pid" + install_exit=$? + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + fi -if [ -f "$HOME/.bashrc" ]; then + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" # shellcheck source=/dev/null - source "$HOME/.bashrc" 2>/dev/null || true -fi -export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" -# shellcheck source=/dev/null -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" -if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then - export PATH="$HOME/.local/bin:$PATH" -fi + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi -if [ "$install_exit" -eq 0 ]; then - if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ]; then - pass "Phase 3: install skipped (FROM_PHASE5); using existing sandbox '${SANDBOX_NAME}'" - elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL:-1}" = "1" ]; then - pass "public install (expect interactive curl|bash) completed (exit 0)" + if [ "$install_exit" -eq 0 ]; then + if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ]; then + pass "Phase 3: install skipped (FROM_PHASE5); using existing sandbox '${SANDBOX_NAME}'" + elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL:-1}" = "1" ]; then + pass "public install (expect interactive curl|bash) completed (exit 0)" + else + pass "public install (curl nemoclaw.sh | bash) completed (exit 0)" + fi else - pass "public install (curl nemoclaw.sh | bash) completed (exit 0)" + if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL:-1}" = "1" ]; then + fail "public install (expect interactive curl|bash) failed (exit $install_exit)" + else + fail "public install (curl nemoclaw.sh | bash) failed (exit $install_exit)" + fi + exit 1 fi -else - if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL:-1}" = "1" ]; then - fail "public install (expect interactive curl|bash) failed (exit $install_exit)" + + if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH ($(command -v nemoclaw))" else - fail "public install (curl nemoclaw.sh | bash) failed (exit $install_exit)" + _e2e_path_ctx="after install" + [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ] && _e2e_path_ctx="required for Phase 5" + fail "nemoclaw not found on PATH (${_e2e_path_ctx})" + exit 1 fi - exit 1 -fi -if command -v nemoclaw >/dev/null 2>&1; then - pass "nemoclaw on PATH ($(command -v nemoclaw))" -else - _e2e_path_ctx="after install" - [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ] && _e2e_path_ctx="required for Phase 5" - fail "nemoclaw not found on PATH (${_e2e_path_ctx})" - exit 1 -fi + if command -v openshell >/dev/null 2>&1; then + pass "openshell on PATH ($(openshell --version 2>&1 || echo unknown))" + else + _e2e_path_ctx="after install" + [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ] && _e2e_path_ctx="required for Phase 5" + fail "openshell not found on PATH (${_e2e_path_ctx})" + exit 1 + fi -if command -v openshell >/dev/null 2>&1; then - pass "openshell on PATH ($(openshell --version 2>&1 || echo unknown))" -else - _e2e_path_ctx="after install" - [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ] && _e2e_path_ctx="required for Phase 5" - fail "openshell not found on PATH (${_e2e_path_ctx})" - exit 1 -fi + if nemoclaw --help >/dev/null 2>&1; then + pass "nemoclaw --help exits 0" + else + fail "nemoclaw --help failed" + exit 1 + fi -if nemoclaw --help >/dev/null 2>&1; then - pass "nemoclaw --help exits 0" -else - fail "nemoclaw --help failed" - exit 1 fi # ══════════════════════════════════════════════════════════════════════ @@ -461,29 +505,33 @@ fi # Ready scripts are sorted by filename; each must exit 0 on success. See e2e-cloud-experimental/README.md. section "Phase 5: Sandbox checks suite (then Phase 5b chat + Phase 5c skill smoke in this script)" -export SANDBOX_NAME CLOUD_EXPERIMENTAL_MODEL REPO NVIDIA_API_KEY +if ! e2e_cloud_experimental_phase_enabled phase5; then + skip "Phase 5: skipped by tag (phase5 — checks/*.sh only; 5b–5f use their own tags)" +else + export SANDBOX_NAME CLOUD_EXPERIMENTAL_MODEL REPO NVIDIA_API_KEY -shopt -s nullglob -case_scripts=("$E2E_CLOUD_EXPERIMENTAL_READY_DIR"/*.sh) -shopt -u nullglob + shopt -s nullglob + case_scripts=("$E2E_CLOUD_EXPERIMENTAL_READY_DIR"/*.sh) + shopt -u nullglob -if [ "${#case_scripts[@]}" -eq 0 ]; then - skip "No checks scripts in ${E2E_CLOUD_EXPERIMENTAL_READY_DIR} (add checks/*.sh)" -else - info "Checks directory: ${E2E_CLOUD_EXPERIMENTAL_READY_DIR} (${#case_scripts[@]} script(s))" - for case_script in "${case_scripts[@]}"; do - info "Running $(basename "$case_script")..." - set +e - bash "$case_script" - c_rc=$? - set -uo pipefail - if [ "$c_rc" -eq 0 ]; then - pass "case $(basename "$case_script" .sh)" - else - fail "case $(basename "$case_script" .sh) exited ${c_rc}" - exit 1 - fi - done + if [ "${#case_scripts[@]}" -eq 0 ]; then + skip "No checks scripts in ${E2E_CLOUD_EXPERIMENTAL_READY_DIR} (add checks/*.sh)" + else + info "Checks directory: ${E2E_CLOUD_EXPERIMENTAL_READY_DIR} (${#case_scripts[@]} script(s))" + for case_script in "${case_scripts[@]}"; do + info "Running $(basename "$case_script")..." + set +e + bash "$case_script" + c_rc=$? + set -uo pipefail + if [ "$c_rc" -eq 0 ]; then + pass "case $(basename "$case_script" .sh)" + else + fail "case $(basename "$case_script" .sh) exited ${c_rc}" + exit 1 + fi + done + fi fi # ══════════════════════════════════════════════════════════════════════ @@ -492,12 +540,15 @@ fi # Same path as test-full-e2e.sh 4b: sandbox → gateway → cloud; model from CLOUD_EXPERIMENTAL_MODEL. section "Phase 5b: Live chat (inference.local /v1/chat/completions)" -if ! command -v python3 >/dev/null 2>&1; then - fail "Phase 5b: python3 not on PATH (needed to parse chat response)" - exit 1 -fi +if ! e2e_cloud_experimental_phase_enabled phase5b; then + skip "Phase 5b: skipped by tag (phase5b — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +else + if ! command -v python3 >/dev/null 2>&1; then + fail "Phase 5b: python3 not on PATH (needed to parse chat response)" + exit 1 + fi -payload=$(CLOUD_EXPERIMENTAL_MODEL="$CLOUD_EXPERIMENTAL_MODEL" python3 -c " + payload=$(CLOUD_EXPERIMENTAL_MODEL="$CLOUD_EXPERIMENTAL_MODEL" python3 -c " import json, os print(json.dumps({ 'model': os.environ['CLOUD_EXPERIMENTAL_MODEL'], @@ -505,75 +556,77 @@ print(json.dumps({ 'max_tokens': 100, })) ") || { - fail "Phase 5b: could not build chat JSON payload" - exit 1 -} + fail "Phase 5b: could not build chat JSON payload" + exit 1 + } -PHASE_5B_MAX="${E2E_PHASE_5B_MAX_ATTEMPTS:-3}" -PHASE_5B_SLEEP="${E2E_PHASE_5B_RETRY_SLEEP_SEC:-5}" -# Clamp to at least 1 attempt -if ! [[ "$PHASE_5B_MAX" =~ ^[1-9][0-9]*$ ]]; then - PHASE_5B_MAX=3 -fi -info "POST chat completion inside sandbox (model ${CLOUD_EXPERIMENTAL_MODEL}, up to ${PHASE_5B_MAX} attempt(s), ${PHASE_5B_SLEEP}s between retries)..." + PHASE_5B_MAX="${E2E_PHASE_5B_MAX_ATTEMPTS:-3}" + PHASE_5B_SLEEP="${E2E_PHASE_5B_RETRY_SLEEP_SEC:-5}" + # Clamp to at least 1 attempt + if ! [[ "$PHASE_5B_MAX" =~ ^[1-9][0-9]*$ ]]; then + PHASE_5B_MAX=3 + fi + info "POST chat completion inside sandbox (model ${CLOUD_EXPERIMENTAL_MODEL}, up to ${PHASE_5B_MAX} attempt(s), ${PHASE_5B_SLEEP}s between retries)..." -CHAT_TIMEOUT_CMD="" -command -v timeout >/dev/null 2>&1 && CHAT_TIMEOUT_CMD="timeout 120" -command -v gtimeout >/dev/null 2>&1 && CHAT_TIMEOUT_CMD="gtimeout 120" + CHAT_TIMEOUT_CMD="" + command -v timeout >/dev/null 2>&1 && CHAT_TIMEOUT_CMD="timeout 120" + command -v gtimeout >/dev/null 2>&1 && CHAT_TIMEOUT_CMD="gtimeout 120" -ssh_config_chat="$(mktemp)" -if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config_chat" 2>/dev/null; then - rm -f "$ssh_config_chat" - fail "Phase 5b: openshell sandbox ssh-config failed for '${SANDBOX_NAME}'" - exit 1 -fi + ssh_config_chat="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config_chat" 2>/dev/null; then + rm -f "$ssh_config_chat" + fail "Phase 5b: openshell sandbox ssh-config failed for '${SANDBOX_NAME}'" + exit 1 + fi -phase_5b_attempt=1 -phase_5b_ok=0 -phase_5b_last_fail="" -while [ "$phase_5b_attempt" -le "$PHASE_5B_MAX" ]; do - set +e - sandbox_chat_out=$( - $CHAT_TIMEOUT_CMD ssh -F "$ssh_config_chat" \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=10 \ - -o LogLevel=ERROR \ - "openshell-${SANDBOX_NAME}" \ - "curl -sS --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $(printf '%q' "$payload")" \ - 2>&1 - ) - chat_ssh_rc=$? - set -uo pipefail + phase_5b_attempt=1 + phase_5b_ok=0 + phase_5b_last_fail="" + while [ "$phase_5b_attempt" -le "$PHASE_5B_MAX" ]; do + set +e + sandbox_chat_out=$( + $CHAT_TIMEOUT_CMD ssh -F "$ssh_config_chat" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -sS --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $(printf '%q' "$payload")" \ + 2>&1 + ) + chat_ssh_rc=$? + set -uo pipefail - if [ "$chat_ssh_rc" -ne 0 ]; then - phase_5b_last_fail="Phase 5b: ssh/curl failed (exit ${chat_ssh_rc}): ${sandbox_chat_out:0:400}" - elif [ -z "$sandbox_chat_out" ]; then - phase_5b_last_fail="Phase 5b: empty response from inference.local chat completions" - else - chat_text=$(printf '%s' "$sandbox_chat_out" | parse_chat_content 2>/dev/null) || chat_text="" - if echo "$chat_text" | grep -qi "PONG"; then - pass "Phase 5b: chat completion returned PONG (model ${CLOUD_EXPERIMENTAL_MODEL}, attempt ${phase_5b_attempt}/${PHASE_5B_MAX})" - phase_5b_ok=1 + if [ "$chat_ssh_rc" -ne 0 ]; then + phase_5b_last_fail="Phase 5b: ssh/curl failed (exit ${chat_ssh_rc}): ${sandbox_chat_out:0:400}" + elif [ -z "$sandbox_chat_out" ]; then + phase_5b_last_fail="Phase 5b: empty response from inference.local chat completions" + else + chat_text=$(printf '%s' "$sandbox_chat_out" | parse_chat_content 2>/dev/null) || chat_text="" + if echo "$chat_text" | grep -qi "PONG"; then + pass "Phase 5b: chat completion returned PONG (model ${CLOUD_EXPERIMENTAL_MODEL}, attempt ${phase_5b_attempt}/${PHASE_5B_MAX})" + phase_5b_ok=1 + break + fi + phase_5b_last_fail="Phase 5b: expected PONG in assistant text, got: ${chat_text:0:300} (raw: ${sandbox_chat_out:0:400})" + fi + + if [ "$phase_5b_attempt" -ge "$PHASE_5B_MAX" ]; then break fi - phase_5b_last_fail="Phase 5b: expected PONG in assistant text, got: ${chat_text:0:300} (raw: ${sandbox_chat_out:0:400})" - fi + info "Phase 5b: attempt ${phase_5b_attempt}/${PHASE_5B_MAX} failed — ${phase_5b_last_fail#Phase 5b: }" + info "Phase 5b: sleeping ${PHASE_5B_SLEEP}s before retry..." + sleep "$PHASE_5B_SLEEP" + phase_5b_attempt=$((phase_5b_attempt + 1)) + done - if [ "$phase_5b_attempt" -ge "$PHASE_5B_MAX" ]; then - break - fi - info "Phase 5b: attempt ${phase_5b_attempt}/${PHASE_5B_MAX} failed — ${phase_5b_last_fail#Phase 5b: }" - info "Phase 5b: sleeping ${PHASE_5B_SLEEP}s before retry..." - sleep "$PHASE_5B_SLEEP" - phase_5b_attempt=$((phase_5b_attempt + 1)) -done + rm -f "$ssh_config_chat" -rm -f "$ssh_config_chat" + if [ "$phase_5b_ok" -ne 1 ]; then + fail "$phase_5b_last_fail" + exit 1 + fi -if [ "$phase_5b_ok" -ne 1 ]; then - fail "$phase_5b_last_fail" - exit 1 fi # ══════════════════════════════════════════════════════════════════════ @@ -584,32 +637,37 @@ fi # skills subdir is optional (migration); absent → honest SKIP (not PASS). section "Phase 5c: Skill smoke (repo + sandbox OpenClaw)" -info "Validating repo .agents/skills (SKILL.md frontmatter + body)..." -if ! bash "$E2E_DIR/e2e-cloud-experimental/features/skill/lib/validate_repo_skills.sh" --repo "$REPO"; then - fail "Phase 5c: repo skill validation failed" - exit 1 -fi -pass "Phase 5c: repo agent skills (SKILL.md) valid" +if ! e2e_cloud_experimental_phase_enabled phase5c; then + skip "Phase 5c: skipped by tag (phase5c — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +else + info "Validating repo .agents/skills (SKILL.md frontmatter + body)..." + if ! bash "$E2E_DIR/e2e-cloud-experimental/features/skill/lib/validate_repo_skills.sh" --repo "$REPO"; then + fail "Phase 5c: repo skill validation failed" + exit 1 + fi + pass "Phase 5c: repo agent skills (SKILL.md) valid" -info "Checking /sandbox/.openclaw inside sandbox..." -set +e -sb_out=$(SANDBOX_NAME="$SANDBOX_NAME" bash "$E2E_DIR/e2e-cloud-experimental/features/skill/lib/validate_sandbox_openclaw_skills.sh" 2>/dev/null) -sb_rc=$? -set -uo pipefail + info "Checking /sandbox/.openclaw inside sandbox..." + set +e + sb_out=$(SANDBOX_NAME="$SANDBOX_NAME" bash "$E2E_DIR/e2e-cloud-experimental/features/skill/lib/validate_sandbox_openclaw_skills.sh" 2>/dev/null) + sb_rc=$? + set -uo pipefail -if [ "$sb_rc" -ne 0 ]; then - fail "Phase 5c: sandbox OpenClaw layout check failed (exit ${sb_rc}): ${sb_out:0:240}" - exit 1 -fi -pass "Phase 5c: sandbox /sandbox/.openclaw + openclaw.json OK" + if [ "$sb_rc" -ne 0 ]; then + fail "Phase 5c: sandbox OpenClaw layout check failed (exit ${sb_rc}): ${sb_out:0:240}" + exit 1 + fi + pass "Phase 5c: sandbox /sandbox/.openclaw + openclaw.json OK" + + if echo "$sb_out" | grep -q "SKILLS_SUBDIR=present"; then + pass "Phase 5c: sandbox /sandbox/.openclaw/skills present" + elif echo "$sb_out" | grep -q "SKILLS_SUBDIR=absent"; then + skip "Phase 5c: /sandbox/.openclaw/skills absent (host migration snapshot had no skills dir)" + else + fail "Phase 5c: unexpected sandbox check output: ${sb_out:0:240}" + exit 1 + fi -if echo "$sb_out" | grep -q "SKILLS_SUBDIR=present"; then - pass "Phase 5c: sandbox /sandbox/.openclaw/skills present" -elif echo "$sb_out" | grep -q "SKILLS_SUBDIR=absent"; then - skip "Phase 5c: /sandbox/.openclaw/skills absent (host migration snapshot had no skills dir)" -else - fail "Phase 5c: unexpected sandbox check output: ${sb_out:0:240}" - exit 1 fi # ══════════════════════════════════════════════════════════════════════ @@ -618,19 +676,24 @@ fi # Deploy managed skill fixture into sandbox and verify one agent turn returns token. section "Phase 5d: Skill agent verification (inject + token)" -info "Injecting skill-smoke-fixture into sandbox '${SANDBOX_NAME}'..." -if ! SANDBOX_NAME="$SANDBOX_NAME" SKILL_ID="skill-smoke-fixture" bash "$E2E_DIR/e2e-cloud-experimental/features/skill/add-sandbox-skill.sh"; then - fail "Phase 5d: failed to inject/query skill-smoke-fixture" - exit 1 -fi -pass "Phase 5d: skill-smoke-fixture injected and queryable" +if ! e2e_cloud_experimental_phase_enabled phase5d; then + skip "Phase 5d: skipped by tag (phase5d — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +else + info "Injecting skill-smoke-fixture into sandbox '${SANDBOX_NAME}'..." + if ! SANDBOX_NAME="$SANDBOX_NAME" SKILL_ID="skill-smoke-fixture" bash "$E2E_DIR/e2e-cloud-experimental/features/skill/add-sandbox-skill.sh"; then + fail "Phase 5d: failed to inject/query skill-smoke-fixture" + exit 1 + fi + pass "Phase 5d: skill-smoke-fixture injected and queryable" + + info "Running one openclaw agent turn to verify skill token..." + if ! NVIDIA_API_KEY="$NVIDIA_API_KEY" SANDBOX_NAME="$SANDBOX_NAME" SKILL_ID="skill-smoke-fixture" bash "$E2E_DIR/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh"; then + fail "Phase 5d: agent verification did not return skill token" + exit 1 + fi + pass "Phase 5d: agent returned SKILL_SMOKE_VERIFY_K9X2" -info "Running one openclaw agent turn to verify skill token..." -if ! NVIDIA_API_KEY="$NVIDIA_API_KEY" SANDBOX_NAME="$SANDBOX_NAME" SKILL_ID="skill-smoke-fixture" bash "$E2E_DIR/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh"; then - fail "Phase 5d: agent verification did not return skill token" - exit 1 fi -pass "Phase 5d: agent returned SKILL_SMOKE_VERIFY_K9X2" # ══════════════════════════════════════════════════════════════════════ # Phase 5e: OpenClaw TUI smoke (nemoclaw connect → tui → message → Ctrl+C → exit) @@ -639,7 +702,9 @@ pass "Phase 5d: agent returned SKILL_SMOKE_VERIFY_K9X2" # e2e-cloud-experimental/openclaw-tui-in-sandbox.sh (that wrapper uses `interact` for humans). section "Phase 5e: OpenClaw TUI smoke (expect)" -if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_TUI:-1}" = "0" ]; then +if ! e2e_cloud_experimental_phase_enabled phase5e; then + skip "Phase 5e: skipped by tag (phase5e — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_TUI:-1}" = "0" ]; then skip "Phase 5e: skipped (RUN_E2E_CLOUD_EXPERIMENTAL_TUI=0)" elif ! command -v expect >/dev/null 2>&1; then skip "Phase 5e: expect not on PATH — install expect to run TUI smoke (e.g. apt install expect)" @@ -796,7 +861,9 @@ fi # ══════════════════════════════════════════════════════════════════════ section "Phase 5f: Documentation checks (check-docs.sh)" -if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_CHECK_DOCS:-0}" = "1" ]; then +if ! e2e_cloud_experimental_phase_enabled phase5f; then + skip "Phase 5f: skipped by tag (phase5f — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_CHECK_DOCS:-0}" = "1" ]; then skip "Phase 5f: check-docs skipped (RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_CHECK_DOCS=1)" else info "check-docs.sh (default: curl unique http(s) links; CHECK_DOC_LINKS_REMOTE=0 to skip remote only)" @@ -814,50 +881,18 @@ fi # openshell sandbox delete + forward stop + gateway destroy. section "Phase 6: Final cleanup" -if [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ] && [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5_RUN_CLEANUP:-0}" != "1" ]; then +if ! e2e_cloud_experimental_phase_enabled phase6; then + skip "Phase 6: skipped by tag (phase6 — set E2E_CLOUD_EXPERIMENTAL_ONLY_TAGS / SKIP_TAGS)" +elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5:-0}" = "1" ] && [ "${RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5_RUN_CLEANUP:-0}" != "1" ]; then skip "Phase 6: final cleanup skipped (RUN_E2E_CLOUD_EXPERIMENTAL_FROM_PHASE5=1 — set FROM_PHASE5_RUN_CLEANUP=1 to destroy sandbox)" elif [ "${RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_FINAL_CLEANUP:-${RUN_SCENARIO_A_SKIP_FINAL_CLEANUP:-}}" = "1" ]; then skip "Phase 6: final cleanup skipped (RUN_E2E_CLOUD_EXPERIMENTAL_SKIP_FINAL_CLEANUP=1)" else info "Removing sandbox '${SANDBOX_NAME}', port forward, and nemoclaw gateway..." - - if command -v nemoclaw >/dev/null 2>&1; then - nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true - fi - if command -v openshell >/dev/null 2>&1; then - openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true - openshell forward stop 18789 2>/dev/null || true - openshell gateway destroy -g nemoclaw 2>/dev/null || true - fi - - if command -v openshell >/dev/null 2>&1; then - if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then - fail "openshell sandbox get '${SANDBOX_NAME}' still succeeds after cleanup" - exit 1 - fi - pass "openshell: sandbox '${SANDBOX_NAME}' no longer visible to sandbox get" - else - skip "openshell not on PATH — skipped sandbox get check after cleanup" - fi - - if command -v nemoclaw >/dev/null 2>&1; then - set +e - list_out=$(nemoclaw list 2>&1) - list_rc=$? - set -uo pipefail - if [ "$list_rc" -eq 0 ]; then - if echo "$list_out" | grep -Fq " ${SANDBOX_NAME}"; then - fail "nemoclaw list still lists '${SANDBOX_NAME}' after destroy" - exit 1 - fi - pass "nemoclaw list: '${SANDBOX_NAME}' removed from registry" - else - skip "nemoclaw list failed after cleanup — could not verify registry (exit $list_rc)" - fi - else - skip "nemoclaw not on PATH — skipped list check after cleanup" + if ! SANDBOX_NAME="$SANDBOX_NAME" bash "${E2E_DIR}/e2e-cloud-experimental/cleanup.sh" --verify; then + fail "Phase 6: final cleanup or verification failed" + exit 1 fi - pass "Phase 6: final cleanup complete" fi diff --git a/test/e2e/test-spark-install.sh b/test/e2e/test-spark-install.sh new file mode 100755 index 000000000..bc5f91339 --- /dev/null +++ b/test/e2e/test-spark-install.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# DGX Spark install smoke: setup-spark (Docker cgroupns) + install.sh — parity with +# test/integration/spark-install-cli.test.ts and spark-install.md Quick Start. +# +# Prerequisites: +# - Linux (DGX Spark or similar); other OS exits immediately (fail) +# - Docker running +# - sudo (for scripts/setup-spark.sh) unless NEMOCLAW_E2E_SPARK_SKIP_SETUP=1 +# - Same env your non-interactive install needs (e.g. NEMOCLAW_NON_INTERACTIVE=1, API keys, …) +# +# Environment: +# NEMOCLAW_NON_INTERACTIVE=1 — required (matches full-e2e install phase) +# NEMOCLAW_E2E_SPARK_SKIP_SETUP=1 — skip sudo setup-spark (host already configured) +# NEMOCLAW_E2E_PUBLIC_INSTALL=1 — use curl|bash instead of repo install.sh +# NEMOCLAW_INSTALL_SCRIPT_URL — URL when using public install (default: nemoclaw.sh) +# INSTALL_LOG — log file (default: /tmp/nemoclaw-e2e-spark-install.log) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 bash test/e2e/test-spark-install.sh +# +# See: spark-install.md + +set -uo pipefail + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root (install.sh)." + exit 1 +fi + +SETUP_SCRIPT="$REPO/scripts/setup-spark.sh" +INSTALL_LOG="${INSTALL_LOG:-/tmp/nemoclaw-e2e-spark-install.log}" + +section "Phase 0: Platform" +if [ "$(uname -s)" = "Linux" ]; then + pass "Running on Linux" +else + fail "This script is for DGX Spark (Linux). On other OS use Vitest: NEMOCLAW_E2E_SPARK_INSTALL=1 --project spark-install-cli (skipped there on non-Linux)." + exit 1 +fi + +section "Phase 1: Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if [ -f "$SETUP_SCRIPT" ]; then + pass "Found scripts/setup-spark.sh" +else + fail "Missing $SETUP_SCRIPT" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ]; then + pass "NEMOCLAW_NON_INTERACTIVE=1" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +section "Phase 2: Spark Docker setup (sudo)" +cd "$REPO" || { + fail "cd to repo: $REPO" + exit 1 +} + +if [ "${NEMOCLAW_E2E_SPARK_SKIP_SETUP:-0}" = "1" ]; then + info "Skipping sudo setup-spark (NEMOCLAW_E2E_SPARK_SKIP_SETUP=1)" + pass "setup-spark skipped" +else + info "Running: sudo bash scripts/setup-spark.sh" + if sudo bash "$SETUP_SCRIPT"; then + pass "setup-spark completed" + else + fail "setup-spark failed" + exit 1 + fi +fi + +section "Phase 3: Install NemoClaw (non-interactive)" +info "Log: $INSTALL_LOG" +if [ "${NEMOCLAW_E2E_PUBLIC_INSTALL:-0}" = "1" ]; then + url="${NEMOCLAW_INSTALL_SCRIPT_URL:-https://www.nvidia.com/nemoclaw.sh}" + info "Running: curl -fsSL ... | bash (url=$url)" + curl -fsSL "$url" | bash >"$INSTALL_LOG" 2>&1 & +else + info "Running: bash install.sh --non-interactive" + bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +fi +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +if [ "$install_exit" -ne 0 ]; then + fail "install failed (exit $install_exit); last 80 lines of log:" + tail -n 80 "$INSTALL_LOG" >&2 || true + exit 1 +fi +pass "install completed (exit 0)" + +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +section "Phase 4: Verify CLI" +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH ($(command -v nemoclaw))" +else + fail "nemoclaw not on PATH" + exit 1 +fi + +if command -v openshell >/dev/null 2>&1; then + pass "openshell on PATH" +else + fail "openshell not on PATH" + exit 1 +fi + +if nemoclaw --help >/dev/null 2>&1; then + pass "nemoclaw --help exits 0" +else + fail "nemoclaw --help failed" + exit 1 +fi + +section "Summary" +printf '\033[1;32mOK: spark-install bash smoke (%d checks passed)\033[0m\n' "$PASS" +echo " Log: $INSTALL_LOG" diff --git a/test/install-preflight.test.js b/test/install-preflight.test.js index c65f199c3..8ee8d9408 100644 --- a/test/install-preflight.test.js +++ b/test/install-preflight.test.js @@ -207,7 +207,7 @@ exit 98 expect(result.status).toBe(0); expect(fs.readFileSync(gitLog, "utf-8")).toMatch(/clone.*NemoClaw\.git/); - }); + }, 60_000); it("prints the HTTPS GitHub remediation when the binary is missing", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-install-remediation-")); diff --git a/test/uninstall.test.js b/test/uninstall.test.js index cd0178638..cba70840e 100644 --- a/test/uninstall.test.js +++ b/test/uninstall.test.js @@ -69,7 +69,7 @@ describe("uninstall CLI flags", () => { } finally { fs.rmSync(tmp, { recursive: true, force: true }); } - }); + }, 60_000); }); describe("uninstall helpers", () => {