diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dd84418f9..194fcfd89 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -104,51 +104,6 @@ jobs: echo "ARTEMIS_UUID=$UUID" >> $GITHUB_ENV mv build/snapshots/artemis.snapshot build/snapshots/artemis-$UUID.snapshot - - name: Sign in to Artemis - shell: bash - env: - ARTEMIS_EMAIL: leon@toit.io - ARTEMIS_PASSWORD: ${{ secrets.LEON_ARTEMIS_PW }} - run: | - # We log in with the Artemis executable. - # It will set the authentication in the config file which is also - # used by the uploader. - build/bin/artemis auth login --email "$ARTEMIS_EMAIL" --password "$ARTEMIS_PASSWORD" - - - name: Upload snapshot to Artemis - if: github.event_name == 'release' || inputs.upload_service_snapshot - shell: bash - run: | - # No need to cache the snapshots in a local snapshot directory. - mkdir -p tmp_snapshots - build/bin/uploader cli-snapshot \ - --snapshot-directory tmp_snapshots \ - build/snapshots/*.snapshot - - - name: Build and upload service images - if: github.event_name == 'release' - shell: bash - run: | - # No need to cache the snapshots in a local snapshot directory. - mkdir -p tmp_snapshots - for version in $SUPPORTED_SDK_VERSIONS; do - # For each supported SDK version, build the locally checked out - # version of the service and upload it to Artemis. - build/bin/uploader service \ - --snapshot-directory tmp_snapshots \ - --sdk-version $version \ - --service-version ${{ github.event.release.tag_name }} \ - --local \ - --force - build/bin/uploader service \ - --snapshot-directory tmp_snapshots \ - --sdk-version $version \ - --service-version "${{ github.event.release.tag_name }}+O1" \ - --local \ - --optimization-level=1 \ - --force - done - - name: Upload artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15718d679..a7c952f7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,6 @@ jobs: run_supabase_tests: true - os: ubuntu-latest run_supabase_tests: false - upload-service: true runs-on: ${{ matrix.os }} @@ -185,36 +184,6 @@ jobs: make disable-supabase-tests make test - - name: Upload service for Testing org - if: | - matrix.upload-service && - github.event_name == 'push' && - github.ref == 'refs/heads/main' - shell: bash - env: - ARTEMIS_EMAIL: leon@toit.io - ARTEMIS_PASSWORD: ${{ secrets.LEON_ARTEMIS_PW }} - run: | - # We log in with the Artemis executable. - # It will set the authentication in the config file which is also - # used by the uploader. - build/bin/artemis$BIN_EXTENSION auth login --email "$ARTEMIS_EMAIL" --password "$ARTEMIS_PASSWORD" - - SDK_VERSION=$(make dev-sdk-version) - VERSION=$(cmake -DPRINT_VERSION=1 -P tools/gitversion.cmake) - # It's hard to ensure that we don't upload a service with the same tag - # as a release, and that would lead to trouble. - # Add a T to indicate that this is a test version and avoid this problem. - SERVICE_VERSION=$VERSION-T - - ARTEMIS_TESTING_ORGANIZATION=3ea5b632-5739-4f40-8446-2fc102a5b338 - - build/bin/uploader service \ - --sdk-version $SDK_VERSION \ - --service-version $SERVICE_VERSION \ - --organization-id $ARTEMIS_TESTING_ORGANIZATION \ - --local - - name: Upload binary artifacts if: (!matrix.run_supabase_tests) uses: actions/upload-artifact@v4 @@ -265,8 +234,7 @@ jobs: run: | echo "ARTEMIS_CONFIG=${{ github.workspace }}/test_config" >> $GITHUB_ENV - - name: Install dependencies - Linux - if: runner.os == 'Linux' + - name: Install dependencies run: | sudo apt-get update -y -q sudo apt-get install -y -q --no-install-recommends ninja-build @@ -300,16 +268,20 @@ jobs: uses: pguyot/arm-runner-action@v2 with: cpu: cortex-a7 - base_image: "raspios_lite_arm64:2022-01-28" + base_image: "raspios_lite_arm64:2022-04-04" image_additional_mb: 500 commands: | uname -m + sudo apt-get update -y + sudo apt-get install -y git export HOME=$(pwd) export ARTEMIS_CONFIG=$HOME/config build/bin/artemis --version mkdir fleet export ARTEMIS_FLEET_ROOT=$PWD/fleet build/bin/artemis fleet init + sed -i 's/artemis-version.*//' fleet/my-pod.yaml + echo "artemis-version: ${{ github.ref }}" >> fleet/my-pod.yaml build/bin/artemis pod build -o test.pod fleet/my-pod.yaml - name: Kill http-servers diff --git a/CMakeLists.txt b/CMakeLists.txt index 0902980f8..dea873252 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,5 +41,5 @@ add_subdirectory(tests) add_subdirectory(src/cli) add_subdirectory(tools/http_servers) add_subdirectory(tools/lan_ip) -add_subdirectory(tools/service_image_uploader) +add_subdirectory(tools/service_image_downloader) add_subdirectory(tools/snapshot) diff --git a/Makefile b/Makefile index 6c3105005..a0052b976 100644 --- a/Makefile +++ b/Makefile @@ -175,32 +175,21 @@ setup-local-dev: @ $(TOITRUN) src/cli/cli.toit org add "Test Org" - @ $(MAKE) upload-service - .PHONY: dev-sdk-version dev-sdk-version: @ echo $(LOCAL_DEV_SDK) -.PHONY: upload-service -upload-service: - @ $(TOITRUN) tools/service_image_uploader/uploader.toit service --local --force \ - --sdk-version=$(LOCAL_DEV_SDK) \ - --service-version=$(SETUP_LOCAL_DEV_SERVICE) - @ $(TOITRUN) tools/service_image_uploader/uploader.toit service --local --force \ - --sdk-version=$(LOCAL_DEV_SDK) \ - --service-version=$$(cmake -DPRINT_VERSION=1 -P tools/gitversion.cmake) - .PHONY: download-sdk download-sdk: install-pkgs - @ $(TOITRUN) tools/service_image_uploader/sdk-downloader.toit download \ + @ $(TOITRUN) tools/service_image_downloader/sdk-downloader.toit download \ --version $(LOCAL_DEV_SDK) \ --envelope=esp32,esp32-qemu @ cmake \ -DDEV_SDK_VERSION=$(LOCAL_DEV_SDK) \ - -DDEV_SDK_PATH="$$($(TOITRUN) tools/service_image_uploader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print)" \ - -DDEV_ENVELOPE_ESP32_PATH="$$($(TOITRUN) tools/service_image_uploader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --envelope="esp32")" \ - -DDEV_ENVELOPE_ESP32_QEMU_PATH="$$($(TOITRUN) tools/service_image_uploader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --envelope="esp32-qemu")" \ - -DDEV_ENVELOPE_HOST_PATH="$$($(TOITRUN) tools/service_image_uploader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --host-envelope)" \ + -DDEV_SDK_PATH="$$($(TOITRUN) tools/service_image_downloader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print)" \ + -DDEV_ENVELOPE_ESP32_PATH="$$($(TOITRUN) tools/service_image_downloader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --envelope="esp32")" \ + -DDEV_ENVELOPE_ESP32_QEMU_PATH="$$($(TOITRUN) tools/service_image_downloader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --envelope="esp32-qemu")" \ + -DDEV_ENVELOPE_HOST_PATH="$$($(TOITRUN) tools/service_image_downloader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --host-envelope)" \ build # We rebuild the cmake file all the time. diff --git a/src/cli/artemis.toit b/src/cli/artemis.toit index cf2488c60..6d6744686 100644 --- a/src/cli/artemis.toit +++ b/src/cli/artemis.toit @@ -9,11 +9,16 @@ import uuid show Uuid import encoding.base64 import encoding.ubjson import encoding.json +import fs +import host.os +import system import .cache as cache import .cache show cache-key-service-image import .config import .device +import .git +import .pod-specification import .utils @@ -73,19 +78,6 @@ class Artemis: ensure-authenticated -> none: connected-artemis-server_ - /** - Checks whether the given $sdk version and $service version is supported by - the Artemis server. - */ - check-is-supported-version_ --organization-id/Uuid --sdk/string?=null --service/string?=null: - server := connected-artemis-server_ - versions := server.list-sdk-service-versions - --organization-id=organization-id - --sdk-version=sdk - --service-version=service - if versions.is-empty: - cli_.ui.abort "Unsupported Artemis/SDK versions ($service/$sdk)." - notify-created --hardware-id/Uuid: server := connected-artemis-server_ server.notify-created --hardware-id=hardware-id @@ -95,49 +87,6 @@ class Artemis: --device-id=device-id --organization-id=organization-id - /** - Gets the Artemis service image for the given $sdk and $service versions. - - Returns a path to the cached image. - */ - get-service-image-path -> string - --organization-id/Uuid - --sdk/string - --service/string - --chip-family/string - --word-size/int: - if word-size != 32 and word-size != 64: throw "INVALID_ARGUMENT" - service-key := cache-key-service-image - --service-version=service - --sdk-version=sdk - --artemis-config=server-config - --chip-family=chip-family - --word-size=word-size - return cli_.cache.get-file-path service-key: | store/FileStore | - server := connected-artemis-server_ --no-authenticated - entry := server.list-sdk-service-versions - --organization-id=organization-id - --sdk-version=sdk - --service-version=service - if entry.is-empty: - cli_.ui.abort "Unsupported Artemis/SDK versions." - image-name := entry.first["image"] - service-image-bytes := server.download-service-image image-name - ar-reader := ar.ArReader.from-bytes service-image-bytes - artemis-file := ar-reader.find "artemis" - metadata := json.decode artemis-file.contents - // Reset the reader. The images should be after the metadata, but - // doesn't hurt. - ar-reader = ar.ArReader.from-bytes service-image-bytes - if metadata["version"] == 1: - if chip-family != "esp32": - cli_.ui.abort "Unsupported chip family '$chip-family' for service $service and SDK $sdk." - ar-file := ar-reader.find "service-$(word-size).img" - store.save ar-file.contents - else: - ar-file := ar-reader.find "$(chip-family)-$(word-size).img" - store.save ar-file.contents - /** Fetches the organizations with the given $id. @@ -146,22 +95,34 @@ class Artemis: get-organization --id/Uuid -> OrganizationDetailed?: return connected-artemis-server_.get-organization id - /** - List all SDK/service version combinations. - - Returns a list of maps with the following keys: - - "sdk_version": the SDK version - - "service_version": the service version - - "image": the name of the image - - If provided, the given $sdk-version and $service-version can be - used to filter the results. - */ - list-sdk-service-versions -> List - --organization-id/Uuid - --sdk-version/string? - --service-version/string?: - return connected-artemis-server_.list-sdk-service-versions - --organization-id=organization-id - --sdk-version=sdk-version - --service-version=service-version +service-path-in-repository root/string --chip-family/string -> string: + return "$root/src/service/run/$(chip-family).toit" + +ARTEMIS-SERVICE-GIT-URL ::= "https://github.com/toitware/artemis" + +get-artemis-container version-or-path/string --chip-family/string --cli/Cli -> ContainerPath: + artemis-root-path := os.env.get "ARTEMIS_REPO_PATH" + if artemis-root-path: + entrypoint := service-path-in-repository artemis-root-path --chip-family=chip-family + return ContainerPath "artemis" --entrypoint=entrypoint + if is-dev-setup: + git := Git --cli=cli + artemis-path := fs.dirname system.program-path + root := git.current-repository-root --path=artemis-path + entrypoint := service-path-in-repository root --chip-family=chip-family + return ContainerPath "artemis" --entrypoint=entrypoint + + url/string := ? + if version-or-path.starts-with "http://" or version-or-path.starts-with "https://": + url = version-or-path + else if version-or-path.starts-with "file:/": + return ContainerPath "artemis" --entrypoint=(version-or-path.trim --left "file:/") + else: + // This is a version string. + url = ARTEMIS-SERVICE-GIT-URL + + version := version-or-path + return ContainerPath "artemis" + --entrypoint=(service-path-in-repository "." --chip-family=chip-family) + --git-url=url + --git-ref=version diff --git a/src/cli/artemis_servers/artemis-server.toit b/src/cli/artemis_servers/artemis-server.toit index 3ef5e10e8..2be547cac 100644 --- a/src/cli/artemis_servers/artemis-server.toit +++ b/src/cli/artemis_servers/artemis-server.toit @@ -140,30 +140,6 @@ interface ArtemisServerCli implements Authenticatable: // TODO(florian): add support for changing the email. update-profile --name/string - /** - List all SDK/service version combinations. - - Returns a list of maps with the following keys: - - "sdk_version": the SDK version - - "service_version": the service version - - "image": the name of the image - - If provided, the given $sdk-version and $service-version can be - used to filter the results. - */ - list-sdk-service-versions -> List - --organization-id/Uuid - --sdk-version/string?=null - --service-version/string?=null - - /** - Downloads the given $image. - - The $image must be a valid image name, as returned by - $list-sdk-service-versions. - */ - download-service-image image/string -> ByteArray - with-server server-config/ServerConfig --cli/Cli [block]: network := net.open server/ArtemisServerCli? := null diff --git a/src/cli/artemis_servers/http/base.toit b/src/cli/artemis_servers/http/base.toit index 4ef509bff..9269b61e3 100644 --- a/src/cli/artemis_servers/http/base.toit +++ b/src/cli/artemis_servers/http/base.toit @@ -164,21 +164,6 @@ class ArtemisServerCliHttpToit implements ArtemisServerCli: "name": name, } - list-sdk-service-versions -> List - --organization-id/Uuid - --sdk-version/string?=null - --service-version/string?=null: - return send-request_ COMMAND-LIST-SDK-SERVICE-VERSIONS_ { - "organization_id": "$organization-id", - "sdk_version": sdk-version, - "service_version": service-version, - } - - download-service-image image/string -> ByteArray: - return send-request_ COMMAND-DOWNLOAD-SERVICE-IMAGE_ { - "image": image, - } - send-request_ command/int data/Map -> any: encoded/ByteArray := #[command] + (json.encode data) headers := http.Headers diff --git a/src/cli/artemis_servers/supabase/supabase.toit b/src/cli/artemis_servers/supabase/supabase.toit index 342200e90..3e294b05a 100644 --- a/src/cli/artemis_servers/supabase/supabase.toit +++ b/src/cli/artemis_servers/supabase/supabase.toit @@ -154,20 +154,3 @@ class ArtemisServerCliSupabase implements ArtemisServerCli: client_.rest.update "profiles" { "name": name } --filters=[ equals "id" "$user-id", ] - - list-sdk-service-versions -> List - --organization-id/Uuid - --sdk-version/string?=null - --service-version/string?=null: - filters := [ - orr [ - is-null "organization_id", - equals "organization_id" "$organization-id", - ] - ] - if sdk-version: filters.add (equals "sdk_version" sdk-version) - if service-version: filters.add (equals "service_version" service-version) - return client_.rest.select "sdk_service_versions" --filters=filters - - download-service-image image/string -> ByteArray: - return client_.storage.download --public --path="service-images/$image" diff --git a/src/cli/broker.toit b/src/cli/broker.toit index e22ac00f0..1a160518e 100644 --- a/src/cli/broker.toit +++ b/src/cli/broker.toit @@ -20,6 +20,7 @@ import .pod-specification import .utils import .utils.patch-build show build-diff-patch build-trivial-patch +import ..shared.version import ..shared.utils.patch show Patcher PatchObserver import .brokers.broker @@ -841,19 +842,6 @@ class Broker: service-version := specification.artemis-version sdk-version := specification.sdk-version - checked := false - // We try to check the sdk and service versions as soon as possible to - // avoid downloading expensive assets. - check-sdk-service-version := : - if not checked and sdk-version: - artemis.check-is-supported-version_ - --organization-id=organization-id - --sdk=sdk-version - --service=service-version - checked = true - - check-sdk-service-version.call - envelope-path := get-envelope --specification=specification --cli=cli_ @@ -868,7 +856,6 @@ class Broker: cli_.ui.abort "The envelope uses SDK version $envelope-sdk-version, but $sdk-version was requested." else: sdk-version = envelope-sdk-version - check-sdk-service-version.call envelope-word-bit-size := Sdk.get-word-bit-size-from --envelope=envelope sdk := get-sdk sdk-version --cli=cli_ @@ -982,18 +969,58 @@ class Broker: artemis-assets-path := "$tmp-dir/artemis.assets" sdk.assets-create --output-path=artemis-assets-path artemis-assets - // Get the prebuilt Artemis service. - artemis-service-image-path := artemis.get-service-image-path - --organization-id=organization-id - --chip-family=envelope-chip-family - --word-size=envelope-word-bit-size - --sdk=sdk-version - --service=service-version + + // Build the Artemis service image. + artemis-container := get-artemis-container service-version --chip-family=envelope-chip-family --cli=cli_ + artemis-snapshot-path := "$tmp-dir/artemis.snapshot" + create-version-file := :: | repo-path/string | + // TODO(florian): share this code with the identity creation code. + version-path := "$repo-path/src/shared/version.toit" + if not file.is-file version-path: + // We already have the version form the container. + // We still need to extract a major and minor version. + artemis-version := artemis-container.git-ref + artemis-major/int := ? + artemis-minor/int := ? + if not artemis-version: + cli_.ui.abort "Local Artemis checkouts must have a version.toit file." + parts := (artemis-version.trim --left "v").split "." + if parts.size < 2: + // Probably just a commit hash or full ref. + // This should only happen during development. Use our version instead. + artemis-major = ARTEMIS-VERSION-MAJOR + artemis-minor = ARTEMIS-VERSION-MINOR + else: + had-error := false + artemis-major = int.parse parts[0] --on-error=: + had-error = true + 0 + artemis-minor = int.parse parts[1] --on-error=: + had-error = true + 0 + if had-error: + artemis-major = ARTEMIS-VERSION-MAJOR + artemis-minor = ARTEMIS-VERSION-MINOR + version-contents := """ + // This file is generated by the Toit SDK. + // Do not edit. + ARTEMIS-VERSION ::= "$artemis-version" + ARTEMIS-VERSION-MAJOR ::= $artemis-major + ARTEMIS-VERSION-MINOR ::= $artemis-minor + """ + file.write-contents --path=version-path version-contents + artemis-container.build-snapshot + --pre-compilation-hook=create-version-file + --relative-to=specification.relative-to + --sdk=sdk + --output-path=artemis-snapshot-path + --cli=cli_ + cli_.ui.emit --info "Added Artemis service container to envelope." sdk.firmware-add-container "artemis" --envelope=output-path --assets=artemis-assets-path - --program-path=artemis-service-image-path + --program-path=artemis-snapshot-path --trigger="boot" --critical diff --git a/src/cli/cmds/sdk.toit b/src/cli/cmds/sdk.toit index 8481529af..d83906126 100644 --- a/src/cli/cmds/sdk.toit +++ b/src/cli/cmds/sdk.toit @@ -20,7 +20,7 @@ create-sdk-commands -> List: list-cmd := Command "list" --aliases=["ls"] - --help="List supported SDKs." + --help="List supported SDKs. REMOVED." --options=[ Option "sdk-version" --help="The SDK version to list.", Option "service-version" --help="The service version to list.", @@ -39,31 +39,5 @@ create-sdk-commands -> List: return [sdk-cmd] list-sdks invocation/Invocation: - sdk-version := invocation["sdk-version"] - service-version := invocation["service-version"] - - with-pod-fleet invocation: | fleet/Fleet | - artemis := fleet.artemis - versions/List := artemis.list-sdk-service-versions - --organization-id=fleet.organization-id - --sdk-version=sdk-version - --service-version=service-version - - versions.sort --in-place: | a/Map b/Map | - semver.compare a["sdk_version"] b["sdk_version"] --if-equal=: - semver.compare a["service_version"] b["service_version"] --if-equal=: - // As a last effort compare the strings directly. - // This also includes the build metadata, which is ignored for semver comparisons. - "$a["sdk_version"]-$a["service_version"]".compare-to "$b["sdk_version"]-$b["service_version"]" - - output := versions.map: { - "sdk-version": it["sdk_version"], - "service-version": it["service_version"], - } - - invocation.cli.ui.emit-table --result - --header={ - "sdk-version": "SDK Version", - "service-version": "Service Version", - } - output + print "Artemis now compiles the service locally, and services are not downloaded from the server anymore." + invocation.cli.ui.abort "UNSUPPORTED" diff --git a/src/cli/git.toit b/src/cli/git.toit index 207f7914e..05ba94b7d 100644 --- a/src/cli/git.toit +++ b/src/cli/git.toit @@ -13,13 +13,15 @@ class Git: /** Returns the root of the Git repository that contains the - current working directory. + given $path. If no $path is given, uses the current working directory. */ - current-repository-root -> string: - out := run_ [ + current-repository-root --path/string?=null -> string: + args := path ? ["-C", path] : [] + args.add-all [ "rev-parse", - "--show-toplevel" + "--show-toplevel", ] + out := run_ args return out.trim /** @@ -175,10 +177,7 @@ class Git: /** Runs the command, and only outputs stdout/stderr if there is an error. */ - run_ args/List -> string: - return run_ args --description="Git command" - - run_ args/List --description -> string: + run_ args/List --description/string="Git command" -> string: output := io.Buffer stdout := io.Buffer fork-data := pipe.fork diff --git a/src/cli/pod-specification.toit b/src/cli/pod-specification.toit index a48d201d7..73a6a383c 100644 --- a/src/cli/pod-specification.toit +++ b/src/cli/pod-specification.toit @@ -575,6 +575,14 @@ abstract class ContainerBase implements Container: defines/Map? name/string + constructor .name + --.arguments/List? + --.triggers/List? + --.is-background/bool? + --.is-critical/bool? + --.runlevel/int? + --.defines/Map?: + constructor.from-json .name json-map/JsonMap: arguments = json-map.get-optional-list "arguments" --type="string" @@ -626,7 +634,12 @@ abstract class ContainerBase implements Container: defines = json-map.get-optional-map "defines" abstract type -> string - abstract build-snapshot --output-path/string --relative-to/string --sdk/Sdk --cli/Cli -> none + abstract build-snapshot -> none + --output-path/string + --relative-to/string + --pre-compilation-hook/Lambda?=null + --sdk/Sdk + --cli/Cli class ContainerPath extends ContainerBase: entrypoint/string @@ -634,6 +647,28 @@ class ContainerPath extends ContainerBase: git-ref/string? compile-flags/List? + constructor name/string + --.entrypoint + --.git-url=null + --.git-ref=null + --.compile-flags=null + --arguments/List?=null + --triggers/List?=null + --is-background/bool?=null + --is-critical/bool?=null + --runlevel/int?=null + --defines/Map?=null: + if compile-flags: compile-flags.do: | flag | + if flag is not string: + format-error_ "Compile flags entry must be a list of strings: $compile-flags" + super name + --arguments=arguments + --triggers=triggers + --is-background=is-background + --is-critical=is-critical + --runlevel=runlevel + --defines=defines + constructor.from-json name/string json-map/JsonMap: holder := "container $name" git-ref = json-map.get-optional-string "branch" @@ -648,7 +683,12 @@ class ContainerPath extends ContainerBase: format-error_"In container $name, git entry requires a relative path: $entrypoint" super.from-json name json-map - build-snapshot --output-path/string --relative-to/string --sdk/Sdk --cli/Cli -> none: + build-snapshot -> none + --output-path/string + --relative-to/string + --sdk/Sdk + --pre-compilation-hook/Lambda?=null + --cli/Cli: ui := cli.ui if not git-url: @@ -708,6 +748,8 @@ class ContainerPath extends ContainerBase: --repository-root=clone-dir --ref=git-ref --quiet + if pre-compilation-hook: + pre-compilation-hook.call clone-dir ui.emit --info "Compiling '$git-url'." entrypoint-path := "$clone-dir/$entrypoint" if not file.is-file entrypoint-path: @@ -742,7 +784,14 @@ class ContainerSnapshot extends ContainerBase: snapshot-path = json-map.get-string "snapshot" super.from-json name json-map - build-snapshot --relative-to/string --output-path/string --sdk/Sdk --cli/Cli -> none: + build-snapshot -> none + --output-path/string + --relative-to/string + --pre-compilation-hook/Lambda?=null + --sdk/Sdk + --cli/Cli: + if pre-compilation-hook: + cli.ui.abort "Pre-compilation hook is not supported for snapshot containers." path := snapshot-path if fs.is-relative snapshot-path: path = "$relative-to/$snapshot-path" diff --git a/src/cli/sdk.toit b/src/cli/sdk.toit index 1345fbd56..c9ed5da19 100644 --- a/src/cli/sdk.toit +++ b/src/cli/sdk.toit @@ -24,6 +24,12 @@ class Sdk: constructor .sdk-path .version: + constructor .sdk-path/string: + version = (pipe.backticks [ + "$sdk-path/bin/toit", + "version", + ]).trim + constructor --envelope-path/string --cli/Cli: sdk-version := get-sdk-version-from --envelope-path=envelope-path return get-sdk sdk-version --cli=cli @@ -465,16 +471,28 @@ sdk-url version/string -> string: reported-local-sdk-use_/bool := false -get-sdk version/string --cli/Cli -> Sdk: +get-sdk version-or-path/string --cli/Cli -> Sdk: if is-dev-setup: local-sdk := os.env.get "DEV_TOIT_REPO_PATH" if local-sdk: if not reported-local-sdk-use_: print-on-stderr_ "Using local SDK" reported-local-sdk-use_ = true - return Sdk "$local-sdk/build/host/sdk" version + return Sdk "$local-sdk/build/host/sdk" version-or-path + + url/string := ? + if version-or-path.starts-with "http://" or version-or-path.starts-with "https://": + url = version-or-path + else if version-or-path.starts-with "file:/": + return Sdk (version-or-path.trim --left "file:/") + else if version-or-path.contains "/": + // This is a path to a local SDK. + return Sdk version-or-path + else: + // This is a version string. + url = sdk-url version-or-path - url := sdk-url version + version := version-or-path cache-key := cache-key-sdk --version=version path := cli.cache.get-directory-path cache-key: | store/DirectoryStore | with-tmp-directory: | tmp-dir | @@ -488,4 +506,4 @@ get-sdk version/string --cli/Cli -> Sdk: gunzip gzip-path untar tar-path --target=final-out-dir store.move "$final-out-dir/toit" - return Sdk path version + return Sdk path diff --git a/src/shared/constants.toit b/src/shared/constants.toit index ba3371a3b..b31d313f3 100644 --- a/src/shared/constants.toit +++ b/src/shared/constants.toit @@ -7,12 +7,6 @@ COMMAND-SIGN-UP_ ::= 2 COMMAND-SIGN-IN_ ::= 3 COMMAND-GET-ORGANIZATIONS_ ::= 4 COMMAND-UPDATE-CURRENT-USER_ ::= 18 -/** -Command to notify the Artemis server that a device has been created. - -To avoid accidental confusion with $COMMAND-NOTIFY-BROKER-CREATED_, the - command has the same constants. -*/ COMMAND-NOTIFY-ARTEMIS-CREATED_ ::= 5 COMMAND-GET-ORGANIZATION-DETAILS_ ::= 6 COMMAND-CREATE-ORGANIZATION_ ::= 7 @@ -23,8 +17,11 @@ COMMAND-ORGANIZATION-MEMBER-REMOVE_ ::= 11 COMMAND-ORGANIZATION-MEMBER-SET-ROLE_ ::= 12 COMMAND-GET-PROFILE_ ::= 13 COMMAND-UPDATE-PROFILE_ ::= 14 +/// Deprecated and unused in newer CLIs. COMMAND-LIST-SDK-SERVICE-VERSIONS_ ::= 15 +/// Deprecated and unused in newer CLIs. COMMAND-DOWNLOAD-SERVICE-IMAGE_ ::= 16 +/// Deprecated and unused in newer CLIs. COMMAND-UPLOAD-SERVICE-IMAGE_ ::= 17 ARTEMIS-COMMAND-TO-STRING ::= { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d6ae021d0..acb7f627f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -43,7 +43,6 @@ set(TEST_PREFIX "") include(fail.cmake OPTIONAL) set (SUPABASE_ARTEMIS_TESTS - "/tests/supabase-uploader-test.toit" "/tests/supabase-policies-test.toit" ) diff --git a/tests/artemis-server-test.toit b/tests/artemis-server-test.toit index 53320e0a8..50da7d0fd 100644 --- a/tests/artemis-server-test.toit +++ b/tests/artemis-server-test.toit @@ -44,7 +44,6 @@ run-test artemis-server/TestArtemisServer [--authenticate]: test-organizations server-cli backdoor test-profile server-cli backdoor - test-sdk server-cli backdoor test-create-device-in-organization server-cli/ArtemisServerCli backdoor/ArtemisServerBackdoor -> Uuid: // Test without and with alias. @@ -192,94 +191,3 @@ test-profile server-cli/ArtemisServerCli backdoor/ArtemisServerBackdoor: // and test user into the same organization. profile-demo := server-cli.get-profile --user-id=DEMO-EXAMPLE-COM-UUID expect-equals DEMO-EXAMPLE-COM-NAME profile-demo["name"] - -test-sdk server-cli/ArtemisServerCli backdoor/ArtemisServerBackdoor: - SDK-V1 ::= "v2.0.0-alpha.46" - SDK-V2 ::= "v2.0.0-alpha.47" - SERVICE-V1 ::= "v0.0.1" - SERVICE-V2 ::= "v0.0.2" - - IMAGE-V1-V1 ::= "foobar" - IMAGE-V2-V1 ::= "toto" - IMAGE-V2-V2 ::= "titi" - - CONTENTS-V1-V1 ::= "foobar_contents".to-byte-array - CONTENTS-V2-V1 ::= "toto_contents".to-byte-array - CONTENTS-V2-V2 ::= "titi_contents".to-byte-array - - test-images := [ - { - "sdk_version": SDK-V1, - "service_version": SERVICE-V1, - "image": IMAGE-V1-V1, - "content": CONTENTS-V1-V1, - }, - { - "sdk_version": SDK-V2, - "service_version": SERVICE-V1, - "image": IMAGE-V2-V1, - "content": CONTENTS-V2-V1, - }, - { - "sdk_version": SDK-V2, - "service_version": SERVICE-V2, - "image": IMAGE-V2-V2, - "content": CONTENTS-V2-V2, - }, - ] - backdoor.install-service-images test-images - - images := server-cli.list-sdk-service-versions --organization-id=TEST-ORGANIZATION-UUID - expect-equals test-images.size images.size - - test-images.do: | test-image | - filtered-image := images.filter: | image | - image["sdk_version"] == test-image["sdk_version"] and - image["service_version"] == test-image["service_version"] - expect-equals 1 filtered-image.size - image := filtered-image[0]["image"] - expect-equals test-image["image"] image - - downloaded-contents := server-cli.download-service-image image - expect-equals test-image["content"] downloaded-contents - - // Test that the filters on list_sdk_service_versions work. - images = server-cli.list-sdk-service-versions - --organization-id=TEST-ORGANIZATION-UUID - --sdk-version=SDK-V1 - expect-equals 1 images.size - expect-equals SDK-V1 images[0]["sdk_version"] - expect-equals SERVICE-V1 images[0]["service_version"] - - images = server-cli.list-sdk-service-versions - --organization-id=TEST-ORGANIZATION-UUID - --sdk-version=SDK-V2 - expect-equals 2 images.size - images.do: | image | - expect-equals SDK-V2 image["sdk_version"] - expect (image["service_version"] == SERVICE-V1 or - image["service_version"] == SERVICE-V2) - - images = server-cli.list-sdk-service-versions - --organization-id=TEST-ORGANIZATION-UUID - --service-version=SERVICE-V1 - expect-equals 2 images.size - images.do: | image | - expect-equals SERVICE-V1 image["service_version"] - expect (image["sdk_version"] == SDK-V1 or - image["sdk_version"] == SDK-V2) - - images = server-cli.list-sdk-service-versions - --organization-id=TEST-ORGANIZATION-UUID - --service-version=SERVICE-V2 - expect-equals 1 images.size - expect-equals SERVICE-V2 images[0]["service_version"] - expect-equals SDK-V2 images[0]["sdk_version"] - - images = server-cli.list-sdk-service-versions - --organization-id=TEST-ORGANIZATION-UUID - --sdk-version=SDK-V2 - --service-version=SERVICE-V1 - expect-equals 1 images.size - expect-equals SERVICE-V1 images[0]["service_version"] - expect-equals SDK-V2 images[0]["sdk_version"] diff --git a/tests/cli-device-extract.toit b/tests/cli-device-extract.toit index 39bd6724f..5831e2aa1 100644 --- a/tests/cli-device-extract.toit +++ b/tests/cli-device-extract.toit @@ -30,8 +30,6 @@ upload-pod -> Uuid --pod-spec/Map={:} --pod-spec-filename/string="my-pod.yaml": - fleet.tester.ensure-available-artemis-service - prefix := "--base-root=" base-root/string? := null fleet.args.do: diff --git a/tests/cmd-device-container-test.toit b/tests/cmd-device-container-test.toit index 2e34c9c20..1693684c2 100644 --- a/tests/cmd-device-container-test.toit +++ b/tests/cmd-device-container-test.toit @@ -51,10 +51,6 @@ run-test fleet/TestFleet: "device", "container", "uninstall", "hello", "--force" ] - fleet.tester.ensure-available-artemis-service - --sdk-version=TEST-SDK-VERSION - --artemis-version=TEST-ARTEMIS-VERSION - pod-spec := deep-copy_ INITIAL-POD-SPECIFICATION pod-spec["sdk-version"] = TEST-SDK-VERSION diff --git a/tests/cmd-pod-build-test.toit b/tests/cmd-pod-build-test.toit index 51a89d13a..179c3e1db 100644 --- a/tests/cmd-pod-build-test.toit +++ b/tests/cmd-pod-build-test.toit @@ -20,7 +20,6 @@ run-test fleet/TestFleet: fleet-dir := fleet.fleet-dir tester := fleet.tester ui := TestUi - fleet.tester.ensure-available-artemis-service spec := { "\$schema": "https://toit.io/schemas/artemis/pod-specification/v1.json", diff --git a/tests/cmd-pod-delete-test.toit b/tests/cmd-pod-delete-test.toit index bd5a086d7..49e0e1708 100644 --- a/tests/cmd-pod-delete-test.toit +++ b/tests/cmd-pod-delete-test.toit @@ -49,8 +49,6 @@ create-pods name/string fleet/TestFleet --count/int -> List: return [description-id, spec-ids] run-test fleet/TestFleet: - fleet.tester.ensure-available-artemis-service - pod1-name := "pod1" tmp := create-pods pod1-name fleet --count=3 description1-id := tmp[0] diff --git a/tests/cmd-pod-list-test-slow.toit b/tests/cmd-pod-list-test-slow.toit index 34123298a..82089fd76 100644 --- a/tests/cmd-pod-list-test-slow.toit +++ b/tests/cmd-pod-list-test-slow.toit @@ -11,8 +11,6 @@ main args: run-test fleet run-test fleet/TestFleet: - fleet.tester.ensure-available-artemis-service - name := "test-pod" spec := { diff --git a/tests/cmd-pod-tag-test.toit b/tests/cmd-pod-tag-test.toit index 14d141467..846e20bd3 100644 --- a/tests/cmd-pod-tag-test.toit +++ b/tests/cmd-pod-tag-test.toit @@ -11,8 +11,6 @@ main args: run-test fleet run-test fleet/TestFleet: - fleet.tester.ensure-available-artemis-service - name := "test-pod" name2 := "test-pod2" diff --git a/tests/cmd-pod-upload-tag-test-slow.toit b/tests/cmd-pod-upload-tag-test-slow.toit index dbe40a02d..a22d52b92 100644 --- a/tests/cmd-pod-upload-tag-test-slow.toit +++ b/tests/cmd-pod-upload-tag-test-slow.toit @@ -25,8 +25,6 @@ run-test fleet/TestFleet: tester.replacements["$it["id"]"] = pad-replacement-id "ID $pod-name#$(it["revision"])" output - tester.ensure-available-artemis-service - name := "test-pod" spec := { "\$schema": "https://toit.io/schemas/artemis/pod-specification/v1.json", diff --git a/tests/cmd-sdk-test.toit b/tests/cmd-sdk-test.toit deleted file mode 100644 index 0d504fb8c..000000000 --- a/tests/cmd-sdk-test.toit +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (C) 2023 Toitware ApS. - -// ARTEMIS_TEST_FLAGS: ARTEMIS - -import expect show * -import .utils - -main args: - with-fleet --count=0 --args=args: | fleet/TestFleet | - run-test fleet.tester - -run-test tester/Tester: - test-start := Time.now - - backdoor := tester.artemis.backdoor - backdoor.install-service-images [] - - tester.login - - output := tester.run [ "sdk", "list" ] - /* -┌─────────────┬─────────────────┐ -│ SDK Version Service Version │ -├─────────────┼─────────────────┤ -└─────────────┴─────────────────┘ - */ - expect-not (output.contains "0") - - SDK-V1 ::= "v2.0.0-alpha.46" - SDK-V2 ::= "v2.0.0-alpha.47" - SERVICE-V1 ::= "v0.0.1" - SERVICE-V2 ::= "v0.0.2" - - IMAGE-V1-V1 ::= "foobar" - IMAGE-V2-V1 ::= "toto" - IMAGE-V2-V2 ::= "titi" - - IGNORED-CONTENTS ::= "ignored".to-byte-array - - test-images := [ - { - "sdk_version": SDK-V1, - "service_version": SERVICE-V1, - "image": IMAGE-V1-V1, - "content": IGNORED-CONTENTS, - }, - { - "sdk_version": SDK-V2, - "service_version": SERVICE-V1, - "image": IMAGE-V2-V1, - "content": IGNORED-CONTENTS, - }, - { - "sdk_version": SDK-V2, - "service_version": SERVICE-V2, - "image": IMAGE-V2-V2, - "content": IGNORED-CONTENTS, - }, - ] - backdoor.install-service-images test-images - - output = tester.run [ "sdk", "list" ] - /* - ┌─────────────────┬─────────────────┐ - │ SDK Version Service Version │ - ├─────────────────┼─────────────────┤ - │ v2.0.0-alpha.46 v0.0.1 │ - │ v2.0.0-alpha.47 v0.0.1 │ - │ v2.0.0-alpha.47 v0.0.2 │ - └─────────────────┴─────────────────┘ - */ - lines := output.split "\n" - found-v1-v1 := false - found-v2-v1 := false - found-v2-v2 := false - lines.do: | line/string | - if line.contains SDK-V1 and line.contains SERVICE-V1: - found-v1-v1 = true - if line.contains SDK-V2 and line.contains SERVICE-V1: - found-v2-v1 = true - if line.contains SDK-V2 and line.contains SERVICE-V2: - found-v2-v2 = true - expect found-v1-v1 - expect found-v2-v1 - expect found-v2-v2 - - // Test filtering. - output = tester.run [ "sdk", "list", "--sdk-version", SDK-V2 ] - /* - ┌─────────────────┬─────────────────┐ - │ SDK Version Service Version │ - ├─────────────────┼─────────────────┤ - │ v2.0.0-alpha.47 v0.0.1 │ - │ v2.0.0-alpha.47 v0.0.2 │ - └─────────────────┴─────────────────┘ - */ - lines = output.split "\n" - found-v1-v1 = false - found-v2-v1 = false - found-v2-v2 = false - lines.do: | line/string | - if line.contains SDK-V1 and line.contains SERVICE-V1: - found-v1-v1 = true - if line.contains SDK-V2 and line.contains SERVICE-V1: - found-v2-v1 = true - if line.contains SDK-V2 and line.contains SERVICE-V2: - found-v2-v2 = true - expect-not found-v1-v1 - expect found-v2-v1 - expect found-v2-v2 - - // Same for service version. - output = tester.run [ "sdk", "list", "--service-version", SERVICE-V1 ] - /* - ┌─────────────────┬─────────────────┐ - │ SDK Version Service Version │ - ├─────────────────┼─────────────────┤ - │ v2.0.0-alpha.46 v0.0.1 │ - │ v2.0.0-alpha.47 v0.0.1 │ - └─────────────────┴─────────────────┘ - */ - lines = output.split "\n" - found-v1-v1 = false - found-v2-v1 = false - found-v2-v2 = false - lines.do: | line/string | - if line.contains SDK-V1 and line.contains SERVICE-V1: - found-v1-v1 = true - if line.contains SDK-V2 and line.contains SERVICE-V1: - found-v2-v1 = true - if line.contains SDK-V2 and line.contains SERVICE-V2: - found-v2-v2 = true - expect found-v1-v1 - expect found-v2-v1 - expect-not found-v2-v2 - - // Test sdk and service version. - output = tester.run [ "sdk", "list", "--sdk-version", SDK-V2, "--service-version", SERVICE-V1 ] - /* - ┌─────────────────┬─────────────────┐ - │ SDK Version Service Version │ - ├─────────────────┼─────────────────┤ - │ v2.0.0-alpha.47 v0.0.1 │ - └─────────────────┴─────────────────┘ - */ - lines = output.split "\n" - found-v1-v1 = false - found-v2-v1 = false - found-v2-v2 = false - lines.do: | line/string | - if line.contains SDK-V1 and line.contains SERVICE-V1: - found-v1-v1 = true - if line.contains SDK-V2 and line.contains SERVICE-V1: - found-v2-v1 = true - if line.contains SDK-V2 and line.contains SERVICE-V2: - found-v2-v2 = true - expect-not found-v1-v1 - expect found-v2-v1 - expect-not found-v2-v2 diff --git a/tests/gold/cmd-pod-upload-tag-test-slow/BAA-upload-existing-tag.txt b/tests/gold/cmd-pod-upload-tag-test-slow/BAA-upload-existing-tag.txt index 51e3e64cd..7f2a34cf9 100644 --- a/tests/gold/cmd-pod-upload-tag-test-slow/BAA-upload-existing-tag.txt +++ b/tests/gold/cmd-pod-upload-tag-test-slow/BAA-upload-existing-tag.txt @@ -1,5 +1,6 @@ # Upload a pod with existing tag # [pod, upload, /test-pod.yaml, --tag, some-tag] +Added Artemis service container to envelope. Uploading pod. This may take a while. Uploaded test-pod#2 to fleet -={|~~~~~~~~~~FLEET_ID~~~~~~~~~~|}=-. id: -={|~~~~~~~ID test-pod#2~~~~~~~~|}=- diff --git a/tests/gold/cmd-pod-upload-tag-test-slow/BAC-upload-existing-tag-force.txt b/tests/gold/cmd-pod-upload-tag-test-slow/BAC-upload-existing-tag-force.txt index af3d7eb26..cec5633f3 100644 --- a/tests/gold/cmd-pod-upload-tag-test-slow/BAC-upload-existing-tag-force.txt +++ b/tests/gold/cmd-pod-upload-tag-test-slow/BAC-upload-existing-tag-force.txt @@ -1,5 +1,6 @@ # Upload a pod with existing tag using --force # [pod, upload, /test-pod.yaml, --tag, some-tag, --force] +Added Artemis service container to envelope. Uploading pod. This may take a while. Successfully uploaded test-pod#3 to fleet -={|~~~~~~~~~~FLEET_ID~~~~~~~~~~|}=-. id: -={|~~~~~~~ID test-pod#3~~~~~~~~|}=- diff --git a/tests/gold/serial-full-workflow-test-slow/DAB-upload-firmware.txt b/tests/gold/serial-full-workflow-test-slow/DAB-upload-firmware.txt index 2a4e6e33c..1b6946cfe 100644 --- a/tests/gold/serial-full-workflow-test-slow/DAB-upload-firmware.txt +++ b/tests/gold/serial-full-workflow-test-slow/DAB-upload-firmware.txt @@ -1,5 +1,6 @@ # Upload the firmware. # [--fleet-root, TMP_DIR/fleet, pod, upload, TMP_DIR/firmware.pod] +Added Artemis service container to envelope. Uploading pod. This may take a while. Successfully uploaded my-pod#1 to fleet -={| UUID-FOR-FLEET |}=-. id: -={| UUID-FOR-MY-POD#1 |}=- diff --git a/tests/gold/serial-full-workflow-test-slow/DAD-upload-modified-pod.txt b/tests/gold/serial-full-workflow-test-slow/DAD-upload-modified-pod.txt index 664a1101c..ae146d147 100644 --- a/tests/gold/serial-full-workflow-test-slow/DAD-upload-modified-pod.txt +++ b/tests/gold/serial-full-workflow-test-slow/DAD-upload-modified-pod.txt @@ -1,5 +1,6 @@ # Upload the modified pod. # [--fleet-root, TMP_DIR/fleet, pod, upload, TMP_DIR/firmware.pod] +Added Artemis service container to envelope. Uploading pod. This may take a while. Successfully uploaded my-pod#2 to fleet -={| UUID-FOR-FLEET |}=-. id: -={| UUID-FOR-MY-POD#2 |}=- diff --git a/tests/gold/serial-full-workflow-test-slow/DDB-upload-new-firmware.txt b/tests/gold/serial-full-workflow-test-slow/DDB-upload-new-firmware.txt index 8b5e2dfac..4c8289cbb 100644 --- a/tests/gold/serial-full-workflow-test-slow/DDB-upload-new-firmware.txt +++ b/tests/gold/serial-full-workflow-test-slow/DDB-upload-new-firmware.txt @@ -1,5 +1,6 @@ # Upload a new version of the firmware. # [--fleet-root, TMP_DIR/fleet, pod, upload, TMP_DIR/fleet/FLEET-POD-FILE] +Added Artemis service container to envelope. Uploading pod. This may take a while. Successfully uploaded my-pod#3 to fleet -={| UUID-FOR-FLEET |}=-. id: -={| UUID-FOR-MY-POD#3 |}=- diff --git a/tests/serial-full-workflow-test-slow.toit b/tests/serial-full-workflow-test-slow.toit index 0a8664807..f2dadbcad 100644 --- a/tests/serial-full-workflow-test-slow.toit +++ b/tests/serial-full-workflow-test-slow.toit @@ -9,7 +9,6 @@ import artemis.cli.server-config as cli-server-config import artemis.service import artemis.shared.server-config show ServerConfig ServerConfigHttp import artemis.cli.utils show read-json read-yaml write-yaml-to-file write-blob-to-file -import cli show Cli import encoding.json import host.directory import host.file @@ -19,7 +18,6 @@ import uuid show Uuid import expect show * import .artemis-server show TestArtemisServer import .utils -import ..tools.service-image-uploader.uploader as uploader main args/List: serial-port := os.env.get "ARTEMIS_TEST_SERIAL_PORT" @@ -62,17 +60,6 @@ run-test tester/Tester serial-port/string wifi-ssid/string wifi-password/string: service-version := "v0.0.$(random)-TEST" - cli := tester.cli.with --ui=ui - uploader.main - --cli=cli - [ - "service", - "--sdk-version", tester.sdk-version, - "--service-version", service-version, - "--snapshot-directory", "$tmp-dir/snapshots", - "--local", - ] - email := "test-$(%010d random)@example.com" password := "test-$(%010d random)" tester.replacements[email] = "USER@EMAIL" diff --git a/tests/serial.toit b/tests/serial.toit index b09a9436d..bc92693ff 100644 --- a/tests/serial.toit +++ b/tests/serial.toit @@ -18,7 +18,6 @@ flash-serial -> Uuid --tmp-dir/string="$fleet.tester.tmp-dir/serial": fleet-dir := fleet.fleet-dir - fleet.tester.ensure-available-artemis-service directory.mkdir --recursive tmp-dir files.do: | filename/string blob | diff --git a/tests/supabase-uploader-test.toit b/tests/supabase-uploader-test.toit deleted file mode 100644 index a8bf6b4cf..000000000 --- a/tests/supabase-uploader-test.toit +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright (C) 2022 Toitware ApS. - -import expect show * -import host.directory -import host.file - -import .utils -import .artemis-server -import ..tools.service-image-uploader.uploader as uploader -import ..tools.service-image-uploader.downloader as downloader -import artemis.shared.version show ARTEMIS-VERSION -import artemis.cli.git show Git -import supabase -import supabase.filter show equals - -main args: - // Start a Tester, since that will set up everything the way we want. - with-tester --args=args --artemis-type="supabase": - run-test it - -// Just a commit that exists on main. -// It's safe to update the commit to a newer version. -TEST-COMMIT ::= "a521082f8f6f0ddfaf33ab55007ec8a51b659dff" - -run-main-test - tester/Tester - tmp-dir/string - service-version/string - --keep-service/bool=false - [block]: - block.call - supabase-backdoor := (tester.artemis.backdoor) as SupabaseBackdoor - // Check that the service is available. - supabase-backdoor.with-backdoor-client_: | client/supabase.Client | - available-sdks := client.rest.select "sdk_service_versions" --filters=[ - equals "sdk_version" tester.sdk-version, - equals "service_version" service-version, - ] - expect-not available-sdks.is-empty - - // Check that the snapshots were written into the snapshot directory. - // We have two snapshots since we have two chip-families: esp32 and host. - files-iterator := directory.DirectoryStream tmp-dir - files := [] - while file-name := files-iterator.next: files.add file-name - files-iterator.close - expect-equals 2 files.size - - // Delete the files. - file.delete "$tmp-dir/$files[0]" - file.delete "$tmp-dir/$files[1]" - - // Check that the file was deleted. - files-iterator = directory.DirectoryStream tmp-dir - files = [] - while file-name := files-iterator.next: files.add file-name - files-iterator.close - expect-equals 0 files.size - - if not keep-service: - delete-service-version tester service-version - -delete-service-version tester/Tester service-version/string: - supabase-backdoor := tester.artemis.backdoor as SupabaseBackdoor - supabase-backdoor.with-backdoor-client_: | client/supabase.Client | - client.rest.delete "artemis_services" --filters=[ - equals "version" "$service-version", - ] - -run-test tester/Tester: - sdk-version := tester.sdk-version - with-tmp-directory: | tmp-dir/string | - ui := TestUi --no-quiet - cli := tester.cli.with --ui=ui - git := Git --cli=cli - - // Login using the CLI login. - // The uploader reuses the same credentials. - tester.run [ - "auth", "login", - "--email", ADMIN-EMAIL, - "--password", ADMIN-PASSWORD - ] - - service-version := "v0.0.$(random)-TEST" - - // The test-server could have already been used. - // We want to avoid duplicates, so we create a new version number. - // This means that we create a new tag every time we run this test. - // We try to remove it afterwards, but if the program is interrupted - // we might leave a tag behind. In that case it's safe to delete - // the tag manually. - git.tag --name=service-version --commit=TEST-COMMIT - try: - // We keep the service for the next test. - run-main-test tester tmp-dir service-version --keep-service: - uploader.main - --cli=cli - [ - "service", - "--sdk-version", sdk-version, - "--service-version", service-version, - "--snapshot-directory", tmp-dir, - ] - - // Debug information to analyze flaky tests. - supabase-backdoor := tester.artemis.backdoor as SupabaseBackdoor - supabase-backdoor.with-backdoor-client_: | client/supabase.Client | - service-images := client.rest.select "service_images" - print "Service images before 2nd attempt: $service-images" - - // Without force we can't upload the same version again. - exception := catch: - uploader.main - --cli=cli - [ - "service", - "--sdk-version", sdk-version, - "--service-version", service-version, - "--snapshot-directory", tmp-dir, - ] - if exception is not TestExit: - // Debug information to analyze flaky tests. - supabase-backdoor.with-backdoor-client_: | client/supabase.Client | - service-images := client.rest.select "service_images" - print "Service images after 2nd attempt: $service-images" - expect false - - // We keep the service version for the download test. - run-main-test tester tmp-dir service-version --keep-service: - uploader.main - --cli=cli - [ - "service", - "--sdk-version", sdk-version, - "--service-version", service-version, - "--snapshot-directory", tmp-dir, - "--force", - ] - - // Try with a specific commit. - commit-version := "$service-version-$(TEST-COMMIT)" - run-main-test tester tmp-dir commit-version: - uploader.main - --cli=cli - [ - "service", - "--sdk-version", sdk-version, - "--service-version", service-version, - "--commit", TEST-COMMIT, - "--snapshot-directory", tmp-dir, - ] - - // Try with local. - - // With service version. - local-version := "$service-version-$Time.now" - run-main-test tester tmp-dir local-version: - uploader.main - --cli=cli - [ - "service", - "--sdk-version", sdk-version, - "--service-version", local-version, - "--local", - "--snapshot-directory", tmp-dir, - ] - - // Without service version. - run-main-test tester tmp-dir ARTEMIS-VERSION: - uploader.main - --cli=cli - [ - "service", - "--sdk-version", sdk-version, - "--local", - "--snapshot-directory", tmp-dir, - ] - - finally: - git.tag --delete --name=service-version - - // Download a service. - downloader.main - --cli=cli - [ - "--sdk-version", sdk-version, - "--service-version", service-version, - "--output-directory", tmp-dir, - ] - - // Check that the file was downloaded. - files-iterator := directory.DirectoryStream tmp-dir - files := [] - while file-name := files-iterator.next: files.add file-name - files-iterator.close - // Two different snapshots due to two different chip families. - expect-equals 2 files.size - - delete-service-version tester service-version - -expect-exit-1 [block]: - exception := catch: block.call - expect exception is TestExit diff --git a/tests/utils.toit b/tests/utils.toit index f14481c79..ec3004943 100644 --- a/tests/utils.toit +++ b/tests/utils.toit @@ -28,7 +28,6 @@ import artemis.service.brokers.broker show BrokerConnection BrokerService import artemis.service.device show Device import artemis.service.storage show Storage import artemis.cli.utils show write-blob-to-file read-base64-ubjson -import ..tools.service-image-uploader.uploader as uploader import monitor import .artemis-server import .broker @@ -430,43 +429,6 @@ class Tester: return device-description - /** - Ensures that there exists a service image uploaded. - */ - ensure-available-artemis-service - --sdk-version=TEST-SDK-VERSION - --artemis-version=TEST-ARTEMIS-VERSION: - if TEST-ARTEMIS-VERSION != configured-version.ARTEMIS-VERSION: - // The uploader will call 'make' if the versions don't match. - // That's not safe when multiple tests run in parallel. The - // testing framework has a pre-build step that ensures that the - // versions match. - throw "The configured version is not the test version" - with-tmp-directory: | admin-tmp-dir | - admin-config := cli-pkg.Config - --app-name="test" - --path="$admin-tmp-dir/config" - --data=(deep-copy_ cli.config.data) - ui := TestUi - admin-cli := cli-pkg.Cli "test" - --config=admin-config - --ui=ui - --cache=cli.cache - artemis-pkg.main --cli=admin-cli [ - "auth", "login", - "--email", ADMIN-EMAIL, - "--password", ADMIN-PASSWORD, - ] - uploader.main --cli=admin-cli - [ - "service", - "--sdk-version", sdk-version, - "--service-version", artemis-version, - "--force", - "--local" - ] - - /** Stops the main broker. diff --git a/tools/service_image_uploader/CMakeLists.txt b/tools/service_image_downloader/CMakeLists.txt similarity index 64% rename from tools/service_image_uploader/CMakeLists.txt rename to tools/service_image_downloader/CMakeLists.txt index dea6deef5..15b6335b0 100644 --- a/tools/service_image_uploader/CMakeLists.txt +++ b/tools/service_image_downloader/CMakeLists.txt @@ -2,17 +2,6 @@ toit_project(uploader "${CMAKE_CURRENT_LIST_DIR}") -set(UPLOADER_SOURCE "${CMAKE_CURRENT_LIST_DIR}/uploader.toit") -set(UPLOADER_EXE "${CMAKE_BINARY_DIR}/bin/uploader${CMAKE_EXECUTABLE_SUFFIX}") -set(UPLOADER_DEP "${CMAKE_CURRENT_BINARY_DIR}/uploader.toit.dep") - -ADD_TOIT_EXE( - ${UPLOADER_SOURCE} - ${UPLOADER_EXE} - ${UPLOADER_DEP} - "" -) - set(DOWNLOADER_SOURCE "${CMAKE_CURRENT_LIST_DIR}/downloader.toit") set(DOWNLOADER_EXE "${CMAKE_BINARY_DIR}/bin/downloader${CMAKE_EXECUTABLE_SUFFIX}") set(DOWNLOADER_DEP "${CMAKE_CURRENT_BINARY_DIR}/downloader.toit.dep") diff --git a/tools/service_image_uploader/client.toit b/tools/service_image_downloader/client.toit similarity index 100% rename from tools/service_image_uploader/client.toit rename to tools/service_image_downloader/client.toit diff --git a/tools/service_image_uploader/downloader.toit b/tools/service_image_downloader/downloader.toit similarity index 100% rename from tools/service_image_uploader/downloader.toit rename to tools/service_image_downloader/downloader.toit diff --git a/tools/service_image_uploader/package.lock b/tools/service_image_downloader/package.lock similarity index 100% rename from tools/service_image_uploader/package.lock rename to tools/service_image_downloader/package.lock diff --git a/tools/service_image_uploader/package.yaml b/tools/service_image_downloader/package.yaml similarity index 100% rename from tools/service_image_uploader/package.yaml rename to tools/service_image_downloader/package.yaml diff --git a/tools/service_image_uploader/sdk-downloader.toit b/tools/service_image_downloader/sdk-downloader.toit similarity index 100% rename from tools/service_image_uploader/sdk-downloader.toit rename to tools/service_image_downloader/sdk-downloader.toit diff --git a/tools/service_image_uploader/utils.toit b/tools/service_image_downloader/utils.toit similarity index 100% rename from tools/service_image_uploader/utils.toit rename to tools/service_image_downloader/utils.toit diff --git a/tools/service_image_uploader/uploader.toit b/tools/service_image_uploader/uploader.toit deleted file mode 100755 index b3f1ed84b..000000000 --- a/tools/service_image_uploader/uploader.toit +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env toit.run - -// Copyright (C) 2023 Toitware ApS. All rights reserved. - -import ar -import certificate-roots -import cli show * -import encoding.url as url-encoding - -import artemis.cli.cache show cache-key-service-image -import artemis.cli.git show Git -import artemis.cli.sdk show * -import artemis.shared.version show ARTEMIS-VERSION -import host.file -import host.pipe -import uuid show Uuid -import snapshot show cache-snapshot extract-uuid -import supabase - -import .client -import .utils - -CHIP-FAMILIES ::= [ - "esp32", - "host", -] - -main args/List: - // Use the same application name as the - // This way we get the same config and cache. - // The config gives as the server configurations and oauth tokens. - // The cache the SDKs. - cli := Cli "artemis" - main args --cli=cli - -main args/List --cli/Cli?: - certificate-roots.install-all-trusted-roots - - cmd := Command "uploader" - --help=""" - Administrative tool to upload CLI snapshots and Artemis service - images to the Artemis server. - - Make sure to be authenticated against the Artemis server. - """ - --options=[ - Option "server" - --help="The server to upload to.", - Option "snapshot-directory" - --help="The directory to store the snapshot in.", - ] - - cli-snapshot-cmd := Command "cli-snapshot" - --help=""" - Uploads the CLI snapshot to the Artemis server. - - After downloading it again with the downloader, allows to - decode CLI system messages. - - Also copies the snapshot into the snapshot directory. - """ - --rest=[ - Option "snapshot" - --help="The snapshot to upload." - --type="file" - --required, - ] - --run=:: upload-cli-snapshot it - cmd.add cli-snapshot-cmd - - service-cmd := Command "service" - --help=""" - Builds and uploads the Artemis service image. - - Downloads the SDK if necessary. - - The service is taken from this repository. - - There are three ways to specify which code should be built: - 1. If '--service-version' is specified, the code of the - specified version is built, by cloning this repository - into a temporary directory and checking out the specified - version. - 2. If '--commit' is specified, the code of the specified commit - is built, by cloning this repository into a temporary directory - and checking out the specified commit. The full version string - (as seen in the database) is then '-'. - 3. If '--local' is specified, builds the service from the checked - out code. If no service-version is provided, uses the one in the - version.toit file. - - Note that there can only be one service/sdk version combination. - Even if a version is uploaded to a specific organization-id, there - can't be the same version for other organizations. - - The built image is then uploaded to the Artemis server. - """ - --options=[ - Option "sdk-version" - --help="The version of the SDK to use." - --required, - Option "service-version" - --help="The version of the service to use.", - Option "commit" - --help="The commit to build.", - Flag "local" - --help="Build the service from the checked out code of the current repository.", - Option "organization-id" - --help="The organization ID to upload the service to.", - Flag "force" - --short-name="f" - --help="Force the upload, even if the service already exists." - --default=false, - Option "optimization-level" - --short-name="O" - --help="The optimization level to use." - --default="2", - ] - --examples=[ - Example "Upload the checked out code as version v0.5.5.pre.1+fix.foo to organization 3ea5b632-5739-4f40-8446-2fc102a5b338:" - --arguments="--sdk-version=v2.0.0-alpha.139 --service-version=v0.5.5.pre.1+fix.foo --organization-id=3ea5b632-5739-4f40-8446-2fc102a5b338", - Example "Upload the commit faea5684479957e48b945f7fdf4cbc70c0053225 as version v0.5.5.pre.2+updated to organization 3ea5b632-5739-4f40-8446-2fc102a5b338:" - --arguments="--sdk-version=v2.0.0-alpha.139 --service-version=v0.5.5.pre.2+updated --commit=faea5684479957e48b945f7fdf4cbc70c0053225 --organization-id=3ea5b632-5739-4f40-8446-2fc102a5b338", - Example "Upload the build v2.0.0-alpha.140/v0.5.5 to the server for everyone:" - --arguments="--sdk-version=v2.0.0-alpha.140 --service-version=v0.5.5" - ] - --run=:: build-and-upload it - cmd.add service-cmd - - cmd.run args --cli=cli - -service-path-in-repository root/string --chip-family/string -> string: - return "$root/src/service/run/$(chip-family).toit" - -build-and-upload invocation/Invocation: - params := invocation.parameters - cli := invocation.cli - ui := cli.ui - - sdk-version := params["sdk-version"] - service-version := params["service-version"] - commit := params["commit"] - use-local := params["local"] - snapshot-directory := params["snapshot-directory"] - organization-id := params["organization-id"] - force := params["force"] - optimization-level := params["optimization-level"] - - git := Git --cli=cli - // Get the SDK. - sdk := get-sdk sdk-version --cli=cli - root := git.current-repository-root - - with-tmp-directory: | tmp-dir/string | - full-service-version/string := ? - repo-path/string := ? - - if use-local: - // Build the service from the checked out code. - // No caching is possible. - // The full version string is then '-', - // where the timestamp is the time when the build was started. - - repo-path = root - full-service-version = service-version or ARTEMIS-VERSION - else: - ui.emit --info "Cloning repository and checking out $(commit or service-version)." - repo-path = "$tmp-dir/artemis" - git.init repo-path --origin="file://$(url-encoding.encode root)" - git.config --repository-root=repo-path - --key="advice.detachedHead" - --value="false" - git.fetch - --checkout - --depth=1 - --repository-root=repo-path - --ref=(commit or service-version) - - ui.emit --info "Downloading packages." - sdk.pkg-install --project-root=repo-path - - full-service-version = service-version - if commit: full-service-version += "-$commit" - - artemis-config := get-artemis-config --cli=cli - // Since we are potentially reusing an ID, we need to remove the cached versions. - CHIP-FAMILIES.do: | chip-family | - [32, 64].do: | word-size | - cache-key := cache-key-service-image - --sdk-version=sdk-version - --service-version=full-service-version - --artemis-config=artemis-config - --chip-family=chip-family - --word-size=word-size - cli.cache.remove cache-key - - service-source-paths := CHIP-FAMILIES.map: | chip-family/string | - service-path-in-repository repo-path --chip-family=chip-family - - if full-service-version != ARTEMIS-VERSION: - // Update the version.toit file. - ui.emit --info "Generating version.toit." - exit-status := pipe.run-program - --environment={"ARTEMIS_GIT_VERSION": full-service-version} - ["make", "-C", repo-path, "rebuild-cmake"] - if exit-status != 0: throw "make failed with exit code $(pipe.exit-code exit-status)" - - ar-file := "$tmp-dir/service.ar" - ui.emit --info "Creating snapshot." - - snapshot-paths := {:} - CHIP-FAMILIES.do: | chip-family/string | - service-source-path := service-path-in-repository repo-path --chip-family=chip-family - if not file.is-file service-source-path: - throw "Service source file '$service-source-path' does not exist." - snapshot-path := "$tmp-dir/service-$(chip-family).snapshot" - sdk.compile-to-snapshot service-source-path - --out=snapshot-path - --flags=["-O$optimization-level"] - snapshot-paths[chip-family] = snapshot-path - - create-image-archive snapshot-paths --sdk=sdk --out=ar-file - - with-upload-client invocation: | client/UploadClient | - image-id := (Uuid.uuid5 "artemis" - "$Time.monotonic-us $sdk-version $full-service-version").stringify - - image-contents := file.read-contents ar-file - snapshots := snapshot-paths.map: | _ snapshot-path | file.read-contents snapshot-path - client.upload - --sdk-version=sdk-version - --service-version=full-service-version - --image-id=image-id - --image-contents=image-contents - --snapshots=snapshots - --organization-id=organization-id - --force=force - - snapshots.do: | _ snapshot-contents/ByteArray | - cache-snapshot snapshot-contents - --output-directory=snapshot-directory - -create-image-archive snapshot-paths/Map --sdk/Sdk --out/string: - ar-stream := file.Stream.for-write out - ar-writer := ar.ArWriter ar-stream - - ar-writer.add "artemis" """{ "magic": "�", "version": 2 }""" - - with-tmp-directory: | tmp-dir/string | - snapshot-paths.do: | chip-family/string snapshot-path/string | - [32, 64].do: | word-size | - // Note that 'ar' file names can only be 15 characters long. - image-name := "$(chip-family)-$(word-size).img" - image-path := "$tmp-dir/$image-name" - sdk.compile-snapshot-to-image - --snapshot-path=snapshot-path - --out=image-path - --word-sizes=[word-size] - - ar-writer.add image-name (file.read-contents image-path) - if chip-family == "esp32": - // Add the same image again with the deprecated name. - // TODO(florian): remove deprecated image name without chip-family. - ar-writer.add "service-$(word-size).img" (file.read-contents image-path) - - ar-stream.close - -upload-cli-snapshot invocation/Invocation: - snapshot := invocation["snapshot"] - snapshot-directory := invocation["snapshot-directory"] - - snapshot-contents := file.read-contents snapshot - with-upload-client invocation: | client/UploadClient | - uuid := extract-uuid snapshot-contents - client.upload snapshot-contents --snapshot-uuid=uuid - - cache-snapshot snapshot-contents - --output-directory=snapshot-directory