Build the Artemis service from source.

.github/workflows/build.yml

@@ -104,51 +104,6 @@ jobs:
           echo "ARTEMIS_UUID=$UUID" >> $GITHUB_ENV
           mv build/snapshots/artemis.snapshot build/snapshots/artemis-$UUID.snapshot
 
-      - name: Sign in to Artemis
-        shell: bash
-        env:
-          ARTEMIS_EMAIL: leon@toit.io
-          ARTEMIS_PASSWORD: ${{ secrets.LEON_ARTEMIS_PW }}
-        run: |
-          # We log in with the Artemis executable.
-          # It will set the authentication in the config file which is also
-          # used by the uploader.
-          build/bin/artemis auth login --email "$ARTEMIS_EMAIL" --password "$ARTEMIS_PASSWORD"
-
-      - name: Upload snapshot to Artemis
-        if: github.event_name == 'release' || inputs.upload_service_snapshot
-        shell: bash
-        run: |
-          # No need to cache the snapshots in a local snapshot directory.
-          mkdir -p tmp_snapshots
-          build/bin/uploader cli-snapshot \
-            --snapshot-directory tmp_snapshots \
-            build/snapshots/*.snapshot
-
-      - name: Build and upload service images
-        if: github.event_name == 'release'
-        shell: bash
-        run: |
-          # No need to cache the snapshots in a local snapshot directory.
-          mkdir -p tmp_snapshots
-          for version in $SUPPORTED_SDK_VERSIONS; do
-            # For each supported SDK version, build the locally checked out
-            # version of the service and upload it to Artemis.
-            build/bin/uploader service \
-                --snapshot-directory tmp_snapshots \
-                --sdk-version $version \
-                --service-version ${{ github.event.release.tag_name }} \
-                --local \
-                --force
-            build/bin/uploader service \
-                --snapshot-directory tmp_snapshots \
-                --sdk-version $version \
-                --service-version "${{ github.event.release.tag_name }}+O1" \
-                --local \
-                --optimization-level=1 \
-                --force
-          done
-
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:

.github/workflows/ci.yml

@@ -29,7 +29,6 @@ jobs:
             run_supabase_tests: true
           - os: ubuntu-latest
             run_supabase_tests: false
-            upload-service: true
 
     runs-on: ${{ matrix.os }}
 
@@ -185,36 +184,6 @@ jobs:
           make disable-supabase-tests
           make test
 
-      - name: Upload service for Testing org
-        if: |
-          matrix.upload-service &&
-          github.event_name == 'push' &&
-          github.ref == 'refs/heads/main'
-        shell: bash
-        env:
-          ARTEMIS_EMAIL: leon@toit.io
-          ARTEMIS_PASSWORD: ${{ secrets.LEON_ARTEMIS_PW }}
-        run: |
-          # We log in with the Artemis executable.
-          # It will set the authentication in the config file which is also
-          # used by the uploader.
-          build/bin/artemis$BIN_EXTENSION auth login --email "$ARTEMIS_EMAIL" --password "$ARTEMIS_PASSWORD"
-
-          SDK_VERSION=$(make dev-sdk-version)
-          VERSION=$(cmake -DPRINT_VERSION=1 -P tools/gitversion.cmake)
-          # It's hard to ensure that we don't upload a service with the same tag
-          # as a release, and that would lead to trouble.
-          # Add a T to indicate that this is a test version and avoid this problem.
-          SERVICE_VERSION=$VERSION-T
-
-          ARTEMIS_TESTING_ORGANIZATION=3ea5b632-5739-4f40-8446-2fc102a5b338
-
-          build/bin/uploader service \
-              --sdk-version $SDK_VERSION \
-              --service-version $SERVICE_VERSION \
-              --organization-id $ARTEMIS_TESTING_ORGANIZATION \
-              --local
-
       - name: Upload binary artifacts
         if: (!matrix.run_supabase_tests)
         uses: actions/upload-artifact@v4
@@ -265,8 +234,7 @@ jobs:
         run: |
           echo "ARTEMIS_CONFIG=${{ github.workspace }}/test_config" >> $GITHUB_ENV
 
-      - name: Install dependencies - Linux
-        if: runner.os == 'Linux'
+      - name: Install dependencies
         run: |
           sudo apt-get update -y -q
           sudo apt-get install -y -q --no-install-recommends ninja-build
@@ -300,16 +268,20 @@ jobs:
         uses: pguyot/arm-runner-action@v2
         with:
           cpu: cortex-a7
-          base_image: "raspios_lite_arm64:2022-01-28"
+          base_image: "raspios_lite_arm64:2022-04-04"
           image_additional_mb: 500
           commands: |
               uname -m
+              sudo apt-get update -y
+              sudo apt-get install -y git
               export HOME=$(pwd)
               export ARTEMIS_CONFIG=$HOME/config
               build/bin/artemis --version
               mkdir fleet
               export ARTEMIS_FLEET_ROOT=$PWD/fleet
               build/bin/artemis fleet init
+              sed -i 's/artemis-version.*//' fleet/my-pod.yaml
+              echo "artemis-version: ${{ github.ref }}" >> fleet/my-pod.yaml
               build/bin/artemis pod build -o test.pod fleet/my-pod.yaml
 
       - name: Kill http-servers

CMakeLists.txt

@@ -41,5 +41,5 @@ add_subdirectory(tests)
 add_subdirectory(src/cli)
 add_subdirectory(tools/http_servers)
 add_subdirectory(tools/lan_ip)
-add_subdirectory(tools/service_image_uploader)
+add_subdirectory(tools/service_image_downloader)
 add_subdirectory(tools/snapshot)

Makefile

@@ -175,32 +175,21 @@ setup-local-dev:
 
 	@ $(TOITRUN) src/cli/cli.toit org add "Test Org"
 
-	@ $(MAKE) upload-service
-
 .PHONY: dev-sdk-version
 dev-sdk-version:
 	@ echo $(LOCAL_DEV_SDK)
 
-.PHONY: upload-service
-upload-service:
-	@ $(TOITRUN) tools/service_image_uploader/uploader.toit service --local --force \
-		--sdk-version=$(LOCAL_DEV_SDK) \
-		--service-version=$(SETUP_LOCAL_DEV_SERVICE)
-	@ $(TOITRUN) tools/service_image_uploader/uploader.toit service --local --force \
-		--sdk-version=$(LOCAL_DEV_SDK) \
-		--service-version=$$(cmake -DPRINT_VERSION=1 -P tools/gitversion.cmake)
-
 .PHONY: download-sdk
 download-sdk: install-pkgs
-	@ $(TOITRUN) tools/service_image_uploader/sdk-downloader.toit download \
+	@ $(TOITRUN) tools/service_image_downloader/sdk-downloader.toit download \
 	    --version $(LOCAL_DEV_SDK) \
 			--envelope=esp32,esp32-qemu
 	@ cmake \
 		-DDEV_SDK_VERSION=$(LOCAL_DEV_SDK) \
-		-DDEV_SDK_PATH="$$($(TOITRUN) tools/service_image_uploader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print)" \
-		-DDEV_ENVELOPE_ESP32_PATH="$$($(TOITRUN) tools/service_image_uploader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --envelope="esp32")" \
-		-DDEV_ENVELOPE_ESP32_QEMU_PATH="$$($(TOITRUN) tools/service_image_uploader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --envelope="esp32-qemu")" \
-		-DDEV_ENVELOPE_HOST_PATH="$$($(TOITRUN) tools/service_image_uploader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --host-envelope)" \
+		-DDEV_SDK_PATH="$$($(TOITRUN) tools/service_image_downloader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print)" \
+		-DDEV_ENVELOPE_ESP32_PATH="$$($(TOITRUN) tools/service_image_downloader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --envelope="esp32")" \
+		-DDEV_ENVELOPE_ESP32_QEMU_PATH="$$($(TOITRUN) tools/service_image_downloader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --envelope="esp32-qemu")" \
+		-DDEV_ENVELOPE_HOST_PATH="$$($(TOITRUN) tools/service_image_downloader/sdk-downloader.toit --version $(LOCAL_DEV_SDK) print --host-envelope)" \
 		build
 
 # We rebuild the cmake file all the time.

src/cli/artemis.toit

@@ -9,11 +9,16 @@ import uuid show Uuid
 import encoding.base64
 import encoding.ubjson
 import encoding.json
+import fs
+import host.os
+import system
 
 import .cache as cache
 import .cache show cache-key-service-image
 import .config
 import .device
+import .git
+import .pod-specification
 
 import .utils
 
@@ -73,19 +78,6 @@ class Artemis:
   ensure-authenticated -> none:
     connected-artemis-server_
 
-  /**
-  Checks whether the given $sdk version and $service version is supported by
-    the Artemis server.
-  */
-  check-is-supported-version_ --organization-id/Uuid --sdk/string?=null --service/string?=null:
-    server := connected-artemis-server_
-    versions := server.list-sdk-service-versions
-        --organization-id=organization-id
-        --sdk-version=sdk
-        --service-version=service
-    if versions.is-empty:
-      cli_.ui.abort "Unsupported Artemis/SDK versions ($service/$sdk)."
-
   notify-created --hardware-id/Uuid:
     server := connected-artemis-server_
     server.notify-created --hardware-id=hardware-id
@@ -95,49 +87,6 @@ class Artemis:
         --device-id=device-id
         --organization-id=organization-id
 
-  /**
-  Gets the Artemis service image for the given $sdk and $service versions.
-
-  Returns a path to the cached image.
-  */
-  get-service-image-path -> string
-      --organization-id/Uuid
-      --sdk/string
-      --service/string
-      --chip-family/string
-      --word-size/int:
-    if word-size != 32 and word-size != 64: throw "INVALID_ARGUMENT"
-    service-key := cache-key-service-image
-        --service-version=service
-        --sdk-version=sdk
-        --artemis-config=server-config
-        --chip-family=chip-family
-        --word-size=word-size
-    return cli_.cache.get-file-path service-key: | store/FileStore |
-      server := connected-artemis-server_ --no-authenticated
-      entry := server.list-sdk-service-versions
-          --organization-id=organization-id
-          --sdk-version=sdk
-          --service-version=service
-      if entry.is-empty:
-        cli_.ui.abort "Unsupported Artemis/SDK versions."
-      image-name := entry.first["image"]
-      service-image-bytes := server.download-service-image image-name
-      ar-reader := ar.ArReader.from-bytes service-image-bytes
-      artemis-file := ar-reader.find "artemis"
-      metadata := json.decode artemis-file.contents
-      // Reset the reader. The images should be after the metadata, but
-      // doesn't hurt.
-      ar-reader = ar.ArReader.from-bytes service-image-bytes
-      if metadata["version"] == 1:
-        if chip-family != "esp32":
-          cli_.ui.abort "Unsupported chip family '$chip-family' for service $service and SDK $sdk."
-        ar-file := ar-reader.find "service-$(word-size).img"
-        store.save ar-file.contents
-      else:
-        ar-file := ar-reader.find "$(chip-family)-$(word-size).img"
-        store.save ar-file.contents
-
   /**
   Fetches the organizations with the given $id.
 
@@ -146,22 +95,34 @@ class Artemis:
   get-organization --id/Uuid -> OrganizationDetailed?:
     return connected-artemis-server_.get-organization id
 
-  /**
-  List all SDK/service version combinations.
-
-  Returns a list of maps with the following keys:
-  - "sdk_version": the SDK version
-  - "service_version": the service version
-  - "image": the name of the image
-
-  If provided, the given $sdk-version and $service-version can be
-    used to filter the results.
-  */
-  list-sdk-service-versions -> List
-      --organization-id/Uuid
-      --sdk-version/string?
-      --service-version/string?:
-    return connected-artemis-server_.list-sdk-service-versions
-        --organization-id=organization-id
-        --sdk-version=sdk-version
-        --service-version=service-version
+service-path-in-repository root/string --chip-family/string -> string:
+  return "$root/src/service/run/$(chip-family).toit"
+
+ARTEMIS-SERVICE-GIT-URL ::= "https://github.com/toitware/artemis"
+
+get-artemis-container version-or-path/string --chip-family/string --cli/Cli -> ContainerPath:
+  artemis-root-path := os.env.get "ARTEMIS_REPO_PATH"
+  if artemis-root-path:
+    entrypoint := service-path-in-repository artemis-root-path --chip-family=chip-family
+    return ContainerPath "artemis" --entrypoint=entrypoint
+  if is-dev-setup:
+    git := Git --cli=cli
+    artemis-path := fs.dirname system.program-path
+    root := git.current-repository-root --path=artemis-path
+    entrypoint := service-path-in-repository root --chip-family=chip-family
+    return ContainerPath "artemis" --entrypoint=entrypoint
+
+  url/string := ?
+  if version-or-path.starts-with "http://" or version-or-path.starts-with "https://":
+    url = version-or-path
+  else if version-or-path.starts-with "file:/":
+    return ContainerPath "artemis" --entrypoint=(version-or-path.trim --left "file:/")
+  else:
+    // This is a version string.
+    url = ARTEMIS-SERVICE-GIT-URL
+
+  version := version-or-path
+  return ContainerPath "artemis"
+      --entrypoint=(service-path-in-repository "." --chip-family=chip-family)
+      --git-url=url
+      --git-ref=version

src/cli/artemis_servers/artemis-server.toit

@@ -140,30 +140,6 @@ interface ArtemisServerCli implements Authenticatable:
   // TODO(florian): add support for changing the email.
   update-profile --name/string
 
-  /**
-  List all SDK/service version combinations.
-
-  Returns a list of maps with the following keys:
-  - "sdk_version": the SDK version
-  - "service_version": the service version
-  - "image": the name of the image
-
-  If provided, the given $sdk-version and $service-version can be
-    used to filter the results.
-  */
-  list-sdk-service-versions -> List
-      --organization-id/Uuid
-      --sdk-version/string?=null
-      --service-version/string?=null
-
-  /**
-  Downloads the given $image.
-
-  The $image must be a valid image name, as returned by
-    $list-sdk-service-versions.
-  */
-  download-service-image image/string -> ByteArray
-
 with-server server-config/ServerConfig --cli/Cli [block]:
   network := net.open
   server/ArtemisServerCli? := null

src/cli/artemis_servers/http/base.toit

@@ -164,21 +164,6 @@ class ArtemisServerCliHttpToit implements ArtemisServerCli:
       "name": name,
     }
 
-  list-sdk-service-versions -> List
-      --organization-id/Uuid
-      --sdk-version/string?=null
-      --service-version/string?=null:
-    return send-request_ COMMAND-LIST-SDK-SERVICE-VERSIONS_ {
-      "organization_id": "$organization-id",
-      "sdk_version": sdk-version,
-      "service_version": service-version,
-    }
-
-  download-service-image image/string -> ByteArray:
-    return send-request_ COMMAND-DOWNLOAD-SERVICE-IMAGE_ {
-      "image": image,
-    }
-
   send-request_ command/int data/Map -> any:
     encoded/ByteArray := #[command] + (json.encode data)
     headers := http.Headers

src/cli/artemis_servers/supabase/supabase.toit

@@ -154,20 +154,3 @@ class ArtemisServerCliSupabase implements ArtemisServerCli:
     client_.rest.update "profiles"  { "name": name } --filters=[
       equals "id" "$user-id",
     ]
-
-  list-sdk-service-versions -> List
-      --organization-id/Uuid
-      --sdk-version/string?=null
-      --service-version/string?=null:
-    filters := [
-      orr [
-        is-null "organization_id",
-        equals "organization_id" "$organization-id",
-      ]
-    ]
-    if sdk-version: filters.add (equals "sdk_version" sdk-version)
-    if service-version: filters.add (equals "service_version" service-version)
-    return client_.rest.select "sdk_service_versions" --filters=filters
-
-  download-service-image image/string -> ByteArray:
-    return client_.storage.download --public --path="service-images/$image"