diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 72eb1226..605d8a66 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -99,7 +99,7 @@ jobs: with: name: generated-documentation path: | - system-design/specification/application-package/application-description.md + system-design/specification/applications/application-description.md system-design/specification/margo-management-interface/desired-state.md pages: diff --git a/.gitignore b/.gitignore index e228571f..d687d211 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ myenv # Ignore the generated files: system-design/specification/margo-management-interface/desired-state.md -system-design/specification/application-package/application-description.md -src/specification/application-package/docs +system-design/specification/applications/application-description.md +src/specification/applications/docs src/specification/margo-management-interface/docs *.code-workspace diff --git a/doc-generation/configurations/application-package-api.json b/doc-generation/configurations/application-package-api.json index fdc1e0c9..c6fe0e40 100644 --- a/doc-generation/configurations/application-package-api.json +++ b/doc-generation/configurations/application-package-api.json @@ -1,5 +1,5 @@ { - "root": "src/specification/application-package", + "root": "src/specification/applications", "targetclass": "ApplicationDescription", "schemafile": "application-description.linkml.yaml", "markdowndoc": "application-description.md" diff --git a/mkdocs.yml b/mkdocs.yml index 9433a452..309efb45 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -10,15 +10,15 @@ nav: - personas-and-definitions/software-composition.md - Overview: - overview/envisioned-system-design.md - - overview/workloads.md + - overview/applications.md - overview/workload-fleet-management.md - overview/edge-compute-devices.md - overview/workload-observability.md - Concepts: - - Workloads: - - concepts/workloads/application-package.md - - concepts/workloads/application-registry.md - - concepts/workloads/local-registries.md + - Applications: + - concepts/applications/application-package.md + - concepts/applications/application-registry.md + - concepts/applications/local-registries.md - Workload Fleet Managers: - concepts/workload-fleet-managers/device-client-onboarding.md - concepts/workload-fleet-managers/device-capabilities.md @@ -34,8 +34,9 @@ nav: - specification/margo-management-interface/desired-state.md - specification/margo-management-interface/deployment-status.md - specification/margo-management-interface/management-interface-swagger.md - - Application Package: - - specification/application-package/application-description.md + - Applications: + - specification/applications/application-description.md + - specification/applications/application-registry.md - Margo Devices: - specification/margo-devices/device-requirements.md - Observability: diff --git a/src/specification/application-package/application-description.linkml.yaml b/src/specification/applications/application-description.linkml.yaml similarity index 98% rename from src/specification/application-package/application-description.linkml.yaml rename to src/specification/applications/application-description.linkml.yaml index c95170ba..84567be0 100644 --- a/src/specification/application-package/application-description.linkml.yaml +++ b/src/specification/applications/application-description.linkml.yaml @@ -3,10 +3,8 @@ id: http://specification.margo.org/application-schema name: ApplicationDescription title: Application Description Schema description: >- - Schema for defining application metadata, deployment profiles, parameters, and configuration. - The purpose of this file is to present the application on an application catalog or marketplace - from where an end user selects the application to hand it over to the Workload Orchestration Software, - which configures it and makes it available for installation on the edge device. + Schema for defining application metadata, deployment profiles, and parameters. + The purpose of the Application Description is to enable an application's discovery, configuration, and deployment on edge devices. version: 1.0 prefixes: linkml: https://w3id.org/linkml/ @@ -97,8 +95,7 @@ classes: required: true catalog: description: >- - Catalog element specifying the application catalog details used to display the application - in an application catalog or marketplace. + Catalog element specifying the application's metadata for enabling its discovery. See the [Catalog](#catalog-attributes) section below. rank: 40 range: Catalog diff --git a/src/specification/application-package/resources/class.md.jinja2 b/src/specification/applications/resources/class.md.jinja2 similarity index 100% rename from src/specification/application-package/resources/class.md.jinja2 rename to src/specification/applications/resources/class.md.jinja2 diff --git a/src/specification/application-package/resources/examples/valid/ApplicationDescription-001.yaml b/src/specification/applications/resources/examples/valid/ApplicationDescription-001.yaml similarity index 100% rename from src/specification/application-package/resources/examples/valid/ApplicationDescription-001.yaml rename to src/specification/applications/resources/examples/valid/ApplicationDescription-001.yaml diff --git a/src/specification/application-package/resources/examples/valid/ApplicationDescription-002.yaml b/src/specification/applications/resources/examples/valid/ApplicationDescription-002.yaml similarity index 100% rename from src/specification/application-package/resources/examples/valid/ApplicationDescription-002.yaml rename to src/specification/applications/resources/examples/valid/ApplicationDescription-002.yaml diff --git a/src/specification/application-package/resources/index.md.jinja2 b/src/specification/applications/resources/index.md.jinja2 similarity index 93% rename from src/specification/application-package/resources/index.md.jinja2 rename to src/specification/applications/resources/index.md.jinja2 index 3d3bd68d..7e4e1b3c 100644 --- a/src/specification/application-package/resources/index.md.jinja2 +++ b/src/specification/applications/resources/index.md.jinja2 @@ -1,4 +1,9 @@ -The application description has the purpose of presenting the application, e.g., on an application catalog or marketplace from where an end user selects an application to be installed. The end user defines an `ApplicationDeployment` by specifying [configuration parameters](#defining-configurable-application-parameters) for an `ApplicationDescription`. An `ApplicationDeployment` defines the [desired state](../margo-management-interface/desired-state.md) for an application. +# Application Description + +The purpose of the Application Description is to enable an application's discovery, configuration, and deployment on edge devices. +To deploy an application the end user specifies values for the [parameters](#defining-configurable-application-parameters) given in +an Application Description (e.g., through a UI of the WFM) to instantiate an `ApplicationDeployment`, +which defines the [desired state](../margo-management-interface/desired-state.md) for an application. ### Top-level Attributes diff --git a/system-design/concepts/applications/application-package.md b/system-design/concepts/applications/application-package.md new file mode 100644 index 00000000..362eedae --- /dev/null +++ b/system-design/concepts/applications/application-package.md @@ -0,0 +1,69 @@ +# Application Package + +The Application Package, which is used to [distribute an application](../../overview/applications.md) to be deployed as [workloads](../../personas-and-definitions/technical-lexicon.md#workload) on edge devices, comprises the following elements: + +- The **Application Description**: a YAML document with the element `kind` defined as `ApplicationDescription`, which is stored in a file (for example named `margo.yaml`) and contains information about the application's [metadata](../../specification/applications/application-description.md#metadata-attributes) (e.g., name, version, author, etc.), the application [components](../../personas-and-definitions/technical-lexicon.md#component) (e.g., Helm charts, Compose Archive), which are defined in [deployment configurations](../../specification/applications/application-description.md#deploymentprofile-attributes), and [configurable application parameters](../../specification/applications/application-description.md#defining-configurable-application-parameters). There is only one YAML file in the package root of kind `ApplicationDescription`. +- The **resources**, which are additional files associated with the application (human-readable description, icon, release notes, or license file) that may be used to provide more information about the application (e.g., in a UI such as an [application catalog](../../personas-and-definitions/technical-lexicon.md#application-catalog) of a [WFM](../../personas-and-definitions/technical-lexicon.md#workload-fleet-manager)). + +The Application Package is distributed via an [Application Registry](./application-registry.md) and is bundled within a **repository** according to the [OCI distribution specification](https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md). +All files (i.e., Application Description and resource files) belonging to the Application Package are referenced as *layers* (the equivalent term **blobs** is nowadays preferred and will be used in the rest of this document) by a **manifest** inside the repository. +The blob of the Application Description file is marked with a **mediaType** value as `application/vnd.margo.app.description.v1+yaml`. +The blobs of resource files are marked with an according **mediaType** (see list [here](../../specification/applications/application-registry.md#margo-specific-media-types)), e.g., an icon in jpeg format is marked with the mediaTye `application/vnd.margo.app.icon.v1+jpeg`. +This is illustrated in the figure below and formally defined [here](../../specification/applications/application-registry.md). + + +``` mermaid +classDiagram +direction LR + class `my-application-description: Blob` { + mediaType: "application/vnd.margo.app.description.v1+yaml" + digest: "sha256:f6b7914..." + size: 999 + } + + class `my-resources-icon: Blob` { + mediaType: "application/vnd.margo.app.icon.v1+jpeg" + digest: "sha256:sha256:451410b6adfdce..." + size: 5065 + } + + class `my-margo-app-manifest: Manifest` { + mediaType: "application/vnd.oci.image.manifest.v1+json" + artifactType: "application/vnd.org.margo.app.v1+json" + } + + `my-margo-app-manifest: Manifest` o-- `my-application-description: Blob` + `my-margo-app-manifest: Manifest` o-- `my-resources-icon: Blob` +``` + + +An application aggregates one or more [components](../../personas-and-definitions/technical-lexicon.md#component). Therefore, the [deployment profiles](../../specification/applications/application-description.md#deploymentprofile-attributes) specified in the [Application Description](../../specification/applications/application-description.md) can be defined as Helm Chart components AND/OR [Compose Archive](../../personas-and-definitions/technical-lexicon.md#compose-archive) components. + +Each component links to one or more [OCI Container Images](https://github.com/opencontainers/image-spec). +The components referenced in the [Application Description](../../specification/applications/application-description.md) are stored in a [Component Registry](../../personas-and-definitions/technical-lexicon.md#component-registry), and the linked container images are provided via a [Container Image Registry](../../personas-and-definitions/technical-lexicon.md#container-image-registry). +Registries can be remote (i.e., Internet-accessible) or [local](../../concepts/applications/local-registries.md) (i.e., accessible within a local network infrastructure of the devices). + + The components are being deployed as workloads on the edge devices: + +- To target devices, which deploy workloads using Kubernetes, components need to be defined as Helm charts using [Helm (version 3)](https://helm.sh/docs/topics/charts/). +- To target devices, which deploy workloads using [Compose](https://www.compose-spec.io/), components need to be packaged as [Compose Archives](../../personas-and-definitions/technical-lexicon.md#compose-archive), i.e., a tarball file containing the `compose.yaml` file and any additional artifacts referenced by the Compose file (e.g., configuration files, environment variable files, etc.). Margo recommends to digitally sign this package and to specify the location of the public key in the `ApplicationDescription` (see `keyLocation` [here](../../specification/applications/application-description.md#componentproperties-attributes)). When digitally signing the package PGP encryption MUST be used. + +If either one cannot be implemented it MAY be omitted but Margo RECOMMENDS defining [deployment profiles](../../specification/applications/application-description.md#deploymentprofile-attributes) as both Helm chart **AND** Compose components to strengthen interoperability and applicability. + +> **Note** +> A device running the application will only install the application using either the Compose Archives or the Helm Charts, but not both. + + +Margo will provide more detailed discussion and specification on the following points: + +> **Investigation Needed**: Question: do we need to specify the location of a SHA256 hash for the Compose Archive also (similar to the PGP key) in the ApplicationDescription? +> We will also discuss how we should handle secure container registries that require a username and password. +> +> **Investigation Needed**: We need to determine what impact, if any, using 3rd party helm charts has on being Margo compliant. +> +> **Investigation Needed**: Missing in the current specification are ways to define dependencies (e.g., application dependencies) as well as required infrastructure services such as storage, message queues/bus, reverse proxy, or authentication/authorization/accounting. + +## Relevant Links +Please follow the subsequent link to view the technical reference of the `ApplicationDescription` format: + +- [Application Description](../../specification/applications/application-description.md) diff --git a/system-design/concepts/applications/application-registry.md b/system-design/concepts/applications/application-registry.md new file mode 100644 index 00000000..9fa3b778 --- /dev/null +++ b/system-design/concepts/applications/application-registry.md @@ -0,0 +1,42 @@ +# Application Registry + +The Margo specification differentiates 3 kinds of registries: *Application Registries*, *Component Registries*, and *Container Image Registries*. + +1. An **Application Registry** hosts Application Packages that define through their [Application Description](../../specification/applications/application-description.md) the application as one or multiple [Components](../../personas-and-definitions/technical-lexicon.md#component). +2. A **Component Registry** hosts the [Components](../../personas-and-definitions/technical-lexicon.md#component) (which are deployable as *workloads*) and are provided as **Helm Charts** or **Compose Archives**. +3. A **Container Image Registry** hosts container images referenced by those Components. + +Out of these 3 registries, **only the Application Registry interface is in scope** of the Margo specification and its API definition can be found [here](../../specification/applications/application-registry.md). + +The diagram below illustrates these functionalities and relationships of registries within Margo. + +```mermaid +flowchart + A[WFM, or internal Application Catalog] -- Application Descriptions link to --> B[Component Registry] + C[Application Registry] -- Application Descriptions link to --> B + B -- hosted Components links to --> D[Container Image Registry] + A -->|pulls Application Package | C + F[App Developer] -->|uploads Application Package to| C + C -->|hosts 0..*| E@{ shape: docs, label: "Application Packages"} + C s1@-->|validates token| H[Authentication Service] + A s2@-->|requests token| H + style H stroke-dasharray: 3 6 + style B fill:#ABC + style C fill:#ABC + style D fill:#ABC + classDef dashed stroke-dasharray: 3,6, stroke-dashoffset: 900; + class s1 dashed + class s2 dashed +``` + +As shown in the figure above, an `Application Developer` uploads an [Application Package](application-package.md) to an Application Registry. From there, it is available to a `Workload Fleet Manager` (WFM). +The WFM acts as a client to pull an [Application Package](application-package.md) from the Application Registry. It would then list this Application Package on its UI (e.g., an internal `Application Catalog`) to enable the usage on its managed devices. + +> Note: The figure above shows an `Authentication Service` that manages access control for the Application Registry. This service and interactions with it are not yet formally defined by Margo and needs to be further discussed in the future. The idea here is that the WFM requests a token from the Authentication Service to include in requests to the Application Registry. The received token is then validated by the Application Registry through interaction with the Authentication Service. + +The Application Registry's API is compliant with the [OCI Registry API (v1.1.0)](https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md). +A hosted Application Package is provided by listing its parts as blobs in an [image manifests](https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md) that can be requested through the API. + + +## Relevant Links +* The technical reference of the Application Registry API is defined [here](../../specification/applications/application-registry.md). diff --git a/system-design/concepts/applications/local-registries.md b/system-design/concepts/applications/local-registries.md new file mode 100644 index 00000000..8bd9fe3c --- /dev/null +++ b/system-design/concepts/applications/local-registries.md @@ -0,0 +1,93 @@ +# Local Registries + +The [Component Registry](application-registry.md) as well as the [Container Image Registry](application-registry.md) considered by Margo are the registries that need to be accessed by devices to download components (i.e., Helm Charts or Compose Archives) or container images. +This section describes how to configure those as local registries to avoid reliance on public Internet-accessible registries. + +The reasons for not using public registries are for example: (1) publicly hosted container images or Helm charts could become unavailable at some point, as the owner decides to take the container images or Helm charts off the public registry, (2) Internet connectivity may not be available to the device and hence public registries are not reachable, or (3) end-users want to host their own registries so they can do security scans and package validation. + +In terms of connectivity, we can thereby distinguish mainly between the following device categories: + +1. **Fully connected device**, which means a device is deployed in the field (e.g., a factory shop floor) and has access to the Internet. +2. **Locally connected device**, i.e., the device has connectivity to a local network (e.g., factory- or enterprise-wide) and a local repository can be made reachable. +3. **Air-gapped device**, i.e., the device generally is not connected and must be configured by accessing it directly (via direct network link with an Ethernet cable, or similar) for example through a technician’s laptop. + +Local Component Registries or Container Image Registries can be used for all device categories. +In case of **fully connected devices**, although the device could reach the Internet, a local registry can still be useful (e.g., as a cache for remote registries to save on bandwidth or to have container images or Helm Charts reliably available). +In case of **locally connected devices**, a local registry is required to enable the Workload Fleet Management Client (WFMC) to install a Margo application on the device, as the device/WFMC does not have Internet access. +Thereby, the local registry can be set up as a *pull-through cache* where data (e.g., container images) is cached locally when they are first retrieved from a remote source and subsequent requests for the same data are served from the local cache rather than fetching it again from the remote source. +In case of **air-gapped devices**, a local registry has to be accessible on the technician's laptop (or other directly connected device), which performs the application installation process. + +To setup local registries, different configuration options exist: + +## Option - Container Image Registry Mirror on Kubernetes Level + +Kubernetes supports the configuration of registry mirrors. How this is configured depends on the distribution and the underlying container runtime. Distributions that utilize **containerd** as runtime (e.g., k3s or microk8s) allow the definition of mirrors in a configuration file. For example, in k3s the file `/etc/rancher/k3s/registries.yaml` can be used to set up a mirror for each device's Kubernetes environment: + +```yaml +mirrors: + "docker.io": + endpoint: + - "http://:5000" +configs: + "docker.io": + auth: + username: "" + password: "" +``` + +## Option - Container Image Registry as Pull-through Cache on Docker Level + +To configure a pull-through cache in Docker for the Container Image Registry, a Docker Registry can be setup that acts as caching proxy for a remote Docker Registry. Such a Docker Registry can be defined using the following `config.yml`: + +```yaml +version: 0.1 +log: + fields: + service: registry +storage: + filesystem: + rootdirectory: /var/lib/registry +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +proxy: + remoteurl: https://registry-1.docker.io +``` + +This registry can be started uing the following command: +`docker run -d -p 5000:5000 --restart unless-stopped --name registry -v $(pwd)/config.yml:/etc/docker/registry/config.yml registry:2` + +Then, the Docker daemon needs to be configured to use the private registry as a mirror in the `/etc/docker/daemon.json` file: + +```json +{ + "registry-mirrors": ["http://10.100.2.102:5000"], + "insecure-registries": ["10.100.2.102:5000"] +} +``` + +## Option - Helm Chart Component Registry as Pull-through Cache + +Setting up a pull-through cache for Helm charts in combination with Kubernetes involves configuring a local Helm chart repository, e.g., ChartMuseum, that can be installed with the `PROXY_CACHE` variable set to `true`: + +```bash +helm repo add chartmuseum https://chartmuseum.github.io/charts +helm repo update +helm install my-chartmuseum chartmuseum/chartmuseum --set env.open.DISABLE_API=false --set env.open.PROXY_CACHE=true +``` + +Then, this Helm Chart repository can be added to Helm and chart releases can be installed there: + +```bash +helm repo add my-cached-repo http://:8080 +helm repo update +helm install my-release my-cached-repo/ +``` + +Now, when deploying applications in Kubernetes using Helm, the cached repository is used to serve charts rather than the remote repository. diff --git a/system-design/concepts/workloads/application-package.md b/system-design/concepts/workloads/application-package.md deleted file mode 100644 index dc1dc02d..00000000 --- a/system-design/concepts/workloads/application-package.md +++ /dev/null @@ -1,41 +0,0 @@ -# Application Package - -The application package, which is used to [distribute an application](../../overview/workloads.md), comprises the following elements: - -- The **application description**: a YAML document with the element `kind` defined as `ApplicationDescription`, which is stored in a file (for example named `margo.yaml`) and contains information about the application's [metadata](../../specification/application-package/application-description.md#metadata-attributes) (e.g., description, icon, release notes, license file, etc.), application supported [deployment configurations](../../specification/application-package/application-description.md#deploymentprofile-attributes) (e.g, Helm charts, Compose Archive), and [configurable application parameters](../../specification/application-package/application-description.md#defining-configurable-application-parameters). There SHALL be only one YAML file in the package root of kind `ApplicationDescription`. -- The **resources**, which are additional files associated with the application (e.g., manual, icon, release notes, license file, etc.) that may be used to display more information about the application in a UI such as an [application catalog](../../personas-and-definitions/technical-lexicon.md#application-catalog) or [marketplace](../../personas-and-definitions/technical-lexicon.md#workload-marketplace) or other informative outputs. - -The application package has the following file/folder structure: - -```yaml -/ # REQUIRED top-level directory -└── .yaml # REQUIRED a YAML document with element 'kind' as 'ApplicationDescription' stored in a file (e.g., 'margo.yaml') -└── resources # OPTIONAL folder with application files (e.g., icon, license file, release notes) that may be used for displaying additional information about the application -``` - -An application aggregates one or more [OCI Containers](https://github.com/opencontainers). While the application package is made available in an [application registry](./application-registry.md), the referenced OCI artifacts are stored in a remote or [local registry](../../concepts/workloads/local-registries.md). - -> **Note** -> Application catalogs or marketplaces are out of scope for Margo. The exact requirements of the marketing material shall be defined by the application marketplace beyond outlined mandatory content. - -The [deployment profiles](../../specification/application-package/application-description.md#deploymentprofile-attributes) specified in the application description SHALL be defined as Helm Chart components AND/OR Compose components. - -- To target devices, which run Kubernetes, applications must be packaged as Helm charts using [Helm (version 3)](https://helm.sh/docs/topics/charts/). -- To target devices, which deploy applications using [Compose](https://www.compose-spec.io/), applications must be packaged as what Margo calls a Compose Archive, i.e., a tarball file containing the `compose.yaml` file and any additional artifacts referenced by the Compose file (e.g., configuration files, environment variable files, etc.). Margo recommends to digitally sign this package and to specify the location of the public key in the `ApplicationDescription` (see `keyLocation` [here](../../specification/application-package/application-description.md#componentproperties-attributes)). When digitally signing the package PGP encryption MUST be used. - -> **Investigation Needed**: Question: do we need to specify the location of a SHA256 hash for the Compose Archive also (similar to the PGP key) in the ApplicationDescription? -> We will also discuss how we should handle secure container registries that require a username and password. -> -> **Investigation Needed**: We need to determine what impact, if any, using 3rd party helm charts has on being Margo compliant. -> -> **Investigation Needed**: Missing in the current specification are ways to define dependencies (e.g., application dependencies) as well as required infrastructure services such as storage, message queues/bus, reverse proxy, or authentication/authorization/accounting. - -If either one cannot be implemented it MAY be omitted but Margo RECOMMENDS defining [deployment profiles](../../specification/application-package/application-description.md#deploymentprofile-attributes) as both Helm chart **AND** Compose components to strengthen interoperability and applicability. - -> **Note** -> A device running the application will only install the application using either the Compose Archives or the Helm Charts, but not both. - -## Relevant Links -Please follow the subsequent link to view the technical reference of the `ApplicationDescription` format: - -- [Application Description](../../specification/application-package/application-description.md) diff --git a/system-design/concepts/workloads/application-registry.md b/system-design/concepts/workloads/application-registry.md deleted file mode 100644 index bf33a90f..00000000 --- a/system-design/concepts/workloads/application-registry.md +++ /dev/null @@ -1,32 +0,0 @@ -# Application Registry - -This section describes the Application Registry and the exchange of an [application package](./application-package.md) from an Application Developer to the Workload Fleet Manager (WFM). - -The Application Developer SHALL use a [Git repository](https://git-scm.com/) to share an [application package](./application-package.md). This Git repository is considered the Application Registry. - -The connectivity between the Workload Fleet Manager and the Application Registry SHALL be read-only. - -Upon installation request from the End User, the Workload Fleet Manager SHALL retrieve the [application package](./application-package.md) using a ``git pull`` request from the Application Registry. - -The Workload Fleet Manager reads in the application description file, ``margo.yaml``, and presents a user interface that allows the specification of parameters available according to ``margo.yaml``. - -The End User then specifies the configuration parameters for the [application package](./application-package.md). - -Then, the [application package](./application-package.md) is ready to be passed on to the installation process. - -> **Note** -> The specifics of the installation process are still under discussion: this could be for example a GitOps based approach. - -During this process the containers referenced in the application manifest ([Helm Chart](https://helm.sh/docs/) or [Compose](https://github.com/compose-spec/compose-spec/blob/master/03-compose-file.md)) are retrieved from container/Helm registries. - -At a minimum, a Margo-compliant Workload Fleet Manager SHALL provide a way for an end user to manually set up a connection between the Workload Fleet Manager and an application registry. This is required so as not to prohibit an end user from being able to install any Margo-compliant application they wish, including applications developed by the end user. - -The Workload Fleet Manager MAY provide enhanced user experience options such as the pre-configuring of application registries to monitor. These can include application registries from third-party application vendors or their own applications. - -## Secure Access to the Application Package - -It is expected the connection between the Workload Fleet Manager and the Application developer’s application registry is secured using standard secure connectivity best practices. Some standard practices include the following: - -- Basic authentication via HTTPS -- Bearer token authentication -- TLS certifications \ No newline at end of file diff --git a/system-design/concepts/workloads/local-registries.md b/system-design/concepts/workloads/local-registries.md deleted file mode 100644 index 7cfa1930..00000000 --- a/system-design/concepts/workloads/local-registries.md +++ /dev/null @@ -1,107 +0,0 @@ -# Local Registries - -The margo specification differentiates 3 different kinds of registries: Application, Component, and Container Registries. - -1. A **Application Registry** hosts the Application Packages that defines through its [Application Description](../../specification/application-package/application-description.md) the application as one or multiple Components. -2. A **Component Registry** hosts the Components (which are deployable as *workloads*) and are provided as **Helm Charts** or **Compose Archives**. -3. A **Container Registry** hosts container images referenced by those Components. - -The diagram below illustrates these functionalities and relationships of registries within margo. - -```mermaid -flowchart - A[WFM, or internal Application Catalog] -- Application Description Links to --> B[Component Registry] - C[Application Registry] -- Application Description links to --> B - B -- Component links to --> D[Container Registry] - A -->|pulls in Application Package from| C - F[App Developer] -->|uploads Applications Packages to| C - G["Marketplace (outside Margo's scope)"] -- points to Application Package --> C - -``` - -This section investigates options for configuring the usage of local OCI-based Component and/or Container Registries. The goal of configuring such local registries is to avoid reliance on public, Internet-accessible registries. - -The reasons for not using such public registries are mainly twofold: (1) publicly hosted container images or Helm charts could become unavailable at some point, as the owner decides to take the container images or Helm charts off the public registry, (2) Internet connectivity may not be available to the device and hence public registries are not reachable, or (3) end-users want to host their own registries so they can do security scans and package validation. - -In terms of connectivity, we can thereby distinguish mainly between the following device categories: - -1. **Fully connected device**, which means a device is deployed in the field (e.g., a factory shop floor) and has access to the Internet. -2. **Locally connected device**, i.e., the device has connectivity to a local network (e.g., factory- or enterprise-wide) and a local repository can be made reachable. -3. **Air-gapped device**, i.e., the device generally is not connected and must be configured by accessing it directly (via USB, Bluetooth, or a direct network link, e.g., via Ethernet cable, or similar) for example via a technician’s laptop. - -Local registries for container images and Helm Charts can be used for all three categories of devices. In case of **fully connected devices**, although the device could reach the Internet, a local registry can still be useful, e.g., as a cache for remote registries to save on bandwidth or to have container images and Helm Charts reliably available. In case of **locally connected devices**, a local registry is required to enable the Workload Fleet Management Client (WFMC) to install a margo application on the device, as the device/WFMC does not have Internet access. Thereby, the local registry can be set up as a *pull-through cache* where data (e.g., container images) are cached locally when they are first retrieved from a remote source and subsequent requests for the same data are served from the local cache rather than fetching it again from the remote source. In case of **air-gapped devices**, a local registry has to be accessible on the technician's laptop (or other directly connected device), which performs the application installation process. - -To setup local registries, different configuration options exist: - -## Option - Container Registry Mirror on Kubernetes Level - -Kubernetes supports the configuration of registry mirrors. How this is configured depends on the distribution and the underlying container runtime. Distributions that utilize **containerd** as runtime (e.g., k3s or microk8s) allow the definition of mirrors in a configuration file. For example, in k3s the file `/etc/rancher/k3s/registries.yaml` can be used to set up a mirror for each device's Kubernetes environment: - -```yaml -mirrors: - "docker.io": - endpoint: - - "http://:5000" -configs: - "docker.io": - auth: - username: "" - password: "" -``` - -## Option - Container Registry as Pull-through Cache on Docker Level - -To configure a pull-through cache in Docker for the container registry, a Docker Registry can be setup that acts as caching proxy for a remote Docker registry. Such a Docker Registry container can be defined using the following `config.yml`: - -```yaml -version: 0.1 -log: - fields: - service: registry -storage: - filesystem: - rootdirectory: /var/lib/registry -http: - addr: :5000 - headers: - X-Content-Type-Options: [nosniff] -health: - storagedriver: - enabled: true - interval: 10s - threshold: 3 -proxy: - remoteurl: https://registry-1.docker.io -``` - -This registry can be started uing the following command: -`docker run -d -p 5000:5000 --restart unless-stopped --name registry -v $(pwd)/config.yml:/etc/docker/registry/config.yml registry:2` - -Then, the Docker daemon needs to be configured to use the private registry as a mirror in the `/etc/docker/daemon.json` file: - -```json -{ - "registry-mirrors": ["http://10.100.2.102:5000"], - "insecure-registries": ["10.100.2.102:5000"] -} -``` - -## Option - Helm Chart Component Registry as Pull-through Cache - -Setting up a pull-through cache for Helm charts in combination with Kubernetes involves configuring a local Helm chart repository, e.g., ChartMuseum that can be installed with the `PROXY_CACHE` variable set to `true`: - -```bash -helm repo add chartmuseum https://chartmuseum.github.io/charts -helm repo update -helm install my-chartmuseum chartmuseum/chartmuseum --set env.open.DISABLE_API=false --set env.open.PROXY_CACHE=true -``` - -Then, this Helm Chart repository can be added to Helm and chart releases can be installed there: - -```bash -helm repo add my-cached-repo http://:8080 -helm repo update -helm install my-release my-cached-repo/ -``` - -Now, when deploying applications in Kubernetes using Helm, the cached repository is used to serve charts rather than the remote repository. diff --git a/system-design/figures/System-design.drawio.svg b/system-design/figures/System-design.drawio.svg index a9fcda09..be454350 100644 --- a/system-design/figures/System-design.drawio.svg +++ b/system-design/figures/System-design.drawio.svg @@ -1,1521 +1,4 @@ - - - - - - - - - - - - - - - - - - - - - - -
-
-
- - - Component Registry - - -
-
-
- - - Component Registry - - - - - - - - - - - - - - - - - -
-
-
- Standalone Cluster(Leader and/or Worker) -
-
-
- - - Standalone Cluster(Leader and/or Worker) - - - - - - - - - - - - - - - - - - -
-
-
- WORKLOAD FLEET MANAGER -
-
-
- - - WORKLOAD FLEET MANAGER - - - - - - - - - - - - - - -
-
-
- DEVICE FLEET MANAGER -
-
-
- - - DEVICE FLEET MANAGER - - - - - - - - - - - - - - - - - - -
-
-
- KUBERNETES -
-
-
- - - KUBERNETES - - - - - - - - - - - -
-
-
- OBSERVABILITY PLATFORM -
-
-
- - - OBSERVABILITY PLATFORM - - - - - - - - - - - - - - - - - - -
-
-
- Cluster Worker -
-
-
- - - Cluster Worker - - - - - - - - - - - -
-
-
- Kubelet -
-
-
- - - Kubelet - - - - - - - - - - - - - - - - - - - -
-
-
- Standalone Device -
-
-
- - - Standalone Device - - - - - - - - - - - -
-
-
- OTEL Collector -
-
-
- - - OTEL Collector - - - - - - - - - - - - - - -
-
-
- WORKLOAD FLEET MANAGEMENT CLIENT -
-
-
- - - WORKLOAD FLEET MANAGEMENT C... - - - - - - - - - - - - - - - - - -
-
-
- Policy Agent -
-
-
- - - Policy Agent - - - - - - - - - - - -
-
-
- OTEL Collector -
-
-
- - - OTEL Collector - - - - - - - - - - - -
-
-
- WORKLOAD FLEET MANAGEMENT CLIENT -
-
-
- - - WORKLOAD FLEET MANAGEMENT C... - - - - - - - - - - - - - - - - - - - - - -
-
-
- DEVICE FLEET -
- MANAGEMENT -
- CLIENT -
-
-
- - - DEVICE FLEET... - - - - - - - - - - - -
-
-
- ONBOARDING SERVICE -
-
-
- - - ONBOARDING S... - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- - - Kubernetes Cluster - - -
-
-
- - - Kubernetes Cluster - - - - - - - - - - - - - - - -
-
-
- Device Supplier -
-
-
- - - Device Supplier - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- Cloud or -
- On Prem -
- Datacenter -
-
-
- - - Cloud or... - - - - - - - - - - - -
-
-
- Shop Floor -
-
-
- - - Shop Floor - - - - - - - - - - - - - -
-
-
- Application -
- Deployment -
-
-
-
- - - Application... - - - - - - - - - - - - - - - - - -
-
-
- DEVICE FLEET -
- MANAGEMENT -
- CLIENT -
-
-
- - - DEVICE FLEET... - - - - - - - - - - - -
-
-
- ONBOARDING SERVICE -
-
-
- - - ONBOARDING S... - - - - - - - - - - - -
-
-
- Policy Agent -
-
-
- - - Policy Agent - - - - - - - - - - - -
-
-
- Device Option 1 -
-
-
- - - Device Option 1 - - - - - - - - - - - -
-
-
- Device Option 2 -
-
-
- - - Device Option 2 - - - - - - - - - - - - -
-
-
- OS -
-
-
- - - OS - - - - - - - - - - - -
-
-
- OCI CONTAINER RUNTIME -
-
-
- - - OCI CONTAINER RUNTIME - - - - - - - - - - - -
-
-
- Deployed -
- Workload(s) -
-
-
-
- - - Deployed... - - - - - - - - - - - - -
-
-
- OS -
-
-
- - - OS - - - - - - - - - - - -
-
-
- OCI CONTAINER RUNTIME(Docker/Podman/...) -
-
-
- - - OCI CONTAINER RUNTIME(Docker/Podman/...) - - - - - - - - - - - -
-
-
- Deployed -
- Workload(s) -
-
-
-
- - - Deployed... - - - - - - - - - - - - -
-
-
- OS -
-
-
- - - OS - - - - - - - - - - - -
-
-
- OCI CONTAINER RUNTIME -
-
-
- - - OCI CONTAINER RUNTIME - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- LEGEND -
-
-
- - - LEGEND - - - - - - - - - - - - - - - - - -
-
-
- App Developer -
- Private -
-
-
- - - App Developer... - - - - - - - - - - - -
-
-
- Margo Envisioned Communication Mechanisms -
-
-
- - - Margo Envisioned Communicatio... - - - - - - - - - - - - - - -
-
-
- Device Manufacturer -
- Private -
-
-
- - - Device Manufacturer... - - - - - - - - - - - - - - -
-
-
- Fleet Management Platform Provider -
-
-
- - - Fleet Management Platform Pro... - - - - - - - - - - - - - -
-
-
- xxxxx -
-
-
- - - xxxxx - - - - - - - - - - - -
-
-
- Margo Defined File format -
-
-
- - - Margo Defined File format - - - - - - - - - - - - - - -
-
-
- Communication Mechanisms  Not defined via Margo -
-
-
- - - Communication Mechanisms  Not... - - - - - - - - - - - - - - -
-
-
- Observability Platform -
- Provider -
-
-
- - - Observability Platform... - - - - - - - - - - - -
-
-
- Deployed -
- Workload(s) -
-
-
-
- - - Deployed... - - - - - - - - - - - - - -
-
-
- Application Artifact(s) -
-
-
- - - Application Art... - - - - - - - - - - - - - -
-
-
- Application Manifest(s) -
-
-
- - - Application Man... - - - - - - - - - - - -
-
-
- - - Git Repository - - -
-
-
- - - Git Repository - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- Application Description -
-
-
- - - Application Des... - - - - - - - - - - - - - -
-
-
- Metadata -
- File(s) -
-
-
-
- - - Metadata... - - - - - - - - - - - -
-
-
- - - Application Package - - -
-
-
- - - Application Package - - - - - - - - - - - - - -
-
-
- - Metadata - -
- - File(s) - -
-
-
-
- - - Metadata... - - - - - - - - - - - -
-
-
- - Fleet Management - -
- - Supplier - -
-
-
-
- - - Fleet Management... - - - - - - - - - - - -
-
-
- - - Device Manifests / Components - - -
- - - TDB - - -
-
-
-
-
- - - Device Manifests / Components... - - - - - - - - - - - -
-
-
- - Device Supplier - -
-
-
- - - Device Supplier - - - - - - - - - - - - - -
-
-
- xxxxx -
-
-
- - - xxxxx - - - - - - - - - - - -
-
-
- Non Margo Defined File format -
-
-
- - - Non Margo Defined File format - - - - - - - - - - - -
-
-
- - - Application Supplier - - -
-
-
- - - Application Supplier - - - - - - - - - - - - - -
-
-
- Application Manifest(s) -
-
-
- - - Application Man... - - - - - - - - - - - - - -
-
-
- Application Artifact(s) -
-
-
- - - Application Art... - - - - - - - - - - - - - - Text is not SVG - cannot display - - - - \ No newline at end of file + + + +
Standalone Cluster(Leader and/or Worker)
Standalone Cluster(Leader and/or Worker)
WORKLOAD FLEET MANAGER
WORKLOAD FLEET MANAGER
DEVICE FLEET MANAGER
DEVICE FLEET MANAGER
KUBERNETES 
KUBERNETES 
OBSERVABILITY PLATFORM
OBSERVABILITY PLATFORM
Cluster Worker
Cluster Worker
Kubelet
Kubelet
Standalone Device
Standalone Device
OTEL Collector
OTEL Collector
WORKLOAD FLEET MANAGEMENT CLIENT
WORKLOAD FLEET MANAGEMENT C...
Policy Agent
Policy Agent
OTEL Collector
OTEL Collector
WORKLOAD FLEET MANAGEMENT CLIENT
WORKLOAD FLEET MANAGEMENT C...
DEVICE FLEET 
MANAGEMENT
CLIENT
DEVICE FLEET...
ONBOARDING SERVICE
ONBOARDING S...
Kubernetes Cluster
Kubernetes Cluster
Device Supplier
Device Supplier
Cloud or 
On Prem
Datacenter
Cloud or...
Shop Floor
Shop Floor
Application
Deployment
Application...
DEVICE FLEET 
MANAGEMENT
CLIENT
DEVICE FLEET...
ONBOARDING SERVICE
ONBOARDING S...
Policy Agent
Policy Agent
Device Option 1
Device Option 1
Device Option 2
Device Option 2
OS
OS
OCI CONTAINER RUNTIME
OCI CONTAINER RUNTIME
Deployed
Workload(s)
Deployed...
OS
OS
OCI CONTAINER RUNTIME(Docker/Podman/...)
OCI CONTAINER RUNTIME(Docker/Podman/...)
Deployed
Workload(s)
Deployed...
OS
OS
OCI CONTAINER RUNTIME
OCI CONTAINER RUNTIME
LEGEND
LEGEND
App Developer
Private
App Developer...
Margo Envisioned Communication Mechanisms
Margo Envisioned Communicatio...
Device Manufacturer
Private
Device Manufacturer...
Fleet Management Platform Provider
Fleet Management Platform Pro...
xxxxx
xxxxx
Margo Defined File format
Margo Defined File format
Communication Mechanisms  Not defined via Margo
Communication Mechanisms  Not...
Observability Platform
Provider
Observability Platform...
Deployed
Workload(s)
Deployed...
Application Registry
Application Registry
Application Description
Application Des...
Metadata
File(s)
Metadata...
Application Package
Application Package
Resource
File(s)
Resource...
Fleet Management
Supplier
Fleet Management...
Device Manifests / Components
TDB
Device Manifests / Components...
Device Supplier
Device Supplier
xxxxx
xxxxx
Non Margo Defined File format
Non Margo Defined File format
Application Supplier
Application Supplier
Component Registry
Component Registry
Component
(e.g., Helm Chart or Compse Archive)
Component...
Container Registry
Container Registry
Container Image
Container ImageText is not SVG - cannot display \ No newline at end of file diff --git a/system-design/overview/applications.md b/system-design/overview/applications.md new file mode 100644 index 00000000..7413f25b --- /dev/null +++ b/system-design/overview/applications.md @@ -0,0 +1,80 @@ +# Applications + +An application is defined as an [Application Package](../concepts/applications/application-package.md) described through an [Application Description](../specification/applications/application-description.md) and consists of deployable [components](../personas-and-definitions/technical-lexicon.md#component). A running [component](../personas-and-definitions/technical-lexicon.md#component) is called a [workload](../personas-and-definitions/technical-lexicon.md#workload). A workload runs on a Margo compliant [edge compute device](../personas-and-definitions/technical-lexicon.md#edge-compute-device). + +In order to help achieve Margo's interoperability [mission statement](../index.md#mission-statement) we are initially targeting [containerized](https://github.com/opencontainers) components/workloads capable of running on platforms like Kubernetes, Docker and Podman. The flexibility these platforms provide enables [workload suppliers](../personas-and-definitions/personas.md#workload-supplier) to define and package their components in a common way using [Helm](https://helm.sh/docs/) or the [Compose specification](https://github.com/compose-spec/compose-spec/blob/main/spec.md) so they can more easily be deployed to multiple compatible edge compute devices as workloads. + +While Margo is initially targeting deployments using Helm or Compose, we plan to support other deployment types in the future. One of our design goals is to make it easier for [workload fleet managers](../personas-and-definitions/technical-lexicon.md#workload-fleet-manager) to support the current and future deployment types without having to implement special logic for each type. In order to achieve this, Margo defines an [application description model](../concepts/applications/application-package.md) to abstract away some of the details to make it easier for workload fleet managers to support the different deployment types. + +The three main goals of Margo's application description model is to allow workload fleet managers to do the following: + +- Display information about available applications (e.g., via an [application catalog](../personas-and-definitions/technical-lexicon.md#application-catalog)), which the [OT user](../personas-and-definitions/personas.md#ot-user) can deploy as workloads. +- Determine which edge compute devices are compatible with an application (regarding processor type, GPU present, RAM available, etc.) +- Capture, and validate, configuration information from the OT user when deploying application components as workloads, or updating them. + +Another advantage of Margo's [application description model](../concepts/applications/application-package.md) is to enable workload suppliers to define different deployment profiles in a single application description file to target deploying to different types of edge compute devices (e.g., Arm vs. x86, Kubernetes vs. Docker) instead of needing to maintain multiple application description files. + +## Packaging & Distribution + +To distribute an application consisting of multiple components that are deployable as workloads, they are wrapped in an [application package](../concepts/applications/application-package.md) defined by the workload supplier who aims to provide it to Margo-compliant edge compute devices. +Therefore, the workload supplier creates an [application description YAML document](../specification/applications/application-description.md) containing information about the application and references to its components. +The application package is made available in an [application registry](../concepts/applications/application-registry.md), its components (i.e., Helm Charts or Compose Archives) are stored in a remote or [local](../concepts/applications/local-registries.md) component registry, and the linked container images are provided through a container image registry. + +### Example workflow + +The following diagram provides an example workflow showing one way a workload fleet manager may use the application description information: + +```mermaid +--- +config: + layout: elk + + +--- +sequenceDiagram + actor EndUser as End User + participant frontend as Workload Fleet Manager Frontend + participant fleetmgr as Workload Fleet Manager + participant registry as Application Registry + + autonumber + + EndUser->>frontend: Visits Application Catalog + frontend->>fleetmgr: Get list of available application packages + + alt + fleetmgr ->> registry: Get application descriptions from each known application registry. + else + fleetmgr ->> fleetmgr: Get application descriptions for all cached applications. + end + fleetmgr->>frontend: Return list of application descriptions + + frontend ->> frontend: Read all application descriptions' 'metadata' element + frontend ->> EndUser: Show UI with list of applications + EndUser->>frontend: Select application / components to install + frontend ->> frontend: Read application descriptions 'configuration' element + frontend -->> EndUser: Show UI to fill application configuration + EndUser ->> frontend: fills application configuration + frontend ->> fleetmgr: Create 'ApplicationDeployment' definition + +``` + +1. An end user visits the [application catalog](../personas-and-definitions/technical-lexicon.md#application-catalog) of a Workload Fleet Manager Frontend. +2. This frontend requests all installable [application packages](../concepts/applications/application-package.md) from the Workload Fleet Manager. +3. _Either_: the Workload Fleet Manager requests all application descriptions from each known [application registry](../concepts/applications/application-registry.md). +4. _Or_: the Workload Fleet Manager maintains a cache of application descriptions and services the request from there. +5. The Workload Fleet Manager returns the retrieved application descriptions to the frontend. +6. The frontend parses the [metadata](../specification/applications/application-description.md#metadata-attributes) element of all received application descriptions. +7. The frontend presents the parsed metadata in a UI to the end user. +8. The end user selects the application package to be installed. +9. The frontend parses the [configuration](../specification/applications/application-description.md#configuration-attributes) element of the selected application description. +10. The frontend presents the parsed configuration to the user. +11. The end user fills out the [configurable application parameters](../specification/applications/application-description.md#defining-configurable-application-parameters). +12. The frontend creates an `ApplicationDeployment` definition (from the `ApplicationDescription` and the filled out parameters) and sends it to the Workload Fleet Manager, which executes it as the [desired state](../specification/margo-management-interface/desired-state.md). + +## Relevant Links + +Please follow the subsequent links to view more technical information regarding Margo application packaging: + +- [Application Package](../concepts/applications/application-package.md) +- [Application Registry](../concepts/applications/application-registry.md) diff --git a/system-design/overview/envisioned-system-design.md b/system-design/overview/envisioned-system-design.md index ed30e7c2..f155f35e 100644 --- a/system-design/overview/envisioned-system-design.md +++ b/system-design/overview/envisioned-system-design.md @@ -10,7 +10,9 @@ The envisioned system can be broken down into the following main components: ### Workloads -Workloads are the software deployed to Margo-compliant edge compute devices. See the [workloads overview](workloads.md) page to learn more Margo's supported workloads and how the are packaged. +Workloads are the software deployed to Margo-compliant edge compute devices. +They are deployed [Components](../personas-and-definitions/technical-lexicon.md#component) of an [Application Package](../personas-and-definitions/technical-lexicon.md#application-package). +See the [Applications overview](applications.md) page to learn more Margo's supported workloads and how they are packaged. ### Workload Observability diff --git a/system-design/overview/workloads.md b/system-design/overview/workloads.md deleted file mode 100644 index 9bfec454..00000000 --- a/system-design/overview/workloads.md +++ /dev/null @@ -1,78 +0,0 @@ -# Workloads - -A [workload](../personas-and-definitions/technical-lexicon.md#workload) is software deployed to, and run on, Margo compliant [edge compute devices](../personas-and-definitions/technical-lexicon.md#edge-compute-device). - -In order to help achieve Margo's interoperability [mission statement](../index.md#mission-statement) we are initially targeting [containerized](https://github.com/opencontainers) workloads capable of running on platforms like Kubernetes, Docker and Podman. The flexibility these platforms provide enables [workload suppliers](../personas-and-definitions/personas.md#workload-supplier) to define and package their workloads in a common way using [Helm](https://helm.sh/docs/) or the [Compose specification](https://github.com/compose-spec/compose-spec/blob/main/spec.md) so they can more easily be deployed to multiple compatible edge compute devices. - -While Margo is initially targeting deployments using Helm or Compose, we plan to support other deployment types in the future. One of our design goals is to make it easier for [workload fleet managers](../personas-and-definitions/technical-lexicon.md#workload-fleet-manager) to support the current and future deployment types without having to implement special logic for each type. In order to achieve this, Margo defines an [application description model](../concepts/workloads/application-package.md) to abstract away some of the details to make it easier for workload fleet managers to support the different deployment types. - -The three main goals of Margo's application description model is to allow workload fleet managers to do the following: - -- Display information about the workloads the [OT user](../personas-and-definitions/personas.md#ot-user) can deploy (e.g., a [workload catalog](../personas-and-definitions/technical-lexicon.md#application-catalog)). -- Determine which edge compute devices are compatible with the workloads (e.g., processor types match, GPU present, etc.) -- Capture, and validate, configuration information from the OT user when deploying and updating workloads. - -Another advantage of Margo's [application description model](../concepts/workloads/application-package.md) is to enable workload suppliers to define different deployment profiles in a single application description file to target deploying to different types of edge compute devices (e.g., Arm vs. x86, Kubernetes vs. Docker) instead of needing to maintain multiple application description files. - -## Packaging - -To distribute one, or more, workloads they are wrapped in an [application package](../concepts/workloads/application-package.md) that is provided by the workload supplier who aims to provide it to Margo-compliant edge compute devices. Therefore, the workload supplier creates an application description YAML document containing information about the application and a reference on how to deploy the [OCI-containerized](https://github.com/opencontainers) workloads that make up the application. The application package is made available in an [application registry](../concepts/workloads/application-registry.md) and the OCI artifacts are stored in a remote or [local registry](../concepts/workloads/local-registries.md). - -## Example workflow - -The following diagram provides an example workflow showing one way a workload fleet manager might use the application description information: - -```mermaid ---- -config: - layout: elk - - ---- -sequenceDiagram - actor EndUser as End User - participant frontend as Workload Fleet Manager Frontend - participant fleetmgr as Workload Fleet Manager - participant registry as Application Registry - - autonumber - - EndUser->>frontend: Visits Application Catalog - frontend->>fleetmgr: Get list of available workloads (=Apps) - - alt - fleetmgr ->> registry: Get 'application description' from each known application registry. - else - fleetmgr ->> fleetmgr: Get 'application description' for all cached applications. - end - fleetmgr->>frontend: Return list of 'application description's - - frontend ->> frontend: Read all 'application description's -> 'metadata' element - frontend ->> EndUser: Show UI with list of applications - EndUser->>frontend: Select workload (=App) to install - frontend ->> frontend: Read 'application description' -> 'configuration' element - frontend -->> EndUser: Show UI to fill App configuration - EndUser ->> frontend: Answer configurable questions to be applied to workload(s) - frontend ->> fleetmgr: Create 'ApplicationDeployment' definition - -``` - -1. An end user visits an [workload catalog](../personas-and-definitions/technical-lexicon.md#application-catalog) of the Workload Fleet Manager Frontend. -2. This frontend requests all workloads from the Workload Fleet Manager. -3. _Either_: the Workload Fleet Manager requests all application descriptions from each known Application Registry. -4. _Or_: the Workload Fleet Manager maintains a cache of application descriptions and services the request from there. -5. The Workload Fleet Manager returns the retrieved documents of application descriptions to the frontend. -6. The frontend parses the [metadata](../specification/application-package/application-description.md#metadata-attributes) element of all received application description documents. -7. The frontend presents the parsed metadata in a UI to the end user. -8. The end user selects the workload to be installed. -9. The frontend parses the [configuration](../specification/application-package/application-description.md#configuration-attributes) element of the selected application description. -10. The frontend presents the parsed configuration to the user. -11. The end user fills out the [configurable application parameters](../specification/application-package/application-description.md#defining-configurable-application-parameters) to be applied to the workload. -12. The frontend creates an `ApplicationDeployment` definition (from the `ApplicationDescription` and the filled out parameters) and sends it to the Workload Fleet Manager, which executes it as the [desired state](../specification/margo-management-interface/desired-state.md). - -## Relevant Links - -Please follow the subsequent links to view more technical information regarding Margo application packaging: - -- [Application Package](../concepts/workloads/application-package.md) -- [Application Registry](../concepts/workloads/application-registry.md) diff --git a/system-design/personas-and-definitions/software-composition.md b/system-design/personas-and-definitions/software-composition.md index f0624986..d8e70eba 100644 --- a/system-design/personas-and-definitions/software-composition.md +++ b/system-design/personas-and-definitions/software-composition.md @@ -61,7 +61,7 @@ Software at rest is made available as an Application Package, which is a folder [Application Packages][application-package] and [Components][component] are managed and hosted separately: -- [Application Registries][application-registry] store [Application Descriptions][application-description] and their associated application resources. An [Application Registry][application-registry] is implemented as a git repository. +- [Application Registries][application-registry] store [Application Descriptions][application-description] and their associated application resources. An [Application Registry][application-registry] is compliant with the [OCI Registry API (v1.1.0)](https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md). - [Component Registries][component-registry] store [Components][component] The following diagram shows, at hand of an example, the relationship between an [Application Package][application-package] and the [Components][component] listed within its [Deployment Profiles][deployment-profile]: @@ -70,7 +70,7 @@ The following diagram shows, at hand of an example, the relationship between an C4Component title Application Bundling: Example 1 - Helm and Compose deployment profiles provided - System_Boundary(ar, "Application Registry (Git)") { + System_Boundary(ar, "Application Registry") { System_Boundary(ab1, "Application Package 1") { Component(atb1, "Application Package", "Reference") @@ -124,7 +124,7 @@ C4Component UpdateLayoutConfig($c4BoundaryInRow="1", $c4ShapeInRow="3") - System_Boundary(ar, "Application Registry (Git)") { + System_Boundary(ar, "Application Registry") { System_Boundary(ab1, "Application Package 1") { Component(atb1, "Application Package") @@ -200,9 +200,9 @@ C4Context In this stage the [providers][provider-model] are responsible for managing the individual [Workloads][workload]. -On a Helm v3 [Deployment Profiles][deployment-profile], a [Workload Fleet Management Client][wfma] implementation could utilize the Helm API to start the individual Helm Charts. +On a Helm v3 [Deployment Profiles][deployment-profile], a [Workload Fleet Management Client][wfmc] implementation could utilize the Helm API to start the individual Helm Charts. -On a Compose [Deployment Profiles][deployment-profile], a [Workload Fleet Management Client][wfma] implementation could utilize the Compose CLI to start the individual [Workloads][workload]. +On a Compose [Deployment Profiles][deployment-profile], a [Workload Fleet Management Client][wfmc] implementation could utilize the Compose CLI to start the individual [Workloads][workload]. The following diagram shows the result of reaching the desired state for an [Application][application] with a Helm v3 [Deployment Profile][deployment-profile] (the result of `helm install`). @@ -272,8 +272,8 @@ C4Component UpdateElementStyle(atb1, $fontColor="white", $bgColor="blue", $borderColor="grey") ``` -[application-description]: ../specification/application-package/application-description.md -[application-package]: ../concepts/workloads/application-package.md +[application-description]: ../specification/applications/application-description.md +[application-package]: ../concepts/applications/application-package.md [application-registry]: technical-lexicon.md#application-registry [component]: technical-lexicon.md#component [workload]: technical-lexicon.md#workload @@ -282,4 +282,5 @@ C4Component [deployment-definition]: ../specification/margo-management-interface/desired-state.md#applicationdeployment-yaml-definition [provider-model]: technical-lexicon.md#provider-model [wfmc]: technical-lexicon.md#workload-fleet-management-client -[deployment-profile]: ../specification/application-package/application-description.md#deploymentprofile-attributes +[deployment-profile]: ../specification/applications/application-description.md#deploymentprofile-attributes + diff --git a/system-design/personas-and-definitions/technical-lexicon.md b/system-design/personas-and-definitions/technical-lexicon.md index 0d863379..4f4d41ce 100644 --- a/system-design/personas-and-definitions/technical-lexicon.md +++ b/system-design/personas-and-definitions/technical-lexicon.md @@ -37,12 +37,11 @@ Current providers supported: #### Application -An application is a collection of one, or more, [Components](#component), as defined by an Application Description, and bundled within an [application package](#application-package). +An application is a collection of one, or more, [Components](#component), as defined by an [Application Description](../specification/applications/application-description.md), and bundled within an [Application Package](#application-package). #### Application Package -An Application Package is used to distribute an [application](#application). -According the [specification](../concepts/workloads/application-package.md), it is a folder in a Git repository referenced by URL, which contains the Application Description (that refers to contained and deployable Components) as well as associated resources (e.g., icons). +An Application Package is used to distribute an [application](#application). The parts of an Application Package are: Application Description (that refers to contained and deployable Components) as well as associated resources (e.g., icons). While the application package is made available in an [Application Registry](../concepts/applications/application-registry.md), the referenced [components](#component) are stored in a [Component Registry](#component-registry), and the linked containers are provided via a OCI [Container Image Registry](#container-image-registry). #### Component @@ -50,7 +49,10 @@ A Component is a piece of software tailored to be deployed within a customer's e Currently Margo-supported components are: - Helm Chart -- Compose Archive +- [Compose Archive](#compose-archive) + +#### Compose Archive +A Compose Archive is a tarball file containing the Compose file, `compose.yaml`, which is formatted according the [Compose specification](https://www.compose-spec.io/), and any additional artifacts referenced by the Compose file (e.g., configuration files, environment variable files, etc.). #### Workload @@ -79,36 +81,24 @@ Device Fleet Manager (DFM) represents a software offering that enables End Users > Note: The Device Fleet Manager is a future component of the Margo specification. This section will be expanded as the community defines device management functionality. -##### Application Registry - -An Application Registry holds [Application](#application) Packages. -It is used by developers to make their applications available. -An Application Registries MUST be a Git repository. - -[Workload Fleet Managers](#workload-fleet-manager) cannot access Application Registries directly, they can only access [Application Catalogs](#application-catalog). - -##### Application Catalog - -An Application Catalog holds Application Packages that were preselected to be install-ready for the edge environment of a [Workload Fleet Manager](#workload-fleet-manager) to deploy them to managed [Edge Compute Devices](#edge-compute-device). - -An Application Catalog obtains the offered [Applications](#application) from one or more [Application Registries](#application-registry). +#### Application Registry -##### Component Registry +An [Application Registry](../concepts/applications/application-registry.md) hosts [Application Packages](#application-package) that define, through their [Application Description](../specification/applications/application-description.md), the application as one or multiple [Components](#component). +It is used by application developers to make their applications available. +The [API of the Application Registry](../specification/applications/application-registry.md) is compliant with the [OCI Registry API (v1.1.0)](https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md). -A Component Registry holds [Components](#component) (e.g., Helm Charts and Compose Archives) for Application Packages. -When an application gets deployed through a [Workload Fleet Manager](#workload-fleet-manager), the components (linked within an Application Description) are requested from the Component Registry. -This can be implemented, for example, as an OCI Registry. +#### Application Catalog -#### Workload Marketplace +An Application Catalog is a visual representation of preselected, install-ready applications, the user of the [WFM](#workload-fleet-manager) can deploy to it's managed [Edge Compute Devices](#edge-compute-device). Application Catalogs and how they function within the WFM are out of scope for Margo. -Workload Marketplace is the location where end users purchase the rights to access [Workloads](#workload) from a vendor. -Functional Requirements of the Workload Marketplace: +#### Component Registry -- Provide users with a list of Workloads available for purchase -- Enable users to purchase access rights to a Workload -- Enable users with the meta data to access associated Workload Registries/Repositories +A Component Registry holds [Components](#component) (e.g., Helm Charts and Compose Archives) for [Application Packages](#application-package). +When an application gets deployed through a [Workload Fleet Manager](#workload-fleet-manager), the components (linked within an [Application Description](../specification/applications/application-description.md)) are requested from the Component Registry and then deployed as [workloads](#workload). Components link to containers that are typically provided through [Container Registries](#container-image-registry). +The Component Registry can be implemented, e.g., as an OCI Registry. -> Note: The Workload Marketplace component is out of scope for Project Margo. However, it is necessary to define to clarify the full user workflow. +#### Container Image Registry +A Container Image Registry hosts container images. [Components](#component) which are provided as Helm Charts or Compose Archives link to such container images. diff --git a/system-design/specification/application-package/.gitkeep b/system-design/specification/applications/.gitkeep similarity index 100% rename from system-design/specification/application-package/.gitkeep rename to system-design/specification/applications/.gitkeep diff --git a/system-design/specification/applications/application-registry.md b/system-design/specification/applications/application-registry.md new file mode 100644 index 00000000..3c2c0e7c --- /dev/null +++ b/system-design/specification/applications/application-registry.md @@ -0,0 +1,246 @@ +# Application Registry + +The interactions of an `Application Registry` are described [here](../../concepts/applications/application-registry.md) from a conceptual view. This section formally specifies the API of the `Application Registry` and the exchange of an [Application Package](../../concepts/applications/application-package.md), defined through an [Application Description](../../specification/applications/application-description.md) file, from an Application Developer to the [Workload Fleet Manager](../../personas-and-definitions/technical-lexicon.md#workload-fleet-manager) (WFM). The `Application Registry` is designed as an `OCI Registry`, i.e., it offers an API compliant with the [OCI Distribution Specification](https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md) (called here the "OCI_spec") for digital artifact distribution. This way, the Application Registry hosts the parts of an [Application Package](../../concepts/applications/application-package.md) in the form of `blobs` and uses `image manifests` according to [OCI Image specification](https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md) to list a set of `layers`, each pointing at a blob. + +## Overview of API Endpoints + +The WFM MUST interact with the Application Registry compliant to the OCI Registry API endpoints defined in the [OCI_spec](https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md): + +* Tags: `/v2/{name}/tags/list` for listing available versions of an Application Package +* Manifest: `/v2/{name}/manifests/{reference}` for retrieving an image manifest (identifed through the `{reference}`) that lists layers, which represent the parts of the Application Package. +* Blob: `/v2/{name}/blobs/{digest}` for downloading parts of the Application Package + +`{name}` is the namespace of the repository, which needs to be directly communicated by the App Developer to the WFM vendor. It could be for example a combination of the organization's and application's name. + +Further details on how to use these API endpoints are specified below [towards App Developer](#uploading-an-application-package) and [towards WFM](#retrieving-an-application-package). + +## Authentication, Authorization & Security +Margo recommends the use of an Authentication Service for the interactions between the WFM and Application Registry as conceptually described [here](../../concepts/applications/application-registry.md), e.g., implemented using OAuth 2.0 (see the [Token Authentication Specification](https://distribution.github.io/distribution/spec/auth/token/) provided by the reference implementation of the spec for more information). +This involves the following workflow: + +* WFM obtains credentials during onboarding +* WFM requests a token from an Authentication Service +* WFM uses the token for subsequent API calls to the Application Registry +* Application Registry validates the token and enforces access control + +> Note: Authentication, Authorization & Security mechanisms should be defined centrally and homogeneously across Margo. Hence, it is not further defined here. + +Further, it is recommendable to sign the Application Package. +To do so, the Application Package can be signed as an OCI artifact (with tools such as [cosign](https://github.com/sigstore/cosign)). +Any OCI registry can host those signatures alongside the Application Packages. +Local storing (inside of an archive or in a directory) is also supported over the [OCI Layout](https://github.com/opencontainers/image-spec/blob/main/image-layout.md). +Distributing the signatures alongside the Application Packages they validate can be done with multiple tools available in the OCI ecosystem. + + +## Uploading an Application Package + +This section describes the upload workflow. +However this is an application of the [standard artifact push workflow according the OCI Distribution Specification](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#push) and therefore technical details are to be obtained from that specification. +There are multiple libraries (e.g., [regclient](https://pkg.go.dev/github.com/regclient/regclient) or [oras](https://pkg.go.dev/oras.land/oras-go/v2) for Go, [oras](https://pypi.org/project/oras/) for Python) and tools (e.g., [regctl](https://github.com/regclient/regclient), [skopeo](https://github.com/containers/skopeo), [oras](https://github.com/oras-project/oras), [crane](https://github.com/google/go-containerregistry/tree/main/cmd/crane)) to realize this workflow. + +As shown in the sequence diagram below, the Application Developer uploads the Margo-compliant `Application Package` to the Application Registry. +I.e., all parts of the Application Package MUST be pushed as blobs according the ["Pushing Blobs" section of the OCI_spec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-blobs). +Subsequently, the Application Developer creates an OCI image manifest that lists layers, each of which links to an uploaded blob. +Then the manifest MUST be pushed to the Application Registry according the ["Pushing Manifests" section of the OCI_spec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests). +The uploaded OCI image manifest MUST adhere to the Margo-specific constraints detailed [here](#application-package-manifest). +Subsequently, the App Developer uses a UI or other vendor-specific mechanism to communicate (either directly or indirectly) to the WFM the namespace of the Application Package's repository. + +```mermaid +sequenceDiagram + Note over AppDeveloper: uploads parts of Application Package as blobs: + AppDeveloper->>AppRegistry: push blobs + AppDeveloper->>AppRegistry: push manifest + Note over AppDeveloper: uses vendor-specific upload mechanism (e.g., UI) to enable WFM to find the Application Package: + AppDeveloper->>+WFM: Application Package location is: repository name in Application Registry +``` + + + +## Retrieving an Application Package + +This section describes the retrieval workflow. +However this is an application of the [standard artifact pull workflow according the OCI Distribution Specification](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pull) and therefore technical details are to be obtained from that specification. + +The WFM has received the namespace of the repository of the Application Package at the Application Registry. +Next, as shown in the sequence diagram below, the WFM uses the API endpoints defined in the OCI_spec to retrieve a list of Application Package versions (as detailed [here](#list-margo-application-package-versions)). +Then, the WFM pulls the OCI image manifest of the selected version of the Application Package, with the identifying reference being a tag or digest (as detailed [here](#retrieving-application-package-manifest)). +If high trust requirements apply on the acquisition of the application package versions, state-of-the-art mechanisms can be applied to ensure that sophisticated attacks are not possible (for example a [TUF](https://theupdateframework.com/) implementation). +Finally, all parts (e.g., the Application Description file, the icon, the license, etc.) of the Application Package are retrieved by pulling the respective blobs listed as layers in the image manifest (as detailed [here](#retrieve-parts-of-application-package)). + +```mermaid +sequenceDiagram + Note over WFM: retrieve available versions of an Application Package: + WFM->>+AppRegistry: pull tags list + + Note over WFM: retrieves the OCI image manifest of the selected Application Package version. {reference} is a tag or a digest: + WFM->>+AppRegistry: pull manifest + + AppRegistry-->>+WFM: OCI image manifest + Note over WFM: retrieves parts of the Application Package as listed in OCI image manifest layers: + WFM->>AppRegistry: pull blobs +``` + +### List Margo Application Package Versions + +The list of the available Application Package versions can be obtained as documented in the ["Listing Tags" section of the OCI_spec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-tags). +The API supports filtering and pagination as documented in the linked specification section. +The `{name}` variable is the namespace of the Application Package repository, which was communicated by the App Developer to the WFM vendor. + +The list of Application Packages versions is provided in a JSON document that MUST conform with the above mentioned OCI_spec section and therefore has following format: + +```json +{ + "name": "", + "tags": [ + "", # each tag MUST be the value of the 'metadata.version' of the associated Application Package's Application Description document. + "", + "" + ] +} +``` + +Thereby, a listed tag of an image manifest MUST be the same as the value of the key ``metadata.version`` as specified in the [Application Description document](../../specification/applications/application-description.md) of the associated Application Package. + +#### Example A (no filtering): + +Request: GET ``http://famous-app-registry/v2/northstar-industrial-applications/app1/tags/list`` + +Response: +```json +{ + "name": "northstar-industrial-applications/app1", + "tags": [ + "v1.0.0", + "v1.1.0", + "latest" + ] +} +``` + +#### Example B (filtering to hide versions below v1.0.0): + +Request: GET ``http://famous-app-registry/v2/northstar-industrial-applications/app1/tags/list?n=2&last=v1.0.0`` + +Response: +```json +{ + "name": "northstar-industrial-applications/app1", + "tags": [ + "v1.1.0", + "latest" + ] +} +``` + +### Retrieving Application Package Manifest + +The Application Package Manifest is a JSON document that conforms to the [OCI Manifest specification](https://github.com/opencontainers/image-spec/blob/main/manifest.md) and provides metadata required for the distribution of the Application Package (from pushing to pulling) according to the OCI_spec. +Therefore retrieving an Application Package Manifest MUST be implemented according the ["Pulling manifests" section of the OCISpec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests). + +As mentioned before, different libraries and tools do the heavylifting of implementing these workflow. +An implementation without any of those tools or libraries is possible and MUST rely on the OCI_spec to do so. + +#### Application Package Manifest + +The Application Package Manifest is an OCI image manifest that MUST conform to the [OCI Image Manifest Specification](https://github.com/opencontainers/image-spec/blob/v1.1.0/manifest.md) and MUST contain pointers to all parts of an Application Package within the Application Registry. +This section specifies the Margo requirements on the Application Package Manifest. +Please refer to the [OCI Image Manifest Specification](https://github.com/opencontainers/image-spec/blob/v1.1.0/manifest.md) for all other technical details that are not Margo specific. + +Each version of an Application Package MUST have its own OCI image manifest. + +The ``artifactType`` of the OCI image manifest must be ``application/vnd.margo.app.v1+json``. + +The ``config`` MUST be a so-called "empty config" and be specified in the manifest according the ["Guidance for an empty descriptor" section of the OCI Image specification](https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor). + +Each element of the ``layers`` array contains a reference (so-called `digests`) to an artifact (so-called `blobs`) that is a part of the Application Package. + +Each Application Package part (file) MUST be listed as an element of the ``layers`` array: + +* The [Application Description](../../specification/applications/application-description.md) of the Application Package must be referred to in one element of the ``layers`` array, where the ``mediaType`` of this blob must be ``application/vnd.margo.app.description.v1+yaml``. + +* Each [application resource](../../concepts/applications/application-package.md), which is an additional file associated with the application, must be referred to as a blob listed in the ``layers`` array. + The blobs of resource files must be marked with a **mediaType** specific to the kind of resource file (currently four possible resource files: ``icon``, ``releaseNotes``, ``descriptionFile``, ``licenseFile``), as listed in the [table below](../../specification/applications/application-registry.md#margo-specific-media-types)). E.g., the blob representing the icon file (e.g., in ``jpeg`` format) of an application is marked with the mediaType `application/vnd.margo.app.icon.v1+jpeg`. The [Application Description file](../../specification/applications/application-description.md) lists these resource files in the ```metadata.catalog.application`` element. + +The following response example is a Margo-specific OCI image manifest following the [OCI Image Manifest Specification](https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md) and the above defined specifics: + +```json +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.margo.app.v1+json" # this MUST be the artifactType of an OCI image manifest of a Margo Application Package, + "config": { + "mediaType": "application/vnd.oci.empty.v1+json", # the 'config' object MUST be empty + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "size": 2, + "data": "e30=" + }, + "layers": [ + { + "mediaType": "application/vnd.margo.app.description.v1+yaml", # this MUST be the artifactType of a Margo Application Description + "digest": "sha256:f6b79149e6650b0064c146df7c045d157f3656b5ad1279b5ce9f4446b510bacf", + "size": 999, + "annotations": { + "org.opencontainers.image.title": "margo.yaml" + } + }, + { + "mediaType": "application/vnd.margo.app.descriptionFile.v1+markdown", # this MUST be the correct mediaType for this resource as defined below + "digest": "sha256:e373b123782a2b52483a9124cc9c578e0ed0300cb4131b73b0c79612122b8361", + "size": 1596, + "annotations": { + "org.opencontainers.image.title": "resources/description.md" + } + }, + { + "mediaType": "application/vnd.margo.app.licenseFile.v1+markdown", # this MUST be the correct mediaType for this resource as defined below + "digest": "sha256:af7db4ab9030533b6cda2325247920c3659bc67a7d49f3d5098ae54a64633ec7", + "size": 25, + "annotations": { + "org.opencontainers.image.title": "resources/license.md" + } + }, + { + "mediaType": "application/vnd.margo.app.icon.v1+jpeg", # this MUST be the correct mediaType for this resource as defined below + "digest": "sha256:451410b6adfdce1c974da2275290d9e207911a4023fafea0e283fad0502e5e56", + "size": 5065, + "annotations": { + "org.opencontainers.image.title": "resources/margo.jpg" + } + }, + { + "mediaType": "application/vnd.margo.app.releaseNotes.v1+markdown", # this MUST be the correct mediaType for this resource as defined below + "digest": "sha256:c412d143084c3b051d7ea4b166a7bfffb4550f401d89cae8898991c65e90f736", + "size": 42, + "annotations": { + "org.opencontainers.image.title": "resources/release-notes.md" + } + } + ], +} +``` + +#### Margo-Specific Media Types + +|Media Type|Description| +|----------|----------| +|``application/vnd.margo.app.v1+json`` | MUST be used as the **artifactType** to mark the OCI image manifest as the definition of a Margo Application Package | +|``application/vnd.margo.app.description.v1+yaml`` | MUST be used to mark a layer in the OCI image manifest as pointing to the Margo Application Description file | +|``application/vnd.margo.app.icon.v1+{file format}``| MUST be used to mark a layer in the OCI image manifest as pointing to the icon of a Margo Application Package | +|``application/vnd.margo.app.descriptionFile.v1+{file format}``| MUST be used to mark a layer in the OCI image manifest as pointing to description file of a Margo Application Package | +|``application/vnd.margo.app.licenseFile.v1+{file format}``| MUST be used to mark a layer in the OCI image manifest as pointing to the license file of a Margo Application Package| +|``application/vnd.margo.app.releaseNotes.v1+{file format}``| MUST be used to mark a layer in the OCI image manifest as pointing to the release notes file of a Margo Application Package| + + +#### Margo-Specific Annotation Keys + +|Annotation Key | Description| +|----------|----------| +|``org.margo.app.resource`` | MUST be used to annotate a layer/blob that references a Margo application resource | + + + +### Retrieve Parts of Application Package + +Retrieving the different files that compose an Application Package MUST be implemented according to the ["Pulling blobs" section of the OCI_spec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-blobs). + +Also for this purpose available tools and libraries can be used for an implementation with low complexity. \ No newline at end of file