diff --git a/new-docs/.gitignore b/new-docs/.gitignore new file mode 100644 index 0000000000..b2d6de3062 --- /dev/null +++ b/new-docs/.gitignore @@ -0,0 +1,20 @@ +# Dependencies +/node_modules + +# Production +/build + +# Generated files +.docusaurus +.cache-loader + +# Misc +.DS_Store +.env.local +.env.development.local +.env.test.local +.env.production.local + +npm-debug.log* +yarn-debug.log* +yarn-error.log* diff --git a/new-docs/README.md b/new-docs/README.md new file mode 100644 index 0000000000..b28211a9bb --- /dev/null +++ b/new-docs/README.md @@ -0,0 +1,41 @@ +# Website + +This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator. + +## Installation + +```bash +yarn +``` + +## Local Development + +```bash +yarn start +``` + +This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. + +## Build + +```bash +yarn build +``` + +This command generates static content into the `build` directory and can be served using any static contents hosting service. + +## Deployment + +Using SSH: + +```bash +USE_SSH=true yarn deploy +``` + +Not using SSH: + +```bash +GIT_USER= yarn deploy +``` + +If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. diff --git a/new-docs/docusaurus.config.ts b/new-docs/docusaurus.config.ts new file mode 100644 index 0000000000..4aa920a0c7 --- /dev/null +++ b/new-docs/docusaurus.config.ts @@ -0,0 +1,31 @@ +import type { Config } from "@docusaurus/types"; +import { + createSeqeraConfig, + getSeqeraThemeConfig, + getSeqeraPresetOptions, +} from "@seqeralabs/docusaurus-preset-seqera"; + +export default async function createConfigAsync(): Promise { + return createSeqeraConfig({ + plugins: [], + presets: [ + [ + "@seqeralabs/docusaurus-preset-seqera", + await getSeqeraPresetOptions({ + docs: { + routeBasePath: "/wave", + path: "wave-docs", + sidebarPath: "./sidebars.ts", + showLastUpdateAuthor: false, + showLastUpdateTime: false, + }, + openapi: false, + theme: { + customCss: require.resolve("./src/css/custom.css"), + }, + }), + ], + ], + themeConfig: getSeqeraThemeConfig({}), + }) satisfies Config; +} \ No newline at end of file diff --git a/new-docs/package.json b/new-docs/package.json new file mode 100644 index 0000000000..6b7e852dfa --- /dev/null +++ b/new-docs/package.json @@ -0,0 +1,48 @@ +{ + "name": "wave-docs", + "version": "0.0.0", + "private": true, + "scripts": { + "docusaurus": "docusaurus", + "start": "docusaurus start", + "build": "docusaurus build", + "swizzle": "docusaurus swizzle", + "deploy": "docusaurus deploy", + "clear": "docusaurus clear", + "serve": "docusaurus serve", + "write-translations": "docusaurus write-translations", + "write-heading-ids": "docusaurus write-heading-ids", + "typecheck": "tsc" + }, + "dependencies": { + "@docusaurus/core": "3.9.2", + "@docusaurus/faster": "^3.9.2", + "@docusaurus/preset-classic": "3.9.2", + "@mdx-js/react": "^3.0.0", + "clsx": "^2.0.0", + "prism-react-renderer": "^2.3.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" + }, + "devDependencies": { + "@docusaurus/module-type-aliases": "3.9.2", + "@docusaurus/tsconfig": "3.9.2", + "@docusaurus/types": "3.9.2", + "typescript": "~5.6.2" + }, + "browserslist": { + "production": [ + ">0.5%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 3 chrome version", + "last 3 firefox version", + "last 5 safari version" + ] + }, + "engines": { + "node": ">=20.0" + } +} diff --git a/new-docs/sidebars.ts b/new-docs/sidebars.ts new file mode 100644 index 0000000000..289713975c --- /dev/null +++ b/new-docs/sidebars.ts @@ -0,0 +1,33 @@ +import type {SidebarsConfig} from '@docusaurus/plugin-content-docs'; + +// This runs in Node.js - Don't use client-side code here (browser APIs, JSX...) + +/** + * Creating a sidebar enables you to: + - create an ordered group of docs + - render a sidebar for each doc of that group + - provide next/previous navigation + + The sidebars can be generated from the filesystem, or explicitly defined here. + + Create as many sidebars as you want. + */ +const sidebars: SidebarsConfig = { + // By default, Docusaurus generates a sidebar from the docs folder structure + tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], + + // But you can create a sidebar manually + /* + tutorialSidebar: [ + 'intro', + 'hello', + { + type: 'category', + label: 'Tutorial', + items: ['tutorial-basics/create-a-document'], + }, + ], + */ +}; + +export default sidebars; diff --git a/new-docs/src/css/custom.css b/new-docs/src/css/custom.css new file mode 100644 index 0000000000..e69de29bb2 diff --git a/new-docs/static/.nojekyll b/new-docs/static/.nojekyll new file mode 100644 index 0000000000..e69de29bb2 diff --git a/new-docs/tsconfig.json b/new-docs/tsconfig.json new file mode 100644 index 0000000000..920d7a6523 --- /dev/null +++ b/new-docs/tsconfig.json @@ -0,0 +1,8 @@ +{ + // This file is not used in compilation. It is here just for a nice editor experience. + "extends": "@docusaurus/tsconfig", + "compilerOptions": { + "baseUrl": "." + }, + "exclude": [".docusaurus", "build"] +} diff --git a/new-docs/wave-docs/_images/wave_container_augmentation.png b/new-docs/wave-docs/_images/wave_container_augmentation.png new file mode 100644 index 0000000000..747d8ce397 Binary files /dev/null and b/new-docs/wave-docs/_images/wave_container_augmentation.png differ diff --git a/new-docs/wave-docs/_images/wave_container_build_failure_details.png b/new-docs/wave-docs/_images/wave_container_build_failure_details.png new file mode 100644 index 0000000000..be367f62bb Binary files /dev/null and b/new-docs/wave-docs/_images/wave_container_build_failure_details.png differ diff --git a/new-docs/wave-docs/api.md b/new-docs/wave-docs/api.md new file mode 100644 index 0000000000..40a6d2d96e --- /dev/null +++ b/new-docs/wave-docs/api.md @@ -0,0 +1,710 @@ +--- +title: API +--- + +This page summarizes the API provided by the Wave container service. + +**API limits** + +The Wave service implements API rate limits for API calls. Authenticated users have higher rate limits than anonymous users. + +If an access token is provided, the following rate limits apply: + +- 250 container builds per hour +- 2,000 container pulls per minute + +If an access token isn't provided, the following rate limits apply: + +- 25 container builds per day +- 100 container pulls per hour + +## POST `/container-token` + +Deprecated endpoint allows you to submit a request to access a private container registry via Wave, or build a container image on-the-fly with a Dockerfile or Conda recipe file. + +The endpoint returns the name of the container request made available by Wave. + +:::important + +This API endpoint is deprecated in current versions of Wave. + +::: + +### Request body + +```json +{ + containerImage: string, + containerFile: string, + containerConfig: { + entrypoint: string[], + cmd: string[], + env: string[], + workingDir: string + layers: [ + { + location: string, + gzipDigest: string, + gzipSize: string, + tarDigest: string + }, + ... + ] + }, + condaFile: string, + containerPlatform: string, + buildRepository: string, + cacheRepository: string, + timestamp: string, + fingerprint: string, + freeze: boolean, + towerAccessToken: string, + towerRefreshToken: string, + towerEndpoint: string, + towerWorkspaceId: number, +} +``` + +#### Container token request attributes + +| Attribute | Description | +| ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `containerImage` | Name of the container to be served, e.g., `docker.io/library/ubuntu:latest` (optional). If omitted, the `containerFile` must be provided. | +| `containerConfig.entrypoint` | The container entrypoint command, e.g., `['/bin/bash']`. | +| `containerConfig.cmd` | The launch command to be used by the Wave container, e.g., `['echo', 'Hello world']` (optional). | +| `containerConfig.env` | The environment variables to be defined in the Wave container, e.g., `['FOO=one','BAR=two']` (optional). | +| `containerConfig.workingDir` | The work directory to be used in the Wave container, e.g., `/some/work/dir` (optional). | +| `containerConfig.layers.location` | Specifies a container image layer stored as a tar.gz file (optional). Either a HTTP URL to the file or a base64 encoded string prefixed with `data:`. | +| `containerConfig.layers.tarDigest` | The SHA256checksum of the provided tar file, e.g., `sha256:a7c724b02...`. | +| `containerConfig.layers.gzipDigest` | The SHA256 checksum of the provided layer tar gzip file, e.g., `sha256:a7c724b02...`. | +| `containerConfig.layers.gzipSize` | The size in bytes of the the provided layer tar gzip file. | +| `containerFile` | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted. | +| `condaFile` | Conda environment file encoded as base64 string. | +| `containerPlatform` | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64. | +| `buildRepository` | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional). | +| `cacheRepository` | Container repository used to cache build layers `docker.io/user/my-cache` (optional). | +| `timestamp` | Request submission timestamp using ISO-8601. | +| `fingerprint` | Request unique fingerprint. | +| `freeze` | Freeze requires buildRepository to push the build container to a user-defined repository. This provides the container URL from the user-defined repository, not the Wave generated URL. This URL won't change. | +| `towerEndpoint` | Seqera Platform service endpoint from where container registry credentials are retrieved (optional). Default `https://api.cloud.seqera.io`. | +| `towerAccessToken` | Access token of the user account granting access to the Seqera Platform service specified via `towerEndpoint` (optional). | +| `towerWorkspaceId` | ID of the Seqera Platform workspace from where the container registry credentials are retrieved (optional). When omitted the personal workspace is used. | + +### Response + +```json +{ + containerToken: string, + targetImage: string, + expiration: string +} +``` + +| Attribute | Description | +| ---------------- | ---------------------------------------------------------------------------------------- | +| `containerToken` | The unique token identifying the Wave container request, e.g., `0123456789`. | +| `targetImage` | The Wave container image name, e.g., `wave.seqera.io/wt/0123456789/library/ubuntu:latest`. | +| `expiration` | The expiration timestamp of the Wave container using ISO-8601 format. | + +## POST `/v1alpha2/container` + +This endpoint allows you to submit a request to access a private container registry via Wave, or build a container image on-the-fly with a Dockerfile or Conda recipe file. + +The endpoint returns the name of the container request made available by Wave. + +### Request body + +```json +{ + containerImage: string, + containerFile: string, + containerConfig: { + entrypoint: string[], + cmd: string[], + env: string[], + workingDir: string + layers: [ + { + location: string, + gzipDigest: string, + gzipSize: string, + tarDigest: string + }, + ... + ] + }, + condaFile: string, + containerPlatform: string, + buildRepository: string, + cacheRepository: string, + timestamp: string, + fingerprint: string, + freeze: boolean, + towerAccessToken: string, + towerRefreshToken: string, + towerEndpoint: string, + towerWorkspaceId: number, + packages: { + type: string, + environment: string, + entries: string[], + channels: string[], + condaOpts: { + mambaImage: string, + commands: string[], + basePackages: string + } + + }, + nameStrategy: string +} +``` + +#### Container token request attributes + +| Attribute | Description | +| ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `containerImage` | Name of the container to be served, e.g., `docker.io/library/ubuntu:latest` (optional). If omitted, the `containerFile` must be provided. | +| `containerConfig.entrypoint` | The container entrypoint command, e.g., `['/bin/bash']`. | +| `containerConfig.cmd` | The launch command to be used by the Wave container, e.g., `['echo', 'Hello world']` (optional). | +| `containerConfig.env` | The environment variables to be defined in the Wave container, e.g., `['FOO=one','BAR=two']` (optional). | +| `containerConfig.workingDir` | The work directory to be used in the Wave container, e.g., `/some/work/dir` (optional). | +| `containerConfig.layers.location` | Specifies a container image layer stored as a tar.gz file (optional). Either a HTTP URL to the file or a base64 encoded string prefixed with `data:`. | +| `containerConfig.layers.tarDigest` | The SHA256checksum of the provided tar file, e.g., `sha256:a7c724b02...`. | +| `containerConfig.layers.gzipDigest` | The SHA256 checksum of the provided layer tar gzip file, e.g., `sha256:a7c724b02...`. | +| `containerConfig.layers.gzipSize` | The size in bytes of the the provided layer tar gzip file. | +| `containerFile` | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted. | +| `condaFile` | Conda environment file encoded as base64 string. | +| `containerPlatform` | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64. | +| `buildRepository` | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional). | +| `cacheRepository` | Container repository used to cache build layers `docker.io/user/my-cache` (optional). | +| `timestamp` | Request submission timestamp using ISO-8601. | +| `fingerprint` | Request unique fingerprint. | +| `freeze` | Freeze requires buildRepository to push the build container to a user-defined repository. This provides the container URL from the user-defined repository, not the Wave generated URL. This URL won't change. | +| `towerEndpoint` | Seqera Platform service endpoint from where container registry credentials are retrieved (optional). Default `https://api.cloud.seqera.io`. | +| `towerAccessToken` | Access token of the user account granting access to the Seqera Platform service specified via `towerEndpoint` (optional). | +| `towerWorkspaceId` | ID of the Seqera Platform workspace from where the container registry credentials are retrieved (optional). When omitted the personal workspace is used. | +| `packages` | This object specifies Conda packages environment information. | +| `environment` | The package environment file encoded as a base64 string. | +| `type` | This represents the type of package builder. Use `CONDA`. | +| `entries` | List of the packages names. | +| `channels` | List of Conda channels, which will be used to download packages. | +| `mambaImage` | Name of the Docker image used to build Conda containers. | +| `commands` | Command to be included in the container. | +| `basePackages` | Names of base packages. | +| `nameStrategy` | The name strategy to be used to create the name of the container built by Wave. Its values can be `none`, `tagPrefix`, or `imageSuffix`. | | + +### Response + +```json +{ + containerToken: string, + targetImage: string, + expiration: string, + buildId: string, + cached: boolean +} +``` + +#### Container token response attributes + +| Attribute | Description | +| ---------------- | ---------------------------------------------------------------------------------------- | +| `containerToken` | The unique token identifying the Wave container request, e.g., `0123456789`. | +| `targetImage` | The Wave container image name, e.g., `wave.seqera.io/wt/0123456789/library/ubuntu:latest`. | +| `expiration` | The expiration timestamp of the Wave container using ISO-8601 format. | +| `cached` | Indicates if the requested image is built or in progress. | + +### Examples + +1. Create Docker image with Conda packages: + +##### Request + +```shell +curl --location 'http://localhost:9090/v1alpha2/container' \ +--header 'Content-Type: application/json' \ +--data '{ + "packages":{ + "type": "CONDA", + "entries": ["salmon", "bwa"], + "channels": ["conda-forge", "bioconda"] + } +}' +``` + +#### Response + +```json +{ + "containerToken":"732b73aa17c8", + "targetImage":"0625dce899da.ngrok.app/wt/732b73aa17c8/hrma017/dev:salmon_bwa--5e49881e6ad74121", + "expiration":"2024-04-09T21:19:01.715321Z", + "buildId":"5e49881e6ad74121_1", + "cached":false, + "freeze":false +} +``` + +2. Create Singularity image with Conda packages: + +##### Request + +```shell +curl --location 'http://localhost:9090/v1alpha2/container' \ +--header 'Content-Type: application/json' \ +--data '{ + "format": "sif", + "containerPlatform": "arm64", + "packages":{ + "type": "CONDA", + "entries": ["salmon"], + "channels": ["conda-forge", "bioconda"] + }, + "freeze": true, + "buildRepository": , + "towerAccessToken":, + "towerEndpoint": "http://localhost:8008/api" +}' +``` + +#### Response + +```json +{ + "targetImage":"oras://:salmon--6c084f2e43f86a78", + "buildId":"6c084f2e43f86a78_1", + "cached":false, + "freeze":true +} +``` + +:::note +You must add your container registry credentials in Seqera Platform to use the freeze feature. This is a requirement for Singularity. +::: + +## GET `/v1alpha1/builds/{buildId}/status` + +Provides status of build against buildId passed as path variable + +### Response + +```json +{ + id: string, + status: string, + startTime: string, + duration: string, + succeeded: boolean +} +``` + +:::note +Status can only be `PENDING` or `COMPLETED`. +::: + +### Example + +```shell +% curl --location 'http://localhost:9090/v1alpha1/builds/6c084f2e43f86a78_1/status' +{ + "id":"6c084f2e43f86a78_1", + "status":"COMPLETED", + "startTime":"2024-04-09T20:31:35.355423Z", + "duration":123.914989000, + "succeeded":true +} +``` + +## GET `/v1alpha1/builds/{buildId}/logs` + +Supply logs corresponding to the specified buildId within the API request. + +### Response + +```text +string +``` + +### Example + +``` +% curl --location 'http://localhost:9090/v1alpha1/builds//logs' +INFO[0001] Retrieving image manifest alpine:latest +INFO[0001] Retrieving image alpine:latest from registry index.docker.io +INFO[0002] Retrieving image manifest alpine:latest +INFO[0002] Returning cached image manifest +INFO[0002] Built cross stage deps: map[] +INFO[0002] Retrieving image manifest alpine:latest +INFO[0002] Returning cached image manifest +INFO[0002] Retrieving image manifest alpine:latest +INFO[0002] Returning cached image manifest +INFO[0002] Executing 0 build triggers +INFO[0002] Building stage 'alpine:latest' [idx: '0', base-idx: '-1'] +INFO[0002] Skipping unpacking as no commands require it. +INFO[0002] Pushing image to / +INFO[0005] Pushed index.docker.io// +``` + +## GET `/service-info` + +Provides basic information about the service status. + +### Response + +```json +{ + serviceInfo: { + version: string, + commitId: string + } +} +``` + +## POST `/v1alpha1/inspect` + +This endpoint returns the metadata about provided container image + +### Request + +```json +{ + containerImage: string, + towerAccessToken: string, + towerEndpoint: string, + towerWorkspaceId: string +} +``` + +#### Container inspect request attributes + +| Attribute | Description | +| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `containerImage` | Name of the container to be inpected, e.g., `docker.io/library/ubuntu:latest` | +| `towerEndpoint` | Seqera Platform service endpoint from where container registry credentials are retrieved (optional). Default `https://api.cloud.seqera.io`. | +| `towerAccessToken` | Access token of the user account granting the access to the Seqera Platform service specified via `towerEndpoint` (optional). | +| `towerWorkspaceId` | ID of the Seqera Platform workspace from where the container registry credentials are retrieved (optional). When omitted the personal workspace is used.| + +### Response + +```json +{ + container: { + registry: string, + hostName: string, + imageName: string, + reference: string, + digest: string, + config: { + architecture: string, + config: { + attachStdin: boolean, + attachStdout: boolean, + attachStderr: boolean, + tty: boolean, + env: string[], + cmd: string[], + image: string + }, + container: string, + created: string, + rootfs: { + type: string, + diff_ids: string[] + } + }, + manifest: { + schemaVersion: integer, + mediaType: string, + config: { + mediaType: string, + digest: string, + size: integer + }, + layers: [ + { + mediaType: string, + digest: string, + size: integer + } + ] + }, + v1: boolean, + v2: boolean, + oci: boolean + } +} +``` + +:::note +You can find the explanation of the response attributes (here)[https://github.com/opencontainers/image-spec/blob/main/spec.md] +::: + +### Example + +#### API call + +```shell +curl --location 'http://localhost:9090/v1alpha1/inspect' \ +--header 'Content-Type: application/json' \ +--data '{ + "containerImage": "docker.io//", + "towerAccessToken": "", + "towerEndpoint": "http://localhost:8000/api" +}' +``` + +##### Response + +```json +{ + "container": { + "registry": "docker.io", + "hostName": "https://registry-1.docker.io", + "imageName": "//", + "reference": "9b80535d04eceefd", + "digest": "sha256:1fcabdb850dc7c46646b3796fca01aca5721330252b586058e0d326705374dd5", + "config": { + "architecture": "amd64", + "config": { + "env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "cmd": [ + "/bin/sh" + ], + "image": "sha256:9a5ce069f40cfe0f2270eafbff0a0f2fa08f1add73571af9f78209e96bb8a5e9" + }, + "container": "4189cbc534955765760c227f328ec1cdd52e8550681c2bf9f8f990b27b644f9c", + "created": "2024-04-19T14:38:17.047396956Z", + "rootfs": { + "type": "layers", + "diff_ids": [ + "sha256:d4fc045c9e3a848011de66f34b81f052d4f2c15a17bb196d637e526349601820" + ] + } + }, + "manifest": { + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "digest": "sha256:639823e18eb8b62cf43e92bac114ae35c03c07449e4ee5c10f8ebf8d033877d6", + "size": 774 + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:4abcf20661432fb2d719aaf90656f55c287f8ca915dc1c92ec14ff61e67fbaf8", + "size": 3408729 + } + ] + }, + "v1": false, + "v2": true, + "oci": false + } +} +``` + +## Metrics APIs based on Redis + +These APIs provide usage (builds and pulls) metrics of Wave for a specific date and/or a specific organization. +These APIs require basic authentication, so you must provide a username and password while calling these APIs. + +All Metrics API endpoints use these query parameters: + +| Name | Description | sample Value | +|------|-----------------------------------------------------------------------|--------------| +| date | Format: `yyyy-mm-dd`, The date of the required metrics. | 2024-04-08 | +| org | Domain of the organization used in emails, e.g., `org=seqera.io` | seqera.io | + +### Build Metrics API + +These APIs are used to retrieve metrics about container builds performed by Wave. + +### GET `/v1alpha2/metrics/builds` + +This endpoint is used to retrieve the builds performed by Wave. + +### Response + +```json +{ + metric: "builds", + count: integer, + orgs: { + String: integer, + String: integer, + ... + } +} +``` + +#### Examples + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/builds" +{ + "metric": "builds", + "count": 18, + "orgs": { + "seqera.io": 13, + "gmail.com": 5 + } +} +``` + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/builds?date=2024-04-08&org=seqera.io" +{"count":4} +``` + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/builds?date=2024-04-08" +{ + "metric": "builds", + "count": 8, + "orgs": { + "gmail.com": 4, + "seqera.io": 4 + } +} +``` + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/builds?org=seqera.io" +{ + "metric": "builds", + "count": 13, + "orgs": { + "seqera.io": 13 + } +} +``` +### Pull Metrics API + +These APIs are used to get the metrics about the container pulls through Wave. + +### GET `/v1alpha2/metrics/pulls` + +This endpoint is used to get the pulls performed through Wave. + +### Response + +```json +{ + metric: "pulls", + count: integer, + orgs: { + String: integer, + String: integer, + ... + } +} +``` + +#### Examples + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/pulls" +{ + "metric": "pulls", + "count": 11, + "orgs": { + "seqera.io": 7, + "gmail.com": 4 + } +} +``` + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/pulls?date=2024-04-08&org=seqera.io" +{"count":5} +``` + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/pulls?date=2024-04-08" +{ + "metric": "pulls", + "count": 4, + "orgs": { + "seqera.io": 1, + "gmail.com": 3 + } +} +``` + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/pulls?org=seqera.io" +{ + "metric": "pulls", + "count": 7, + "orgs": { + "seqera.io": 7 + } +} +``` + +### Fusion Pull Metrics API + +These APIs are used to get the metrics about the Fusion-based container pulls through Wave. + +### GET `/v1alpha2/metrics/fusion/pulls` + +This endpoint is used to get the pulls of Fusion-based containers performed through Wave. + +### Response + +```json +{ + metric: "fusion", + count: integer, + orgs: { + String: integer, + String: integer, + ... + } +} +``` + +#### Examples + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/fusion/pulls" +{ + "metric": "fusion", + "count": 2, + "orgs": { + "seqera.io": 1, + "gmail.com": 1 + } +} +``` + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/fusion/pulls?date=2024-04-08&org=seqera.io" +{"count":2} +``` + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/fusion/pulls?date=2024-04-08" +{ + "metric": "fusion", + "count": 1, + "orgs": { + "gmail.com": 1 + } +} +``` + +```shell +curl -u foo:bar "http://localhost:9090/v1alpha2/metrics/fusion/pulls?org=seqera.io" +{ + "metric": "fusion", + "count": 1, + "orgs": { + "seqera.io": 1 + } +} +``` diff --git a/new-docs/wave-docs/cli/index.md b/new-docs/wave-docs/cli/index.md new file mode 100644 index 0000000000..90c215338c --- /dev/null +++ b/new-docs/wave-docs/cli/index.md @@ -0,0 +1,95 @@ +--- +title: Overview +--- + +The Wave CLI is a command-line interface that provides convenient access to Wave's container provisioning capabilities through a simple wrapper around the Wave API. + +Wave command line tool includes functionality to: + +- Build container images on-demand using Dockerfiles or Conda packages +- Augment existing images with additional layers +- Generate multi-platform containers for linux/amd64 and linux/arm64 architectures +- Create Singularity native images and push them to OCI-compliant registries +- Mirror containers between registries by copying images on-demand to target repositories +- Scan containers for security vulnerabilities automatically during provisioning +- Push and cache built containers to user-specified repositories +- Integrate with Seqera Platform for credentials management and persistent storage + +## Container URLs + +The Wave CLI is designed for composability with other command-line tools. Upon successful completion, it outputs the container URL to stdout in this format: + +```console +wave.seqera.io/wt/xxxxxxxxxxxx/wave/build:xxxxxxxxxxxxxxxx +``` + +In the above example: + +- The first 12-character sequence is a unique access token +- The second 16-character sequence is a checksum that uniquely identifies the build + +## Container persistence + +By default, Wave container builds are ephemeral. You can persist, or _freeze_, a container build by saving it in a private container registry. For Wave to freeze your container image, you must provide a Seqera access token so that Wave can access your private registry credentials. + +## Seqera Platform integration + +You can integrate Wave CLI with your Seqera Platform instance by specifying a token and optional workspace ID and endpoint. Each value can be specified either as an environment variable or as an argument to the CLI. An environment variable is always overridden by the equivalent CLI argument, so if the `TOWER_ACCESS_TOKEN` environment variable is set and the `--tower-token` is specified, the value used for the CLI argument takes precedence. + +### CLI arguments + +The following CLI arguments are available for Seqera Platform integration: + +- `--tower-token`: A Seqera Platform auth token so that Wave can access your private registry credentials. +- `--tower-endpoint`: For Enterprise customers, the URL endpoint for your instance, such as `https://api.cloud.seqera.io`. +- `--tower-workspace-id`: A Seqera Platform workspace ID, such as `1234567890`, where credentials may be stored. + +### Environment variables + +The following environment variables are available for Seqera Platform integration: + +- `TOWER_ACCESS_TOKEN`: A Seqera Platform auth token so that Wave can access your private registry credentials. +- `TOWER_API_ENDPOINT`: For Enterprise customers, the URL endpoint for your instance, such as `https://api.cloud.seqera.io`. +- `TOWER_WORKSPACE_ID`: A Seqera Platform workspace ID, such as `1234567890`, where credentials may be stored. + +## Usage limits + +The following usage limits apply: + +- Anonymous usage + - 25 container builds per day + - 250 container pulls per hour +- Seqera Platform authenticated users + - 100 container builds per hour + - 2,000 container pulls per minute + +To authenticate with Seqera, define an access token in the `TOWER_ACCESS_TOKEN` environment variable or specify the token with the `--tower-token` CLI argument. + +## Context directory size limits + +The following file size limits apply when specifying a [context] directory: + +- A file must be no larger than 1 MB each. +- A directory must be no larger than 10 MB, inclusive of all files. + +## Image layer caching + +When you build a container with the CLI, a successful build returns a unique `wave.seqera.io` URL each time. Unchanged image layers retain their digest SHA and therefore can be reused from image layer caches. + +If you specify the `--cache-repo` argument, Wave caches new image layers that it creates to the container registry that you specify. + +:::important +This caching behavior doesn't apply when building Singularity containers, where the image file name is considered authoritative. For example, the `wave.seqera.io-wt-53014394cbda-nextflow-rnaseq-nf-v1.1.img` file is the result of a Singularity container build. +::: + +To avoid unnecessary image downloads, you can freeze the built container to provide a stable URL. + +## Image security scans + +As part of the build process, for containers that build successfully, Wave conducts a vulnerability scan using the [Trivy](https://trivy.dev/) security scanner. You must specify a Seqera access token to receive an email that links to the result of the security scan. + +:::note +Singularity containers are not currently scanned. +::: + +[context]: https://docs.docker.com/build/building/context/ diff --git a/new-docs/wave-docs/cli/installation.mdx b/new-docs/wave-docs/cli/installation.mdx new file mode 100644 index 0000000000..b079355e3f --- /dev/null +++ b/new-docs/wave-docs/cli/installation.mdx @@ -0,0 +1,84 @@ +--- +title: Installation +--- + +The Wave CLI is distributed as an easy to use self-installing package and via Homebrew. + +The following sections describe how to install the Wave CLI on your system. + +## Self-install + +To self-install the latest Wave release from GitHub: + +1. Download the [latest version of the Wave CLI][download] for your platform. + +1. In a new terminal, complete the following steps: + + 1. Move the executable from your downloads folder to a location in your `PATH`, such as `~/bin`. For example: + + ```bash + mv wave-cli-0.8.0-macos-x86_64 ~/bin/wave + ``` + + 1. Ensure that the executable permission is set. For example: + + ```bash + chmod u+x ~/bin/wave + ``` + +1. Verify that you can build containers with Wave: + + 1. Create a basic `Dockerfile`: + + ```Dockerfile + cat << EOF > ./Dockerfile + FROM busybox:latest + EOF + ``` + + 1. Use the CLI to build the container: + + ```bash + wave -f Dockerfile + ``` + + Example output: + + ```console + wave.seqera.io/wt/xxxxxxxxxxxx/wave/build:xxxxxxxxxxxxxxxx + ``` + +## Homebrew + +To install the latest version with [Homebrew]: + +1. Install Wave CLI with the following command: + + ```bash + brew install seqeralabs/tap/wave-cli + ``` + +1. Verify that you can build containers with Wave: + + 1. Create a basic `Dockerfile`: + + ```dockerfile + cat << EOF > ./Dockerfile + FROM busybox:latest + EOF + ``` + + 1. Use the CLI to build the container: + + ```bash + wave -f Dockerfile + ``` + + Example output: + + ```console + wave.seqera.io/wt/xxxxxxxxxxxx/wave/build:xxxxxxxxxxxxxxxx + ``` + +[download]: https://github.com/seqeralabs/wave-cli/releases +[Homebrew]: https://brew.sh/ diff --git a/new-docs/wave-docs/cli/use-cases.md b/new-docs/wave-docs/cli/use-cases.md new file mode 100644 index 0000000000..f3ee816528 --- /dev/null +++ b/new-docs/wave-docs/cli/use-cases.md @@ -0,0 +1,272 @@ +--- +title: Use cases +description: Learn how to use Wave CLI for building Docker and Singularity containers from various sources +date: "2024-08-22" +tags: [wave cli, use cases, containers, docker, singularity] +--- + +The Wave CLI enables you to build Docker and Singularity containers from various sources, including Dockerfiles, Singularity definition files, file system directories, and Conda packages. The following sections describe several common use cases. + +:::tip +To get started with an example Nextflow pipeline that uses Wave CLI, see [Wave CLI][start]. +::: + +## Augment a container with a directory + +The Wave CLI supports container augmentation with a specified directory. You can use container augmentation to dynamically add layers, such as scripts or configuration files, to your container. + +
+**Augment a container with a directory** + +**Related CLI arguments** + +Directory builds support the following arguments: + +- `--layer`: Specifies a directory containing layer content. +- `--image`, `-i`: Specifies an existing container image (default: `docker.io`). Accepts image names (e.g., `alpine:latest`) or image URLs (e.g., `public.ecr.aws/docker/library/busybox`). + +**Limitations** + +The following limitations apply: + +- A base image must be specified. +- Each file must be no larger than 1 MB. +- A directory must be no larger than 10 MB, inclusive of all files. + +**Example usage** + +Create a custom layer with a shell script and add it to an Alpine base image: + +1. Create a new context directory: + + ```bash + mkdir -p new-layer/usr/local/bin + printf 'echo Hello world!' > new-layer/usr/local/bin/hello.sh + chmod +x new-layer/usr/local/bin/hello.sh + ``` + +1. Build and run the image: + + ```bash + docker run $(wave -i alpine --layer new-layer) sh -c hello.sh + ``` +
+ +## Build a container from Conda packages + +The Wave CLI supports building a container from a list of [Conda][conda] packages. + +
+**Build a container from Conda packages** + +**Related CLI arguments** + +Conda builds support the following arguments: + +- `--conda-base-image`: Specifies the base image for installing Conda packages (default: `mambaorg/micromamba:1.5.10-noble`). +- `--conda-channels`: Specifies one or more comma-separated channels (default: `seqera,bioconda,conda-forge,defaults`). +- `--conda-file`: Specifies a [Conda lock file][conda-lock] path or URL. +- `--conda-package`: Specifies Conda packages to install. Supports expressions such as `bioconda::samtools=1.17` or `samtools>=1.0,<1.17`. Accepts a comma-separated list or can be specified multiple times. +- `--conda-run-command`: Specifies a Docker `RUN` command to execute during the build. Can be specified multiple times. + +**Example usage** + +Build a container with specific versions of the `samtools` and `bamtools` packages: + +```bash +wave \ + --conda-package bamtools=2.5.2 \ + --conda-package samtools=1.17 +``` +
+ +## Build a container from a Dockerfile + +The Wave CLI supports building a container from a `Dockerfile`. Specifying an optional build context allows the use of `ADD` and `COPY` commands in a Dockerfile. + +:::note +Dockerfile builds that require `--build-arg` for build time variables aren't currently supported. +::: + +
+**Build a container from a Dockerfile** + +**Related CLI arguments** + +Container builds support the following arguments: + +- `--containerfile`, `-f`: A Dockerfile to build. Build arguments aren't currently supported. +- `--context`: A directory that contains the context for the build. + +**Example usage** + +Build a container that installs several packages from a `Dockerfile`: + +1. Create a Dockerfile: + + ```bash + cat << EOF > ./Dockerfile + FROM alpine + + RUN apk update && apk add bash cowsay \ + --update-cache \ + --repository https://alpine.global.ssl.fastly.net/alpine/edge/community \ + --repository https://alpine.global.ssl.fastly.net/alpine/edge/main \ + --repository https://dl-3.alpinelinux.org/alpine/edge/testing + EOF + ``` + +1. Build and run the container: + + ``` + container=$(wave --containerfile ./Dockerfile) + docker run --rm $container cowsay "Hello world" + ``` + +Build a container from a Dockerfile with a local build context: + +1. Create a Dockerfile that references a local file: + + ```bash + cat << EOF > ./Dockerfile + FROM alpine + ADD hello.sh /usr/local/bin/ + EOF + ``` + +1. Create the shell script in the build context directory: + + ```bash + mkdir -p build-context/ + printf 'echo Hello world!' > build-context/hello.sh + chmod +x build-context/hello.sh + ``` + +1. Build and run the container with the build context: + + ```bash + docker run $(wave -f Dockerfile --context build-context) sh -c hello.sh + ``` +
+ +## Build a Singularity container + +The Wave CLI supports building [Singularity][singularity] containers. A target build repository, specified with the `--build-repo` argument, is required to build Singularity containers. + +
+**Build a Singularity container** + +**Related CLI arguments** + +Singularity container builds support the following arguments: + +- `--build-repo`: Specifies the target repository to save the built container. +- `--freeze`: Enables container freeze mode. +- `--singularity`, `-s`: Enables Singularity container builds. +- `--tower-token`: Specifies a Seqera access token to access private registry credentials stored in Platform (not required if the `TOWER_ACCESS_TOKEN` environment variable is set). +- `--tower-workspace-id`: Specifies a Seqera workspace ID (e.g., `1234567890`) where credentials are stored. Requires `--tower-token` flag or `TOWER_ACCESS_TOKEN` environment variable to be set. + +**Limitations** + +The following limitations apply: + +- The `linux/arm64` platform is not currently supported. + +**Example usage** + +Augment a Docker base image and save it as a Singularity container: + +```bash +wave -i alpine --layer context-dir/ --build-repo docker.io/user/repo +``` + +Build a Singularity container from a SingularityCE definition (`.def`) file: + +```bash +wave -f hello-world.def --singularity --freeze --build-repo docker.io/user/repo +``` + +Build a Singularity container from Conda packages: + +```bash +wave --conda-package bamtools=2.5.2 --conda-package samtools=1.17 --freeze --singularity --build-repo docker.io/user/repo +``` + +
+ +## Build a container and freeze to a container registry + +The Wave CLI supports building a container and persisting the container to a container registry, such as DockerHub. You can refer to this frozen container image in a Dockerfile or [Nextflow][nextflow] pipeline in the same way as any other container. + +
+**Build a container and freeze to a container registry** + +**Prerequisites** + +Ensure the following conditions are met: + +- You created a Seqera access token +- You specified the destination container registry credentials in Seqera +- You specified the Seqera access token via either the `TOWER_ACCESS_TOKEN` environment variable or the `--tower-token` Wave command-line option + +**Related CLI arguments** + +Container freeze builds support the following arguments: + +- `--build-repo`: Specifies the target repository to save the built container. +- `--freeze`: Enables container freeze mode. +- `--tower-token`: Specifies a Seqera access token for accessing private registry credentials (not required if the `TOWER_ACCESS_TOKEN` environment variable is set). +- `--tower-workspace-id`: Specifies a Seqera workspace ID (e.g., `1234567890`) where credentials are stored. + +**Example usage** + +Freeze the `alpine` container image to a private Docker Hub registry: + +```bash +wave -i alpine --freeze \ + --build-repo docker.io//repo --tower-token +``` + +
+ +## Mirror a container image to another registry + +The Wave CLI supports mirroring, i.e., copying containers to a container registry of your choice. + +
+**Mirror a container image to another registry** + +**Prerequisites** + +Ensure the following conditions are met: + +- You created a Seqera access token. +- You specified the destination container registry credentials in Seqera. +- You specified the Seqera access token via either the `TOWER_ACCESS_TOKEN` environment variable or the `--tower-token` Wave command-line option. + +**Related CLI arguments** + +Container mirroring supports the following arguments: + +- `--mirror`: Enables container mirror mode. +- `--build-repo`: Specifies the target repository to save the mirrored container. +- `--tower-token`: Specifies a Seqera access token to access private registry credentials stored in Platform (not required if the `TOWER_ACCESS_TOKEN` environment variable is set). + +**Example usage** + +Mirror the [`samtools:0.1.16--2`][samtools] container image to a private Docker Hub registry: + +```bash +wave -i quay.io/biocontainers/samtools:0.1.16--2 --mirror \ + --build-repo docker.io//containers --tower-token +``` + +
+ +[conda]: https://anaconda.org/anaconda/repo +[conda-lock]: https://github.com/conda/conda-lock +[nextflow]: https://www.nextflow.io/ +[samtools]: https://quay.io/repository/biocontainers/samtools?tab=tags +[singularity]: https://docs.sylabs.io/guides/latest/user-guide/introduction.html +[singularityce]: https://docs.sylabs.io/guides/latest/user-guide/definition_files.html +[start]: /wave_docs/wave_repo/docs/tutorials/wave-cli.mdx diff --git a/new-docs/wave-docs/configuration.md b/new-docs/wave-docs/configuration.md new file mode 100644 index 0000000000..b379435c84 --- /dev/null +++ b/new-docs/wave-docs/configuration.md @@ -0,0 +1,386 @@ +--- +title: Configuration reference +tags: [configuration, reference, wave] +--- + +This page documents configuration options for self-hosted Wave deployments. +If you are using Wave as a service, these configuration options do not apply to your setup. + +Configure Wave by setting YAML values in the [`config.yml`](https://github.com/seqeralabs/wave/blob/master/config.yml) configuration file: + +```yml +wave: + mail: + from: "wave-app@seqera.io" +``` + +Configuration paths in this reference use dot notation to represent nested YAML keys. In the example above, the `from` value nested under the `mail` section is referenced as `wave.mail.from`. + +You can configure Wave using either the `config.yml` file or environment variables. Environment variables are provided where available, though not all configuration options support them. + +## General + +Configure general Wave application settings, such as application name, port, anonymous access permissions, and platform URLs with the following options: + +`tower.endpoint.url` *(optional)* +: Specifies the URL pointing to the Seqera platform API service (default: [`https://api.cloud.seqera.io`](https://api.cloud.seqera.io)). + Can be set using the `${TOWER_ENDPOINT_URL}` environment variable. + +`wave.allowAnonymous` *(required)* +: Enables anonymous access to the Wave server (default: `false`). +: Modify this option based on your security requirements. + +`wave.denyPaths` *(optional)* +: Filter out API calls for specific artifacts, like manifests, that don't exist. + +`wave.server.url` *(required)* +: Specifies the URL for the Wave server. + Can be set using the `${WAVE_SERVER_URL}` environment variable. + +`wave.tokens.cache.duration` *(optional)* +: Sets the duration for cached tokens generated by Wave (default: `1h`). + +## Container registry + +Wave uses the generic format `wave.registries..username` and `wave.registries..password` for registry authentication. +You must specify all repositories used in your Wave installation. + +The examples below show standard formats for known registries, but you can customize the registry name (for example, change `azurecr.io` to `seqeralabs.azurecr.io`). + +:::note +You can also define container registry credentials in the [`config.yml`](https://github.com/seqeralabs/wave/blob/master/config.yml) file. +These configurations enable Wave authentication for repositories used to push or pull artifacts. +::: + +Configure container registry authentication with the following options: + +`wave.registries..password` *(optional)* +: Specifies the AWS ECR password for authentication. + For example: `195996028523.dkr.ecr.eu-west-1.amazonaws.com`. + Can be set using the `${AWS_SECRET_ACCESS_KEY}` environment variable. + +`wave.registries..username` *(optional)* +: Specifies the AWS ECR (Elastic Container Registry) username for authentication. + For example, `195996028523.dkr.ecr.eu-west-1.amazonaws.com`. + Can be set using the `${AWS_ACCESS_KEY_ID}` environment variable. + +`wave.registries.default` *(optional)* +: Sets the default container registry for Wave (default: `docker.io`). + +`wave.registries.docker.io.password` *(optional)* +: Specifies the Docker Hub password or PAT (Personal Access Token) for authentication. + Can be set using the `${DOCKER_PAT}` environment variable. + +`wave.registries.docker.io.username` *(optional)* +: Specifies the Docker Hub username for authentication. + Can be set using the `${DOCKER_USER}` environment variable. + +`wave.registries.quay.io.password` *(optional)* +: Specifies the Quay.io password or PAT for authentication. + Can be set using the `${QUAY_PAT}` environment variable. + +`wave.registries.quay.io.username` *(optional)* +: Specifies the Quay.io username for authentication. + Can be set using the `${QUAY_USER}` environment variable. + +## HTTP client + +Configure the HTTP client with the following options: + +`wave.httpclient.connectTimeout` *(optional)* +: Sets the connection timeout for the HTTP client (default: `20s`). + +`wave.httpclient.retry.attempts` *(optional)* +: Sets the number of HTTP client retry attempts (default: `5`). + +`wave.httpclient.retry.delay` *(optional)* +: Sets the delay for HTTP client retries (default: `1s`). + +`wave.httpclient.retry.jitter` *(optional)* +: Sets the jitter for HTTP client retries (default: `0.25`). + +`wave.httpclient.retry.maxDelay` *(optional)* +: Sets the maximum delay for HTTP client retries. + +`wave.httpclient.retry.multiplier` *(optional)* +: Sets the multiplier for HTTP client retries (default: `1.0`). + +## Container build process + +Configure how Wave builds container images and manages associated logs for monitoring, troubleshooting, and delivery with the following options: + +`wave.build.buildkit-image` *(required)* +: Sets the [Buildkit](https://github.com/moby/buildkit) container image used in the Wave build process (default: `moby/buildkit:v0.13.2-rootless`). + +`wave.build.cache` *(required)* +: Sets the container repository used to cache layers of images built by Wave. + +`wave.build.cleanup` *(optional)* +: Sets the cleanup strategy after the build process. + For example, set to `OnSuccess` for cleanup only if a build is successful. + +`wave.build.compression` *(optional)* +: Sets which type of compression will be applied to cache layers (default: `gzip`). + Options include: `uncompressed`,`estargz`, and `zstd`. + +`wave.build.force-compression` *(optional)* +: Enables forced compression for each cache layers produced by the build process (default: `false`). + +`wave.build.oci-mediatypes` *(optional)* +: Enables OCI media types in exported manifests (default: `true`). + +`wave.build.public-repo` *(optional)* +: Specifies a public repository for the container images built by Wave. + Wave uses this repository when you freeze container images without specifying `wave.build.repo`. + When not configured, you must specify `wave.build.repo` for freeze mode operations. + +`wave.build.repo` *(required)* +: Sets the Docker container repository for the container images built by Wave. + +`wave.build.singularity-image` *(optional)* +: Sets the [Singularity](https://quay.io/repository/singularity/singularity?tab=tags) image used in the build process (default: `quay.io/singularity/singularity:v3.11.4-slim`). + +`wave.build.singularity-image-arm64` *(optional)* +: Sets the ARM64 version of the Singularity image for the build process (default: `quay.io/singularity/singularity:v3.11.4-slim-arm64`). + +`wave.build.status.delay` *(optional)* +: Sets the delay between build status checks (default: `5s`). + +`wave.build.status.duration` *(optional)* +: Sets the duration for build status checks (default: `1d`). + +`wave.build.timeout` *(optional)* +: Sets the maximum duration for the build process (default: `5m`). + +`wave.build.trusted-timeout` *(optional)* +: Sets the maximum duration for the build process when you are authenticated and freeze mode is enabled (default: `10m`). + If set to less than `wave.build.timeout`, the longer `wave.build.timeout` value is used. + +`wave.build.workspace` *(required)* +: Sets the path to the directory used by Wave to store artifacts such as Containerfiles, Trivy cache for scan, Buildkit context, and authentication configuration files. + For example, `/efs/wave/build`. + +### Build process logs + +Configure how Wave stores and delivers build logs from containers and Kubernetes pods, which can be retrieved later or included in build completion emails, with the following options: + +`wave.build.locks.path` *(required)* +: Sets the path inside `wave.build.logs.bucket` where Wave will store Conda lock files. + +`wave.build.logs.bucket` *(required)* +: Sets the AWS S3 bucket where Wave will store build process logs. + +`wave.build.logs.maxLength` *(optional)* +: Determines the maximum number of bytes that can be read from a log file. If a log file exceeds this limit, it will be truncated (default: `100000` (100 KB)). + +`wave.build.logs.path` *(required)* +: Sets the path inside `wave.build.logs.bucket` where Wave will store build logs. + +`wave.build.logs.prefix` *(optional)* +: Sets the prefix for build process log files in the specified S3 bucket. + +### Kubernetes container build process + +Configure Kubernetes-specific settings for Wave, where build and scan processes share most configurations except for CPU and memory requirements, with the following options: + +`wave.build.k8s.labels` *(optional)* +: Sets labels on Wave build Kubernetes pods. + +`wave.build.k8s.namespace` *(required)* +: Sets the Kubernetes namespace where Wave will run its build pods. + +`wave.build.k8s.node-selector` *(optional)* +: Sets the node selector for Wave build Kubernetes pods. + +`wave.build.k8s.resources.requests.cpu` *(optional)* +: Sets the [CPU resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes) to allocate to Wave build processes. + For example, set to `2` (2 CPU cores) or `1500Mi` (1.5 CPU cores). + +`wave.build.k8s.resources.requests.memory` *(optional)* +: Sets the [memory resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes) to allocate to Wave build processes. + For example, set to `3Gi` (3 Gigabytes) or `2000Mi` (2000 Megabytes). + +`wave.build.k8s.service-account` *(optional)* +: Specifies the Kubernetes service account name to be used by Wave build pods. + +`wave.build.k8s.storage.claimName` *(optional)* +: Sets the volume claim name for the Wave build Kubernetes pods. + +`wave.build.k8s.storage.mountPath` *(optional)* +: Sets the volume mount path on Wave build Kubernetes pods. + +## Container scan process + +Configure how Wave's vulnerability scanning process uses a [Trivy Docker image](https://hub.docker.com/r/aquasec/trivy) with customizable tags and severity levels with the following options: + +`wave.scan.enabled` *(optional)* +: Enables vulnerability scanning (default: `true`). + +`wave.scan.image.name`  *(optional)* +: Sets the [Trivy Docker image](https://hub.docker.com/r/aquasec/trivy) to use for container security scanning (default: `aquasec/trivy:0.47.0`). + +`wave.scan.reports.path` *(required)* +: Sets the path inside the S3 bucket where Wave will store SBOM reports. + For example, `s3://wave-store/scan-reports`. + +`wave.scan.severity` *(optional)* +: Sets the [severity levels](https://aquasecurity.github.io/trivy/v0.22.0/vulnerability/examples/filter/) to report in vulnerability scanning. +: Options include: `MEDIUM`,`HIGH`, and `CRITICAL`. + +### Kubernetes Wave scan process + +Configure Wave scanning process resource requirements for Kubernetes deployments with the following options: + +`wave.scan.k8s.resources.requests.cpu` *(optional)* +: Sets the amount of [CPU resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes) allocated to Wave scan processes. + For example, set to `2` (2 CPU cores) or `1500Mi` (1.5 CPU cores). + +`wave.scan.k8s.resources.requests.memory` *(optional)* +: Sets the [memory resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes) allocated to Wave scan processes. + For example, set to `3Gi` (3 Gigabytes) or `2000Mi` (2000 Megabytes). + +## Rate limits + +Configure how Wave controls rate limits for anonymous and authenticated user access with the following options: + +`rate-limit.build.anonymous` *(required)* +: Sets the rate limit for build requests from anonymous users (default: 25 build requests per day (`25/1d`); max: 25). + +`rate-limit.build.authenticated` *(required)* +: Sets the rate limit for build requests from authenticated users (default: 100 build requests per hour (`100/1h`); max: 100). + +`rate-limit.pull.anonymous` *(required)* +: Sets the rate limit for anonymous pull requests from anonymous users (default: 250 pull requests per hour (`250/1h`); max: 250). + +`rate-limit.pull.authenticated` *(required)* +: Sets the rate limit for authenticated pull requests from authenticated users (default: 2k pull requests per minute (`2000/1m`); max: 2k). + +## Database and cache + +Wave uses external database and caching services to store application data and improve performance. + +### Redis + +Configure Redis with the following options: + +`redis.client.timeout` *(optional)* +: Sets the timeout duration (in milliseconds) for Redis client operations (default: `5000` (5 seconds)). + +`redis.password` *(optional)* +: Specifies the password used to authenticate with the Redis server. + Can be set using the `${REDIS_PASSWORD}` environment variable. + +`redis.pool.enabled` *(optional)* +: Enables the Redis pool (default: `true`). + +`redis.pool.maxIdle` *(optional)* +: Sets the maximum number of idle connections to maintain in the Redis connection pool (default: `10`). + +`redis.pool.maxTotal` *(optional)* +: Sets the maximum number of connections that can be maintained in the Redis connection pool (default: `50`). + +`redis.pool.minIdle` *(optional)* +: Sets the minimum number of idle connections to maintain in the Redis connection pool (default: `0`). + +`redis.uri` *(required)* +: Specifies the URI for connecting to Redis (default format: `redis://${REDIS_HOST:redis}:${REDIS_PORT:6379}`). + Can be set using the `${REDIS_URI}` environment variable. + +### PostgreSQL + +Configure PostgreSQL with the following options: + +`wave.db.password` *(required)* +: Specifies the password for the PostgreSQL database user. + Can be set using the `${WAVE_DB_PASSWORD}` environment variable. + +`wave.db.uri` *(required)* +: Specifies the JDBC connection string for the PostgreSQL database. + For example, `jdbc:postgresql://localhost:5432/wave`. + Can be set using the `${WAVE_DB_URI}` environment variable. + +`wave.db.user` *(required)* +: Specifies the username for authenticating with the PostgreSQL database. + Can be set using the `${WAVE_DB_USER}` environment variable. + +## Blob cache + +Configure how Wave caches container blobs to improve client performance and optionally delegates transfer tasks to Kubernetes pods for scalability with the following options: + +`wave.blobCache.baseUrl` *(optional)* +: Specifies the URL, which will override the base URL (part of URL before the blob path) of blobs sent to the end client. + +`wave.blobCache.cloudflare.lifetime` *(optional)* +: Sets the validity of the Cloudflare WAF token. + +`wave.blobCache.cloudflare.secret-key` *(optional)* +: Specifies the [Cloudflare secret](https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/) to create the WAF token. + +`wave.blobCache.cloudflare.urlSignatureDuration` *(optional)* +: Sets the validity of the AWS S3 URL signature (default: `30m`). + +`wave.blobCache.enabled` *(optional)* +: Enables the blob cache (default: `false`). + +`wave.blobCache.requestsCpu` *(optional)* +: Sets the amount of [CPU resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes) allocated to the k8s pod used for blob binary transfers. + +`wave.blobCache.requestsMemory` *(optional)* +: Sets the [memory resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes) allocated to the k8s pod used for blob binary transfers. + +`wave.blobCache.s5cmdImage` *(optional)* +: Sets the container image that supplies the [s5cmd tool](https://github.com/peak/s5cmd) to upload blob binaries to the S3 bucket (default: `public.cr.seqera.io/wave/s5cmd:v2.2.2`). + +`wave.blobCache.signing-strategy` *(optional)* +: Sets the URL signing strategy for different services. + Currently supports AWS S3 and Cloudflare service. + Options include: `aws-presigned-url` and `cloudflare-waf-token`. + +`wave.blobCache.status.delay` *(optional)* +: Sets the time delay in checking the status of the transfer of the blob binary from the repository to the cache (default: `5s`). + +`wave.blobCache.status.duration` *(optional)* +: Sets the time for which Wave will store the blob binary in cache (default: `5d`). + +`wave.blobCache.storage.accessKey` *(optional)* +: Specifies the access key (part of credentials) to access the resources of the service used for caching. + +`wave.blobCache.storage.bucket` *(required)* +: Sets the name of Cloudflare or S3 bucket. + For example, `s3://wave-blob-cache`. + +`wave.blobCache.storage.endpoint` *(optional)* +: Specifies the URL for the storage location for the download or upload of blob binaries. + +`wave.blobCache.storage.region` *(required)* +: Sets the AWS region where the bucket is created. + +`wave.blobCache.storage.secretKey` *(optional)* +: Specifies the secret key (part of credentials) to access the resources of the service used for caching. + +`wave.blobCache.timeout` *(optional)* +: Sets the timeout for blob binary transfer, after which Wave will throw a `TransferTimeoutException` exception (default: `5m`). + +## Email configuration + +Configure how Wave sends email notifications on behalf of the service with the following options: + +`mail.from` *(required)* +: Specifies the sender's email address for Wave notifications. + Can be set using the `${MAIL_FROM}` environment variable. + +## Metrics + +Configure how Wave Metrics service provides data about container builds and pulls per organization and date with the following options: + +`wave.metrics.enabled` *(optional)* +: Enables Wave metrics (default: `false`). + +## Accounts + +Configure user credentials for accessing authenticated Wave APIs and services with the following options: + +`wave.accounts` *(required)* +: Sets a list of credentials to access authenticated Wave APIs like metrics APIs. + Format of the credential list: `- :` + diff --git a/new-docs/wave-docs/configure-wave.md b/new-docs/wave-docs/configure-wave.md new file mode 100644 index 0000000000..d5b207265b --- /dev/null +++ b/new-docs/wave-docs/configure-wave.md @@ -0,0 +1,242 @@ +--- +title: Configure Wave +--- + +This page describes common operations to set up and configure Wave. + +:::info +See [Configuration reference](./configuration.md) for a full list configuration options for self-hosted Wave deployments. +::: + +## Email notifications + +Wave can be configured to send email notifications for various build related events. + +### SMTP + +#### Configuration + +Add `mail` to your Micronaut environments and configure the SMTP settings in your Wave configuration: + +**Environment configuration:** + +```yaml +# Add 'mail' to your existing environments +MICRONAUT_ENVIRONMENTS: "postgres,redis,lite,mail" +``` + +**SMTP configuration:** + +```yaml +mail: + from: "wave-notifications@your-domain.com" + smtp: + host: "smtp.your-provider.com" + port: "587" + user: "your-smtp-username" + password: "your-smtp-password" + auth: true + starttls: + enable: true + required: true + ssl: + protocols: "TLSv1.2" +``` + +#### Configuration Options + +| Setting | Description | Example Values | +| ------------------- | ------------------------------------ | ------------------------------------------- | +| `from` | Email address that appears as sender | `wave@company.com` | +| `host` | SMTP server hostname | `smtp.gmail.com`, `smtp.office365.com` | +| `port` | SMTP server port | `587` (STARTTLS), `465` (SSL), `25` (plain) | +| `user` | SMTP authentication username | Usually your email address | +| `password` | SMTP authentication password | App password or account password | +| `auth` | Enable SMTP authentication | `true` (recommended) | +| `starttls.enable` | Enable STARTTLS encryption | `true` (recommended) | +| `starttls.required` | Require STARTTLS encryption | `true` (recommended) | +| `ssl.protocols` | Supported SSL/TLS protocols | `TLSv1.2`, `TLSv1.3` | + +### SES + +For AWS environments, Wave supports direct integration with Amazon Simple Email Service (SES) using IAM authentication instead of SMTP credentials. + +#### Requirements + +- AWS SES must be configured in the same region as your Wave deployment +- Wave must have appropriate IAM permissions to send emails via SES +- The IAM role or user must have `ses:SendEmail` and `ses:SendRawEmail` permissions + +#### Configuration + +Add `aws-ses` to your Micronaut environments along with `mail`: + +**Environment configuration:** + +```yaml +# Add both 'mail' and 'aws-ses' to your existing environments +MICRONAUT_ENVIRONMENTS: "postgres,redis,lite,mail,aws-ses" +``` + +**SES configuration:** + +```yaml +mail: + from: "wave-notifications@your-domain.com" +``` + +#### IAM permissions + +Wave requires the following IAM permissions for SES integration: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["ses:SendEmail", "ses:SendRawEmail"], + "Resource": "*" + } + ] +} +``` + +#### SES setup requirements + +Before configuring Wave with SES: + +1. **Verify your sending domain** in the AWS SES console +2. **Move out of SES sandbox** if sending to unverified email addresses +3. **Configure appropriate sending limits** for your use case +4. **Ensure SES is available** in your Wave deployment region + +#### Regional considerations + +Wave will automatically use SES in the same AWS region where it's deployed. Ensure SES is: + +- Available and configured in your deployment region +- Has verified domains/addresses for your `from` email address +- Not in sandbox mode if sending to external recipients + +**Note:** No SMTP configuration is needed when using SES with IAM authentication - Wave will use the AWS SDK to send emails directly through the SES API. + +## Security scanning + +Wave can perform security scanning on container builds. This feature requires the build service to be enabled and additional scanning infrastructure. + +**Prerequisites:** + +- Wave build service must be enabled (`wave.build.enabled: true`) +- Scanning backend must be configured and accessible +- Appropriate compute resources for scanning workloads + +``` +wave: + build: + enabled: true + scan: true +``` + +## ECR cache repository + +Wave supports using Amazon Elastic Container Registry (ECR) as a cache repository to store and reuse **build** layers, improving build performance and reducing bandwidth usage. + +**Prerequisites:** + +- AWS ECR repository configured in the same region as Wave +- Wave must have appropriate IAM permissions to push/pull from ECR +- ECR repository must be accessible from Wave build infrastructure + +#### Configuration + +Configure ECR cache repository in your Wave configuration: + +```yaml +wave: + build: + enabled: true + cache: + enabled: true + repository: "123456789012.dkr.ecr.us-east-1.amazonaws.com/wave-cache" +``` + +#### IAM permissions + +Wave requires the following IAM permissions for ECR cache operations: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload" + ], + "Resource": ["arn:aws:ecr:us-east-1:123456789012:repository/wave-cache"] + }, + { + "Effect": "Allow", + "Action": ["ecr:GetAuthorizationToken"], + "Resource": "*" + } + ] +} +``` + +#### ECR repository setup + +Create and configure your ECR cache repository: + +1. **Create ECR repository:** + + ```bash + aws ecr create-repository --repository-name wave-cache --region us-east-1 + ``` + +2. **Configure lifecycle policy** to manage cache storage costs: + ```json + { + "rules": [ + { + "rulePriority": 1, + "selection": { + "tagStatus": "untagged", + "countType": "sinceImagePushed", + "countUnit": "days", + "countNumber": 7 + }, + "action": { + "type": "expire" + } + } + ] + } + ``` + +#### Benefits + +Using ECR as a cache repository provides: + +- **Faster builds** by reusing cached layers +- **Reduced bandwidth** usage for repeated builds +- **Cost optimization** through efficient layer storage +- **Regional performance** with ECR in the same region as Wave +- **Integrated security** with AWS IAM and ECR security features + +#### Configuration Options + +| Setting | Description | Example | +| ------------------ | -------------------- | --------------------------------------------------------- | +| `cache.enabled` | Enable build caching | `true` | +| `cache.repository` | ECR repository URL | `123456789012.dkr.ecr.us-east-1.amazonaws.com/wave-cache` | + +**Note:** ECR cache requires Wave build service to be enabled and is only available in AWS deployments with proper ECR access configured. diff --git a/new-docs/wave-docs/db-migration.md b/new-docs/wave-docs/db-migration.md new file mode 100644 index 0000000000..c132e5afaf --- /dev/null +++ b/new-docs/wave-docs/db-migration.md @@ -0,0 +1,123 @@ +--- +title: Wave database migration +description: Migrate your Wave installation from SurrealDB to PostgreSQL +tags: [wave, surrealdb, postgresql, migration] +--- + +[Wave 1.21.0](https://docs.seqera.io/changelog/wave/v1.21.0) introduces support for PostgreSQL as the primary database backend, replacing SurrealDB. + +This guide outlines the steps to migrate your existing Wave installation from SurrealDB to PostgreSQL. + +:::info[**Prerequisites**] +You will need the following to get started: + +- [Wave CLI](./cli/index.md) +- A PostgreSQL database accessible to Wave +::: + +## Database migration + +1. Generate build ID, scan ID, token, and mirror ID data to verify your migration in step 3. + + 1. Run a Wave build operation and capture the `buildId`: + + ```bash + wave --conda-package bwa --wave-endpoint + ``` + + 1. Verify the build and record the build ID: + + ```bash + curl /view/builds/ + ``` + + 1. Verify the scan and record the scan ID: + + ```bash + curl /view/scans/ + ``` + + 1. Create a container augmentation and record the token: + + ```bash + wave -i ubuntu --config-file --wave-endpoint + ``` + + 1. Verify the container and record the token: + + ```bash + curl /view/containers/ + ``` + + 1. Create a mirror operation and note the mirror ID: + + ```bash + wave --mirror -i ubuntu --build-repo --wave-endpoint + ``` + + 1. Verify the mirror and record the mirror ID: + + ```bash + curl /view/mirrors/ + ``` + +1. Migrate your database: + + 1. Add the following to your `MICRONAUT_ENVIRONMENTS`: + - `postgres` + - `surrealdb` + - `migrate` + - `redis` + + 1. Start the Wave application: + + ```console + INFO i.s.w.s.p.m.DataMigrationService - Data migration service initialized + ``` + + 1. Check the logs for the migration status: + + ```console + INFO i.s.w.s.p.m.DataMigrationService - All wave_request records migrated. + INFO i.s.w.s.p.m.DataMigrationService - All wave_scan records migrated. + INFO i.s.w.s.p.m.DataMigrationService - All wave_build records migrated. + INFO i.s.w.s.p.m.DataMigrationService - All wave_mirror records migrated. + ``` + + 1. When all records are migrated, remove `migrate` and `surrealdb` from your `MICRONAUT_ENVIRONMENTS`, then restart Wave. + +1. Use the build ID, scan ID, token, and mirror ID you generated in step 1 to verify the migration: + + 1. Verify the build data: + + ```bash + curl /view/builds/ + ``` + + 1. Verify the scan data: + + ```bash + curl /view/scans/ + ``` + + 1. Verify the container data: + + ```bash + curl /view/containers/ + ``` + + 1. Verify the mirror data: + + ```bash + curl /view/mirrors/ + ``` + +## Wave configuration + +Add the following properties to your Wave configuration file: + +`wave.build.logs.path` +: Sets the path inside `wave.build.logs.bucket`, where build logs will be stored. + +`wave.build.locks.path` +: Sets the path inside `wave.build.logs.bucket`, where conda lock files will be stored. diff --git a/new-docs/wave-docs/faq.md b/new-docs/wave-docs/faq.md new file mode 100644 index 0000000000..1459afd5a5 --- /dev/null +++ b/new-docs/wave-docs/faq.md @@ -0,0 +1,68 @@ +--- +title: Frequently asked questions +description: Find answers to common questions about Wave container provisioning +date: "2023-11-11" +tags: [wave, containers, nextflow, faq] +--- + +## Is Wave a container registry? + +Wave implements the Docker registry pull API, and therefore allows Docker and other compatible container engines to pull container images from it. However, strictly speaking it's not a container registry because it does not store container images, and it's not possible to push an image into it. + +From a technical point of view, Wave behaves as a proxy server that intermediates container pull requests from the Docker client and the target registry where the container image is stored. + +## Why should I use Wave instead of a container registry? + +Wave has been designed to streamline the use of software containers with Nextflow data analysis pipelines, in three ways: + +1. Safely handle the authentication of container repositories to allow access to private registries and the ability to pull from public registries without being affected by service rate limits. +2. Dynamically include in the container execution context pipeline scripts and infrastructure-related dependencies, without the need to rebuild the corresponding containers. +3. Building the pipeline containers on-demand, using Dockerfiles associated with the pipeline modules or Conda packages specified in the pipeline configuration. + +## How can I specify the credentials of my private container registry when using Wave? + +Store the container registry credentials in your [Seqera Platform](https://cloud.seqera.io) account. + +If you launch the Nextflow pipeline with Seqera, there's nothing else to do (other than using a compute environment with Wave support enabled). +If you're launching the pipeline with the Nextflow command line, add your [Seqera access token](https://docs.seqera.io/platform/latest/api/overview#authentication) to your Nextflow configuration. See [Access private container repositories](./nextflow/use-cases.md#access-private-container-repositories) for more information. + +## Does Wave modify my container images? + +No. Wave doesn't alter or modify your container images. Wave acts as proxy server between the Docker client (or equivalent container engine) and the target registry where the container image is hosted. + +With Wave container augmentation, you can include extra content required by the pipeline execution when you pull container images. + +## Can the container built by Wave be stored in my container registry? + +Yes. You can specify the container registry where Wave stores built containers by adding the following setting in your Nextflow configuration: + +```groovy +wave.build.repository = 'example.com/your/build-repo' +``` + +## Which container registries are supported by Wave? + +Wave has been tested with the following container registries: + +- [AWS Elastic Container Registry](https://aws.amazon.com/ecr/) +- [Azure Container Registry](https://azure.microsoft.com/en-us/products/container-registry) +- [Docker Hub](https://hub.docker.com/) +- [Quay.io](https://quay.io/) +- [Gitea Container Registry](https://docs.gitea.io/en-us/packages/container/) +- [GitHub Container Registry](https://github.blog/2020-09-01-introducing-github-container-registry/) +- [Google Artifact Registry](https://cloud.google.com/artifact-registry) + +## Can I use SHA256 digests in image names with Wave? + +Wave does not support using an SHA256 digest in the image name (for example, `ubuntu@sha256:3235...ce8f`) when using the augmentation process to extend container images. + +To reference a container by SHA256 digest in the image name with Wave, enable freeze mode and force the creation of a new container image using your specified base image by adding the following to your Nextflow configuration: + +```groovy +wave.enabled = true +wave.freeze = true +wave.strategy = ['dockerfile'] +wave.build.repository = '' +``` + +Replace `` with the repository where your image files should be uploaded. diff --git a/new-docs/wave-docs/index.md b/new-docs/wave-docs/index.md new file mode 100644 index 0000000000..3897406ffe --- /dev/null +++ b/new-docs/wave-docs/index.md @@ -0,0 +1,100 @@ +--- +title: Wave containers +--- + +Containers are an essential part of modern data analysis pipelines in bioinformatics. They encapsulate applications and dependencies in portable, self-contained packages that can be easily distributed across diverse computing environments. Containers are also key to enabling predictable and reproducible scientific results. + +However, workflows can comprise dozens of distinct container images. Pipeline developers must manage and maintain these container images and ensure that their functionality precisely aligns with the requirements of every pipeline task, creating unnecessary friction in the maintenance and deployment of data pipelines. + +Wave solves this problem by provisioning containers on-demand during pipeline execution. This allows the delivery of container images that are defined precisely depending on the requirements of each pipeline task in terms of dependencies and platform architecture. This process is completely transparent and fully automated, removing all the plumbing and friction commonly needed to create, upload, and maintain multiple container images required for pipeline execution. + +To get started with Wave: + +1. See the [Get started][started] guides. +1. Learn about [Nextflow integration][nf]. +1. Learn about the [Wave CLI][cli]. + +:::note +Wave is also available as a hosted service on [Seqera Platform](https://cloud.seqera.io/). For Seqera Enterprise customers, a licensed self-hosted Wave solution is also available. [Contact us](https://seqera.io/contact-us/) for more information. +::: + +[started]: ./tutorials/index.md +[nf]: ./nextflow/index.md +[cli]: ./cli/index.md + +## Wave features + +### Container registries + +#### Private container registries + +Wave integrates with [Seqera Platform credentials management][private] enabling seamless access and publishing to private registries. + +[private]: ./nextflow/use-cases.md#access-private-container-repositories + +#### Seqera Containers - The community container registry + +[Seqera Containers] is a free community service operated by Seqera. + +It uses Wave to build images from Conda / PyPI packages on demand, either through the [web interface](https://seqera.io/containers/) or using the [Wave CLI](./cli/index.md) / [Nextflow integration](./nextflow/index.md). + +These images are cached and hosted permanently, being served through a [Docker Distribution][docker] registry and hosted on AWS infrastructure. Images are cached and served via Cloudflare CDN. + +Images are publicly accessible to anyone for free and will be stored for at least 5 years. They can be pulled using any infrastructure (local, HPC, cloud) as Docker or native Singularity images. Images can be built for both `linux/aarch64` and `linux/arm64` architectures. + +:::note +Seqera Containers does not work with custom container files, augmentation, or authorization. It provides only Conda based containers. +::: + +[docker]: https://github.com/distribution/distribution +[Seqera Containers]: https://seqera.io/containers/ + +### Augment existing containers + +Wave offers a flexible approach to container image management. It allows you to [dynamically add custom layers][augment] to existing Docker images, creating new images tailored to your specific needs. +Any existing container can be extended without rebuilding it. You can add user-provided content such as custom scripts and logging agents, providing greater flexibility in the container’s configuration. + +[augment]: ./provisioning.md#container-augmentation + +### Conda-based containers + +Package management systems such as Conda and Bioconda simplify the installation of scientific software. +Wave enables dynamic provisioning of container images from any Conda or Bioconda recipe. Just [declare the Conda packages][conda] in your Nextflow pipeline and Wave will assemble the required container. + +[conda]: ./nextflow/use-cases.md#build-conda-based-containers + +### Singularity containers + +Singularity and Apptainer use a proprietary format called _Singularity Image Format_ (SIF). The Wave service can [provision containers based on the Singularity image format][singularity] either by using a `Singularityfile` file or Conda packages. The resulting Singularity image file is stored as an ORAS artifact in an OCI-compliant container registry of your choice or the Wave Community registry. + +The advantage of this approach is that Singularity and Apptainer engines can pull and execute container images natively without the extra conversion steps needed when using Docker images with these engines. + +:::note +Due to the Singularity image format's peculiarities, Wave's freeze mode is mandatory when provisioning Singularity images. +::: + +[singularity]: ./nextflow/use-cases.md#build-singularity-containers + +### Deploying containers across multi-clouds + +Cloud vendors provide integrated container registries with better performance and cost-efficiency than central, remote registries. +Storing container images in a private registry also enhances security and provide faster access with greater control. +Wave mirroring addresses these needs by copying containers to your chosen registry while preserving the original manifest, image name, and hash, and ensuring images remain unmodified and accessible via the original build hash. + +### Container security scanning + +Builds for OCI-compliant container images are automatically scanned for known security vulnerabilities. Wave conducts a vulnerability scan using the [Trivy](https://trivy.dev/) security scanner. Seqera Platform customers receive an email that includes a link to the security report listing any vulnerabilities discovered. + +### Optimize workloads for specific architectures + +Modern data pipelines can be deployed across different data centers having different hardware architectures such as AMD64, ARM64, and others. This requires curating different collections of containers for each architecture. +Wave allows for the on-demand provisioning of containers, depending on the target execution platform (in development). + +### Near caching + +The deployment of production pipelines at scale often requires the use of multiple cloud regions to enable efficient resource allocation. +However, this can result in an increased overhead when pulling container images from a central container registry. Wave allows the transparent caching of container images in the same region where computation occurs, reducing data transfer costs and time (in development). + +### Wave Lite + +Wave can be used in [Lite](./wave-lite.md) mode for container augmentation and inspection capabilities. Wave Lite can be deployed in both Docker Compose and Kubernetes installations. diff --git a/new-docs/wave-docs/install/_templates/wave.env b/new-docs/wave-docs/install/_templates/wave.env new file mode 100644 index 0000000000..5f383081d2 --- /dev/null +++ b/new-docs/wave-docs/install/_templates/wave.env @@ -0,0 +1,40 @@ +# +# Base URL that will be used to reach the Wave service. +# +WAVE_SERVER_URL=http://localhost:9090 + +# +# JDBC connection URI to connect the PostgreSQL database +# required by the Wave service. Note the database name is expected +# to be provided in place of the "changeme" suffix. +# +WAVE_DB_URI=jdbc:postgresql://host.docker.internal:5432/changeme + +# +# User name required to access the PostgreSQL database +# used by the Wave service. +# +WAVE_DB_USER=changeme + +# +# Password required to access the PostgreSQL database +# used by the Wave service. +# +WAVE_DB_PASSWORD=changeme + +# +# Connection URI to connect Wave with the Redis cache +# +REDIS_URI=redis://host.docker.internal:6379 + +# +# Connection URL to connect Wave service with Platform instance. +# Replace this variable with your own Platform endpoint. +# +TOWER_ENDPOINT_URL=http://localhost:8000/api + +# +# Email address that will be used by the Wave service to send notification emails. +# Replace this variable with your own mail address. +# +MAIL_FROM=wave-app+changeme@seqera.io diff --git a/new-docs/wave-docs/install/configure-wave-build.md b/new-docs/wave-docs/install/configure-wave-build.md new file mode 100644 index 0000000000..1d61cf7083 --- /dev/null +++ b/new-docs/wave-docs/install/configure-wave-build.md @@ -0,0 +1,407 @@ +--- +title: Configure Wave build +--- + +This guide covers extending your existing Wave installation on Kubernetes to support container build capabilities. This enables Wave's full feature set including container building, freezing, and advanced caching. + +## Prerequisites + +Before extending Wave for build support, ensure you have: + +- **Existing Wave installation** - Basic Wave deployment already running in augmentation-only mode +- **AWS EKS cluster** - Build capabilities require AWS-specific integrations +- **EFS filesystem** - Configured and accessible from your EKS cluster for shared build storage +- **Cluster admin permissions** - Required to create RBAC policies and storage resources + +## Create Kubernetes Service Account & RBAC Policies + +Wave's build service needs permissions to create and manage build pods. Create the necessary RBAC configuration: + +```yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: wave-sa + namespace: wave +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: wave-role +rules: + - apiGroups: [""] + resources: [pods, pods/status, pods/log, pods/exec] + verbs: [get, list, watch, create, delete] + - apiGroups: ["batch"] + resources: [jobs, jobs/status] + verbs: [get, list, watch, create, delete] + - apiGroups: [""] + resources: [configmaps, secrets] + verbs: [get, list] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: wave-rolebind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: wave-role +subjects: + - kind: ServiceAccount + name: wave-sa + namespace: wave +``` + +## Configure EFS Storage + +Wave builds require shared storage accessible across multiple pods. Configure EFS with the AWS EFS CSI driver: + +### Storage Class + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: efs-wave-sc +provisioner: efs.csi.aws.com +parameters: + provisioningMode: efs-ap + fileSystemId: "REPLACE_ME_EFS_ID" + directoryPerms: "0755" +``` + +### Persistent Volume + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: wave-build-pv +spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: efs-wave-sc + csi: + driver: efs.csi.aws.com + volumeHandle: "REPLACE_ME_EFS_ID" +``` + +### Persistent Volume Claim + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + namespace: wave + name: wave-build-pvc + labels: + app: wave-app +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Gi + storageClassName: efs-wave-sc +``` + +**Configuration Notes:** +- Replace `REPLACE_ME_EFS_ID` with your actual EFS filesystem ID +- EFS must be in the same VPC as your EKS cluster +- Ensure EFS security groups allow NFS traffic from EKS worker nodes + +## Update Wave Configuration + +Update your existing Wave ConfigMap to enable build features and configure storage paths: + +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: wave-cfg + namespace: wave + labels: + app: wave-cfg +data: + config.yml: | + wave: + # Enable build service + build: + enabled: true + workspace: '/build/workspace' + # Optional: Retain failed builds to gather logs & inspect + cleanup: "OnSuccess" + # Optional: Configure build timeouts + timeout: '15m' + # Example additional kubernetes configuration for wave-build + k8s: + dns: + servers: + - "1.1.1.1" + - "8.8.8.8" + namespace: "wave-build" + storage: + mountPath: "/build" + # Relevant volume claim name should match the + claimName: "wave-build-pvc" + serviceAccount: "wave-build-sa" + resources: + requests: + memory: '1Gi' + cpu: '500m' + limits: + memory: '4Gi' + cpu: '2000m' + nodeSelector: + # this node selector binds the build pods to a separate cluster node group + linux/amd64: 'service=wave-build' + linux/arm64: 'service=wave-build-arm64' + # Enable other build-dependent features + mirror: + enabled: true + scan: + enabled: true + blobCache: + enabled: true + + # Existing database, redis, and platform configuration... + db: + uri: "jdbc:postgresql://your-postgres-host:5432/wave" + user: "wave_user" + password: "your_secure_password_here" + + redis: + uri: "redis://your-redis-host:6379" + + tower: + endpoint: + url: "https://your-platform-instance.com/api" + +``` + +## Update Wave Deployment + +Modify your existing Wave deployment to include the service account and EFS storage: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: wave + namespace: wave + labels: + app: wave-app +spec: + replicas: 1 + selector: + matchLabels: + app: wave-app + template: + metadata: + labels: + app: wave-app + spec: + serviceAccountName: wave-sa # Add service account + containers: + - image: your-registry.com/wave:latest + name: wave-app + ports: + - containerPort: 9090 + name: http + env: + - name: MICRONAUT_ENVIRONMENTS + value: "postgres,redis,k8s" # Add k8s environment + - name: WAVE_JVM_OPTS + value: "-Xmx3g -Xms1g -XX:+UseG1GC" + resources: + requests: + memory: "4Gi" + cpu: "1000m" + limits: + memory: "4Gi" + cpu: "2000m" + workingDir: "/work" + volumeMounts: + - name: wave-cfg + mountPath: /work/config.yml + subPath: "config.yml" + - name: build-storage + mountPath: /build + readinessProbe: + httpGet: + path: /health + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 9090 + initialDelaySeconds: 60 + timeoutSeconds: 10 + volumes: + - name: wave-cfg + configMap: + name: wave-cfg + - name: build-storage # Add EFS volume + persistentVolumeClaim: + claimName: wave-build-pvc + restartPolicy: Always +``` + +## Deploy the Updates + +Apply the configuration changes to enable build support: + +```bash +# Apply RBAC configuration +kubectl apply -f wave-rbac.yaml + +# Apply storage configuration +kubectl apply -f wave-storage.yaml + +# Update the ConfigMap +kubectl apply -f wave-configmap.yaml + +# Update the deployment +kubectl apply -f wave-deployment.yaml + +# Verify the deployment +kubectl get pods -n wave +kubectl logs -f deployment/wave -n wave + +# Check that EFS is mounted correctly +kubectl exec -it deployment/wave -n wave -- df -h /build +``` + +## Verify Build Functionality + +Test that Wave build capabilities are working: + +1. **Check Wave health endpoint** for build service status +2. **Monitor logs** for build service initialization messages +3. **Test a simple build** through the Wave API or Platform integration + +```bash +curl http://wave-service.wave.svc.cluster.local:9090/health + +kubectl logs -f deployment/wave -n wave | grep -i build +``` + +## Recommended Production Enhancements + +### Dedicated Node Pools + +Create dedicated node pools for Wave build workloads to isolate build processes and optimize resource allocation: + + +### Build Pod Resource Management + +Configure resource quotas and limits for build pods: + +```yaml +apiVersion: v1 +kind: ResourceQuota +metadata: + name: wave-build-quota + namespace: wave +spec: + hard: + requests.cpu: "10" + requests.memory: 20Gi + limits.cpu: "20" + limits.memory: 40Gi + pods: "10" +``` + +### Monitoring and Alerting + +Set up monitoring for build operations: + +- **Build success/failure rates** +- **Build duration metrics** +- **EFS storage usage** +- **Node resource utilization** +- **Build queue length** + +## Security Considerations + +- **EFS Access Points** - Use EFS access points to isolate build workspaces +- **Network Policies** - Restrict network access for build pods +- **Pod Security Standards** - Apply appropriate security contexts to build pods +- **Image Scanning** - Enable security scanning for built images +- **RBAC Minimization** - Regularly review and minimize Wave's cluster permissions + +## Troubleshooting + +**Common issues and solutions:** + +- **EFS mount failures** - Check security groups and VPC configuration +- **Build pod creation failures** - Verify RBAC permissions and node selectors +- **Storage access issues** - Ensure EFS access points are configured correctly +- **Build timeouts** - Adjust build timeout settings based on workload requirements + +For additional configuration options and advanced features, see [Configuring Wave](../configure-wave.md). + +## Bottlerocket support + +Buildkit requires user namespaces. However, Bottlerocket sets `user.max_user_namespaces=0` by default for security. + +To use Buildkit with Bottlerocket, enable user namespaces for container builds by setting `user.max_user_namespaces=N` on your host nodes, where `N` is a positive integer. Use a sufficiently high value (e.g., `62000`) to avoid build failures. Values that are too low (e.g., `10`) will limit concurrent build capacity and may cause build failures. + +You can configure this setting in two ways: + +#### Recommended: Node startup configuration + +Configure the user namespace setting in your node group's startup script or user data. This approach applies the configuration at boot time and doesn't require privileged containers in your cluster. + +#### Alternative: DaemonSet + +If you can't control the node configuration directly, use DaemonSet. This approach requires running a privileged container. We recommend you deploy it only on wave-build nodes and use a dedicated namespace for isolation. + +**Example manifest:** + +```yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: sysctl-userns + name: sysctl-userns +spec: + selector: + matchLabels: + app: sysctl-userns + template: + metadata: + labels: + app: sysctl-userns + spec: + containers: + - name: sysctl-userns + image: busybox + command: ["sh", "-euxc", "sysctl -w user.max_user_namespaces=63359 && sleep infinity"] + securityContext: + privileged: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: service + operator: In + values: ["wave-build", "wave-build-arm64"] +``` + +#### Additional resources + +For more information about Bottlerocket, see: + +- [Bottlerocket FAQs](https://bottlerocket.dev/en/faq/) +- [Amazon Bottlerocket FAQs](https://aws.amazon.com/bottlerocket/faqs/) diff --git a/new-docs/wave-docs/install/docker-compose.md b/new-docs/wave-docs/install/docker-compose.md new file mode 100644 index 0000000000..bf1c64f775 --- /dev/null +++ b/new-docs/wave-docs/install/docker-compose.md @@ -0,0 +1,220 @@ +--- +title: Docker Compose installation +--- + +Wave enables you to provision container images on demand, removing the need to build and upload them manually to a container registry. Wave can provision both ephemeral and regular registry-persisted container images. + +Docker Compose installations support Wave Lite, a configuration mode for Wave that includes only container augmentation and inspection capabilities, and enables the use of Fusion file system in Nextflow pipelines. + +## Prerequisites + +Before installing Wave, you need the following infrastructure components: + +- **PostgreSQL instance** - Version 12, or higher +- **Redis instance** - Version 6.2, or higher + +## System requirements + +The minimum system requirements for self-hosted Wave in Docker Compose are: + +- Current, supported versions of **Docker Engine** and **Docker Compose**. +- Compute instance minimum requirements: + - **Memory**: 32 GB RAM available to be used by the Wave application on the host system. + - **CPU**: 8 CPU cores available on the host system. + - **Storage**: 10 GB in addition to sufficient disk space for your container images and temporary files. + - For example, in AWS EC2, `m5a.2xlarge` or greater + - **Network**: Connectivity to your PostgreSQL and Redis instances. + +## Database configuration + +Wave requires a PostgreSQL database to operate. + +Create a dedicated `wave` database and user account with the appropriate privileges: + +```sql +-- Create a dedicated user for Wave +CREATE ROLE wave_user LOGIN PASSWORD 'your_secure_password'; + +-- Create the Wave database +CREATE DATABASE wave; + +-- Connect to the wave database +\c wave; + +-- Grant basic schema access +GRANT USAGE, CREATE ON SCHEMA public TO wave_user; + +-- Grant privileges on existing tables and sequences +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO wave_user; +GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO wave_user; + +-- Grant privileges on future tables and sequences +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO wave_user; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO wave_user; +``` + +Wave will automatically handle schema migrations on startup and create the required database objects. + +## Wave config + +Create a configuration file that defines Wave's behavior and integrations. Save this as `config/wave-config.yml` in your Docker Compose directory. + +```yaml +wave: + # Build service configuration - disabled for Docker Compose + build: + enabled: false + # Mirror service configuration - disabled for Docker Compose + mirror: + enabled: false + # Security scanning configuration - disabled for Docker Compose + scan: + enabled: false + # Blob caching configuration - disabled for Docker Compose + blobCache: + enabled: false + # Database connection settings + db: + uri: "jdbc:postgresql://your-postgres-host:5432/wave" + user: "wave_user" + password: "your_secure_password" + +# Redis configuration for caching and session management +redis: + uri: "redis://your-redis-host:6379" + +# Platform integration (optional) +tower: + endpoint: + url: "https://your-platform-server.com" + +# Micronaut framework configuration +micronaut: + # Netty HTTP server configuration + netty: + event-loops: + default: + num-threads: 64 + stream-pool: + executor: stream-executor + # HTTP client configuration + http: + services: + stream-client: + read-timeout: 30s + read-idle-timeout: 5m + event-loop-group: stream-pool + +# Management endpoints configuration +loggers: + env: + enabled: false + bean: + enabled: false + caches: + enabled: false + refresh: + enabled: false + loggers: + enabled: false + info: + enabled: false + # Enable metrics for monitoring + metrics: + enabled: true + # Enable health checks + health: + enabled: true + disk-space: + enabled: false + jdbc: + enabled: false +``` + +Configuration notes: + +- Replace `your-postgres-host` and `your-redis-host` with your service endpoints. +- Adjust `number-of-threads` (16) and `num-threads` (64) based on your CPU cores — Use between 2x and 4x your CPU core count. + +## Docker Compose + +Add the following to your `docker-compose.yml`: + +```yaml +services: + wave-app: + image: your-registry.com/wave:latest + container_name: wave-app + ports: + # Bind to the host on 9100 vs 9090 + - "9100:9090" + environment: + - MICRONAUT_ENVIRONMENTS=lite,redis,postgres + volumes: + - ./config/wave-config.yml:/work/config.yml:ro + deploy: + mode: replicated + replicas: 2 + resources: + limits: + memory: 4G + cpus: '1.0' + reservations: + memory: 4G + cpus: '1' + # Health check configuration + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9090/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + # Restart policy + restart: unless-stopped +``` + +## Deploy Wave + +1. Download and populate the [wave.env](./_templates/wave.env) file with the settings corresponding to your system. + +1. Use Docker Swarm to deploy Wave Lite. See [Create a swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/) for detailed setup instructions. + + +1. Deploy the Wave service, running two replicas: + + ```bash + docker stack deploy -c docker-compose.yml mystack + ``` + + :::note + Wave is available at `http://localhost:9090` once the container is running and healthy. The application may take 30-60 seconds to fully initialize on first startup, as it performs database migrations. + ::: + +1. Check the current status: + + ```bash + docker service ls + ``` + +1. Check the logs: + + ```bash + docker service logs mystack_wave + ``` + +1. Tear down the service when it's no longer needed: + + ```bash + docker stack rm mystack + ``` + + :::warning + If Wave Lite is running in the same container as Platform Connect for [Studios](https://docs.seqera.io/platform-enterprise/25.2/enterprise/studios#docker-compose), tearing down the service will also interrupt Connect services. + ::: + +### Advanced configuration + +See [Configure Wave](../configure-wave.md) for advanced Wave features, scaling guidance, and integration options. diff --git a/new-docs/wave-docs/install/kubernetes.md b/new-docs/wave-docs/install/kubernetes.md new file mode 100644 index 0000000000..f3671ed75f --- /dev/null +++ b/new-docs/wave-docs/install/kubernetes.md @@ -0,0 +1,354 @@ +--- +title: Kubernetes installation +--- + +Wave enables you to provision container images on-demand, removing the need to build and upload them manually to a container registry. Wave can can provision both disposable containers that are only accessible for a short period, and regular registry-persisted container images. + +This installation guide covers Wave in [Lite](../wave-lite.md) mode. Wave Lite provides container augmentation and inspection capabilities on AWS, Azure, and GCP cloud deployments, and enables the use of Fusion file system in Nextflow pipelines. + +:::info +Wave's full build capabilities require specific integrations with Kubernetes and AWS EFS Storage, making EKS and AWS a hard dependency for fully-featured deployments. After you have configured a base Wave Lite installation on AWS with this guide, see [Configure Wave Build](./configure-wave-build.md) to extend your installation to support build capabilities. +::: + +## Prerequisites + +**Required infrastructure:** +- **Kubernetes cluster** - Version 1.31 or higher (any distribution) +- **PostgreSQL instance** - Version 12 or higher (managed externally) +- **Redis instance** - Version 6.0 or higher (managed externally) + +## System requirements + +The minimum system requirements for a Wave Kubernetes installation are: + +- **Memory**: Minimum 4GB RAM per Wave pod +- **CPU**: Minimum 1 CPU core per pod +- **Network**: Connectivity to your external PostgreSQL and Redis instances +- **Storage**: Sufficient storage for your container images and temporary files + +:::info +See [Configure Wave](../configure-wave.md) for detailed scaling and performance tuning guidance. +::: + +## Assumptions + +This guide assumes: +- You have already deployed Seqera Platform Enterprise +- You will deploy Wave into the `wave` namespace +- You have appropriate cluster permissions to create namespaces, deployments, and services +- Your PostgreSQL and Redis instances are accessible from the Kubernetes cluster + +## Database configuration + +Wave requires a PostgreSQL database to operate. + +Create a dedicated `wave` database and user account with the appropriate privileges: + +```sql +-- Create a dedicated user for Wave +CREATE ROLE wave_user LOGIN PASSWORD 'your_secure_password'; + +-- Create the Wave database +CREATE DATABASE wave; + +-- Connect to the wave database +\c wave; + +-- Grant basic schema access +GRANT USAGE, CREATE ON SCHEMA public TO wave_user; + +-- Grant privileges on existing tables and sequences +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO wave_user; +GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO wave_user; + +-- Grant privileges on future tables and sequences +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO wave_user; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO wave_user; +``` + +## Create namespace + +```yaml +--- +apiVersion: v1 +kind: Namespace +metadata: + name: "wave" + labels: + app: wave-app +``` + +## Configure Wave + +Create a ConfigMap containing Wave's configuration. Update the following values to match your environment: + +- Database connection details (`uri`, `user`, `password`) +- Redis connection string +- Seqera Platform API endpoint + +:::warning +This configuration contains sensitive values. It is recommended to use Kubernetes Secrets for sensitive data, instead of embedding them directly in the ConfigMap. See the [Kubernetes Secrets documentation](https://kubernetes.io/docs/concepts/configuration/secret/) for more details. + +Example using environment variables with secrets: + +```yaml +env: + - name: WAVE_DB_PASSWORD + valueFrom: + secretKeyRef: + name: wave-secrets + key: db-password +``` +::: + +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: wave-cfg + namespace: "wave" + labels: + app: wave-cfg +data: + config.yml: | + wave: + # Build service configuration - disabled for Wave base installation + build: + enabled: false + # Mirror service configuration - disabled for Wave base installation + mirror: + enabled: false + # Security scanning configuration - disabled for Wave base installation + scan: + enabled: false + # Blob caching configuration - disabled for Wave base installation + blobCache: + enabled: false + # Database connection settings + db: + uri: "jdbc:postgresql://your-postgres-host:5432/wave" + user: "wave_user" + password: "your_secure_password" + + # Redis configuration for caching and session management + redis: + uri: "rediss://your-redis-host:6379" + + # Platform integration (optional) + tower: + endpoint: + url: "https://your-platform-server.com" + + # Micronaut framework configuration + micronaut: + # Executor configuration for handling concurrent requests + executors: + stream-executor: + type: FIXED + number-of-threads: 16 + # Netty HTTP server configuration + netty: + event-loops: + default: + num-threads: 64 + stream-pool: + executor: stream-executor + # HTTP client configuration + http: + services: + stream-client: + read-timeout: 30s + read-idle-timeout: 5m + event-loop-group: stream-pool + + # Management endpoints configuration + loggers: + # Enable metrics for monitoring + metrics: + enabled: true + # Enable health checks + health: + enabled: true + disk-space: + enabled: false + jdbc: + enabled: false +``` + +## Create deployment + +Deploy Wave using the following Deployment manifest: + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: wave + namespace: "wave" + labels: + app: wave-app +spec: + replicas: 1 + selector: + matchLabels: + app: wave-app + template: + metadata: + labels: + app: wave-app + spec: + containers: + - image: REPLACE_ME_AWS_ACCOUNT.dkr.ecr.us-east-1.amazonaws.com/nf-tower-enterprise/wave:REPLACE_ME_WAVE_IMAGE_TAG + name: wave-app + ports: + - containerPort: 9090 + env: + - name: MICRONAUT_ENVIRONMENTS + value: "postgres,redis,lite" + resources: + requests: + memory: "4000Mi" + limits: + memory: "4000Mi" + workingDir: "/work" + volumeMounts: + - name: wave-cfg + mountPath: /work/config.yml + subPath: "config.yml" + readinessProbe: + httpGet: + path: /health + port: 9090 + initialDelaySeconds: 5 + timeoutSeconds: 3 + livenessProbe: + httpGet: + path: /health + port: 9090 + initialDelaySeconds: 5 + timeoutSeconds: 3 + failureThreshold: 10 + volumes: + - name: wave-cfg + configMap: + name: wave-cfg + restartPolicy: Always +``` + +## Create Service + +Expose Wave within the cluster using a Service: + +```yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: wave-service + namespace: "wave" + labels: + app: wave-app +spec: + selector: + app: wave-app + ports: + - name: http + port: 9090 + targetPort: 9090 + protocol: TCP + type: ClusterIP +``` + + +## Next steps + +### Configure Seqera Platform to integrate with Wave + +Configure your Seqera Platform Enterprise deployment to integrate with Wave by setting the Wave server endpoint in your `tower.yml` [configuration](https://docs.seqera.io/platform-enterprise/latest/enterprise/configuration/wave). + +### Networking + +Wave must be accessible from: + +- Seqera Platform services +- Compute environments (for container image access) + +Configure external access using a Kubernetes ingress. + +Update the following example ingress with your provider-specific annotations: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: wave-ingress + namespace: wave +spec: + rules: + - host: wave.your-domain.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: wave-service + port: + number: 9090 +``` + +### TLS + +Wave does not handle TLS termination directly. Configure TLS at your ingress controller or load balancer level. Most ingress controllers support automatic certificate provisioning through provider integrations. + + +### Production environments + +Consider implementing the following for production deployments: + +**Reliability:** +- Pod Disruption Budgets, for availability during cluster maintenance +- Horizontal Pod Autoscaler, for automatic scaling based on load +- Multiple replicas with anti-affinity rules, for high availability + +**Resource management:** +- Node selectors or affinity rules for optimal pod placement +- Resource quotas and limit ranges for the Wave namespace + +### AWS credentials to access ECR +Wave requires access to AWS ECR for container image management. Create an IAM role with the following permissions: + +```json +"Statement": [ + { + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages", + "ecr:BatchGetImage", + "ecr:GetLifecyclePolicy", + "ecr:GetLifecyclePolicyPreview", + "ecr:ListTagsForResource", + "ecr:DescribeImageScanFindings", + "ecr:CompleteLayerUpload", + "ecr:UploadLayerPart", + "ecr:InitiateLayerUpload", + "ecr:PutImage" + ], + "Effect": "Allow", + "Resource": [ + "/wave/*" + ] + } + ``` + +### Advanced configuration + +See [Configure Wave](../configure-wave.md) for advanced Wave features, scaling guidance, and integration options. diff --git a/new-docs/wave-docs/migrations/1-21-0.md b/new-docs/wave-docs/migrations/1-21-0.md new file mode 100644 index 0000000000..a06a16452b --- /dev/null +++ b/new-docs/wave-docs/migrations/1-21-0.md @@ -0,0 +1,18 @@ +--- +title: Migrating to 1.21.0 +tags: [migration notes, wave] +--- + +Wave 1.21.0 was released on May 29, 2025. + +## Mandatory steps + +Wave 1.21.0 introduces support for PostgreSQL as the primary database backend, replacing SurrealDB. + +To upgrade your existing data from SurrealDB to PostgreSQL: + +1. Follow the steps in the [Wave database migration](../db-migration.md) guide. +2. Add the following properties to your Wave configuration file: + + - `wave.build.logs.path`: Sets the path inside `wave.build.logs.bucket`, where build logs will be stored. + - `wave.build.locks.path`: Sets the path inside `wave.build.logs.bucket`, where conda lock files will be stored. diff --git a/new-docs/wave-docs/migrations/1-24-0.md b/new-docs/wave-docs/migrations/1-24-0.md new file mode 100644 index 0000000000..acc83fb017 --- /dev/null +++ b/new-docs/wave-docs/migrations/1-24-0.md @@ -0,0 +1,14 @@ +--- +title: Migrating to 1.24.0 +tags: [migration notes, wave] +--- + +Wave 1.24.0 was released on August 12, 2025. + +## Mandatory steps + +Wave 1.24.0 adds support for SBOM (Software Bill of Materials) generation using Syft. + +To use SBOM generation, add the following property to your Wave configuration: + +- `wave.scan.reports.path`: Sets the path inside the S3 bucket where Wave will store SBOM reports. For example, `s3://wave-store/scan-reports`. diff --git a/new-docs/wave-docs/migrations/1-25-0.md b/new-docs/wave-docs/migrations/1-25-0.md new file mode 100644 index 0000000000..7d6c250ea6 --- /dev/null +++ b/new-docs/wave-docs/migrations/1-25-0.md @@ -0,0 +1,20 @@ +--- +title: Migrating to 1.25.0 +tags: [migration notes, wave] +--- + +Wave 1.25.0 was released on September 2, 2025. + +## Mandatory steps + +Wave 1.25.0 upgrades Micronaut to 4.9.2 and Netty to 4.2.0. This changes the default Netty ByteBuf allocator from `PooledByteBufAllocator` to `AdaptiveRecvByteBufAllocator`, which may impact your memory usage patterns. + +The Netty ByteBuf allocator must be explicitly set back to `PooledByteBufAllocator` to maintain stable memory usage patterns. + +If you are passing custom `WAVE_JVM_OPTS`, add the following option to your Wave configuration: + +``` +-Dio.netty.allocator.type=pooled +``` + +See [Micronaut 4.9.2 Netty Memory Issue Analysis](https://github.com/seqeralabs/wave/blob/master/adr/mv-4.9-netty-memory.md) for more information. diff --git a/new-docs/wave-docs/migrations/index.md b/new-docs/wave-docs/migrations/index.md new file mode 100644 index 0000000000..2d9316abea --- /dev/null +++ b/new-docs/wave-docs/migrations/index.md @@ -0,0 +1,10 @@ +--- +title: Migration notes +tags: [migration notes, wave] +--- + +This section covers mandatory steps for migrating to new versions of Wave. + +- [Migrating to 1.25.0](./1-25-0.md) +- [Migrating to 1.24.0](./1-24-0.md) +- [Migrating to 1.21.0](./1-21-0.md) diff --git a/new-docs/wave-docs/nextflow/configuration.md b/new-docs/wave-docs/nextflow/configuration.md new file mode 100644 index 0000000000..4bd9d3fc8e --- /dev/null +++ b/new-docs/wave-docs/nextflow/configuration.md @@ -0,0 +1,135 @@ +--- +title: Configuration reference +description: Wave configuration variables for customizing behavior in Nextflow pipelines +tags: [nextflow, wave, use configuration] +--- + +This reference lists the Wave configuration variables that can be used to customize Wave behavior in your Nextflow pipelines. Configure these settings in your Nextflow configuration file. + +:::tip +For a full list of Nextflow configuration options, see [Configuration options](https://www.nextflow.io/docs/latest/reference/config.html). +::: + +:::note +Nextflow integration with Wave requires Nextflow 22.10.0 or later. +::: + +## General + +Use the following options to configure general Wave settings: + +`wave.enabled` +: Enables the execution of Wave containers (default: `false`). + +`wave.endpoint` +: Specifies the Wave service endpoint (default: `https://wave.seqera.io`). + +`wave.freeze` +: _Requires Nextflow version 23.07.0-edge or later._ +: Enables freeze mode to permanently store provisioned Wave containers (default: `false`). + The target registry must be specified by `wave.build.repository`. + Specifying a custom cache repository using `wave.build.cacheRepository` is also recommended. + +`wave.mirror` +: _Requires Nextflow version 24.09.1-edge or later._ +: Enables Wave container mirroring (default: `false`). + The target registry must be specified by `wave.build.repository`. + This option is only compatible with `wave.strategy = 'container'` and cannot be used with `wave.freeze`. + +`wave.strategy` +: Sets the strategy used when resolving ambiguous Wave container requirements (default: `'container,dockerfile,conda'`). + +## Build + +Use the following options to configure Wave build settings: + +`wave.build.cacheRepository` +: Specifies the container repository URL for caching image layers built by the Wave service. + Requires corresponding credentials to be configured in your Seqera account. + +`wave.build.compression.force` +: _Requires Nextflow version 25.05.0-edge or later._ +: Enables forceful compression for all layers, including existing layers (default: `false`). + +`wave.build.compression.level` +: _Requires Nextflow version 25.05.0-edge or later._ +: Sets the compression level used when building containers. Valid ranges depend on compression type: `0`-`9` for `gzip` and `estargz`, and `0`-`22` for `zstd`. Higher values provide better compression but slower build times. + +`wave.build.compression.mode` +: _Requires Nextflow version 25.05.0-edge or later._ +: Sets the compression algorithm for container builds. Options include: `gzip`, `estargz`, or `zstd` (default: `gzip`). + +`wave.build.conda.basePackages` +: Sets Conda packages to always include in the container (default: `conda-forge::procps-ng`). + +`wave.build.conda.commands` +: Sets commands to add to the Dockerfile for Conda-based images. + +`wave.build.conda.mambaImage` +: Specifies the Mamba container image used for building Conda-based containers. + Must be a [micromamba-docker](https://github.com/mamba-org/micromamba-docker) compatible image. + +`wave.build.repository` +: Specifies the container repository URL where Wave uploads built images. + Requires corresponding credentials to be configured in your Seqera account. + +## Conda + +Use the following options to configure Wave Conda settings: + +`wave.conda.mambaImage` +: Specifies the Mamba container image used for building Conda-based containers. + Must be a [micromamba-docker](https://github.com/mamba-org/micromamba-docker) compatible image. + +`wave.conda.commands` +: Specifies additional Dockerfile commands to include when building Conda-based container images. + Commands are inserted into the generated Dockerfile during the build process. + +## HTTP client + +Use the following options to configure Wave HTTP client settings: + +`wave.httpClient.connectTimeout` +: _Requires Nextflow version 22.06.0-edge or later._ +: Sets the connection timeout for the Wave HTTP client (default: `30s`). + +`wave.httpClient.maxRate` +: _Requires Nextflow version 22.06.0-edge or later._ +: Sets the maximum request rate for the Wave HTTP client (default: `1/sec`). + +## Retry policy + +Use the following options to configure Wave retry policy settings: + +`wave.retryPolicy.delay` +: _Requires Nextflow version 22.06.0-edge or later._ +: Sets the delay when retrying failing HTTP requests (default: `450ms`). + +`wave.retryPolicy.jitter` +: _Requires Nextflow version 22.06.0-edge or later._ +: Sets the jitter factor for randomly varying retry delays (default: `0.25`). + +`wave.retryPolicy.maxAttempts` +: _Requires Nextflow version 22.06.0-edge or later._ +: Sets the maximum retry attempts for failing HTTP requests (default: `5`). + +`wave.retryPolicy.maxDelay` +: _Requires Nextflow version 22.06.0-edge or later._ +: Sets the maximum delay when retrying failing HTTP requests (default: `90s`). + +## Scan + +Use the following options to configure Wave scan settings: + +`wave.scan.allowedLevels` +: _Requires Nextflow version 24.09.1-edge or later._ +: Specifies a comma-separated list of allowed vulnerability levels for container scanning. Requires `wave.scan.mode = 'required'` to be set. + Options include: `low`, `medium`, `high`, and `critical`. + +`wave.scan.mode` +: _Requires Nextflow version 24.09.1-edge or later._ +: Sets the Wave container security scanning mode. +: Options include: +: - `none`: No container security scanning. +: - `async`: Containers are scanned for security vulnerabilities. The task is executed regardless of the scan result. +: - `required`: Containers are scanned for security vulnerabilities. The task is executed only if the container is free of vulnerabilities. diff --git a/new-docs/wave-docs/nextflow/index.md b/new-docs/wave-docs/nextflow/index.md new file mode 100644 index 0000000000..4fd0fd5665 --- /dev/null +++ b/new-docs/wave-docs/nextflow/index.md @@ -0,0 +1,69 @@ +--- +title: Overview +description: Learn about using Wave with Nextflow +tags: [nextflow, wave, get started] +--- + +Wave's container provisioning service seamlessly integrates with Nextflow pipelines to streamline container management and deployment. When used with Nextflow, Wave automatically handles container building, provisioning, and optimization, eliminating the complexity of manual container management in your workflow execution. + +:::note +Nextflow integration with Wave requires Nextflow 22.10.0 or later. +::: + +## Get started + +To enable Wave in your Nextflow pipeline, add the following to your Nextflow configuration file: + +```groovy +wave.enabled = true +tower.accessToken = '' +``` + +Replace `` with your Seqera access token. + +:::note + +Using a Seqera access token is optional but provides additional capabilities: + +- Access to private repositories +- Higher API request limits than anonymous users + +::: + +## Use cases + +Wave supports several key use cases that enhance your Nextflow pipeline capabilities: + +### Private repository access + +Securely access private container repositories using credentials managed through the Seqera Platform. Wave handles authentication automatically, allowing your pipelines to pull from private registries without exposing sensitive credentials. See [Access private container repositories](./use-cases.md#access-private-container-repositories) for more information. + +### Container building and provisioning + +Wave can automatically build container images from Dockerfiles placed in your module directories or provision containers based on Conda package specifications. This eliminates the need to pre-build and manage container images manually. See [Build Nextflow module containers](./use-cases.md#build-nextflow-module-containers) for more information. + +### Conda package integration + +Use Conda packages directly in cloud-native environments that don't natively support Conda. Wave builds containers with your specified Conda dependencies, enabling portable execution across different compute platforms. See [Build Conda-based containers](./use-cases.md#build-conda-based-containers) for more information. + +### Singularity container support + +Build and provision Singularity native images for HPC environments while maintaining compatibility with container registries through ORAS artifacts. See [Build Singularity containers](./use-cases.md#build-singularity-containers) for more information. + +### Container mirroring + +Mirror containers from public registries to your own private registry for improved performance, compliance, and availability. This ensures your pipelines can access required containers even when upstream registries are unavailable. See [Mirror containers across registries](./use-cases.md#mirror-containers-across-registries) for more information. + +### Security scanning + +Scan your Nextflow pipeline containers automatically with Wave to identify security vulnerabilities before they become a problem. This proactive security check gives you confidence that your workflows run on secure images. It helps you address potential risks early and maintain robust, trustworthy pipelines. See [Security scan containers](./use-cases.md#security-scan-containers) for more information. + +### Fusion file system integration + +Combine Wave containers with Fusion to use cloud object storage as your pipeline work directory, simplifying data management and improving performance on cloud platforms. See [Use Wave with Fusion](./use-cases.md#use-wave-with-fusion) for more information. + +## Configuration + +Wave provides extensive configuration options to customize container provisioning behavior, scanning behavior, caching strategies, and repository settings. + +For detailed configuration options and advanced settings, see [Configuration reference](./configuration.md). diff --git a/new-docs/wave-docs/nextflow/use-cases.md b/new-docs/wave-docs/nextflow/use-cases.md new file mode 100644 index 0000000000..00beef4bdb --- /dev/null +++ b/new-docs/wave-docs/nextflow/use-cases.md @@ -0,0 +1,277 @@ +--- +title: Use cases +description: Learn how to use Wave with Nextflow for container management, building, and security scanning +date: "2024-08-22" +tags: [nextflow, wave, use cases] +--- + +With Nextflow and Wave, you can build, upload, and manage the container images automatically and on demand during pipeline execution. +The following sections describe several common use cases. + +:::tip +To get started with an example Nextflow pipeline that uses Wave, see [Nextflow and Wave](../tutorials/nextflow-wave.mdx). +::: + +:::note +Nextflow integration with Wave requires Nextflow 22.10.0 or later. +::: + +## Access private container repositories + +Use Wave to access private repositories for your Nextflow pipelines. + +
+**Access private container repositories** + +To enable private container repository access: + +1. Add the following to your Nextflow configuration: + + ```groovy + wave.enabled = true + tower.accessToken = '' + ``` + + Replace `` with your [Seqera access token](../tutorials/nextflow-wave.mdx#create-your-seqera-access-token). + +1. (Optional) If you created your credentials in an organization workspace, add your workspace ID to your Nextflow configuration: + + ```groovy + tower.workspaceId = '' + ``` + + Replace `` with your Seqera workspace ID. + +1. Configure your repository access in Seqera. See [Seqera Platform credentials](https://docs.seqera.io/platform/latest/credentials/overview) for more information. + +1. Add your build and build cache repository to your Nextflow configuration: + + ```groovy + wave.build.repository = '' + wave.build.cacheRepository = '' + ``` + + Replace the following: + + - ``: the repository to store your built container images + - ``: the repository to store image layers for caching + +
+ +## Build Nextflow module containers + +Wave can build and provision container images on demand for your Nextflow pipelines. + +
+**Build Nextflow module containers** + +To enable Wave to build Nextflow module containers: + +1. Add your Dockerfile to the [module directory](https://www.nextflow.io/docs/latest/module.html#module-directory) where you define the pipeline process. + +1. Enable Wave in your Nextflow configuration: + + ```groovy + wave.enabled = true + ``` + +1. (Optional) Set your Wave strategy to prioritize Dockerfiles in your Nextflow configuration: + + ```groovy + wave.strategy = ['dockerfile','container'] + ``` + + :::note + The `container` directive takes precedence over a Dockerfile by default. + ::: + +:::warning +Wave does not support `ADD`, `COPY`, or other Dockerfile commands that access files in the host file system. +::: + +
+ +## Build Conda-based containers + +Wave can provision containers based on the [`conda` directive](https://www.nextflow.io/docs/latest/process.html#conda). +This allows you to use Conda packages in your pipeline, even on cloud-native platforms like AWS Batch and Kubernetes, which do not support the Conda package manager directly. + +
+**Build Conda-based containers** + +To enable Wave to provision Conda package containers: + +1. Define your Conda packages using the `conda` directive in your pipeline processes. + +1. Enable Wave in your Nextflow configuration: + + ```groovy + wave.enabled = true + ``` + +1. (Optional) Set your Wave strategy to prioritize `conda` in your Nextflow configuration: + + ```groovy + wave.strategy = ['conda'] + ``` + + :::note + The `container` directive or a Dockerfile takes precedence over the `conda` directive by default. + ::: + + :::info + Nextflow 23.10.0 or later automatically includes the `conda-forge::procps-ng` package in provisioned containers. This package includes the `ps` command. + ::: + +1. Set your Conda channel priority: + + ```groovy + conda.channels = '' + ``` + + Replace `` with a comma-separated list of your channel priorities. + +
+ +## Build Singularity containers + +Nextflow can build Singularity native images on demand using a `Singularityfile` or Conda packages. +Images are uploaded to an OCI-compliant container registry of your choice and stored as an [ORAS artifact](https://oras.land/). + +:::note +Requires Nextflow version 23.09.0-edge or later and a version of Singularity (or Apptainer) that supports pulling images using the `oras:` pseudo-protocol. +::: + +
+**Build Singularity containers** + +To enable provisioning of Singularity images: + +1. Add the following to your Nextflow configuration: + + ```groovy + wave.enabled = true + wave.freeze = true + wave.strategy = ['conda'] + singularity.enabled = true + ``` + + +1. (Optional) To store your Singularity image files in a private registry: + + 1. Configure your repository access in Seqera. See [Seqera Platform credentials](https://docs.seqera.io/platform-cloud/credentials/overview) for more information. + + 1. Add your build repository to your Nextflow configuration: + + ```groovy + wave.build.repository = '' + ``` + + Replace `` with your OCI-compliant container registry. + + +1. Grant access to the repository on compute nodes: + + ```bash + singularity remote login + ``` + + Replace `` with your Singularity remote endpoint. See [Singularity remote login](https://docs.sylabs.io/guides/3.1/user-guide/cli/singularity_remote_login.html) for more information. + +1. (Optional) To build Singularity native images, disable both `singularity.ociAutoPull` and `singularity.ociMode` in your Nextflow configuration. See [Nextflow configuration](https://www.nextflow.io/docs/latest/config.html#config-singularity) for more information. + +
+ +## Mirror containers across registries + +Wave enables mirroring by copying containers used by your pipeline to a container registry of your choice. +Your pipeline can then pull containers from the target registry instead of the original registry. + +
+**Mirror containers across registries** + +To enable container mirroring: + +1. Add the following to your Nextflow configuration: + + ```groovy + wave.enabled = true + wave.mirror = true + tower.accessToken = '' + ``` + + Replace `` with your [Seqera access token](../tutorials/nextflow-wave.mdx#create-your-seqera-access-token). + + +1. Configure your private repository access in Seqera. See [Seqera Platform credentials](https://docs.seqera.io/platform-cloud/credentials/overview) for more information. + +1. Add your build repository to your Nextflow configuration: + + ```groovy + wave.build.repository = '' + ``` + + Replace `` with your container registry. + +
+ +## Security scan containers + +Wave scans containers used in your Nextflow pipelines for security vulnerabilities. This feature helps you ensure that your pipelines use secure container images by identifying potential security risks before and during pipeline execution. + +
+**Security scan containers** + +To enable container security scanning: + +1. Add the following to your Nextflow configuration: + + ```groovy + wave.enabled = true + wave.scan.mode = 'required' + tower.accessToken = '' + ``` + + Replace `` with your [Seqera access token](../tutorials/nextflow-wave.mdx#create-your-seqera-access-token). + +1. Add the acceptable vulnerability levels to your Nextflow configuration: + + ```groovy + wave.scan.allowedLevels = 'low,medium' + ``` + + Accepted vulnerability levels include: `low`, `medium`, `high`, and `critical`. + +:::note +When you set `wave.scan.mode` to `required`, Wave blocks pipeline execution if containers have vulnerabilities above the specified threshold. +The scanning uses the [Common Vulnerabilities Scoring System (CVSS)](https://en.wikipedia.org/wiki/Common_Vulnerability_Scoring_System) to assess security risks. +::: + +:::note +Scan results expire after seven days. When a container is accessed after this period, Wave automatically re-scans it to ensure up-to-date security assessments. +::: + +
+ +## Use Wave with Fusion + +Wave containers allow you to run your containerized pipelines with the Fusion file system. Wave with Fusion enables you to use an object storage bucket, such as AWS S3 or Google Cloud Storage, as your pipeline work directory. + +
+**Use Wave with Fusion** + +To enable Wave with Fusion, add the following to your Nextflow configuration: + + ```groovy + wave.enabled = true + fusion.enabled = true + tower.accessToken = '' + ``` + + Replace `` with your [Seqera access token](../tutorials/nextflow-wave.mdx#create-your-seqera-access-token). + +:::note +For more information about Fusion capabilities and configuration options, see the [Fusion file system documentation](https://docs.seqera.io/fusion). +::: + +
diff --git a/new-docs/wave-docs/provisioning.md b/new-docs/wave-docs/provisioning.md new file mode 100644 index 0000000000..ffe075cb3c --- /dev/null +++ b/new-docs/wave-docs/provisioning.md @@ -0,0 +1,88 @@ +--- +title: Container provisioning +--- + +In the container lifecycle, images are generally created (*built*) and uploaded (*pushed*) to a container registry, and then these images are downloaded (*pulled*) for the execution of a specific pipeline. + +The Wave container provisioning process streamlines the container lifecycle by delivering container images on-demand during the pipeline execution and making sure each container includes the dependencies required by the requesting actor, such as a pipeline task or a user. + +Wave provides the following container provisioning capabilities: + +- Container augmentation +- Container freezing + +## Container augmentation + +The container augmentation provisioning mode allows _extending_ the content of a container image without rebuilding it. Instead, this mechanism modifies a container image during the pull phase made by a Docker-compatible client. Augmented containers are ephemeral: they are not stored in a container repository, and they can only be accessed for a short period of time. The extended content added by Wave is served from a CDN. The augmentation process does not perform any _build_ operation behind the scenes. + +This approach supports use cases such as the following: + +- Authenticate access to your private repositories with Seqera Platform credentials +- Extend existing containers by adding infrastructure and pipeline dependencies on the fly, without rebuilding and maintaining additional container images + +Container augmentation works as follows: + +1. The client, either Nextflow or Wave CLI, submits a container request specifying: + 1. The Seqera Platform user identity + 1. The container image to be augmented + 1. The container extension configuration, which can be either a custom payload, one or more extension layers, or container images. +1. The Wave service validates the request and authorizes the user submitting a request to the Platform service. +1. The Wave service responds with an ephemeral container image name e.g. `wave.seqera.io/wt//library/alpine:latest`. The `` is uniquely assigned and is used to identify and authorize the following container request. +1. The Docker client uses the returned image name to pull the container binary content of the upstream image directly from the target registry. +1. The content added by Wave as one or more layer extensions is shipped by the Wave service. + +Notable parts of this workflow include: + +- Wave acts as a proxy between the Docker client and the target registry that hosts the container image. +- Wave modifies, if needed, the container manifest to add the new content as specified by the request, but it does not (and cannot) alter the container layer blob files that have a unique checksum, which is preserved. +- Image blobs are downloaded directly from the target registry, not from Wave. + +## Container freezes + +The container _freeze mode_ allows the provisioning of non-ephemeral containers that are stored permanently in a container registry of your choice. When using the freeze mode, the Wave service transparently carries out a regular container build. + +This approach supports use cases such as the following: + +- Create container images on-demand from Conda packages +- Deliver multi-architecture (AMD64 and ARM64) and multi-format (Docker and Singularity) container collections +- Deliver container images in the same region where compute is performed + +Wave freeze mode works as follows: + +1. The client, either Nextflow or the Wave CLI, submits a container request specifying: + 1. The Seqera Platform user identity + 1. The container image to augment + 1. The container extension configuration, which can be either a custom payload, one or more extension layers, or container images + 1. The target repository where the built container should be uploaded +1. The Wave service validates the request and authorizes the user via a request to the Platform service. +1. The Wave service checks if the container image already exists in the target registry. +1. If the image does not exist, Wave launches a container build job and pushes the resulting image to the target registry. +1. The Wave service responds with the container image name e.g. `example.com/some/image/build:1234567`. + +Notable parts of this workflow include: + +- Container images provisioned with freeze mode are regular container builds. +- Each container image is associated with a unique ID that is obtained by hashing the following elements: + - The Container file + - Any package dependencies + - The target platform, which is either AMD64 or ARM64 + - The target repository name +- When a request for the same container is made, the same ID is assigned to it and therefore, the build is skipped. +- The resulting images are hosted in your selected repository and not cached locally, unless a cache repository is specified. +- The container images are stored permanently unless the repository owner deletes them. + +## Container provisioning capability matrix + +Wave supports the following types of container builds: + +|Type|Provisioning mode|Source|Freeze|Build repo|Accessibility|Format| +|--- |--- |--- |--- |--- |--- |--- | +|Ephemeral|Augmentation|Container image|No|n/a|Temporary token|Docker| +|Ephemeral|Build|Container file|No|Default|Temporary token|Docker| +|Ephemeral|Build|Conda package|No|Default|Temporary token|Docker| +|Ephemeral|Build|Container file|No|Custom|Temporary token|Docker| +|Ephemeral|Build|Conda package|No|Custom|Temporary token|Docker| +|Durable|Build|Container file|Yes|Custom|Docker auth|Docker /Singularity| +|Durable|Build|Conda package|Yes|Custom|Docker auth|Docker /Singularity| +|Community (durable)|Build|Container file|Yes|Default|Public|Docker /Singularity| +|Community (durable)|Build|Conda package|Yes|Default|Public|Docker /Singularity| diff --git a/new-docs/wave-docs/sidebar.json b/new-docs/wave-docs/sidebar.json new file mode 100644 index 0000000000..4b273ed2a2 --- /dev/null +++ b/new-docs/wave-docs/sidebar.json @@ -0,0 +1,51 @@ +{ + "sidebar": [ + { + "type": "category", + "label": "Wave", + "collapsed": false, + "items": [ + "index", + "get-started", + "wave-lite", + "provisioning" + ] + }, + { + "type": "category", + "label": "Installation", + "collapsed": false, + "items": [ + "install/docker-compose", + "install/kubernetes" + ] + }, + { + "type": "category", + "label": "Configuration", + "collapsed": false, + "items": [ + "wave-lite", + "install/configure-wave", + "install/configure-wave-build" + ] + }, + "nextflow", + { + "type": "category", + "label": "Developer tools", + "collapsed": false, + "items": [ + "cli/index", + "cli/reference", + "api" + ] + }, + "faq", + { + "type": "link", + "label": "Changelog", + "href": "/changelog/tags/wave" + } + ] +} diff --git a/new-docs/wave-docs/troubleshoot.md b/new-docs/wave-docs/troubleshoot.md new file mode 100644 index 0000000000..16a7c36e12 --- /dev/null +++ b/new-docs/wave-docs/troubleshoot.md @@ -0,0 +1,26 @@ +## Troubleshoot guide + +1. How to troubleshoot container build failure? + +If your container build fails, you can check the build details by checking the logs in build details email as shown in below screenshot. + +#### email screenshot: +![](_images/wave_container_build_failure_details.png) + +If there is nothing conclusive in logs, you can check the exit status, e.g. if it is 137 that means out of memory error. +Wave run build process in Kubernetes pod, you can check this [link](https://komodor.com/learn/exit-codes-in-containers-and-kubernetes-the-complete-guide/) for more details on exit codes. + +2. How to solve buildkit error, while running wave build on Docker desktop in mac os? + +#### error: +``` +could not connect to unix:///run/user/1000/buildkit/buildkitd.sock after 10 trials +========== log ========== +[rootlesskit:parent] error: failed to start the child: fork/exec /proc/self/exe: invalid argument +sh: can't kill pid 14: No such process +``` + +#### Solution: +- In case of wave cli use `--platform linux/arm64` flag with wave build command. +- In case of API call use `containerPlatform: linux/arm64` in the request body. + diff --git a/new-docs/wave-docs/tutorials/index.md b/new-docs/wave-docs/tutorials/index.md new file mode 100644 index 0000000000..35ac396375 --- /dev/null +++ b/new-docs/wave-docs/tutorials/index.md @@ -0,0 +1,15 @@ +--- +title: Tutorials +description: Guides to get started with Wave. +tags: [containers, nextflow, seqera containers, wave, wave cli] +--- + +You can leverage Wave in your Nextflow pipelines in several ways. The following guides show you how to get started with different Wave integration approaches: + +- [Nextflow and Wave][nextflow-wave-page]: Provision containers with Wave directly from your Nextflow pipelines +- [Nextflow and Seqera Containers][seqera-containers-page]: Provision containers through Seqera Containers and use them in your Nextflow pipelines +- [Wave CLI][wave-cli-page]: Provision containers with the Wave CLI + +[seqera-containers-page]: /wave_docs/wave_repo/docs/tutorials/nextflow-seqera-containers.mdx +[nextflow-wave-page]: /wave_docs/wave_repo/docs/tutorials/nextflow-wave.mdx +[wave-cli-page]: /wave_docs/wave_repo/docs/tutorials/wave-cli.mdx diff --git a/new-docs/wave-docs/tutorials/nextflow-seqera-containers.mdx b/new-docs/wave-docs/tutorials/nextflow-seqera-containers.mdx new file mode 100644 index 0000000000..fbc8800905 --- /dev/null +++ b/new-docs/wave-docs/tutorials/nextflow-seqera-containers.mdx @@ -0,0 +1,101 @@ +--- +title: Nextflow and Seqera Containers +description: Provision containers through Seqera Containers and use them in your Nextflow pipelines +tags: [containers, nextflow, seqera containers, wave] +--- + +[Seqera Containers][sc] is a free, publicly available bioinformatics container solution launched by Seqera in collaboration with AWS that allows researchers to generate custom containers on demand for any combination of Conda and PyPI packages at the click of a button. + +This guide describes how to provision containers that include Conda packages through [Seqera Containers][sc] and use them directly in your Nextflow pipelines. It includes: + +- Requesting a Conda package from Seqera Containers. +- Creating a Nextflow pipeline that uses the container. +- Running the Nextflow pipeline. + +:::info[**Prerequisites**] +You will need the following to get started: + +- Either [Docker Desktop] or [Podman] installed locally. +- [Nextflow] 23.10.x or newer installed locally. +- [Singularity][Singularity] 4.x or newer to run the Singularity containers. +::: + +## Request a Conda package from Seqera Containers + +Seqera Containers enables users to generate custom Docker or Singularity containers by selecting Conda and PyPI packages through a web interface. + +To request a Conda package from Seqera Containers: + +1. Open [Seqera Containers][sc] in a browser. +1. In the search box, enter `samtools`. +1. In the search results, select **Add** in the `bioconda::samtools` result, and then **Get Container** to initiate the container build. +1. From the **Fetching container** modal, copy the container image URI that Seqera Containers provides. +1. Optional: Select **View build details** to watch Seqera Containers build the requested container in real time. + +## Create a Nextflow pipeline that uses the container + +Nextflow can use the container that Seqera Containers built in the previous section. Use the container URI from Seqera Containers in the `container` directive. + +1. In a terminal window, create a new directory for the Nextflow pipeline. +1. Create a `nextflow.config` file with the following contents: + + ```groovy + docker { + enabled = true + } + ``` + +1. Create a `main.nf` file with the following contents: + + ```nextflow + process SAMTOOLS { + container '' + debug true + """ + samtools --version-only + """ + } + workflow { + SAMTOOLS() + } + ``` + + Replace `` with the container URI that you received from Seqera Containers in the previous section. For example: + + - `community.wave.seqera.io/library/samtools:1.20--b5dfbd93de237464` for linux/amd64. + - `community.wave.seqera.io/library/samtools:1.20--497854c5df637867` for linux/arm64. + +## Run the Nextflow pipeline + +To confirm that the `samtools` command is available from your pipeline, run the following command: + +```bash +nextflow run main.nf +``` + +If successful, you will see output similar to the following: + +```console +N E X T F L O W ~ version 24.04.4 + +Launching `main.nf` [furious_carlsson] DSL2 - revision: 04817f962f + +executor > local (1) +[2f/d2ccc7] process > SAMTOOLS [100%] 1 of 1 ✔ +1.20+htslib-1.20 +``` + +## Next steps + +Visit [Seqera Containers][sc] to learn more about Seqera's container solutions. + +[sc]: https://seqera.io/containers/ +[conda]: https://docs.conda.io/en/latest/ +[Nextflow]: https://www.nextflow.io/docs/latest/install.html +[Singularity]: https://docs.sylabs.io/guides/latest/user-guide/introduction.html +[Podman]: https://podman.io/docs/installation +[Docker Desktop]: https://www.docker.com/products/docker-desktop/ + +[wave-build]: /wave_docs/wave_repo/docs/cli/use-cases.md#build-a-container +[nf-config]: /wave_docs/wave_repo/docs/nextflow/configuration.md +[wave-cli]: /wave_docs/wave_repo/docs/cli/index.md diff --git a/new-docs/wave-docs/tutorials/nextflow-wave.mdx b/new-docs/wave-docs/tutorials/nextflow-wave.mdx new file mode 100644 index 0000000000..4ed71672fe --- /dev/null +++ b/new-docs/wave-docs/tutorials/nextflow-wave.mdx @@ -0,0 +1,183 @@ +--- +title: Nextflow and Wave +description: Provision containers with Wave directly from your Nextflow pipelines +tags: [containers, nextflow, wave] +--- + +Wave integrates seamlessly into Nextflow pipelines by automatically provisioning containers on demand based on your specified Conda dependencies. + +This guide describes how to provision containers with Wave directly from your Nextflow pipelines. It includes: + +- Creating a Seqera access token. +- Adding your container registry credentials to Seqera. +- Creating a Nextflow pipeline that uses the container. +- Running the Nextflow pipeline. + +:::info[**Prerequisites**] +You will need the following to get started: + +- An account with a container registry, such as DockerHub, and an access token that provides write access to your container repository. +- A Seqera account to log into Platform. +- Either [Docker Desktop] or [Podman] installed locally. +- [Nextflow] 23.10.x or newer installed locally. +::: + +## Create your Seqera access token + +A Seqera access token is your personal authentication key that enables access to Seqera Platform services. + +To create a Seqera access token: + +1. Log in to [Seqera](https://cloud.seqera.io/login). +1. From your personal workspace: Go to the user menu and select **Settings > Your tokens**. +1. Select **Add token**. +1. Enter a unique name for your token, then select **Add**. +1. Copy and store your token securely. + + :::caution + The access token is displayed only once. Save the token value before you close the **Personal Access Token** window. + ::: + +1. In a terminal window, assign your access token to the `TOWER_ACCESS_TOKEN` environment variable: + + ```bash + export TOWER_ACCESS_TOKEN= + ``` + + Replace `` with the Seqera access token you created. + +## Add your container registry credentials to Seqera + +When freezing a container to the build repository that you specify, Wave uses Seqera to obtain your registry access credentials. + +To create your access token in Docker Hub: + +1. Log in to [Docker Hub](https://hub.docker.com/). +2. Select your username in the top right corner and select **Account Settings**. +3. Select **Security > New Access Token**. +4. Enter a token description. Select **Access permissions > Read-only**, then select **Generate**. +5. Copy and save the generated access token (this is only displayed once). + +To add your credentials to Seqera: + +1. Add your credentials to your organization or personal workspace: + - From an organization workspace: Go to **Credentials > Add Credentials**. + - From your personal workspace: From the user menu, go to **Your credentials > Add credentials**. + +2. Complete the following fields: + + - **Name**: Specify a unique name for the credentials using alphanumeric characters, dashes, or underscores. For example, `my-registry-creds`. + - **Provider**: Select **Container registry**. + - **User name**: Specify your Docker username. For example, `user1`. + - **Password**: Specify your personal access token (PAT). For example, `1fcd02dc-...215bc3f3`. + - **Registry server**: Specify the container registry hostname, excluding the protocol. For example, `docker.io`. + +3. Select **Add**. The new credential will be listed under the **Credentials** tab. + +Seqera supports other container registries, such as GitHub and Quay.io. + +## Create a Nextflow pipeline that uses Wave + +Nextflow can use Wave to seamlessly build a container directly from a Dockerfile in your pipeline. + +1. In a terminal window, create a new directory for the Nextflow pipeline. + +1. Create a `nextflow.config` file with the following contents: + + ```groovy + docker { + enabled = true + } + + wave { + build.repository = '' + wave.freeze = true + } + + tower { + accessToken = "$TOWER_ACCESS_TOKEN" + } + ``` + + Replace `` with your private container repository. + +1. Create a `main.nf` file with the following contents: + + ```nextflow + include { HELLO } from './modules/gamma' + + workflow { + HELLO() + } + ``` + +1. Create a module directory: + + ```bash + mkdir -p modules/gamma + ``` + +1. Create the `modules/gamma/main.nf` file for the module with the following contents: + + ```nextflow + process HELLO { + debug true + + """ + cowsay Hello! + """ + } + ``` + +1. Create the `modules/gamma/Dockerfile` file for the module and add following contents: + + ```docker + FROM alpine + + RUN apk update && apk add bash cowsay \ + --update-cache \ + --repository https://alpine.global.ssl.fastly.net/alpine/edge/community \ + --repository https://alpine.global.ssl.fastly.net/alpine/edge/main \ + --repository https://dl-3.alpinelinux.org/alpine/edge/testing + + RUN echo hello + ``` + +## Run the Nextflow pipeline + +To run the pipeline, and initiate the Wave container build, run the following command: + +```bash +nextflow run main.nf -with-wave +``` + +If successful, you will see output similar to the following: + +```console +Launching `main.nf` [naughty_wiles] DSL2 - revision: 3756d705d9 + +executor > local (1) +[c1/6d7d9d] HELLO | 1 of 1 ✔ + ________ +< Hello! > + -------- + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || +``` + +## Next steps + +Learn more about [Nextflow's integration][nextflow-int] with Wave. + +[sc]: https://seqera.io/containers/ +[conda]: https://docs.conda.io/en/latest/ +[Nextflow]: https://www.nextflow.io/docs/latest/install.html +[nextflow-int]: /wave_docs/wave_repo/docs/nextflow/index.md +[Podman]: https://podman.io/docs/installation +[Docker Desktop]: https://www.docker.com/products/docker-desktop/ +[wave-build]: /wave_docs/wave_repo/docs/cli/use-cases.md#build-a-container +[nf-config]: /wave_docs/wave_repo/docs/nextflow/configuration.md +[wave-cli]: /wave_docs/wave_repo/docs/cli/index.md diff --git a/new-docs/wave-docs/tutorials/wave-cli.mdx b/new-docs/wave-docs/tutorials/wave-cli.mdx new file mode 100644 index 0000000000..e20389201f --- /dev/null +++ b/new-docs/wave-docs/tutorials/wave-cli.mdx @@ -0,0 +1,165 @@ +--- +title: Wave CLI +description: Provision containers with the Wave CLI +tags: [containers, nextflow, wave, wave cli] +--- + +Wave CLI is a command line tool for the Wave containers provisioning service that allows you to build container images on demand for various sources. + +This guide describes how to build containers using the Wave CLI and freeze them in your private container repository. It includes: + +- Creating a Seqera access token. +- Adding your container registry credentials to Seqera. +- Creating a Nextflow pipeline that uses the container. +- Running the Nextflow pipeline. + +:::info[**Prerequisites**] +You will need the following to get started: + +- An account with a container registry, such as DockerHub, and an access token that provides write access to your container repository. +- A [Seqera](https://cloud.seqera.io/login) account, to store your container registry credentials for Wave to use. +- [Docker Desktop] installed locally. +- [Nextflow] 23.10.x or newer installed locally. +- The [Wave CLI][wave-cli] installed locally. +::: + +## Create your Seqera access token + +A Seqera access token is your personal authentication key that enables access to Seqera services. + +To create a Seqera access token: + +1. Log in to [Seqera](https://cloud.seqera.io/login). +1. From your personal workspace: Go to the user menu and select **Settings > Your tokens**. +1. Select **Add token**. +1. Enter a unique name for your token, then select **Add**. +1. Copy and store your token securely. + + :::caution + The access token is displayed only once. Save the token value before you close the **Personal Access Token** window. + ::: + +1. In a terminal window, assign your access token to the `TOWER_ACCESS_TOKEN` environment variable: + + ```bash + export TOWER_ACCESS_TOKEN= + ``` + + Replace `` with the access token you created. + +## Add your container registry credentials to Seqera + +When freezing a container to the build repository that you specify, Wave uses Seqera to obtain your registry access credentials. If you use Docker as your container registry, complete the following steps: + +To create your access token in Docker Hub: + +1. Log in to [Docker Hub](https://hub.docker.com/). +2. Select your username in the top right corner and select **Account Settings**. +3. Select **Security > New Access Token**. +4. Enter a token description. Select **Access permissions > Read-only**, then select **Generate**. +5. Copy and save the generated access token (this is only displayed once). + +To add your credentials to Seqera: + +1. Add your credentials to your organization or personal workspace: + - From an organization workspace: Go to **Credentials > Add Credentials**. + - From your personal workspace: From the user menu, go to **Your credentials > Add credentials**. + +2. Complete the following fields: + + - **Name**: Specify a unique name for the credentials using alphanumeric characters, dashes, or underscores. For example, `my-registry-creds`. + - **Provider**: Select **Container registry**. + - **User name**: Specify your Docker username. For example, `user1`. + - **Password**: Specify your personal access token (PAT). For example, `1fcd02dc-...215bc3f3`. + - **Registry server**: Specify the container registry hostname, excluding the protocol. For example, `docker.io`. + +3. Select **Add**. The new credential will be listed under the **Credentials** tab. + +Seqera supports other container registries, such as GitHub and Quay.io. + +## Create and freeze a container + +Wave lets you build a container from a Conda package or a set of Conda packages that you specify. In this section, you'll use Wave to build a container that includes the `faker` Conda package. + +In the same terminal window from the previous section, run the Wave CLI to build and freeze a container with the `faker` Conda package to the repository that you specify: + +```bash +wave --conda-package 'faker' --freeze --build-repo --await +``` + +Replace `` with the URI for the repository for which you added an access token to Seqera. + +If successful, you will see output similar to the following: + +```console +docker.io/example-user/repo:faker--2aa7a4d826a76301 +``` + +After the container build completes, Seqera emails you a build status report, including the build logs for the container. + +## Create a Nextflow pipeline that uses the container + +Nextflow can use the container that Wave froze to the build repository that you provided in the previous section. The Wave CLI outputs the URI for the container image and the image tag. Use these values in the `container` directive. + +1. In a terminal window, create a new directory for the Nextflow pipeline. +1. Create a `nextflow.config` file with the following contents: + + ```groovy + docker { + enabled = true + } + + tower { + accessToken = "$TOWER_ACCESS_TOKEN" + } + ``` + +1. Create a `main.nf` file with the following contents: + + ```nextflow + process FAKER { + container 'docker.io/example-user/repo:faker--2aa7a4d826a76301' + debug true + + """ + faker address + """ + } + + workflow { + FAKER() + } + ``` + +## Run the Nextflow pipeline + +To confirm that the `faker` command is available from your pipeline, run the following command: + +```bash +nextflow run main.nf +``` + +If successful, you will see output similar to the following: + +```console +Launching `./main.nf` [happy_leavitt] DSL2 - revision: 03b4e42ba3 + +executor > local (1) +[1d/120069] FAKER | 1 of 1 ✔ +1287 Eric Grove +Reneechester, AK 75644 +``` + +## Next steps + +Learn more about Wave CLI and how to use it to build containers in the [Wave CLI documentation][wave-cli]. + +[sc]: https://seqera.io/containers/ +[conda]: https://docs.conda.io/en/latest/ +[Nextflow]: https://www.nextflow.io/docs/latest/install.html +[Podman]: https://podman.io/docs/installation +[Docker Desktop]: https://www.docker.com/products/docker-desktop/ + +[wave-build]: /wave_docs/wave_repo/docs/cli/use-cases.md#build-a-container +[nf-config]: /wave_docs/wave_repo/docs/nextflow/configuration.md +[wave-cli]: /wave_docs/wave_repo/docs/cli/index.md diff --git a/new-docs/wave-docs/wave-lite.md b/new-docs/wave-docs/wave-lite.md new file mode 100644 index 0000000000..ba3ff12932 --- /dev/null +++ b/new-docs/wave-docs/wave-lite.md @@ -0,0 +1,27 @@ +# Wave Lite + +Wave Lite is a configuration mode for Wave containers that provides the following features: + +- Container augmentation +- Container inspection + +Wave Lite enables the use of [Fusion file system](https://docs.seqera.io/fusion) in Nextflow pipelines on AWS, Azure, and GCP cloud deployments. + +## Installation + +- [Docker Compose](./install/docker-compose.md) +- [Kubernetes](./install/docker-compose.md) + +:::info +Docker Compose installations only support Wave in Lite mode. Wave's full build capabilities require specific integrations with Kubernetes and AWS EFS Storage, making EKS and AWS a hard dependency for fully-featured deployments. After you have successfully deployed Wave Lite in Kubernetes, see [Configure Wave Build](./install/configure-wave-build.md) to extend your installation to support build capabilities. +::: + +## Limitations + +The following Wave features are **not** available in Lite configuration: + +- **Container Freeze** +- **Container Build service** +- **Container Mirror service** +- **Container Security scanning** +- **Container blobs caching**