diff --git a/jobs/snapshot-s3-archiver/Dockerfile b/jobs/snapshot-s3-archiver/Dockerfile new file mode 100644 index 0000000..c95d9d6 --- /dev/null +++ b/jobs/snapshot-s3-archiver/Dockerfile @@ -0,0 +1,16 @@ +# Using apline/golang image +FROM golang:1.25-alpine + +# Set destination for COPY +WORKDIR /app + +# Copy required files +COPY go.mod ./ +COPY go.sum ./ +COPY main.go ./ + +# Build the executable +RUN go build -o /jobs-snapshot-s3 + +# Run the executable +ENTRYPOINT [ "/jobs-snapshot-s3" ] diff --git a/jobs/snapshot-s3-archiver/README.md b/jobs/snapshot-s3-archiver/README.md new file mode 100644 index 0000000..8f4cf60 --- /dev/null +++ b/jobs/snapshot-s3-archiver/README.md @@ -0,0 +1,133 @@ +# Scaleway Instance Snapshot Backup to S3 + +This project exports available Scaleway Instance snapshots to an S3-compatible bucket (e.g., Scaleway Object Storage), and optionally deletes the snapshot afterward if it's already backed up. It's designed to run as a **Scaleway Serverless Job**, making it ideal for automated, scheduled backups. + +--- + +## ๐Ÿ“ฆ Features + +- Lists all available block storage snapshots in a project. +- Checks if a snapshot with the same name already exists in the target bucket. +- Exports missing snapshots to the bucket in `.qcow2` format. +- Deletes local snapshot after successful export (if not already in bucket). +- Uses environment variables for full configuration. +- Built to run in a container on [Scaleway Serverless Jobs](https://www.scaleway.com/en/serverless-jobs/). + +--- + +## โš™๏ธ Environment Variables + +You must set the following environment variables when deploying the job: + +| Variable | Description | +|--------|-------------| +| `SCW_DEFAULT_ORGANIZATION_ID` | Your Scaleway Organization ID (legacy; prefer project ID). | +| `SCW_DEFAULT_PROJECT_ID` | Your Scaleway Project ID (preferred way to group resources). | +| `SCW_ACCESS_KEY` | API access key (from IAM). | +| `SCW_SECRET_KEY` | API secret key (from IAM). | +| `SCW_ZONE` | Zone where your snapshots are located (e.g., `fr-par-1`). | +| `SCW_BUCKET_NAME` | Name of the S3 bucket to store exported snapshots. | +| `SCW_BUCKET_ENDPOINT` | S3 endpoint (e.g., `s3.fr-par.scw.cloud`). | + +> ๐Ÿ” **Security Tip**: Use IAM API keys with minimal required permissions. + +--- + +## ๐Ÿ› ๏ธ Build & Deploy to Scaleway Serverless Jobs + +### 1. Build the Docker Image + +```bash +docker build -t snapshot-s3-backup . +``` + +### 2. Tag and Push to Scaleway Container Registry (or any registry) + +```bash +# Example using Scaleway CR +docker tag snapshot-s3-backup fr-par.scw.cloud/your-registry/snapshot-s3-backup:v1 +docker push fr-par.scw.cloud/your-registry/snapshot-s3-backup:v1 +``` + +> Replace `your-registry` with your actual container registry name. + +### 3. Create the Serverless Job + +Use the Scaleway CLI or Console: + +#### Using `scw` CLI: + +```bash +scw job create \ + name=backup-snapshots \ + image=fr-par.scw.cloud/your-registry/snapshot-s3-backup:v1 \ + memory-limit=512Mi \ + cpu-limit=500m \ + environment='{ + "SCW_DEFAULT_PROJECT_ID": "your-project-id", + "SCW_ACCESS_KEY": "your-access-key", + "SCW_SECRET_KEY": "your-secret-key", + "SCW_ZONE": "fr-par-1", + "SCW_BUCKET_NAME": "my-backup-bucket", + "SCW_BUCKET_ENDPOINT": "s3.fr-par.scw.cloud" + }' +``` + +### 4. (Optional) Schedule the Job + +Schedule it to run daily using a cron trigger: + +```bash +scw scheduler trigger create-cron \ + job-id=your-job-id \ + schedule="0 2 * * *" \ + name=daily-snapshot-backup +``` + +This runs the job every day at 2 AM. + +--- + +## ๐Ÿ“ Output Format + +Each snapshot is exported as: +``` +.qcow2 +``` + +Example: +``` +my-server-disk-2025-04-05.qcow2 +``` + +--- + +## โœ… Example Use Case + +Run nightly to: +1. Export new snapshots to object storage. +2. Clean up old snapshots once safely backed up. +3. Reduce storage costs and improve disaster recovery. + +--- + +## ๐Ÿงช Local Testing (Optional) + +Set environment variables: + +```bash +export SCW_DEFAULT_PROJECT_ID=... +export SCW_ACCESS_KEY=... +export SCW_SECRET_KEY=... +export SCW_ZONE=fr-par-1 +export SCW_BUCKET_NAME=my-backup-bucket +export SCW_BUCKET_ENDPOINT=s3.fr-par.scw.cloud +``` + +Run: + +```bash +go run main.go +``` + +--- \ No newline at end of file diff --git a/jobs/snapshot-s3-archiver/go.mod b/jobs/snapshot-s3-archiver/go.mod new file mode 100644 index 0000000..c420369 --- /dev/null +++ b/jobs/snapshot-s3-archiver/go.mod @@ -0,0 +1,31 @@ +module github.com/scaleway/serverless-examples/jobs/snapshot-s3-archiver + +go 1.25 + +require ( + github.com/minio/minio-go/v7 v7.0.95 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tinylib/msgp v1.4.0 // indirect + golang.org/x/crypto v0.42.0 // indirect + golang.org/x/net v0.44.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/jobs/snapshot-s3-archiver/go.sum b/jobs/snapshot-s3-archiver/go.sum new file mode 100644 index 0000000..79cac08 --- /dev/null +++ b/jobs/snapshot-s3-archiver/go.sum @@ -0,0 +1,62 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35 h1:8xfn1RzeI9yoCUuEwDy08F+No6PcKZGEDOQ6hrRyLts= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35/go.mod h1:47B1d/YXmSAxlJxUJxClzHR6b3T4M1WyCvwENPQNBWc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.4.0 h1:SYOeDRiydzOw9kSiwdYp9UcBgPFtLU2WDHaJXyHruf8= +github.com/tinylib/msgp v1.4.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jobs/snapshot-s3-archiver/main.go b/jobs/snapshot-s3-archiver/main.go new file mode 100644 index 0000000..d068d72 --- /dev/null +++ b/jobs/snapshot-s3-archiver/main.go @@ -0,0 +1,173 @@ +package main + +import ( + "context" + "fmt" + "os" + "slices" + + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/scaleway/scaleway-sdk-go/scw" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// Environment variable constants used to configure the Scaleway API client. +// These must be set in the environment for the application to authenticate and interact with Scaleway services. +const ( + // envOrgID is the Scaleway Organization ID, used for billing and resource ownership (legacy; prefer Project ID). + envOrgID = "SCW_DEFAULT_ORGANIZATION_ID" + + // envAccessKey is the API access key for authenticating requests to Scaleway. + envAccessKey = "SCW_ACCESS_KEY" + + // envSecretKey is the secret key associated with the access key, used for signing requests. + envSecretKey = "SCW_SECRET_KEY" + + // envProjectID is the Scaleway Project ID, which groups resources and is the preferred way to organize infrastructure. + envProjectID = "SCW_DEFAULT_PROJECT_ID" + + // envZone specifies the geographical region/zone where resources will be created (e.g., fr-par-1). + envZone = "SCW_ZONE" + + // envBucket is a custom environment variable for specifying the name of an S3-compatible bucket. + // This is not a standard Scaleway variable and is application-specific. + envBucket = "SCW_BUCKET_NAME" + + envBucketEndpoint = "SCW_BUCKET_ENDPOINT" +) + +func main() { + // Create a Scaleway client with credentials from environment variables. + client, err := scw.NewClient( + // Get your organization ID at https://console.scaleway.com/organization/settings + scw.WithDefaultOrganizationID(os.Getenv(envOrgID)), + + // Get your credentials at https://console.scaleway.com/iam/api-keys + scw.WithAuth(os.Getenv(envAccessKey), os.Getenv(envSecretKey)), + + // Set the default project ID to organize resources under a specific project + scw.WithDefaultProjectID(os.Getenv(envProjectID)), + + // Set the default zone where resources such as block volumes and snapshots are located + scw.WithDefaultZone(scw.Zone(os.Getenv(envZone))), + ) + if err != nil { + panic(err) + } + + fmt.Println("Initializing instance API...") + + instanceAPI := instance.NewAPI(client) + + fmt.Println("Reading all snapshots for the project...") + + snapList, err := instanceAPI.ListSnapshots(&instance.ListSnapshotsRequest{}, scw.WithAllPages()) + if err != nil { + panic(err) + } + + fmt.Println("Reading all snapshots already in the bucket...") + + filesInBucket, err := listBucketFiles() + if err != nil { + panic(err) + } + + const snapshotExtension = ".qcow2" + + for _, snapshot := range snapList.Snapshots { + fmt.Printf("Checking for snapshot %s\n", snapshot.Name) + + if snapshot.State == instance.SnapshotStateAvailable { + // Check if file already exists in bucket + if slices.Contains(filesInBucket, snapshot.Name+snapshotExtension) { + fmt.Printf("File %s already exists in bucket, can delete the snapshot and skip it\n", snapshot.Name+snapshotExtension) + + err = instanceAPI.DeleteSnapshot(&instance.DeleteSnapshotRequest{ + SnapshotID: snapshot.ID, + }) + if err != nil { + panic(err) + } + + continue + } + + fmt.Printf("File %s not present in the bucket, expording it to the bucket...\n", snapshot.Name+".qcow2") + + snap, err := instanceAPI.ExportSnapshot(&instance.ExportSnapshotRequest{ + SnapshotID: snapshot.ID, + Bucket: os.Getenv(envBucket), + Key: snapshot.Name + snapshotExtension, + }) + if err != nil { + fmt.Printf("Failed to export snapshot %s: %v\n", snapshot.Name, err) + + continue + } + + fmt.Printf("Successfully started export of %s to %s/%s\n", snap.Task.ID, os.Getenv(envBucket), snap.Task.Description) + } else { + fmt.Printf("Skipping snapshot %s (ID: %s) - status is %s, not available\n", snapshot.Name, snapshot.ID, snapshot.State.String()) + } + } +} + +// Check for mandatory variables before starting to work. +func init() { + mandatoryVariables := [...]string{ + envOrgID, + envAccessKey, + envSecretKey, + envZone, + envProjectID, + envBucket, + envBucketEndpoint, + } + + for idx := range mandatoryVariables { + if os.Getenv(mandatoryVariables[idx]) == "" { + panic("missing environment variable " + mandatoryVariables[idx]) + } + } +} + +func listBucketFiles() ([]string, error) { + // Retrieve S3-compatible endpoint and credentials from environment + endpoint := os.Getenv(envBucketEndpoint) + accessKeyID := os.Getenv(envAccessKey) + secretAccessKey := os.Getenv(envSecretKey) + + // Create new MinIO client + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: true, + }) + if err != nil { + return nil, err + } + + // Set up context and result slice + ctx := context.Background() + var files []string + + // Channel to signal listing completion + doneCh := make(chan struct{}) + defer close(doneCh) + + // List all objects in the bucket + for object := range minioClient.ListObjects(ctx, os.Getenv(envBucket), minio.ListObjectsOptions{ + Recursive: false, + WithMetadata: true, + }) { + if object.Err != nil { + return nil, object.Err + } + + files = append(files, object.Key) + } + + return files, nil +}