feat(artifacts): Object storage integration and artifacts key

.gitignore

@@ -38,6 +38,7 @@ secrets.env
 
 # Dotenv environment file
 .env
+.env_*
 .env.test
 
 # Files to be excluded.

.golangci.yml

@@ -157,7 +157,7 @@ linters:
     wsl_v5:
       allow-first-in-block: true
       allow-whole-block: false
-      branch-max-lines: 2 
+      branch-max-lines: 2
   exclusions:
     generated: lax
     presets:

DOCS.md

@@ -56,6 +56,16 @@ echo "VELA_SCM_CLIENT=<Github OAuth Client ID>" >> .env
 echo "VELA_SCM_SECRET=<Github OAuth Client Secret>" >> .env
 ```
 
+* Add `minio` to `/etc/hosts` for nginx to resolve the local minio service when running Vela:
+
+```bash
+ sudo sh -c 'echo "127.0.0.1 minio" >> /etc/hosts'
+````
+
+* Using Artifacts:
+* Set `VELA_STORAGE_ENABLE: true` in the docker-compose file.
+* Create a bucket in Minio UI and add the bucket in docker-compose file as `VELA_STORAGE_BUCKET`.
+
 ## Start
 
 **NOTE: Please review the [setup section](#setup) before moving forward.**

api/storage/doc.go

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: Apache-2.0
+
+// Package storage provides the storage handlers for the Vela API.
+//
+// Usage:
+//
+//	import "github.com/go-vela/server/api/storage"
+package storage

api/storage/storage.go

@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package storage
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/router/middleware/build"
+	"github.com/go-vela/server/router/middleware/repo"
+	"github.com/go-vela/server/storage"
+)
+
+// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/storage storage ListBuildObjectNames
+//
+// List object names for a specific build in a bucket.
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+//   - name: org
+//     in: path
+//     description: Organization name
+//     required: true
+//     type: string
+//   - name: repo
+//     in: path
+//     description: Repository name
+//     required: true
+//     type: string
+//   - name: build
+//     in: path
+//     description: Build number
+//     required: true
+//     type: integer
+//     format: int64
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   200:
+//     description: Successfully listed object names for the build
+//   400:
+//     description: Bad request due to invalid parameters
+//     schema:
+//       $ref: '#/definitions/Error'
+//   403:
+//     description: Storage is not enabled or invalid token
+//     schema:
+//       $ref: '#/definitions/Error'
+//   404:
+//     description: Repo not found
+//     schema:
+//       $ref: '#/definitions/Error'
+//   500:
+//     description: Unexpected server error
+//     schema:
+//       $ref: '#/definitions/Error'
+
+// ListBuildObjectNames represents the API handler to list object names for a specific build.
+func ListBuildObjectNames(c *gin.Context) {
+	enable := c.MustGet("storage-enable").(bool)
+	if !enable {
+		l := c.MustGet("logger").(*logrus.Entry)
+		l.Info("storage is not enabled, skipping credentials request")
+		c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"})
+
+		return
+	}
+
+	l := c.MustGet("logger").(*logrus.Entry)
+
+	r := repo.Retrieve(c)
+	b := build.Retrieve(c)
+	org := r.GetOrg()
+	buildNum := b.GetNumber()
+
+	l.Debugf("listing object names in bucket for %s/%s build #%d", org, r.GetName(), buildNum)
+
+	// Call the ListBuildObjectNames method that handles prefix filtering
+	objectNames, err := storage.FromGinContext(c).ListBuildObjectNames(
+		c.Request.Context(),
+		org,
+		r.GetName(),
+		strconv.FormatInt(buildNum, 10),
+	)
+	if err != nil {
+		l.Errorf("unable to list objects for %s/%s build #%d: %v", org, r.GetName(), buildNum, err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"names": objectNames})
+}

api/storage/sts.go

@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package storage
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/router/middleware/build"
+	"github.com/go-vela/server/router/middleware/repo"
+	"github.com/go-vela/server/storage"
+)
+
+// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/storage/sts storage GetSTSCreds
+//
+// Get temporary STS credentials for build storage uploads.
+//
+// Generates temporary AWS STS credentials scoped to allow PUT operations
+// into the configured storage bucket under the build-specific prefix.
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+//   - name: org
+//     in: path
+//     description: Organization name
+//     required: true
+//     type: string
+//   - name: repo
+//     in: path
+//     description: Repository name
+//     required: true
+//     type: string
+//   - name: build
+//     in: path
+//     description: Build number
+//     required: true
+//     type: integer
+//     format: int64
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   200:
+//     description: Successfully generated temporary STS credentials
+//     schema:
+//       $ref: '#/definitions/STSCreds'
+//   400:
+//     description: Bad request due to invalid parameters
+//     schema:
+//       $ref: '#/definitions/Error'
+//   403:
+//     description: Storage is not enabled or invalid token
+//     schema:
+//       $ref: '#/definitions/Error'
+//   404:
+//     description: Repo not found
+//     schema:
+//       $ref: '#/definitions/Error'
+//   500:
+//     description: Unable to assume role or generate credentials
+//     schema:
+//       $ref: '#/definitions/Error'
+
+// GetSTSCreds represents the API handler to generate temporary STS credentials for build storage uploads.
+func GetSTSCreds(c *gin.Context) {
+	l := c.MustGet("logger").(*logrus.Entry)
+
+	enabled := c.MustGet("storage-enable").(bool)
+	if !enabled {
+		l.Info("storage is not enabled, skipping credentials request")
+		c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"})
+
+		return
+	}
+
+	r := repo.Retrieve(c)
+	org := r.GetOrg()
+	b := build.Retrieve(c)
+	repoName := r.GetName()
+	buildNum := b.GetNumber()
+	ctx := c.Request.Context()
+
+	prefix := fmt.Sprintf("%s/%s/%d/", org, repoName, buildNum)
+
+	sessionName := fmt.Sprintf("vela-%s-%s-%d", org, repoName, buildNum)
+
+	creds, err := storage.FromGinContext(c).AssumeRole(ctx, int(r.GetTimeout())*60, prefix, sessionName)
+	if creds == nil {
+		l.Errorf("unable to assume role and generate temporary credentials without error %s", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "unable to assume role and generate temporary credentials"})
+
+		return
+	}
+
+	if err != nil {
+		l.Errorf("unable to assume role: %v", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+
+		return
+	}
+
+	c.JSON(http.StatusOK, creds)
+}

api/types/pipeline.go

@@ -24,6 +24,7 @@ type Pipeline struct {
 	Stages          *bool     `json:"stages,omitempty"`
 	Steps           *bool     `json:"steps,omitempty"`
 	Templates       *bool     `json:"templates,omitempty"`
+	Artifact        *bool     `json:"artifacts,omitempty"`
 	Warnings        *[]string `json:"warnings,omitempty"`
 	// swagger:strfmt base64
 	Data *[]byte `json:"data,omitempty"`
@@ -211,6 +212,19 @@ func (p *Pipeline) GetTemplates() bool {
 	return *p.Templates
 }
 
+// GetArtifact returns the Artifact field.
+//
+// When the provided Pipeline type is nil, or the field within
+// the type is nil, it returns the zero value for the field.
+func (p *Pipeline) GetArtifact() bool {
+	// return zero value if Pipeline type or Artifact field is nil
+	if p == nil || p.Artifact == nil {
+		return false
+	}
+
+	return *p.Artifact
+}
+
 // GetWarnings returns the Warnings field.
 //
 // When the provided Pipeline type is nil, or the field within
@@ -419,6 +433,19 @@ func (p *Pipeline) SetTemplates(v bool) {
 	p.Templates = &v
 }
 
+// SetArtifact sets the Artifact field.
+//
+// When the provided Pipeline type is nil, it
+// will set nothing and immediately return.
+func (p *Pipeline) SetArtifact(v bool) {
+	// return if Pipeline type is nil
+	if p == nil {
+		return
+	}
+
+	p.Artifact = &v
+}
+
 // SetWarnings sets the Warnings field.
 //
 // When the provided Pipeline type is nil, it
@@ -461,6 +488,7 @@ func (p *Pipeline) String() string {
   Stages: %t,
   Steps: %t,
   Templates: %t,
+  Artifacts: %t,
   Type: %s,
   Version: %s,
   Warnings: %v,
@@ -478,6 +506,7 @@ func (p *Pipeline) String() string {
 		p.GetStages(),
 		p.GetSteps(),
 		p.GetTemplates(),
+		p.GetArtifact(),
 		p.GetType(),
 		p.GetVersion(),
 		p.GetWarnings(),

api/types/pipeline_test.go

@@ -214,6 +214,7 @@ func TestAPI_Pipeline_String(t *testing.T) {
   Stages: %t,
   Steps: %t,
   Templates: %t,
+  Artifacts: %t,
   Type: %s,
   Version: %s,
   Warnings: %v,
@@ -231,6 +232,7 @@ func TestAPI_Pipeline_String(t *testing.T) {
 		p.GetStages(),
 		p.GetSteps(),
 		p.GetTemplates(),
+		p.GetArtifact(),
 		p.GetType(),
 		p.GetVersion(),
 		p.GetWarnings(),
@@ -263,6 +265,7 @@ func testPipeline() *Pipeline {
 	p.SetStages(false)
 	p.SetSteps(true)
 	p.SetTemplates(false)
+	p.SetArtifact(false)
 	p.SetData(testPipelineData())
 	p.SetWarnings([]string{"42:this is a warning"})
 

api/types/storage.go

@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package types
+
+import (
+	"github.com/minio/minio-go/v7"
+)
+
+// Bucket is the API types representation of an object storage.
+//
+// swagger:model CreateBucket
+type Bucket struct {
+	BucketName         string                   `json:"bucket_name,omitempty"`
+	MakeBucketOptions  minio.MakeBucketOptions  `json:"make_bucket_options,omitempty"`
+	ListObjectsOptions minio.ListObjectsOptions `json:"list_objects_options,omitempty"`
+	Recursive          bool                     `json:"recursive"`
+}
+
+type Object struct {
+	ObjectName string `json:"object_name,omitempty"`
+	Bucket     Bucket `json:"bucket,omitempty"`
+	FilePath   string `json:"file_path,omitempty"`
+}

api/types/storage_sts.go

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package types
+
+import "time"
+
+// STSCreds defines the structure for temporary credentials used for object storage access.
+//
+// swagger:model STSCreds
+type STSCreds struct {
+	AccessKey    string `json:"access_key"`
+	SecretKey    string `json:"secret_key"`
+	SessionToken string `json:"session_token"`
+
+	Endpoint string `json:"endpoint"`
+	Bucket   string `json:"bucket"`
+	Region   string `json:"region,omitempty"`
+	Prefix   string `json:"prefix,omitempty"`
+	Secure   bool   `json:"secure,omitempty"`
+
+	ExpiresAt time.Time `json:"expires_at,omitempty"`
+}