diff --git a/.gitignore b/.gitignore index a0b951aa0..29f8fa84c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ secrets.env # Dotenv environment file .env +.env_* .env.test # Files to be excluded. diff --git a/.golangci.yml b/.golangci.yml index 8d40ccb9a..d62717145 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -157,7 +157,7 @@ linters: wsl_v5: allow-first-in-block: true allow-whole-block: false - branch-max-lines: 2 + branch-max-lines: 2 exclusions: generated: lax presets: diff --git a/DOCS.md b/DOCS.md index 940ee02ba..8a3124551 100644 --- a/DOCS.md +++ b/DOCS.md @@ -56,6 +56,16 @@ echo "VELA_SCM_CLIENT=" >> .env echo "VELA_SCM_SECRET=" >> .env ``` +* Add `minio` to `/etc/hosts` for nginx to resolve the local minio service when running Vela: + +```bash + sudo sh -c 'echo "127.0.0.1 minio" >> /etc/hosts' +```` + +* Using Artifacts: +* Set `VELA_STORAGE_ENABLE: true` in the docker-compose file. +* Create a bucket in Minio UI and add the bucket in docker-compose file as `VELA_STORAGE_BUCKET`. + ## Start **NOTE: Please review the [setup section](#setup) before moving forward.** diff --git a/api/storage/doc.go b/api/storage/doc.go new file mode 100644 index 000000000..aae3e99dd --- /dev/null +++ b/api/storage/doc.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package storage provides the storage handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/storage" +package storage diff --git a/api/storage/storage.go b/api/storage/storage.go new file mode 100644 index 000000000..bc6fa6c85 --- /dev/null +++ b/api/storage/storage.go @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/storage" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/storage storage ListBuildObjectNames +// +// List object names for a specific build in a bucket. +// +// --- +// produces: +// - application/json +// parameters: +// - name: org +// in: path +// description: Organization name +// required: true +// type: string +// - name: repo +// in: path +// description: Repository name +// required: true +// type: string +// - name: build +// in: path +// description: Build number +// required: true +// type: integer +// format: int64 +// security: +// - ApiKeyAuth: [] +// responses: +// 200: +// description: Successfully listed object names for the build +// 400: +// description: Bad request due to invalid parameters +// schema: +// $ref: '#/definitions/Error' +// 403: +// description: Storage is not enabled or invalid token +// schema: +// $ref: '#/definitions/Error' +// 404: +// description: Repo not found +// schema: +// $ref: '#/definitions/Error' +// 500: +// description: Unexpected server error +// schema: +// $ref: '#/definitions/Error' + +// ListBuildObjectNames represents the API handler to list object names for a specific build. +func ListBuildObjectNames(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + + r := repo.Retrieve(c) + b := build.Retrieve(c) + org := r.GetOrg() + buildNum := b.GetNumber() + + l.Debugf("listing object names in bucket for %s/%s build #%d", org, r.GetName(), buildNum) + + // Call the ListBuildObjectNames method that handles prefix filtering + objectNames, err := storage.FromGinContext(c).ListBuildObjectNames( + c.Request.Context(), + org, + r.GetName(), + strconv.FormatInt(buildNum, 10), + ) + if err != nil { + l.Errorf("unable to list objects for %s/%s build #%d: %v", org, r.GetName(), buildNum, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + + return + } + + c.JSON(http.StatusOK, gin.H{"names": objectNames}) +} diff --git a/api/storage/sts.go b/api/storage/sts.go new file mode 100644 index 000000000..a3a3b10b3 --- /dev/null +++ b/api/storage/sts.go @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/storage" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/storage/sts storage GetSTSCreds +// +// Get temporary STS credentials for build storage uploads. +// +// Generates temporary AWS STS credentials scoped to allow PUT operations +// into the configured storage bucket under the build-specific prefix. +// +// --- +// produces: +// - application/json +// parameters: +// - name: org +// in: path +// description: Organization name +// required: true +// type: string +// - name: repo +// in: path +// description: Repository name +// required: true +// type: string +// - name: build +// in: path +// description: Build number +// required: true +// type: integer +// format: int64 +// security: +// - ApiKeyAuth: [] +// responses: +// 200: +// description: Successfully generated temporary STS credentials +// schema: +// $ref: '#/definitions/STSCreds' +// 400: +// description: Bad request due to invalid parameters +// schema: +// $ref: '#/definitions/Error' +// 403: +// description: Storage is not enabled or invalid token +// schema: +// $ref: '#/definitions/Error' +// 404: +// description: Repo not found +// schema: +// $ref: '#/definitions/Error' +// 500: +// description: Unable to assume role or generate credentials +// schema: +// $ref: '#/definitions/Error' + +// GetSTSCreds represents the API handler to generate temporary STS credentials for build storage uploads. +func GetSTSCreds(c *gin.Context) { + l := c.MustGet("logger").(*logrus.Entry) + + enabled := c.MustGet("storage-enable").(bool) + if !enabled { + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + r := repo.Retrieve(c) + org := r.GetOrg() + b := build.Retrieve(c) + repoName := r.GetName() + buildNum := b.GetNumber() + ctx := c.Request.Context() + + prefix := fmt.Sprintf("%s/%s/%d/", org, repoName, buildNum) + + sessionName := fmt.Sprintf("vela-%s-%s-%d", org, repoName, buildNum) + + creds, err := storage.FromGinContext(c).AssumeRole(ctx, int(r.GetTimeout())*60, prefix, sessionName) + if creds == nil { + l.Errorf("unable to assume role and generate temporary credentials without error %s", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "unable to assume role and generate temporary credentials"}) + + return + } + + if err != nil { + l.Errorf("unable to assume role: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + + return + } + + c.JSON(http.StatusOK, creds) +} diff --git a/api/types/pipeline.go b/api/types/pipeline.go index dd0ff617d..f054ff9eb 100644 --- a/api/types/pipeline.go +++ b/api/types/pipeline.go @@ -24,6 +24,7 @@ type Pipeline struct { Stages *bool `json:"stages,omitempty"` Steps *bool `json:"steps,omitempty"` Templates *bool `json:"templates,omitempty"` + Artifact *bool `json:"artifacts,omitempty"` Warnings *[]string `json:"warnings,omitempty"` // swagger:strfmt base64 Data *[]byte `json:"data,omitempty"` @@ -211,6 +212,19 @@ func (p *Pipeline) GetTemplates() bool { return *p.Templates } +// GetArtifact returns the Artifact field. +// +// When the provided Pipeline type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (p *Pipeline) GetArtifact() bool { + // return zero value if Pipeline type or Artifact field is nil + if p == nil || p.Artifact == nil { + return false + } + + return *p.Artifact +} + // GetWarnings returns the Warnings field. // // When the provided Pipeline type is nil, or the field within @@ -419,6 +433,19 @@ func (p *Pipeline) SetTemplates(v bool) { p.Templates = &v } +// SetArtifact sets the Artifact field. +// +// When the provided Pipeline type is nil, it +// will set nothing and immediately return. +func (p *Pipeline) SetArtifact(v bool) { + // return if Pipeline type is nil + if p == nil { + return + } + + p.Artifact = &v +} + // SetWarnings sets the Warnings field. // // When the provided Pipeline type is nil, it @@ -461,6 +488,7 @@ func (p *Pipeline) String() string { Stages: %t, Steps: %t, Templates: %t, + Artifacts: %t, Type: %s, Version: %s, Warnings: %v, @@ -478,6 +506,7 @@ func (p *Pipeline) String() string { p.GetStages(), p.GetSteps(), p.GetTemplates(), + p.GetArtifact(), p.GetType(), p.GetVersion(), p.GetWarnings(), diff --git a/api/types/pipeline_test.go b/api/types/pipeline_test.go index 8c14c5af2..4e6359922 100644 --- a/api/types/pipeline_test.go +++ b/api/types/pipeline_test.go @@ -214,6 +214,7 @@ func TestAPI_Pipeline_String(t *testing.T) { Stages: %t, Steps: %t, Templates: %t, + Artifacts: %t, Type: %s, Version: %s, Warnings: %v, @@ -231,6 +232,7 @@ func TestAPI_Pipeline_String(t *testing.T) { p.GetStages(), p.GetSteps(), p.GetTemplates(), + p.GetArtifact(), p.GetType(), p.GetVersion(), p.GetWarnings(), @@ -263,6 +265,7 @@ func testPipeline() *Pipeline { p.SetStages(false) p.SetSteps(true) p.SetTemplates(false) + p.SetArtifact(false) p.SetData(testPipelineData()) p.SetWarnings([]string{"42:this is a warning"}) diff --git a/api/types/storage.go b/api/types/storage.go new file mode 100644 index 000000000..0c6913282 --- /dev/null +++ b/api/types/storage.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "github.com/minio/minio-go/v7" +) + +// Bucket is the API types representation of an object storage. +// +// swagger:model CreateBucket +type Bucket struct { + BucketName string `json:"bucket_name,omitempty"` + MakeBucketOptions minio.MakeBucketOptions `json:"make_bucket_options,omitempty"` + ListObjectsOptions minio.ListObjectsOptions `json:"list_objects_options,omitempty"` + Recursive bool `json:"recursive"` +} + +type Object struct { + ObjectName string `json:"object_name,omitempty"` + Bucket Bucket `json:"bucket,omitempty"` + FilePath string `json:"file_path,omitempty"` +} diff --git a/api/types/storage_sts.go b/api/types/storage_sts.go new file mode 100644 index 000000000..40e560751 --- /dev/null +++ b/api/types/storage_sts.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import "time" + +// STSCreds defines the structure for temporary credentials used for object storage access. +// +// swagger:model STSCreds +type STSCreds struct { + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + SessionToken string `json:"session_token"` + + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` + Region string `json:"region,omitempty"` + Prefix string `json:"prefix,omitempty"` + Secure bool `json:"secure,omitempty"` + + ExpiresAt time.Time `json:"expires_at,omitempty"` +} diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 1fdae07e7..b06974327 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -18,6 +18,7 @@ import ( "github.com/go-vela/server/queue" "github.com/go-vela/server/scm" "github.com/go-vela/server/secret" + "github.com/go-vela/server/storage" "github.com/go-vela/server/tracing" "github.com/go-vela/server/version" ) @@ -62,6 +63,9 @@ func main() { // Add Tracing Flags cmd.Flags = append(cmd.Flags, tracing.Flags...) + // Add S3 Flags + cmd.Flags = append(cmd.Flags, storage.Flags...) + if err = cmd.Run(context.Background(), os.Args); err != nil { logrus.Fatal(err) } diff --git a/cmd/vela-server/metadata.go b/cmd/vela-server/metadata.go index f23e309c1..bcfbb4dfa 100644 --- a/cmd/vela-server/metadata.go +++ b/cmd/vela-server/metadata.go @@ -45,6 +45,13 @@ func setupMetadata(c *cli.Command) (*internal.Metadata, error) { m.Vela = vela + storage, err := metadataStorage(c) + if err != nil { + return nil, err + } + + m.Storage = storage + return m, nil } @@ -93,6 +100,21 @@ func metadataSource(c *cli.Command) (*internal.Source, error) { }, nil } +// helper function to capture the storage metadata from the CLI arguments. +func metadataStorage(c *cli.Command) (*internal.Storage, error) { + logrus.Trace("creating storage metadata from CLI configuration") + + u, err := url.Parse(c.String("storage.addr")) + if err != nil { + return nil, err + } + + return &internal.Storage{ + Driver: c.String("storage.driver"), + Host: u.Host, + }, nil +} + // helper function to capture the Vela metadata from the CLI arguments. // //nolint:unparam // ignore unparam for now diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 8c3870088..3c262b8f2 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -113,6 +113,11 @@ func server(ctx context.Context, cmd *cli.Command) error { return err } + st, err := setupStorage(ctx, cmd) + if err != nil { + return err + } + metadata, err := setupMetadata(cmd) if err != nil { return err @@ -197,6 +202,7 @@ func server(ctx context.Context, cmd *cli.Command) error { middleware.Secret(cmd.String("vela-secret")), middleware.Secrets(secrets), middleware.Scm(scm), + middleware.Storage(st), middleware.QueueSigningPrivateKey(cmd.String("queue.private-key")), middleware.QueueSigningPublicKey(cmd.String("queue.public-key")), middleware.QueueAddress(cmd.String("queue.addr")), @@ -213,6 +219,7 @@ func server(ctx context.Context, cmd *cli.Command) error { middleware.ScheduleFrequency(cmd.Duration("schedule-minimum-frequency")), middleware.TracingClient(tc), middleware.TracingInstrumentation(tc), + middleware.StorageEnable(cmd.Bool("storage.enable")), ) addr, err := url.Parse(cmd.String("server-addr")) diff --git a/cmd/vela-server/storage.go b/cmd/vela-server/storage.go new file mode 100644 index 000000000..3c6c0ce97 --- /dev/null +++ b/cmd/vela-server/storage.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" + + "github.com/go-vela/server/storage" +) + +func setupStorage(_ context.Context, c *cli.Command) (storage.Storage, error) { + logrus.Debug("creating storage client from CLI configuration") + + if !c.Bool("storage.enable") { + logrus.Debug("storage is not enabled from CLI configuration") + + return nil, nil + } + // storage configuration + _setup := &storage.Setup{ + Enable: c.Bool("storage.enable"), + Driver: c.String("storage.driver"), + Endpoint: c.String("storage.addr"), + AccessKey: c.String("storage.access.key"), + SecretKey: c.String("storage.secret.key"), + Bucket: c.String("storage.bucket.name"), + Secure: c.Bool("storage.use.ssl"), + } + // setup the storage + // + // https://pkg.go.dev/github.com/go-vela/server/storage#New + return storage.New(_setup) +} diff --git a/compiler/native/validate_test.go b/compiler/native/validate_test.go index 0897191f9..29ed8747b 100644 --- a/compiler/native/validate_test.go +++ b/compiler/native/validate_test.go @@ -743,7 +743,35 @@ func TestNative_Validate_Steps_StepNameConflict(t *testing.T) { t.Errorf("Validate should have returned err") } } +func TestNative_Validate_Artifact(t *testing.T) { + // setup types + str := "foo" + p := &yaml.Build{ + Version: "v1", + Steps: yaml.StepSlice{ + &yaml.Step{ + Commands: raw.StringSlice{"echo hello"}, + Image: "alpine", + Name: str, + Pull: "always", + Artifacts: yaml.Artifacts{ + Paths: []string{"results.xml", "artifacts.png"}, + }, + }, + }, + } + + // run test + compiler, err := FromCLICommand(context.Background(), testCommand(t, "http://foo.example.com")) + if err != nil { + t.Errorf("Unable to create new compiler: %v", err) + } + err = compiler.ValidateYAML(p) + if err != nil { + t.Errorf("Validate returned err: %v", err) + } +} func TestNative_Validate_Secrets_SecretOriginNameConflict(t *testing.T) { // setup types str := "foo" diff --git a/compiler/types/pipeline/artifact.go b/compiler/types/pipeline/artifact.go new file mode 100644 index 000000000..0f9f11eb9 --- /dev/null +++ b/compiler/types/pipeline/artifact.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pipeline + +// ArtifactSlice is the pipeline representation +// of a slice of artifacts. +// +// swagger:model PipelineArtifactSlice +type ArtifactSlice []*Artifacts + +// Artifact is the pipeline representation +// of artifacts for a pipeline. +// +// swagger:model PipelineArtifact +type Artifacts struct { + Paths []string `yaml:"paths,omitempty" json:"paths,omitempty"` +} + +// Empty returns true if the provided Artifact is empty. +func (a *Artifacts) Empty() bool { + // return true if paths field is empty + if len(a.Paths) == 0 { + return true + } + + // return false if Paths are provided + return false +} diff --git a/compiler/types/pipeline/artifact_test.go b/compiler/types/pipeline/artifact_test.go new file mode 100644 index 000000000..bb0ae397f --- /dev/null +++ b/compiler/types/pipeline/artifact_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pipeline + +import "testing" + +func TestPipeline_Artifacts_Empty(t *testing.T) { + // setup tests + tests := []struct { + artifacts *Artifacts + want bool + }{ + { + artifacts: &Artifacts{Paths: []string{"foo"}}, + want: false, + }, + { + artifacts: new(Artifacts), + want: true, + }, + } + + // run tests + for _, test := range tests { + got := test.artifacts.Empty() + + if got != test.want { + t.Errorf("Empty is %v, want %t", got, test.want) + } + } +} diff --git a/compiler/types/pipeline/container.go b/compiler/types/pipeline/container.go index 34a10dad1..e83b17c95 100644 --- a/compiler/types/pipeline/container.go +++ b/compiler/types/pipeline/container.go @@ -48,6 +48,7 @@ type ( Pull string `json:"pull,omitempty" yaml:"pull,omitempty"` Ruleset Ruleset `json:"ruleset,omitempty" yaml:"ruleset,omitempty"` Secrets StepSecretSlice `json:"secrets,omitempty" yaml:"secrets,omitempty"` + Artifacts Artifacts `json:"artifacts,omitempty" yaml:"artifacts,omitempty"` Ulimits UlimitSlice `json:"ulimits,omitempty" yaml:"ulimits,omitempty"` Volumes VolumeSlice `json:"volumes,omitempty" yaml:"volumes,omitempty"` User string `json:"user,omitempty" yaml:"user,omitempty"` @@ -140,7 +141,8 @@ func (c *Container) Empty() bool { len(c.Volumes) == 0 && len(c.User) == 0 && len(c.ReportAs) == 0 && - len(c.IDRequest) == 0 { + len(c.IDRequest) == 0 && + reflect.DeepEqual(c.Artifacts, Artifacts{}) { return true } diff --git a/compiler/types/yaml/artifacts.go b/compiler/types/yaml/artifacts.go new file mode 100644 index 000000000..5bacf657c --- /dev/null +++ b/compiler/types/yaml/artifacts.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" +) + +// Artifacts represents the structure for artifacts configuration. +type Artifacts struct { + Paths raw.StringSlice `yaml:"paths,omitempty" json:"paths,omitempty"` +} + +// ToPipeline converts the Artifact type +// to a pipeline Artifact type. +func (a *Artifacts) ToPipeline() *pipeline.Artifacts { + return &pipeline.Artifacts{ + Paths: a.Paths, + } +} diff --git a/compiler/types/yaml/build.go b/compiler/types/yaml/build.go index 8b9a25a44..a4b7494f7 100644 --- a/compiler/types/yaml/build.go +++ b/compiler/types/yaml/build.go @@ -33,6 +33,7 @@ func (b *Build) ToPipelineAPI() *api.Pipeline { pipeline.SetStages(len(b.Stages) > 0) pipeline.SetSteps(len(b.Steps) > 0) pipeline.SetTemplates(len(b.Templates) > 0) + pipeline.SetArtifact(b.hasArtifacts()) // set default for external and internal secrets external := false @@ -62,6 +63,28 @@ func (b *Build) ToPipelineAPI() *api.Pipeline { return pipeline } +func (b *Build) hasArtifacts() bool { + for _, step := range b.Steps { + if step != nil && len(step.Artifacts.Paths) > 0 { + return true + } + } + + for _, stage := range b.Stages { + if stage == nil { + continue + } + + for _, step := range stage.Steps { + if step != nil && len(step.Artifacts.Paths) > 0 { + return true + } + } + } + + return false +} + // UnmarshalYAML implements the Unmarshaler interface for the Build type. func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { // build we try unmarshalling to diff --git a/compiler/types/yaml/build_test.go b/compiler/types/yaml/build_test.go index 737a77fa4..84353473e 100644 --- a/compiler/types/yaml/build_test.go +++ b/compiler/types/yaml/build_test.go @@ -25,6 +25,7 @@ func TestYaml_Build_ToAPI(t *testing.T) { build.SetStages(false) build.SetSteps(true) build.SetTemplates(true) + build.SetArtifact(false) stages := new(api.Pipeline) stages.SetFlavor("") @@ -36,6 +37,7 @@ func TestYaml_Build_ToAPI(t *testing.T) { stages.SetStages(true) stages.SetSteps(false) stages.SetTemplates(false) + stages.SetArtifact(false) steps := new(api.Pipeline) steps.SetFlavor("") @@ -47,6 +49,19 @@ func TestYaml_Build_ToAPI(t *testing.T) { steps.SetStages(false) steps.SetSteps(true) steps.SetTemplates(false) + steps.SetArtifact(false) + + artifacts := new(api.Pipeline) + artifacts.SetFlavor("") + artifacts.SetPlatform("") + artifacts.SetVersion("1") + artifacts.SetExternalSecrets(false) + artifacts.SetInternalSecrets(false) + artifacts.SetServices(false) + artifacts.SetStages(false) + artifacts.SetSteps(true) + artifacts.SetTemplates(false) + artifacts.SetArtifact(true) // setup tests tests := []struct { @@ -69,6 +84,11 @@ func TestYaml_Build_ToAPI(t *testing.T) { file: "testdata/build_anchor_step.yml", want: steps, }, + { + name: "artifacts", + file: "testdata/build_artifacts.yml", + want: artifacts, + }, } // run tests diff --git a/compiler/types/yaml/secret.go b/compiler/types/yaml/secret.go index bd2b2bb04..37f408fb7 100644 --- a/compiler/types/yaml/secret.go +++ b/compiler/types/yaml/secret.go @@ -158,7 +158,7 @@ func (o *Origin) Empty() bool { // MergeEnv takes a list of environment variables and attempts // to set them in the secret environment. If the environment -// variable already exists in the secret, than this will +// variable already exists in the secret, then this will // overwrite the existing environment variable. func (o *Origin) MergeEnv(environment map[string]string) error { // check if the secret container is empty diff --git a/compiler/types/yaml/step.go b/compiler/types/yaml/step.go index 405e6d598..d588222d7 100644 --- a/compiler/types/yaml/step.go +++ b/compiler/types/yaml/step.go @@ -24,6 +24,7 @@ type ( Entrypoint raw.StringSlice `yaml:"entrypoint,omitempty" json:"entrypoint,omitempty" jsonschema:"description=Command to execute inside the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-entrypoint-key"` Secrets StepSecretSlice `yaml:"secrets,omitempty" json:"secrets,omitempty" jsonschema:"description=Sensitive variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-secrets-key"` Template StepTemplate `yaml:"template,omitempty" json:"template,omitempty" jsonschema:"oneof_required=template,description=Name of template to expand in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-template-key"` + Artifacts Artifacts `yaml:"artifacts,omitempty" json:"artifacts,omitempty" jsonschema:"description=Artifacts configuration for the step.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-artifacts-key"` Ulimits UlimitSlice `yaml:"ulimits,omitempty" json:"ulimits,omitempty" jsonschema:"description=Set the user limits for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` Volumes VolumeSlice `yaml:"volumes,omitempty" json:"volumes,omitempty" jsonschema:"description=Mount volumes for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` Image string `yaml:"image,omitempty" json:"image,omitempty" jsonschema:"oneof_required=image,minLength=1,description=Docker image to use to create the ephemeral container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-image-key"` @@ -59,6 +60,7 @@ func (s *StepSlice) ToPipeline() *pipeline.ContainerSlice { Pull: step.Pull, Ruleset: *step.Ruleset.ToPipeline(), Secrets: *step.Secrets.ToPipeline(), + Artifacts: *step.Artifacts.ToPipeline(), Ulimits: *step.Ulimits.ToPipeline(), Volumes: *step.Volumes.ToPipeline(), User: step.User, diff --git a/compiler/types/yaml/step_test.go b/compiler/types/yaml/step_test.go index b570c0c7d..e4c1901a6 100644 --- a/compiler/types/yaml/step_test.go +++ b/compiler/types/yaml/step_test.go @@ -77,6 +77,9 @@ func TestYaml_StepSlice_ToPipeline(t *testing.T) { AccessMode: "ro", }, }, + Artifacts: Artifacts{ + Paths: []string{"test-results/*.xml", "screenshots/**/*.png", " video/*.mp4"}, + }, }, }, want: &pipeline.ContainerSlice{ @@ -136,6 +139,9 @@ func TestYaml_StepSlice_ToPipeline(t *testing.T) { AccessMode: "ro", }, }, + Artifacts: pipeline.Artifacts{ + Paths: []string{"test-results/*.xml", "screenshots/**/*.png", " video/*.mp4"}, + }, }, }, }, @@ -215,6 +221,14 @@ func TestYaml_StepSlice_UnmarshalYAML(t *testing.T) { }, }, }, + { + Name: "artifact", + Image: "golang:1.20", + Pull: "always", + Artifacts: Artifacts{ + Paths: []string{"test-results/*.xml", "screenshots/**/*.png", " video/*.mp4"}, + }, + }, }, }, { diff --git a/compiler/types/yaml/testdata/build_artifacts.yml b/compiler/types/yaml/testdata/build_artifacts.yml new file mode 100644 index 000000000..f3969b163 --- /dev/null +++ b/compiler/types/yaml/testdata/build_artifacts.yml @@ -0,0 +1,13 @@ +--- +version: '1' + +steps: + - name: test + image: alpine:latest + commands: + - echo "artifact test" + artifacts: + paths: + - 'test-results.xml' + - 'coverage.html' + - 'junit-report.json' diff --git a/compiler/types/yaml/testdata/step.yml b/compiler/types/yaml/testdata/step.yml index 1d6d9cc93..a115689f1 100644 --- a/compiler/types/yaml/testdata/step.yml +++ b/compiler/types/yaml/testdata/step.yml @@ -43,4 +43,10 @@ vars: registry: index.docker.io repo: github/octocat - tags: [ latest, dev ] + tags: [latest, dev] + +- name: artifact + image: golang:1.20 + pull: true + artifacts: + paths: ['test-results/*.xml', 'screenshots/**/*.png', ' video/*.mp4'] diff --git a/constants/driver.go b/constants/driver.go index e42b924c5..0b06bd4e5 100644 --- a/constants/driver.go +++ b/constants/driver.go @@ -62,3 +62,9 @@ const ( // DriverGitLab defines the driver type when integrating with a Gitlab source code system. DriverGitlab = "gitlab" ) + +// Server storage drivers. +const ( + // DriverMinio defines the driver type when integrating with a local storage system. + DriverMinio = "minio" +) diff --git a/constants/table.go b/constants/table.go index cd1000469..fe0b73d88 100644 --- a/constants/table.go +++ b/constants/table.go @@ -31,6 +31,9 @@ const ( // TableRepo defines the table type for the database repos table. TableRepo = "repos" + // TableArtifact defines the table type for the database artifacts table. + TableArtifact = "artifacts" + // TableSchedule defines the table type for the database schedules table. TableSchedule = "schedules" diff --git a/database/integration_test.go b/database/integration_test.go index 519aba166..5367517fd 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -3125,6 +3125,7 @@ func newResources() *Resources { pipelineOne.SetStages(false) pipelineOne.SetSteps(true) pipelineOne.SetTemplates(false) + pipelineOne.SetArtifact(false) pipelineOne.SetWarnings([]string{}) pipelineOne.SetData([]byte("version: 1")) @@ -3143,6 +3144,7 @@ func newResources() *Resources { pipelineTwo.SetStages(false) pipelineTwo.SetSteps(true) pipelineTwo.SetTemplates(false) + pipelineTwo.SetArtifact(false) pipelineTwo.SetWarnings([]string{"42:this is a warning"}) pipelineTwo.SetData([]byte("version: 1")) diff --git a/database/pipeline/create_test.go b/database/pipeline/create_test.go index 8a7d1a82a..7f23f6ae7 100644 --- a/database/pipeline/create_test.go +++ b/database/pipeline/create_test.go @@ -35,9 +35,9 @@ func TestPipeline_Engine_CreatePipeline(t *testing.T) { // ensure the mock expects the query _mock.ExpectQuery(`INSERT INTO "pipelines" -("repo_id","commit","flavor","platform","ref","type","version","external_secrets","internal_secrets","services","stages","steps","templates","warnings","data","id") -VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) RETURNING "id"`). - WithArgs(1, "48afb5bdc41ad69bf22588491333f7cf71135163", nil, nil, "refs/heads/main", "yaml", "1", false, false, false, false, false, false, nil, AnyArgument{}, 1). +("repo_id","commit","flavor","platform","ref","type","version","external_secrets","internal_secrets","services","stages","steps","templates","artifacts","warnings","data","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17) RETURNING "id"`). + WithArgs(1, "48afb5bdc41ad69bf22588491333f7cf71135163", nil, nil, "refs/heads/main", "yaml", "1", false, false, false, false, false, false, false, nil, AnyArgument{}, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/pipeline/table.go b/database/pipeline/table.go index 93ea90b30..c24212da6 100644 --- a/database/pipeline/table.go +++ b/database/pipeline/table.go @@ -28,6 +28,7 @@ pipelines ( stages BOOLEAN, steps BOOLEAN, templates BOOLEAN, + artifacts BOOLEAN, warnings VARCHAR(5000), data BYTEA, UNIQUE(repo_id, commit) @@ -53,6 +54,7 @@ pipelines ( stages BOOLEAN, steps BOOLEAN, templates BOOLEAN, + artifacts BOOLEAN, warnings TEXT, data BLOB, UNIQUE(repo_id, 'commit') diff --git a/database/pipeline/update_test.go b/database/pipeline/update_test.go index 878fe66c0..9e2623ffc 100644 --- a/database/pipeline/update_test.go +++ b/database/pipeline/update_test.go @@ -32,9 +32,9 @@ func TestPipeline_Engine_UpdatePipeline(t *testing.T) { // ensure the mock expects the query _mock.ExpectExec(`UPDATE "pipelines" -SET "repo_id"=$1,"commit"=$2,"flavor"=$3,"platform"=$4,"ref"=$5,"type"=$6,"version"=$7,"external_secrets"=$8,"internal_secrets"=$9,"services"=$10,"stages"=$11,"steps"=$12,"templates"=$13,"warnings"=$14,"data"=$15 -WHERE "id" = $16`). - WithArgs(1, "48afb5bdc41ad69bf22588491333f7cf71135163", nil, nil, "refs/heads/main", "yaml", "1", false, false, false, false, false, false, nil, AnyArgument{}, 1). +SET "repo_id"=$1,"commit"=$2,"flavor"=$3,"platform"=$4,"ref"=$5,"type"=$6,"version"=$7,"external_secrets"=$8,"internal_secrets"=$9,"services"=$10,"stages"=$11,"steps"=$12,"templates"=$13,"artifacts"=$14,"warnings"=$15,"data"=$16 +WHERE "id" = $17`). + WithArgs(1, "48afb5bdc41ad69bf22588491333f7cf71135163", nil, nil, "refs/heads/main", "yaml", "1", false, false, false, false, false, false, false, nil, AnyArgument{}, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/resource_test.go b/database/resource_test.go index c852c8bf0..5f68b4844 100644 --- a/database/resource_test.go +++ b/database/resource_test.go @@ -83,6 +83,7 @@ func TestDatabase_Engine_NewResources(t *testing.T) { // create a test database without mocking the call _unmocked, _ := testPostgres(t) + defer _unmocked.Close() _sqlite := testSqlite(t) defer _sqlite.Close() diff --git a/database/testutils/api_resources.go b/database/testutils/api_resources.go index e0b590b92..ce3c8a164 100644 --- a/database/testutils/api_resources.go +++ b/database/testutils/api_resources.go @@ -275,6 +275,7 @@ func APIPipeline() *api.Pipeline { Stages: new(bool), Steps: new(bool), Templates: new(bool), + Artifact: new(bool), Warnings: new([]string), Data: new([]byte), } diff --git a/database/types/pipeline.go b/database/types/pipeline.go index f5edfd7c7..cf0c4200d 100644 --- a/database/types/pipeline.go +++ b/database/types/pipeline.go @@ -55,6 +55,7 @@ type Pipeline struct { Stages sql.NullBool `sql:"stages"` Steps sql.NullBool `sql:"steps"` Templates sql.NullBool `sql:"templates"` + Artifacts sql.NullBool `sql:"artifacts"` Warnings pq.StringArray `sql:"warnings" gorm:"type:varchar(5000)"` Data []byte `sql:"data"` @@ -168,6 +169,7 @@ func (p *Pipeline) ToAPI() *api.Pipeline { pipeline.SetStages(p.Stages.Bool) pipeline.SetSteps(p.Steps.Bool) pipeline.SetTemplates(p.Templates.Bool) + pipeline.SetArtifact(p.Artifacts.Bool) pipeline.SetWarnings(p.Warnings) pipeline.SetData(p.Data) @@ -246,6 +248,7 @@ func PipelineFromAPI(p *api.Pipeline) *Pipeline { Stages: sql.NullBool{Bool: p.GetStages(), Valid: true}, Steps: sql.NullBool{Bool: p.GetSteps(), Valid: true}, Templates: sql.NullBool{Bool: p.GetTemplates(), Valid: true}, + Artifacts: sql.NullBool{Bool: p.GetArtifact(), Valid: true}, Warnings: pq.StringArray(p.GetWarnings()), Data: p.GetData(), } diff --git a/database/types/pipeline_test.go b/database/types/pipeline_test.go index 2a03afff1..b78c99243 100644 --- a/database/types/pipeline_test.go +++ b/database/types/pipeline_test.go @@ -285,6 +285,7 @@ func TestDatabase_Pipeline_ToAPI(t *testing.T) { want.SetStages(false) want.SetSteps(true) want.SetTemplates(false) + want.SetArtifact(false) want.SetWarnings([]string{"42:this is a warning"}) want.SetData(testPipelineData()) @@ -394,6 +395,7 @@ func TestDatabase_PipelineFromAPI(t *testing.T) { Stages: sql.NullBool{Bool: false, Valid: true}, Steps: sql.NullBool{Bool: true, Valid: true}, Templates: sql.NullBool{Bool: false, Valid: true}, + Artifacts: sql.NullBool{Bool: false, Valid: true}, Warnings: []string{"42:this is a warning"}, Data: testPipelineData(), } @@ -414,6 +416,7 @@ func TestDatabase_PipelineFromAPI(t *testing.T) { p.SetStages(false) p.SetSteps(true) p.SetTemplates(false) + p.SetArtifact(false) p.SetWarnings([]string{"42:this is a warning"}) p.SetData(testPipelineData()) @@ -443,6 +446,7 @@ func testPipeline() *Pipeline { Stages: sql.NullBool{Bool: false, Valid: true}, Steps: sql.NullBool{Bool: true, Valid: true}, Templates: sql.NullBool{Bool: false, Valid: true}, + Artifacts: sql.NullBool{Bool: false, Valid: true}, Warnings: []string{"42:this is a warning"}, Data: testPipelineData(), diff --git a/docker-compose.yml b/docker-compose.yml index c6ffb64c9..a1acfc0ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,13 @@ services: VELA_OTEL_TRACING_ENABLE: true VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100 + VELA_STORAGE_ENABLE: true + VELA_STORAGE_DRIVER: minio + VELA_STORAGE_ADDRESS: 'http://minio:9001' # Address of the MinIO server + VELA_STORAGE_ACCESS_KEY: minioadmin + VELA_STORAGE_SECRET_KEY: minioadmin + VELA_STORAGE_USE_SSL: 'false' + VELA_STORAGE_BUCKET: vela env_file: - .env restart: always @@ -86,6 +93,7 @@ services: VELA_SERVER_SECRET: 'zB7mrKDTZqNeNTD8z47yG4DHywspAh' WORKER_ADDR: 'http://worker:8080' WORKER_CHECK_IN: 2m + VELA_EXECUTOR_OUTPUTS_IMAGE: 'alpine:latest' restart: always ports: - '8081:8080' @@ -156,18 +164,18 @@ services: # # https://www.vaultproject.io/ vault: - image: hashicorp/vault:latest - container_name: vault - command: server -dev - networks: - - vela - environment: - VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 - VAULT_DEV_ROOT_TOKEN_ID: vela - ports: - - '8200:8200' - cap_add: - - IPC_LOCK + image: hashicorp/vault:latest + container_name: vault + command: server -dev + networks: + - vela + environment: + VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 + VAULT_DEV_ROOT_TOKEN_ID: vela + ports: + - '8200:8200' + cap_add: + - IPC_LOCK jaeger: image: jaegertracing/all-in-one:latest @@ -180,5 +188,19 @@ services: - '16686:16686' - '4318:4318' + minio: + container_name: minio + image: minio/minio + restart: always + ports: + - '9001:9001' + - '9002:9002' + networks: + - vela + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + command: minio server --address ":9001" --console-address ":9002" /data + networks: vela: diff --git a/go.mod b/go.mod index 66b9097ea..5582ddb07 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/lestrrat-go/jwx/v3 v3.0.10 github.com/lib/pq v1.10.9 github.com/microcosm-cc/bluemonday v1.0.27 + github.com/minio/minio-go/v7 v7.0.83 github.com/prometheus/client_golang v1.23.0 github.com/redis/go-redis/v9 v9.12.1 github.com/sirupsen/logrus v1.9.3 @@ -82,12 +83,15 @@ require ( github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -114,6 +118,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect @@ -124,6 +129,7 @@ require ( github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.28 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -133,9 +139,11 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.65.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect diff --git a/go.sum b/go.sum index 0bda9e7a3..b7981a5a9 100644 --- a/go.sum +++ b/go.sum @@ -78,8 +78,9 @@ github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCy github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= @@ -88,6 +89,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/expr-lang/expr v1.17.7 h1:Q0xY/e/2aCIp8g9s/LGvMDCC5PxYlvHgDZRQ4y16JX8= github.com/expr-lang/expr v1.17.7/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -106,6 +109,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0FdY= github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -200,6 +205,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -238,6 +244,10 @@ github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEu github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.83 h1:W4Kokksvlz3OKf3OqIlzDNKd4MERlC2oN8YptwJ0+GA= +github.com/minio/minio-go/v7 v7.0.83/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -258,8 +268,9 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -272,6 +283,8 @@ github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgv github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= diff --git a/internal/metadata.go b/internal/metadata.go index 92616ed83..eb92e1508 100644 --- a/internal/metadata.go +++ b/internal/metadata.go @@ -23,6 +23,12 @@ type ( Host string `json:"host"` } + // Storage is the extra set of Storage data passed to the compiler. + Storage struct { + Driver string `json:"driver"` + Host string `json:"host"` + } + // Vela is the extra set of Vela data passed to the compiler. Vela struct { Address string `json:"address"` @@ -41,5 +47,6 @@ type ( Queue *Queue `json:"queue"` Source *Source `json:"source"` Vela *Vela `json:"vela"` + Storage *Storage `json:"storage"` } ) diff --git a/mock/server/pipeline.go b/mock/server/pipeline.go index d3723fad2..4544c8041 100644 --- a/mock/server/pipeline.go +++ b/mock/server/pipeline.go @@ -169,6 +169,7 @@ templates: "stages": false, "steps": true, "templates": false, + "artifacts": false, "warnings": [ "42:this is a warning" ], @@ -244,6 +245,7 @@ templates: "stages": false, "steps": true, "templates": false, + "artifacts": false, "data": "LS0tCnZlcnNpb246ICIxIgoKc3RlcHM6CiAgLSBuYW1lOiBlY2hvCiAgICBpbWFnZTogYWxwaW5lOmxhdGVzdAogICAgY29tbWFuZHM6IFtlY2hvIGZvb10=" }, { @@ -313,6 +315,7 @@ templates: "stages": false, "steps": true, "templates": false, + "artifacts": false, "data": "LS0tCnZlcnNpb246ICIxIgoKc3RlcHM6CiAgLSBuYW1lOiBlY2hvCiAgICBpbWFnZTogYWxwaW5lOmxhdGVzdAogICAgY29tbWFuZHM6IFtlY2hvIGZvb10=" } ]` diff --git a/mock/server/server.go b/mock/server/server.go index 0d17c368f..e4d4da532 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -159,5 +159,8 @@ func FakeHandler() http.Handler { // mock endpoint for queue credentials e.GET("/api/v1/queue/info", getQueueCreds) + // mock endpoint for storage sts credentials + e.GET("/api/v1/repos/:org/:repo/builds/:build/storage/sts", getStorageCreds) + return e } diff --git a/mock/server/worker.go b/mock/server/worker.go index 6003d2919..70eb0c292 100644 --- a/mock/server/worker.go +++ b/mock/server/worker.go @@ -186,6 +186,13 @@ const ( "queue_public_key": "DXeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ98zmko=", "queue_address": "redis://redis:6000" }` + + // StorageSTSResp represents a JSON return for an admin requesting a storage sts creds. + StorageSTSResp = `{ + "storage_access_key": "DXeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ98zmko=", + "storage_secret_key": "DXeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ98zmko=", + "storage_address": "http://storage:9000" + }` ) // getWorkers returns mock JSON for a http GET. @@ -339,3 +346,25 @@ func getQueueCreds(c *gin.Context) { c.JSON(http.StatusCreated, body) } + +// getStorageCreds returns mock JSON for a http GET. +// Pass "" to Authorization header to test receiving a http 401 response. +func getStorageCreds(c *gin.Context) { + token := c.Request.Header.Get("Authorization") + // verify token if empty + if token == "" { + msg := "unable get storage credentials; invalid registration token" + + c.AbortWithStatusJSON(http.StatusUnauthorized, api.Error{Message: &msg}) + + return + } + + data := []byte(StorageSTSResp) + + var body api.STSCreds + + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusCreated, body) +} diff --git a/router/build.go b/router/build.go index 97cb678c5..667313544 100644 --- a/router/build.go +++ b/router/build.go @@ -43,7 +43,13 @@ import ( // POST /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs // GET /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs // PUT /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs -// DELETE /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs . +// DELETE /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs +// GET /api/v1/repos/:org/:repo/builds/:build/graph +// GET /api/v1/repos/:org/:repo/builds/:build/id_token +// GET /api/v1/repos/:org/:repo/builds/:build/id_request_token +// GET /api/v1/repos/:org/:repo/builds/:build/install_token +// GET /api/v1/repos/:org/:repo/builds/:build/storage/sts +// GET /api/v1/repos/:org/:repo/builds/:build/storage/ . func BuildHandlers(base *gin.RouterGroup) { // Builds endpoints builds := base.Group("/builds") @@ -75,6 +81,8 @@ func BuildHandlers(base *gin.RouterGroup) { // Step endpoints // * Log endpoints StepHandlers(b) + + StorageHandlers(b) } // end of build endpoints } // end of builds endpoints } diff --git a/router/middleware/pipeline/pipeline_test.go b/router/middleware/pipeline/pipeline_test.go index 7edfcc9ec..b18b7566d 100644 --- a/router/middleware/pipeline/pipeline_test.go +++ b/router/middleware/pipeline/pipeline_test.go @@ -98,6 +98,7 @@ func TestPipeline_Establish(t *testing.T) { want.SetStages(false) want.SetSteps(false) want.SetTemplates(false) + want.SetArtifact(false) want.SetWarnings([]string{}) want.SetData([]byte{}) diff --git a/router/middleware/storage.go b/router/middleware/storage.go new file mode 100644 index 000000000..c3f2a73cd --- /dev/null +++ b/router/middleware/storage.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/storage" +) + +// Storage is a middleware function that initializes the object storage and +// attaches to the context of every http.Request. +func Storage(q storage.Storage) gin.HandlerFunc { + return func(c *gin.Context) { + // attach the object storage to the context + storage.WithGinContext(c, q) + storage.ToContext(c, q) + + c.Next() + } +} + +// StorageEnable is a middleware function that sets a flag in the context +// to determined if storage is enabled. +func StorageEnable(enabled bool) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("storage-enable", enabled) + c.Next() + } +} diff --git a/router/middleware/storage_test.go b/router/middleware/storage_test.go new file mode 100644 index 000000000..dbfaa51d4 --- /dev/null +++ b/router/middleware/storage_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/storage" + "github.com/go-vela/server/storage/minio" +) + +func TestMiddleware_Storage(t *testing.T) { + // setup types + var got storage.Storage + + want, _ := minio.NewTest("", "", "", "", false) + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(Storage(want)) + engine.GET("/health", func(c *gin.Context) { + got = storage.FromGinContext(c) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("Storage returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("Storage is %v, want %v", got, want) + } +} + +func TestMiddleware_StorageEnable(t *testing.T) { + // setup types + got := false + want := true + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/health", nil) + // setup mock server + engine.Use(StorageEnable(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("storage-enable").(bool) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageEnable returned %v, want %v", resp.Code, http.StatusOK) + } + + if got != want { + t.Errorf("StorageEnable is %v, want %v", got, want) + } +} diff --git a/router/storage.go b/router/storage.go new file mode 100644 index 000000000..ce3a951f9 --- /dev/null +++ b/router/storage.go @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 + +package router + +import ( + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/api/storage" + "github.com/go-vela/server/router/middleware/perm" +) + +// StorageHandlers is a function that extends the provided base router group +// with the API handlers for storage functionality. +// +// GET /api/v1/repos/:org/:repo/builds/:build/storage/sts +// GET /api/v1/repos/:org/:repo/builds/:build/storage/. +func StorageHandlers(base *gin.RouterGroup) { + // Storage endpoints + _storage := base.Group("/storage") + { + _storage.GET("/", perm.MustRead(), storage.ListBuildObjectNames) + _storage.GET("/sts", perm.MustBuildAccess(), storage.GetSTSCreds) + } // end of storage endpoints +} diff --git a/storage/context.go b/storage/context.go new file mode 100644 index 000000000..4f215b192 --- /dev/null +++ b/storage/context.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + + "github.com/gin-gonic/gin" +) + +// key is the key used to store minio service in context. +const key = "minio" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext retrieves minio service from the context. +func FromContext(ctx context.Context) Storage { + // get minio value from context.Context + v := ctx.Value(key) + if v == nil { + return nil + } + + // cast minio value to expected Storage type + s, ok := v.(Storage) + if !ok { + return nil + } + + return s +} + +// FromGinContext retrieves the S3 Service from the gin.Context. +func FromGinContext(c *gin.Context) Storage { + // get minio value from gin.Context + // + // https://pkg.go.dev/github.com/gin-gonic/gin?tab=doc#Context.Get + v, ok := c.Get(key) + if !ok { + return nil + } + + // cast minio value to expected Service type + s, ok := v.(Storage) + if !ok { + return nil + } + + return s +} + +// ToContext adds the secret Service to this +// context if it supports the Setter interface. +func ToContext(c Setter, s Storage) { + c.Set(key, s) +} + +// WithContext adds the minio Storage to the context. +func WithContext(ctx context.Context, s Storage) context.Context { + // set the storage Service in the context.Context + // + // https://pkg.go.dev/context?tab=doc#WithValue + return context.WithValue(ctx, key, s) +} + +// WithGinContext inserts the minio Storage into the gin.Context. +func WithGinContext(c *gin.Context, s Storage) { + // set the minio Storage in the gin.Context + // + // https://pkg.go.dev/github.com/gin-gonic/gin?tab=doc#Context.Set + c.Set(key, s) +} diff --git a/storage/context_test.go b/storage/context_test.go new file mode 100644 index 000000000..025f35274 --- /dev/null +++ b/storage/context_test.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "reflect" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestExecutor_FromContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + // setup tests + tests := []struct { + context context.Context + want Storage + }{ + { + + context: context.WithValue(context.Background(), key, _service), + want: _service, + }, + { + context: context.Background(), + want: nil, + }, + { + + context: context.WithValue(context.Background(), key, "foo"), + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := FromContext(test.context) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromContext is %v, want %v", got, test.want) + } + } +} + +func TestExecutor_FromGinContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + // setup tests + tests := []struct { + context *gin.Context + value interface{} + want Storage + }{ + { + context: new(gin.Context), + value: _service, + want: _service, + }, + { + context: new(gin.Context), + value: nil, + want: nil, + }, + { + context: new(gin.Context), + value: "foo", + want: nil, + }, + } + + // run tests + for _, test := range tests { + if test.value != nil { + test.context.Set(key, test.value) + } + + got := FromGinContext(test.context) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromGinContext is %v, want %v", got, test.want) + } + } +} + +func TestExecutor_WithContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + want := context.WithValue(context.Background(), key, _service) + + // run test + got := WithContext(context.Background(), _service) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WithContext is %v, want %v", got, want) + } +} + +func TestExecutor_WithGinContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + want := new(gin.Context) + want.Set(key, _service) + + // run test + got := new(gin.Context) + WithGinContext(got, _service) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WithGinContext is %v, want %v", got, want) + } +} diff --git a/storage/flags.go b/storage/flags.go new file mode 100644 index 000000000..a0776fa77 --- /dev/null +++ b/storage/flags.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "fmt" + "strings" + + "github.com/urfave/cli/v3" +) + +var Flags = []cli.Flag{ + // STORAGE Flags + + &cli.BoolFlag{ + Name: "storage.enable", + Usage: "enable object storage", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_ENABLE"), + cli.File("vela/storage/enable"), + ), + }, + &cli.StringFlag{ + Name: "storage.driver", + Usage: "object storage driver", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_DRIVER"), + cli.EnvVar("STORAGE_DRIVER"), + cli.File("vela/storage/driver"), + ), + }, + &cli.StringFlag{ + Name: "storage.addr", + Usage: "set the storage endpoint (ex. scheme://host:port)", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_ADDRESS"), + cli.EnvVar("STORAGE_ADDRESS"), + cli.File("vela/storage/addr"), + ), + Action: func(_ context.Context, _ *cli.Command, v string) error { + // check if the storage address has a scheme + if !strings.Contains(v, "://") { + return fmt.Errorf("storage address must be fully qualified (://)") + } + + // check if the queue address has a trailing slash + if strings.HasSuffix(v, "/") { + return fmt.Errorf("storage address must not have trailing slash") + } + + return nil + }, + }, + + &cli.StringFlag{ + Name: "storage.access.key", + Usage: "set storage access key", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_ACCESS_KEY"), + cli.EnvVar("STORAGE_ACCESS_KEY"), + cli.File("vela/storage/access_key"), + ), + }, + &cli.StringFlag{ + Name: "storage.secret.key", + Usage: "set storage secret key", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_SECRET_KEY"), + cli.EnvVar("STORAGE_SECRET_KEY"), + cli.File("vela/storage/secret_key"), + ), + }, + &cli.StringFlag{ + Name: "storage.bucket.name", + Usage: "set storage bucket name", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_BUCKET"), + cli.File("vela/storage/bucket"), + ), + }, + &cli.BoolFlag{ + Name: "storage.use.ssl", + Usage: "enable storage to use SSL", + Value: false, + Sources: cli.EnvVars("VELA_STORAGE_USE_SSL"), + }, +} diff --git a/storage/flags_test.go b/storage/flags_test.go new file mode 100644 index 000000000..09bef4e6c --- /dev/null +++ b/storage/flags_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "maps" + "testing" + + "github.com/urfave/cli/v3" +) + +func TestStorage_Flags(t *testing.T) { + // deep copy flags since they are global variables and will hold onto modifications during testing + deepCopyFlags := func(flags []cli.Flag) []cli.Flag { + copiedFlags := make([]cli.Flag, len(flags)) + for i, flag := range flags { + switch f := flag.(type) { + case *cli.StringFlag: + copyFlag := *f + copiedFlags[i] = ©Flag + case *cli.BoolFlag: + copyFlag := *f + copiedFlags[i] = ©Flag + default: + t.Fatalf("unsupported flag type: %T", f) + } + } + + return copiedFlags + } + + validFlags := map[string]string{ + "storage.enable": "true", + "storage.driver": "s3", + "storage.addr": "https://s3.amazonaws.com", + "storage.access.key": "test-access-key", + "storage.secret.key": "test-secret-key", + "storage.bucket.name": "test-bucket", + "storage.use.ssl": "true", + } + + // Define test cases + tests := []struct { + name string + override map[string]string + wantErr bool + }{ + { + name: "happy path", + wantErr: false, + }, + { + name: "invalid storage addr - no scheme", + override: map[string]string{ + "storage.addr": "s3.amazonaws.com", + }, + wantErr: true, + }, + { + name: "invalid storage addr - trailing slash", + override: map[string]string{ + "storage.addr": "https://s3.amazonaws.com/", + }, + wantErr: true, + }, + { + name: "valid storage addr with port", + override: map[string]string{ + "storage.addr": "https://localhost:9000", + }, + wantErr: false, + }, + { + name: "valid storage addr with http scheme", + override: map[string]string{ + "storage.addr": "http://minio.local:9000", + }, + wantErr: false, + }, + } + + // Run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Create a new command with a deep copy of the Flags slice + cmd := cli.Command{ + Name: "test", + Action: func(_ context.Context, _ *cli.Command) error { + return nil + }, + Flags: deepCopyFlags(Flags), + } + + copyMap := maps.Clone(validFlags) + + maps.Copy(copyMap, test.override) + + args := []string{"test"} + // Set command line arguments + for key, value := range copyMap { + if len(value) == 0 { + continue + } + + args = append(args, `--`+key+"="+value) + } + + // Run command + err := cmd.Run(context.Background(), args) + + // Check the result + if (err != nil) != test.wantErr { + t.Errorf("error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} diff --git a/storage/minio/assume_role.go b/storage/minio/assume_role.go new file mode 100644 index 000000000..485f691fc --- /dev/null +++ b/storage/minio/assume_role.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "fmt" + "time" + + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/api/types" +) + +func (c *Client) AssumeRole(_ context.Context, durationSeconds int, prefix, sessionName string) (*types.STSCreds, error) { + c.Logger.WithFields(logrus.Fields{ + "sessionName": sessionName, + }).Tracef("creating STS assume role credentials") + + if durationSeconds <= 0 { + durationSeconds = 900 + } + + opts := credentials.STSAssumeRoleOptions{ + AccessKey: c.GetAccessKey(), // server long-lived + SecretKey: c.GetSecretKey(), // server long-lived + RoleARN: "arn:minio:iam:::role/vela-uploader", + RoleSessionName: sessionName, + DurationSeconds: durationSeconds, + Policy: c.GetPolicy(prefix), + } + + // using GetEndpoint because STS needs full URL, not just host:port + stsCreds, err := credentials.NewSTSAssumeRole(c.GetEndpoint(), opts) + if err != nil { + return nil, fmt.Errorf("unable to assume role: %w", err) + } + + val, err := stsCreds.GetWithContext(c.client.CredContext()) + if err != nil { + return nil, fmt.Errorf("unable to get credentials: %w", err) + } + + return &types.STSCreds{ + AccessKey: val.AccessKeyID, + SecretKey: val.SecretAccessKey, + SessionToken: val.SessionToken, + ExpiresAt: time.Now().Add(time.Duration(durationSeconds) * time.Second), + Endpoint: c.GetAddress(), + Bucket: c.config.Bucket, + Secure: c.config.Secure, + }, nil +} diff --git a/storage/minio/doc.go b/storage/minio/doc.go new file mode 100644 index 000000000..e1b1afd52 --- /dev/null +++ b/storage/minio/doc.go @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio diff --git a/storage/minio/get_access_key.go b/storage/minio/get_access_key.go new file mode 100644 index 000000000..a6555232c --- /dev/null +++ b/storage/minio/get_access_key.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +// GetAccessKey returns the configured access key. +func (c *Client) GetAccessKey() string { + if c == nil || c.config == nil { + return "" + } + + return c.config.AccessKey +} diff --git a/storage/minio/get_address.go b/storage/minio/get_address.go new file mode 100644 index 000000000..9a304fa20 --- /dev/null +++ b/storage/minio/get_address.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import "net/url" + +// GetAddress returns the endpoint address for the MinIO client. +func (c *Client) GetAddress() string { + if c == nil || c.config == nil { + return "" + } + // Parse the configured endpoint to extract just the host:port + if c.config.Endpoint == "" { + return "" + } + + u, err := url.Parse(c.config.Endpoint) + if err != nil { + // If parsing fails, return the endpoint as-is + return c.config.Endpoint + } + + return u.Host +} + +// GetEndpoint returns the configured endpoint. +func (c *Client) GetEndpoint() string { + if c == nil || c.config == nil { + return "" + } + + return c.config.Endpoint +} diff --git a/storage/minio/get_address_test.go b/storage/minio/get_address_test.go new file mode 100644 index 000000000..0b8115915 --- /dev/null +++ b/storage/minio/get_address_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestMinioClient_GetAddress_ReturnsConfiguredBucket(t *testing.T) { + gin.SetMode(gin.TestMode) + + _, engine := gin.CreateTestContext(httptest.NewRecorder()) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Fatalf("failed to create minio test client: %v", err) + } + + got := client.GetAddress() + want := fake.Listener.Addr().String() + + if got != want { + t.Fatalf("GetAddress() = %q, want %q", got, want) + } +} + +func TestMinioClient_GetAddress_EmptyWhenUnset(t *testing.T) { + gin.SetMode(gin.TestMode) + + _, engine := gin.CreateTestContext(httptest.NewRecorder()) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Fatalf("failed to create minio test client: %v", err) + } + + client.config.Endpoint = "" + + got := client.GetAddress() + if got != "" { + t.Fatalf("GetAddress() = %q, want empty string", got) + } +} diff --git a/storage/minio/get_bucket.go b/storage/minio/get_bucket.go new file mode 100644 index 000000000..214551624 --- /dev/null +++ b/storage/minio/get_bucket.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +// GetBucket returns the configured bucket name. +func (c *Client) GetBucket() string { + if c == nil || c.config == nil { + return "" + } + + return c.config.Bucket +} diff --git a/storage/minio/get_bucket_test.go b/storage/minio/get_bucket_test.go new file mode 100644 index 000000000..04a265391 --- /dev/null +++ b/storage/minio/get_bucket_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestMinioClient_GetBucket_ReturnsConfiguredBucket(t *testing.T) { + gin.SetMode(gin.TestMode) + + _, engine := gin.CreateTestContext(httptest.NewRecorder()) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Fatalf("failed to create minio test client: %v", err) + } + + got := client.GetBucket() + want := "foo" + + if got != want { + t.Fatalf("GetBucket() = %q, want %q", got, want) + } +} + +func TestMinioClient_GetBucket_EmptyWhenUnset(t *testing.T) { + gin.SetMode(gin.TestMode) + + _, engine := gin.CreateTestContext(httptest.NewRecorder()) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Fatalf("failed to create minio test client: %v", err) + } + + client.config.Bucket = "" + + got := client.GetBucket() + if got != "" { + t.Fatalf("GetBucket() = %q, want empty string", got) + } +} diff --git a/storage/minio/get_policy.go b/storage/minio/get_policy.go new file mode 100644 index 000000000..74a804be2 --- /dev/null +++ b/storage/minio/get_policy.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "encoding/json" + "strings" +) + +type policyDoc struct { + Version string `json:"Version"` + Statement []statement `json:"Statement"` +} + +type statement struct { + Effect string `json:"Effect"` + Action []string `json:"Action"` + Resource []string `json:"Resource"` +} + +func (c *Client) GetPolicy(prefix string) string { + policy, err := buildPutOnlyPolicy(c.GetBucket(), prefix) + if err != nil { + c.Logger.Debugf("failed to build policy: %v", err) + return "" + } + + return policy +} + +func buildPutOnlyPolicy(bucket, prefix string) (string, error) { + // Normalize prefix + prefix = strings.TrimPrefix(prefix, "/") + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + doc := policyDoc{ + Version: "2012-10-17", + Statement: []statement{ + { + Effect: "Allow", + Action: []string{ + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + }, + Resource: []string{ + "arn:aws:s3:::" + bucket + "/" + prefix + "*", + }, + }, + }, + } + + b, err := json.Marshal(doc) + if err != nil { + return "", err + } + + return string(b), nil +} diff --git a/storage/minio/get_secret_key.go b/storage/minio/get_secret_key.go new file mode 100644 index 000000000..23feea442 --- /dev/null +++ b/storage/minio/get_secret_key.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +// GetSecretKey returns the secret key for the MinIO client. +func (c *Client) GetSecretKey() string { + if c == nil || c.config == nil { + return "" + } + + return c.config.SecretKey +} diff --git a/storage/minio/list_objects.go b/storage/minio/list_objects.go new file mode 100644 index 000000000..5ba664554 --- /dev/null +++ b/storage/minio/list_objects.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "fmt" + + "github.com/minio/minio-go/v7" + + api "github.com/go-vela/server/api/types" +) + +// ListBuildObjectNames lists the names of objects in a bucket for a specific build. +func (c *Client) ListBuildObjectNames(ctx context.Context, org, repo, build string) (map[string]string, error) { + objectsWithURLs := make(map[string]string) + // Construct the prefix path for filtering + prefix := org + "/" + repo + "/" + build + "/" + + c.Logger.Tracef("listing object names in bucket %s with prefix %s", c.config.Bucket, prefix) + + b := api.Bucket{ + BucketName: c.GetBucket(), + ListObjectsOptions: minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }, + } + + objectCh := c.client.ListObjects(ctx, c.config.Bucket, b.ListObjectsOptions) + + var objectNames []string + + for object := range objectCh { + if object.Err != nil { + return nil, object.Err + } + + objectNames = append(objectNames, object.Key) + // Generate presigned URL for each object + obj := &api.Object{ + ObjectName: object.Key, + Bucket: b, + } + + url, err := c.PresignedGetObject(ctx, obj) + if err != nil { + return nil, fmt.Errorf("failed to generate presigned URL for object %s: %w", object.Key, err) + } + + objectsWithURLs[object.Key] = url + } + + return objectsWithURLs, nil +} diff --git a/storage/minio/list_objects_test.go b/storage/minio/list_objects_test.go new file mode 100644 index 000000000..df7d1f180 --- /dev/null +++ b/storage/minio/list_objects_test.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestMinioClient_ListBuildObjectNames_Success(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.XML(http.StatusOK, gin.H{ + "bucketName": "foo", + "bucketLocation": "snowball", + "objectName": "test.xml", + }) + }) + + // Mock bucket location check + engine.GET("/foo/", func(c *gin.Context) { + if _, ok := c.GetQuery("location"); ok { + c.Data(http.StatusOK, "application/xml", []byte(`us-east-1`)) + return + } + + // Handle list objects request + prefix := c.Query("prefix") + t.Logf("ListObjects called with prefix: %s", prefix) + + xmlResponse := ` + + foo + ` + prefix + ` + 2 + 1000 + false + + octocat/hello-world/1/test.xml + 2025-03-20T19:01:40.968Z + 558677 + + octocat/hello-world/1/coverage.xml + 2025-03-20T19:02:40.968Z + 123456 + +` + + c.Data(http.StatusOK, "application/xml", []byte(xmlResponse)) + }) + + // mock stat object call + engine.HEAD("/foo/octocat/hello-world/1/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.Header("Last-Modified", "Mon, 2 Jan 2006 15:04:05 GMT") + c.XML(200, gin.H{ + "name": "test.xml", + }) + }) + // Mock presigned URL requests + engine.GET("/foo/octocat/hello-world/1/test.xml", func(c *gin.Context) { + c.Redirect(http.StatusTemporaryRedirect, "http://presigned.url/test.xml") + }) + + // mock stat object call + engine.HEAD("/foo/octocat/hello-world/1/coverage.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.Header("Last-Modified", "Mon, 2 Jan 2006 15:04:05 GMT") + c.XML(200, gin.H{ + "name": "test.xml", + }) + }) + engine.GET("/foo/octocat/hello-world/1/coverage.xml", func(c *gin.Context) { + c.Redirect(http.StatusTemporaryRedirect, "http://presigned.url/coverage.xml") + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Fatalf("Failed to create MinIO client: %v", err) + } + + results, err := client.ListBuildObjectNames(ctx, "octocat", "hello-world", "1") + if err != nil { + t.Fatalf("ListBuildObjectNames returned err: %v", err) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 results, got %d", len(results)) + } + + expectedNames := []string{ + "octocat/hello-world/1/test.xml", + "octocat/hello-world/1/coverage.xml", + } + + for _, expected := range expectedNames { + if _, found := results[expected]; !found { + t.Errorf("Expected object name %q not found in results", expected) + } + } +} + +func TestMinioClient_ListBuildObjectNames_Failure(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock bucket endpoint + engine.PUT("/foo/", func(c *gin.Context) { + c.XML(http.StatusOK, gin.H{ + "bucketName": "foo", + "bucketLocation": "snowball", + "objectName": "test.xml", + }) + }) + + // Return error for GET request + engine.GET("/foo/", func(c *gin.Context) { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal Server Error"}) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Errorf("Failed to create MinIO client: %v", err) + } + + // Run test + _, err = client.ListBuildObjectNames(ctx, "octocat", "hello-world", "1") + if err == nil { + t.Errorf("ListBuildObjectNames should have returned an error") + } +} diff --git a/storage/minio/minio.go b/storage/minio/minio.go new file mode 100644 index 000000000..6c42376d7 --- /dev/null +++ b/storage/minio/minio.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "net/url" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/sirupsen/logrus" +) + +// config holds the configuration for the MinIO client. +// +// but it is necessary for the MinIO client to function properly. +type config struct { + Enable bool + Endpoint string + AccessKey string + SecretKey string + Bucket string + Secure bool + Token string +} + +// Client implements the Storage interface using MinIO. +type Client struct { + config *config + client *minio.Client + Options *minio.Options + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + Logger *logrus.Entry +} + +// New creates a new MinIO client. +func New(endpoint string, opts ...ClientOpt) (*Client, error) { + // create new Minio client + c := new(Client) + + // create new fields + c.config = new(config) + c.Options = new(minio.Options) + + // create new logger for the client + logger := logrus.StandardLogger() + c.Logger = logrus.NewEntry(logger).WithField("minio", "minio") + + // apply all provided configuration options + for _, opt := range opts { + err := opt(c) + if err != nil { + return nil, err + } + } + + c.Options.Creds = credentials.NewStaticV4(c.config.AccessKey, c.config.SecretKey, c.config.Token) + c.Options.Secure = c.config.Secure + + urlEndpoint, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + // create the Minio client from the provided endpoint and options + minioClient, err := minio.New(urlEndpoint.Host, c.Options) + if err != nil { + return nil, err + } + + c.client = minioClient + + return c, nil +} + +// NewTest returns a Storage implementation that +// integrates with a local MinIO instance. +// +// This function is intended for running tests only. +func NewTest(endpoint, accessKey, secretKey, bucket string, secure bool) (*Client, error) { + return New(endpoint, + WithOptions(true, secure, + endpoint, accessKey, secretKey, bucket, "")) +} diff --git a/storage/minio/minio_test.go b/storage/minio/minio_test.go new file mode 100644 index 000000000..0ad6233de --- /dev/null +++ b/storage/minio/minio_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "testing" +) + +var ( + endpoint = "http://localhost:9000" + _accessKey = "minio_access_user" + _secretKey = "minio_secret_key" + _bucket = "minio_bucket" + _useSSL = false +) + +func TestMinio_New(t *testing.T) { + tests := []struct { + failure bool + endpoint string + }{ + { + failure: false, + endpoint: endpoint, + }, + { + failure: true, + endpoint: "", + }, + } + + // run tests + for _, test := range tests { + _, err := New( + test.endpoint, + WithOptions(true, _useSSL, + test.endpoint, _accessKey, _secretKey, _bucket, ""), + ) + + if test.failure { + if err == nil { + t.Errorf("New should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("New returned err: %v", err) + } + } +} diff --git a/storage/minio/opts.go b/storage/minio/opts.go new file mode 100644 index 000000000..aae928661 --- /dev/null +++ b/storage/minio/opts.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "fmt" +) + +// ClientOpt represents a configuration option to initialize the MinIO client. +type ClientOpt func(client *Client) error + +// WithOptions sets multiple options in the MinIO client. +func WithOptions(enable, secure bool, endpoint, accessKey, secretKey, bucket, token string) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring multiple options in minio client") + + if len(accessKey) == 0 { + return fmt.Errorf("no MinIO access key provided") + } + // check if the secret key provided is empty + if len(secretKey) == 0 { + return fmt.Errorf("no MinIO secret key provided") + } + // check if the bucket name provided is empty + if len(bucket) == 0 { + return fmt.Errorf("no MinIO bucket name provided") + } + // set the enable flag in the minio client + c.config.Enable = enable + // set the endpoint in the minio client + c.config.Endpoint = endpoint + // set the secret key in the minio client + c.config.SecretKey = secretKey + // set the access key in the minio client + c.config.AccessKey = accessKey + // set the secure connection mode in the minio client + c.config.Secure = secure + // set the bucket name in the minio client + c.config.Bucket = bucket + // set the token in the minio client + c.config.Token = token + + return nil + } +} diff --git a/storage/minio/opts_test.go b/storage/minio/opts_test.go new file mode 100644 index 000000000..f4e410c82 --- /dev/null +++ b/storage/minio/opts_test.go @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "testing" +) + +func TestWithAccessKey(t *testing.T) { + // setup tests + tests := []struct { + failure bool + accessKey string + want string + }{ + { + failure: false, + accessKey: "validAccessKey", + want: "validAccessKey", + }, + { + failure: true, + accessKey: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + test.accessKey, + "miniosecret", + "foo", + false) + + if test.failure { + if err == nil { + t.Errorf("WithAddress should have returned err") + } + + continue + } + + if err != nil && test.accessKey != "" { + t.Errorf("WithAccessKey returned err: %v", err) + } + + if client.config.AccessKey != test.want { + t.Errorf("WithAccessKey is %v, want %v", client.config.AccessKey, test.want) + } + } +} + +func TestWithSecretKey(t *testing.T) { + // setup tests + tests := []struct { + failure bool + secretKey string + want string + }{ + { + failure: false, + secretKey: "validSecretKey", + want: "validSecretKey", + }, + { + failure: true, + secretKey: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + "minioaccess", + test.secretKey, + "foo", + false) + + if test.failure { + if err == nil { + t.Errorf("WithSecretKey should have returned err") + } + + continue + } + + if err != nil && test.secretKey != "" { + t.Errorf("WithSecretKey returned err: %v", err) + } + + if client.config.SecretKey != test.want { + t.Errorf("WithSecretKey is %v, want %v", client.config.SecretKey, test.want) + } + } +} + +func TestWithSecure(t *testing.T) { + // setup tests + tests := []struct { + failure bool + secure bool + want bool + }{ + { + failure: false, + secure: true, + want: true, + }, + { + failure: false, + secure: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + "minioaccess", + "miniosecret", + "foo", + test.secure) + + if test.failure { + if err == nil { + t.Errorf("WithSecure should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithSecure returned err: %v", err) + } + + if client.config.Secure != test.want { + t.Errorf("WithSecure is %v, want %v", client.config.Secure, test.want) + } + } +} + +func TestWithBucket(t *testing.T) { + // setup tests + tests := []struct { + failure bool + bucket string + want string + }{ + { + failure: false, + bucket: "validBucket", + want: "validBucket", + }, + { + failure: true, + bucket: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + "minioaccess", + "miniosecret", + test.bucket, + false) + + if test.failure { + if err == nil { + t.Errorf("WithBucket should have returned err") + } + + continue + } + + if err != nil && test.bucket != "" { + t.Errorf("WithBucket returned err: %v", err) + } + + if client.config.Bucket != test.want { + t.Errorf("WithBucket is %v, want %v", client.config.Bucket, test.want) + } + } +} diff --git a/storage/minio/presigned_get_object.go b/storage/minio/presigned_get_object.go new file mode 100644 index 000000000..1dcdcf8f2 --- /dev/null +++ b/storage/minio/presigned_get_object.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "fmt" + "time" + + "github.com/minio/minio-go/v7" + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" +) + +// PresignedGetObject generates a presigned URL for downloading an object. +func (c *Client) PresignedGetObject(ctx context.Context, object *api.Object) (string, error) { + c.Logger.Tracef("generating presigned URL for object %s in bucket %s", object.ObjectName, object.Bucket.BucketName) + + var url string + // collect metadata on the object + // make sure the object exists before generating the presigned URL + objInfo, err := c.client.StatObject(ctx, object.Bucket.BucketName, object.ObjectName, minio.StatObjectOptions{}) + if objInfo.Key == "" { + logrus.Errorf("unable to get object info %s from bucket %s: %v", object.ObjectName, object.Bucket.BucketName, err) + return "", err + } + + // Generate presigned URL for downloading the object. + // The URL is valid for 2 minutes. + presignedURL, err := c.client.PresignedGetObject(ctx, object.Bucket.BucketName, object.ObjectName, 2*time.Minute, nil) + if err != nil { + return fmt.Sprintf("Unable to generate presigned URL for object %s", object.ObjectName), err + } + + url = presignedURL.String() + + return url, nil +} diff --git a/storage/minio/presigned_get_object_test.go b/storage/minio/presigned_get_object_test.go new file mode 100644 index 000000000..12508dcbc --- /dev/null +++ b/storage/minio/presigned_get_object_test.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func Test_PresignedGetObject_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // mock stat object call + engine.HEAD("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.Header("Last-Modified", "Mon, 2 Jan 2006 15:04:05 GMT") + c.XML(200, gin.H{ + "name": "test.xml", + }) + }) + // mock presigned get object call + engine.GET("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(200, gin.H{ + "bucketName": "foo", + }) + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + ctx := context.TODO() + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + url, err := client.PresignedGetObject(ctx, object) + if err != nil { + t.Errorf("PresignedGetObject returned err: %v", err) + } + + // check if URL is valid + if url == "" { + t.Errorf("PresignedGetObject returned empty URL") + } +} + +func Test_PresignedGetObject_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock presigned get object call + engine.GET("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(500, gin.H{ + "error": "Internal Server Error", + }) + c.Status(http.StatusInternalServerError) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + url, err := client.PresignedGetObject(ctx, object) + if err == nil { + t.Errorf("PresignedGetObject expected error but got none") + } + + if url != "" { + t.Errorf("PresignedGetObject returned URL when it should have failed %s", url) + } +} diff --git a/storage/minio/stat_object.go b/storage/minio/stat_object.go new file mode 100644 index 000000000..966a17699 --- /dev/null +++ b/storage/minio/stat_object.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "fmt" + + "github.com/minio/minio-go/v7" + + "github.com/go-vela/server/api/types" +) + +// StatObject retrieves the metadata of an object from the MinIO storage. +func (c *Client) StatObject(ctx context.Context, object *types.Object) (*types.Object, error) { + c.Logger.Tracef("retrieving metadata for object %s from bucket %s", object.ObjectName, object.Bucket.BucketName) + + // Get object info + info, err := c.client.StatObject(ctx, object.Bucket.BucketName, object.ObjectName, minio.StatObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("unable to get object info %s from bucket %s: %w", object.ObjectName, object.Bucket.BucketName, err) + } + + // Map MinIO object info to API object + return &types.Object{ + ObjectName: info.Key, + }, nil +} diff --git a/storage/minio/stat_object_test.go b/storage/minio/stat_object_test.go new file mode 100644 index 000000000..41a413e4d --- /dev/null +++ b/storage/minio/stat_object_test.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/minio/minio-go/v7" + + api "github.com/go-vela/server/api/types" +) + +func Test_StatObject_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock create bucket call + engine.GET("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(200, gin.H{ + "Buckets": []minio.BucketInfo{ + { + Name: "foo", + }, + }, + }) + }) + // mock stat object call + engine.HEAD("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.Header("Last-Modified", "Mon, 2 Jan 2006 15:04:05 GMT") + c.XML(200, gin.H{ + "etag": "982beba05db8083656a03f544c8c7927", + "name": "test.xml", + "lastModified": "2025-03-20T19:01:40.968Z", + "size": 558677, + "contentType": "", + "expires": time.Now(), + "metadata": "null", + "UserTagCount": 0, + "Owner": gin.H{ + "owner": gin.H{ + "Space": "http://s3.amazonaws.com/doc/2006-03-01/", + "Local": "Owner", + }, + "name": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4", + "id": "minio", + }, + "Grant": "null", + "storageClass": "STANDARD", + "IsLatest": false, + "IsDeleteMarker": false, + "VersionID": "", + "ReplicationStatus": "", + "ReplicationReady": false, + "Expiration": time.Now(), + "ExpirationRuleID": "", + "Restore": "null", + "ChecksumCRC32": "", + "ChecksumCRC32C": "", + "ChecksumSHA1": "", + "ChecksumSHA256": "", + "ChecksumCRC64NVME": "", + "Internal": "null", + }) + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + result, err := client.StatObject(ctx, object) + if err != nil { + t.Errorf("StatObject returned err: %v", err) + } + + if "test.xml" != result.ObjectName { + t.Errorf("StatObject is %v, want \"test.xml\"", result.ObjectName) + } +} + +func Test_StatObject_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock stat object call + engine.HEAD("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(500, gin.H{ + "error": "Internal Server Error", + }) + c.Status(http.StatusInternalServerError) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + result, err := client.StatObject(ctx, object) + if err == nil { + t.Errorf("StatObject should have returned err: %v", err) + } + + if result != nil { + t.Errorf("StatObject should have returned nil result, got: %v", result) + } +} diff --git a/storage/minio/test_data/create_bucket.json b/storage/minio/test_data/create_bucket.json new file mode 100644 index 000000000..b1e77da2f --- /dev/null +++ b/storage/minio/test_data/create_bucket.json @@ -0,0 +1,3 @@ +{ + "bucket_name": "foo" +} \ No newline at end of file diff --git a/storage/minio/test_data/test.xml b/storage/minio/test_data/test.xml new file mode 100644 index 000000000..66c653d49 --- /dev/null +++ b/storage/minio/test_data/test.xml @@ -0,0 +1,7 @@ + + + TEST + Upload + Reminder + Please upload me! + \ No newline at end of file diff --git a/storage/service.go b/storage/service.go new file mode 100644 index 000000000..8c3da4c84 --- /dev/null +++ b/storage/service.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + + api "github.com/go-vela/server/api/types" +) + +// Storage defines the service interface for object storage operations. +type Storage interface { + GetAddress() string + GetBucket() string + GetPolicy(string) string + StatObject(context.Context, *api.Object) (*api.Object, error) + ListBuildObjectNames(context.Context, string, string, string) (map[string]string, error) + PresignedGetObject(context.Context, *api.Object) (string, error) + AssumeRole(ctx context.Context, durationSeconds int, prefix, sessionName string) (*api.STSCreds, error) +} diff --git a/storage/setup.go b/storage/setup.go new file mode 100644 index 000000000..e47075e7d --- /dev/null +++ b/storage/setup.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "fmt" + "net/url" + + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/constants" + "github.com/go-vela/server/storage/minio" +) + +// Setup represents the configuration necessary for +// creating a Vela service capable of integrating +// with a configured S3 environment. +type Setup struct { + Enable bool + Driver string + Endpoint string + AccessKey string + SecretKey string + Bucket string + Region string + Secure bool + Token string +} + +// Minio creates and returns a Vela service capable +// of integrating with an S3 environment. +func (s *Setup) Minio() (Storage, error) { + return minio.New( + s.Endpoint, + minio.WithOptions( + s.Enable, + s.Secure, + s.Endpoint, + s.AccessKey, + s.SecretKey, + s.Bucket, + s.Token), + ) +} + +// Validate verifies the necessary fields for the +// provided configuration are populated correctly. +func (s *Setup) Validate() error { + logrus.Trace("validating Storage setup for client") + + // storage disabled: nothing to validate + if s.Enable { + if s.Driver != "" && s.Driver != constants.DriverMinio { + return fmt.Errorf("storage driver should be set to %s (got %q)", + constants.DriverMinio, s.Driver) + } + + if s.Bucket == "" { + return fmt.Errorf("storage is enabled but no bucket provided") + } + + if s.Endpoint == "" { + return fmt.Errorf("storage is enabled but no endpoint provided") + } + + if s.AccessKey == "" || s.SecretKey == "" { + return fmt.Errorf("storage is enabled but no access key or secret key provided") + } + + if _, err := url.ParseRequestURI(s.Endpoint); err != nil { + return fmt.Errorf("storage is enabled but endpoint is invalid") + } + } + + // setup is valid + return nil +} diff --git a/storage/setup_test.go b/storage/setup_test.go new file mode 100644 index 000000000..12b495a4c --- /dev/null +++ b/storage/setup_test.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "testing" + + "github.com/go-vela/server/constants" +) + +func TestSetup_Minio(t *testing.T) { + setup := &Setup{ + Enable: true, + Driver: constants.DriverMinio, + Endpoint: "http://minio.example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + Secure: true, + } + + storageClient, err := setup.Minio() + if err != nil { + t.Errorf("unable to create minio client: %v", err) + } + + if storageClient == nil { + t.Error("expected minio client, got nil") + } +} + +func TestSetup_Validate(t *testing.T) { + tests := []struct { + name string + setup *Setup + wantErr bool + }{ + { + name: "storage disabled", + setup: &Setup{ + Enable: false, + }, + wantErr: false, + }, + { + name: "valid config", + setup: &Setup{ + Enable: true, + Driver: constants.DriverMinio, + Endpoint: "http://example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + }, + wantErr: false, + }, + { + name: "missing bucket", + setup: &Setup{ + Enable: true, + Endpoint: "http://example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + }, + wantErr: true, + }, + { + name: "driver set", + setup: &Setup{ + Enable: true, + Driver: constants.DriverMinio, + Endpoint: "http://example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + }, + wantErr: false, + }, + { + name: "missing endpoint", + setup: &Setup{ + Enable: true, + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + }, + wantErr: true, + }, + { + name: "missing credentials", + setup: &Setup{ + Enable: true, + Endpoint: "http://example.com", + Bucket: "bucket-name", + }, + wantErr: true, + }, + { + name: "invalid endpoint URL", + setup: &Setup{ + Enable: true, + Endpoint: "://bad-url", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + }, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.setup.Validate() + if tc.wantErr { + if err == nil { + t.Errorf("Validate() expected error, got nil") + } + + return + } + + // success case + if err != nil { + t.Errorf("Validate() unexpected error: %v", err) + } + }) + } +} diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 000000000..bde8809ec --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/constants" +) + +// New creates and returns a Vela service capable of +// integrating with the configured storage environment. +// Currently, the following storages are supported: +// +// * minio +// . +func New(s *Setup) (Storage, error) { + // validate the setup being provided + // + // https://pkg.go.dev/github.com/go-vela/server/storage#Setup.Validate + if s.Enable { + err := s.Validate() + if err != nil { + return nil, fmt.Errorf("unable to validate storage setup: %w", err) + } + + logrus.Debug("creating storage client from setup") + // process the storage driver being provided + switch s.Driver { + case constants.DriverMinio: + // handle the storage driver being provided + // + // https://pkg.go.dev/github.com/go-vela/server/storage?tab=doc#Setup.Minio + return s.Minio() + default: + // handle an invalid storage driver being provided + return nil, fmt.Errorf("invalid storage driver provided: %s", s.Driver) + } + } + + return nil, nil +} diff --git a/storage/storage_test.go b/storage/storage_test.go new file mode 100644 index 000000000..212fab3b0 --- /dev/null +++ b/storage/storage_test.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "testing" + + "github.com/go-vela/server/constants" +) + +func TestStorage_New(t *testing.T) { + tests := []struct { + name string + failure bool + setup *Setup + }{ + { + name: "valid-minio-config", + failure: false, + setup: &Setup{ + Driver: constants.DriverMinio, + Enable: true, + Endpoint: "http://minio.example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + Secure: true, + }, + }, + { + name: "invalid-driver", + failure: true, + setup: &Setup{ + Driver: "invalid-driver", + Enable: true, + Endpoint: "http://invalid.example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + Secure: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := New(tt.setup) + + if tt.failure { + if err == nil { + t.Errorf("New() expected error, got nil") + } + + return + } + + // success case + if err != nil { + t.Errorf("New() unexpected error: %v", err) + } + }) + } +}