From 9c4996990724fa11d1f5e2f23b6b2775a20e7d65 Mon Sep 17 00:00:00 2001 From: DanKirberger Date: Thu, 9 Apr 2026 21:39:26 -0500 Subject: [PATCH] feat(storage): add secured=false option for public artifact access Adds a `secured: false` field to the artifacts block in .vela.yml pipelines that routes artifact objects under a `public/` prefix in MinIO, enabling direct (non-presigned) GET URLs for unauthenticated artifact downloads. Uploads remain worker-authenticated. Includes `--storage-public-policy` flag to optionally auto-configure bucket policy for anonymous GET on public/*. Co-authored-by: Dayton <31824+SVDEA001@users.noreply.git.target.com> --- api/storage/storage.go | 23 ++++++- cmd/vela-server/storage.go | 15 ++--- compiler/native/compile_test.go | 89 +++++++++++++++++++++++++++ compiler/native/transform_test.go | 4 ++ compiler/native/validate_test.go | 16 +++++ compiler/types/pipeline/artifact.go | 3 +- compiler/types/yaml/artifacts.go | 12 +++- compiler/types/yaml/artifacts_test.go | 54 ++++++++++++++++ compiler/types/yaml/stage_test.go | 3 + compiler/types/yaml/step.go | 2 +- compiler/types/yaml/step_test.go | 3 +- storage/flags.go | 8 +++ storage/minio/direct_url.go | 15 +++++ storage/minio/list_objects.go | 70 +++++++++++++-------- storage/minio/minio.go | 45 +++++++++++--- storage/minio/minio_test.go | 2 +- storage/minio/opts.go | 4 +- storage/service.go | 1 + storage/setup.go | 22 ++++--- 19 files changed, 330 insertions(+), 61 deletions(-) create mode 100644 compiler/types/yaml/artifacts_test.go create mode 100644 storage/minio/direct_url.go diff --git a/api/storage/storage.go b/api/storage/storage.go index 4ccd14949..6f53cc965 100644 --- a/api/storage/storage.go +++ b/api/storage/storage.go @@ -110,6 +110,11 @@ func ListBuildObjectNames(c *gin.Context) { // The URL is valid for a limited time and can be used to securely upload files directly // to the storage service without exposing credentials. // +// When the optional `secured` query parameter is set to `false`, the object is stored under +// a `public/` prefix in the bucket, making it downloadable via a direct URL without authentication. +// This requires the storage bucket to have a public-read policy configured for the `public/*` prefix. +// Defaults to `true` (authenticated presigned GET URLs) when omitted. +// // --- // produces: // - application/json @@ -135,6 +140,11 @@ func ListBuildObjectNames(c *gin.Context) { // description: Object name for the PUT URL // required: true // type: string +// - name: secured +// in: query +// description: "When false, stores the object under the public/ prefix for unauthenticated downloads. Defaults to true." +// required: false +// type: boolean // security: // - ApiKeyAuth: [] // responses: @@ -185,7 +195,18 @@ func GetPresignedPutURL(c *gin.Context) { return } - path := fmt.Sprintf("%s/%s/%d/%s", org, repoName, buildNum, objName) + // when secured=false the object is stored under the public/ prefix, making + // it accessible without authentication via a direct (non-presigned) URL. + // defaults to true (authenticated presigned GET) when the param is absent. + secured := c.Query("secured") != "false" + + var path string + if secured { + path = fmt.Sprintf("%s/%s/%d/%s", org, repoName, buildNum, objName) + } else { + path = fmt.Sprintf("public/%s/%s/%d/%s", org, repoName, buildNum, objName) + } + timeout := time.Duration(r.GetTimeout()) * time.Minute putURL, err := storage.FromGinContext(c).PresignedPutObject(c, path, timeout) diff --git a/cmd/vela-server/storage.go b/cmd/vela-server/storage.go index 3c6c0ce97..a64a28c72 100644 --- a/cmd/vela-server/storage.go +++ b/cmd/vela-server/storage.go @@ -21,13 +21,14 @@ func setupStorage(_ context.Context, c *cli.Command) (storage.Storage, error) { } // storage configuration _setup := &storage.Setup{ - Enable: c.Bool("storage.enable"), - Driver: c.String("storage.driver"), - Endpoint: c.String("storage.addr"), - AccessKey: c.String("storage.access.key"), - SecretKey: c.String("storage.secret.key"), - Bucket: c.String("storage.bucket.name"), - Secure: c.Bool("storage.use.ssl"), + Enable: c.Bool("storage.enable"), + Driver: c.String("storage.driver"), + Endpoint: c.String("storage.addr"), + AccessKey: c.String("storage.access.key"), + SecretKey: c.String("storage.secret.key"), + Bucket: c.String("storage.bucket.name"), + Secure: c.Bool("storage.use.ssl"), + PublicPolicy: c.Bool("storage.public.policy"), } // setup the storage // diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index a77faf090..f74bdc55d 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -103,6 +103,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -118,6 +119,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -135,6 +137,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { Name: "install", Number: 3, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -152,6 +155,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { Name: "test", Number: 4, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -169,6 +173,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { Name: "build", Number: 5, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -195,6 +200,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { Target: "REGISTRY_PASSWORD", }, }, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -501,6 +507,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -510,6 +517,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -520,6 +528,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { Name: "install", Number: 3, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -530,6 +539,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { Name: "test", Number: 4, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -548,6 +558,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { Matcher: "filepath", }, }, + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -567,6 +578,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { Target: "REGISTRY_PASSWORD", }, }, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, Secrets: pipeline.SecretSlice{ @@ -714,6 +726,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -729,6 +742,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -746,6 +760,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Name: "sample_install", Number: 3, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -756,6 +771,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Name: "sample_test", Number: 4, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -766,6 +782,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Name: "sample_build", Number: 5, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -792,6 +809,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Target: "REGISTRY_PASSWORD", }, }, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -960,6 +978,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -969,6 +988,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -979,6 +999,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Name: "sample_install", Number: 3, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -989,6 +1010,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Name: "sample_test", Number: 4, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -999,6 +1021,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Name: "sample_build", Number: 5, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1018,6 +1041,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Target: "REGISTRY_PASSWORD", }, }, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, Secrets: pipeline.SecretSlice{ @@ -1145,6 +1169,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1154,6 +1179,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1164,6 +1190,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi Name: "sample_hello", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -1254,6 +1281,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1263,6 +1291,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1273,6 +1302,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t Name: "inline_templatename_hello", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -1413,6 +1443,7 @@ func TestNative_Compile_Clone(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1422,6 +1453,7 @@ func TestNative_Compile_Clone(t *testing.T) { Name: "foo", Number: 2, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -1444,6 +1476,7 @@ func TestNative_Compile_Clone(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1453,6 +1486,7 @@ func TestNative_Compile_Clone(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1462,6 +1496,7 @@ func TestNative_Compile_Clone(t *testing.T) { Name: "foo", Number: 3, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -1484,6 +1519,7 @@ func TestNative_Compile_Clone(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1493,6 +1529,7 @@ func TestNative_Compile_Clone(t *testing.T) { Name: "clone", Number: 2, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1502,6 +1539,7 @@ func TestNative_Compile_Clone(t *testing.T) { Name: "foo", Number: 3, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -1597,6 +1635,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1606,6 +1645,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1615,6 +1655,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Name: "foo", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -1643,6 +1684,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1652,6 +1694,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1661,6 +1704,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Name: "foo", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -1689,6 +1733,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1698,6 +1743,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -1707,6 +1753,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Name: "foo", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -1870,6 +1917,7 @@ func TestNative_Compile_StageNameCollisionPurged(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -1885,6 +1933,7 @@ func TestNative_Compile_StageNameCollisionPurged(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -1902,6 +1951,7 @@ func TestNative_Compile_StageNameCollisionPurged(t *testing.T) { Name: "word_key_build", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2047,6 +2097,7 @@ func TestNative_Compile_StepNameCollisionPurged(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -2056,6 +2107,7 @@ func TestNative_Compile_StepNameCollisionPurged(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -2066,6 +2118,7 @@ func TestNative_Compile_StepNameCollisionPurged(t *testing.T) { Name: "build", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ ID: "", @@ -2076,6 +2129,7 @@ func TestNative_Compile_StepNameCollisionPurged(t *testing.T) { Name: "test", Number: 4, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -2536,6 +2590,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2551,6 +2606,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2568,6 +2624,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "test", Pull: "not_present", Number: 3, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2585,6 +2642,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "starlark_build_foo", Pull: "not_present", Number: 4, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2602,6 +2660,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "starlark_build_bar", Pull: "not_present", Number: 5, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2635,6 +2694,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2650,6 +2710,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2667,6 +2728,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "test", Pull: "not_present", Number: 3, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2684,6 +2746,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "nested_test", Pull: "not_present", Number: 4, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2701,6 +2764,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "nested_starlark_build_foo", Pull: "not_present", Number: 5, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2718,6 +2782,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "nested_starlark_build_bar", Pull: "not_present", Number: 6, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2747,6 +2812,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "#init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2756,6 +2822,7 @@ func Test_Compile_Inline(t *testing.T) { Image: defaultCloneImage, Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2766,6 +2833,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2776,6 +2844,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Number: 4, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2786,6 +2855,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Number: 5, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2796,6 +2866,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Number: 6, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2806,6 +2877,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Number: 7, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2816,6 +2888,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Number: 8, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -2858,6 +2931,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "#init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2867,6 +2941,7 @@ func Test_Compile_Inline(t *testing.T) { Image: defaultCloneImage, Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2877,6 +2952,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, Secrets: pipeline.SecretSlice{ @@ -2962,6 +3038,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "#init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2971,6 +3048,7 @@ func Test_Compile_Inline(t *testing.T) { Image: defaultCloneImage, Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -2981,6 +3059,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, Services: []*pipeline.Container{ @@ -3037,6 +3116,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "#init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -3046,6 +3126,7 @@ func Test_Compile_Inline(t *testing.T) { Image: defaultCloneImage, Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, { ID: "", @@ -3055,6 +3136,7 @@ func Test_Compile_Inline(t *testing.T) { Image: "alpine", Number: 3, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -3086,6 +3168,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "init", Number: 1, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -3101,6 +3184,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "clone", Number: 2, Pull: "not_present", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -3118,6 +3202,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "foo", Pull: "not_present", Number: 3, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -3135,6 +3220,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "bar", Pull: "not_present", Number: 4, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -3152,6 +3238,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "star", Pull: "not_present", Number: 5, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -3169,6 +3256,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "starlark_build_foo", Pull: "not_present", Number: 6, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -3186,6 +3274,7 @@ func Test_Compile_Inline(t *testing.T) { Name: "starlark_build_bar", Pull: "not_present", Number: 7, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index 10798d3f1..0d8b0a0a6 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -136,6 +136,7 @@ func TestNative_TransformStages(t *testing.T) { Name: "install", Number: 1, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -192,6 +193,7 @@ func TestNative_TransformStages(t *testing.T) { Name: "install", Number: 1, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -358,6 +360,7 @@ func TestNative_TransformSteps(t *testing.T) { Name: "install deps", Number: 1, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, Secrets: pipeline.SecretSlice{ @@ -409,6 +412,7 @@ func TestNative_TransformSteps(t *testing.T) { Name: "install deps", Number: 1, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, Secrets: pipeline.SecretSlice{ diff --git a/compiler/native/validate_test.go b/compiler/native/validate_test.go index f782d2e2b..2d5951a71 100644 --- a/compiler/native/validate_test.go +++ b/compiler/native/validate_test.go @@ -132,6 +132,7 @@ func TestNative_ValidatePipeline_Services(t *testing.T) { Image: "postgres", Name: str, Ports: raw.StringSlice{"8080:8080"}, + Artifacts: pipeline.Artifacts{Secured: true}, }, }, Steps: pipeline.ContainerSlice{ @@ -140,6 +141,7 @@ func TestNative_ValidatePipeline_Services(t *testing.T) { Image: "alpine", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -238,6 +240,7 @@ func TestNative_Validate_Stages(t *testing.T) { Image: "alpine", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -272,6 +275,7 @@ func TestNative_Validate_StagesSameName(t *testing.T) { Image: "alpine", Name: strFoo, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -283,6 +287,7 @@ func TestNative_Validate_StagesSameName(t *testing.T) { Image: "alpine", Name: strBar, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -439,6 +444,7 @@ func TestNative_Validate_Stages_StepNameConflict(t *testing.T) { Image: "alpine", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -450,6 +456,7 @@ func TestNative_Validate_Stages_StepNameConflict(t *testing.T) { Image: "alpine", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, }, @@ -512,6 +519,7 @@ func TestNative_Validate_Steps(t *testing.T) { Image: "alpine", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -566,6 +574,7 @@ func TestNative_Validate_Services_NameCollision(t *testing.T) { Image: "postgres", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ Environment: raw.StringSliceMap{ @@ -574,6 +583,7 @@ func TestNative_Validate_Services_NameCollision(t *testing.T) { Image: "kafka", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -655,6 +665,7 @@ func TestNative_Validate_Steps_ExceedReportAs(t *testing.T) { Name: fmt.Sprintf("%s-%d", str, i), Pull: "always", ReportAs: fmt.Sprintf("step-%d", i), + Artifacts: pipeline.Artifacts{Secured: true}, } reportSteps = append(reportSteps, reportStep) } @@ -688,6 +699,7 @@ func TestNative_Validate_MultiReportAs(t *testing.T) { Name: str, Pull: "always", ReportAs: "bar", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ Commands: raw.StringSlice{"echo hello"}, @@ -695,6 +707,7 @@ func TestNative_Validate_MultiReportAs(t *testing.T) { Name: str + "-2", Pull: "always", ReportAs: "bar", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -722,12 +735,14 @@ func TestNative_Validate_Steps_StepNameConflict(t *testing.T) { Image: "alpine", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, &pipeline.Container{ Commands: raw.StringSlice{"echo goodbye"}, Image: "alpine", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } @@ -797,6 +812,7 @@ func TestNative_Validate_Secrets_SecretOriginNameConflict(t *testing.T) { Image: "alpine", Name: str, Pull: "always", + Artifacts: pipeline.Artifacts{Secured: true}, }, }, } diff --git a/compiler/types/pipeline/artifact.go b/compiler/types/pipeline/artifact.go index 0f9f11eb9..4c214a40e 100644 --- a/compiler/types/pipeline/artifact.go +++ b/compiler/types/pipeline/artifact.go @@ -13,7 +13,8 @@ type ArtifactSlice []*Artifacts // // swagger:model PipelineArtifact type Artifacts struct { - Paths []string `yaml:"paths,omitempty" json:"paths,omitempty"` + Paths []string `yaml:"paths,omitempty" json:"paths,omitempty"` + Secured bool `yaml:"secured" json:"secured"` } // Empty returns true if the provided Artifact is empty. diff --git a/compiler/types/yaml/artifacts.go b/compiler/types/yaml/artifacts.go index 5bacf657c..5ace78c26 100644 --- a/compiler/types/yaml/artifacts.go +++ b/compiler/types/yaml/artifacts.go @@ -9,13 +9,21 @@ import ( // Artifacts represents the structure for artifacts configuration. type Artifacts struct { - Paths raw.StringSlice `yaml:"paths,omitempty" json:"paths,omitempty"` + Paths raw.StringSlice `yaml:"paths,omitempty" json:"paths,omitempty"` + Secured *bool `yaml:"secured,omitempty" json:"secured,omitempty"` } // ToPipeline converts the Artifact type // to a pipeline Artifact type. func (a *Artifacts) ToPipeline() *pipeline.Artifacts { + // default secured to true when not explicitly set + secured := true + if a.Secured != nil { + secured = *a.Secured + } + return &pipeline.Artifacts{ - Paths: a.Paths, + Paths: a.Paths, + Secured: secured, } } diff --git a/compiler/types/yaml/artifacts_test.go b/compiler/types/yaml/artifacts_test.go new file mode 100644 index 000000000..de6c85280 --- /dev/null +++ b/compiler/types/yaml/artifacts_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func boolPtr(b bool) *bool { return &b } + +func TestYaml_Artifacts_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + name string + artifacts *Artifacts + want *pipeline.Artifacts + }{ + { + name: "nil Secured defaults to true", + artifacts: &Artifacts{Paths: []string{"test-results.xml"}}, + want: &pipeline.Artifacts{Paths: []string{"test-results.xml"}, Secured: true}, + }, + { + name: "Secured explicitly true", + artifacts: &Artifacts{Paths: []string{"coverage.html"}, Secured: boolPtr(true)}, + want: &pipeline.Artifacts{Paths: []string{"coverage.html"}, Secured: true}, + }, + { + name: "Secured explicitly false", + artifacts: &Artifacts{Paths: []string{"junit-report.json"}, Secured: boolPtr(false)}, + want: &pipeline.Artifacts{Paths: []string{"junit-report.json"}, Secured: false}, + }, + { + name: "no paths", + artifacts: &Artifacts{}, + want: &pipeline.Artifacts{Paths: nil, Secured: true}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := test.artifacts.ToPipeline() + + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("ToPipeline mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/compiler/types/yaml/stage_test.go b/compiler/types/yaml/stage_test.go index d082daca5..e4317c6b1 100644 --- a/compiler/types/yaml/stage_test.go +++ b/compiler/types/yaml/stage_test.go @@ -142,6 +142,9 @@ func TestYaml_StageSlice_ToPipeline(t *testing.T) { AccessMode: "ro", }, }, + Artifacts: pipeline.Artifacts{ + Secured: true, + }, }, }, }, diff --git a/compiler/types/yaml/step.go b/compiler/types/yaml/step.go index 79f311999..cee735ac3 100644 --- a/compiler/types/yaml/step.go +++ b/compiler/types/yaml/step.go @@ -25,7 +25,7 @@ type ( Entrypoint raw.StringSlice `yaml:"entrypoint,omitempty" json:"entrypoint,omitempty" jsonschema:"description=Command to execute inside the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-entrypoint-key"` Secrets StepSecretSlice `yaml:"secrets,omitempty" json:"secrets,omitempty" jsonschema:"description=Sensitive variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-secrets-key"` Template StepTemplate `yaml:"template,omitempty" json:"template" jsonschema:"oneof_required=template,description=Name of template to expand in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-template-key"` - Artifacts Artifacts `yaml:"artifacts,omitempty" json:"artifacts" jsonschema:"description=Artifacts configuration for the step.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-artifacts-key"` + Artifacts Artifacts `yaml:"artifacts,omitempty" json:"artifacts" jsonschema:"description=Artifacts configuration for the step. Set secured: false to allow unauthenticated downloads.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-artifacts-key"` Ulimits UlimitSlice `yaml:"ulimits,omitempty" json:"ulimits,omitempty" jsonschema:"description=Set the user limits for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` Volumes VolumeSlice `yaml:"volumes,omitempty" json:"volumes,omitempty" jsonschema:"description=Mount volumes for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` Image string `yaml:"image,omitempty" json:"image,omitempty" jsonschema:"oneof_required=image,minLength=1,description=Docker image to use to create the ephemeral container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-image-key"` diff --git a/compiler/types/yaml/step_test.go b/compiler/types/yaml/step_test.go index b27b24829..84158b95d 100644 --- a/compiler/types/yaml/step_test.go +++ b/compiler/types/yaml/step_test.go @@ -152,7 +152,8 @@ func TestYaml_StepSlice_ToPipeline(t *testing.T) { }, }, Artifacts: pipeline.Artifacts{ - Paths: []string{"test-results/*.xml", "screenshots/**/*.png", " video/*.mp4"}, + Paths: []string{"test-results/*.xml", "screenshots/**/*.png", " video/*.mp4"}, + Secured: true, }, }, }, diff --git a/storage/flags.go b/storage/flags.go index a0776fa77..21bd1a769 100644 --- a/storage/flags.go +++ b/storage/flags.go @@ -85,4 +85,12 @@ var Flags = []cli.Flag{ Value: false, Sources: cli.EnvVars("VELA_STORAGE_USE_SSL"), }, + &cli.BoolFlag{ + Name: "storage.public.policy", + Usage: "automatically configure a bucket policy allowing anonymous GET on the public/* prefix, enabling unauthenticated artifact downloads when secured: false is set", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_PUBLIC_POLICY"), + cli.File("vela/storage/public_policy"), + ), + }, } diff --git a/storage/minio/direct_url.go b/storage/minio/direct_url.go new file mode 100644 index 000000000..41c5993b7 --- /dev/null +++ b/storage/minio/direct_url.go @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import "fmt" + +// DirectObjectURL returns a non-presigned direct URL for an object in the bucket. +// This is used for objects stored under the public/ prefix that are accessible +// without authentication when the bucket has a public-read policy configured +// for that prefix. +func (c *Client) DirectObjectURL(objectKey string) string { + // c.config.Endpoint is a fully-qualified URL (e.g. "http://minio:9000"), + // so we just append the bucket and object key directly. + return fmt.Sprintf("%s/%s/%s", c.config.Endpoint, c.config.Bucket, objectKey) +} diff --git a/storage/minio/list_objects.go b/storage/minio/list_objects.go index 92e649641..1532501fc 100644 --- a/storage/minio/list_objects.go +++ b/storage/minio/list_objects.go @@ -11,44 +11,60 @@ import ( api "github.com/go-vela/server/api/types" ) -// ListBuildObjectNames lists the names of objects in a bucket for a specific build. +// ListBuildObjectNames lists all artifact objects for a build, returning a map of +// object key to download URL. Objects stored under the standard prefix receive a +// 2-minute presigned GET URL (authenticated). Objects stored under the public/ prefix +// receive a direct, non-presigned URL that is accessible without authentication, +// provided the bucket has a public-read policy on the public/* prefix. func (c *Client) ListBuildObjectNames(ctx context.Context, org, repo, build string) (map[string]string, error) { objectsWithURLs := make(map[string]string) - // Construct the prefix path for filtering - prefix := org + "/" + repo + "/" + build + "/" - c.Logger.Tracef("listing object names in bucket %s with prefix %s", c.config.Bucket, prefix) - - b := api.Bucket{ - BucketName: c.config.Bucket, - ListObjectsOptions: minio.ListObjectsOptions{ - Prefix: prefix, - Recursive: true, - }, + type prefixEntry struct { + prefix string + public bool } - objectCh := c.client.ListObjects(ctx, c.config.Bucket, b.ListObjectsOptions) + prefixes := []prefixEntry{ + {prefix: fmt.Sprintf("%s/%s/%s/", org, repo, build), public: false}, + {prefix: fmt.Sprintf("public/%s/%s/%s/", org, repo, build), public: true}, + } - var objectNames []string + for _, p := range prefixes { + c.Logger.Tracef("listing object names in bucket %s with prefix %s", c.config.Bucket, p.prefix) - for object := range objectCh { - if object.Err != nil { - return nil, object.Err + opts := minio.ListObjectsOptions{ + Prefix: p.prefix, + Recursive: true, } - objectNames = append(objectNames, object.Key) - // Generate presigned URL for each object - obj := &api.Object{ - ObjectName: object.Key, - Bucket: b, - } + for object := range c.client.ListObjects(ctx, c.config.Bucket, opts) { + if object.Err != nil { + return nil, object.Err + } - url, err := c.PresignedGetObject(ctx, obj) - if err != nil { - return nil, fmt.Errorf("failed to generate presigned URL for object %s: %w", object.Key, err) - } + var ( + url string + err error + ) + + if p.public { + url = c.DirectObjectURL(object.Key) + } else { + obj := &api.Object{ + ObjectName: object.Key, + Bucket: api.Bucket{ + BucketName: c.config.Bucket, + }, + } - objectsWithURLs[object.Key] = url + url, err = c.PresignedGetObject(ctx, obj) + if err != nil { + return nil, fmt.Errorf("failed to generate presigned URL for object %s: %w", object.Key, err) + } + } + + objectsWithURLs[object.Key] = url + } } return objectsWithURLs, nil diff --git a/storage/minio/minio.go b/storage/minio/minio.go index 290012e55..96dec2646 100644 --- a/storage/minio/minio.go +++ b/storage/minio/minio.go @@ -3,6 +3,8 @@ package minio import ( + "context" + "fmt" "net/url" "github.com/minio/minio-go/v7" @@ -16,14 +18,15 @@ import ( // // but it is necessary for the MinIO client to function properly. type config struct { - Enable bool - Endpoint string - AccessKey string - SecretKey string - Bucket string - Secure bool - Token string - Driver string + Enable bool + Endpoint string + AccessKey string + SecretKey string + Bucket string + Secure bool + Token string + Driver string + PublicPolicy bool } // Client implements the Storage interface using MinIO. @@ -72,9 +75,33 @@ func New(endpoint string, opts ...ClientOpt) (*Client, error) { c.client = minioClient + if c.config.PublicPolicy { + if err := c.applyPublicPolicy(context.Background()); err != nil { + logrus.Warnf("storage: failed to apply public bucket policy: %v", err) + } + } + return c, nil } +// applyPublicPolicy sets an anonymous read-only policy on the public/* prefix of the bucket, +// allowing unauthenticated GET requests for objects stored under that prefix. +func (c *Client) applyPublicPolicy(ctx context.Context) error { + policy := fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"AWS": ["*"]}, + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::%s/public/*"] + }] +}`, c.config.Bucket) + + c.Logger.Infof("storage: applying public-read policy for public/* prefix in bucket %s", c.config.Bucket) + + return c.client.SetBucketPolicy(ctx, c.config.Bucket, policy) +} + // NewTest returns a Storage implementation that // integrates with a local MinIO instance. // @@ -82,5 +109,5 @@ func New(endpoint string, opts ...ClientOpt) (*Client, error) { func NewTest(endpoint, accessKey, secretKey, bucket string, secure bool) (*Client, error) { return New(endpoint, WithOptions(true, secure, - endpoint, accessKey, secretKey, bucket, "", constants.DriverMinio)) + endpoint, accessKey, secretKey, bucket, "", constants.DriverMinio, false)) } diff --git a/storage/minio/minio_test.go b/storage/minio/minio_test.go index 9676c85df..b1998d6ba 100644 --- a/storage/minio/minio_test.go +++ b/storage/minio/minio_test.go @@ -36,7 +36,7 @@ func TestMinio_New(t *testing.T) { _, err := New( test.endpoint, WithOptions(true, _useSSL, - test.endpoint, _accessKey, _secretKey, _bucket, "", constants.DriverMinio), + test.endpoint, _accessKey, _secretKey, _bucket, "", constants.DriverMinio, false), ) if test.failure { diff --git a/storage/minio/opts.go b/storage/minio/opts.go index d72d5aac6..9878e4ac8 100644 --- a/storage/minio/opts.go +++ b/storage/minio/opts.go @@ -10,7 +10,7 @@ import ( type ClientOpt func(client *Client) error // WithOptions sets multiple options in the MinIO client. -func WithOptions(enable, secure bool, endpoint, accessKey, secretKey, bucket, token, driver string) ClientOpt { +func WithOptions(enable, secure bool, endpoint, accessKey, secretKey, bucket, token, driver string, publicPolicy bool) ClientOpt { return func(c *Client) error { c.Logger.Trace("configuring multiple options in minio client") @@ -41,6 +41,8 @@ func WithOptions(enable, secure bool, endpoint, accessKey, secretKey, bucket, to c.config.Token = token // set the driver in the minio client c.config.Driver = driver + // set the public policy flag in the minio client + c.config.PublicPolicy = publicPolicy return nil } diff --git a/storage/service.go b/storage/service.go index 29cc4a521..53ad615cc 100644 --- a/storage/service.go +++ b/storage/service.go @@ -15,4 +15,5 @@ type Storage interface { ListBuildObjectNames(context.Context, string, string, string) (map[string]string, error) PresignedGetObject(context.Context, *api.Object) (string, error) PresignedPutObject(context.Context, string, time.Duration) (string, error) + DirectObjectURL(objectKey string) string } diff --git a/storage/setup.go b/storage/setup.go index 4d8c5222b..8a51f9902 100644 --- a/storage/setup.go +++ b/storage/setup.go @@ -16,15 +16,16 @@ import ( // creating a Vela service capable of integrating // with a configured S3 environment. type Setup struct { - Enable bool - Driver string - Endpoint string - AccessKey string - SecretKey string - Bucket string - Region string - Secure bool - Token string + Enable bool + Driver string + Endpoint string + AccessKey string + SecretKey string + Bucket string + Region string + Secure bool + Token string + PublicPolicy bool } // Minio creates and returns a Vela service capable @@ -40,7 +41,8 @@ func (s *Setup) Minio() (Storage, error) { s.SecretKey, s.Bucket, s.Token, - s.Driver), + s.Driver, + s.PublicPolicy), ) }