From b9125ea55de1cddf0e5c6f2d8056e8f18af1ed52 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 24 Feb 2022 16:40:58 -0600 Subject: [PATCH 01/11] Replace "Pulling" with "Preparing" in init step logs Pulling is a Docker-specific term that does not apply to the Kubernetes runtime. Preparing is more generic and can apply to both. --- executor/linux/build.go | 10 +++++----- executor/linux/stage.go | 2 +- executor/local/build.go | 6 +++--- executor/local/stage.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/executor/linux/build.go b/executor/linux/build.go index fa31f65d..c61734d3 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -153,7 +153,7 @@ func (c *client) PlanBuild(ctx context.Context) error { // update the init log with progress // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData - _log.AppendData([]byte("> Pulling secrets...\n")) + _log.AppendData([]byte("> Preparing secrets...\n")) // iterate through each secret provided in the pipeline for _, secret := range c.pipeline.Secrets { @@ -233,7 +233,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { // update the init log with progress // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData - _log.AppendData([]byte("> Pulling service images...\n")) + _log.AppendData([]byte("> Preparing service images...\n")) // create the services for the pipeline for _, s := range c.pipeline.Services { @@ -264,7 +264,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { // update the init log with progress // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData - _log.AppendData([]byte("> Pulling stage images...\n")) + _log.AppendData([]byte("> Preparing stage images...\n")) // create the stages for the pipeline for _, s := range c.pipeline.Stages { @@ -286,7 +286,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { // update the init log with progress // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData - _log.AppendData([]byte("> Pulling step images...\n")) + _log.AppendData([]byte("> Preparing step images...\n")) // create the steps for the pipeline for _, s := range c.pipeline.Steps { @@ -319,7 +319,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { // update the init log with progress // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData - _log.AppendData([]byte("> Pulling secret images...\n")) + _log.AppendData([]byte("> Preparing secret images...\n")) // create the secrets for the pipeline for _, s := range c.pipeline.Secrets { diff --git a/executor/linux/stage.go b/executor/linux/stage.go index 21b738f0..6618ae57 100644 --- a/executor/linux/stage.go +++ b/executor/linux/stage.go @@ -31,7 +31,7 @@ func (c *client) CreateStage(ctx context.Context, s *pipeline.Stage) error { // update the init log with progress // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData - _log.AppendData([]byte(fmt.Sprintf("> Pulling step images for stage %s...\n", s.Name))) + _log.AppendData([]byte(fmt.Sprintf("> Preparing step images for stage %s...\n", s.Name))) // create the steps for the stage for _, _step := range s.Steps { diff --git a/executor/local/build.go b/executor/local/build.go index 312ba8c8..9d32cafd 100644 --- a/executor/local/build.go +++ b/executor/local/build.go @@ -164,7 +164,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { } // output init progress to stdout - fmt.Fprintln(os.Stdout, _pattern, "> Pulling service images...") + fmt.Fprintln(os.Stdout, _pattern, "> Preparing service images...") // create the services for the pipeline for _, _service := range c.pipeline.Services { @@ -189,7 +189,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { } // output init progress to stdout - fmt.Fprintln(os.Stdout, _pattern, "> Pulling stage images...") + fmt.Fprintln(os.Stdout, _pattern, "> Preparing stage images...") // create the stages for the pipeline for _, _stage := range c.pipeline.Stages { @@ -208,7 +208,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { } // output init progress to stdout - fmt.Fprintln(os.Stdout, _pattern, "> Pulling step images...") + fmt.Fprintln(os.Stdout, _pattern, "> Preparing step images...") // create the steps for the pipeline for _, _step := range c.pipeline.Steps { diff --git a/executor/local/stage.go b/executor/local/stage.go index 16d27ee7..668580d1 100644 --- a/executor/local/stage.go +++ b/executor/local/stage.go @@ -23,7 +23,7 @@ func (c *client) CreateStage(ctx context.Context, s *pipeline.Stage) error { _pattern := fmt.Sprintf(stagePattern, c.init.Name, c.init.Name) // output init progress to stdout - fmt.Fprintln(os.Stdout, _pattern, "> Pulling step images for stage", s.Name, "...") + fmt.Fprintln(os.Stdout, _pattern, "> Preparing step images for stage", s.Name, "...") // create the steps for the stage for _, _step := range s.Steps { From 545a2ab30279e4de45f79f4756bf52b5c9505d15 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 24 Feb 2022 17:22:59 -0600 Subject: [PATCH 02/11] More image validation in kubernetes runtime --- runtime/kubernetes/container.go | 21 ++++++++++++++++++++- runtime/kubernetes/image.go | 16 +++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 752c7316..32454309 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -74,8 +74,15 @@ func (c *client) RemoveContainer(ctx context.Context, ctn *pipeline.Container) e // nolint: lll // ignore long line length func (c *client) RunContainer(ctx context.Context, ctn *pipeline.Container, b *pipeline.Build) error { c.Logger.Tracef("running container %s", ctn.ID) + // validate the container image + err := c.CreateImage(ctx, ctn) + if err != nil { + return err + } + // parse image from step - _image, err := image.ParseWithError(ctn.Image) + var _image string + _image, err = image.ParseWithError(ctn.Image) if err != nil { return err } @@ -98,6 +105,8 @@ func (c *client) RunContainer(ctx context.Context, ctn *pipeline.Container, b *p return err } + // TODO: watch k8s events for errors pulling container. + // Only return nil once the image has been pulled successfully. return nil } @@ -134,6 +143,11 @@ func (c *client) SetupContainer(ctx context.Context, ctn *pipeline.Container) er case constants.PullAlways: // set the pod container pull policy to always container.ImagePullPolicy = v1.PullAlways + // validate ctn.Image + err := c.CreateImage(ctx, ctn) + if err != nil { + return err + } case constants.PullNever: // set the pod container pull policy to never container.ImagePullPolicy = v1.PullNever @@ -149,6 +163,11 @@ func (c *client) SetupContainer(ctx context.Context, ctn *pipeline.Container) er default: // default the pod container pull policy to if not present container.ImagePullPolicy = v1.PullIfNotPresent + // validate ctn.Image + err := c.CreateImage(ctx, ctn) + if err != nil { + return err + } } // fill in the VolumeMounts including workspaceMount diff --git a/runtime/kubernetes/image.go b/runtime/kubernetes/image.go index 0fe98dd2..1e64bf4e 100644 --- a/runtime/kubernetes/image.go +++ b/runtime/kubernetes/image.go @@ -12,6 +12,7 @@ import ( "github.com/go-vela/types/constants" "github.com/go-vela/types/pipeline" + "github.com/go-vela/worker/internal/image" ) const imagePatch = ` @@ -29,8 +30,21 @@ const imagePatch = ` // CreateImage creates the pipeline container image. func (c *client) CreateImage(ctx context.Context, ctn *pipeline.Container) error { - c.Logger.Tracef("no-op: creating image for container %s", ctn.ID) + c.Logger.Tracef("creating image for container %s", ctn.ID) + // parse/validate image from container + // + // https://pkg.go.dev/github.com/go-vela/worker/internal/image#ParseWithError + _, err := image.ParseWithError(ctn.Image) + if err != nil { + return err + } + + // Kubernetes does not have an API to make sure it can access an image, + // so we have to query the appropriate docker registry ourselves. + // TODO: query docker registry for the image (if possible) + // this might require retrieving the pullSecrets from k8s + // or have the admin add a Vela accessible secret as well. return nil } From 0d9c8270091b681f32187d7c719247a3227ad12e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 25 Feb 2022 17:14:03 -0600 Subject: [PATCH 03/11] Rename watch->podWatch for planned imports --- runtime/kubernetes/container.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 32454309..671f0788 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -344,7 +344,7 @@ func (c *client) WaitContainer(ctx context.Context, ctn *pipeline.Container) err // https://pkg.go.dev/k8s.io/client-go/kubernetes/typed/core/v1?tab=doc#PodInterface // -> // https://pkg.go.dev/k8s.io/apimachinery/pkg/watch?tab=doc#Interface - watch, err := c.Kubernetes.CoreV1().Pods(c.config.Namespace).Watch(context.Background(), opts) + podWatch, err := c.Kubernetes.CoreV1().Pods(c.config.Namespace).Watch(context.Background(), opts) if err != nil { return err } @@ -353,7 +353,7 @@ func (c *client) WaitContainer(ctx context.Context, ctn *pipeline.Container) err // capture new result from the channel // // https://pkg.go.dev/k8s.io/apimachinery/pkg/watch?tab=doc#Interface - result := <-watch.ResultChan() + result := <-podWatch.ResultChan() // convert the object from the result to a pod pod, ok := result.Object.(*v1.Pod) From 2b1735be70115c528b58120646ea97bb39dba84f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 25 Feb 2022 17:26:50 -0600 Subject: [PATCH 04/11] Extract Kubernetes Client + Watch mocking logic --- runtime/kubernetes/container_test.go | 58 ++++++++++++++++------------ 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/runtime/kubernetes/container_test.go b/runtime/kubernetes/container_test.go index 07728893..11481007 100644 --- a/runtime/kubernetes/container_test.go +++ b/runtime/kubernetes/container_test.go @@ -226,34 +226,11 @@ func TestKubernetes_TailContainer(t *testing.T) { func TestKubernetes_WaitContainer(t *testing.T) { // setup types - _engine, err := NewMock(_pod) + _engine, _watch, err := newMockWithWatch(_pod, "pods") if err != nil { t.Errorf("unable to create runtime engine: %v", err) } - // create a new fake kubernetes client - // - // https://pkg.go.dev/k8s.io/client-go/kubernetes/fake?tab=doc#NewSimpleClientset - _kubernetes := fake.NewSimpleClientset(_pod) - - // create a new fake watcher - // - // https://pkg.go.dev/k8s.io/apimachinery/pkg/watch?tab=doc#NewFake - _watch := watch.NewFake() - - // create a new watch reactor with the fake watcher - // - // https://pkg.go.dev/k8s.io/client-go/testing?tab=doc#DefaultWatchReactor - reactor := testcore.DefaultWatchReactor(_watch, nil) - - // add watch reactor to beginning of the client chain - // - // https://pkg.go.dev/k8s.io/client-go/testing?tab=doc#Fake.PrependWatchReactor - _kubernetes.PrependWatchReactor("pods", reactor) - - // overwrite the mock kubernetes client - _engine.Kubernetes = _kubernetes - // setup tests tests := []struct { failure bool @@ -334,3 +311,36 @@ func TestKubernetes_WaitContainer(t *testing.T) { } } } + +func newMockWithWatch(pod *v1.Pod, watchResource string, opts ...ClientOpt) (*client, *watch.RaceFreeFakeWatcher, error) { + // setup types + _engine, err := NewMock(pod, opts...) + if err != nil { + return nil, nil, err + } + + // create a new fake kubernetes client + // + // https://pkg.go.dev/k8s.io/client-go/kubernetes/fake?tab=doc#NewSimpleClientset + _kubernetes := fake.NewSimpleClientset(pod) + + // create a new fake watcher + // + // https://pkg.go.dev/k8s.io/apimachinery/pkg/watch?tab=doc#NewRaceFreeFake + _watch := watch.NewRaceFreeFake() + + // create a new watch reactor with the fake watcher + // + // https://pkg.go.dev/k8s.io/client-go/testing?tab=doc#DefaultWatchReactor + reactor := testcore.DefaultWatchReactor(_watch, nil) + + // add watch reactor to beginning of the client chain + // + // https://pkg.go.dev/k8s.io/client-go/testing?tab=doc#Fake.PrependWatchReactor + _kubernetes.PrependWatchReactor(watchResource, reactor) + + // overwrite the mock kubernetes client + _engine.Kubernetes = _kubernetes + + return _engine, _watch, nil +} From ec5b3d1cfbfecc1a75d18d51e70dc02cbb50ce01 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 25 Feb 2022 18:19:21 -0600 Subject: [PATCH 05/11] Escape selector just in case --- runtime/kubernetes/container.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 671f0788..be635805 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -19,6 +19,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" ) @@ -331,7 +332,7 @@ func (c *client) WaitContainer(ctx context.Context, ctn *pipeline.Container) err c.Logger.Tracef("waiting for container %s", ctn.ID) // create label selector for watching the pod - selector := fmt.Sprintf("pipeline=%s", c.Pod.ObjectMeta.Name) + selector := fmt.Sprintf("pipeline=%s", fields.EscapeValue(c.Pod.ObjectMeta.Name)) // create options for watching the container opts := metav1.ListOptions{ From b3fc92b35bd0c99f48e7b61991dcef3a2c336de0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 25 Feb 2022 18:20:05 -0600 Subject: [PATCH 06/11] Move newMockWithWatch to kubernetes_test.go --- runtime/kubernetes/container_test.go | 36 ----------------------- runtime/kubernetes/kubernetes_test.go | 41 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/runtime/kubernetes/container_test.go b/runtime/kubernetes/container_test.go index 11481007..86a4553e 100644 --- a/runtime/kubernetes/container_test.go +++ b/runtime/kubernetes/container_test.go @@ -13,9 +13,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes/fake" - testcore "k8s.io/client-go/testing" ) func TestKubernetes_InspectContainer(t *testing.T) { @@ -311,36 +308,3 @@ func TestKubernetes_WaitContainer(t *testing.T) { } } } - -func newMockWithWatch(pod *v1.Pod, watchResource string, opts ...ClientOpt) (*client, *watch.RaceFreeFakeWatcher, error) { - // setup types - _engine, err := NewMock(pod, opts...) - if err != nil { - return nil, nil, err - } - - // create a new fake kubernetes client - // - // https://pkg.go.dev/k8s.io/client-go/kubernetes/fake?tab=doc#NewSimpleClientset - _kubernetes := fake.NewSimpleClientset(pod) - - // create a new fake watcher - // - // https://pkg.go.dev/k8s.io/apimachinery/pkg/watch?tab=doc#NewRaceFreeFake - _watch := watch.NewRaceFreeFake() - - // create a new watch reactor with the fake watcher - // - // https://pkg.go.dev/k8s.io/client-go/testing?tab=doc#DefaultWatchReactor - reactor := testcore.DefaultWatchReactor(_watch, nil) - - // add watch reactor to beginning of the client chain - // - // https://pkg.go.dev/k8s.io/client-go/testing?tab=doc#Fake.PrependWatchReactor - _kubernetes.PrependWatchReactor(watchResource, reactor) - - // overwrite the mock kubernetes client - _engine.Kubernetes = _kubernetes - - return _engine, _watch, nil -} diff --git a/runtime/kubernetes/kubernetes_test.go b/runtime/kubernetes/kubernetes_test.go index 45d08068..46483c7c 100644 --- a/runtime/kubernetes/kubernetes_test.go +++ b/runtime/kubernetes/kubernetes_test.go @@ -11,6 +11,11 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + testcore "k8s.io/client-go/testing" + "k8s.io/client-go/tools/clientcmd/api/latest" + "k8s.io/client-go/tools/reference" ) func TestKubernetes_New(t *testing.T) { @@ -314,3 +319,39 @@ var ( }, } ) + +// newMockWithWatch returns an Engine implementation that +// integrates with a Kubernetes runtime and a FakeWatcher +// that can be used to inject resource events into it. +func newMockWithWatch(pod *v1.Pod, watchResource string, opts ...ClientOpt) (*client, *watch.RaceFreeFakeWatcher, error) { + // setup types + _engine, err := NewMock(pod, opts...) + if err != nil { + return nil, nil, err + } + + // create a new fake kubernetes client + // + // https://pkg.go.dev/k8s.io/client-go/kubernetes/fake?tab=doc#NewSimpleClientset + _kubernetes := fake.NewSimpleClientset(pod) + + // create a new fake watcher + // + // https://pkg.go.dev/k8s.io/apimachinery/pkg/watch?tab=doc#NewRaceFreeFake + _watch := watch.NewRaceFreeFake() + + // create a new watch reactor with the fake watcher + // + // https://pkg.go.dev/k8s.io/client-go/testing?tab=doc#DefaultWatchReactor + reactor := testcore.DefaultWatchReactor(_watch, nil) + + // add watch reactor to beginning of the client chain + // + // https://pkg.go.dev/k8s.io/client-go/testing?tab=doc#Fake.PrependWatchReactor + _kubernetes.PrependWatchReactor(watchResource, reactor) + + // overwrite the mock kubernetes client + _engine.Kubernetes = _kubernetes + + return _engine, _watch, nil +} From f958f85f96c883ba5a7ccde45e18f94f2f938638 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 25 Feb 2022 18:32:39 -0600 Subject: [PATCH 07/11] Close the podWatch in WaitContainer --- runtime/kubernetes/container.go | 1 + runtime/kubernetes/container_test.go | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index be635805..007e9088 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -349,6 +349,7 @@ func (c *client) WaitContainer(ctx context.Context, ctn *pipeline.Container) err if err != nil { return err } + defer podWatch.Stop() for { // capture new result from the channel diff --git a/runtime/kubernetes/container_test.go b/runtime/kubernetes/container_test.go index 86a4553e..a7c38442 100644 --- a/runtime/kubernetes/container_test.go +++ b/runtime/kubernetes/container_test.go @@ -222,12 +222,6 @@ func TestKubernetes_TailContainer(t *testing.T) { } func TestKubernetes_WaitContainer(t *testing.T) { - // setup types - _engine, _watch, err := newMockWithWatch(_pod, "pods") - if err != nil { - t.Errorf("unable to create runtime engine: %v", err) - } - // setup tests tests := []struct { failure bool @@ -288,12 +282,18 @@ func TestKubernetes_WaitContainer(t *testing.T) { // run tests for _, test := range tests { + // setup types + _engine, _watch, err := newMockWithWatch(_pod, "pods") + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + go func() { // simulate adding a pod to the watcher _watch.Add(test.object) }() - err := _engine.WaitContainer(context.Background(), test.container) + err = _engine.WaitContainer(context.Background(), test.container) if test.failure { if err == nil { From 133cbd1b62b3cf50dea9d055de35bf7377cc0315 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 25 Feb 2022 18:46:46 -0600 Subject: [PATCH 08/11] Enhance RunContainer to watch for image pull events --- runtime/kubernetes/container.go | 74 +++++++++++++++++++++++++-- runtime/kubernetes/container_test.go | 11 +++- runtime/kubernetes/kubernetes_test.go | 27 ++++++++++ 3 files changed, 107 insertions(+), 5 deletions(-) diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 007e9088..577b21a6 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" ) // InspectContainer inspects the pipeline container. @@ -75,8 +76,33 @@ func (c *client) RemoveContainer(ctx context.Context, ctn *pipeline.Container) e // nolint: lll // ignore long line length func (c *client) RunContainer(ctx context.Context, ctn *pipeline.Container, b *pipeline.Build) error { c.Logger.Tracef("running container %s", ctn.ID) + + field := fields.Set{ + "involvedObject.name": fields.EscapeValue(c.Pod.ObjectMeta.Name), + "involvedObject.namespace": fields.EscapeValue(c.config.Namespace), + "involvedObject.fieldPath": fmt.Sprintf("spec.container{%s}", fields.EscapeValue(ctn.ID)), + } + fieldSelector := field.AsSelector() + + // create options for watching the container events + opts := metav1.ListOptions{ + FieldSelector: fieldSelector.String(), + Watch: true, + } + + // Call Kubernetes API to watch for Image Pull Errors + eventWatch, err := c.Kubernetes.CoreV1().Events(c.config.Namespace).Watch(context.Background(), opts) + if err != nil { + return fmt.Errorf( + "unable to watch events for pod %s and container %s", + c.Pod.ObjectMeta.Name, + ctn.ID, + ) + } + defer eventWatch.Stop() + // validate the container image - err := c.CreateImage(ctx, ctn) + err = c.CreateImage(ctx, ctn) if err != nil { return err } @@ -106,9 +132,49 @@ func (c *client) RunContainer(ctx context.Context, ctn *pipeline.Container, b *p return err } - // TODO: watch k8s events for errors pulling container. - // Only return nil once the image has been pulled successfully. - return nil + for { + // capture new result from the channel + // + // https://pkg.go.dev/k8s.io/apimachinery/pkg/watch?tab=doc#Interface + result := <-eventWatch.ResultChan() + + // events get deleted after some time so ignore them + if result.Type == watch.Deleted { + continue + } + + // convert the object from the result to a pod + event := result.Object.(*v1.Event) + + // check if the event mentions the target image + if !(strings.Contains(event.Message, ctn.Image) || strings.Contains(event.Message, _image)) { + // if the relevant messages does not include our image + // it is probably for "kubernetes/pause:latest" + // or it is a generic message that is basically useless like: + // event.Reason => event.Message + // Failed => Error: ErrImagePull + // BackOff => Error: ImagePullBackOff + continue + } + + // TODO: probably need a timeout of some kind... + + switch event.Reason { + // examples: event.Reason => event.Message + case "Failed", "BackOff": + // Failed => Failed to pull image "image:tag": + // BackOff => Back-off pulling image "image:tag" + return fmt.Errorf("failed to run container %s in %s: %s", ctn.ID, c.Pod.ObjectMeta.Name, event.Message) + case "Pulled": + // Pulled => Successfully pulled image "image:tag" in