diff --git a/README.md b/README.md index aa3bf862..1dd3182c 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Cyclops can either be installed manually by applying the latest manifest, by usi To install Cyclops using `kubectl` into your cluster, run the commands below: ```bash -kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/cyclops-install.yaml && kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/demo-templates.yaml +kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/cyclops-install.yaml && kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/demo-templates.yaml ``` It will create a new namespace called `cyclops` and deploy everything you need for your Cyclops instance to run. diff --git a/cyclops-ctrl/api/v1alpha1/module_types.go b/cyclops-ctrl/api/v1alpha1/module_types.go index 52e0cc1c..9050a8e3 100644 --- a/cyclops-ctrl/api/v1alpha1/module_types.go +++ b/cyclops-ctrl/api/v1alpha1/module_types.go @@ -44,9 +44,10 @@ const ( TemplateSourceTypeHelm TemplateSourceType = "helm" TemplateSourceTypeOCI TemplateSourceType = "oci" - GitOpsWriteRepoAnnotation = "cyclops-ui.com/write-repo" - GitOpsWritePathAnnotation = "cyclops-ui.com/write-path" - GitOpsWriteRevisionAnnotation = "cyclops-ui.com/write-revision" + GitOpsWriteRepoAnnotation = "cyclops-ui.com/write-repo" + GitOpsWritePathAnnotation = "cyclops-ui.com/write-path" + GitOpsWriteRevisionAnnotation = "cyclops-ui.com/write-revision" + GitOpsWriteResourcesAnnotation = "cyclops-ui.com/write-child-resources" ModuleManagerLabel = "cyclops-ui.com/module-manager" @@ -60,6 +61,9 @@ type GitOpsWriteDestination struct { Repo string `json:"repo"` Path string `json:"path"` Version string `json:"version"` + + // +kubebuilder:validation:Optional + WriteResources bool `json:"writeResources"` } type TemplateRef struct { diff --git a/cyclops-ctrl/cmd/main/main.go b/cyclops-ctrl/cmd/main/main.go index f8e4ca71..10dc00d0 100644 --- a/cyclops-ctrl/cmd/main/main.go +++ b/cyclops-ctrl/cmd/main/main.go @@ -142,6 +142,7 @@ func main() { mgr.GetScheme(), templatesRepo, k8sClient, + gitWriteClient, renderer, getMaxConcurrentReconciles(), telemetryClient, diff --git a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml index 13b59d88..d7a3eedc 100644 --- a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml +++ b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml @@ -112,6 +112,8 @@ spec: type: string version: type: string + writeResources: + type: boolean required: - path - repo diff --git a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml index 9a856ec0..f9716311 100644 --- a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml +++ b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml @@ -63,6 +63,8 @@ spec: type: string version: type: string + writeResources: + type: boolean required: - path - repo diff --git a/cyclops-ctrl/internal/controller/modules.go b/cyclops-ctrl/internal/controller/modules.go index cb25e3ed..d6546936 100644 --- a/cyclops-ctrl/internal/controller/modules.go +++ b/cyclops-ctrl/internal/controller/modules.go @@ -314,12 +314,11 @@ func (m *Modules) CreateModule(ctx *gin.Context) { m.telemetryClient.ModuleCreation() if module.GetAnnotations() != nil && len(module.GetAnnotations()[v1alpha1.GitOpsWriteRepoAnnotation]) != 0 { - err := m.gitWriteClient.Write(module) + err := m.gitWriteClient.WriteModule(module) if err != nil { fmt.Println(err) ctx.JSON(http.StatusInternalServerError, dto.NewError("Error pushing to git", err.Error())) } - return } err = m.kubernetesClient.CreateModule(module) @@ -400,7 +399,7 @@ func (m *Modules) UpdateModule(ctx *gin.Context) { module.SetAnnotations(annotations) if len(module.GetAnnotations()[v1alpha1.GitOpsWriteRepoAnnotation]) != 0 { - err := m.gitWriteClient.Write(module) + err := m.gitWriteClient.WriteModule(module) if err != nil { fmt.Println(err) ctx.JSON(http.StatusInternalServerError, dto.NewError("Error pushing to git", err.Error())) diff --git a/cyclops-ctrl/internal/git/writeclient.go b/cyclops-ctrl/internal/git/writeclient.go index 63a493a1..4125daa9 100644 --- a/cyclops-ctrl/internal/git/writeclient.go +++ b/cyclops-ctrl/internal/git/writeclient.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "fmt" - json "github.com/json-iterator/go" path2 "path" "text/template" "time" @@ -18,6 +17,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" + json "github.com/json-iterator/go" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" cyclopsv1alpha1 "github.com/cyclops-ui/cyclops/cyclops-ctrl/api/v1alpha1" @@ -38,47 +39,7 @@ func NewWriteClient(templatesResolver auth.TemplatesResolver, commitMessageTempl } } -func getCommitMessageTemplate(commitMessageTemplate string, logger logr.Logger) *template.Template { - if commitMessageTemplate == "" { - return template.Must(template.New("commitMessage").Parse(_defaultCommitMessageTemplate)) - } - - tmpl, err := template.New("commitMessage").Parse(commitMessageTemplate) - if err != nil { - logger.Error(err, "failed to parse commit message template, falling back to the default commit message", "template", commitMessageTemplate) - return template.Must(template.New("commitMessage").Parse(_defaultCommitMessageTemplate)) - } - - return tmpl -} - -func getModulePath(module cyclopsv1alpha1.Module) (string, error) { - path := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWritePathAnnotation] - - tmpl, err := template.New("modulePath").Parse(path) - if err != nil { - return "", err - } - - moduleMap := make(map[string]interface{}) - moduleData, err := json.Marshal(module) - if err != nil { - return "", err - } - if err := json.Unmarshal(moduleData, &moduleMap); err != nil { - return "", err - } - - var o bytes.Buffer - err = tmpl.Execute(&o, moduleMap) - if err != nil { - return "", err - } - - return o.String(), nil -} - -func (c *WriteClient) Write(module cyclopsv1alpha1.Module) error { +func (c *WriteClient) WriteModule(module cyclopsv1alpha1.Module) error { module.Status.ReconciliationStatus = nil module.Status.ManagedGVRs = nil @@ -103,29 +64,9 @@ func (c *WriteClient) Write(module cyclopsv1alpha1.Module) error { return errors.New(fmt.Sprintf("failed to fetch creds for repo %v: check template auth rules", repoURL)) } - storer := memory.NewStorage() - fs := memfs.New() - - repo, worktree, err := cloneRepo(repoURL, revision, storer, &fs, creds) + _, fs, repo, worktree, err := c.clone(repoURL, revision, creds) if err != nil { - if errors.Is(err, git.NoMatchingRefSpecError{}) { - storer = memory.NewStorage() - fs = memfs.New() - repo, worktree, err = cloneRepo(repoURL, "", storer, &fs, creds) - if err != nil { - return err - } - - err = worktree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName(revision), - Create: true, - }) - if err != nil { - return err - } - } else { - return err - } + return err } path = moduleFilePath(path, module.Name) @@ -145,34 +86,63 @@ func (c *WriteClient) Write(module cyclopsv1alpha1.Module) error { } file.Close() - if _, err := worktree.Add(path); err != nil { - fmt.Println("err worktree.Add", path) - return fmt.Errorf("failed to add file to worktree: %w", err) + return c.commitPush(path, module, repo, worktree, creds) +} + +func (c *WriteClient) WriteModuleResources(module cyclopsv1alpha1.Module, resources []unstructured.Unstructured) error { + if _, ok := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteResourcesAnnotation]; !ok { + return nil } - var o bytes.Buffer - err = c.commitMessageTemplate.Execute(&o, module.ObjectMeta) + module.Status.ReconciliationStatus = nil + module.Status.ManagedGVRs = nil + + repoURL, exists := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteRepoAnnotation] + if !exists { + return errors.New(fmt.Sprintf("module passed to write without git repository; set cyclops-ui.com/write-repo annotation in module %v", module.Name)) + } + + basePath, err := getModulePath(module) if err != nil { return err } - _, err = worktree.Commit(o.String(), &git.CommitOptions{ - Author: &object.Signature{ - Name: creds.Username, - When: time.Now(), - }, - }) + revision := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteRevisionAnnotation] + + creds, err := c.templatesResolver.RepoAuthCredentials(repoURL) if err != nil { - return fmt.Errorf("failed to commit changes: %w", err) + return err } - if err := repo.Push(&git.PushOptions{ - Auth: httpBasicAuthCredentials(creds), - }); err != nil { - return fmt.Errorf("failed to push changes: %w", err) + if creds == nil { + return errors.New(fmt.Sprintf("failed to fetch creds for repo %v: check template auth rules", repoURL)) + } + + _, fs, repo, worktree, err := c.clone(repoURL, revision, creds) + if err != nil { + return err } - return nil + for _, resource := range resources { + path := moduleResourceFilePath(basePath, module.Name, resource.GetKind(), resource.GetName()) + + file, err := fs.Create(path) + if err != nil { + return fmt.Errorf("failed to create file in repository: %w", err) + } + + moduleData, err := yaml.Marshal(resource.Object) + if err != nil { + return err + } + + if _, err := file.Write(moduleData); err != nil { + return fmt.Errorf("failed to write JSON data to file: %w", err) + } + file.Close() + } + + return c.commitPush(basePath, module, repo, worktree, creds) } func (c *WriteClient) DeleteModule(module cyclopsv1alpha1.Module) error { @@ -189,29 +159,9 @@ func (c *WriteClient) DeleteModule(module cyclopsv1alpha1.Module) error { return err } - storer := memory.NewStorage() - fs := memfs.New() - - repo, worktree, err := cloneRepo(repoURL, revision, storer, &fs, creds) + _, fs, repo, worktree, err := c.clone(repoURL, revision, creds) if err != nil { - if errors.Is(err, git.NoMatchingRefSpecError{}) { - storer = memory.NewStorage() - fs = memfs.New() - repo, worktree, err = cloneRepo(repoURL, "", storer, &fs, creds) - if err != nil { - return err - } - - err = worktree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName(revision), - Create: true, - }) - if err != nil { - return err - } - } else { - return err - } + return err } path = moduleFilePath(path, module.Name) @@ -221,33 +171,47 @@ func (c *WriteClient) DeleteModule(module cyclopsv1alpha1.Module) error { return fmt.Errorf("failed to remove file from repository: %w", err) } - if _, err := worktree.Add(path); err != nil { - return fmt.Errorf("failed to add changes to worktree: %w", err) + return c.commitPush(path, module, repo, worktree, creds) +} + +func getCommitMessageTemplate(commitMessageTemplate string, logger logr.Logger) *template.Template { + if commitMessageTemplate == "" { + return template.Must(template.New("commitMessage").Parse(_defaultCommitMessageTemplate)) } - var o bytes.Buffer - err = c.commitMessageTemplate.Execute(&o, module.ObjectMeta) + tmpl, err := template.New("commitMessage").Parse(commitMessageTemplate) if err != nil { - return err + logger.Error(err, "failed to parse commit message template, falling back to the default commit message", "template", commitMessageTemplate) + return template.Must(template.New("commitMessage").Parse(_defaultCommitMessageTemplate)) } - _, err = worktree.Commit(o.String(), &git.CommitOptions{ - Author: &object.Signature{ - Name: creds.Username, - When: time.Now(), - }, - }) + return tmpl +} + +func getModulePath(module cyclopsv1alpha1.Module) (string, error) { + path := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWritePathAnnotation] + + tmpl, err := template.New("modulePath").Parse(path) if err != nil { - return fmt.Errorf("failed to commit changes: %w", err) + return "", err } - if err := repo.Push(&git.PushOptions{ - Auth: httpBasicAuthCredentials(creds), - }); err != nil { - return fmt.Errorf("failed to push changes: %w", err) + moduleMap := make(map[string]interface{}) + moduleData, err := json.Marshal(module) + if err != nil { + return "", err + } + if err := json.Unmarshal(moduleData, &moduleMap); err != nil { + return "", err + } + + var o bytes.Buffer + err = tmpl.Execute(&o, moduleMap) + if err != nil { + return "", err } - return nil + return o.String(), nil } func moduleFilePath(path, moduleName string) string { @@ -258,7 +222,44 @@ func moduleFilePath(path, moduleName string) string { return path } -func cloneRepo(url, revision string, storer *memory.Storage, fs *billy.Filesystem, creds *auth.Credentials) (*git.Repository, *git.Worktree, error) { +func moduleResourceFilePath(path, moduleName, resourceKind, resourceName string) string { + if path2.Ext(path) != "yaml" || path2.Ext(path) != "yml" { + path = path2.Join(path, fmt.Sprintf("%v_resources", moduleName), fmt.Sprintf("%v_%v.yaml", resourceKind, resourceName)) + } + + return path +} + +func (c *WriteClient) clone(repoURL, revision string, creds *auth.Credentials) (*memory.Storage, billy.Filesystem, *git.Repository, *git.Worktree, error) { + storer := memory.NewStorage() + fs := memfs.New() + + repo, worktree, err := c.cloneRepo(repoURL, revision, storer, &fs, creds) + if err != nil { + if errors.Is(err, git.NoMatchingRefSpecError{}) { + storer = memory.NewStorage() + fs = memfs.New() + repo, worktree, err = c.cloneRepo(repoURL, "", storer, &fs, creds) + if err != nil { + return nil, nil, nil, nil, err + } + + err = worktree.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewBranchReferenceName(revision), + Create: true, + }) + if err != nil { + return nil, nil, nil, nil, err + } + } else { + return nil, nil, nil, nil, err + } + } + + return storer, fs, repo, worktree, nil +} + +func (c *WriteClient) cloneRepo(url, revision string, storer *memory.Storage, fs *billy.Filesystem, creds *auth.Credentials) (*git.Repository, *git.Worktree, error) { cloneOpts := git.CloneOptions{ URL: url, Auth: httpBasicAuthCredentials(creds), @@ -282,6 +283,31 @@ func cloneRepo(url, revision string, storer *memory.Storage, fs *billy.Filesyste return repo, worktree, nil } +func (c *WriteClient) commitPush(path string, module cyclopsv1alpha1.Module, repo *git.Repository, worktree *git.Worktree, creds *auth.Credentials) error { + if _, err := worktree.Add(path); err != nil { + fmt.Println("err worktree.Add", path) + return fmt.Errorf("failed to add file to worktree: %w", err) + } + + var o bytes.Buffer + if err := c.commitMessageTemplate.Execute(&o, module.ObjectMeta); err != nil { + return err + } + + if _, err := worktree.Commit(o.String(), &git.CommitOptions{ + Author: &object.Signature{ + Name: creds.Username, + When: time.Now(), + }, + }); err != nil { + return fmt.Errorf("failed to commit changes: %w", err) + } + + return repo.Push(&git.PushOptions{ + Auth: httpBasicAuthCredentials(creds), + }) +} + func httpBasicAuthCredentials(creds *auth.Credentials) *http.BasicAuth { if creds == nil { return nil diff --git a/cyclops-ctrl/internal/mapper/modules.go b/cyclops-ctrl/internal/mapper/modules.go index 2d204544..0318986e 100644 --- a/cyclops-ctrl/internal/mapper/modules.go +++ b/cyclops-ctrl/internal/mapper/modules.go @@ -24,6 +24,10 @@ func RequestToModule(req dto.Module) (cyclopsv1alpha1.Module, error) { annotations[cyclopsv1alpha1.GitOpsWriteRepoAnnotation] = req.GitOpsWrite.Repo annotations[cyclopsv1alpha1.GitOpsWritePathAnnotation] = req.GitOpsWrite.Path annotations[cyclopsv1alpha1.GitOpsWriteRevisionAnnotation] = req.GitOpsWrite.Branch + + if req.GitOpsWrite.WriteResources { + annotations[cyclopsv1alpha1.GitOpsWriteResourcesAnnotation] = "true" + } } return cyclopsv1alpha1.Module{ diff --git a/cyclops-ctrl/internal/models/dto/modules.go b/cyclops-ctrl/internal/models/dto/modules.go index 10ef9f45..a7be6f22 100644 --- a/cyclops-ctrl/internal/models/dto/modules.go +++ b/cyclops-ctrl/internal/models/dto/modules.go @@ -37,9 +37,10 @@ type Template struct { } type GitOpsWrite struct { - Repo string `json:"repo"` - Path string `json:"path"` - Branch string `json:"branch"` + Repo string `json:"repo"` + Path string `json:"path"` + Branch string `json:"branch"` + WriteResources bool `json:"writeResources"` } type TemplatesResponse struct { diff --git a/cyclops-ctrl/internal/modulecontroller/module_controller.go b/cyclops-ctrl/internal/modulecontroller/module_controller.go index e0cc66ea..95568090 100644 --- a/cyclops-ctrl/internal/modulecontroller/module_controller.go +++ b/cyclops-ctrl/internal/modulecontroller/module_controller.go @@ -19,6 +19,7 @@ package modulecontroller import ( "context" "fmt" + "github.com/cyclops-ui/cyclops/cyclops-ctrl/internal/git" "sort" "strings" "time" @@ -53,6 +54,7 @@ type ModuleReconciler struct { templatesRepo templaterepo.ITemplateRepo kubernetesClient k8sclient.IKubernetesClient + gitWriteClient *git.WriteClient renderer *render.Renderer maxConcurrentReconciles int @@ -67,6 +69,7 @@ func NewModuleReconciler( scheme *runtime.Scheme, templatesRepo templaterepo.ITemplateRepo, kubernetesClient k8sclient.IKubernetesClient, + gitWriteClient *git.WriteClient, renderer *render.Renderer, maxConcurrentReconciles int, telemetryClient telemetry.Client, @@ -77,6 +80,7 @@ func NewModuleReconciler( Scheme: scheme, templatesRepo: templatesRepo, kubernetesClient: kubernetesClient, + gitWriteClient: gitWriteClient, renderer: renderer, telemetryClient: telemetryClient, maxConcurrentReconciles: maxConcurrentReconciles, @@ -254,6 +258,8 @@ func (r *ModuleReconciler) generateResources( installErrors := make([]string, 0) childrenGVRs := make([]cyclopsv1alpha1.GroupVersionResource, 0) + childObjs := make([]unstructured.Unstructured, 0) + for _, s := range strings.Split(out, "\n---\n") { s := strings.TrimSpace(s) if len(s) == 0 { @@ -320,6 +326,35 @@ func (r *ModuleReconciler) generateResources( } childrenGVRs = append(childrenGVRs, gvr) + childObjs = append(childObjs, obj) + } + + if _, ok := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteResourcesAnnotation]; ok { + return []string{}, childrenGVRs, r.gitWriteClient.WriteModuleResources(module, childObjs) + } + + for _, obj := range childObjs { + resourceName, err := kClient.GVKtoAPIResourceName(obj.GroupVersionKind().GroupVersion(), obj.GroupVersionKind().Kind) + if err != nil { + installErrors = append(installErrors, fmt.Sprintf( + "%v%v/%v %v/%v failed to apply: %v", + obj.GroupVersionKind().Group, + obj.GroupVersionKind().Version, + obj.GroupVersionKind().Kind, + obj.GetNamespace(), + obj.GetName(), + err.Error(), + )) + + continue + } + + gvr := cyclopsv1alpha1.GroupVersionResource{ + Group: obj.GroupVersionKind().Group, + Version: obj.GroupVersionKind().Version, + Resource: resourceName, + } + if err := kClient.CreateDynamic(gvr, &obj, module.Spec.TargetNamespace); err != nil { installErrors = append(installErrors, fmt.Sprintf( "%v%v/%v %v/%v failed to apply: %v", diff --git a/cyclops-ui/src/components/shared/CreateModule/CreateModule.tsx b/cyclops-ui/src/components/shared/CreateModule/CreateModule.tsx index 2e832cdb..0bff358f 100644 --- a/cyclops-ui/src/components/shared/CreateModule/CreateModule.tsx +++ b/cyclops-ui/src/components/shared/CreateModule/CreateModule.tsx @@ -55,6 +55,7 @@ interface templateStoreOption { repo: string; path: string; branch: string; + writeResources: boolean; }; } @@ -150,6 +151,7 @@ export const CreateModuleComponent = ({ writeRepo: string, writePath: string, writeBranch: string, + writeResources: boolean, ) => { if (template.enforceGitOpsWrite !== undefined) { return template.enforceGitOpsWrite; @@ -160,6 +162,7 @@ export const CreateModuleComponent = ({ repo: writeRepo, path: writePath, branch: writeBranch, + writeResources: writeResources, } : null; }; @@ -196,6 +199,7 @@ export const CreateModuleComponent = ({ const gitopsWriteRepo = values["gitops-repo"]; const gitopsWritePath = values["gitops-path"]; const gitopsWriteBranch = values["gitops-branch"]; + const gitopsWriteResources = values["gitops-write-resources"]; values = findMaps(config.root.properties, values, initialValuesRaw); @@ -209,7 +213,12 @@ export const CreateModuleComponent = ({ sourceType: template.ref.sourceType, }, values, - resolveGitOpsWrite(gitopsWriteRepo, gitopsWritePath, gitopsWriteBranch), + resolveGitOpsWrite( + gitopsWriteRepo, + gitopsWritePath, + gitopsWriteBranch, + gitopsWriteResources, + ), ) .then(() => { onSubmitModuleSuccess(moduleName); @@ -666,6 +675,26 @@ export const CreateModuleComponent = ({ > + + Write child resources to Git? +

+ Instead of applying created resources to the + cluster, Cyclops will push all the created + resources to a git repository on path specified + above. +

+ + } + style={{ padding: "0px 12px 0px 12px" }} + > + +
{

Install it with a single command

- {"kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/cyclops-install.yaml && \n" + - "kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/demo-templates.yaml"} + {"kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/cyclops-install.yaml && \n" + + "kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/demo-templates.yaml"}
);