From f7e02c43507e1a86e086ea5db4acd9403e20120a Mon Sep 17 00:00:00 2001 From: petar-cvit Date: Fri, 27 Jun 2025 18:50:14 +0200 Subject: [PATCH 01/11] add write child resources to crd --- cyclops-ctrl/api/v1alpha1/module_types.go | 14 ++++++++------ .../config/crd/bases/cyclops-ui.com_modules.yaml | 3 +++ .../crd/bases/cyclops-ui.com_templatestores.yaml | 3 +++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cyclops-ctrl/api/v1alpha1/module_types.go b/cyclops-ctrl/api/v1alpha1/module_types.go index 52e0cc1c..c2b597ca 100644 --- a/cyclops-ctrl/api/v1alpha1/module_types.go +++ b/cyclops-ctrl/api/v1alpha1/module_types.go @@ -44,9 +44,10 @@ const ( TemplateSourceTypeHelm TemplateSourceType = "helm" TemplateSourceTypeOCI TemplateSourceType = "oci" - GitOpsWriteRepoAnnotation = "cyclops-ui.com/write-repo" - GitOpsWritePathAnnotation = "cyclops-ui.com/write-path" - GitOpsWriteRevisionAnnotation = "cyclops-ui.com/write-revision" + GitOpsWriteRepoAnnotation = "cyclops-ui.com/write-repo" + GitOpsWritePathAnnotation = "cyclops-ui.com/write-path" + GitOpsWriteRevisionAnnotation = "cyclops-ui.com/write-revision" + GitOpsWriteResourcesAnnotation = "cyclops-ui.com/write-child-resources" ModuleManagerLabel = "cyclops-ui.com/module-manager" @@ -57,9 +58,10 @@ const ( ) type GitOpsWriteDestination struct { - Repo string `json:"repo"` - Path string `json:"path"` - Version string `json:"version"` + Repo string `json:"repo"` + Path string `json:"path"` + Version string `json:"version"` + WriteResources bool `json:"writeResources"` } type TemplateRef struct { diff --git a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml index 13b59d88..cd9614d1 100644 --- a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml +++ b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml @@ -112,10 +112,13 @@ spec: type: string version: type: string + writeResources: + type: boolean required: - path - repo - version + - writeResources type: object path: type: string diff --git a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml index 9a856ec0..4a0a7df5 100644 --- a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml +++ b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml @@ -63,10 +63,13 @@ spec: type: string version: type: string + writeResources: + type: boolean required: - path - repo - version + - writeResources type: object path: type: string From 1a8d0d213c43b4e84d4685403a2302e1a4d42eb0 Mon Sep 17 00:00:00 2001 From: petar-cvit Date: Fri, 27 Jun 2025 18:52:22 +0200 Subject: [PATCH 02/11] add toggle for pushing child resources to git --- .../shared/CreateModule/CreateModule.tsx | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/cyclops-ui/src/components/shared/CreateModule/CreateModule.tsx b/cyclops-ui/src/components/shared/CreateModule/CreateModule.tsx index 2e832cdb..0bff358f 100644 --- a/cyclops-ui/src/components/shared/CreateModule/CreateModule.tsx +++ b/cyclops-ui/src/components/shared/CreateModule/CreateModule.tsx @@ -55,6 +55,7 @@ interface templateStoreOption { repo: string; path: string; branch: string; + writeResources: boolean; }; } @@ -150,6 +151,7 @@ export const CreateModuleComponent = ({ writeRepo: string, writePath: string, writeBranch: string, + writeResources: boolean, ) => { if (template.enforceGitOpsWrite !== undefined) { return template.enforceGitOpsWrite; @@ -160,6 +162,7 @@ export const CreateModuleComponent = ({ repo: writeRepo, path: writePath, branch: writeBranch, + writeResources: writeResources, } : null; }; @@ -196,6 +199,7 @@ export const CreateModuleComponent = ({ const gitopsWriteRepo = values["gitops-repo"]; const gitopsWritePath = values["gitops-path"]; const gitopsWriteBranch = values["gitops-branch"]; + const gitopsWriteResources = values["gitops-write-resources"]; values = findMaps(config.root.properties, values, initialValuesRaw); @@ -209,7 +213,12 @@ export const CreateModuleComponent = ({ sourceType: template.ref.sourceType, }, values, - resolveGitOpsWrite(gitopsWriteRepo, gitopsWritePath, gitopsWriteBranch), + resolveGitOpsWrite( + gitopsWriteRepo, + gitopsWritePath, + gitopsWriteBranch, + gitopsWriteResources, + ), ) .then(() => { onSubmitModuleSuccess(moduleName); @@ -666,6 +675,26 @@ export const CreateModuleComponent = ({ > + + Write child resources to Git? +

+ Instead of applying created resources to the + cluster, Cyclops will push all the created + resources to a git repository on path specified + above. +

+ + } + style={{ padding: "0px 12px 0px 12px" }} + > + +
Date: Mon, 30 Jun 2025 21:18:31 +0200 Subject: [PATCH 03/11] pass git write client to reconciler --- cyclops-ctrl/cmd/main/main.go | 1 + cyclops-ctrl/internal/mapper/modules.go | 4 +++ cyclops-ctrl/internal/models/dto/modules.go | 7 ++--- .../modulecontroller/module_controller.go | 27 +++++++++++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/cyclops-ctrl/cmd/main/main.go b/cyclops-ctrl/cmd/main/main.go index f8e4ca71..10dc00d0 100644 --- a/cyclops-ctrl/cmd/main/main.go +++ b/cyclops-ctrl/cmd/main/main.go @@ -142,6 +142,7 @@ func main() { mgr.GetScheme(), templatesRepo, k8sClient, + gitWriteClient, renderer, getMaxConcurrentReconciles(), telemetryClient, diff --git a/cyclops-ctrl/internal/mapper/modules.go b/cyclops-ctrl/internal/mapper/modules.go index 2d204544..0318986e 100644 --- a/cyclops-ctrl/internal/mapper/modules.go +++ b/cyclops-ctrl/internal/mapper/modules.go @@ -24,6 +24,10 @@ func RequestToModule(req dto.Module) (cyclopsv1alpha1.Module, error) { annotations[cyclopsv1alpha1.GitOpsWriteRepoAnnotation] = req.GitOpsWrite.Repo annotations[cyclopsv1alpha1.GitOpsWritePathAnnotation] = req.GitOpsWrite.Path annotations[cyclopsv1alpha1.GitOpsWriteRevisionAnnotation] = req.GitOpsWrite.Branch + + if req.GitOpsWrite.WriteResources { + annotations[cyclopsv1alpha1.GitOpsWriteResourcesAnnotation] = "true" + } } return cyclopsv1alpha1.Module{ diff --git a/cyclops-ctrl/internal/models/dto/modules.go b/cyclops-ctrl/internal/models/dto/modules.go index 10ef9f45..a7be6f22 100644 --- a/cyclops-ctrl/internal/models/dto/modules.go +++ b/cyclops-ctrl/internal/models/dto/modules.go @@ -37,9 +37,10 @@ type Template struct { } type GitOpsWrite struct { - Repo string `json:"repo"` - Path string `json:"path"` - Branch string `json:"branch"` + Repo string `json:"repo"` + Path string `json:"path"` + Branch string `json:"branch"` + WriteResources bool `json:"writeResources"` } type TemplatesResponse struct { diff --git a/cyclops-ctrl/internal/modulecontroller/module_controller.go b/cyclops-ctrl/internal/modulecontroller/module_controller.go index e0cc66ea..45999751 100644 --- a/cyclops-ctrl/internal/modulecontroller/module_controller.go +++ b/cyclops-ctrl/internal/modulecontroller/module_controller.go @@ -19,6 +19,7 @@ package modulecontroller import ( "context" "fmt" + "github.com/cyclops-ui/cyclops/cyclops-ctrl/internal/git" "sort" "strings" "time" @@ -53,6 +54,7 @@ type ModuleReconciler struct { templatesRepo templaterepo.ITemplateRepo kubernetesClient k8sclient.IKubernetesClient + gitWriteClient *git.WriteClient renderer *render.Renderer maxConcurrentReconciles int @@ -67,6 +69,7 @@ func NewModuleReconciler( scheme *runtime.Scheme, templatesRepo templaterepo.ITemplateRepo, kubernetesClient k8sclient.IKubernetesClient, + gitWriteClient *git.WriteClient, renderer *render.Renderer, maxConcurrentReconciles int, telemetryClient telemetry.Client, @@ -77,6 +80,7 @@ func NewModuleReconciler( Scheme: scheme, templatesRepo: templatesRepo, kubernetesClient: kubernetesClient, + gitWriteClient: gitWriteClient, renderer: renderer, telemetryClient: telemetryClient, maxConcurrentReconciles: maxConcurrentReconciles, @@ -260,6 +264,23 @@ func (r *ModuleReconciler) generateResources( continue } + if _, ok := module.Annotations[cyclopsv1alpha1.GitOpsWriteResourcesAnnotation]; ok { + err := r.writeResourceToGit(s) + + r.logger.Error(err, "failed to write child resource to git", + "module namespaced name", + module.Name, + ) + + installErrors = append(installErrors, fmt.Sprintf( + "failed to write child resource to git for module %v :%v", + module.Name, + err.Error(), + )) + + continue + } + var obj unstructured.Unstructured decoder := yaml.NewYAMLOrJSONDecoder(strings.NewReader(s), len(s)) if err := decoder.Decode(&obj); err != nil { @@ -338,6 +359,12 @@ func (r *ModuleReconciler) generateResources( return installErrors, childrenGVRs, nil } +func (r *ModuleReconciler) writeResourceToGit( + obj string, +) error { + r.gitWriteClient.Write() +} + func (r *ModuleReconciler) applyCRDs(template *models.Template) []string { installErrors := make([]string, 0) From d0168384ea0c939367ef36dc85242e1aa6a15546 Mon Sep 17 00:00:00 2001 From: petar-cvit Date: Tue, 1 Jul 2025 14:21:24 +0200 Subject: [PATCH 04/11] push resources to separate folder --- cyclops-ctrl/internal/controller/modules.go | 4 +- cyclops-ctrl/internal/git/writeclient.go | 365 ++++++++++++------ .../modulecontroller/module_controller.go | 54 +-- 3 files changed, 277 insertions(+), 146 deletions(-) diff --git a/cyclops-ctrl/internal/controller/modules.go b/cyclops-ctrl/internal/controller/modules.go index cb25e3ed..ce9444d4 100644 --- a/cyclops-ctrl/internal/controller/modules.go +++ b/cyclops-ctrl/internal/controller/modules.go @@ -314,7 +314,7 @@ func (m *Modules) CreateModule(ctx *gin.Context) { m.telemetryClient.ModuleCreation() if module.GetAnnotations() != nil && len(module.GetAnnotations()[v1alpha1.GitOpsWriteRepoAnnotation]) != 0 { - err := m.gitWriteClient.Write(module) + err := m.gitWriteClient.WriteModule(module) if err != nil { fmt.Println(err) ctx.JSON(http.StatusInternalServerError, dto.NewError("Error pushing to git", err.Error())) @@ -400,7 +400,7 @@ func (m *Modules) UpdateModule(ctx *gin.Context) { module.SetAnnotations(annotations) if len(module.GetAnnotations()[v1alpha1.GitOpsWriteRepoAnnotation]) != 0 { - err := m.gitWriteClient.Write(module) + err := m.gitWriteClient.WriteModule(module) if err != nil { fmt.Println(err) ctx.JSON(http.StatusInternalServerError, dto.NewError("Error pushing to git", err.Error())) diff --git a/cyclops-ctrl/internal/git/writeclient.go b/cyclops-ctrl/internal/git/writeclient.go index 63a493a1..61dcb0ea 100644 --- a/cyclops-ctrl/internal/git/writeclient.go +++ b/cyclops-ctrl/internal/git/writeclient.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "fmt" - json "github.com/json-iterator/go" path2 "path" "text/template" "time" @@ -18,6 +17,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" + json "github.com/json-iterator/go" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" cyclopsv1alpha1 "github.com/cyclops-ui/cyclops/cyclops-ctrl/api/v1alpha1" @@ -38,47 +39,7 @@ func NewWriteClient(templatesResolver auth.TemplatesResolver, commitMessageTempl } } -func getCommitMessageTemplate(commitMessageTemplate string, logger logr.Logger) *template.Template { - if commitMessageTemplate == "" { - return template.Must(template.New("commitMessage").Parse(_defaultCommitMessageTemplate)) - } - - tmpl, err := template.New("commitMessage").Parse(commitMessageTemplate) - if err != nil { - logger.Error(err, "failed to parse commit message template, falling back to the default commit message", "template", commitMessageTemplate) - return template.Must(template.New("commitMessage").Parse(_defaultCommitMessageTemplate)) - } - - return tmpl -} - -func getModulePath(module cyclopsv1alpha1.Module) (string, error) { - path := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWritePathAnnotation] - - tmpl, err := template.New("modulePath").Parse(path) - if err != nil { - return "", err - } - - moduleMap := make(map[string]interface{}) - moduleData, err := json.Marshal(module) - if err != nil { - return "", err - } - if err := json.Unmarshal(moduleData, &moduleMap); err != nil { - return "", err - } - - var o bytes.Buffer - err = tmpl.Execute(&o, moduleMap) - if err != nil { - return "", err - } - - return o.String(), nil -} - -func (c *WriteClient) Write(module cyclopsv1alpha1.Module) error { +func (c *WriteClient) WriteModule(module cyclopsv1alpha1.Module) error { module.Status.ReconciliationStatus = nil module.Status.ManagedGVRs = nil @@ -103,29 +64,9 @@ func (c *WriteClient) Write(module cyclopsv1alpha1.Module) error { return errors.New(fmt.Sprintf("failed to fetch creds for repo %v: check template auth rules", repoURL)) } - storer := memory.NewStorage() - fs := memfs.New() - - repo, worktree, err := cloneRepo(repoURL, revision, storer, &fs, creds) + _, fs, repo, worktree, err := c.clone(repoURL, revision, creds) if err != nil { - if errors.Is(err, git.NoMatchingRefSpecError{}) { - storer = memory.NewStorage() - fs = memfs.New() - repo, worktree, err = cloneRepo(repoURL, "", storer, &fs, creds) - if err != nil { - return err - } - - err = worktree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName(revision), - Create: true, - }) - if err != nil { - return err - } - } else { - return err - } + return err } path = moduleFilePath(path, module.Name) @@ -145,34 +86,63 @@ func (c *WriteClient) Write(module cyclopsv1alpha1.Module) error { } file.Close() - if _, err := worktree.Add(path); err != nil { - fmt.Println("err worktree.Add", path) - return fmt.Errorf("failed to add file to worktree: %w", err) + return c.commitPush(path, module, repo, worktree, creds) +} + +func (c *WriteClient) WriteModuleResources(module cyclopsv1alpha1.Module, resources []unstructured.Unstructured) error { + if _, ok := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteResourcesAnnotation]; !ok { + return nil } - var o bytes.Buffer - err = c.commitMessageTemplate.Execute(&o, module.ObjectMeta) + module.Status.ReconciliationStatus = nil + module.Status.ManagedGVRs = nil + + repoURL, exists := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteRepoAnnotation] + if !exists { + return errors.New(fmt.Sprintf("module passed to write without git repository; set cyclops-ui.com/write-repo annotation in module %v", module.Name)) + } + + basePath, err := getModulePath(module) if err != nil { return err } - _, err = worktree.Commit(o.String(), &git.CommitOptions{ - Author: &object.Signature{ - Name: creds.Username, - When: time.Now(), - }, - }) + revision := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteRevisionAnnotation] + + creds, err := c.templatesResolver.RepoAuthCredentials(repoURL) if err != nil { - return fmt.Errorf("failed to commit changes: %w", err) + return err } - if err := repo.Push(&git.PushOptions{ - Auth: httpBasicAuthCredentials(creds), - }); err != nil { - return fmt.Errorf("failed to push changes: %w", err) + if creds == nil { + return errors.New(fmt.Sprintf("failed to fetch creds for repo %v: check template auth rules", repoURL)) + } + + _, fs, repo, worktree, err := c.clone(repoURL, revision, creds) + if err != nil { + return err } - return nil + for _, resource := range resources { + path := moduleResourceFilePath(basePath, module.Name, resource.GetKind(), resource.GetName()) + + file, err := fs.Create(path) + if err != nil { + return fmt.Errorf("failed to create file in repository: %w", err) + } + + moduleData, err := yaml.Marshal(resource.Object) + if err != nil { + return err + } + + if _, err := file.Write(moduleData); err != nil { + return fmt.Errorf("failed to write JSON data to file: %w", err) + } + file.Close() + } + + return c.commitPush(basePath, module, repo, worktree, creds) } func (c *WriteClient) DeleteModule(module cyclopsv1alpha1.Module) error { @@ -189,29 +159,9 @@ func (c *WriteClient) DeleteModule(module cyclopsv1alpha1.Module) error { return err } - storer := memory.NewStorage() - fs := memfs.New() - - repo, worktree, err := cloneRepo(repoURL, revision, storer, &fs, creds) + _, fs, repo, worktree, err := c.clone(repoURL, revision, creds) if err != nil { - if errors.Is(err, git.NoMatchingRefSpecError{}) { - storer = memory.NewStorage() - fs = memfs.New() - repo, worktree, err = cloneRepo(repoURL, "", storer, &fs, creds) - if err != nil { - return err - } - - err = worktree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName(revision), - Create: true, - }) - if err != nil { - return err - } - } else { - return err - } + return err } path = moduleFilePath(path, module.Name) @@ -221,35 +171,146 @@ func (c *WriteClient) DeleteModule(module cyclopsv1alpha1.Module) error { return fmt.Errorf("failed to remove file from repository: %w", err) } - if _, err := worktree.Add(path); err != nil { - return fmt.Errorf("failed to add changes to worktree: %w", err) + return c.commitPush(path, module, repo, worktree, creds) +} + +func getCommitMessageTemplate(commitMessageTemplate string, logger logr.Logger) *template.Template { + if commitMessageTemplate == "" { + return template.Must(template.New("commitMessage").Parse(_defaultCommitMessageTemplate)) } - var o bytes.Buffer - err = c.commitMessageTemplate.Execute(&o, module.ObjectMeta) + tmpl, err := template.New("commitMessage").Parse(commitMessageTemplate) if err != nil { - return err + logger.Error(err, "failed to parse commit message template, falling back to the default commit message", "template", commitMessageTemplate) + return template.Must(template.New("commitMessage").Parse(_defaultCommitMessageTemplate)) } - _, err = worktree.Commit(o.String(), &git.CommitOptions{ - Author: &object.Signature{ - Name: creds.Username, - When: time.Now(), - }, - }) + return tmpl +} + +func getModulePath(module cyclopsv1alpha1.Module) (string, error) { + path := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWritePathAnnotation] + + tmpl, err := template.New("modulePath").Parse(path) if err != nil { - return fmt.Errorf("failed to commit changes: %w", err) + return "", err } - if err := repo.Push(&git.PushOptions{ - Auth: httpBasicAuthCredentials(creds), - }); err != nil { - return fmt.Errorf("failed to push changes: %w", err) + moduleMap := make(map[string]interface{}) + moduleData, err := json.Marshal(module) + if err != nil { + return "", err + } + if err := json.Unmarshal(moduleData, &moduleMap); err != nil { + return "", err + } + + var o bytes.Buffer + err = tmpl.Execute(&o, moduleMap) + if err != nil { + return "", err } - return nil + return o.String(), nil } +//func (c *WriteClient) Write(module cyclopsv1alpha1.Module) error { +// module.Status.ReconciliationStatus = nil +// module.Status.ManagedGVRs = nil +// +// repoURL, exists := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteRepoAnnotation] +// if !exists { +// return errors.New(fmt.Sprintf("module passed to write without git repository; set cyclops-ui.com/write-repo annotation in module %v", module.Name)) +// } +// +// path, err := getModulePath(module) +// if err != nil { +// return err +// } +// +// revision := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteRevisionAnnotation] +// +// creds, err := c.templatesResolver.RepoAuthCredentials(repoURL) +// if err != nil { +// return err +// } +// +// if creds == nil { +// return errors.New(fmt.Sprintf("failed to fetch creds for repo %v: check template auth rules", repoURL)) +// } +// +// storer := memory.NewStorage() +// fs := memfs.New() +// +// repo, worktree, err := cloneRepo(repoURL, revision, storer, &fs, creds) +// if err != nil { +// if errors.Is(err, git.NoMatchingRefSpecError{}) { +// storer = memory.NewStorage() +// fs = memfs.New() +// repo, worktree, err = cloneRepo(repoURL, "", storer, &fs, creds) +// if err != nil { +// return err +// } +// +// err = worktree.Checkout(&git.CheckoutOptions{ +// Branch: plumbing.NewBranchReferenceName(revision), +// Create: true, +// }) +// if err != nil { +// return err +// } +// } else { +// return err +// } +// } +// +// path = moduleFilePath(path, module.Name) +// +// file, err := fs.Create(path) +// if err != nil { +// return fmt.Errorf("failed to create file in repository: %w", err) +// } +// +// moduleData, err := yaml.Marshal(module) +// if err != nil { +// return err +// } +// +// if _, err := file.Write(moduleData); err != nil { +// return fmt.Errorf("failed to write JSON data to file: %w", err) +// } +// file.Close() +// +// if _, err := worktree.Add(path); err != nil { +// fmt.Println("err worktree.Add", path) +// return fmt.Errorf("failed to add file to worktree: %w", err) +// } +// +// var o bytes.Buffer +// err = c.commitMessageTemplate.Execute(&o, module.ObjectMeta) +// if err != nil { +// return err +// } +// +// _, err = worktree.Commit(o.String(), &git.CommitOptions{ +// Author: &object.Signature{ +// Name: creds.Username, +// When: time.Now(), +// }, +// }) +// if err != nil { +// return fmt.Errorf("failed to commit changes: %w", err) +// } +// +// if err := repo.Push(&git.PushOptions{ +// Auth: httpBasicAuthCredentials(creds), +// }); err != nil { +// return fmt.Errorf("failed to push changes: %w", err) +// } +// +// return nil +//} + func moduleFilePath(path, moduleName string) string { if path2.Ext(path) != "yaml" || path2.Ext(path) != "yml" { path = path2.Join(path, fmt.Sprintf("%v.yaml", moduleName)) @@ -258,7 +319,44 @@ func moduleFilePath(path, moduleName string) string { return path } -func cloneRepo(url, revision string, storer *memory.Storage, fs *billy.Filesystem, creds *auth.Credentials) (*git.Repository, *git.Worktree, error) { +func moduleResourceFilePath(path, moduleName, resourceKind, resourceName string) string { + if path2.Ext(path) != "yaml" || path2.Ext(path) != "yml" { + path = path2.Join(path, fmt.Sprintf("%v_resources", moduleName), fmt.Sprintf("%v_%v.yaml", resourceKind, resourceName)) + } + + return path +} + +func (c *WriteClient) clone(repoURL, revision string, creds *auth.Credentials) (*memory.Storage, billy.Filesystem, *git.Repository, *git.Worktree, error) { + storer := memory.NewStorage() + fs := memfs.New() + + repo, worktree, err := c.cloneRepo(repoURL, revision, storer, &fs, creds) + if err != nil { + if errors.Is(err, git.NoMatchingRefSpecError{}) { + storer = memory.NewStorage() + fs = memfs.New() + repo, worktree, err = c.cloneRepo(repoURL, "", storer, &fs, creds) + if err != nil { + return nil, nil, nil, nil, err + } + + err = worktree.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewBranchReferenceName(revision), + Create: true, + }) + if err != nil { + return nil, nil, nil, nil, err + } + } else { + return nil, nil, nil, nil, err + } + } + + return storer, fs, repo, worktree, nil +} + +func (c *WriteClient) cloneRepo(url, revision string, storer *memory.Storage, fs *billy.Filesystem, creds *auth.Credentials) (*git.Repository, *git.Worktree, error) { cloneOpts := git.CloneOptions{ URL: url, Auth: httpBasicAuthCredentials(creds), @@ -282,6 +380,31 @@ func cloneRepo(url, revision string, storer *memory.Storage, fs *billy.Filesyste return repo, worktree, nil } +func (c *WriteClient) commitPush(path string, module cyclopsv1alpha1.Module, repo *git.Repository, worktree *git.Worktree, creds *auth.Credentials) error { + if _, err := worktree.Add(path); err != nil { + fmt.Println("err worktree.Add", path) + return fmt.Errorf("failed to add file to worktree: %w", err) + } + + var o bytes.Buffer + if err := c.commitMessageTemplate.Execute(&o, module.ObjectMeta); err != nil { + return err + } + + if _, err := worktree.Commit(o.String(), &git.CommitOptions{ + Author: &object.Signature{ + Name: creds.Username, + When: time.Now(), + }, + }); err != nil { + return fmt.Errorf("failed to commit changes: %w", err) + } + + return repo.Push(&git.PushOptions{ + Auth: httpBasicAuthCredentials(creds), + }) +} + func httpBasicAuthCredentials(creds *auth.Credentials) *http.BasicAuth { if creds == nil { return nil diff --git a/cyclops-ctrl/internal/modulecontroller/module_controller.go b/cyclops-ctrl/internal/modulecontroller/module_controller.go index 45999751..95568090 100644 --- a/cyclops-ctrl/internal/modulecontroller/module_controller.go +++ b/cyclops-ctrl/internal/modulecontroller/module_controller.go @@ -258,29 +258,14 @@ func (r *ModuleReconciler) generateResources( installErrors := make([]string, 0) childrenGVRs := make([]cyclopsv1alpha1.GroupVersionResource, 0) + childObjs := make([]unstructured.Unstructured, 0) + for _, s := range strings.Split(out, "\n---\n") { s := strings.TrimSpace(s) if len(s) == 0 { continue } - if _, ok := module.Annotations[cyclopsv1alpha1.GitOpsWriteResourcesAnnotation]; ok { - err := r.writeResourceToGit(s) - - r.logger.Error(err, "failed to write child resource to git", - "module namespaced name", - module.Name, - ) - - installErrors = append(installErrors, fmt.Sprintf( - "failed to write child resource to git for module %v :%v", - module.Name, - err.Error(), - )) - - continue - } - var obj unstructured.Unstructured decoder := yaml.NewYAMLOrJSONDecoder(strings.NewReader(s), len(s)) if err := decoder.Decode(&obj); err != nil { @@ -341,6 +326,35 @@ func (r *ModuleReconciler) generateResources( } childrenGVRs = append(childrenGVRs, gvr) + childObjs = append(childObjs, obj) + } + + if _, ok := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteResourcesAnnotation]; ok { + return []string{}, childrenGVRs, r.gitWriteClient.WriteModuleResources(module, childObjs) + } + + for _, obj := range childObjs { + resourceName, err := kClient.GVKtoAPIResourceName(obj.GroupVersionKind().GroupVersion(), obj.GroupVersionKind().Kind) + if err != nil { + installErrors = append(installErrors, fmt.Sprintf( + "%v%v/%v %v/%v failed to apply: %v", + obj.GroupVersionKind().Group, + obj.GroupVersionKind().Version, + obj.GroupVersionKind().Kind, + obj.GetNamespace(), + obj.GetName(), + err.Error(), + )) + + continue + } + + gvr := cyclopsv1alpha1.GroupVersionResource{ + Group: obj.GroupVersionKind().Group, + Version: obj.GroupVersionKind().Version, + Resource: resourceName, + } + if err := kClient.CreateDynamic(gvr, &obj, module.Spec.TargetNamespace); err != nil { installErrors = append(installErrors, fmt.Sprintf( "%v%v/%v %v/%v failed to apply: %v", @@ -359,12 +373,6 @@ func (r *ModuleReconciler) generateResources( return installErrors, childrenGVRs, nil } -func (r *ModuleReconciler) writeResourceToGit( - obj string, -) error { - r.gitWriteClient.Write() -} - func (r *ModuleReconciler) applyCRDs(template *models.Template) []string { installErrors := make([]string, 0) From 4c575a232bfb968c27687b9c924234bc227905e1 Mon Sep 17 00:00:00 2001 From: petar-cvit Date: Tue, 1 Jul 2025 14:24:18 +0200 Subject: [PATCH 05/11] update CRD install --- install/cyclops-install.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/install/cyclops-install.yaml b/install/cyclops-install.yaml index 679785f0..1f301a08 100644 --- a/install/cyclops-install.yaml +++ b/install/cyclops-install.yaml @@ -1,3 +1,4 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -111,10 +112,13 @@ spec: type: string version: type: string + writeResources: + type: boolean required: - path - repo - version + - writeResources type: object path: type: string @@ -358,10 +362,13 @@ spec: type: string version: type: string + writeResources: + type: boolean required: - path - repo - version + - writeResources type: object path: type: string From 26a2b5813711c08f998448836cdb47a01fa5dbf6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 1 Jul 2025 15:52:45 +0000 Subject: [PATCH 06/11] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20update=20cyclops=20t?= =?UTF-8?q?o=20v0.22.0-rc.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- install/cyclops-install.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/cyclops-install.yaml b/install/cyclops-install.yaml index 1f301a08..4375e876 100644 --- a/install/cyclops-install.yaml +++ b/install/cyclops-install.yaml @@ -465,7 +465,7 @@ spec: spec: containers: - name: cyclops-ui - image: cyclopsui/cyclops-ui:v0.21.1 + image: cyclopsui/cyclops-ui:v0.22.0-rc.2 ports: - containerPort: 80 env: @@ -530,7 +530,7 @@ spec: serviceAccountName: cyclops-ctrl containers: - name: cyclops-ctrl - image: cyclopsui/cyclops-ctrl:v0.21.1 + image: cyclopsui/cyclops-ctrl:v0.22.0-rc.2 ports: - containerPort: 8080 env: From 3926135936f5d8a58ec0f6d324536625260089c7 Mon Sep 17 00:00:00 2001 From: petar-cvit Date: Mon, 7 Jul 2025 17:03:57 +0200 Subject: [PATCH 07/11] bump to v0.21.1 --- README.md | 2 +- web/docs/installation/install/manifest.md | 2 +- web/src/components/Install/Install/index.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index aa3bf862..1dd3182c 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Cyclops can either be installed manually by applying the latest manifest, by usi To install Cyclops using `kubectl` into your cluster, run the commands below: ```bash -kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/cyclops-install.yaml && kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/demo-templates.yaml +kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/cyclops-install.yaml && kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/demo-templates.yaml ``` It will create a new namespace called `cyclops` and deploy everything you need for your Cyclops instance to run. diff --git a/web/docs/installation/install/manifest.md b/web/docs/installation/install/manifest.md index 525beeba..6cbe0789 100644 --- a/web/docs/installation/install/manifest.md +++ b/web/docs/installation/install/manifest.md @@ -7,7 +7,7 @@ sidebar_label: Using kubectl To install Cyclops in your cluster, run commands below: ```bash -kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/cyclops-install.yaml && kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/demo-templates.yaml +kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/cyclops-install.yaml && kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/demo-templates.yaml ``` It will create a new namespace called `cyclops` and deploy everything you need for your Cyclops instance to run. diff --git a/web/src/components/Install/Install/index.js b/web/src/components/Install/Install/index.js index 6d6ea9ec..46394497 100644 --- a/web/src/components/Install/Install/index.js +++ b/web/src/components/Install/Install/index.js @@ -8,8 +8,8 @@ const InstallCmd = () => {

Install it with a single command

- {"kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/cyclops-install.yaml && \n" + - "kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.0/install/demo-templates.yaml"} + {"kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/cyclops-install.yaml && \n" + + "kubectl apply -f https://raw.githubusercontent.com/cyclops-ui/cyclops/v0.21.1/install/demo-templates.yaml"}
); From 4e92dceea2ea9e09a468410a3b9876d0193158c5 Mon Sep 17 00:00:00 2001 From: petar-cvit Date: Mon, 7 Jul 2025 17:09:35 +0200 Subject: [PATCH 08/11] create module in local cluster --- cyclops-ctrl/internal/controller/modules.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cyclops-ctrl/internal/controller/modules.go b/cyclops-ctrl/internal/controller/modules.go index ce9444d4..d6546936 100644 --- a/cyclops-ctrl/internal/controller/modules.go +++ b/cyclops-ctrl/internal/controller/modules.go @@ -319,7 +319,6 @@ func (m *Modules) CreateModule(ctx *gin.Context) { fmt.Println(err) ctx.JSON(http.StatusInternalServerError, dto.NewError("Error pushing to git", err.Error())) } - return } err = m.kubernetesClient.CreateModule(module) From a4d5d8154777511a34c28cd3ebc851dc198b7206 Mon Sep 17 00:00:00 2001 From: petar-cvit Date: Tue, 22 Jul 2025 20:02:10 +0200 Subject: [PATCH 09/11] remove unused write method --- cyclops-ctrl/internal/git/writeclient.go | 97 ------------------------ 1 file changed, 97 deletions(-) diff --git a/cyclops-ctrl/internal/git/writeclient.go b/cyclops-ctrl/internal/git/writeclient.go index 61dcb0ea..4125daa9 100644 --- a/cyclops-ctrl/internal/git/writeclient.go +++ b/cyclops-ctrl/internal/git/writeclient.go @@ -214,103 +214,6 @@ func getModulePath(module cyclopsv1alpha1.Module) (string, error) { return o.String(), nil } -//func (c *WriteClient) Write(module cyclopsv1alpha1.Module) error { -// module.Status.ReconciliationStatus = nil -// module.Status.ManagedGVRs = nil -// -// repoURL, exists := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteRepoAnnotation] -// if !exists { -// return errors.New(fmt.Sprintf("module passed to write without git repository; set cyclops-ui.com/write-repo annotation in module %v", module.Name)) -// } -// -// path, err := getModulePath(module) -// if err != nil { -// return err -// } -// -// revision := module.GetAnnotations()[cyclopsv1alpha1.GitOpsWriteRevisionAnnotation] -// -// creds, err := c.templatesResolver.RepoAuthCredentials(repoURL) -// if err != nil { -// return err -// } -// -// if creds == nil { -// return errors.New(fmt.Sprintf("failed to fetch creds for repo %v: check template auth rules", repoURL)) -// } -// -// storer := memory.NewStorage() -// fs := memfs.New() -// -// repo, worktree, err := cloneRepo(repoURL, revision, storer, &fs, creds) -// if err != nil { -// if errors.Is(err, git.NoMatchingRefSpecError{}) { -// storer = memory.NewStorage() -// fs = memfs.New() -// repo, worktree, err = cloneRepo(repoURL, "", storer, &fs, creds) -// if err != nil { -// return err -// } -// -// err = worktree.Checkout(&git.CheckoutOptions{ -// Branch: plumbing.NewBranchReferenceName(revision), -// Create: true, -// }) -// if err != nil { -// return err -// } -// } else { -// return err -// } -// } -// -// path = moduleFilePath(path, module.Name) -// -// file, err := fs.Create(path) -// if err != nil { -// return fmt.Errorf("failed to create file in repository: %w", err) -// } -// -// moduleData, err := yaml.Marshal(module) -// if err != nil { -// return err -// } -// -// if _, err := file.Write(moduleData); err != nil { -// return fmt.Errorf("failed to write JSON data to file: %w", err) -// } -// file.Close() -// -// if _, err := worktree.Add(path); err != nil { -// fmt.Println("err worktree.Add", path) -// return fmt.Errorf("failed to add file to worktree: %w", err) -// } -// -// var o bytes.Buffer -// err = c.commitMessageTemplate.Execute(&o, module.ObjectMeta) -// if err != nil { -// return err -// } -// -// _, err = worktree.Commit(o.String(), &git.CommitOptions{ -// Author: &object.Signature{ -// Name: creds.Username, -// When: time.Now(), -// }, -// }) -// if err != nil { -// return fmt.Errorf("failed to commit changes: %w", err) -// } -// -// if err := repo.Push(&git.PushOptions{ -// Auth: httpBasicAuthCredentials(creds), -// }); err != nil { -// return fmt.Errorf("failed to push changes: %w", err) -// } -// -// return nil -//} - func moduleFilePath(path, moduleName string) string { if path2.Ext(path) != "yaml" || path2.Ext(path) != "yml" { path = path2.Join(path, fmt.Sprintf("%v.yaml", moduleName)) From 26d45c7dd612e6ec7b028455cff0540829301985 Mon Sep 17 00:00:00 2001 From: petar-cvit Date: Tue, 22 Jul 2025 20:03:44 +0200 Subject: [PATCH 10/11] remove yaml separator --- install/cyclops-install.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/install/cyclops-install.yaml b/install/cyclops-install.yaml index 4375e876..f838b5d3 100644 --- a/install/cyclops-install.yaml +++ b/install/cyclops-install.yaml @@ -1,4 +1,3 @@ ---- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: From 7c26a6e40208b333df8fe693170fd9af77404339 Mon Sep 17 00:00:00 2001 From: petar-cvit Date: Mon, 28 Jul 2025 17:48:52 +0200 Subject: [PATCH 11/11] optional write child resouces flag --- cyclops-ctrl/api/v1alpha1/module_types.go | 10 ++++++---- .../config/crd/bases/cyclops-ui.com_modules.yaml | 1 - .../crd/bases/cyclops-ui.com_templatestores.yaml | 1 - install/cyclops-install.yaml | 3 +-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cyclops-ctrl/api/v1alpha1/module_types.go b/cyclops-ctrl/api/v1alpha1/module_types.go index c2b597ca..9050a8e3 100644 --- a/cyclops-ctrl/api/v1alpha1/module_types.go +++ b/cyclops-ctrl/api/v1alpha1/module_types.go @@ -58,10 +58,12 @@ const ( ) type GitOpsWriteDestination struct { - Repo string `json:"repo"` - Path string `json:"path"` - Version string `json:"version"` - WriteResources bool `json:"writeResources"` + Repo string `json:"repo"` + Path string `json:"path"` + Version string `json:"version"` + + // +kubebuilder:validation:Optional + WriteResources bool `json:"writeResources"` } type TemplateRef struct { diff --git a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml index cd9614d1..d7a3eedc 100644 --- a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml +++ b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_modules.yaml @@ -118,7 +118,6 @@ spec: - path - repo - version - - writeResources type: object path: type: string diff --git a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml index 4a0a7df5..f9716311 100644 --- a/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml +++ b/cyclops-ctrl/config/crd/bases/cyclops-ui.com_templatestores.yaml @@ -69,7 +69,6 @@ spec: - path - repo - version - - writeResources type: object path: type: string diff --git a/install/cyclops-install.yaml b/install/cyclops-install.yaml index f838b5d3..f8a5eb61 100644 --- a/install/cyclops-install.yaml +++ b/install/cyclops-install.yaml @@ -1,3 +1,4 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -117,7 +118,6 @@ spec: - path - repo - version - - writeResources type: object path: type: string @@ -367,7 +367,6 @@ spec: - path - repo - version - - writeResources type: object path: type: string