diff --git a/.golangci.yml b/.golangci.yml index 5a659553..600bef78 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,13 +1,2 @@ -version: "2" - run: timeout: 5m - -linters-settings: - staticcheck: - go: "1.25" - checks: ["all", "-ST1005"] - -issues: - exclude: - - "ST1005" \ No newline at end of file diff --git a/generators/github/git_repo.go b/generators/github/git_repo.go index a57c6217..577fdec4 100644 --- a/generators/github/git_repo.go +++ b/generators/github/git_repo.go @@ -6,8 +6,10 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strings" + giturlparse "github.com/git-download-manager/git-url-parse" "github.com/meshery/meshkit/generators/models" "github.com/meshery/meshkit/utils" "github.com/meshery/meshkit/utils/helm" @@ -50,13 +52,29 @@ func (gr GitRepo) GetContent() (models.Package, error) { _ = br.Flush() _ = fd.Close() }() + + // If root is not specified, enable recursive traversal from root to discover CRDs automatically + // This makes the generator robust to repository structure changes + rootPath := root + isAutoDiscovery := rootPath == "" + if isAutoDiscovery { + // Use "/**" to enable recursive traversal from repository root + rootPath = "/**" + } + gw := gitWalker. Owner(owner). Repo(repo). Branch(branch). - Root(root). - RegisterFileInterceptor(fileInterceptor(br)). - RegisterDirInterceptor(dirInterceptor(br)) + Root(rootPath). + RegisterFileInterceptor(crdAwareFileInterceptor(br)) + + // Register dirInterceptor to handle Helm charts which may contain CRDs + // Note: When doing automatic discovery (recurse mode), dirInterceptor processes directories + // and fileInterceptor processes files. For Helm charts, dirInterceptor extracts CRDs from + // the chart structure, while fileInterceptor finds standalone CRD files. This ensures we + // discover CRDs in both formats without missing any. + gw = gw.RegisterDirInterceptor(dirInterceptor(br)) if version != "" { gw = gw.ReferenceName(fmt.Sprintf("refs/tags/%s", version)) @@ -77,18 +95,38 @@ func (gr GitRepo) GetContent() (models.Package, error) { }, nil } +// parseGitURL parses a git URL and extracts owner, repo, branch, and path components +func parseGitURL(rawURL *url.URL) (owner, repo, branch, path string, err error) { + gitRepository := giturlparse.NewGitRepository("", "", rawURL.String(), "") + if err := gitRepository.Parse("", 0, ""); err != nil { + return "", "", "", "", err + } + + owner = gitRepository.Owner + repo = gitRepository.Name + branch = gitRepository.Branch + if branch == "" { + branch = "main" + } + path = gitRepository.Path + + if owner == "" || repo == "" { + return "", "", "", "", fmt.Errorf("invalid git URL format: must have at least owner/repo in path: %s", rawURL.String()) + } + + return owner, repo, branch, path, nil +} + func (gr GitRepo) extractRepoDetailsFromSourceURL() (owner, repo, branch, root string, err error) { - parts := strings.SplitN(strings.TrimPrefix(gr.URL.Path, "/"), "/", 4) - size := len(parts) - if size > 3 { - owner = parts[0] - repo = parts[1] - branch = parts[2] - root = parts[3] - - } else { - err = ErrInvalidGitHubSourceURL(fmt.Errorf("Source URL %s is invalid, specify owner, repo, branch and filepath in the url according to the specified source url format", gr.URL.String())) + owner, repo, branch, root, err = parseGitURL(gr.URL) + if err != nil { + err = ErrInvalidGitHubSourceURL(err) + return } + + // If root is empty, we'll use "/**" for recursive traversal in GetContent + // This enables automatic CRD discovery + return } @@ -96,6 +134,7 @@ func (gr GitRepo) ExtractRepoDetailsFromSourceURL() (owner, repo, branch, root s return gr.extractRepoDetailsFromSourceURL() } +// fileInterceptor processes all files (original behavior) func fileInterceptor(br *bufio.Writer) walker.FileInterceptor { return func(file walker.File) error { tempPath := filepath.Join(os.TempDir(), utils.GetRandomAlphabetsOfDigit(5)) @@ -103,6 +142,81 @@ func fileInterceptor(br *bufio.Writer) walker.FileInterceptor { } } +// crdAwareFileInterceptor only processes files that contain CRDs +// This enables automatic CRD discovery without requiring specific directory paths +func crdAwareFileInterceptor(br *bufio.Writer) walker.FileInterceptor { + return func(file walker.File) error { + // Check if the file is a YAML/JSON file that might contain CRDs + fileName := strings.ToLower(file.Name) + isYAML := strings.HasSuffix(fileName, ".yaml") || strings.HasSuffix(fileName, ".yml") + isJSON := strings.HasSuffix(fileName, ".json") + + if !isYAML && !isJSON { + // Skip non-YAML/JSON files + return nil + } + + // Check if the file content contains a CRD + // Handle both single-document and multi-document YAML files + content := file.Content + + // For multi-document YAML, split by document separator and check each + documents := strings.Split(content, "\n---\n") + // Also handle documents separated by "---" at the start of a line + if len(documents) == 1 { + // Try splitting by lines starting with "---" + lines := strings.Split(content, "\n") + var docs []string + var currentDoc strings.Builder + for _, line := range lines { + if strings.TrimSpace(line) == "---" && currentDoc.Len() > 0 { + docs = append(docs, currentDoc.String()) + currentDoc.Reset() + } else { + if currentDoc.Len() > 0 { + currentDoc.WriteString("\n") + } + currentDoc.WriteString(line) + } + } + if currentDoc.Len() > 0 { + docs = append(docs, currentDoc.String()) + } + if len(docs) > 1 { + documents = docs + } + } + + // Check each document for CRD + hasCRD := false + for _, doc := range documents { + doc = strings.TrimSpace(doc) + if doc == "" { + continue + } + // Check for YAML format + if match, _ := regexp.MatchString(`kind:\s*CustomResourceDefinition`, doc); match { + hasCRD = true + break + } + // Check for JSON format + if match, _ := regexp.MatchString(`"kind"\s*:\s*"CustomResourceDefinition"`, doc); match { + hasCRD = true + break + } + } + + if !hasCRD { + // File doesn't contain a CRD, skip it + return nil + } + + // File contains a CRD, process it + tempPath := filepath.Join(os.TempDir(), utils.GetRandomAlphabetsOfDigit(5)) + return ProcessContent(br, tempPath, file.Path) + } +} + // When passing a directory to extract charts and the format introspector is provided as file/dir interceptor i.e. ConvertToK8sManifest ensure the Recurese is off. It is required othweise we will process the dir as well as process the file in that dir separately. // Add more calrifying commment and entry inside docs. func dirInterceptor(br *bufio.Writer) walker.DirInterceptor { diff --git a/generators/github/scheme_interface.go b/generators/github/scheme_interface.go index cda361e7..56d18acd 100644 --- a/generators/github/scheme_interface.go +++ b/generators/github/scheme_interface.go @@ -3,6 +3,7 @@ package github import ( "net/url" + giturlparse "github.com/git-download-manager/git-url-parse" "github.com/meshery/meshkit/generators/models" ) @@ -11,6 +12,14 @@ type DownloaderScheme interface { } func NewDownloaderForScheme(scheme string, url *url.URL, packageName string) DownloaderScheme { + // Check if this is a GitHub URL - route to GitRepo for automatic CRD discovery + if isGitHubURL(url) { + return GitRepo{ + URL: url, + PackageName: packageName, + } + } + switch scheme { case "git": return GitRepo{ @@ -27,3 +36,16 @@ func NewDownloaderForScheme(scheme string, url *url.URL, packageName string) Dow } return nil } + +func isGitHubURL(u *url.URL) bool { + gitRepository := giturlparse.NewGitRepository("", "", u.String(), "") + if err := gitRepository.Parse("", 0, ""); err != nil { + return false + } + + if gitRepository.Hostname != "github.com" { + return false + } + + return gitRepository.Owner != "" && gitRepository.Name != "" +} diff --git a/go.mod b/go.mod index bb542625..11c763fb 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/docker/cli v27.5.1+incompatible github.com/fluxcd/pkg/oci v0.43.1 github.com/fluxcd/pkg/tar v0.10.0 + github.com/git-download-manager/git-url-parse v1.0.3 github.com/go-git/go-git/v5 v5.16.4 github.com/go-logr/logr v1.4.3 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1