diff --git a/generators/generator.go b/generators/generator.go index cbf4fe1a..4ce4ac9f 100644 --- a/generators/generator.go +++ b/generators/generator.go @@ -14,7 +14,17 @@ const ( gitHub = "github" ) +type GeneratorOptions struct { + Recursive bool + MaxDepth int + Extensions []string +} + func NewGenerator(registrant, url, packageName string) (models.PackageManager, error) { + return NewGeneratorWithOptions(registrant, url, packageName, GeneratorOptions{}) +} + +func NewGeneratorWithOptions(registrant, url, packageName string, opts GeneratorOptions) (models.PackageManager, error) { registrant = utils.ReplaceSpacesAndConvertToLowercase(registrant) switch registrant { case artifactHub: @@ -26,6 +36,9 @@ func NewGenerator(registrant, url, packageName string) (models.PackageManager, e return github.GitHubPackageManager{ PackageName: packageName, SourceURL: url, + Recursive: opts.Recursive, + MaxDepth: opts.MaxDepth, + Extensions: opts.Extensions, }, nil } return nil, ErrUnsupportedRegistrant(fmt.Errorf("generator not implemented for the registrant %s", registrant)) diff --git a/generators/generator_test.go b/generators/generator_test.go new file mode 100644 index 00000000..ba446223 --- /dev/null +++ b/generators/generator_test.go @@ -0,0 +1,64 @@ +package generators + +import ( + "testing" + + "github.com/meshery/meshkit/generators/github" +) + +func TestNewGeneratorWithOptions(t *testing.T) { + tests := []struct { + name string + registrant string + url string + packageName string + opts GeneratorOptions + wantRec bool + wantDepth int + }{ + { + name: "Github Recursive", + registrant: "github", + url: "https://github.com/owner/repo", + packageName: "test", + opts: GeneratorOptions{ + Recursive: true, + MaxDepth: 5, + }, + wantRec: true, + wantDepth: 5, + }, + { + name: "Github Default", + registrant: "github", + url: "https://github.com/owner/repo", + packageName: "test", + opts: GeneratorOptions{ + Recursive: false, + MaxDepth: 0, + }, + wantRec: false, + wantDepth: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pm, err := NewGeneratorWithOptions(tt.registrant, tt.url, tt.packageName, tt.opts) + if err != nil { + t.Fatalf("NewGeneratorWithOptions() error = %v", err) + } + + if ghpm, ok := pm.(github.GitHubPackageManager); ok { + if ghpm.Recursive != tt.wantRec { + t.Errorf("NewGeneratorWithOptions() Recursive = %v, want %v", ghpm.Recursive, tt.wantRec) + } + if ghpm.MaxDepth != tt.wantDepth { + t.Errorf("NewGeneratorWithOptions() MaxDepth = %v, want %v", ghpm.MaxDepth, tt.wantDepth) + } + } else { + t.Errorf("NewGeneratorWithOptions() returned unexpected type") + } + }) + } +} diff --git a/generators/github/git_repo.go b/generators/github/git_repo.go index a57c6217..b34864dd 100644 --- a/generators/github/git_repo.go +++ b/generators/github/git_repo.go @@ -18,6 +18,9 @@ type GitRepo struct { // URL *url.URL PackageName string + Recursive bool + MaxDepth int + Extensions []string } // Assumpations: 1. Always a K8s manifest @@ -54,10 +57,23 @@ func (gr GitRepo) GetContent() (models.Package, error) { Owner(owner). Repo(repo). Branch(branch). - Root(root). + MaxDepth(gr.MaxDepth). + AllowedExtensions(gr.Extensions). RegisterFileInterceptor(fileInterceptor(br)). RegisterDirInterceptor(dirInterceptor(br)) + effectiveRoot := root + if gr.Recursive { + if !strings.HasSuffix(effectiveRoot, "/**") { + effectiveRoot += "/**" + } + } else { + if strings.HasSuffix(effectiveRoot, "/**") { + effectiveRoot = strings.TrimSuffix(effectiveRoot, "/**") + } + } + gw = gw.Root(effectiveRoot) + if version != "" { gw = gw.ReferenceName(fmt.Sprintf("refs/tags/%s", version)) } diff --git a/generators/github/package_manager.go b/generators/github/package_manager.go index df79bcae..1de97800 100644 --- a/generators/github/package_manager.go +++ b/generators/github/package_manager.go @@ -11,6 +11,9 @@ import ( type GitHubPackageManager struct { PackageName string SourceURL string + Recursive bool + MaxDepth int + Extensions []string } func (ghpm GitHubPackageManager) GetPackage() (models.Package, error) { @@ -21,7 +24,7 @@ func (ghpm GitHubPackageManager) GetPackage() (models.Package, error) { } protocol := url.Scheme - downloader := NewDownloaderForScheme(protocol, url, ghpm.PackageName) + downloader := NewDownloaderForScheme(protocol, url, ghpm) if downloader == nil { err = errors.New("unsupported protocol") return nil, ErrGenerateGitHubPackage(err, ghpm.PackageName) diff --git a/generators/github/recursive_test.go b/generators/github/recursive_test.go new file mode 100644 index 00000000..bf2adc0c --- /dev/null +++ b/generators/github/recursive_test.go @@ -0,0 +1,186 @@ +package github + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/meshery/meshkit/utils/walker" +) + +func TestRecursiveWalkFunctional(t *testing.T) { + tempDir, err := ioutil.TempDir("", "test-recursive-walk") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tempDir) + + r, err := git.PlainInit(tempDir, false) + if err != nil { + t.Fatalf("Failed to init git repo: %v", err) + } + w, err := r.Worktree() + if err != nil { + t.Fatalf("Failed to get worktree: %v", err) + } + + createFile(t, tempDir, "root.yaml", "content") + createFile(t, tempDir, "root.json", "content") + createFile(t, tempDir, "level1/level1.yaml", "content") + createFile(t, tempDir, "level1/level1.txt", "content") + createFile(t, tempDir, "level1/level2/level2.yaml", "content") + + _, err = w.Add(".") + if err != nil { + t.Fatalf("Failed to add files: %v", err) + } + _, err = w.Commit("Initial commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "Test", + Email: "test@example.com", + When: time.Now(), + }, + }) + if err != nil { + t.Fatalf("Failed to commit: %v", err) + } + + tests := []struct { + name string + recursive bool + maxDepth int + extensions []string + wantFiles []string + }{ + { + name: "Non-recursive (Root only, all types)", + recursive: false, + maxDepth: 0, + extensions: nil, + wantFiles: []string{"root.yaml", "root.json"}, + }, + { + name: "Recursive Unlimited (All types)", + recursive: true, + maxDepth: 0, + extensions: nil, + wantFiles: []string{"root.yaml", "root.json", "level1.yaml", "level1.txt", "level2.yaml"}, + }, + { + name: "Recursive MaxDepth 1 (All types)", + recursive: true, + maxDepth: 1, + extensions: nil, + wantFiles: []string{"root.yaml", "root.json", "level1.yaml", "level1.txt"}, + }, + { + name: "Recursive MaxDepth 2 (All types)", + recursive: true, + maxDepth: 2, + extensions: nil, + wantFiles: []string{"root.yaml", "root.json", "level1.yaml", "level1.txt", "level2.yaml"}, + }, + { + name: "Recursive Filtered (.yaml only)", + recursive: true, + maxDepth: 0, + extensions: []string{".yaml"}, + wantFiles: []string{"root.yaml", "level1.yaml", "level2.yaml"}, + }, + { + name: "Recursive Filtered (.yaml, .json)", + recursive: true, + maxDepth: 0, + extensions: []string{".yaml", ".json"}, + wantFiles: []string{"root.yaml", "root.json", "level1.yaml", "level2.yaml"}, + }, + { + name: "Recursive MaxDepth 1 Filtered (.yaml)", + recursive: true, + maxDepth: 1, + extensions: []string{".yaml"}, + wantFiles: []string{"root.yaml", "level1.yaml"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repoPath := filepath.ToSlash(tempDir) + if !strings.HasPrefix(repoPath, "/") { + repoPath = "/" + repoPath + } + fileURL := "file://" + repoPath + + walkerInst := walker.NewGit(). + Owner(""). + Repo(""). + BaseURL(fileURL). + Branch("master"). + Root(""). + MaxDepth(tt.maxDepth). + AllowedExtensions(tt.extensions) + + var collectedFiles []string + walkerInst.RegisterFileInterceptor(func(f walker.File) error { + collectedFiles = append(collectedFiles, f.Name) + return nil + }) + + if tt.recursive { + walkerInst.Root("/**") + } else { + walkerInst.Root("") + } + + err := walkerInst.Walk() + if err != nil { + t.Fatalf("Walk failed: %v", err) + } + + assertFilesEqual(t, collectedFiles, tt.wantFiles) + }) + } +} + +func assertFilesEqual(t *testing.T, got, want []string) { + gotMap := make(map[string]struct{}) + for _, f := range got { + gotMap[f] = struct{}{} + } + failed := false + for _, f := range want { + if _, ok := gotMap[f]; !ok { + t.Errorf("missing expected file: %s", f) + failed = true + } else { + delete(gotMap, f) + } + } + if len(gotMap) > 0 { + for f := range gotMap { + t.Errorf("unexpected file collected: %s", f) + failed = true + } + } + if failed { + t.Errorf("Got: %v", got) + t.Errorf("Want: %v", want) + } +} + +func createFile(t *testing.T, base, path, content string) { + fullPath := filepath.Join(base, path) + err := os.MkdirAll(filepath.Dir(fullPath), 0755) + if err != nil { + t.Fatalf("Failed to create dirs: %v", err) + } + err = ioutil.WriteFile(fullPath, []byte(content), 0644) + if err != nil { + t.Fatalf("Failed to create file: %v", err) + } +} diff --git a/generators/github/scheme_interface.go b/generators/github/scheme_interface.go index cda361e7..382c3196 100644 --- a/generators/github/scheme_interface.go +++ b/generators/github/scheme_interface.go @@ -10,19 +10,22 @@ type DownloaderScheme interface { GetContent() (models.Package, error) } -func NewDownloaderForScheme(scheme string, url *url.URL, packageName string) DownloaderScheme { +func NewDownloaderForScheme(scheme string, url *url.URL, gp GitHubPackageManager) DownloaderScheme { switch scheme { case "git": return GitRepo{ URL: url, - PackageName: packageName, + PackageName: gp.PackageName, + Recursive: gp.Recursive, + MaxDepth: gp.MaxDepth, + Extensions: gp.Extensions, } case "http": fallthrough case "https": return URL{ URL: url, - PackageName: packageName, + PackageName: gp.PackageName, } } return nil diff --git a/utils/walker/git.go b/utils/walker/git.go index 59687a92..f6c3ed21 100644 --- a/utils/walker/git.go +++ b/utils/walker/git.go @@ -29,6 +29,8 @@ type Git struct { fileInterceptor FileInterceptor dirInterceptor DirInterceptor referenceName plumbing.ReferenceName + maxDepth int + allowedExtensions []string } // NewGit returns a pointer to an instance of Git @@ -66,6 +68,30 @@ func (g *Git) MaxFileSize(size int64) *Git { return g } +func (g *Git) MaxDepth(depth int) *Git { + g.maxDepth = depth + return g +} + +func (g *Git) AllowedExtensions(ext []string) *Git { + g.allowedExtensions = ext + return g +} + +func (g *Git) isAllowedFile(name string) bool { + if len(g.allowedExtensions) == 0 { + return true // no filtering + } + + ext := strings.ToLower(filepath.Ext(name)) + for _, allowed := range g.allowedExtensions { + if ext == allowed { + return true + } + } + return false +} + // ShowLogs enable the logs and returns a pointer // to the same Git instance func (g *Git) ShowLogs() *Git { @@ -184,13 +210,29 @@ func clonewalk(g *Git) error { // If recurse mode is on, we will walk the tree if g.recurse { err = filepath.WalkDir(rootPath, func(path string, d fs.DirEntry, er error) error { - if d.IsDir() && g.dirInterceptor != nil { - return g.dirInterceptor(Directory{ - Name: d.Name(), - Path: path, - }) - } if d.IsDir() { + if d.Name() == ".git" { + return filepath.SkipDir + } + if g.maxDepth > 0 { + rel, err := filepath.Rel(rootPath, path) + if err == nil && rel != "." { + currentDepth := strings.Count(rel, string(os.PathSeparator)) + 1 + if currentDepth > g.maxDepth { + return filepath.SkipDir + } + } + } + + if g.dirInterceptor != nil { + return g.dirInterceptor(Directory{ + Name: d.Name(), + Path: path, + }) + } + return nil + } + if !g.isAllowedFile(d.Name()) { return nil } f, errInfo := d.Info()