diff --git a/README.md b/README.md index 6ad6899..775f2e0 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Flags: -f, --force Force overwriting existing repositories -h, --help help for import -i, --input .repos Path to input .repos file + -s, --recurse-submodules Recursively clone submodules -r, --recursive .repos Recursively search of other .repos file in the cloned repositories -n, --retry int Number of attempts to import repositories (default 2) -l, --shallow Clone repositories with a depth of 1 diff --git a/cmd/import.go b/cmd/import.go index a8635e8..66ffc94 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -40,11 +40,12 @@ import cycle.`, depthRecursive, _ := cmd.Flags().GetInt("depth-recursive") numWorkers, _ := cmd.Flags().GetInt("workers") excludeList, _ := cmd.Flags().GetStringSlice("exclude") + recurseSubmodules, _ := cmd.Flags().GetBool("recurse-submodules") var hardCodedExcludeList = []string{} // Import repository files in the given file - validFile, hardCodedExcludeList := singleCloneSweep(cloningPath, filePath, numWorkers, overwriteExisting, shallowClone, numRetries) + validFile, hardCodedExcludeList := singleCloneSweep(cloningPath, filePath, numWorkers, overwriteExisting, shallowClone, numRetries, recurseSubmodules) if !validFile { os.Exit(1) } @@ -52,7 +53,7 @@ import cycle.`, os.Exit(0) } excludeList = append(excludeList, hardCodedExcludeList...) - nestedImportClones(cloningPath, filePath, depthRecursive, numWorkers, overwriteExisting, shallowClone, numRetries, excludeList) + nestedImportClones(cloningPath, filePath, depthRecursive, numWorkers, overwriteExisting, shallowClone, numRetries, excludeList, recurseSubmodules) }, } @@ -68,9 +69,10 @@ func init() { importCmd.Flags().BoolP("shallow", "l", false, "Clone repositories with a depth of 1") importCmd.Flags().IntP("workers", "w", 8, "Number of concurrent workers to use") importCmd.Flags().StringSliceP("exclude", "x", []string{}, "List of files and/or directories to exclude when performing a recursive import") + importCmd.Flags().BoolP("recurse-submodules", "s", false, "Recursively clone submodules") } -func singleCloneSweep(root string, filePath string, numWorkers int, overwriteExisting bool, shallowClone bool, numRetries int) (bool, []string) { +func singleCloneSweep(root string, filePath string, numWorkers int, overwriteExisting bool, shallowClone bool, numRetries int, recurseSubmodules bool) (bool, []string) { utils.PrintSeparator() utils.PrintSection(fmt.Sprintf("Importing from %s", filePath)) utils.PrintSeparator() @@ -102,7 +104,7 @@ func singleCloneSweep(root string, filePath string, numWorkers int, overwriteExi } else { success := false for range numRetries { - success = utils.PrintGitClone(job.Repo.URL, job.Repo.Version, job.RepoPath, overwriteExisting, shallowClone, false) + success = utils.PrintGitClone(job.Repo.URL, job.Repo.Version, job.RepoPath, overwriteExisting, shallowClone, false, recurseSubmodules) if success { break } @@ -142,7 +144,7 @@ func singleCloneSweep(root string, filePath string, numWorkers int, overwriteExi return validFile, allExcludes } -func nestedImportClones(cloningPath string, initialFilePath string, depthRecursive int, numWorkers int, overwriteExisting bool, shallowClone bool, numRetries int, excludeList []string) { +func nestedImportClones(cloningPath string, initialFilePath string, depthRecursive int, numWorkers int, overwriteExisting bool, shallowClone bool, numRetries int, excludeList []string, recurseSubmodules bool) { // Recursively import .repos files found clonedReposFiles := map[string]bool{initialFilePath: true} validFiles := true @@ -201,7 +203,7 @@ func nestedImportClones(cloningPath string, initialFilePath string, depthRecursi clonedReposFiles[filePathToClone] = false continue } - validFiles, hardCodedExcludeList = singleCloneSweep(cloningPath, filePathToClone, numWorkers, overwriteExisting, shallowClone, numRetries) + validFiles, hardCodedExcludeList = singleCloneSweep(cloningPath, filePathToClone, numWorkers, overwriteExisting, shallowClone, numRetries, recurseSubmodules) clonedReposFiles[filePathToClone] = true newReposFileFound = true if !validFiles { diff --git a/test/git_helpers_test.go b/test/git_helpers_test.go index 7935b34..b5b3674 100644 --- a/test/git_helpers_test.go +++ b/test/git_helpers_test.go @@ -85,14 +85,14 @@ func TestGitStatus(t *testing.T) { func TestGetGitBranch(t *testing.T) { testingBranch := "jazzy" repoPath := "/tmp/testdata/demos_branch" - if utils.GitClone("https://github.com/ros2/demos.git", testingBranch, repoPath, true, false, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/demos.git", testingBranch, repoPath, true, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } if utils.GetGitBranch(repoPath) != testingBranch { t.Errorf("Failed to get main branch for valid git repository") } testingTag := "0.34.0" - if utils.GitClone("https://github.com/ros2/demos.git", testingTag, repoPath, true, false, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/demos.git", testingTag, repoPath, true, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } obtainedTag := utils.GetGitBranch(repoPath) @@ -104,7 +104,7 @@ func TestGetGitBranch(t *testing.T) { func TestGetGitCommitSha(t *testing.T) { testingSha := "839b622bc40ec62307d6ba0615adb9b8bd1cbc30" repoPath := "/tmp/testdata/demos_sha" - if utils.GitClone("https://github.com/ros2/demos.git", testingSha, repoPath, false, false, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/demos.git", testingSha, repoPath, false, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } if utils.GetGitCommitSha(repoPath) != testingSha { @@ -115,7 +115,7 @@ func TestGetGitCommitSha(t *testing.T) { func TestGetGitRemoteURL(t *testing.T) { repoPath := "/tmp/testdata/demos_url" remoteUrl := "https://github.com/ros2/demos.git" - if utils.GitClone(remoteUrl, "", repoPath, false, false, false) != utils.SuccessfullClone { + if utils.GitClone(remoteUrl, "", repoPath, false, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } if utils.GetGitRemoteURL(repoPath) != remoteUrl { @@ -125,7 +125,7 @@ func TestGetGitRemoteURL(t *testing.T) { func TestGitPull(t *testing.T) { repoPath := "/tmp/testdata/demos_pull" - if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } msg := utils.PullGitRepo(repoPath) @@ -185,38 +185,44 @@ func TestIsValidCommitSha(t *testing.T) { func TestCloneGitRepo(t *testing.T) { repoPath := "/tmp/testdata/demos_clone" - if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } - if utils.GitClone("https://github.com/ros2/ros2cli", "", "/tmp/testdata/ros2cli", false, false, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/ros2cli", "", "/tmp/testdata/ros2cli", false, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } - if utils.GitClone("https://github.com/ros2/sadasdasd.git", "", "/tmp/testdata/sdasda", false, false, false) != utils.FailedClone { + if utils.GitClone("https://github.com/ros2/sadasdasd.git", "", "/tmp/testdata/sdasda", false, false, false, false) != utils.FailedClone { t.Errorf("Expected to fail to clone git repository") } - if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, false, false, false) != utils.SkippedClone { + if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, false, false, false, false) != utils.SkippedClone { t.Errorf("Expected to skip to clone git repository") } - if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, true, false, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, true, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to overwrite found git repository") } - if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, true, true, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, true, true, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully to clone git repository with shallow enabled") } + if utils.GitClone("https://github.com/cyberbotics/webots_ros2.git", "", "/tmp/testdata/webots_ros2", false, false, false, true) != utils.SuccessfullClone { + t.Errorf("Expected to successfully to clone git repository with submodules") + } + if utils.GitClone("https://github.com/cyberbotics/webots_ros2.git", "", "/tmp/testdata/webots_ros2", true, true, false, true) != utils.SuccessfullClone { + t.Errorf("Expected to successfully to clone git repository with submodules and shallow enabled") + } count, err := utils.RunGitCmd(repoPath, "rev-list", nil, []string{"--all", "--count"}...) if err != nil || strings.TrimSpace(count) != "1" { t.Errorf("Expected to have a shallow clone of the git repository") } testingSha := "839b622bc40ec62307d6ba0615adb9b8bd1cbc30" - if utils.GitClone("https://github.com/ros2/demos.git", testingSha, repoPath, true, false, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/demos.git", testingSha, repoPath, true, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository given a SHA") } } func TestGitSwitch(t *testing.T) { repoPath := "/tmp/testdata/switch_test" - if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false) != utils.SuccessfullClone { + if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } _, err := utils.GitSwitch(repoPath, "humble", false, false) diff --git a/test/repos_helpers_test.go b/test/repos_helpers_test.go index e8e9de2..803278e 100644 --- a/test/repos_helpers_test.go +++ b/test/repos_helpers_test.go @@ -139,7 +139,7 @@ func TestParseRepositoryInfo(t *testing.T) { repoPath := "/tmp/testdata/demos_parse" repoURL := "https://github.com/ros2/demos.git" repoVersion := "rolling" - if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false) != utils.SuccessfullClone { + if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } @@ -149,7 +149,7 @@ func TestParseRepositoryInfo(t *testing.T) { } repoVersion = "839b622bc40ec62307d6ba0615adb9b8bd1cbc30" - if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false) != utils.SuccessfullClone { + if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } repository = utils.ParseRepositoryInfo(repoPath, true) @@ -158,7 +158,7 @@ func TestParseRepositoryInfo(t *testing.T) { } repoVersion = "0.34.0" - if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false) != utils.SuccessfullClone { + if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false, false) != utils.SuccessfullClone { t.Errorf("Expected to successfully clone git repository") } repository = utils.ParseRepositoryInfo(repoPath, false) diff --git a/utils/git_helpers.go b/utils/git_helpers.go index 17e0a20..72e0543 100644 --- a/utils/git_helpers.go +++ b/utils/git_helpers.go @@ -210,7 +210,7 @@ func IsValidSha(sha string) bool { } // GitClone Clone a given repository URL -func GitClone(url string, version string, clonePath string, overwriteExisting bool, shallowClone bool, enablePrompt bool) int { +func GitClone(url string, version string, clonePath string, overwriteExisting bool, shallowClone bool, enablePrompt bool, recurseSubmodules bool) int { // Check if clonePath exists if _, err := os.Stat(clonePath); err == nil { @@ -244,6 +244,12 @@ func GitClone(url string, version string, clonePath string, overwriteExisting bo if shallowClone { cmdArgs = append(cmdArgs, "--depth", "1") } + if recurseSubmodules { + cmdArgs = append(cmdArgs, "--recurse-submodules") + if shallowClone { + cmdArgs = append(cmdArgs, "--shallow-submodules") + } + } if _, err := RunGitCmd(".", "clone", envConfig, cmdArgs...); err != nil { return FailedClone } @@ -308,10 +314,10 @@ func PrintCheckGit(path string, url string, version string, enablePrompt bool) b } // PrintGitClone Pretty print git clone -func PrintGitClone(url string, version string, path string, overwriteExisting bool, shallowClone bool, enablePrompt bool) bool { +func PrintGitClone(url string, version string, path string, overwriteExisting bool, shallowClone bool, enablePrompt bool, recurseSubmodules bool) bool { var cloneMsg string var cloneSuccessful bool - statusClone := GitClone(url, version, path, overwriteExisting, shallowClone, enablePrompt) + statusClone := GitClone(url, version, path, overwriteExisting, shallowClone, enablePrompt, recurseSubmodules) switch statusClone { case SuccessfullClone: cloneMsg = fmt.Sprintf("Successfully cloned git repository '%s' with version '%s'\n", url, version)