Add recurse-submodules option

README.md

@@ -91,6 +91,7 @@ Flags:
   -f, --force                 Force overwriting existing repositories
   -h, --help                  help for import
   -i, --input .repos          Path to input .repos file
+  -s, --recurse-submodules    Recursively clone submodules
   -r, --recursive .repos      Recursively search of other .repos file in the cloned repositories
   -n, --retry int             Number of attempts to import repositories (default 2)
   -l, --shallow               Clone repositories with a depth of 1

cmd/import.go

@@ -40,19 +40,20 @@ import cycle.`,
 		depthRecursive, _ := cmd.Flags().GetInt("depth-recursive")
 		numWorkers, _ := cmd.Flags().GetInt("workers")
 		excludeList, _ := cmd.Flags().GetStringSlice("exclude")
+		recurseSubmodules, _ := cmd.Flags().GetBool("recurse-submodules")
 
 		var hardCodedExcludeList = []string{}
 
 		// Import repository files in the given file
-		validFile, hardCodedExcludeList := singleCloneSweep(cloningPath, filePath, numWorkers, overwriteExisting, shallowClone, numRetries)
+		validFile, hardCodedExcludeList := singleCloneSweep(cloningPath, filePath, numWorkers, overwriteExisting, shallowClone, numRetries, recurseSubmodules)
 		if !validFile {
 			os.Exit(1)
 		}
 		if !recursiveFlag {
 			os.Exit(0)
 		}
 		excludeList = append(excludeList, hardCodedExcludeList...)
-		nestedImportClones(cloningPath, filePath, depthRecursive, numWorkers, overwriteExisting, shallowClone, numRetries, excludeList)
+		nestedImportClones(cloningPath, filePath, depthRecursive, numWorkers, overwriteExisting, shallowClone, numRetries, excludeList, recurseSubmodules)
 
 	},
 }
@@ -68,9 +69,10 @@ func init() {
 	importCmd.Flags().BoolP("shallow", "l", false, "Clone repositories with a depth of 1")
 	importCmd.Flags().IntP("workers", "w", 8, "Number of concurrent workers to use")
 	importCmd.Flags().StringSliceP("exclude", "x", []string{}, "List of files and/or directories to exclude when performing a recursive import")
+	importCmd.Flags().BoolP("recurse-submodules", "s", false, "Recursively clone submodules")
 }
 
-func singleCloneSweep(root string, filePath string, numWorkers int, overwriteExisting bool, shallowClone bool, numRetries int) (bool, []string) {
+func singleCloneSweep(root string, filePath string, numWorkers int, overwriteExisting bool, shallowClone bool, numRetries int, recurseSubmodules bool) (bool, []string) {
 	utils.PrintSeparator()
 	utils.PrintSection(fmt.Sprintf("Importing from %s", filePath))
 	utils.PrintSeparator()
@@ -102,7 +104,7 @@ func singleCloneSweep(root string, filePath string, numWorkers int, overwriteExi
 				} else {
 					success := false
 					for range numRetries {
-						success = utils.PrintGitClone(job.Repo.URL, job.Repo.Version, job.RepoPath, overwriteExisting, shallowClone, false)
+						success = utils.PrintGitClone(job.Repo.URL, job.Repo.Version, job.RepoPath, overwriteExisting, shallowClone, false, recurseSubmodules)
 						if success {
 							break
 						}
@@ -142,7 +144,7 @@ func singleCloneSweep(root string, filePath string, numWorkers int, overwriteExi
 	return validFile, allExcludes
 }
 
-func nestedImportClones(cloningPath string, initialFilePath string, depthRecursive int, numWorkers int, overwriteExisting bool, shallowClone bool, numRetries int, excludeList []string) {
+func nestedImportClones(cloningPath string, initialFilePath string, depthRecursive int, numWorkers int, overwriteExisting bool, shallowClone bool, numRetries int, excludeList []string, recurseSubmodules bool) {
 	// Recursively import .repos files found
 	clonedReposFiles := map[string]bool{initialFilePath: true}
 	validFiles := true
@@ -201,7 +203,7 @@ func nestedImportClones(cloningPath string, initialFilePath string, depthRecursi
 					clonedReposFiles[filePathToClone] = false
 					continue
 				}
-				validFiles, hardCodedExcludeList = singleCloneSweep(cloningPath, filePathToClone, numWorkers, overwriteExisting, shallowClone, numRetries)
+				validFiles, hardCodedExcludeList = singleCloneSweep(cloningPath, filePathToClone, numWorkers, overwriteExisting, shallowClone, numRetries, recurseSubmodules)
 				clonedReposFiles[filePathToClone] = true
 				newReposFileFound = true
 				if !validFiles {

test/git_helpers_test.go

@@ -85,14 +85,14 @@ func TestGitStatus(t *testing.T) {
 func TestGetGitBranch(t *testing.T) {
 	testingBranch := "jazzy"
 	repoPath := "/tmp/testdata/demos_branch"
-	if utils.GitClone("https://github.com/ros2/demos.git", testingBranch, repoPath, true, false, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", testingBranch, repoPath, true, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
 	if utils.GetGitBranch(repoPath) != testingBranch {
 		t.Errorf("Failed to get main branch for valid git repository")
 	}
 	testingTag := "0.34.0"
-	if utils.GitClone("https://github.com/ros2/demos.git", testingTag, repoPath, true, false, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", testingTag, repoPath, true, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
 	obtainedTag := utils.GetGitBranch(repoPath)
@@ -104,7 +104,7 @@ func TestGetGitBranch(t *testing.T) {
 func TestGetGitCommitSha(t *testing.T) {
 	testingSha := "839b622bc40ec62307d6ba0615adb9b8bd1cbc30"
 	repoPath := "/tmp/testdata/demos_sha"
-	if utils.GitClone("https://github.com/ros2/demos.git", testingSha, repoPath, false, false, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", testingSha, repoPath, false, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
 	if utils.GetGitCommitSha(repoPath) != testingSha {
@@ -115,7 +115,7 @@ func TestGetGitCommitSha(t *testing.T) {
 func TestGetGitRemoteURL(t *testing.T) {
 	repoPath := "/tmp/testdata/demos_url"
 	remoteUrl := "https://github.com/ros2/demos.git"
-	if utils.GitClone(remoteUrl, "", repoPath, false, false, false) != utils.SuccessfullClone {
+	if utils.GitClone(remoteUrl, "", repoPath, false, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
 	if utils.GetGitRemoteURL(repoPath) != remoteUrl {
@@ -125,7 +125,7 @@ func TestGetGitRemoteURL(t *testing.T) {
 
 func TestGitPull(t *testing.T) {
 	repoPath := "/tmp/testdata/demos_pull"
-	if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
 	msg := utils.PullGitRepo(repoPath)
@@ -185,38 +185,44 @@ func TestIsValidCommitSha(t *testing.T) {
 
 func TestCloneGitRepo(t *testing.T) {
 	repoPath := "/tmp/testdata/demos_clone"
-	if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
-	if utils.GitClone("https://github.com/ros2/ros2cli", "", "/tmp/testdata/ros2cli", false, false, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/ros2cli", "", "/tmp/testdata/ros2cli", false, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
-	if utils.GitClone("https://github.com/ros2/sadasdasd.git", "", "/tmp/testdata/sdasda", false, false, false) != utils.FailedClone {
+	if utils.GitClone("https://github.com/ros2/sadasdasd.git", "", "/tmp/testdata/sdasda", false, false, false, false) != utils.FailedClone {
 		t.Errorf("Expected to fail to clone git repository")
 	}
-	if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, false, false, false) != utils.SkippedClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, false, false, false, false) != utils.SkippedClone {
 		t.Errorf("Expected to skip to clone git repository")
 	}
-	if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, true, false, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, true, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to overwrite found git repository")
 	}
-	if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, true, true, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", "", repoPath, true, true, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully to clone git repository with shallow enabled")
 	}
+	if utils.GitClone("https://github.com/cyberbotics/webots_ros2.git", "", "/tmp/testdata/webots_ros2", false, false, false, true) != utils.SuccessfullClone {
+		t.Errorf("Expected to successfully to clone git repository with submodules")
+	}
+	if utils.GitClone("https://github.com/cyberbotics/webots_ros2.git", "", "/tmp/testdata/webots_ros2", true, true, false, true) != utils.SuccessfullClone {
+		t.Errorf("Expected to successfully to clone git repository with submodules and shallow enabled")
+	}
 	count, err := utils.RunGitCmd(repoPath, "rev-list", nil, []string{"--all", "--count"}...)
 	if err != nil || strings.TrimSpace(count) != "1" {
 		t.Errorf("Expected to have a shallow clone of the git repository")
 	}
 
 	testingSha := "839b622bc40ec62307d6ba0615adb9b8bd1cbc30"
-	if utils.GitClone("https://github.com/ros2/demos.git", testingSha, repoPath, true, false, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", testingSha, repoPath, true, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository given a SHA")
 	}
 }
 
 func TestGitSwitch(t *testing.T) {
 	repoPath := "/tmp/testdata/switch_test"
-	if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false) != utils.SuccessfullClone {
+	if utils.GitClone("https://github.com/ros2/demos.git", "rolling", repoPath, false, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
 	_, err := utils.GitSwitch(repoPath, "humble", false, false)

test/repos_helpers_test.go

@@ -139,7 +139,7 @@ func TestParseRepositoryInfo(t *testing.T) {
 	repoPath := "/tmp/testdata/demos_parse"
 	repoURL := "https://github.com/ros2/demos.git"
 	repoVersion := "rolling"
-	if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false) != utils.SuccessfullClone {
+	if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
 
@@ -149,7 +149,7 @@ func TestParseRepositoryInfo(t *testing.T) {
 	}
 
 	repoVersion = "839b622bc40ec62307d6ba0615adb9b8bd1cbc30"
-	if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false) != utils.SuccessfullClone {
+	if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
 	repository = utils.ParseRepositoryInfo(repoPath, true)
@@ -158,7 +158,7 @@ func TestParseRepositoryInfo(t *testing.T) {
 	}
 
 	repoVersion = "0.34.0"
-	if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false) != utils.SuccessfullClone {
+	if utils.GitClone(repoURL, repoVersion, repoPath, true, false, false, false) != utils.SuccessfullClone {
 		t.Errorf("Expected to successfully clone git repository")
 	}
 	repository = utils.ParseRepositoryInfo(repoPath, false)

utils/git_helpers.go

@@ -210,7 +210,7 @@ func IsValidSha(sha string) bool {
 }
 
 // GitClone Clone a given repository URL
-func GitClone(url string, version string, clonePath string, overwriteExisting bool, shallowClone bool, enablePrompt bool) int {
+func GitClone(url string, version string, clonePath string, overwriteExisting bool, shallowClone bool, enablePrompt bool, recurseSubmodules bool) int {
 
 	// Check if clonePath exists
 	if _, err := os.Stat(clonePath); err == nil {
@@ -244,6 +244,12 @@ func GitClone(url string, version string, clonePath string, overwriteExisting bo
 	if shallowClone {
 		cmdArgs = append(cmdArgs, "--depth", "1")
 	}
+	if recurseSubmodules {
+		cmdArgs = append(cmdArgs, "--recurse-submodules")
+		if shallowClone {
+			cmdArgs = append(cmdArgs, "--shallow-submodules")
+		}
+	}
 	if _, err := RunGitCmd(".", "clone", envConfig, cmdArgs...); err != nil {
 		return FailedClone
 	}
@@ -308,10 +314,10 @@ func PrintCheckGit(path string, url string, version string, enablePrompt bool) b
 }
 
 // PrintGitClone Pretty print git clone
-func PrintGitClone(url string, version string, path string, overwriteExisting bool, shallowClone bool, enablePrompt bool) bool {
+func PrintGitClone(url string, version string, path string, overwriteExisting bool, shallowClone bool, enablePrompt bool, recurseSubmodules bool) bool {
 	var cloneMsg string
 	var cloneSuccessful bool
-	statusClone := GitClone(url, version, path, overwriteExisting, shallowClone, enablePrompt)
+	statusClone := GitClone(url, version, path, overwriteExisting, shallowClone, enablePrompt, recurseSubmodules)
 	switch statusClone {
 	case SuccessfullClone:
 		cloneMsg = fmt.Sprintf("Successfully cloned git repository '%s' with version '%s'\n", url, version)