[imp] add sparse checkout

README.rst

@@ -206,6 +206,33 @@ A real life example: applying a patch
         shell_command_after:
             - git am "$(git format-patch -1 XXXXXX -o ../patches)"
 
+Sparse Checkout
+---------------
+
+Git provides sparse-checkout to only checkout a set of files/directories which is
+very useful for more granular control over what we should keep. Especially useful
+when repository is getting big, or when you want to automatically install only
+specific modules from the directory.
+
+Looking at the example below, only ``product_brand`` will be checkout from remote.
+
+.. code-block:: yaml
+
+    ./product_attribute:
+        remotes:
+            oca: https://github.com/OCA/product-attribute.git
+        merges:
+            - oca 8.0
+        target: oca 8.0
+        sparse-checkout:
+            - product_brand
+
+You can skip sparse-checkout for all repositories using the ``--no-sparse-checkout`` flag:
+
+.. code-block:: bash
+
+    $ gitaggregate -c repos.yaml --no-sparse-checkout
+
 Command line Usage
 ==================
 

git_aggregator/config.py

@@ -13,10 +13,11 @@
 log = logging.getLogger(__name__)
 
 
-def get_repos(config, force=False):
+def get_repos(config, force=False, no_sparse_checkout=False):
     """Return a :py:obj:`list` list of repos from config file.
     :param config: the repos config in :py:class:`dict` format.
     :param bool force: Force aggregate dirty repos or not.
+    :param bool no_sparse_checkout: Disable sparse checkout regardless of config.
     :type config: dict
     :rtype: list
     """
@@ -28,6 +29,7 @@ def get_repos(config, force=False):
             'cwd': directory,
             'defaults': repo_data.get('defaults', dict()),
             'force': force,
+            'no_sparse_checkout': no_sparse_checkout,
         }
         remote_names = set()
         if 'remotes' in repo_data:
@@ -126,11 +128,20 @@ def get_repos(config, force=False):
                     cmds = [cmds]
                 commands = cmds
         repo_dict['shell_command_after'] = commands
+        # Handle sparse-checkout configuration
+        sparse_checkout = repo_data.get('sparse-checkout', None)
+        if sparse_checkout:
+            if isinstance(sparse_checkout, string_types):
+                sparse_checkout = [sparse_checkout]
+            elif not isinstance(sparse_checkout, list):
+                raise ConfigException(
+                    '%s: sparse-checkout must be a string or list of strings.' % directory)
+        repo_dict['sparse_checkout'] = sparse_checkout
         repo_list.append(repo_dict)
     return repo_list
 
 
-def load_config(config, expand_env=False, env_file=None, force=False):
+def load_config(config, expand_env=False, env_file=None, force=False, no_sparse_checkout=False):
     """Return repos from a directory and fnmatch. Not recursive.
 
     :param config: paths to config file
@@ -140,6 +151,7 @@ def load_config(config, expand_env=False, env_file=None, force=False):
     :param env_file: path to file with variables to add to the environment.
     :type env_file: str or None
     :param bool force: True to aggregate even if repo is dirty.
+    :param bool no_sparse_checkout: True to disable sparse checkout regardless of config.
     :returns: expanded config dict item
     :rtype: iter(dict)
     """
@@ -173,4 +185,4 @@ def load_config(config, expand_env=False, env_file=None, force=False):
 
     conf = yaml.load(config, Loader=yaml.SafeLoader)
 
-    return get_repos(conf or {}, force)
+    return get_repos(conf or {}, force, no_sparse_checkout)

git_aggregator/main.py

@@ -122,6 +122,14 @@ def get_parser():
         help='Force cleanup and aggregation on dirty repositories.',
     )
 
+    main_parser.add_argument(
+        '--no-sparse-checkout',
+        dest='no_sparse_checkout',
+        default=False,
+        action='store_true',
+        help='Skip sparse-checkout for all repositories.',
+    )
+
     main_parser.add_argument(
         '-j', '--jobs',
         dest='jobs',
@@ -245,7 +253,7 @@ def run(args):
     in args.command"""
 
     repos = load_config(
-        args.config, args.expand_env, args.env_file, args.force)
+        args.config, args.expand_env, args.env_file, args.force, args.no_sparse_checkout)
 
     jobs = max(args.jobs, 1)
     threads = []

git_aggregator/repo.py

@@ -36,7 +36,7 @@ class Repo:
 
     def __init__(self, cwd, remotes, merges, target,
                  shell_command_after=None, fetch_all=False, defaults=None,
-                 force=False):
+                 force=False, sparse_checkout=None, no_sparse_checkout=False):
         """Initialize a git repository aggregator
 
         :param cwd: path to the directory where to initialize the repository
@@ -55,6 +55,9 @@ def __init__(self, cwd, remotes, merges, target,
             Collection of default parameters to be passed to git.
         :param bool force:
             When ``False``, it will stop if repo is dirty.
+        :param sparse_checkout: list of paths to include in sparse-checkout
+        :param bool no_sparse_checkout:
+            When ``True``, disable sparse checkout regardless of configuration.
         """
         self.cwd = cwd
         self.remotes = remotes
@@ -67,6 +70,8 @@ def __init__(self, cwd, remotes, merges, target,
         self.shell_command_after = shell_command_after or []
         self.defaults = defaults or dict()
         self.force = force
+        self.no_sparse_checkout = no_sparse_checkout
+        self.sparse_checkout = sparse_checkout
 
     @property
     def git_version(self):
@@ -219,13 +224,19 @@ def init_repository(self, target_dir):
             repository,
             target_dir,
         )
+        if self.no_sparse_checkout and self.sparse_checkout:
+            logger.info('Sparse checkout is disabled (ignoring config: %s)',
+                        self.sparse_checkout)
         cmd = ('git', 'clone')
         if self.git_version >= (2, 17):
             # Git added support for partial clone in 2.17
             # https://git-scm.com/docs/partial-clone
             # Speeds up cloning by functioning without a complete copy of
             # repository
             cmd += ('--filter=blob:none',)
+        # Enable sparse-checkout if configured
+        if self.sparse_checkout and not self.no_sparse_checkout:
+            cmd += ('--no-checkout',)
         # Try to clone target branch, if it exists
         rtype, _sha = self.query_remote_ref(repository, branch)
         if rtype in {'branch', 'tag'}:
@@ -234,6 +245,17 @@ def init_repository(self, target_dir):
         cmd += self._fetch_options({})
         cmd += (repository, target_dir)
         self.log_call(cmd)
+
+        # Configure and apply sparse-checkout if specified
+        if self.sparse_checkout and not self.no_sparse_checkout:
+            logger.info('Configuring sparse-checkout for %s', self.sparse_checkout)
+            # Enable sparse-checkout
+            self.log_call(['git', 'sparse-checkout', 'init', '--cone'], cwd=target_dir)
+            # Set the paths to include
+            self.log_call(['git', 'sparse-checkout', 'set'] + self.sparse_checkout, cwd=target_dir)
+            # Checkout the files
+            self.log_call(['git', 'checkout'], cwd=target_dir)
+
         return True
 
     def fetch(self):

tests/test_config.py

@@ -435,3 +435,91 @@ def test_fetch_all_true(self):
         config_yaml = dedent(config_yaml)
         repos = config.get_repos(self._parse_config(config_yaml))
         self.assertIs(repos[0]["fetch_all"], True)
+
+    def test_sparse_checkout_string(self):
+        """Test sparse-checkout with a single string path."""
+        config_yaml = """
+            ./test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                sparse-checkout: src/module1
+            """
+        config_yaml = dedent(config_yaml)
+        repos = config.get_repos(self._parse_config(config_yaml))
+        self.assertEqual(repos[0]["sparse_checkout"], ["src/module1"])
+
+    def test_sparse_checkout_list(self):
+        """Test sparse-checkout with a list of paths."""
+        config_yaml = """
+            ./test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                sparse-checkout:
+                    - src/module1
+                    - src/module2
+                    - docs
+            """
+        config_yaml = dedent(config_yaml)
+        repos = config.get_repos(self._parse_config(config_yaml))
+        self.assertEqual(repos[0]["sparse_checkout"], ["src/module1", "src/module2", "docs"])
+
+    def test_sparse_checkout_invalid_type(self):
+        """Test sparse-checkout with invalid type raises ConfigException."""
+        config_yaml = """
+            /test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                sparse-checkout: 123
+            """
+        config_yaml = dedent(config_yaml)
+        with self.assertRaises(ConfigException) as ex:
+            config.get_repos(self._parse_config(config_yaml))
+        self.assertEqual(
+            ex.exception.args[0],
+            '/test: sparse-checkout must be a string or list of strings.'
+        )
+
+    def test_sparse_checkout_none(self):
+        """Test sparse-checkout with None value (not configured)."""
+        config_yaml = """
+            ./test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+            """
+        config_yaml = dedent(config_yaml)
+        repos = config.get_repos(self._parse_config(config_yaml))
+        self.assertIsNone(repos[0]["sparse_checkout"])
+
+    def test_no_sparse_checkout_parameter(self):
+        """Test no_sparse_checkout parameter is passed through get_repos."""
+        config_yaml = """
+            ./test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                sparse-checkout: src/module1
+            """
+        config_yaml = dedent(config_yaml)
+        repos = config.get_repos(
+            self._parse_config(config_yaml),
+            force=False,
+            no_sparse_checkout=True
+        )
+        # Verify no_sparse_checkout is passed to the repo configuration
+        self.assertEqual(repos[0]["no_sparse_checkout"], True)
+        # Verify sparse_checkout is still in config (not removed)
+        self.assertEqual(repos[0]["sparse_checkout"], ["src/module1"])