From e95f7b80cef89b8120b0bed67f9f63cd59d1c0ee Mon Sep 17 00:00:00 2001 From: George Pickering <9803299+bigpick@users.noreply.github.com> Date: Tue, 30 Jan 2024 09:22:00 -0500 Subject: [PATCH 1/4] feat: plugins reuse excludes CLI arg Signed-off-by: George Pickering <9803299+bigpick@users.noreply.github.com> --- detect_secrets/core/usage.py | 17 +++++++++++++++++ detect_secrets/main.py | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/detect_secrets/core/usage.py b/detect_secrets/core/usage.py index 5d532f3d0..c174a6d81 100644 --- a/detect_secrets/core/usage.py +++ b/detect_secrets/core/usage.py @@ -5,6 +5,14 @@ from detect_secrets.constants import DEFAULT_GHE_INSTANCE +def add_plugins_reuse_exclude_flag(parser): + parser.add_argument( + '--plugins-reuse-excludes', + action='store_true', + help='Force plugins to try re-using existing exclude contents.' + ) + + def add_exclude_lines_argument(parser): parser.add_argument( '--exclude-lines', @@ -89,6 +97,7 @@ def add_default_arguments(self): def add_pre_commit_arguments(self): self._add_filenames_argument()\ + ._add_plugins_reuse_exclude_flag()\ ._add_set_baseline_argument()\ ._add_exclude_lines_argument()\ ._add_word_list_argument()\ @@ -154,6 +163,10 @@ def _add_set_baseline_argument(self): ) return self + def _add_plugins_reuse_exclude_flag(self): + add_plugins_reuse_exclude_flag(self.parser) + return self + def _add_exclude_lines_argument(self): add_exclude_lines_argument(self.parser) return self @@ -222,6 +235,10 @@ def _add_initialize_baseline_argument(self): ), ) + # Pairing `--plugins-reuse-excludes` to + # both pre-commit and `--scan` because it can be used for both. + add_plugins_reuse_exclude_flag(self.parser) + # Pairing `--exclude-lines` and `--word-list` to # both pre-commit and `--scan` because it can be used for both. add_exclude_lines_argument(self.parser) diff --git a/detect_secrets/main.py b/detect_secrets/main.py index 7bbc8f8f6..477d33514 100644 --- a/detect_secrets/main.py +++ b/detect_secrets/main.py @@ -17,6 +17,19 @@ def parse_args(argv, parserBuilder): return parserBuilder.add_console_use_arguments().parse_args(argv) +def maybe_get_existing_exclude(exclude_files, exclude_lines, old_baseline): + if not old_baseline: + return exclude_lines + + previously_included = old_baseline.get("exclude", None) + if not previously_included: + return exclude_files, exclude_lines + + files = "|".join(filter(None, (exclude_files, previously_included.get("files",None)))) + lines = "|".join(filter(None, (exclude_lines, previously_included.get("lines",None)))) + + return files, lines + def main(argv=None): if len(sys.argv) == 1: # pragma: no cover sys.argv.append('-h') @@ -36,6 +49,9 @@ def main(argv=None): if args.word_list_file: automaton, word_list_hash = build_automaton(args.word_list_file) + if args.plugins_reuse_excludes: + args.exclude_files, args.exclude_lines = maybe_get_existing_exclude(args.exclude_files, args.exclude_lines, _get_existing_baseline(args.import_filename)) + # Plugins are *always* rescanned with fresh settings, because # we want to get the latest updates. plugins = initialize.from_parser_builder( From d0ee995a5e5cf19b3d134179cfd08d0470e44748 Mon Sep 17 00:00:00 2001 From: George Pickering <9803299+bigpick@users.noreply.github.com> Date: Tue, 30 Jan 2024 09:48:24 -0500 Subject: [PATCH 2/4] feat: always persist plugins_reuse_excludes state from .secrets.baseline Signed-off-by: George Pickering <9803299+bigpick@users.noreply.github.com> --- detect_secrets/core/baseline.py | 5 +++++ detect_secrets/core/secrets_collection.py | 3 +++ detect_secrets/main.py | 9 +++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/detect_secrets/core/baseline.py b/detect_secrets/core/baseline.py index 41692f266..7afb3c8a6 100644 --- a/detect_secrets/core/baseline.py +++ b/detect_secrets/core/baseline.py @@ -16,6 +16,7 @@ def initialize( path, plugins, + plugins_reuse_exclude=None, exclude_files_regex=None, exclude_lines_regex=None, word_list_file=None, @@ -33,6 +34,9 @@ def initialize( :type plugins: tuple of detect_secrets.plugins.base.BasePlugin :param plugins: rules to initialize the SecretsCollection with. + :type plugins_reuse_exclude: bool|None + :param plugins_reuse_exclude optional bool indicating whether plugins were forced to reuse excludes. + :type exclude_files_regex: str|None :type exclude_lines_regex: str|None @@ -53,6 +57,7 @@ def initialize( """ output = SecretsCollection( plugins, + plugins_reuse_exclude=plugins_reuse_exclude, exclude_files=exclude_files_regex, exclude_lines=exclude_lines_regex, word_list_file=word_list_file, diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index d3948c647..e03ef1726 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -21,6 +21,7 @@ class SecretsCollection: def __init__( self, plugins=(), + plugins_reuse_exclude=None, exclude_files=None, exclude_lines=None, word_list_file=None, @@ -46,6 +47,7 @@ def __init__( """ self.data = {} self.plugins = plugins + self.plugins_reuse_exclude = plugins_reuse_exclude self.exclude_files = exclude_files self.exclude_lines = exclude_lines self.word_list_file = word_list_file @@ -342,6 +344,7 @@ def format_for_baseline_output(self): plugins_used = sorted(plugins_used, key=lambda x: x['name']) return { + **({"plugins_reuse_excludes": True} if self.plugins_reuse_exclude else {}), 'generated_at': strftime('%Y-%m-%dT%H:%M:%SZ', gmtime()), 'exclude': { 'files': self.exclude_files, diff --git a/detect_secrets/main.py b/detect_secrets/main.py index 477d33514..90c9830e2 100644 --- a/detect_secrets/main.py +++ b/detect_secrets/main.py @@ -49,8 +49,9 @@ def main(argv=None): if args.word_list_file: automaton, word_list_hash = build_automaton(args.word_list_file) - if args.plugins_reuse_excludes: - args.exclude_files, args.exclude_lines = maybe_get_existing_exclude(args.exclude_files, args.exclude_lines, _get_existing_baseline(args.import_filename)) + _baseline = _get_existing_baseline(args.import_filename) + if args.plugins_reuse_excludes or (_baseline and _baseline.get("plugins_reuse_excludes", False)): + args.exclude_files, args.exclude_lines = maybe_get_existing_exclude(args.exclude_files, args.exclude_lines, _baseline) # Plugins are *always* rescanned with fresh settings, because # we want to get the latest updates. @@ -189,6 +190,9 @@ def _perform_scan(args, plugins, automaton, word_list_hash): if not args.word_list_file and old_baseline.get('word_list'): args.word_list_file = old_baseline['word_list']['file'] + if not args.plugins_reuse_excludes: + args.plugins_reuse_excludes = old_baseline.get('plugins_reuse_excludes', False) + # If we have knowledge of an existing baseline file, we should use # that knowledge and add it to our exclude_files regex. if args.import_filename: @@ -196,6 +200,7 @@ def _perform_scan(args, plugins, automaton, word_list_hash): new_baseline = baseline.initialize( plugins=plugins, + plugins_reuse_exclude=args.plugins_reuse_excludes, exclude_files_regex=args.exclude_files, exclude_lines_regex=args.exclude_lines, word_list_file=args.word_list_file, From ad381f164b6e45fac9635ad6dbdb0c63402de8b4 Mon Sep 17 00:00:00 2001 From: George Pickering <9803299+bigpick@users.noreply.github.com> Date: Tue, 30 Jan 2024 09:59:36 -0500 Subject: [PATCH 3/4] fix: return both excludes in no baseline case Signed-off-by: George Pickering <9803299+bigpick@users.noreply.github.com> --- detect_secrets/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detect_secrets/main.py b/detect_secrets/main.py index 90c9830e2..67354044f 100644 --- a/detect_secrets/main.py +++ b/detect_secrets/main.py @@ -19,7 +19,7 @@ def parse_args(argv, parserBuilder): def maybe_get_existing_exclude(exclude_files, exclude_lines, old_baseline): if not old_baseline: - return exclude_lines + return exclude_files, exclude_lines previously_included = old_baseline.get("exclude", None) if not previously_included: From db334ff8febd04e5a3962a317cb71305eb58f9d2 Mon Sep 17 00:00:00 2001 From: George Pickering <9803299+bigpick@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:13:45 -0500 Subject: [PATCH 4/4] fix: version and tests Signed-off-by: George Pickering <9803299+bigpick@users.noreply.github.com> --- detect_secrets/__init__.py | 2 +- tests/main_test.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/detect_secrets/__init__.py b/detect_secrets/__init__.py index d654b7290..f5eee76e3 100644 --- a/detect_secrets/__init__.py +++ b/detect_secrets/__init__.py @@ -1 +1 @@ -VERSION = '0.13.1+ibm.62.dss' +VERSION = '0.13.1+ibm.63.dss' diff --git a/tests/main_test.py b/tests/main_test.py index 0b0368b24..f215afe79 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -87,6 +87,7 @@ def test_scan_basic(self, mock_baseline_initialize): mock_baseline_initialize.assert_called_once_with( plugins=Any(tuple), + plugins_reuse_excludes=None, exclude_files_regex=None, exclude_lines_regex=None, path='.', @@ -104,6 +105,7 @@ def test_scan_with_rootdir(self, mock_baseline_initialize): mock_baseline_initialize.assert_called_once_with( plugins=Any(tuple), + plugins_reuse_excludes=None, exclude_files_regex=None, exclude_lines_regex=None, path=['test_data'], @@ -123,6 +125,7 @@ def test_scan_with_exclude_args(self, mock_baseline_initialize): mock_baseline_initialize.assert_called_once_with( plugins=Any(tuple), + plugins_reuse_excludes=None, exclude_files_regex='some_pattern_here', exclude_lines_regex='other_patt', path='.', @@ -208,6 +211,7 @@ def test_scan_with_all_files_flag(self, mock_baseline_initialize): mock_baseline_initialize.assert_called_once_with( plugins=Any(tuple), + plugins_reuse_excludes=None, exclude_files_regex=None, exclude_lines_regex=None, path='.', @@ -265,6 +269,7 @@ def test_reads_non_existed_baseline_from_file( mock_baseline_initialize.assert_called_once_with( plugins=Any(tuple), + plugins_reuse_excludes=None, exclude_files_regex='^non_existed_baseline_file$', exclude_lines_regex=None, path='.',