chore: reduce FPs in whitespace PR by considering ; statement (#1186)

Reduce false positives in the whitespace semgrep rule by considering the ; statement.

Signed-off-by: Carl Flottmann <carl.flottmann@oracle.com>

Author: art1f1c3R

src/macaron/config/defaults.ini

@@ -629,12 +629,13 @@ check_deliverability = True
 # custom rulesets: this is a collection of user-provided rulesets, living inside the path provided to 'custom_semgrep_rules_path'.
 
 # disable default semgrep rulesets here (i.e. all rule IDs in a Semgrep .yaml file) using ruleset names, the name
-# without the .yaml prefix. Currently, we disable the exfiltration rulesets by default due to a high false positive rate.
-# This list may not contain duplicated elements. Macaron's default ruleset names are all unique.
+# without the .yaml prefix (e.g. "obfuscation" for "obfuscation.yaml"). Currently, we disable the exfiltration rulesets
+# by default due to a high false positive rate. This list may not contain duplicated elements. Macaron's default ruleset
+# names are all unique.
 disabled_default_rulesets = exfiltration
-# disable individual rules here (i.e. individual rule IDs inside a Semgrep .yaml file) using rule IDs. You may also
-# provide the IDs of your custom semgrep rules here too, as all Semgrep rule IDs must be unique. This list may not contain
-# duplicated elements.
+# disable individual rules here (i.e. individual rule IDs inside a Semgrep .yaml file, specified under the "rules" header in the
+# .yaml file, with each rule ID under "- id") using rule IDs. You may also provide the IDs of your custom semgrep rules here too,
+# as all Semgrep rule IDs must be unique. This list may not contain duplicated elements.
 disabled_rules =
 # absolute path to a directory where a custom set of semgrep rules for source code analysis are stored. These will be included
 # with Macaron's default rules. The path will be normalised to the OS path type.

src/macaron/malware_analyzer/README.md

@@ -101,6 +101,15 @@ This feature is currently a work in progress, and supports detection of code obf
 - `custom_semgrep_rules`: supply to this an absolute path to a directory containing custom Semgrep `.yaml` files to be run alongside the default ones.
 - `disabled_custom_rulesets`: supply to this a comma separated list of the names of custom Semgrep rule files (excluding the `.yaml` extension) to disable all rule IDs in that file.
 
+Here, a "semgrep ruleset" refers to the name of a Semgrep `.yaml` file without the extension. For example, the name of one of the default rulesets is `obfuscation`, as the file name is `obfuscation.yaml`. To disable all rules in that `.yaml` file would look like this:
+```
+disabled_default_rulesets = obfuscation
+```
+A "semgrep rule", or "rule ID", refers to an `- id` entry under the `rules:` heading in a Semgrep `.yaml` file. For example, the name of a rule in `obfuscation.yaml` would be `obfuscation_excessive-spacing`, which is the name specified under the `- id` entry for that rule. Disabling it would look like this:
+```
+disabled_rules = obfuscation_excessive-spacing
+```
+
 ### Contributing
 
 When contributing an analyzer, it must meet the following requirements:

src/macaron/resources/pypi_malware_rules/obfuscation.yaml

@@ -319,6 +319,8 @@ rules:
   languages:
   - python
   severity: ERROR
-  patterns:
-  - pattern-regex: '[\s]{50,}(\S)+' # The 50 here is the threshold for excessive spacing , more than that is considered obfuscation
-  - pattern-not-regex: '"""[\s\S]*"""'
+  pattern-either: # The 50 here is the threshold for excessive spacing , more than that is considered obfuscation
+    # there is excessive spacing after a ";", marking the end of a statement, then additional code.
+  - pattern-regex: ;[\s]{50,}(\S)+
+    # there is excessive spacing before a ";", and any amount of whitespace before additional code.
+  - pattern-regex: '[\s]{50,};[\s]*(\S)+'

tests/malware_analyzer/pypi/resources/sourcecode_samples/obfuscation/excessive_spacing.py

@@ -20,6 +20,7 @@ def test_function():
     """
     sys.exit()
 
-    # excessive spacing obfuscation
-    def excessive_spacing_flow():
-                                                                                                                                                                                                                                                       print("Hello world!")
+    # excessive spacing obfuscation. The second line here will trigger two detections, which is expected since it matches both patterns.
+    print("hello");                                                                                                                                                         __import__('os')
+    print("hi")                                                                                         ;                                                                   __import__('base64')
+    print("things")                                                                                                                                                         ;__import__('zlib')

tests/malware_analyzer/pypi/resources/sourcecode_samples/obfuscation/expected_results.json

@@ -53,6 +53,21 @@
                     "start": 44,
                     "end": 44
                 },
+                {
+                    "file": "obfuscation/excessive_spacing.py",
+                    "start": 24,
+                    "end": 24
+                },
+                {
+                    "file": "obfuscation/excessive_spacing.py",
+                    "start": 25,
+                    "end": 25
+                },
+                {
+                    "file": "obfuscation/excessive_spacing.py",
+                    "start": 26,
+                    "end": 26
+                },
                 {
                     "file": "obfuscation/inline_imports.py",
                     "start": 23,
@@ -105,6 +120,36 @@
                 }
             ]
         },
+        "src.macaron.resources.pypi_malware_rules.obfuscation_excessive-spacing": {
+            "message": "Hidden code after excessive spacing",
+            "detections": [
+                {
+                    "file": "obfuscation/excessive_spacing.py",
+                    "start": 24,
+                    "end": 24
+                },
+                {
+                    "file": "obfuscation/excessive_spacing.py",
+                    "start": 25,
+                    "end": 25
+                },
+                {
+                    "file": "obfuscation/excessive_spacing.py",
+                    "start": 25,
+                    "end": 25
+                },
+                {
+                    "file": "obfuscation/excessive_spacing.py",
+                    "start": 26,
+                    "end": 26
+                },
+                {
+                    "file": "obfuscation/inline_imports.py",
+                    "start": 27,
+                    "end": 27
+                }
+            ]
+        },
         "src.macaron.resources.pypi_malware_rules.obfuscation_obfuscation-tools": {
             "message": "Found an indicator of the use of a python code obfuscation tool",
             "detections": [
@@ -229,21 +274,6 @@
                     "end": 68
                 }
             ]
-        },
-        "src.macaron.resources.pypi_malware_rules.obfuscation_excessive-spacing": {
-            "message": "Hidden code after excessive spacing",
-            "detections": [
-                {
-                    "file": "obfuscation/excessive_spacing.py",
-                    "start": 24,
-                    "end": 25
-                },
-                {
-                   "file": "obfuscation/inline_imports.py",
-                    "start": 27,
-                    "end": 27
-                }
-            ]
         }
     },
     "disabled_sourcecode_rule_findings": {}