[bug] fixed #14, UnboundLocalError: cannot access local variable 'ldap_session' where it is not associated with a value

.github/workflows/auto_prefix_issues.yml

@@ -0,0 +1,86 @@
+name: Auto-prefix & Label Issues
+
+on:
+  issues:
+    types: [opened, edited]
+
+jobs:
+  prefix_and_label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Ensure labels exist, then prefix titles & add labels
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const owner = context.repo.owner;
+            const repo  = context.repo.repo;
+
+            // 1. Ensure required labels exist
+            const required = [
+              { name: 'bug',         color: 'd73a4a', description: 'Something isn\'t working' },
+              { name: 'enhancement', color: 'a2eeef', description: 'New feature or request' }
+            ];
+
+            // Fetch current labels in the repo
+            const { data: existingLabels } = await github.rest.issues.listLabelsForRepo({
+              owner, repo, per_page: 100
+            });
+            const existingNames = new Set(existingLabels.map(l => l.name));
+
+            // Create any missing labels
+            for (const lbl of required) {
+              if (!existingNames.has(lbl.name)) {
+                await github.rest.issues.createLabel({
+                  owner,
+                  repo,
+                  name: lbl.name,
+                  color: lbl.color,
+                  description: lbl.description
+                });
+                console.log(`Created label "${lbl.name}"`);
+              }
+            }
+
+            // 2. Fetch all open issues
+            const issues = await github.paginate(
+              github.rest.issues.listForRepo,
+              { owner, repo, state: 'open', per_page: 100 }
+            );
+
+            // 3. Keyword sets
+            const enhancementWords = ["add", "added", "improve", "improved"];
+            const bugWords         = ["bug", "error", "problem", "crash", "failed", "fix", "fixed"];
+
+            // 4. Process each issue
+            for (const issue of issues) {
+              const origTitle = issue.title;
+              const lower     = origTitle.toLowerCase();
+
+              // skip if already prefixed
+              if (/^\[(bug|enhancement)\]/i.test(origTitle)) continue;
+
+              let prefix, labelToAdd;
+              if (enhancementWords.some(w => lower.includes(w))) {
+                prefix     = "[enhancement]";
+                labelToAdd = "enhancement";
+              } else if (bugWords.some(w => lower.includes(w))) {
+                prefix     = "[bug]";
+                labelToAdd = "bug";
+              }
+
+              if (prefix) {
+                // update title
+                await github.rest.issues.update({
+                  owner, repo, issue_number: issue.number,
+                  title: `${prefix} ${origTitle}`
+                });
+                console.log(`Prefixed title of #${issue.number}`);
+
+                // add label
+                await github.rest.issues.addLabels({
+                  owner, repo, issue_number: issue.number,
+                  labels: [labelToAdd]
+                });
+                console.log(`Added label "${labelToAdd}" to #${issue.number}`);
+              }
+            }

.github/workflows/python-pip-build.yml

@@ -18,9 +18,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python 3.11
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: "3.11"
     - name: Install dependencies

.github/workflows/release-tests.yml

@@ -0,0 +1,70 @@
+name: Release Tests
+
+on:
+  release:
+    types: [ published, created ]
+
+jobs:
+  comprehensive-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+
+    - name: Run all tests
+      run: |
+        cd sectools/tests
+        python run_tests.py
+
+    - name: Run tests with verbose output
+      run: |
+        cd sectools/tests
+        python -m unittest discover -v
+
+    - name: Run tests with coverage
+      run: |
+        pip install coverage
+        coverage run --source=sectools sectools/tests/run_tests.py
+        coverage report
+
+  build-package:
+    runs-on: ubuntu-latest
+    needs: comprehensive-test
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.12"
+
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build wheel
+
+    - name: Build package
+      run: |
+        python -m build
+
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: sectools-package
+        path: dist/

Makefile

@@ -1,16 +1,71 @@
-.PHONY : all clean build upload
+.PHONY : all clean build upload test test-verbose test-coverage lint lint-fix fix
+
+PROJECTNAME := "sectools"
 
 all: install clean
 
 clean:
 	@rm -rf `find ./ -type d -name "*__pycache__"`
-	@rm -rf ./build/ ./dist/ ./sectools.egg-info/
+	@rm -rf ./build/ ./dist/ ./$(PROJECTNAME).egg-info/
+	@rm -rf ./$(PROJECTNAME)/tests/datasets/dataset_n*.json
+
+generate-docs:
+	@python3 -m pip install pdoc --break-system-packages
+	@echo "[$(shell date)] Generating docs ..."
+	@PDOC_ALLOW_EXEC=1 python3 -m pdoc -d markdown -o ./documentation/ ./$(PROJECTNAME)/
+	@echo "[$(shell date)] Done!"
+
+uninstall:
+	pip uninstall $(PROJECTNAME) --yes --break-system-packages
 
 install: build
-	python3 setup.py install
+	python3 -m pip install . --break-system-packages
 
 build:
-	python3 setup.py sdist bdist_wheel
+	python3 -m pip uninstall $(PROJECTNAME) --yes --break-system-packages
+	python3 -m pip install build --break-system-packages
+	python3 -m build --wheel
 
 upload: build
-	twine upload dist/*
+	python3 -m pip install twine --break-system-packages
+	python3 -m twine upload dist/*
+
+test:
+	@echo "[$(shell date)] Running tests ..."
+	@cd $(PROJECTNAME)/tests && python3 run_tests.py
+	@echo "[$(shell date)] Tests completed!"
+
+test-verbose:
+	@echo "[$(shell date)] Running tests with verbose output ..."
+	@cd $(PROJECTNAME)/tests && python3 -m unittest discover -v
+	@echo "[$(shell date)] Tests completed!"
+
+test-coverage:
+	@echo "[$(shell date)] Installing coverage and running tests with coverage ..."
+	@python3 -m pip install coverage --break-system-packages
+	@coverage run --source=$(PROJECTNAME) $(PROJECTNAME)/tests/run_tests.py
+	@coverage report
+	@coverage html
+	@echo "[$(shell date)] Coverage report generated in htmlcov/"
+
+lint:
+	@echo "[$(shell date)] Installing linting tools ..."
+	@python3 -m pip install flake8 black isort --break-system-packages
+	@echo "[$(shell date)] Running flake8 linting ..."
+	@python3 -m flake8 $(PROJECTNAME)/ --max-line-length=88 --extend-ignore=E501
+	@echo "[$(shell date)] Running black code formatting check ..."
+	@python3 -m black --check --diff $(PROJECTNAME)/
+	@echo "[$(shell date)] Running isort import sorting check ..."
+	@python3 -m isort --check-only --diff $(PROJECTNAME)/
+	@echo "[$(shell date)] Linting completed!"
+
+lint-fix:
+	@echo "[$(shell date)] Installing linting tools ..."
+	@python3 -m pip install flake8 black isort --break-system-packages
+	@echo "[$(shell date)] Running black to fix formatting issues ..."
+	@python3 -m black $(PROJECTNAME)/
+	@echo "[$(shell date)] Running isort to fix import sorting ..."
+	@python3 -m isort $(PROJECTNAME)/
+	@echo "[$(shell date)] Running flake8 to check remaining issues ..."
+	@python3 -m flake8 $(PROJECTNAME)/ --max-line-length=88 --extend-ignore=E501
+	@echo "[$(shell date)] Code formatting fixes completed!"

sectools/__init__.py

@@ -3,4 +3,3 @@
 # File name          : __init__.py
 # Author             : Podalirius (@podalirius_)
 # Date created       : 28 Jul 2022
-

sectools/data/regex.py

@@ -5,26 +5,28 @@
 # Date created       : 25 Sep 2021
 
 # source: https://stackoverflow.com/questions/201323/how-can-i-validate-an-email-address-using-a-regular-expression
-regex_email = r'''(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])'''
+regex_email = r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])"""
 regex_email_b = bytes(regex_email, "UTF-8")
 
-regex_domain = r'(([a-zA-Z0-9\-_]+\.)+([a-zA-Z0-9\-_]+\.[a-zA-Z0-9\-_]+))'
+regex_domain = r"(([a-zA-Z0-9\-_]+\.)+([a-zA-Z0-9\-_]+\.[a-zA-Z0-9\-_]+))"
 regex_domain_b = bytes(regex_domain, "UTF-8")
 
-regex_url = r'((http|https|ftp)://[a-zA-Z0-9\-]+\.[a-zA-Z0-9\-]+([:][0-9]+)?(/[a-zA-Z0-9_\-\.]+)+([/])?)'
+regex_url = r"((http|https|ftp)://[a-zA-Z0-9\-]+\.[a-zA-Z0-9\-]+([:][0-9]+)?(/[a-zA-Z0-9_\-\.]+)+([/])?)"
 regex_url_b = bytes(regex_url, "UTF-8")
 
-regex_ipv4 = r'(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])'
+regex_ipv4 = r"(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])"
 regex_ipv4_b = bytes(regex_ipv4, "UTF-8")
 
-regex_ipv4_cidr = r'((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9]))/([123][0-9]|[1-9])'
+regex_ipv4_cidr = r"((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9]))/([123][0-9]|[1-9])"
 regex_ipv4_cidr_b = bytes(regex_ipv4_cidr, "UTF-8")
 
-regex_ipv6 = r'((([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}:[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){5}:([0-9A-Fa-f]{1,4}:)?[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){4}:([0-9A-Fa-f]{1,4}:){0,2}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){3}:([0-9A-Fa-f]{1,4}:){0,3}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){2}:([0-9A-Fa-f]{1,4}:){0,4}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(([0-9A-Fa-f]{1,4}:){0,5}:((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(::([0-9A-Fa-f]{1,4}:){0,5}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|([0-9A-Fa-f]{1,4}::([0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4})|(::([0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){1,7}:))'
+regex_ipv6 = r"((([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}:[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){5}:([0-9A-Fa-f]{1,4}:)?[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){4}:([0-9A-Fa-f]{1,4}:){0,2}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){3}:([0-9A-Fa-f]{1,4}:){0,3}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){2}:([0-9A-Fa-f]{1,4}:){0,4}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(([0-9A-Fa-f]{1,4}:){0,5}:((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(::([0-9A-Fa-f]{1,4}:){0,5}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|([0-9A-Fa-f]{1,4}::([0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4})|(::([0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){1,7}:))"
 regex_ipv6_b = bytes(regex_ipv6, "UTF-8")
 
-regex_mac = r'([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})'
+regex_mac = r"([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})"
 regex_mac_b = bytes(regex_mac, "UTF-8")
 
-regex_phone_france = r'(0[1-9]([0-9]{8}))|(0[1-9][ .\-]([0-9][0-9][ .\-]){3}([0-9][0-9]))'
+regex_phone_france = (
+    r"(0[1-9]([0-9]{8}))|(0[1-9][ .\-]([0-9][0-9][ .\-]){3}([0-9][0-9]))"
+)
 regex_phone_france_b = bytes(regex_phone_france, "UTF-8")

sectools/fileformats/Markdown.py

@@ -22,7 +22,7 @@ def __init__(self, data):
     @classmethod
     def fromFile(cls, path_to_file):
         if os.path.exists(path_to_file):
-            f = open(path_to_file, 'r')
+            f = open(path_to_file, "r")
             self = cls(data=f.read())
             f.close()
             return self
@@ -37,7 +37,9 @@ def fromData(cls, data):
     def extract_links(self):
         found = []
         if self.data is not None:
-            for match in re.findall(r'[^!](\[([^\]]*)\]\(([^)]*)\))|(^\[([^\]]*)\]\(([^)]*)\))', self.data):
+            for match in re.findall(
+                r"[^!](\[([^\]]*)\]\(([^)]*)\))|(^\[([^\]]*)\]\(([^)]*)\))", self.data
+            ):
                 if len(match[0]) != 0:
                     md_format, text, link = match[:3]
                     found.append({"markdown": md_format, "text": text, "link": link})
@@ -49,7 +51,7 @@ def extract_links(self):
     def extract_images(self):
         found = []
         if self.data is not None:
-            for match in re.findall(r'(!\[([^\]]*)\]\(([^)]*)\))', self.data):
+            for match in re.findall(r"(!\[([^\]]*)\]\(([^)]*)\))", self.data):
                 md_format, text, link = match
                 found.append({"markdown": md_format, "text": text, "link": link})
         return found

sectools/fileformats/__init__.py

@@ -4,4 +4,6 @@
 # Author             : Podalirius (@podalirius_)
 # Date created       : 6 Nov 2022
 
-from .Markdown import Markdown
+from .Markdown import Markdown
+
+__all__ = ["Markdown"]

sectools/network/domains.py

@@ -5,10 +5,20 @@
 # Date created       : 2 Aug 2022
 
 import re
+
 from sectools.data.regex import regex_domain
 
 
 def is_fqdn(target):
+    """
+    Check if the target is a valid FQDN (Fully Qualified Domain Name).
+
+    Args:
+        target (str): The string to check
+
+    Returns:
+        bool: True if target is valid FQDN, False otherwise
+    """
     outcome = False
     matched = re.match("^" + regex_domain + "$", target.strip())
     if matched is not None: