diff --git a/.github/scripts/progressive_lint.py b/.github/scripts/progressive_lint.py new file mode 100644 index 000000000..c31f061cc --- /dev/null +++ b/.github/scripts/progressive_lint.py @@ -0,0 +1,199 @@ +import subprocess +import sys +import os +import xml.sax +import textwrap + +# --- Configuration --- +CONFIG = { + "indent_char": " ", # Use " " for spaces, "\t" for tabs + "indent_size": 4, # How many spaces per level? + "max_line_length": 120, # Maximum characters per line + "extensions": ('.xml', '.xsd', '.wsdl'), + # Map of { Bad_Byte : Replacement_Bytes } + "replacements": { + b'\x85': b'...', # Horizontal ellipsis + b'\x91': b"'", # Left single quote + b'\x92': b"'", # Right single quote + b'\x93': b'"', # Left double quote + b'\x94': b'"', # Right double quote + b'\x95': b'*', # Bullet + b'\x96': b'-', # En dash + b'\x97': b'-', # Em dash + b'\x84': b'"', # Low double quote + b'\x99': b'\xe2\x84\xa2' # Trademark (TM) + } +} + +class DepthMapHandler(xml.sax.ContentHandler): + def __init__(self): + self.line_depths = {} + self.protected_lines = set() + self.current_depth = 0 + self.locator = None + self.in_programlisting = False + self.programlisting_start = -1 + + def setDocumentLocator(self, locator): + self.locator = locator + + def startElement(self, name, attrs): + line_num = self.locator.getLineNumber() + if line_num not in self.line_depths: + self.line_depths[line_num] = self.current_depth + self.current_depth += 1 + + if name == 'programlisting': + self.in_programlisting = True + self.programlisting_start = line_num + + def endElement(self, name): + self.current_depth -= 1 + line_num = self.locator.getLineNumber() + if line_num not in self.line_depths: + self.line_depths[line_num] = self.current_depth + + if name == 'programlisting' and self.in_programlisting: + self.in_programlisting = False + self.protected_lines.update(range(self.programlisting_start, line_num + 1)) + + def characters(self, content): + if content.strip(): + line_num = self.locator.getLineNumber() + if line_num not in self.line_depths: + self.line_depths[line_num] = self.current_depth + +def get_changed_lines(file_path, base_ref): + """ + Returns a set of 1-based line numbers changed in the file. + Uses the triple-dot syntax to compare against the merge base. + """ + # The '...' tells Git to compare the working branch against the point where it diverged from base_ref + cmd = ["git", "diff", "-U0", f"{base_ref}...", "--", file_path] + try: + output = subprocess.check_output(cmd).decode("utf-8", errors="replace") + except subprocess.CalledProcessError: + return set() + + changed_lines = set() + for line in output.splitlines(): + if line.startswith("@@"): + try: + plus_part = line.split("+")[1].split()[0] + if "," in plus_part: + start, count = map(int, plus_part.split(",")) + else: + start, count = int(plus_part), 1 + + if count > 0: + changed_lines.update(range(start, start + count)) + except (IndexError, ValueError): + continue + return changed_lines + +def smart_wrap(content, indent_str, max_len): + clean_content = ' '.join(content.split()) + + full_line = indent_str + clean_content + if len(full_line) <= max_len: + return full_line + "\n" + + wrapped_lines = textwrap.wrap( + clean_content, + width=max_len, + initial_indent=indent_str, + subsequent_indent=indent_str, + break_long_words=False, + break_on_hyphens=False + ) + return "\n".join(wrapped_lines) + "\n" + +def process_file(file_path, base_ref): + # 1. Get changed lines (from the dirty file on disk, relative to merge base) + changed_lines = get_changed_lines(file_path, base_ref) + if not changed_lines: + return + + # 2. Read file as BINARY + with open(file_path, "rb") as f: + original_lines_bytes = f.readlines() + + # 3. Create a 'Clean-in-Memory' version for parser + cleaned_lines_bytes = list(original_lines_bytes) + + for i in range(len(cleaned_lines_bytes)): + line_num = i + 1 + if line_num in changed_lines: + current_line = cleaned_lines_bytes[i] + for bad_byte, replacement in CONFIG["replacements"].items(): + current_line = current_line.replace(bad_byte, replacement) + cleaned_lines_bytes[i] = current_line + + # 4. Parse the Cleaned Content + full_clean_content = b"".join(cleaned_lines_bytes) + + handler = DepthMapHandler() + parser = xml.sax.make_parser() + parser.setContentHandler(handler) + try: + xml.sax.parseString(full_clean_content, handler) + except xml.sax.SAXException: + print(f"Skipping {file_path}: Malformed XML.") + return + + # 5. Format and Reconstruct + final_output_lines = [] + + for i, line_bytes in enumerate(cleaned_lines_bytes): + line_num = i + 1 + + # Protected Block: Keep original bytes exactly + if line_num in handler.protected_lines: + final_output_lines.append(original_lines_bytes[i]) + continue + + # Changed Block: Format + if line_num in changed_lines and line_num in handler.line_depths: + try: + line_text = line_bytes.decode('utf-8') + except UnicodeDecodeError: + final_output_lines.append(original_lines_bytes[i]) + continue + + depth = handler.line_depths[line_num] + correct_indent = CONFIG["indent_char"] * (CONFIG["indent_size"] * depth) + + formatted_text = smart_wrap(line_text, correct_indent, CONFIG["max_line_length"]) + final_output_lines.append(formatted_text.encode('utf-8')) + + else: + # Unchanged Block: Keep original bytes exactly + final_output_lines.append(original_lines_bytes[i]) + + # 6. Write back as BINARY + with open(file_path, "wb") as f: + f.writelines(final_output_lines) + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python progressive_lint.py ") + sys.exit(1) + + base_ref = sys.argv[1] + + try: + # Use triple dots to compare against the merge base + diff_cmd = ["git", "diff", "--name-only", f"{base_ref}..."] + files = subprocess.check_output(diff_cmd).decode("utf-8").splitlines() + + target_files = [ + f for f in files + if f.lower().endswith(CONFIG["extensions"]) and os.path.exists(f) + ] + + for file_path in target_files: + process_file(file_path, base_ref) + + except Exception as e: + print(f"Error: {e}") + sys.exit(1) diff --git a/.github/workflows/gsoap-wsdl-check.yaml b/.github/workflows/gsoap-wsdl-check.yaml deleted file mode 100644 index 7b6c1d457..000000000 --- a/.github/workflows/gsoap-wsdl-check.yaml +++ /dev/null @@ -1,54 +0,0 @@ -name: gsoap-wsdl-check -run-name: "WSDL files syntax check with gSoap" - -on: - push: - paths: - - '**.wsdl' - - '**.xsd' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - gsoap-wsdl-check: - strategy: - fail-fast: true - runs-on: ubuntu-24.04 - steps: - - name: Check that remote files are reachable - id: vars - run: | - export MISSING_FILES=0 - curl --head http://docs.oasis-open.org/wsn/b-2.xsd || export MISSING_FILES=$((MISSING_FILES+1)) - echo "MISSING_FILES=$MISSING_FILES" >> $GITHUB_OUTPUT - echo "MISSING_FILES=$MISSING_FILES" - - name: Install missing software - if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} - run: | - sudo apt update && sudo apt install gcc g++ curl autoconf automake cmake bison flex libssl-dev zlib1g-dev make - - name: Checkout tools repo - uses: actions/checkout@v6 - if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} - with: - repository: onvif/gsoap-wsdl-checker - path: gsoap-wsdl-checker - ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} - - name: Get branch names. - if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} - id: branch-names - uses: tj-actions/branch-names@v9.0.2 - - name: Install gsoap - if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} - run: | - cd gsoap-wsdl-checker - cmake CMakeLists.txt -DBRANCH=${{ steps.branch-names.outputs.current_branch }} - make gsoap - - name: Compile the WDSL files - if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} - run: | - cd gsoap-wsdl-checker - make - diff --git a/.github/workflows/dotnet-wsdl-check.yaml b/.github/workflows/source-lint-and-check.yaml similarity index 70% rename from .github/workflows/dotnet-wsdl-check.yaml rename to .github/workflows/source-lint-and-check.yaml index 065d07feb..d4e4a3179 100644 --- a/.github/workflows/dotnet-wsdl-check.yaml +++ b/.github/workflows/source-lint-and-check.yaml @@ -1,11 +1,12 @@ -name: dotnet-wsdl-check -run-name: "WSDL files syntax check with .Net 9.0" +name: source-lint-and-check +run-name: "Source lint and check" on: push: paths: - '**.wsdl' - '**.xsd' + - '**.xml' workflow_dispatch: concurrency: @@ -13,7 +14,79 @@ concurrency: cancel-in-progress: true jobs: + source-lint: + strategy: + fail-fast: true + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # This ensures it checks out the branch that triggered the workflow + ref: ${{ github.ref }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Run progressive lint script + # Assuming the script modifies files in place + run: python .github/scripts/progressive_lint.py + + - name: Commit and push changes + uses: EndBug/add-and-commit@v9 + with: + # EndBug automatically checks if files changed. If no files changed, it does nothing. + message: 'chore: apply progressive linting fixes' + committer_name: GitHub Actions + committer_email: actions@github.com + # By default, this action pushes back to the branch checked out by actions/checkout + + gsoap-wsdl-check: + needs: source-lint + strategy: + fail-fast: true + runs-on: ubuntu-24.04 + steps: + - name: Check that remote files are reachable + id: vars + run: | + export MISSING_FILES=0 + curl --head http://docs.oasis-open.org/wsn/b-2.xsd || export MISSING_FILES=$((MISSING_FILES+1)) + echo "MISSING_FILES=$MISSING_FILES" >> $GITHUB_OUTPUT + echo "MISSING_FILES=$MISSING_FILES" + - name: Install missing software + if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} + run: | + sudo apt update && sudo apt install gcc g++ curl autoconf automake cmake bison flex libssl-dev zlib1g-dev make + - name: Checkout tools repo + uses: actions/checkout@v6 + if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} + with: + repository: onvif/gsoap-wsdl-checker + path: gsoap-wsdl-checker + ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} + - name: Get branch names. + if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} + id: branch-names + uses: tj-actions/branch-names@v9.0.2 + - name: Install gsoap + if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} + run: | + cd gsoap-wsdl-checker + cmake CMakeLists.txt -DBRANCH=${{ steps.branch-names.outputs.current_branch }} + make gsoap + - name: Compile the WDSL files + if: ${{ steps.vars.outputs.MISSING_FILES == 0 }} + run: | + cd gsoap-wsdl-checker + make + dotnet-wsdl-check: + needs: source-lint strategy: fail-fast: true runs-on: ubuntu-24.04