Makefiles testing - CMake Source of Truth

.github/build_tools/find_cmake_built_dirs.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Find source directories whose Makefile examples were also built by CMake.
+
+Scans all Makefiles for 'EXAMPLE := <name>', then checks if that executable
+exists anywhere in the CMake build tree. Outputs the list of source directories
+whose Makefile targets have a matching CMake-built executable.
+
+This lets us selectively build only Makefiles for examples whose dependencies
+CMake confirmed are available, without requiring the directory structures to
+match between source and build trees.
+"""
+
+import argparse
+import os
+import re
+import stat
+
+
+def find_makefile_examples(source_dir):
+    """Walk source tree, find Makefiles with 'EXAMPLE := <name>'.
+
+    Returns dict mapping example_name -> relative source directory.
+    """
+    examples = {}
+    for root, _dirs, files in os.walk(source_dir):
+        if "Makefile" not in files:
+            continue
+        makefile_path = os.path.join(root, "Makefile")
+        try:
+            with open(makefile_path) as f:
+                for line in f:
+                    m = re.match(r"^EXAMPLE\s*:=\s*(\S+)", line)
+                    if m:
+                        name = m.group(1)
+                        rel = os.path.relpath(root, source_dir)
+                        examples[name] = rel
+                        break
+        except OSError:
+            continue
+    return examples
+
+
+def find_cmake_executables(build_dir):
+    """Walk CMake build tree and collect names of executable files."""
+    executables = set()
+    for root, _dirs, files in os.walk(build_dir):
+        for fname in files:
+            fpath = os.path.join(root, fname)
+            try:
+                st = os.stat(fpath)
+            except OSError:
+                continue
+            if not (st.st_mode & stat.S_IXUSR):
+                continue
+            # Skip non-executable artifacts
+            if fname.endswith((".so", ".a", ".cmake", ".sh", ".py", ".txt")):
+                continue
+            if fname.startswith("lib"):
+                continue
+            executables.add(fname)
+    return executables
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Find source dirs whose Makefile examples were built by CMake."
+    )
+    parser.add_argument("--build-dir", required=True, help="CMake build directory")
+    parser.add_argument("--source-dir", required=True, help="Source root directory")
+    parser.add_argument(
+        "--output", required=True, help="Output file (one directory per line)"
+    )
+    args = parser.parse_args()
+
+    build_dir = os.path.abspath(args.build_dir)
+    source_dir = os.path.abspath(args.source_dir)
+
+    makefile_examples = find_makefile_examples(source_dir)
+    cmake_executables = find_cmake_executables(build_dir)
+
+    matched_dirs = []
+    for name, rel_dir in sorted(makefile_examples.items()):
+        if name in cmake_executables:
+            matched_dirs.append(rel_dir)
+
+    with open(args.output, "w") as f:
+        for d in matched_dirs:
+            f.write(d + "\n")
+
+    print(
+        f"Found {len(matched_dirs)} Makefile directories matching CMake-built "
+        f"executables (out of {len(makefile_examples)} Makefiles total)"
+    )
+
+
+if __name__ == "__main__":
+    main()

.github/build_tools/generate_skip_tests.py

@@ -8,6 +8,17 @@
 import argparse
 import os
 
+# Tests to always skip in Makefile runs. These require test data or arguments
+# that are only set up by the CMake build system (e.g. ROCDECODE_TEST_FRAMES_DIR).
+# ctest handles them via add_test() COMMAND args; the Makefile runner cannot.
+MAKEFILE_SKIP_TESTS = [
+    "rocdecode_rocdec_decode",
+]
+
+# Tests to always skip (all targets, all distros).
+ALWAYS_SKIP_TESTS = [
+]
+
 # Tests to skip per GPU target (one list per target that has skips)
 SKIP_TESTS = {
     "gfx1151": [
@@ -29,8 +40,6 @@
 # Tests to skip for a specific GPU target + distro combination.
 # Keys are "<gpu_target>:<distro_key>", e.g. "gfx1151:sles-15.7".
 DISTRO_SKIP_TESTS = {
-    # Example:
-    # "gfx1151:sles-15.7": ["some_test"],
 }
 
 
@@ -53,9 +62,22 @@ def main():
         default="",
         help="Distro key for distro-specific skips (e.g. sles-15.7)",
     )
+    parser.add_argument(
+        "--makefile",
+        action="store_true",
+        help="Include Makefile-only skips (tests needing CMake test data)",
+    )
     args = parser.parse_args()
 
-    lines = list(SKIP_TESTS.get(args.target, []))
+    lines = list(ALWAYS_SKIP_TESTS)
+    for test in SKIP_TESTS.get(args.target, []):
+        if test not in lines:
+            lines.append(test)
+
+    if args.makefile:
+        for test in MAKEFILE_SKIP_TESTS:
+            if test not in lines:
+                lines.append(test)
 
     if args.distro:
         combo_key = f"{args.target}:{args.distro}"

.github/build_tools/parse_ctest_results.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Parse ctest JUnit XML results and extract test names that actually ran.
+
+Used by the Makefile test step to only run tests that ctest executed,
+ensuring parity between CMake and Makefile test sets.
+"""
+
+import argparse
+import sys
+import xml.etree.ElementTree as ET
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Extract test names from ctest JUnit XML results."
+    )
+    parser.add_argument(
+        "--junit", required=True, help="Path to ctest JUnit XML file"
+    )
+    parser.add_argument(
+        "--output", required=True, help="Output file (one test name per line)"
+    )
+    args = parser.parse_args()
+
+    try:
+        tree = ET.parse(args.junit)
+    except FileNotFoundError:
+        print(f"Warning: {args.junit} not found (ctest may not have run)")
+        print("Creating empty allow list — no Makefile tests will run")
+        with open(args.output, "w") as f:
+            pass
+        return
+    except ET.ParseError as e:
+        print(f"Error parsing {args.junit}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    root = tree.getroot()
+    passed = []
+    failed = []
+    for testcase in root.iter("testcase"):
+        name = testcase.get("name")
+        if not name:
+            continue
+        # In JUnit XML, failed tests have a <failure> child element
+        if testcase.find("failure") is not None:
+            failed.append(name)
+        else:
+            passed.append(name)
+
+    with open(args.output, "w") as f:
+        for name in passed:
+            f.write(name + "\n")
+
+    total = len(passed) + len(failed)
+    print(f"Extracted {len(passed)} passed tests from {total} ctest results ({len(failed)} failed, excluded)")
+
+
+if __name__ == "__main__":
+    main()