From 7495a09f80d967a55c2203116a9f8aac5247d0b4 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 25 Mar 2026 12:25:15 -0400 Subject: [PATCH] feat: add redhat/builder to leaderboard Score: 78.6/100 (Gold) Repository: https://gitlab.com/redhat/rhel-ai/wheels/builder --- .../2026-03-25T12-00-00-assessment.json | 667 ++++++++++++++++++ 1 file changed, 667 insertions(+) create mode 100644 submissions/redhat/builder/2026-03-25T12-00-00-assessment.json diff --git a/submissions/redhat/builder/2026-03-25T12-00-00-assessment.json b/submissions/redhat/builder/2026-03-25T12-00-00-assessment.json new file mode 100644 index 00000000..d7f93605 --- /dev/null +++ b/submissions/redhat/builder/2026-03-25T12-00-00-assessment.json @@ -0,0 +1,667 @@ +{ + "schema_version": "1.0.0", + "metadata": { + "agentready_version": "2.30.1", + "research_version": "1.0.1", + "assessment_timestamp": "2026-03-24T15:28:44.569307", + "assessment_timestamp_human": "March 24, 2026 at 3:28 PM", + "executed_by": "redacted", + "command": "agentready assess ./builder --config agentready-config.yml", + "working_directory": "redacted" + }, + "repository": { + "path": "redacted", + "name": "builder", + "url": "git@gitlab.com:redhat/rhel-ai/wheels/builder.git", + "branch": "main", + "commit_hash": "d7aaaa2c99309458db7abcd781d89fef81755b3d", + "languages": { + "Markdown": 14, + "JSON": 5, + "YAML": 205, + "Shell": 32, + "Python": 117 + }, + "total_files": 620, + "total_lines": 43684 + }, + "timestamp": "2026-03-24T15:28:44.569307", + "overall_score": 78.6, + "certification_level": "Gold", + "attributes_assessed": 16, + "attributes_skipped": 2, + "attributes_total": 18, + "findings": [ + { + "attribute": { + "id": "claude_md_file", + "name": "CLAUDE.md Configuration Files", + "category": "Context Window Optimization", + "tier": 1, + "description": "Project-specific configuration for Claude Code", + "criteria": "CLAUDE.md file exists in repository root", + "default_weight": 0.1 + }, + "status": "pass", + "score": 100.0, + "measured_value": "present", + "threshold": "present", + "evidence": [ + "CLAUDE.md found at builder/CLAUDE.md", + "Symlink to AGENTS.md (38309 bytes)", + "AGENTS.md also present (cross-tool compatibility)" + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "readme_structure", + "name": "README Structure", + "category": "Documentation Standards", + "tier": 1, + "description": "Well-structured README with key sections", + "criteria": "README.md with installation, usage, and development sections", + "default_weight": 0.1 + }, + "status": "pass", + "score": 100.0, + "measured_value": "3/3 sections", + "threshold": "3/3 sections", + "evidence": [ + "Found 3/3 essential sections", + "Installation: \u2713", + "Usage: \u2713", + "Development: \u2713" + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "type_annotations", + "name": "Type Annotations", + "category": "Code Quality", + "tier": 1, + "description": "Type hints in function signatures", + "criteria": ">80% of functions have type annotations", + "default_weight": 0.1 + }, + "status": "pass", + "score": 100.0, + "measured_value": "87.3%", + "threshold": "\u226580%", + "evidence": [ + "Typed functions: 475/544", + "Coverage: 87.3%" + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "standard_layout", + "name": "Standard Project Layouts", + "category": "Repository Structure", + "tier": 1, + "description": "Follows standard project structure for language", + "criteria": "Standard directories (src/ or project-named, tests/) present", + "default_weight": 0.1 + }, + "status": "fail", + "score": 50.0, + "measured_value": "1/2 directories", + "threshold": "2/2 directories", + "evidence": [ + "Found 1/2 standard directories", + "source directory: \u2717 (no src/ or project-named dir)", + "tests/: \u2713" + ], + "remediation": { + "summary": "Organize code into standard directories", + "steps": [ + "Create a source directory for your code", + "Option A: Use src/ layout (recommended for packages)", + "Option B: Use project-named directory (e.g., mypackage/)", + "Ensure your package has __init__.py" + ], + "tools": [], + "commands": [ + "# Option A: src layout", + "mkdir -p src/mypackage", + "touch src/mypackage/__init__.py", + "# ---", + "# Option B: flat layout (project-named)", + "mkdir -p mypackage", + "touch mypackage/__init__.py" + ], + "examples": [ + "# src layout (recommended for distributable packages)\nproject/\n\u251c\u2500\u2500 src/\n\u2502 \u2514\u2500\u2500 mypackage/\n\u2502 \u251c\u2500\u2500 __init__.py\n\u2502 \u2514\u2500\u2500 module.py\n\u251c\u2500\u2500 tests/\n\u2502 \u2514\u2500\u2500 test_module.py\n\u2514\u2500\u2500 pyproject.toml\n\n# flat layout (common in major projects like pandas, numpy)\nproject/\n\u251c\u2500\u2500 mypackage/\n\u2502 \u251c\u2500\u2500 __init__.py\n\u2502 \u2514\u2500\u2500 module.py\n\u251c\u2500\u2500 tests/\n\u2502 \u2514\u2500\u2500 test_module.py\n\u2514\u2500\u2500 pyproject.toml\n" + ], + "citations": [ + { + "source": "Python Packaging Authority", + "title": "src layout vs flat layout", + "url": "https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/", + "relevance": "Official guidance on Python project layouts" + } + ] + }, + "error_message": null + }, + { + "attribute": { + "id": "lock_files", + "name": "Dependency Pinning for Reproducibility", + "category": "Dependency Management", + "tier": 1, + "description": "Dependencies pinned to exact versions in lock files", + "criteria": "Lock file with pinned versions, updated within 6 months", + "default_weight": 0.1 + }, + "status": "fail", + "score": 70.0, + "measured_value": "requirements.txt", + "threshold": "lock file with pinned versions, < 6 months old", + "evidence": [ + "Found requirements.txt: 7 pinned, 3 unpinned", + "\u26a0\ufe0f 3 dependencies not pinned (use '==' not '>=')" + ], + "remediation": { + "summary": "Improve dependency version pinning", + "steps": [ + "Use exact version pinning (== not >=) in requirements.txt", + "Or switch to poetry.lock or Pipfile.lock for automatic pinning", + "Update dependencies regularly (at least every 6 months)" + ], + "tools": [ + "pip", + "poetry", + "pipenv" + ], + "commands": [ + "pip freeze > requirements.txt # Exact versions", + "poetry lock # Auto-managed lock file" + ], + "examples": [], + "citations": [] + }, + "error_message": null + }, + { + "attribute": { + "id": "dependency_security", + "name": "Dependency Security & Vulnerability Scanning", + "category": "Security", + "tier": 1, + "description": "Security scanning tools configured for dependencies and code", + "criteria": "Dependabot, Renovate, CodeQL, or SAST tools configured; secret detection enabled", + "default_weight": 0.04 + }, + "status": "pass", + "score": 35, + "measured_value": "Security tools configured: Renovate", + "threshold": "\u226560 points (Dependabot/Renovate + SAST or multiple scanners)", + "evidence": [ + "\u2713 Renovate configured for dependency updates", + " Meaningful Renovate configuration detected" + ], + "remediation": { + "summary": "Add more security scanning tools for comprehensive coverage", + "steps": [ + "Enable Dependabot alerts in GitHub repository settings (or configure Renovate: add renovate.json to repository root)", + "Add CodeQL scanning workflow for SAST", + "Configure secret detection (detect-secrets, gitleaks)", + "Set up language-specific scanners (pip-audit, npm audit, Snyk)" + ], + "tools": [ + "Dependabot", + "Renovate", + "CodeQL", + "detect-secrets", + "pip-audit", + "npm audit" + ], + "commands": [ + "gh repo edit --enable-security", + "pip install detect-secrets # Python secret detection", + "npm audit # JavaScript dependency audit" + ], + "examples": [ + "# .github/dependabot.yml\nversion: 2\nupdates:\n - package-ecosystem: pip\n directory: /\n schedule:\n interval: weekly" + ], + "citations": [ + { + "source": "OWASP", + "title": "Dependency-Check Project", + "url": "https://owasp.org/www-project-dependency-check/", + "relevance": "Open-source tool for detecting known vulnerabilities in dependencies" + }, + { + "source": "GitHub", + "title": "Dependabot Documentation", + "url": "https://docs.github.com/en/code-security/dependabot", + "relevance": "Official guide for configuring automated dependency updates and security alerts" + } + ] + }, + "error_message": null + }, + { + "attribute": { + "id": "gitignore_completeness", + "name": ".gitignore Completeness", + "category": "Git & Version Control", + "tier": 2, + "description": "Comprehensive .gitignore file with language-specific patterns", + "criteria": ".gitignore exists and includes language-specific patterns from GitHub templates", + "default_weight": 0.03 + }, + "status": "fail", + "score": 41.66666666666667, + "measured_value": "5/12 patterns", + "threshold": "\u226570% of language-specific patterns", + "evidence": [ + ".gitignore found (402 bytes)", + "Pattern coverage: 5/12 (42%)", + "Missing 7 recommended patterns" + ], + "remediation": { + "summary": "Add missing language-specific ignore patterns", + "steps": [ + "Review GitHub's gitignore templates for your language", + "Add the 7 missing patterns", + "Ensure editor/IDE patterns are included" + ], + "tools": [], + "commands": [], + "examples": [ + "# Missing patterns:\n.DS_Store\n*.swo\n.idea/\n*.py[cod]\n.pytest_cache/" + ], + "citations": [ + { + "source": "GitHub", + "title": "gitignore Templates Collection", + "url": "https://github.com/github/gitignore", + "relevance": "Comprehensive collection of language-specific gitignore patterns" + } + ] + }, + "error_message": null + }, + { + "attribute": { + "id": "one_command_setup", + "name": "One-Command Build/Setup", + "category": "Build & Development", + "tier": 2, + "description": "Single command to set up development environment from fresh clone", + "criteria": "Single command (make setup, npm install, etc.) documented prominently", + "default_weight": 0.03 + }, + "status": "pass", + "score": 100, + "measured_value": "make wheels-builder-cuda", + "threshold": "single command", + "evidence": [ + "Setup command found in README: 'make wheels-builder-cuda'", + "Setup automation found: Makefile, pyproject.toml", + "Setup instructions in prominent location" + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "file_size_limits", + "name": "File Size Limits", + "category": "Context Window Optimization", + "tier": 2, + "description": "Files are reasonably sized for AI context windows", + "criteria": "<5% of files >500 lines, no files >1000 lines", + "default_weight": 0.03 + }, + "status": "pass", + "score": 75, + "measured_value": "0 huge, 6 large out of 117", + "threshold": "<5% files >500 lines, 0 files >1000 lines", + "evidence": [ + "Found 6 files 500-1000 lines (5.1% of 117 files)" + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "separation_of_concerns", + "name": "Separation of Concerns", + "category": "Code Organization", + "tier": 2, + "description": "Code organized with single responsibility per module", + "criteria": "Feature-based organization, cohesive modules, low coupling", + "default_weight": 0.03 + }, + "status": "pass", + "score": 92.46153846153847, + "measured_value": "organization:100, cohesion:95, naming:80", + "threshold": "\u226575 overall", + "evidence": [ + "Good directory organization (feature-based or flat)", + "File cohesion: 6/117 files >500 lines", + "Anti-pattern files found: utils.py" + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "concise_documentation", + "name": "Concise Documentation", + "category": "Documentation", + "tier": 2, + "description": "Documentation maximizes information density while minimizing token consumption", + "criteria": "README <500 lines with clear structure, bullet points over prose", + "default_weight": 0.03 + }, + "status": "fail", + "score": 50.0, + "measured_value": "826 lines, 54 headings, 64 bullets", + "threshold": "<500 lines, structured format", + "evidence": [ + "README length: 826 lines (excessive)", + "Heading density: 6.5 per 100 lines (target: 3-5)", + "4 paragraphs exceed 10 lines (walls of text)" + ], + "remediation": { + "summary": "Make documentation more concise and structured", + "steps": [ + "Break long README into multiple documents (docs/ directory)", + "Add clear Markdown headings (##, ###) for structure", + "Convert prose paragraphs to bullet points where possible", + "Add table of contents for documents >100 lines", + "Use code blocks instead of describing commands in prose", + "Move detailed content to wiki or docs/, keep README focused" + ], + "tools": [], + "commands": [ + "# Check README length", + "wc -l README.md", + "", + "# Count headings", + "grep -c '^#' README.md" + ], + "examples": [ + "# Good: Concise with structure\n\n## Quick Start\n```bash\npip install -e .\nagentready assess .\n```\n\n## Features\n- Fast repository scanning\n- HTML and Markdown reports\n- 25 agent-ready attributes\n\n## Documentation\nSee [docs/](docs/) for detailed guides.\n", + "# Bad: Verbose prose\n\nThis project is a tool that helps you assess your repository\nagainst best practices for AI-assisted development. It works by\nscanning your codebase and checking for various attributes that\nmake repositories more effective when working with AI coding\nassistants like Claude Code...\n\n[Many more paragraphs of prose...]\n" + ], + "citations": [ + { + "source": "ArXiv", + "title": "LongCodeBench: Evaluating Coding LLMs at 1M Context Windows", + "url": "https://arxiv.org/abs/2501.00343", + "relevance": "Research showing performance degradation with long contexts" + }, + { + "source": "Markdown Guide", + "title": "Basic Syntax", + "url": "https://www.markdownguide.org/basic-syntax/", + "relevance": "Best practices for Markdown formatting" + } + ] + }, + "error_message": null + }, + { + "attribute": { + "id": "inline_documentation", + "name": "Inline Documentation", + "category": "Documentation", + "tier": 2, + "description": "Function, class, and module-level documentation using language-specific conventions", + "criteria": "\u226580% of public functions/classes have docstrings", + "default_weight": 0.03 + }, + "status": "fail", + "score": 56.67562724014337, + "measured_value": "45.3%", + "threshold": "\u226580%", + "evidence": [ + "Documented items: 253/558", + "Coverage: 45.3%", + "Many public functions/classes lack docstrings" + ], + "remediation": { + "summary": "Add docstrings to public functions and classes", + "steps": [ + "Identify functions/classes without docstrings", + "Add PEP 257 compliant docstrings for Python", + "Add JSDoc comments for JavaScript/TypeScript", + "Include: description, parameters, return values, exceptions", + "Add examples for complex functions", + "Run pydocstyle to validate docstring format" + ], + "tools": [ + "pydocstyle", + "jsdoc" + ], + "commands": [ + "# Install pydocstyle", + "pip install pydocstyle", + "", + "# Check docstring coverage", + "pydocstyle src/", + "", + "# Generate documentation", + "pip install sphinx", + "sphinx-apidoc -o docs/ src/" + ], + "examples": [ + "# Python - Good docstring\ndef calculate_discount(price: float, discount_percent: float) -> float:\n \"\"\"Calculate discounted price.\n\n Args:\n price: Original price in USD\n discount_percent: Discount percentage (0-100)\n\n Returns:\n Discounted price\n\n Raises:\n ValueError: If discount_percent not in 0-100 range\n\n Example:\n >>> calculate_discount(100.0, 20.0)\n 80.0\n \"\"\"\n if not 0 <= discount_percent <= 100:\n raise ValueError(\"Discount must be 0-100\")\n return price * (1 - discount_percent / 100)\n", + "// JavaScript - Good JSDoc\n/**\n * Calculate discounted price\n *\n * @param {number} price - Original price in USD\n * @param {number} discountPercent - Discount percentage (0-100)\n * @returns {number} Discounted price\n * @throws {Error} If discountPercent not in 0-100 range\n * @example\n * calculateDiscount(100.0, 20.0)\n * // Returns: 80.0\n */\nfunction calculateDiscount(price, discountPercent) {\n if (discountPercent < 0 || discountPercent > 100) {\n throw new Error(\"Discount must be 0-100\");\n }\n return price * (1 - discountPercent / 100);\n}\n" + ], + "citations": [ + { + "source": "Python.org", + "title": "PEP 257 - Docstring Conventions", + "url": "https://peps.python.org/pep-0257/", + "relevance": "Python docstring standards" + }, + { + "source": "TypeScript", + "title": "TSDoc Reference", + "url": "https://tsdoc.org/", + "relevance": "TypeScript documentation standard" + } + ] + }, + "error_message": null + }, + { + "attribute": { + "id": "cyclomatic_complexity", + "name": "Cyclomatic Complexity Thresholds", + "category": "Code Quality", + "tier": 3, + "description": "Cyclomatic complexity thresholds enforced", + "criteria": "Average complexity <10, no functions >15", + "default_weight": 0.03 + }, + "status": "pass", + "score": 100.0, + "measured_value": "3.3", + "threshold": "<10.0", + "evidence": [ + "Average cyclomatic complexity: 3.3" + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "cicd_pipeline_visibility", + "name": "CI/CD Pipeline Visibility", + "category": "Testing & CI/CD", + "tier": 3, + "description": "Clear, well-documented CI/CD configuration files", + "criteria": "CI config with descriptive names, caching, parallelization", + "default_weight": 0.015 + }, + "status": "fail", + "score": 70, + "measured_value": "basic config", + "threshold": "CI with best practices", + "evidence": [ + "CI config found: .gitlab-ci.yml", + "Generic job names (consider more descriptive names)", + "No caching detected", + "No parallelization detected", + "Config includes comments", + "Artifacts uploaded" + ], + "remediation": { + "summary": "Add or improve CI/CD pipeline configuration", + "steps": [ + "Create CI config for your platform (GitHub Actions, GitLab CI, etc.)", + "Define jobs: lint, test, build", + "Use descriptive job and step names", + "Configure dependency caching", + "Enable parallel job execution", + "Upload artifacts: test results, coverage reports", + "Add status badge to README" + ], + "tools": [ + "github-actions", + "gitlab-ci", + "circleci" + ], + "commands": [ + "# Create GitHub Actions workflow", + "mkdir -p .github/workflows", + "touch .github/workflows/ci.yml", + "", + "# Validate workflow", + "gh workflow view ci.yml" + ], + "examples": [ + "# .github/workflows/ci.yml - Good example\n\nname: CI Pipeline\n\non:\n push:\n branches: [main]\n pull_request:\n branches: [main]\n\njobs:\n lint:\n name: Lint Code\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: '3.11'\n cache: 'pip' # Caching\n\n - name: Install dependencies\n run: pip install -r requirements.txt\n\n - name: Run linters\n run: |\n black --check .\n isort --check .\n ruff check .\n\n test:\n name: Run Tests\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: '3.11'\n cache: 'pip'\n\n - name: Install dependencies\n run: pip install -r requirements.txt\n\n - name: Run tests with coverage\n run: pytest --cov --cov-report=xml\n\n - name: Upload coverage reports\n uses: codecov/codecov-action@v3\n with:\n files: ./coverage.xml\n\n build:\n name: Build Package\n runs-on: ubuntu-latest\n needs: [lint, test] # Runs after lint/test pass\n steps:\n - uses: actions/checkout@v4\n\n - name: Build package\n run: python -m build\n\n - name: Upload build artifacts\n uses: actions/upload-artifact@v3\n with:\n name: dist\n path: dist/\n" + ], + "citations": [ + { + "source": "GitHub", + "title": "GitHub Actions Documentation", + "url": "https://docs.github.com/en/actions", + "relevance": "Official GitHub Actions guide" + }, + { + "source": "CircleCI", + "title": "CI/CD Best Practices", + "url": "https://circleci.com/blog/ci-cd-best-practices/", + "relevance": "Industry best practices for CI/CD" + } + ] + }, + "error_message": null + }, + { + "attribute": { + "id": "semantic_naming", + "name": "Semantic Naming", + "category": "Code Quality", + "tier": 3, + "description": "Systematic naming patterns following language conventions", + "criteria": "Language conventions followed, avoid generic names", + "default_weight": 0.015 + }, + "status": "pass", + "score": 100.0, + "measured_value": "functions:100%, classes:100%", + "threshold": "\u226575% compliance", + "evidence": [ + "Functions: 187/187 follow snake_case (100.0%)", + "Classes: 6/6 follow PascalCase (100.0%)", + "No generic names (temp, data, obj) detected" + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "branch_protection", + "name": "Branch Protection Rules", + "category": "Git & Version Control", + "tier": 4, + "description": "Required status checks and review approvals before merging", + "criteria": "Branch protection enabled with status checks and required reviews", + "default_weight": 0.005 + }, + "status": "not_applicable", + "score": null, + "measured_value": null, + "threshold": null, + "evidence": [ + "Requires GitHub API integration for branch protection checks. Future implementation will verify: required status checks, required reviews, force push prevention, and branch update requirements." + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "code_smells", + "name": "Code Smell Elimination", + "category": "Code Quality", + "tier": 4, + "description": "Linter configuration for detecting code smells and anti-patterns", + "criteria": "Language-specific linters configured (pylint, ESLint, RuboCop, etc.)", + "default_weight": 0.01 + }, + "status": "pass", + "score": 80.0, + "measured_value": "pylint, ruff", + "threshold": "\u226560% of applicable linters configured", + "evidence": [ + "Linters configured: pylint, ruff", + "Coverage: 40/50 points (80%)" + ], + "remediation": null, + "error_message": null + }, + { + "attribute": { + "id": "container_setup", + "name": "Container/Virtualization Setup", + "category": "Build & Development", + "tier": 4, + "description": "Container configuration for consistent development environments", + "criteria": "Dockerfile/Containerfile, docker-compose.yml, .dockerignore, multi-stage builds", + "default_weight": 0.01 + }, + "status": "not_applicable", + "score": null, + "measured_value": null, + "threshold": null, + "evidence": [ + "Not applicable to ['Markdown', 'JSON', 'YAML', 'Shell', 'Python']" + ], + "remediation": null, + "error_message": null + } + ], + "config": { + "weights": {}, + "excluded_attributes": [ + "test_coverage", + "conventional_commits", + "architecture_decisions", + "structured_logging", + "openapi_specs", + "issue_pr_templates", + "precommit_hooks" + ], + "language_overrides": {}, + "output_dir": null, + "report_theme": "default", + "custom_theme": null + }, + "duration_seconds": 0.5, + "discovered_skills": [] +} \ No newline at end of file