diff --git a/README.md b/README.md index 157bea9..b92ef17 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,10 @@ The config file will be created at `{TLC_CODE_DIR}/config.toml`. See [Configuration Schema](./docs/project-overview-pdr.md#configuration-schema) for all options and validation rules. +## System Packages + +When `install-tools` installs system packages, it uses a curated list that goes beyond what Odoo itself requires. The goal is to pre-install all system-level dependencies needed to compile and run any OCA module out of the box — things like `libcups2-dev` (for `pycups`), `libgeos-dev` (for `shapely`), `libxmlsec1-dev` (for `pysaml2`), `libzbar-dev` (for `pyzbar`), and more. This avoids compilation errors when installing OCA module requirements, without needing to know in advance which modules will be used. + ## System Requirements - Python 3.10+ diff --git a/assets/oca_contributor.toml b/assets/oca_contributor.toml new file mode 100644 index 0000000..6b840ed --- /dev/null +++ b/assets/oca_contributor.toml @@ -0,0 +1,207 @@ +# Exhaustive config to contribute to OCA +# Place this file at ~/code/config.toml (or set TLC_CODE_DIR) + +versions = ["18.0", "19.0"] + +[tools] +uv = [ + "copier", + "oca-port", + "odooly", + "odoo-venv", + "odoo-addons-path" +] +system_packages = ["postgresql"] + +[repos] +odoo = ["odoo"] +oca = [ + # addons repositories + "account-analytic", + "account-budgeting", + "account-closing", + "account-consolidation", + "account-financial-reporting", + "account-financial-tools", + "account-fiscal-rule", + "account-invoice-reporting", + "account-invoicing", + "account-payment", + "account-reconcile", + "agreement", + "ai", + "apps-store", + "automation", + "bank-payment", + "bank-payment-alternative", + "bank-statement-import", + "brand", + "business-requirement", + "calendar", + "cim", + "commission", + "community-data-files", + "connector", + "connector-accountedge", + "connector-cmis", + "connector-ecommerce", + "connector-infor", + "connector-interfaces", + "connector-jira", + "connector-lengow", + "connector-lims", + "connector-magento", + "connector-odoo2odoo", + "connector-prestashop", + "connector-redmine", + "connector-sage", + "connector-salesforce", + "connector-spscommerce", + "connector-telephony", + "connector-woocommerce", + "contract", + "cooperative", + "credit-control", + "crm", + "crowdfunding", + "currency", + "data-protection", + "ddmrp", + "delivery-carrier", + "department", + "dms", + "donation", + "dotnet", + "e-commerce", + "e-learning", + "edi", + "edi-ediversa", + "edi-framework", + "edi-voxel", + "event", + "field-service", + "fleet", + "geospatial", + "helpdesk", + "hr", + "hr-attendance", + "hr-expense", + "hr-holidays", + "infrastructure", + "interface-github", + "intrastat-extrastat", + "iot", + "knowledge", + "l10n-brazil", + "l10n-france", + "l10n-usa", + "mail", + "maintenance", + "management-system", + "manufacture", + "manufacture-reporting", + "margin-analysis", + "mass-mailing", + "mis-builder", + "mis-builder-contrib", + "multi-company", + "odoo-pim", + "operating-unit", + "partner-contact", + "payroll", + "pms", + "pos", + "product-attribute", + "product-configurator", + "product-kitting", + "product-pack", + "product-variant", + "program", + "project", + "project-agile", + "project-reporting", + "purchase-reporting", + "purchase-workflow", + "pwa-builder", + "queue", + "repair", + "report-print-send", + "reporting-engine", + "resource", + "rest-api", + "rest-framework", + "rma", + "role-policy", + "sale-blanket", + "sale-channel", + "sale-financial", + "sale-prebook", + "sale-promotion", + "sale-reporting", + "sale-workflow", + "search-engine", + "server-auth", + "server-backend", + "server-brand", + "server-env", + "server-tools", + "server-ux", + "shift-planning", + "shopfloor-app", + "sign", + "social", + "spreadsheet", + "stock-logistics-availability", + "stock-logistics-barcode", + "stock-logistics-interfaces", + "stock-logistics-orderpoint", + "stock-logistics-putaway", + "stock-logistics-release-channel", + "stock-logistics-reporting", + "stock-logistics-request", + "stock-logistics-reservation", + "stock-logistics-shopfloor", + "stock-logistics-tracking", + "stock-logistics-transport", + "stock-logistics-warehouse", + "stock-logistics-workflow", + "stock-weighing", + "storage", + "survey", + "timesheet", + "vertical-abbey", + "vertical-agriculture", + "vertical-association", + "vertical-community", + "vertical-construction", + "vertical-cooperative-supermarket", + "vertical-edition", + "vertical-education", + "vertical-hotel", + "vertical-isp", + "vertical-medical", + "vertical-ngo", + "vertical-realestate", + "vertical-rental", + "vertical-travel", + "wallet", + "web", + "web-api", + "web-api-contrib", + "webhook", + "webkit-tools", + "website", + "website-cms", + "website-themes", + "wms", + # exceptions, new repositories + ["oca-custom", ["18.0"]], + ["tier-validation", ["19.0"]], + # tooling + ["oca-ci", ["master"]], + ["oca-github-bot", ["master"]], + ["oca-port", ["main"]], + ["odoo-module-migrator", ["master"]], +] +camptocamp = ["odoo-cloud-platform"] +forgeflow = ["stock-rma"] diff --git a/tests/test_pull_repos.py b/tests/test_pull_repos.py index 134d874..92325e0 100644 --- a/tests/test_pull_repos.py +++ b/tests/test_pull_repos.py @@ -161,7 +161,7 @@ def test_get_tasks_generates_correct_list(mock_config, tmp_path): { "repo_name": "server-tools", "repo_path": code_root / "oca" / "16.0" / "server-tools", - "repo_url": "git@github.com:OCA/server-tools.git", + "repo_url": "git@github.com:oca/server-tools.git", "version": "16.0", }, { @@ -173,7 +173,7 @@ def test_get_tasks_generates_correct_list(mock_config, tmp_path): { "repo_name": "server-tools", "repo_path": code_root / "oca" / "17.0" / "server-tools", - "repo_url": "git@github.com:OCA/server-tools.git", + "repo_url": "git@github.com:oca/server-tools.git", "version": "17.0", }, ] @@ -209,3 +209,31 @@ def test_get_tasks_with_filter(mock_config, tmp_path): task["repo_path"] = str(task["repo_path"]) assert tasks == expected_tasks + + +def test_get_tasks_inline_branch_override(mock_config, tmp_path): + odoo_versions = ["17.0", "18.0"] + repos_config = { + "oca": [ + "server-tools", + ["oca-port", ["main"]], + ["oca-custom", ["17.0", "18.0"]], + ] + } + code_root = tmp_path / "code" + + tasks = _get_tasks(odoo_versions, repos_config, code_root, None) + + paths = {(t["repo_name"], t["version"]): t["repo_path"] for t in tasks} + + # plain string → one task per configured version + assert (code_root / "oca" / "17.0" / "server-tools") == paths[("server-tools", "17.0")] + assert (code_root / "oca" / "18.0" / "server-tools") == paths[("server-tools", "18.0")] + # inline branch override → only the specified branches, no version loop + assert ("oca-port", "17.0") not in paths + assert ("oca-port", "18.0") not in paths + assert (code_root / "oca" / "main" / "oca-port") == paths[("oca-port", "main")] + # multiple explicit branches + assert (code_root / "oca" / "17.0" / "oca-custom") == paths[("oca-custom", "17.0")] + assert (code_root / "oca" / "18.0" / "oca-custom") == paths[("oca-custom", "18.0")] + assert len(tasks) == 5 diff --git a/trobz_local/concurrency.py b/trobz_local/concurrency.py index 941d7b1..f98c237 100644 --- a/trobz_local/concurrency.py +++ b/trobz_local/concurrency.py @@ -14,24 +14,31 @@ class TaskResult: def run_tasks(tasks, max_workers: int = 4): results = [] + total = len(tasks) + completed_count = 0 + with Progress( TextColumn("[progress.description]{task.description}"), BarColumn(), TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), ) as progress: + overall = progress.add_task(f"[cyan]0/{total} done", total=total) + future_to_task = {} with ThreadPoolExecutor(max_workers=max_workers) as executor: try: - # submit tasks + # Submit tasks with hidden progress rows; reveal them on start for task_info in tasks: name = task_info["name"] func = task_info["func"] kwargs = task_info.get("args", {}) - task_id = progress.add_task( - name, - total=100, - ) - future = executor.submit(func, progress, task_id, **kwargs) + task_id = progress.add_task(name, total=100, visible=False) + + def _run(f=func, tid=task_id, kw=kwargs): + progress.update(tid, visible=True) + return f(progress, tid, **kw) + + future = executor.submit(_run) future_to_task[future] = {"name": name, "task_id": task_id} # Wait for all tasks to complete @@ -42,7 +49,14 @@ def run_tasks(tasks, max_workers: int = 4): try: future.result() - progress.update(task_id, completed=100) + task = next(t for t in progress.tasks if t.id == task_id) + label = task.description or name + progress.update(task_id, completed=100, visible=False) + completed_count += 1 + progress.update( + overall, completed=completed_count, description=f"[cyan]{completed_count}/{total} done" + ) + progress.console.print(label) results.append(TaskResult(name=name, success=True, message="Completed")) except Exception as e: diff --git a/trobz_local/main.py b/trobz_local/main.py index 8c41ca0..6795340 100644 --- a/trobz_local/main.py +++ b/trobz_local/main.py @@ -193,27 +193,43 @@ def pull_repos( # noqa: C901 typer.secho("\nAll repositories updated successfully.", fg=typer.colors.GREEN) +def _iter_org_entries(org_repos, odoo_versions): + """Yield (repo_name, branch) pairs for an org's repo list. + + Plain strings use all configured versions; [name, [branch, ...]] entries + use their explicit branch list. + """ + for entry in org_repos: + if isinstance(entry, str): + for version in odoo_versions: + yield entry, str(version) + else: + for branch in entry[1]: + yield entry[0], str(branch) + + def _get_tasks(odoo_versions, repos_config, code_root, repo_filter): tasks = [] for version in odoo_versions: - if "odoo" in repos_config: - for repo_name in repos_config["odoo"]: - if repo_name in ODOO_URLS and (not repo_filter or repo_name in repo_filter): - tasks.append({ - "repo_name": repo_name, - "repo_path": code_root / "odoo" / repo_name / version, - "repo_url": ODOO_URLS[repo_name], - "version": str(version), - }) - if "oca" in repos_config: - for repo_name in repos_config["oca"]: - if not repo_filter or repo_name in repo_filter: - tasks.append({ - "repo_name": repo_name, - "repo_path": code_root / "oca" / str(version) / repo_name, - "repo_url": f"git@github.com:OCA/{repo_name}.git", - "version": str(version), - }) + for repo_name in repos_config.get("odoo", []): + if repo_name in ODOO_URLS and (not repo_filter or repo_name in repo_filter): + tasks.append({ + "repo_name": repo_name, + "repo_path": code_root / "odoo" / repo_name / version, + "repo_url": ODOO_URLS[repo_name], + "version": str(version), + }) + for org, org_repos in repos_config.items(): + if org == "odoo": + continue + for repo_name, branch in _iter_org_entries(org_repos, odoo_versions): + if not repo_filter or repo_name in repo_filter: + tasks.append({ + "repo_name": repo_name, + "repo_path": code_root / org / branch / repo_name, + "repo_url": f"git@github.com:{org}/{repo_name}.git", + "version": branch, + }) return tasks diff --git a/trobz_local/utils.py b/trobz_local/utils.py index 88ebdd9..4ef6bec 100644 --- a/trobz_local/utils.py +++ b/trobz_local/utils.py @@ -8,7 +8,7 @@ import git import tomli import typer -from pydantic import BaseModel, Field, ValidationError, field_validator +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator from rich import print as rprint from rich.progress import ( Progress, @@ -19,15 +19,30 @@ VERSION_REGEX = re.compile(r"^(?:\d+\.\d+|master)$") ARCH_PACKAGES = [ + "git", "gcc", - "postgresql", + "cyrus-sasl", + "libldap", + "openssl", # cryptography + "libffi", # cairosvg + "libxml2", # lxml, pysaml2 + "libxslt", # lxml + "libjpeg-turbo", "postgresql-libs", - "libxml2", - "libxslt", - "libjpeg", "libsass", + "cracklib", + "geos", # shapely + "xmlsec", # pysaml2 + "zbar", # pyzbar + "cairo", # cairosvg + "cups", # pycups + "fontconfig", + "graphviz", + "ghostscript", + "gsfonts", + "poppler", # pdf2image + "postgresql", "base-devel", - "git", ] UBUNTU_PACKAGES = [ @@ -35,13 +50,27 @@ "gcc", "libsasl2-dev", "libldap2-dev", - "libssl-dev", - "libffi-dev", - "libxml2-dev", - "libxslt1-dev", + "libssl-dev", # cryptography + "libffi-dev", # cairosvg + "libxml2-dev", # lxml, pysaml2 + "libxslt1-dev", # lxml "libjpeg-dev", "libpq-dev", "libsass-dev", + "libcrack2-dev", + "libgeos-dev", # shapely + "libxmlsec1-dev", # pysaml2 + "libxmlsec1-openssl", # pysaml2 + "libzbar0", # pyzbar + "libzbar-dev", # pyzbar + "libcairo2", # cairosvg + "libcups2-dev", # pycups + "fontconfig", + "fontconfig-config", + "graphviz", + "ghostscript", + "gsfonts", + "poppler-utils", # pdf2image "postgresql", "postgresql-client", "postgresql-contrib", @@ -79,18 +108,58 @@ def __init__(self, pkg: str): super().__init__(f"Invalid npm package name: {pkg}") +class InvalidRepoOrgConfigError(TypeError): + def __init__(self, org: str): + super().__init__(f"[repos.{org}] must be a list") + + +class InvalidRepoEntryError(ValueError): + def __init__(self, org: str): + super().__init__(f"Invalid entry in [repos.{org}]: must be a name or [name, [branch, ...]]") + + +_REPO_NAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$") + + +def _validate_repo_entry(entry: object, org: str) -> None: + if isinstance(entry, str): + if not _REPO_NAME_RE.match(entry): + raise InvalidRepoNameError(entry) + elif ( + isinstance(entry, list) + and len(entry) == 2 + and isinstance(entry[0], str) + and isinstance(entry[1], list) + and all(isinstance(b, str) for b in entry[1]) + ): + if not _REPO_NAME_RE.match(entry[0]): + raise InvalidRepoNameError(entry[0]) + else: + raise InvalidRepoEntryError(org) + + class RepoConfig(BaseModel): + model_config = ConfigDict(extra="allow") + odoo: list[str] = [] - oca: list[str] = [] - @field_validator("*") + @field_validator("odoo") @classmethod - def validate_repo_names(cls, v: list[str]): + def validate_odoo_repos(cls, v: list[str]) -> list[str]: for name in v: - if not re.match(r"^[a-zA-Z0-9._-]+$", name): + if not _REPO_NAME_RE.match(name): raise InvalidRepoNameError(name) return v + @model_validator(mode="after") + def validate_orgs(self): + for org, repos in self.model_extra.items(): + if not isinstance(repos, list): + raise InvalidRepoOrgConfigError(org) + for entry in repos: + _validate_repo_entry(entry, org) + return self + class ScriptItem(BaseModel): """Configuration for a script to download and execute."""