From 46678c0d5fc286f096132034a46e9b97203f193b Mon Sep 17 00:00:00 2001 From: Vainock Date: Sun, 17 Mar 2024 13:50:04 +0100 Subject: [PATCH 1/3] frontend: Remove obsolete Language string --- frontend/data/locale/en-US.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/frontend/data/locale/en-US.ini b/frontend/data/locale/en-US.ini index 15b1f4aed5988c..ca72226d53ad7e 100644 --- a/frontend/data/locale/en-US.ini +++ b/frontend/data/locale/en-US.ini @@ -2,9 +2,6 @@ # Pull requests for translations outside of Crowdin will not be accepted. # Read this forum post for more instructions on submitting translations: https://obsproject.com/forum/threads/how-to-contribute-translations-for-obs.16327/ -# Language of this file -Language="English" - # commonly shared locale OK="OK" Apply="Apply" From d5d8f06d78938a323ed88821c02a9a3fe33324d2 Mon Sep 17 00:00:00 2001 From: Vainock Date: Sun, 17 Mar 2024 13:52:21 +0100 Subject: [PATCH 2/3] frontend: Update URLs in source locale file --- frontend/data/locale/en-US.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/data/locale/en-US.ini b/frontend/data/locale/en-US.ini index ca72226d53ad7e..bb6f637d938cc9 100644 --- a/frontend/data/locale/en-US.ini +++ b/frontend/data/locale/en-US.ini @@ -1,6 +1,6 @@ -# Note to translators: *DO NOT* translate this file directly. Instead, visit http://crowdin.com/project/obs-studio and submit your translations there. +# Note to translators: *DO NOT* translate this file directly. Instead, visit https://crowdin.com/project/obs-studio and submit your translations there. # Pull requests for translations outside of Crowdin will not be accepted. -# Read this forum post for more instructions on submitting translations: https://obsproject.com/forum/threads/how-to-contribute-translations-for-obs.16327/ +# Read this forum post for more instructions on submitting translations: https://obsproject.com/wiki/How-To-Contribute-Translations-For-OBS # commonly shared locale OK="OK" From a2457cd443244b0f3c027c0af815687ca69f679a Mon Sep 17 00:00:00 2001 From: Vainock Date: Sat, 20 Apr 2024 18:59:24 +0200 Subject: [PATCH 3/3] CI: Rewrite obs-crowdin-sync in-tree --- .../update-crowdin-locales/action.yaml | 33 ++ .../actions/update-translations/action.yaml | 46 +++ .../utils.py/update-crowdin-locales.py | 110 +++++++ .../scripts/utils.py/update-translations.py | 308 ++++++++++++++++++ .github/workflows/dispatch.yaml | 15 +- .github/workflows/scheduled.yaml | 14 +- 6 files changed, 513 insertions(+), 13 deletions(-) create mode 100644 .github/actions/update-crowdin-locales/action.yaml create mode 100644 .github/actions/update-translations/action.yaml create mode 100644 .github/scripts/utils.py/update-crowdin-locales.py create mode 100644 .github/scripts/utils.py/update-translations.py diff --git a/.github/actions/update-crowdin-locales/action.yaml b/.github/actions/update-crowdin-locales/action.yaml new file mode 100644 index 00000000000000..f44c9b4ef77b8e --- /dev/null +++ b/.github/actions/update-crowdin-locales/action.yaml @@ -0,0 +1,33 @@ +name: Update locales on Crowdin +description: Updates the US English locales on Crowdin. +inputs: + crowdinSecret: + description: Crowdin Token for API access + required: true + changedFiles: + description: List of changed en-US.ini files + required: true +runs: + using: composite + steps: + - name: Check Runner Operating System πŸƒβ€β™‚οΈ + if: runner.os != 'Linux' + shell: bash + run: | + echo "::warning::Action requires Linux-based runner." + exit 2 + + - name: Install and Configure Python 🐍 + shell: bash + run: | + python3 -m venv .venv + source .venv/bin/activate + python3 -m pip install crowdin-api-client + + - name: Update Locales on Crowdin πŸ‡ΊπŸ‡Έ + shell: bash + run: | + source .venv/bin/activate + python3 .github/scripts/utils.py/update-crowdin-locales.py \ + ${{ inputs.crowdinSecret }} \ + ${{ inputs.changedFiles }} diff --git a/.github/actions/update-translations/action.yaml b/.github/actions/update-translations/action.yaml new file mode 100644 index 00000000000000..50cacf2e4c253e --- /dev/null +++ b/.github/actions/update-translations/action.yaml @@ -0,0 +1,46 @@ +name: Update Translations and AUTHORS +description: Creates a Pull Request updating translations and AUTHORS. +inputs: + crowdinSecret: + description: Crowdin Token for API access + required: true +runs: + using: composite + steps: + - name: Check Runner Operating System πŸƒβ€β™‚οΈ + if: runner.os != 'Linux' + shell: bash + run: | + echo "::warning::Action requires Linux-based runner." + exit 2 + + - name: Install and Configure Python 🐍 + shell: bash + run: | + python3 -m venv .venv + source .venv/bin/activate + python3 -m pip install lxml requests crowdin-api-client + + - name: Update Translations 🌐 + shell: bash + run: | + source .venv/bin/activate + python3 .github/scripts/utils.py/update-translations.py \ + ${{ inputs.crowdinSecret }} + + - name: Create Pull Request πŸ”§ + uses: peter-evans/create-pull-request@6ce4eca6b6db0ff4f4d1b542dce50e785446dc27 + with: + author: OBS Translators + commit-message: Update Translations and AUTHORS + title: Update Translations and AUTHORS + branch: automated/update-translations + body: | + This updates the translations and the `AUTHORS` file. + + Translations are pulled from https://crowdin.com/project/obs-studio. + Top translators are pulled from https://crowdin.com/project/obs-studio/reports/top-members. + Top Git contributors are generated using `git shortlog --all -sn --no-merges` + + The creation of this Pull Request was triggered manually. + delete-branch: true diff --git a/.github/scripts/utils.py/update-crowdin-locales.py b/.github/scripts/utils.py/update-crowdin-locales.py new file mode 100644 index 00000000000000..06b5c4a2d612fa --- /dev/null +++ b/.github/scripts/utils.py/update-crowdin-locales.py @@ -0,0 +1,110 @@ +import os +import logging +import json +import argparse +import sys + +from crowdin_api import CrowdinClient +from fnmatch import fnmatch + + +def get_dir_id(name: str) -> int: + crowdin_api.source_files.list_directories(filter=name)["data"][0]["data"]["id"] + + +def add_to_crowdin_storage(file_path: str) -> int: + return crowdin_api.storages.add_storage(open(file_path))["data"]["id"] + + +def update_crowdin_file(file_id: int, file_path: str) -> None: + logger = logging.getLogger() + + storage_id = add_to_crowdin_storage(file_path) + crowdin_api.source_files.update_file(file_id, storage_id) + + logger.info(f"{file_path} updated on Crowdin.") + + +def create_crowdin_file(file_path: str, name: str, directory_name: str, export_pattern: str): + logger = logging.getLogger() + + storage_id = add_to_crowdin_storage(file_path) + + crowdin_api.source_files.add_file(storage_id, + name, + directoryId=get_dir_id(directory_name), + exportOptions={"exportPattern": export_pattern}) + + logger.info(f"{file_path} created on Crowdin in {directory_name} as {name}.") + + +def upload(updated_locales: list[str]) -> None: + default_locale = "en-US" + + logger = logging.getLogger() + export_paths_map = dict() + + for source_file_data in crowdin_api.source_files.list_files()["data"]: + source_file = source_file_data["data"] + + if "exportOptions" not in source_file: + continue + + export_path: str = source_file["exportOptions"]["exportPattern"] + export_path = export_path[1:] + export_path = export_path.replace("%file_name%", os.path.basename(source_file["name"]).split(".")[0]) + export_path = export_path.replace("%locale%", default_locale) + + export_paths_map[export_path] = source_file["id"] + + for locale_path in updated_locales: + if not os.path.exists(locale_path): + logger.warning(f"Unable to find {locale_path} in working directory.") + continue + + path_parts = locale_path.split("/") + + if locale_path in export_paths_map: + crowdin_file_id = export_paths_map[locale_path] + update_crowdin_file(crowdin_file_id, locale_path) + elif fnmatch(locale_path, f"plugins/*/data/locale/{default_locale}.ini"): + create_crowdin_file(locale_path, f"{path_parts[1]}.ini", + path_parts[0], "/plugins/%file_name%/data/locale/%locale%.ini") + elif fnmatch(locale_path, f"frontend/plugins/*/data/locale/{default_locale}.ini"): + create_crowdin_file(locale_path, f"{path_parts[2]}.ini", + path_parts[1], "/frontend/plugins/%file_name%/data/locale/%locale%.ini") + else: + logger.error(f"Unable to create {locale_path} on Crowdin due to its unexpected location.") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Update Crowdin source files based on provided list of updated locales") + parser.add_argument("crowdin_secret", type=str, help="Crowdin API Token with manager access") + parser.add_argument("updated_locales", type=str, help="JSON array of updated locales") + parser.add_argument("--project_id", type=int, default=51028, required=False) + parser.add_argument("--loglevel", type=str, default="INFO", required=False) + arguments = parser.parse_args() + + logging.basicConfig(level=arguments.loglevel) + logger = logging.getLogger() + + global crowdin_api + crowdin_api = CrowdinClient(token=arguments.crowdin_secret, project_id=arguments.project_id, page_size=500) + + try: + updated_locales = json.loads(arguments.updated_locales) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse {e.doc}: {e}") + return 1 + + if len(updated_locales) != 0: + upload(updated_locales) + else: + logger.error("List of updated locales is empty.") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/utils.py/update-translations.py b/.github/scripts/utils.py/update-translations.py new file mode 100644 index 00000000000000..2f91a509a120da --- /dev/null +++ b/.github/scripts/utils.py/update-translations.py @@ -0,0 +1,308 @@ +import time +import requests +import fnmatch +import logging +import subprocess +import os +import sys +import argparse + +from lxml import etree +from io import BytesIO +from crowdin_api import CrowdinClient +from zipfile import ZipFile +from typing import NewType + + +CrowdinBuild = NewType("CrowdinBuild", dict[str, dict[str, str]]) # file path mapped to string keys & translations + + +def generate_build() -> int: + build_res = crowdin_api.translations.build_project_translation(dict({"skipUntranslatedStrings": True}))["data"] + + is_finished = build_res["status"] == "finished" + while not is_finished: + time.sleep(5.0) + is_finished = crowdin_api.translations.check_project_build_status(build_res["id"])["data"]["status"] == "finished" + + return build_res["id"] + + +def download_build(build_id: int) -> CrowdinBuild: + logger = logging.getLogger() + build: CrowdinBuild = dict() + + download_url = crowdin_api.translations.download_project_translations(build_id)["data"]["url"] + build_download_req = requests.get(download_url) + + with ZipFile(BytesIO(build_download_req.content)) as build_zip: + file_list = build_zip.namelist() + file_list.sort() + + for file_name in file_list: + if not file_name.endswith(".ini"): + continue + + content = build_zip.read(file_name).decode() + + content = content.strip() + if content == "": + continue + + content = content.replace("\r\n", "\n") + content = content.replace("\r", "\n") + + while "\n\n" in content: + content = content.replace("\n\n", "\n") + + # Manually placed linebreaks in translations break the parsing + fixed_linebreaks = "" + previous_line = "" + + for line in content.splitlines(): + if '="' in line and not line.endswith('="'): + fixed_linebreaks += "\n" + else: + logger.warning(f"{file_name} contains a manually placed linebreak which should be fixed on Crowdin. Line: {previous_line}") + fixed_linebreaks += line + + previous_line = line + + fixed_linebreaks = fixed_linebreaks.strip() + + # Parse to CrowdinBuild + build[file_name] = {line[:line.index("=")]: line[line.index('"') + 1:-1] for line in fixed_linebreaks.splitlines()} + + return build + + +def get_locale_from_path(file_path: str) -> str: + return file_path[file_path.rindex("/")+1:file_path.rindex(".")] + + +def filter_build(build: CrowdinBuild, patterns: list[str]) -> CrowdinBuild: + filtered_build: CrowdinBuild = dict() + + for pattern in patterns: + filtered_build = filtered_build | {key: build[key] for key in fnmatch.filter(list(build), pattern)} + + return filtered_build + + +def write_core_translations(build: CrowdinBuild) -> None: + for file_path, translations in build.items(): + with open(file_path, "w") as file: + output = "" + for key, translation in translations.items(): + output += f'{key}="{translation}"\n' + file.write(output) + + +def write_locale_file(build: CrowdinBuild) -> None: + logger = logging.getLogger() + + locale_file_path = "frontend/data/locale.ini" + default_language_name = "English (USA)" + default_locale = "en-US" + language_progress_threshold = 35 # Required percentage of language progress for a language to be added to the locale.ini + + locales_to_write: list[str] = [] + + with open(locale_file_path, encoding="utf-8") as file: + for line in file: + line = line.strip() + if line.startswith("["): + locales_to_write.append(line[1:-1]) + + for progress_data in crowdin_api.translation_status.get_project_progress()["data"]: + locale = progress_data["data"]["language"]["locale"] + if not locale in locales_to_write: + if progress_data["data"]["translationProgress"] >= language_progress_threshold: + locales_to_write.append(locale) + + build[f"language-name/{default_locale}.ini"] = {"Language": default_language_name} + if not default_locale in locales_to_write: + locales_to_write.append(default_locale) + + locales_to_write.sort() # Sort again because we manually inserted en-US + output = "" + + for locale in locales_to_write: + file_path = f"language-name/{locale}.ini" + if not file_path in build: + logger.warning(f"Unable to add {locale} to locale.ini because it is missing the translation file.") + continue + + translations = build[file_path] + + if not "Language" in translations: + logger.warning(f"Unable to add {locale} to locale.ini because it is missing the language name.") + continue + + output += f"[{locale}]\n" + output += f'Name={translations["Language"]}\n' + output += "\n" + + output = output[:-1] + with open(locale_file_path, "w", encoding="utf-8") as file: + file.write(output) + + +def write_desktop_entry(build: CrowdinBuild) -> None: + desktop_entry_file_path = "frontend/cmake/linux/com.obsproject.Studio.desktop" + + content = "" + + with open(desktop_entry_file_path, encoding="utf-8") as file: + # Remove previous translations + content = file.read().split("\n\n")[0] + + content += "\n\n" + + for file_path, translations in build.items(): + locale = get_locale_from_path(file_path) + + for key, translation in translations.items(): + content += f'{key}[{locale}]={translation}\n' + + with open(desktop_entry_file_path, "w", encoding="utf-8") as file: + file.write(content) + + +def write_meta_info(build: CrowdinBuild) -> None: + meta_file_path = "frontend/cmake/linux/com.obsproject.Studio.metainfo.xml.in" + meta_file_id = 758 # id of the file probably called "Metadata" + language_code_map = {"en_GB": "en-GB", "nb": "nb-NO", "pt_BR": "pt-BR"} # Flathub and Crowdin don't share exactly the same language codes + + logger = logging.getLogger() + + source_strings_data = crowdin_api.source_strings.list_strings(fileId=meta_file_id)["data"] + xpath_list = [string["data"]["identifier"] for string in source_strings_data] + + read_file = "" + with open(meta_file_path, encoding="utf-8") as file: + for line in file.read().splitlines(): + if ' xml:lang="' in line: + continue + + read_file += line + + xml_tree = etree.ElementTree(etree.fromstring(read_file.encode())) + + for xpath in reversed(xpath_list): + for file_path, translations in reversed(build.items()): + if not xpath in translations: + continue + + source_element = xml_tree.find(f".{xpath}") + + if source_element is None: + logger.warning(f"Unable to find {xpath} from {file_path} in meta-info.") + continue + + translated_element = etree.Element(source_element.tag) + translated_element.text = translations[xpath] + + locale = get_locale_from_path(file_path) + + if locale in language_code_map: + locale = language_code_map[locale] + + translated_element.set("xmllang", locale) + source_element.addnext(translated_element) + + etree.indent(xml_tree) + + output = '\n' + output += etree.tostring(xml_tree, encoding="utf-8").decode() + output = output.replace(' xmllang="', ' xml:lang="') + output += "\n" + + with open(meta_file_path, "w", encoding="utf-8") as file: + file.write(output) + + +def write_authors() -> None: + authors_file_path = "AUTHORS" + excluded_commiters = ["Translation Updater", "Service Checker"] + + output = "Original Author: Lain Bailey\n\nContributors are sorted by their amount of commits / translated words.\n\n" + output += "Git Contributors:\n" + + git_output = subprocess.check_output(["git", "shortlog", "--all", "-sn", "--no-merges"]).decode() + for line in git_output.splitlines(): + committer_name = line.split("\t")[1] + if committer_name not in excluded_commiters: + output += f" {committer_name}\n" + + output += "\nTranslators:\n" + + languages_data = crowdin_api.projects.get_project()["data"]["targetLanguages"] + target_languages = {language["name"]: language["id"] for language in languages_data} + + blocked_users_data = crowdin_api.users.list_project_members(role="blocked")["data"] + blocked_users_ids = [user["data"]["id"] for user in blocked_users_data] + + for language_name in sorted(target_languages): + generate_report_data = crowdin_api.reports.generate_top_members_report(format="json", languageId=target_languages[language_name])["data"] + + report_id = generate_report_data["identifier"] + report_finished = generate_report_data["status"] == "finished" + while not report_finished: + time.sleep(1) + report_finished = crowdin_api.reports.check_report_generation_status(report_id)["data"]["status"] == "finished" + + report_download_url = crowdin_api.reports.download_report(report_id)["data"]["url"] + report_download_res = requests.get(report_download_url).json() + + if not "data" in report_download_res: + continue + + output += f" {language_name}:\n" + + for user_data in report_download_res["data"]: + user = user_data["user"] + full_name = user["fullName"] + + if full_name == "REMOVED_USER": + continue + + if user["id"] in blocked_users_ids: + continue + + if user_data["translated"] == 0 and user_data["approved"] == 0: + continue + + output += f" {full_name}\n" + + with open(authors_file_path, "w", encoding="utf-8") as file: + file.write(output) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Update translations from Crowdin and update the AUTHORS file") + parser.add_argument("crowdin_secret", type=str, help="Crowdin API Token with manager access") + parser.add_argument("--project_id", type=int, default=51028, required=False) + parser.add_argument("--loglevel", type=str, default="INFO", required=False) + arguments = parser.parse_args() + + logging.basicConfig(level=arguments.loglevel) + + global crowdin_api + crowdin_api = CrowdinClient(token=arguments.crowdin_secret, project_id=arguments.project_id, page_size=500) + + build_id = generate_build() + build = download_build(build_id) + + write_core_translations(filter_build(build, ["frontend/data/locale/*.ini", + "plugins/*/data/locale/*.ini", + "plugins/mac-virtualcam/src/obs-plugin/data/locale/*.ini", + "frontend/plugins/*/data/locale/*.ini"])) + write_locale_file(filter_build(build, ["language-name/*.ini"])) + write_desktop_entry(filter_build(build, ["desktop-entry/*.ini"])) + write_meta_info(filter_build(build, ["meta-info/*.ini"])) + write_authors() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/dispatch.yaml b/.github/workflows/dispatch.yaml index e7b06898b66f42..9ea63abe2b6a07 100644 --- a/.github/workflows/dispatch.yaml +++ b/.github/workflows/dispatch.yaml @@ -56,19 +56,22 @@ jobs: runServiceChecks: true createPullRequest: true - download-language-files: - name: Download Language Files 🌐 + update-language-files: + name: Update Language Files 🌐 if: github.repository_owner == 'obsproject' && inputs.job == 'translations' runs-on: ubuntu-24.04 - env: - CROWDIN_PAT: ${{ secrets.CROWDIN_SYNC_CROWDIN_PAT }} + permissions: + pull-requests: write + contents: write steps: - uses: actions/checkout@v4 with: submodules: recursive - token: ${{ secrets.CROWDIN_SYNC_GITHUB_PAT }} fetch-depth: 0 - - uses: obsproject/obs-crowdin-sync/download@84628cad04d2423e02443c64965cb834b4c5a245 + - name: Update Language Files 🌐 + uses: ./.github/actions/update-translations + with: + crowdinSecret: ${{ secrets.CROWDIN_SYNC_CROWDIN_PAT }} steam-upload: name: Upload Steam Builds πŸš‚ diff --git a/.github/workflows/scheduled.yaml b/.github/workflows/scheduled.yaml index 022b7fa8d5b809..fba95141529bf1 100644 --- a/.github/workflows/scheduled.yaml +++ b/.github/workflows/scheduled.yaml @@ -106,8 +106,8 @@ jobs: permissions: security-events: write - upload-language-files: - name: Upload Language Files 🌐 + update-crowdin-locales: + name: Update Locales on Crowdin πŸ‡ΊπŸ‡Έ if: github.repository_owner == 'obsproject' && github.ref_name == 'master' runs-on: ubuntu-24.04 steps: @@ -152,12 +152,12 @@ jobs: baseRef: ${{ steps.nightly-checks.outputs.lastNightly }} checkGlob: '**/en-US.ini' - - name: Upload US English Language Files πŸ‡ΊπŸ‡Έ + - name: Update Locales on Crowdin πŸ‡ΊπŸ‡Έ if: steps.checks.outcome == 'success' && fromJSON(steps.checks.outputs.hasChangedFiles) - uses: obsproject/obs-crowdin-sync/upload@84628cad04d2423e02443c64965cb834b4c5a245 - env: - CROWDIN_PAT: ${{ secrets.CROWDIN_SYNC_CROWDIN_PAT }} - GITHUB_EVENT_BEFORE: ${{ steps.nightly-checks.outputs.lastNightly }} + uses: ./.github/actions/update-crowdin-locales + with: + crowdinSecret: ${{ secrets.CROWDIN_SYNC_CROWDIN_PAT }} + changedFiles: ${{ steps.checks.outputs.changedFiles }} steam-upload: name: Upload Steam Builds πŸš‚