chore: automatic crowdin sync via gh actions (#5630)

.github/workflows/locale-sync.yml

@@ -0,0 +1,103 @@
+name: Automatic Locale Sync
+
+on:
+  schedule:
+    # Run every Sunday at 2 AM UTC
+    - cron: '0 2 * * 0'
+  workflow_dispatch:
+    # Allow manual triggering from the GitHub UI
+
+jobs:
+  sync-locales:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Check venv cache
+        id: cache-validate
+        if: steps.cached-poetry-dependencies.outputs.cache-hit == 'true'
+        run: |
+          echo "import fastapi;print('venv good?')" > test.py && poetry run python test.py && echo "cache-hit-success=true" >> $GITHUB_OUTPUT
+          rm test.py
+        continue-on-error: true
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libsasl2-dev libldap2-dev libssl-dev
+          poetry install
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+
+      - name: Run locale generation
+        run: |
+          cd dev/code-generation
+          poetry run python main.py locales
+        env:
+          CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
+
+      - name: Check for changes
+        id: changes
+        run: |
+          if git diff --quiet; then
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+          else
+            echo "has_changes=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Commit and create PR
+        if: steps.changes.outputs.has_changes == 'true'
+        run: |
+          # Configure git
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+
+          # Create a new branch
+          BRANCH_NAME="auto-locale-sync-$(date +%Y%m%d-%H%M%S)"
+          git checkout -b "$BRANCH_NAME"
+
+          # Add and commit changes
+          git add .
+          git commit -m "chore: automatic locale sync"
+
+          # Push the branch
+          git push origin "$BRANCH_NAME"
+
+          # Create PR using GitHub CLI
+          gh pr create --title "chore: automatic locale sync" --body "## Summary
+
+          Automatically generated locale updates from the weekly sync job.
+
+          ## Changes
+          - Updated frontend locale files
+          - Generated from latest translation sources
+
+          ## Test plan
+          - [ ] Verify locale files are properly formatted
+          - [ ] Test that translations load correctly in the frontend" --base dev --head "$BRANCH_NAME"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: No changes detected
+        if: steps.changes.outputs.has_changes == 'false'
+        run: echo "No locale changes detected, skipping PR creation"

Taskfile.yml

@@ -70,7 +70,7 @@ tasks:
   dev:generate:
     desc: run code generators
     cmds:
-      - poetry run python dev/code-generation/main.py
+      - poetry run python dev/code-generation/main.py {{ .CLI_ARGS }}
       - task: py:format
 
   dev:services:

dev/code-generation/gen_ts_locales.py

@@ -23,19 +23,22 @@ class LocaleData:
 
 
 LOCALE_DATA: dict[str, LocaleData] = {
-    "en-US": LocaleData(name="American English"),
-    "en-GB": LocaleData(name="British English"),
     "af-ZA": LocaleData(name="Afrikaans (Afrikaans)"),
     "ar-SA": LocaleData(name="العربية (Arabic)", dir="rtl"),
+    "bg-BG": LocaleData(name="Български (Bulgarian)"),
     "ca-ES": LocaleData(name="Català (Catalan)"),
     "cs-CZ": LocaleData(name="Čeština (Czech)"),
     "da-DK": LocaleData(name="Dansk (Danish)"),
     "de-DE": LocaleData(name="Deutsch (German)"),
     "el-GR": LocaleData(name="Ελληνικά (Greek)"),
+    "en-GB": LocaleData(name="British English"),
+    "en-US": LocaleData(name="American English"),
     "es-ES": LocaleData(name="Español (Spanish)"),
+    "et-EE": LocaleData(name="Eesti (Estonian)"),
     "fi-FI": LocaleData(name="Suomi (Finnish)"),
-    "fr-FR": LocaleData(name="Français (French)"),
     "fr-BE": LocaleData(name="Belge (Belgian)"),
+    "fr-CA": LocaleData(name="Français canadien (Canadian French)"),
+    "fr-FR": LocaleData(name="Français (French)"),
     "gl-ES": LocaleData(name="Galego (Galician)"),
     "he-IL": LocaleData(name="עברית (Hebrew)", dir="rtl"),
     "hr-HR": LocaleData(name="Hrvatski (Croatian)"),
@@ -53,6 +56,7 @@ LOCALE_DATA: dict[str, LocaleData] = {
     "pt-PT": LocaleData(name="Português (Portuguese)"),
     "ro-RO": LocaleData(name="Română (Romanian)"),
     "ru-RU": LocaleData(name="Pусский (Russian)"),
+    "sk-SK": LocaleData(name="Slovenčina (Slovak)"),
     "sl-SI": LocaleData(name="Slovenščina (Slovenian)"),
     "sr-SP": LocaleData(name="српски (Serbian)"),
     "sv-SE": LocaleData(name="Svenska (Swedish)"),
@@ -93,8 +97,8 @@ class CrowdinApi:
     project_id = "451976"
     api_key = API_KEY
 
-    def __init__(self, api_key: str):
-        api_key = api_key
+    def __init__(self, api_key: str | None):
+        self.api_key = api_key or API_KEY
 
     @property
     def headers(self) -> dict:
@@ -196,7 +200,7 @@ def inject_registration_validation_values():
 
 
 def generate_locales_ts_file():
-    api = CrowdinApi("")
+    api = CrowdinApi(None)
     models = api.get_languages()
     tmpl = Template(LOCALE_TEMPLATE)
     rendered = tmpl.render(locales=models)

dev/code-generation/main.py

@@ -1,3 +1,4 @@
+import argparse
 from pathlib import Path
 
 import gen_py_pytest_data_paths
@@ -11,15 +12,39 @@ CWD = Path(__file__).parent
 
 
 def main():
-    items = [
-        (gen_py_schema_exports.main, "schema exports"),
-        (gen_ts_types.main, "frontend types"),
-        (gen_ts_locales.main, "locales"),
-        (gen_py_pytest_data_paths.main, "test data paths"),
-        (gen_py_pytest_routes.main, "pytest routes"),
-    ]
+    parser = argparse.ArgumentParser(description="Run code generators")
+    parser.add_argument(
+        "generators",
+        nargs="*",
+        help="Specific generators to run (schema, types, locales, data-paths, routes). If none specified, all will run.",  # noqa: E501 - long line
+    )
+    args = parser.parse_args()
 
-    for func, name in items:
+    # Define all available generators
+    all_generators = {
+        "schema": (gen_py_schema_exports.main, "schema exports"),
+        "types": (gen_ts_types.main, "frontend types"),
+        "locales": (gen_ts_locales.main, "locales"),
+        "data-paths": (gen_py_pytest_data_paths.main, "test data paths"),
+        "routes": (gen_py_pytest_routes.main, "pytest routes"),
+    }
+
+    # Determine which generators to run
+    if args.generators:
+        # Validate requested generators
+        invalid_generators = [g for g in args.generators if g not in all_generators]
+        if invalid_generators:
+            log.error(f"Invalid generator(s): {', '.join(invalid_generators)}")
+            log.info(f"Available generators: {', '.join(all_generators.keys())}")
+            return
+
+        generators_to_run = [(all_generators[g][0], all_generators[g][1]) for g in args.generators]
+    else:
+        # Run all generators (default behavior)
+        generators_to_run = list(all_generators.values())
+
+    # Run the selected generators
+    for func, name in generators_to_run:
         log.info(f"Generating {name}...")
         func()
 

dev/code-generation/utils/template.py

@@ -1,5 +1,4 @@
 import logging
-import re
 import subprocess
 from dataclasses import dataclass
 from pathlib import Path
@@ -35,7 +34,7 @@ class CodeSlicer:
     start: int
     end: int
 
-    indentation: str
+    indentation: str | None
     text: list[str]
 
     _next_line = None
@@ -47,15 +46,24 @@ class CodeSlicer:
 
     def push_line(self, string: str) -> None:
         self._next_line = self._next_line or self.start + 1
-        self.text.insert(self._next_line, self.indentation + string + "\n")
+        self.text.insert(self._next_line, (self.indentation or "") + string + "\n")
         self._next_line += 1
 
 
-def get_indentation_of_string(line: str, comment_char: str = "//|#") -> str:
-    return re.sub(rf"{comment_char}.*", "", line).removesuffix("\n")
+def get_indentation_of_string(line: str) -> str:
+    # Extract everything before the comment
+    if "//" in line:
+        indentation = line.split("//")[0]
+    elif "#" in line:
+        indentation = line.split("#")[0]
+    else:
+        indentation = line
+
+    # Keep only the whitespace, remove any non-whitespace characters
+    return "".join(c for c in indentation if c.isspace())
 
 
-def find_start_end(file_text: list[str], gen_id: str) -> tuple[int, int, str]:
+def find_start_end(file_text: list[str], gen_id: str) -> tuple[int, int, str | None]:
     start = None
     end = None
     indentation = None