hacktricks/.github/workflows/translate_all.yml

name: Translate All

on:
  push:
    branches:
      - master
    paths-ignore:
      - 'scripts/**'
      - '.gitignore'
      - '.github/**'
      - Dockerfile
  workflow_dispatch:

permissions:
  packages: write
  id-token: write
  contents: write

jobs:
  translate:
    name: Translate → ${{ matrix.name }} (${{ matrix.branch }})
    runs-on: ubuntu-latest

    # Run N languages in parallel (tune max-parallel if needed)
    strategy:
      fail-fast: false
      # max-parallel: 3 #Nothing to run all in parallel
      matrix:
        include:
          - { name: "Afrikaans", language: "Afrikaans", branch: "af" }
          - { name: "German", language: "German", branch: "de" }
          - { name: "Greek", language: "Greek", branch: "el" }
          - { name: "Spanish", language: "Spanish", branch: "es" }
          - { name: "French",  language: "French",  branch: "fr" }
          - { name: "Hindi",  language: "Hindi",  branch: "hi" }
          - { name: "Italian", language: "Italian", branch: "it" }
          - { name: "Japanese", language: "Japanese", branch: "ja" }
          - { name: "Korean", language: "Korean", branch: "ko" }
          - { name: "Polish", language: "Polish", branch: "pl" }
          - { name: "Portuguese", language: "Portuguese", branch: "pt" }
          - { name: "Serbian", language: "Serbian", branch: "sr" }
          - { name: "Swahili", language: "Swahili", branch: "sw" }
          - { name: "Turkish", language: "Turkish", branch: "tr" }
          - { name: "Ukrainian", language: "Ukrainian", branch: "uk" }
          - { name: "Chinese", language: "Chinese", branch: "zh" }

    # Ensure only one job per branch runs at a time (even across workflow runs)
    concurrency:
      group: translate-cloud-${{ matrix.branch }}
      cancel-in-progress: false

    container:
      image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest

    env:
      LANGUAGE: ${{ matrix.language }}
      BRANCH:   ${{ matrix.branch }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      
      - name: Update and download scripts
        run: |
          sudo apt-get update
          sudo apt-get install wget -y
          mkdir scripts
          cd scripts
          wget -O get_and_save_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/get_and_save_refs.py
          wget -O compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py
          wget -O translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py
          cd ..

      - name: Run get_and_save_refs.py
        run: |
          python scripts/get_and_save_refs.py

      - name: Download language branch & update refs
        run: |
          git config --global --add safe.directory /__w/hacktricks/hacktricks
          git config --global user.name 'Translator'
          git config --global user.email 'github-actions@github.com'
          git config pull.rebase false
          git checkout $BRANCH
          git pull
          python scripts/compare_and_fix_refs.py --files-unmatched-paths /tmp/file_paths.txt
          git add .
          git commit -m "Fix unmatched refs" || echo "No changes to commit"
          git push || echo "No changes to push"
      
      - name: Run translation script on changed files
        run: |
          git checkout master
          export OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}
          git diff --name-only HEAD~1 | grep -v "SUMMARY.md" | while read -r file; do
            if echo "$file" | grep -qE '\.md$'; then
              echo -n ",$file" >> /tmp/file_paths.txt
            fi
          done

          echo "Files to translate:"
          cat /tmp/file_paths.txt
          echo ""
          echo ""
          touch /tmp/file_paths.txt

          if [ -s /tmp/file_paths.txt ]; then
            python scripts/translator.py \
              --language "$LANGUAGE" \
              --branch "$BRANCH" \
              --api-key "$OPENAI_API_KEY" \
              -f "$(cat /tmp/file_paths.txt)" \
              -t 3
          else
            echo "No markdown files changed, skipping translation."
          fi

      - name: Build mdBook
        run: |
          git checkout "$BRANCH"
          git pull
          MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
      
      - name: Publish search index release asset
        shell: bash
        env:
          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
        run: |
          set -euo pipefail

          ASSET="book/searchindex.js"
          TAG="searchindex-${BRANCH}"
          TITLE="Search Index (${BRANCH})"

          if [ ! -f "$ASSET" ]; then
            echo "Expected $ASSET to exist after build" >&2
            exit 1
          fi

          TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
          if [ -z "$TOKEN" ]; then
            echo "No token available for GitHub CLI" >&2
            exit 1
          fi
          export GH_TOKEN="$TOKEN"

          if ! gh release view "$TAG" >/dev/null 2>&1; then
            gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY"
          else
            gh release upload "$TAG" "$ASSET" --clobber --repo "$GITHUB_REPOSITORY"
          fi

      # Login in AWs
      - name: Configure AWS credentials using OIDC
        uses: aws-actions/configure-aws-credentials@v3
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1
      
      # Sync the build to S3
      - name: Sync to S3
        run: |
          echo "Current branch:"
          git rev-parse --abbrev-ref HEAD
          echo "Syncing $BRANCH to S3"
          aws s3 sync ./book s3://hacktricks-wiki/$BRANCH --delete
          echo "Sync completed"
          echo "Cat 3 files from the book"
          find . -type f -name 'index.html' -print | head -n 3 | xargs -r cat