From d047f63fa78101d7c81d7242d55c5d4805f58db7 Mon Sep 17 00:00:00 2001 From: carlospolop Date: Tue, 30 Sep 2025 01:19:57 +0200 Subject: [PATCH] upload searchindex to releases --- .github/workflows/auto_merge_approved_prs.yml | 32 +--- .github/workflows/build_master.yml | 63 +++----- .github/workflows/translate_all.yml | 60 +++----- .gitignore | 1 + resolve_searchindex_conflicts.sh | 139 ------------------ theme/ht_searcher.js | 52 +++++-- 6 files changed, 80 insertions(+), 267 deletions(-) delete mode 100644 resolve_searchindex_conflicts.sh diff --git a/.github/workflows/auto_merge_approved_prs.yml b/.github/workflows/auto_merge_approved_prs.yml index 3a567a9fb..21d92a19f 100644 --- a/.github/workflows/auto_merge_approved_prs.yml +++ b/.github/workflows/auto_merge_approved_prs.yml @@ -26,9 +26,6 @@ jobs: git config --global user.email "action@github.com" git config --global user.name "GitHub Action" - - name: Make conflict resolution script executable - run: chmod +x ./resolve_searchindex_conflicts.sh - - name: Check for running workflows id: check_workflows run: | @@ -138,34 +135,7 @@ jobs: echo "Failed to merge PR #$pr_number: $pr_title" fi elif [ "$pr_mergeable" = "CONFLICTED" ] || [ "$pr_mergeable" = "CONFLICTING" ]; then - echo "PR #$pr_number has conflicts. Attempting to resolve searchindex.js conflicts..." - - # Try to resolve conflicts by updating the branch - export GITHUB_REPOSITORY="$GITHUB_REPOSITORY" - export GH_TOKEN="${{ secrets.PAT_TOKEN }}" - if ./resolve_searchindex_conflicts.sh "$pr_number" "$head_branch" "$base_branch"; then - echo "Successfully resolved searchindex.js conflicts for PR #$pr_number" - - # Wait for GitHub to update the mergeable status after conflict resolution - echo "Waiting for GitHub to process the conflict resolution..." - sleep 15 - - # Check if it's now mergeable - pr_mergeable_after=$(gh pr view "$pr_number" --json mergeable --jq '.mergeable' --repo "$GITHUB_REPOSITORY") - if [ "$pr_mergeable_after" = "MERGEABLE" ]; then - if gh pr merge "$pr_number" --merge --delete-branch --repo "$GITHUB_REPOSITORY"; then - echo "Successfully merged PR #$pr_number after resolving conflicts: $pr_title" - current_count=$(cat /tmp/merged_count) - echo $((current_count + 1)) > /tmp/merged_count - else - echo "Failed to merge PR #$pr_number after conflict resolution: $pr_title" - fi - else - echo "PR #$pr_number still not mergeable after conflict resolution (status: $pr_mergeable_after)" - fi - else - echo "Failed to resolve conflicts for PR #$pr_number" - fi + echo "PR #$pr_number has conflicts. Skipping auto-merge so it can be resolved manually." else echo "PR #$pr_number is not mergeable (status: $pr_mergeable)" fi diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index 99f80c75e..60843b805 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -35,60 +35,33 @@ jobs: - name: Build mdBook run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1) - - name: Update searchindex in repo (purge history, keep current on HEAD) + - name: Publish search index release asset shell: bash + env: + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} run: | set -euo pipefail - ls -la - ls -la book + ASSET="book/searchindex.js" + TAG="searchindex-en" + TITLE="Search Index (en)" - git config --global --add safe.directory /__w/hacktricks/hacktricks - git config --global user.email "build@example.com" - git config --global user.name "Build master" - git config pull.rebase false - - # Ensure we're on the target branch and up to date - git fetch origin - git reset --hard origin/master - - # Choose the file to keep at HEAD: - # 1) Prefer freshly built version from book/ - # 2) Fallback to the file currently at HEAD (if it exists) - HAS_FILE=0 - if [ -f "book/searchindex.js" ]; then - cp "book/searchindex.js" /tmp/sidx.js - HAS_FILE=1 - elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then - git show "HEAD:searchindex.js" > /tmp/sidx.js - HAS_FILE=1 + if [ ! -f "$ASSET" ]; then + echo "Expected $ASSET to exist after build" >&2 + exit 1 fi - # Skip if there's nothing to purge AND nothing to keep - if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 HEAD -- "searchindex.js" >/dev/null 2>&1; then - # Fail early if working tree is dirty (avoid confusing rewrites) - git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; } + TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}" + if [ -z "$TOKEN" ]; then + echo "No token available for GitHub CLI" >&2 + exit 1 + fi + export GH_TOKEN="$TOKEN" - # Install git-filter-repo and ensure it's on PATH - python -m pip install --quiet --user git-filter-repo - export PATH="$HOME/.local/bin:$PATH" - - # Rewrite ONLY the current branch, dropping all historical blobs of searchindex.js - git filter-repo --force --path "searchindex.js" --invert-paths --refs "$(git symbolic-ref -q HEAD)" - - # Re-add the current version on top of rewritten history (keep it in HEAD) - if [ "$HAS_FILE" = "1" ]; then - mv /tmp/sidx.js "searchindex.js" - git add "searchindex.js" - git commit -m "Update searchindex (purged history; keep current)" - else - echo "No current searchindex.js to re-add after purge." - fi - - # Safer force push (only updates if remote hasn't advanced) - git push --force-with-lease + if ! gh release view "$TAG" >/dev/null 2>&1; then + gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY" else - echo "Nothing to purge; skipping." + gh release upload "$TAG" "$ASSET" --clobber --repo "$GITHUB_REPOSITORY" fi diff --git a/.github/workflows/translate_all.yml b/.github/workflows/translate_all.yml index f472a42f9..3e0addf29 100644 --- a/.github/workflows/translate_all.yml +++ b/.github/workflows/translate_all.yml @@ -123,57 +123,35 @@ jobs: git pull MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1) - - name: Update searchindex.js in repo (purge history, keep current on HEAD) + - name: Publish search index release asset shell: bash + env: + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} run: | set -euo pipefail - # Be explicit about workspace trust (avoids "dubious ownership") - git config --global --add safe.directory "$GITHUB_WORKSPACE" + ASSET="book/searchindex.js" + TAG="searchindex-${BRANCH}" + TITLE="Search Index (${BRANCH})" - git checkout "$BRANCH" - git fetch origin "$BRANCH" --quiet - git pull --ff-only - - # Choose the file to keep at HEAD: - # 1) Prefer freshly built version from book/ - # 2) Fallback to the file currently at HEAD (if it exists) - HAS_FILE=0 - if [ -f "book/searchindex.js" ]; then - cp "book/searchindex.js" /tmp/sidx.js - HAS_FILE=1 - elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then - git show "HEAD:searchindex.js" > /tmp/sidx.js - HAS_FILE=1 + if [ ! -f "$ASSET" ]; then + echo "Expected $ASSET to exist after build" >&2 + exit 1 fi - # Skip if there's nothing to purge AND nothing to keep - if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 "$BRANCH" -- "searchindex.js" >/dev/null 2>&1; then - # **Fail early if working tree is dirty** (prevents confusing filter results) - git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; } + TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}" + if [ -z "$TOKEN" ]; then + echo "No token available for GitHub CLI" >&2 + exit 1 + fi + export GH_TOKEN="$TOKEN" - # Make sure git-filter-repo is callable via `git filter-repo` - python -m pip install --quiet --user git-filter-repo - export PATH="$HOME/.local/bin:$PATH" - - # Rewrite ONLY this branch, dropping all historical blobs of searchindex.js - git filter-repo --force --path "searchindex.js" --invert-paths --refs "refs/heads/$BRANCH" - - # Re-add the current version on top of rewritten history (keep it in HEAD) - if [ "$HAS_FILE" = "1" ]; then - mv /tmp/sidx.js "searchindex.js" - git add "searchindex.js" - git commit -m "Update searchindex (purged history; keep current)" - else - echo "No current searchindex.js to re-add after purge." - fi - - # **Safer force push** (prevents clobbering unexpected remote updates) - git push --force-with-lease origin "$BRANCH" + if ! gh release view "$TAG" >/dev/null 2>&1; then + gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY" else - echo "Nothing to purge; skipping." + gh release upload "$TAG" "$ASSET" --clobber --repo "$GITHUB_REPOSITORY" fi - + # Login in AWs - name: Configure AWS credentials using OIDC uses: aws-actions/configure-aws-credentials@v3 diff --git a/.gitignore b/.gitignore index 77edd91a0..ca8613877 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ book book/* hacktricks-preprocessor.log hacktricks-preprocessor-error.log +searchindex.js diff --git a/resolve_searchindex_conflicts.sh b/resolve_searchindex_conflicts.sh deleted file mode 100644 index e096f8dbe..000000000 --- a/resolve_searchindex_conflicts.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash - -# Script to resolve searchindex.js conflicts by accepting master branch version -# This script is designed to handle merge conflicts that occur when PRs become -# desynchronized due to the auto-generated searchindex.js file. -# -# The searchindex.js file is automatically generated by the build process and -# frequently causes conflicts when multiple PRs are waiting to be merged. -# This script automatically resolves those conflicts by accepting the master -# branch version of the file. -# -# Usage: resolve_searchindex_conflicts.sh - -set -euo pipefail - -# Validate arguments -if [ $# -ne 3 ]; then - echo "Usage: $0 " - exit 1 -fi - -PR_NUMBER="$1" -HEAD_BRANCH="$2" -BASE_BRANCH="$3" - -# Validate required environment variables -if [ -z "${GITHUB_REPOSITORY:-}" ]; then - echo "Error: GITHUB_REPOSITORY environment variable is required" - exit 1 -fi - -if [ -z "${GH_TOKEN:-}" ]; then - echo "Error: GH_TOKEN environment variable is required" - exit 1 -fi - -echo "Resolving conflicts for PR #$PR_NUMBER (branch: $HEAD_BRANCH -> $BASE_BRANCH)" - -# Get current directory for safety -ORIGINAL_DIR=$(pwd) - -# Create a temporary directory for the operation -TEMP_DIR=$(mktemp -d) -echo "Working in temporary directory: $TEMP_DIR" - -cleanup() { - echo "Cleaning up..." - cd "$ORIGINAL_DIR" - rm -rf "$TEMP_DIR" -} -trap cleanup EXIT - -# Clone the repository to the temp directory -echo "Cloning repository..." -cd "$TEMP_DIR" -gh repo clone "$GITHUB_REPOSITORY" . -- --branch "$HEAD_BRANCH" - -# Configure git -git config user.email "action@github.com" -git config user.name "GitHub Action" - -# Fetch all branches -git fetch origin - -# Make sure we're on the correct branch -git checkout "$HEAD_BRANCH" - -# Try to merge the base branch -echo "Attempting to merge $BASE_BRANCH into $HEAD_BRANCH..." -if git merge "origin/$BASE_BRANCH" --no-edit; then - echo "No conflicts found, merge successful" - - # Push the updated branch - echo "Pushing merged branch..." - git push origin "$HEAD_BRANCH" - exit 0 -fi - -# Check what files have conflicts -echo "Checking for conflicts..." -conflicted_files=$(git diff --name-only --diff-filter=U) -echo "Conflicted files: $conflicted_files" - -# Check if searchindex.js is the only conflict or if conflicts are only in acceptable files -acceptable_conflicts=true -searchindex_conflict=false - -for file in $conflicted_files; do - case "$file" in - "searchindex.js") - searchindex_conflict=true - echo "Found searchindex.js conflict (acceptable)" - ;; - *) - echo "Found unacceptable conflict in: $file" - acceptable_conflicts=false - ;; - esac -done - -if [ "$acceptable_conflicts" = false ]; then - echo "Cannot auto-resolve: conflicts found in files other than searchindex.js" - git merge --abort - exit 1 -fi - -if [ "$searchindex_conflict" = false ]; then - echo "No searchindex.js conflicts found, but merge failed for unknown reason" - git merge --abort - exit 1 -fi - -echo "Resolving searchindex.js conflict by accepting $BASE_BRANCH version..." - -# Accept the base branch version of searchindex.js (--theirs refers to the branch being merged in) -git checkout --theirs searchindex.js -git add searchindex.js - -# Check if there are any other staged changes from the merge -staged_files=$(git diff --cached --name-only || true) -echo "Staged files after resolution: $staged_files" - -# Complete the merge -if git commit --no-edit; then - echo "Successfully resolved merge conflicts" - - # Push the updated branch - echo "Pushing resolved branch..." - if git push origin "$HEAD_BRANCH"; then - echo "Successfully pushed resolved branch" - exit 0 - else - echo "Failed to push resolved branch" - exit 1 - fi -else - echo "Failed to commit merge resolution" - exit 1 -fi diff --git a/theme/ht_searcher.js b/theme/ht_searcher.js index f47ba384a..77f10f607 100644 --- a/theme/ht_searcher.js +++ b/theme/ht_searcher.js @@ -24,13 +24,15 @@ /* 2 — load a single index (remote → local) */ async function loadIndex(remote, local, isCloud=false){ let rawLoaded = false; - try { - const r = await fetch(remote,{mode:'cors'}); - if (!r.ok) throw new Error('HTTP '+r.status); - importScripts(URL.createObjectURL(new Blob([await r.text()],{type:'application/javascript'}))); - rawLoaded = true; - } catch(e){ console.warn('remote',remote,'failed →',e); } - if(!rawLoaded){ + if(remote){ + try { + const r = await fetch(remote,{mode:'cors'}); + if (!r.ok) throw new Error('HTTP '+r.status); + importScripts(URL.createObjectURL(new Blob([await r.text()],{type:'application/javascript'}))); + rawLoaded = true; + } catch(e){ console.warn('remote',remote,'failed →',e); } + } + if(!rawLoaded && local){ try { importScripts(abs(local)); rawLoaded = true; } catch(e){ console.error('local',local,'failed →',e); } } @@ -40,13 +42,41 @@ return data; } + async function loadWithFallback(remotes, local, isCloud=false){ + if(remotes.length){ + const [primary, ...secondary] = remotes; + const primaryData = await loadIndex(primary, null, isCloud); + if(primaryData) return primaryData; + + if(local){ + const localData = await loadIndex(null, local, isCloud); + if(localData) return localData; + } + + for (const remote of secondary){ + const data = await loadIndex(remote, null, isCloud); + if(data) return data; + } + } + + return local ? loadIndex(null, local, isCloud) : null; + } + (async () => { - const MAIN_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks/refs/heads/master/searchindex.js'; - const CLOUD_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/refs/heads/master/searchindex.js'; + const htmlLang = (document.documentElement.lang || 'en').toLowerCase(); + const lang = htmlLang.split('-')[0]; + const mainReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks/releases/download'; + const cloudReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks-cloud/releases/download'; + + const mainTags = Array.from(new Set([`searchindex-${lang}`, 'searchindex-en', 'searchindex-master'])); + const cloudTags = Array.from(new Set([`searchindex-${lang}`, 'searchindex-en', 'searchindex-master'])); + + const MAIN_REMOTE_SOURCES = mainTags.map(tag => `${mainReleaseBase}/${tag}/searchindex.js`); + const CLOUD_REMOTE_SOURCES = cloudTags.map(tag => `${cloudReleaseBase}/${tag}/searchindex.js`); const indices = []; - const main = await loadIndex(MAIN_RAW , '/searchindex.js', false); if(main) indices.push(main); - const cloud= await loadIndex(CLOUD_RAW, '/searchindex-cloud.js', true ); if(cloud) indices.push(cloud); + const main = await loadWithFallback(MAIN_REMOTE_SOURCES , '/searchindex.js', false); if(main) indices.push(main); + const cloud= await loadWithFallback(CLOUD_REMOTE_SOURCES, '/searchindex-cloud.js', true ); if(cloud) indices.push(cloud); if(!indices.length){ postMessage({ready:false, error:'no-index'}); return; }