upload searchindex to releases

.github/workflows/auto_merge_approved_prs.yml

@@ -26,9 +26,6 @@ jobs:
           git config --global user.email "action@github.com"
           git config --global user.name "GitHub Action"
 
-      - name: Make conflict resolution script executable
-        run: chmod +x ./resolve_searchindex_conflicts.sh
-
       - name: Check for running workflows
         id: check_workflows
         run: |
@@ -138,34 +135,7 @@ jobs:
                   echo "Failed to merge PR #$pr_number: $pr_title"
                 fi
               elif [ "$pr_mergeable" = "CONFLICTED" ] || [ "$pr_mergeable" = "CONFLICTING" ]; then
-                echo "PR #$pr_number has conflicts. Attempting to resolve searchindex.js conflicts..."
+                echo "PR #$pr_number has conflicts. Skipping auto-merge so it can be resolved manually."
-                
-                # Try to resolve conflicts by updating the branch
-                export GITHUB_REPOSITORY="$GITHUB_REPOSITORY"
-                export GH_TOKEN="${{ secrets.PAT_TOKEN }}"
-                if ./resolve_searchindex_conflicts.sh "$pr_number" "$head_branch" "$base_branch"; then
-                  echo "Successfully resolved searchindex.js conflicts for PR #$pr_number"
-                  
-                  # Wait for GitHub to update the mergeable status after conflict resolution
-                  echo "Waiting for GitHub to process the conflict resolution..."
-                  sleep 15
-                  
-                  # Check if it's now mergeable
-                  pr_mergeable_after=$(gh pr view "$pr_number" --json mergeable --jq '.mergeable' --repo "$GITHUB_REPOSITORY")
-                  if [ "$pr_mergeable_after" = "MERGEABLE" ]; then
-                    if gh pr merge "$pr_number" --merge --delete-branch --repo "$GITHUB_REPOSITORY"; then
-                      echo "Successfully merged PR #$pr_number after resolving conflicts: $pr_title"
-                      current_count=$(cat /tmp/merged_count)
-                      echo $((current_count + 1)) > /tmp/merged_count
-                    else
-                      echo "Failed to merge PR #$pr_number after conflict resolution: $pr_title"
-                    fi
-                  else
-                    echo "PR #$pr_number still not mergeable after conflict resolution (status: $pr_mergeable_after)"
-                  fi
-                else
-                  echo "Failed to resolve conflicts for PR #$pr_number"
-                fi
               else
                 echo "PR #$pr_number is not mergeable (status: $pr_mergeable)"
               fi

.github/workflows/build_master.yml

@@ -35,60 +35,33 @@ jobs:
       - name: Build mdBook
         run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
       
-      - name: Update searchindex in repo (purge history, keep current on HEAD)
+      - name: Publish search index release asset
         shell: bash
+        env:
+          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
         run: |
           set -euo pipefail
 
-          ls -la
+          ASSET="book/searchindex.js"
-          ls -la book
+          TAG="searchindex-en"
+          TITLE="Search Index (en)"
 
-          git config --global --add safe.directory /__w/hacktricks/hacktricks
+          if [ ! -f "$ASSET" ]; then
-          git config --global user.email "build@example.com"
+            echo "Expected $ASSET to exist after build" >&2
-          git config --global user.name "Build master"
+            exit 1
-          git config pull.rebase false
-
-          # Ensure we're on the target branch and up to date
-          git fetch origin
-          git reset --hard origin/master
-
-          # Choose the file to keep at HEAD:
-          # 1) Prefer freshly built version from book/
-          # 2) Fallback to the file currently at HEAD (if it exists)
-          HAS_FILE=0
-          if [ -f "book/searchindex.js" ]; then
-            cp "book/searchindex.js" /tmp/sidx.js
-            HAS_FILE=1
-          elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
-            git show "HEAD:searchindex.js" > /tmp/sidx.js
-            HAS_FILE=1
           fi
 
-          # Skip if there's nothing to purge AND nothing to keep
+          TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
-          if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 HEAD -- "searchindex.js" >/dev/null 2>&1; then
+          if [ -z "$TOKEN" ]; then
-            # Fail early if working tree is dirty (avoid confusing rewrites)
+            echo "No token available for GitHub CLI" >&2
-            git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
+            exit 1
-
-            # Install git-filter-repo and ensure it's on PATH
-            python -m pip install --quiet --user git-filter-repo
-            export PATH="$HOME/.local/bin:$PATH"
-
-            # Rewrite ONLY the current branch, dropping all historical blobs of searchindex.js
-            git filter-repo --force --path "searchindex.js" --invert-paths --refs "$(git symbolic-ref -q HEAD)"
-
-            # Re-add the current version on top of rewritten history (keep it in HEAD)
-            if [ "$HAS_FILE" = "1" ]; then
-              mv /tmp/sidx.js "searchindex.js"
-              git add "searchindex.js"
-              git commit -m "Update searchindex (purged history; keep current)"
-            else
-              echo "No current searchindex.js to re-add after purge."
           fi
+          export GH_TOKEN="$TOKEN"
 
-            # Safer force push (only updates if remote hasn't advanced)
+          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            git push --force-with-lease
+            gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY"
           else
-            echo "Nothing to purge; skipping."
+            gh release upload "$TAG" "$ASSET" --clobber --repo "$GITHUB_REPOSITORY"
           fi
 
 

.github/workflows/translate_all.yml

@@ -123,55 +123,33 @@ jobs:
           git pull
           MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
       
-      - name: Update searchindex.js in repo (purge history, keep current on HEAD)
+      - name: Publish search index release asset
         shell: bash
+        env:
+          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
         run: |
           set -euo pipefail
 
-          # Be explicit about workspace trust (avoids "dubious ownership")
+          ASSET="book/searchindex.js"
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          TAG="searchindex-${BRANCH}"
+          TITLE="Search Index (${BRANCH})"
 
-          git checkout "$BRANCH"
+          if [ ! -f "$ASSET" ]; then
-          git fetch origin "$BRANCH" --quiet
+            echo "Expected $ASSET to exist after build" >&2
-          git pull --ff-only
+            exit 1
-
-          # Choose the file to keep at HEAD:
-          # 1) Prefer freshly built version from book/
-          # 2) Fallback to the file currently at HEAD (if it exists)
-          HAS_FILE=0
-          if [ -f "book/searchindex.js" ]; then
-            cp "book/searchindex.js" /tmp/sidx.js
-            HAS_FILE=1
-          elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
-            git show "HEAD:searchindex.js" > /tmp/sidx.js
-            HAS_FILE=1
           fi
 
-          # Skip if there's nothing to purge AND nothing to keep
+          TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
-          if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 "$BRANCH" -- "searchindex.js" >/dev/null 2>&1; then
+          if [ -z "$TOKEN" ]; then
-            # **Fail early if working tree is dirty** (prevents confusing filter results)
+            echo "No token available for GitHub CLI" >&2
-            git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
+            exit 1
-
-            # Make sure git-filter-repo is callable via `git filter-repo`
-            python -m pip install --quiet --user git-filter-repo
-            export PATH="$HOME/.local/bin:$PATH"
-
-            # Rewrite ONLY this branch, dropping all historical blobs of searchindex.js
-            git filter-repo --force --path "searchindex.js" --invert-paths --refs "refs/heads/$BRANCH"
-
-            # Re-add the current version on top of rewritten history (keep it in HEAD)
-            if [ "$HAS_FILE" = "1" ]; then
-              mv /tmp/sidx.js "searchindex.js"
-              git add "searchindex.js"
-              git commit -m "Update searchindex (purged history; keep current)"
-            else
-              echo "No current searchindex.js to re-add after purge."
           fi
+          export GH_TOKEN="$TOKEN"
 
-            # **Safer force push** (prevents clobbering unexpected remote updates)
+          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            git push --force-with-lease origin "$BRANCH"
+            gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY"
           else
-            echo "Nothing to purge; skipping."
+            gh release upload "$TAG" "$ASSET" --clobber --repo "$GITHUB_REPOSITORY"
           fi
 
       # Login in AWs

.gitignore

@@ -11,3 +11,4 @@ book
 book/*
 hacktricks-preprocessor.log
 hacktricks-preprocessor-error.log
+searchindex.js

resolve_searchindex_conflicts.sh

@@ -1,139 +0,0 @@
-#!/bin/bash
-
-# Script to resolve searchindex.js conflicts by accepting master branch version
-# This script is designed to handle merge conflicts that occur when PRs become
-# desynchronized due to the auto-generated searchindex.js file.
-# 
-# The searchindex.js file is automatically generated by the build process and
-# frequently causes conflicts when multiple PRs are waiting to be merged.
-# This script automatically resolves those conflicts by accepting the master
-# branch version of the file.
-#
-# Usage: resolve_searchindex_conflicts.sh <pr_number> <head_branch> <base_branch>
-
-set -euo pipefail
-
-# Validate arguments
-if [ $# -ne 3 ]; then
-    echo "Usage: $0 <pr_number> <head_branch> <base_branch>"
-    exit 1
-fi
-
-PR_NUMBER="$1"
-HEAD_BRANCH="$2"
-BASE_BRANCH="$3"
-
-# Validate required environment variables
-if [ -z "${GITHUB_REPOSITORY:-}" ]; then
-    echo "Error: GITHUB_REPOSITORY environment variable is required"
-    exit 1
-fi
-
-if [ -z "${GH_TOKEN:-}" ]; then
-    echo "Error: GH_TOKEN environment variable is required"
-    exit 1
-fi
-
-echo "Resolving conflicts for PR #$PR_NUMBER (branch: $HEAD_BRANCH -> $BASE_BRANCH)"
-
-# Get current directory for safety
-ORIGINAL_DIR=$(pwd)
-
-# Create a temporary directory for the operation
-TEMP_DIR=$(mktemp -d)
-echo "Working in temporary directory: $TEMP_DIR"
-
-cleanup() {
-    echo "Cleaning up..."
-    cd "$ORIGINAL_DIR"
-    rm -rf "$TEMP_DIR"
-}
-trap cleanup EXIT
-
-# Clone the repository to the temp directory
-echo "Cloning repository..."
-cd "$TEMP_DIR"
-gh repo clone "$GITHUB_REPOSITORY" . -- --branch "$HEAD_BRANCH"
-
-# Configure git
-git config user.email "action@github.com"
-git config user.name "GitHub Action"
-
-# Fetch all branches
-git fetch origin
-
-# Make sure we're on the correct branch
-git checkout "$HEAD_BRANCH"
-
-# Try to merge the base branch
-echo "Attempting to merge $BASE_BRANCH into $HEAD_BRANCH..."
-if git merge "origin/$BASE_BRANCH" --no-edit; then
-    echo "No conflicts found, merge successful"
-    
-    # Push the updated branch
-    echo "Pushing merged branch..."
-    git push origin "$HEAD_BRANCH"
-    exit 0
-fi
-
-# Check what files have conflicts
-echo "Checking for conflicts..."
-conflicted_files=$(git diff --name-only --diff-filter=U)
-echo "Conflicted files: $conflicted_files"
-
-# Check if searchindex.js is the only conflict or if conflicts are only in acceptable files
-acceptable_conflicts=true
-searchindex_conflict=false
-
-for file in $conflicted_files; do
-    case "$file" in
-        "searchindex.js")
-            searchindex_conflict=true
-            echo "Found searchindex.js conflict (acceptable)"
-            ;;
-        *)
-            echo "Found unacceptable conflict in: $file"
-            acceptable_conflicts=false
-            ;;
-    esac
-done
-
-if [ "$acceptable_conflicts" = false ]; then
-    echo "Cannot auto-resolve: conflicts found in files other than searchindex.js"
-    git merge --abort
-    exit 1
-fi
-
-if [ "$searchindex_conflict" = false ]; then
-    echo "No searchindex.js conflicts found, but merge failed for unknown reason"
-    git merge --abort
-    exit 1
-fi
-
-echo "Resolving searchindex.js conflict by accepting $BASE_BRANCH version..."
-
-# Accept the base branch version of searchindex.js (--theirs refers to the branch being merged in)
-git checkout --theirs searchindex.js
-git add searchindex.js
-
-# Check if there are any other staged changes from the merge
-staged_files=$(git diff --cached --name-only || true)
-echo "Staged files after resolution: $staged_files"
-
-# Complete the merge
-if git commit --no-edit; then
-    echo "Successfully resolved merge conflicts"
-    
-    # Push the updated branch
-    echo "Pushing resolved branch..."
-    if git push origin "$HEAD_BRANCH"; then
-        echo "Successfully pushed resolved branch"
-        exit 0
-    else
-        echo "Failed to push resolved branch"
-        exit 1
-    fi
-else
-    echo "Failed to commit merge resolution"
-    exit 1
-fi

theme/ht_searcher.js

@@ -24,13 +24,15 @@
     /* 2 — load a single index (remote → local) */
     async function loadIndex(remote, local, isCloud=false){
       let rawLoaded = false;
+      if(remote){
         try {
           const r = await fetch(remote,{mode:'cors'});
           if (!r.ok) throw new Error('HTTP '+r.status);
           importScripts(URL.createObjectURL(new Blob([await r.text()],{type:'application/javascript'})));
           rawLoaded = true;
         } catch(e){ console.warn('remote',remote,'failed →',e); }
-      if(!rawLoaded){
+      }
+      if(!rawLoaded && local){
         try { importScripts(abs(local)); rawLoaded = true; }
         catch(e){ console.error('local',local,'failed →',e); }
       }
@@ -40,13 +42,41 @@
       return data;
     }
 
+    async function loadWithFallback(remotes, local, isCloud=false){
+      if(remotes.length){
+        const [primary, ...secondary] = remotes;
+        const primaryData = await loadIndex(primary, null, isCloud);
+        if(primaryData) return primaryData;
+
+        if(local){
+          const localData = await loadIndex(null, local, isCloud);
+          if(localData) return localData;
+        }
+
+        for (const remote of secondary){
+          const data = await loadIndex(remote, null, isCloud);
+          if(data) return data;
+        }
+      }
+
+      return local ? loadIndex(null, local, isCloud) : null;
+    }
+
     (async () => {
-      const MAIN_RAW  = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks/refs/heads/master/searchindex.js';
+      const htmlLang = (document.documentElement.lang || 'en').toLowerCase();
-      const CLOUD_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/refs/heads/master/searchindex.js';
+      const lang = htmlLang.split('-')[0];
+      const mainReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks/releases/download';
+      const cloudReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks-cloud/releases/download';
+
+      const mainTags = Array.from(new Set([`searchindex-${lang}`, 'searchindex-en', 'searchindex-master']));
+      const cloudTags = Array.from(new Set([`searchindex-${lang}`, 'searchindex-en', 'searchindex-master']));
+
+      const MAIN_REMOTE_SOURCES  = mainTags.map(tag => `${mainReleaseBase}/${tag}/searchindex.js`);
+      const CLOUD_REMOTE_SOURCES = cloudTags.map(tag => `${cloudReleaseBase}/${tag}/searchindex.js`);
 
       const indices = [];
-      const main = await loadIndex(MAIN_RAW , '/searchindex.js',        false); if(main)  indices.push(main);
+      const main = await loadWithFallback(MAIN_REMOTE_SOURCES , '/searchindex.js',        false); if(main)  indices.push(main);
-      const cloud= await loadIndex(CLOUD_RAW, '/searchindex-cloud.js',  true ); if(cloud) indices.push(cloud);
+      const cloud= await loadWithFallback(CLOUD_REMOTE_SOURCES, '/searchindex-cloud.js',  true ); if(cloud) indices.push(cloud);
 
       if(!indices.length){ postMessage({ready:false, error:'no-index'}); return; }