mirror of
https://github.com/HackTricks-wiki/hacktricks.git
synced 2025-10-10 18:36:50 +00:00
upload searchindex to releases
This commit is contained in:
carlospolop
parent
52c76a2513
commit
d047f63fa7
32
.github/workflows/auto_merge_approved_prs.yml
vendored
32
.github/workflows/auto_merge_approved_prs.yml
vendored
@ -26,9 +26,6 @@ jobs:
|
||||
git config --global user.email "action@github.com"
|
||||
git config --global user.name "GitHub Action"
|
||||
|
||||
- name: Make conflict resolution script executable
|
||||
run: chmod +x ./resolve_searchindex_conflicts.sh
|
||||
|
||||
- name: Check for running workflows
|
||||
id: check_workflows
|
||||
run: |
|
||||
@ -138,34 +135,7 @@ jobs:
|
||||
echo "Failed to merge PR #$pr_number: $pr_title"
|
||||
fi
|
||||
elif [ "$pr_mergeable" = "CONFLICTED" ] || [ "$pr_mergeable" = "CONFLICTING" ]; then
|
||||
echo "PR #$pr_number has conflicts. Attempting to resolve searchindex.js conflicts..."
|
||||
|
||||
# Try to resolve conflicts by updating the branch
|
||||
export GITHUB_REPOSITORY="$GITHUB_REPOSITORY"
|
||||
export GH_TOKEN="${{ secrets.PAT_TOKEN }}"
|
||||
if ./resolve_searchindex_conflicts.sh "$pr_number" "$head_branch" "$base_branch"; then
|
||||
echo "Successfully resolved searchindex.js conflicts for PR #$pr_number"
|
||||
|
||||
# Wait for GitHub to update the mergeable status after conflict resolution
|
||||
echo "Waiting for GitHub to process the conflict resolution..."
|
||||
sleep 15
|
||||
|
||||
# Check if it's now mergeable
|
||||
pr_mergeable_after=$(gh pr view "$pr_number" --json mergeable --jq '.mergeable' --repo "$GITHUB_REPOSITORY")
|
||||
if [ "$pr_mergeable_after" = "MERGEABLE" ]; then
|
||||
if gh pr merge "$pr_number" --merge --delete-branch --repo "$GITHUB_REPOSITORY"; then
|
||||
echo "Successfully merged PR #$pr_number after resolving conflicts: $pr_title"
|
||||
current_count=$(cat /tmp/merged_count)
|
||||
echo $((current_count + 1)) > /tmp/merged_count
|
||||
else
|
||||
echo "Failed to merge PR #$pr_number after conflict resolution: $pr_title"
|
||||
fi
|
||||
else
|
||||
echo "PR #$pr_number still not mergeable after conflict resolution (status: $pr_mergeable_after)"
|
||||
fi
|
||||
else
|
||||
echo "Failed to resolve conflicts for PR #$pr_number"
|
||||
fi
|
||||
echo "PR #$pr_number has conflicts. Skipping auto-merge so it can be resolved manually."
|
||||
else
|
||||
echo "PR #$pr_number is not mergeable (status: $pr_mergeable)"
|
||||
fi
|
||||
|
||||
63
.github/workflows/build_master.yml
vendored
63
.github/workflows/build_master.yml
vendored
@ -35,60 +35,33 @@ jobs:
|
||||
- name: Build mdBook
|
||||
run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
|
||||
|
||||
- name: Update searchindex in repo (purge history, keep current on HEAD)
|
||||
- name: Publish search index release asset
|
||||
shell: bash
|
||||
env:
|
||||
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
ls -la
|
||||
ls -la book
|
||||
ASSET="book/searchindex.js"
|
||||
TAG="searchindex-en"
|
||||
TITLE="Search Index (en)"
|
||||
|
||||
git config --global --add safe.directory /__w/hacktricks/hacktricks
|
||||
git config --global user.email "build@example.com"
|
||||
git config --global user.name "Build master"
|
||||
git config pull.rebase false
|
||||
|
||||
# Ensure we're on the target branch and up to date
|
||||
git fetch origin
|
||||
git reset --hard origin/master
|
||||
|
||||
# Choose the file to keep at HEAD:
|
||||
# 1) Prefer freshly built version from book/
|
||||
# 2) Fallback to the file currently at HEAD (if it exists)
|
||||
HAS_FILE=0
|
||||
if [ -f "book/searchindex.js" ]; then
|
||||
cp "book/searchindex.js" /tmp/sidx.js
|
||||
HAS_FILE=1
|
||||
elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
|
||||
git show "HEAD:searchindex.js" > /tmp/sidx.js
|
||||
HAS_FILE=1
|
||||
if [ ! -f "$ASSET" ]; then
|
||||
echo "Expected $ASSET to exist after build" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Skip if there's nothing to purge AND nothing to keep
|
||||
if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 HEAD -- "searchindex.js" >/dev/null 2>&1; then
|
||||
# Fail early if working tree is dirty (avoid confusing rewrites)
|
||||
git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
|
||||
TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
|
||||
if [ -z "$TOKEN" ]; then
|
||||
echo "No token available for GitHub CLI" >&2
|
||||
exit 1
|
||||
fi
|
||||
export GH_TOKEN="$TOKEN"
|
||||
|
||||
# Install git-filter-repo and ensure it's on PATH
|
||||
python -m pip install --quiet --user git-filter-repo
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
|
||||
# Rewrite ONLY the current branch, dropping all historical blobs of searchindex.js
|
||||
git filter-repo --force --path "searchindex.js" --invert-paths --refs "$(git symbolic-ref -q HEAD)"
|
||||
|
||||
# Re-add the current version on top of rewritten history (keep it in HEAD)
|
||||
if [ "$HAS_FILE" = "1" ]; then
|
||||
mv /tmp/sidx.js "searchindex.js"
|
||||
git add "searchindex.js"
|
||||
git commit -m "Update searchindex (purged history; keep current)"
|
||||
else
|
||||
echo "No current searchindex.js to re-add after purge."
|
||||
fi
|
||||
|
||||
# Safer force push (only updates if remote hasn't advanced)
|
||||
git push --force-with-lease
|
||||
if ! gh release view "$TAG" >/dev/null 2>&1; then
|
||||
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY"
|
||||
else
|
||||
echo "Nothing to purge; skipping."
|
||||
gh release upload "$TAG" "$ASSET" --clobber --repo "$GITHUB_REPOSITORY"
|
||||
fi
|
||||
|
||||
|
||||
|
||||
60
.github/workflows/translate_all.yml
vendored
60
.github/workflows/translate_all.yml
vendored
@ -123,57 +123,35 @@ jobs:
|
||||
git pull
|
||||
MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
|
||||
|
||||
- name: Update searchindex.js in repo (purge history, keep current on HEAD)
|
||||
- name: Publish search index release asset
|
||||
shell: bash
|
||||
env:
|
||||
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Be explicit about workspace trust (avoids "dubious ownership")
|
||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
ASSET="book/searchindex.js"
|
||||
TAG="searchindex-${BRANCH}"
|
||||
TITLE="Search Index (${BRANCH})"
|
||||
|
||||
git checkout "$BRANCH"
|
||||
git fetch origin "$BRANCH" --quiet
|
||||
git pull --ff-only
|
||||
|
||||
# Choose the file to keep at HEAD:
|
||||
# 1) Prefer freshly built version from book/
|
||||
# 2) Fallback to the file currently at HEAD (if it exists)
|
||||
HAS_FILE=0
|
||||
if [ -f "book/searchindex.js" ]; then
|
||||
cp "book/searchindex.js" /tmp/sidx.js
|
||||
HAS_FILE=1
|
||||
elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
|
||||
git show "HEAD:searchindex.js" > /tmp/sidx.js
|
||||
HAS_FILE=1
|
||||
if [ ! -f "$ASSET" ]; then
|
||||
echo "Expected $ASSET to exist after build" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Skip if there's nothing to purge AND nothing to keep
|
||||
if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 "$BRANCH" -- "searchindex.js" >/dev/null 2>&1; then
|
||||
# **Fail early if working tree is dirty** (prevents confusing filter results)
|
||||
git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
|
||||
TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
|
||||
if [ -z "$TOKEN" ]; then
|
||||
echo "No token available for GitHub CLI" >&2
|
||||
exit 1
|
||||
fi
|
||||
export GH_TOKEN="$TOKEN"
|
||||
|
||||
# Make sure git-filter-repo is callable via `git filter-repo`
|
||||
python -m pip install --quiet --user git-filter-repo
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
|
||||
# Rewrite ONLY this branch, dropping all historical blobs of searchindex.js
|
||||
git filter-repo --force --path "searchindex.js" --invert-paths --refs "refs/heads/$BRANCH"
|
||||
|
||||
# Re-add the current version on top of rewritten history (keep it in HEAD)
|
||||
if [ "$HAS_FILE" = "1" ]; then
|
||||
mv /tmp/sidx.js "searchindex.js"
|
||||
git add "searchindex.js"
|
||||
git commit -m "Update searchindex (purged history; keep current)"
|
||||
else
|
||||
echo "No current searchindex.js to re-add after purge."
|
||||
fi
|
||||
|
||||
# **Safer force push** (prevents clobbering unexpected remote updates)
|
||||
git push --force-with-lease origin "$BRANCH"
|
||||
if ! gh release view "$TAG" >/dev/null 2>&1; then
|
||||
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY"
|
||||
else
|
||||
echo "Nothing to purge; skipping."
|
||||
gh release upload "$TAG" "$ASSET" --clobber --repo "$GITHUB_REPOSITORY"
|
||||
fi
|
||||
|
||||
|
||||
# Login in AWs
|
||||
- name: Configure AWS credentials using OIDC
|
||||
uses: aws-actions/configure-aws-credentials@v3
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -11,3 +11,4 @@ book
|
||||
book/*
|
||||
hacktricks-preprocessor.log
|
||||
hacktricks-preprocessor-error.log
|
||||
searchindex.js
|
||||
|
||||
@ -1,139 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script to resolve searchindex.js conflicts by accepting master branch version
|
||||
# This script is designed to handle merge conflicts that occur when PRs become
|
||||
# desynchronized due to the auto-generated searchindex.js file.
|
||||
#
|
||||
# The searchindex.js file is automatically generated by the build process and
|
||||
# frequently causes conflicts when multiple PRs are waiting to be merged.
|
||||
# This script automatically resolves those conflicts by accepting the master
|
||||
# branch version of the file.
|
||||
#
|
||||
# Usage: resolve_searchindex_conflicts.sh <pr_number> <head_branch> <base_branch>
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Validate arguments
|
||||
if [ $# -ne 3 ]; then
|
||||
echo "Usage: $0 <pr_number> <head_branch> <base_branch>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PR_NUMBER="$1"
|
||||
HEAD_BRANCH="$2"
|
||||
BASE_BRANCH="$3"
|
||||
|
||||
# Validate required environment variables
|
||||
if [ -z "${GITHUB_REPOSITORY:-}" ]; then
|
||||
echo "Error: GITHUB_REPOSITORY environment variable is required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "Error: GH_TOKEN environment variable is required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Resolving conflicts for PR #$PR_NUMBER (branch: $HEAD_BRANCH -> $BASE_BRANCH)"
|
||||
|
||||
# Get current directory for safety
|
||||
ORIGINAL_DIR=$(pwd)
|
||||
|
||||
# Create a temporary directory for the operation
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
echo "Working in temporary directory: $TEMP_DIR"
|
||||
|
||||
cleanup() {
|
||||
echo "Cleaning up..."
|
||||
cd "$ORIGINAL_DIR"
|
||||
rm -rf "$TEMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Clone the repository to the temp directory
|
||||
echo "Cloning repository..."
|
||||
cd "$TEMP_DIR"
|
||||
gh repo clone "$GITHUB_REPOSITORY" . -- --branch "$HEAD_BRANCH"
|
||||
|
||||
# Configure git
|
||||
git config user.email "action@github.com"
|
||||
git config user.name "GitHub Action"
|
||||
|
||||
# Fetch all branches
|
||||
git fetch origin
|
||||
|
||||
# Make sure we're on the correct branch
|
||||
git checkout "$HEAD_BRANCH"
|
||||
|
||||
# Try to merge the base branch
|
||||
echo "Attempting to merge $BASE_BRANCH into $HEAD_BRANCH..."
|
||||
if git merge "origin/$BASE_BRANCH" --no-edit; then
|
||||
echo "No conflicts found, merge successful"
|
||||
|
||||
# Push the updated branch
|
||||
echo "Pushing merged branch..."
|
||||
git push origin "$HEAD_BRANCH"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Check what files have conflicts
|
||||
echo "Checking for conflicts..."
|
||||
conflicted_files=$(git diff --name-only --diff-filter=U)
|
||||
echo "Conflicted files: $conflicted_files"
|
||||
|
||||
# Check if searchindex.js is the only conflict or if conflicts are only in acceptable files
|
||||
acceptable_conflicts=true
|
||||
searchindex_conflict=false
|
||||
|
||||
for file in $conflicted_files; do
|
||||
case "$file" in
|
||||
"searchindex.js")
|
||||
searchindex_conflict=true
|
||||
echo "Found searchindex.js conflict (acceptable)"
|
||||
;;
|
||||
*)
|
||||
echo "Found unacceptable conflict in: $file"
|
||||
acceptable_conflicts=false
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$acceptable_conflicts" = false ]; then
|
||||
echo "Cannot auto-resolve: conflicts found in files other than searchindex.js"
|
||||
git merge --abort
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$searchindex_conflict" = false ]; then
|
||||
echo "No searchindex.js conflicts found, but merge failed for unknown reason"
|
||||
git merge --abort
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Resolving searchindex.js conflict by accepting $BASE_BRANCH version..."
|
||||
|
||||
# Accept the base branch version of searchindex.js (--theirs refers to the branch being merged in)
|
||||
git checkout --theirs searchindex.js
|
||||
git add searchindex.js
|
||||
|
||||
# Check if there are any other staged changes from the merge
|
||||
staged_files=$(git diff --cached --name-only || true)
|
||||
echo "Staged files after resolution: $staged_files"
|
||||
|
||||
# Complete the merge
|
||||
if git commit --no-edit; then
|
||||
echo "Successfully resolved merge conflicts"
|
||||
|
||||
# Push the updated branch
|
||||
echo "Pushing resolved branch..."
|
||||
if git push origin "$HEAD_BRANCH"; then
|
||||
echo "Successfully pushed resolved branch"
|
||||
exit 0
|
||||
else
|
||||
echo "Failed to push resolved branch"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Failed to commit merge resolution"
|
||||
exit 1
|
||||
fi
|
||||
@ -24,13 +24,15 @@
|
||||
/* 2 — load a single index (remote → local) */
|
||||
async function loadIndex(remote, local, isCloud=false){
|
||||
let rawLoaded = false;
|
||||
try {
|
||||
const r = await fetch(remote,{mode:'cors'});
|
||||
if (!r.ok) throw new Error('HTTP '+r.status);
|
||||
importScripts(URL.createObjectURL(new Blob([await r.text()],{type:'application/javascript'})));
|
||||
rawLoaded = true;
|
||||
} catch(e){ console.warn('remote',remote,'failed →',e); }
|
||||
if(!rawLoaded){
|
||||
if(remote){
|
||||
try {
|
||||
const r = await fetch(remote,{mode:'cors'});
|
||||
if (!r.ok) throw new Error('HTTP '+r.status);
|
||||
importScripts(URL.createObjectURL(new Blob([await r.text()],{type:'application/javascript'})));
|
||||
rawLoaded = true;
|
||||
} catch(e){ console.warn('remote',remote,'failed →',e); }
|
||||
}
|
||||
if(!rawLoaded && local){
|
||||
try { importScripts(abs(local)); rawLoaded = true; }
|
||||
catch(e){ console.error('local',local,'failed →',e); }
|
||||
}
|
||||
@ -40,13 +42,41 @@
|
||||
return data;
|
||||
}
|
||||
|
||||
async function loadWithFallback(remotes, local, isCloud=false){
|
||||
if(remotes.length){
|
||||
const [primary, ...secondary] = remotes;
|
||||
const primaryData = await loadIndex(primary, null, isCloud);
|
||||
if(primaryData) return primaryData;
|
||||
|
||||
if(local){
|
||||
const localData = await loadIndex(null, local, isCloud);
|
||||
if(localData) return localData;
|
||||
}
|
||||
|
||||
for (const remote of secondary){
|
||||
const data = await loadIndex(remote, null, isCloud);
|
||||
if(data) return data;
|
||||
}
|
||||
}
|
||||
|
||||
return local ? loadIndex(null, local, isCloud) : null;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const MAIN_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks/refs/heads/master/searchindex.js';
|
||||
const CLOUD_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/refs/heads/master/searchindex.js';
|
||||
const htmlLang = (document.documentElement.lang || 'en').toLowerCase();
|
||||
const lang = htmlLang.split('-')[0];
|
||||
const mainReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks/releases/download';
|
||||
const cloudReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks-cloud/releases/download';
|
||||
|
||||
const mainTags = Array.from(new Set([`searchindex-${lang}`, 'searchindex-en', 'searchindex-master']));
|
||||
const cloudTags = Array.from(new Set([`searchindex-${lang}`, 'searchindex-en', 'searchindex-master']));
|
||||
|
||||
const MAIN_REMOTE_SOURCES = mainTags.map(tag => `${mainReleaseBase}/${tag}/searchindex.js`);
|
||||
const CLOUD_REMOTE_SOURCES = cloudTags.map(tag => `${cloudReleaseBase}/${tag}/searchindex.js`);
|
||||
|
||||
const indices = [];
|
||||
const main = await loadIndex(MAIN_RAW , '/searchindex.js', false); if(main) indices.push(main);
|
||||
const cloud= await loadIndex(CLOUD_RAW, '/searchindex-cloud.js', true ); if(cloud) indices.push(cloud);
|
||||
const main = await loadWithFallback(MAIN_REMOTE_SOURCES , '/searchindex.js', false); if(main) indices.push(main);
|
||||
const cloud= await loadWithFallback(CLOUD_REMOTE_SOURCES, '/searchindex-cloud.js', true ); if(cloud) indices.push(cloud);
|
||||
|
||||
if(!indices.length){ postMessage({ready:false, error:'no-index'}); return; }
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user