Skip to content

Commit 0683e03

Browse files
carlospolopcarlospolop
committed
f
1 parent 3b40ab6 commit 0683e03

File tree

3 files changed

+314
-206
lines changed

3 files changed

+314
-206
lines changed

‎.github/workflows/build_master.yml‎

Lines changed: 36 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -43,51 +43,60 @@ jobs:
4343
&& sudo apt update \
4444
&& sudo apt install gh -y
4545
46-
- name: Publish search index release asset
46+
- name: Push search index to hacktricks-searchindex repo
4747
shell: bash
4848
env:
4949
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
5050
run: |
5151
set -euo pipefail
5252
5353
ASSET="book/searchindex.js"
54-
TAG="searchindex-en"
55-
TITLE="Search Index (en)"
54+
TARGET_REPO="HackTricks-wiki/hacktricks-searchindex"
55+
FILENAME="searchindex-en.js"
5656
5757
if [ ! -f "$ASSET" ]; then
5858
echo "Expected $ASSET to exist after build" >&2
5959
exit 1
6060
fi
6161
62-
TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
62+
TOKEN="${PAT_TOKEN}"
6363
if [ -z "$TOKEN" ]; then
64-
echo "No token available for GitHub CLI" >&2
64+
echo "No PAT_TOKEN available" >&2
6565
exit 1
6666
fi
67-
export GH_TOKEN="$TOKEN"
6867
69-
# Delete the release if it exists
70-
echo "Checking if release $TAG exists..."
71-
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
72-
echo "Release $TAG already exists, deleting it..."
73-
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag || {
74-
echo "Failed to delete release, trying without cleanup-tag..."
75-
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" || {
76-
echo "Warning: Could not delete existing release, will try to recreate..."
77-
}
78-
}
79-
sleep 2 # Give GitHub API a moment to process the deletion
80-
else
81-
echo "Release $TAG does not exist, proceeding with creation..."
82-
fi
68+
# Clone the searchindex repo
69+
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo
70+
71+
cd /tmp/searchindex-repo
72+
git config user.name "GitHub Actions"
73+
git config user.email "github-actions@github.com"
74+
75+
# Compress the searchindex file
76+
cd "${GITHUB_WORKSPACE}"
77+
gzip -9 -k -f "$ASSET"
78+
79+
# Show compression stats
80+
ORIGINAL_SIZE=$(wc -c < "$ASSET")
81+
COMPRESSED_SIZE=$(wc -c < "${ASSET}.gz")
82+
RATIO=$(awk "BEGIN {printf \"%.1f\", ($COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100}")
83+
echo "Compression: ${ORIGINAL_SIZE} bytes -> ${COMPRESSED_SIZE} bytes (${RATIO}%)"
84+
85+
# Copy the .gz version to the searchindex repo
86+
cd /tmp/searchindex-repo
87+
cp "${GITHUB_WORKSPACE}/${ASSET}.gz" "${FILENAME}.gz"
88+
89+
# Stage the updated file
90+
git add "${FILENAME}.gz"
91+
92+
# Commit with timestamp if there are changes
93+
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
94+
git commit -m "Update searchindex files - ${TIMESTAMP}" || echo "No changes to commit"
95+
96+
# Push to master branch
97+
git push origin master
8398
84-
# Create new release (with force flag to overwrite if deletion failed)
85-
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY" || {
86-
echo "Failed to create release, trying with force flag..."
87-
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag >/dev/null 2>&1 || true
88-
sleep 2
89-
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY"
90-
}
99+
echo "Successfully pushed searchindex files"
91100
92101
93102
# Login in AWs

‎.github/workflows/translate_all.yml‎

Lines changed: 76 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -129,39 +129,101 @@ jobs:
129129
git pull
130130
MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
131131
132-
- name: Publish search index release asset
132+
- name: Push search index to hacktricks-searchindex repo
133133
shell: bash
134134
env:
135135
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
136136
run: |
137137
set -euo pipefail
138138
139139
ASSET="book/searchindex.js"
140-
TAG="searchindex-${BRANCH}"
141-
TITLE="Search Index (${BRANCH})"
140+
TARGET_REPO="HackTricks-wiki/hacktricks-searchindex"
141+
FILENAME="searchindex-${BRANCH}.js"
142142
143143
if [ ! -f "$ASSET" ]; then
144144
echo "Expected $ASSET to exist after build" >&2
145145
exit 1
146146
fi
147147
148-
TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
148+
TOKEN="${PAT_TOKEN}"
149149
if [ -z "$TOKEN" ]; then
150-
echo "No token available for GitHub CLI" >&2
150+
echo "No PAT_TOKEN available" >&2
151151
exit 1
152152
fi
153-
export GH_TOKEN="$TOKEN"
154153
155-
# Delete the release if it exists
156-
if gh release view "$TAG" >/dev/null 2>&1; then
157-
echo "Release $TAG already exists, deleting it..."
158-
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY"
159-
fi
154+
# Clone the searchindex repo
155+
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo
156+
157+
# Compress the searchindex file
158+
gzip -9 -k -f "$ASSET"
159+
160+
# Show compression stats
161+
ORIGINAL_SIZE=$(wc -c < "$ASSET")
162+
COMPRESSED_SIZE=$(wc -c < "${ASSET}.gz")
163+
RATIO=$(awk "BEGIN {printf \"%.1f\", ($COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100}")
164+
echo "Compression: ${ORIGINAL_SIZE} bytes -> ${COMPRESSED_SIZE} bytes (${RATIO}%)"
165+
166+
# Copy ONLY the .gz version to the searchindex repo (no uncompressed .js)
167+
cp "${ASSET}.gz" "/tmp/searchindex-repo/${FILENAME}.gz"
160168
161-
# Create new release
162-
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY"
169+
# Commit and push with retry logic
170+
cd /tmp/searchindex-repo
171+
git config user.name "GitHub Actions"
172+
git config user.email "github-actions@github.com"
173+
git add "${FILENAME}.gz"
174+
175+
if git diff --staged --quiet; then
176+
echo "No changes to commit"
177+
else
178+
git commit -m "Update ${FILENAME} from hacktricks-cloud build"
179+
180+
# Retry push up to 20 times with pull --rebase between attempts
181+
MAX_RETRIES=20
182+
RETRY_COUNT=0
183+
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
184+
if git push origin master; then
185+
echo "Successfully pushed on attempt $((RETRY_COUNT + 1))"
186+
break
187+
else
188+
RETRY_COUNT=$((RETRY_COUNT + 1))
189+
if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
190+
echo "Push failed, attempt $RETRY_COUNT/$MAX_RETRIES. Pulling and retrying..."
191+
192+
# Try normal rebase first
193+
if git pull --rebase origin master 2>&1 | tee /tmp/pull_output.txt; then
194+
echo "Rebase successful, retrying push..."
195+
else
196+
# If rebase fails due to divergent histories (orphan branch reset), re-clone
197+
if grep -q "unrelated histories\|refusing to merge\|fatal: invalid upstream\|couldn't find remote ref" /tmp/pull_output.txt; then
198+
echo "Detected history rewrite, re-cloning repository..."
199+
cd /tmp
200+
rm -rf searchindex-repo
201+
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git searchindex-repo
202+
cd searchindex-repo
203+
git config user.name "GitHub Actions"
204+
git config user.email "github-actions@github.com"
205+
206+
# Re-copy ONLY the .gz version (no uncompressed .js)
207+
cp "${ASSET}.gz" "${FILENAME}.gz"
208+
209+
git add "${FILENAME}.gz"
210+
git commit -m "Update ${FILENAME}.gz from hacktricks-cloud build"
211+
echo "Re-cloned and re-committed, will retry push..."
212+
else
213+
echo "Rebase failed for unknown reason, retrying anyway..."
214+
fi
215+
fi
216+
217+
sleep 1
218+
else
219+
echo "Failed to push after $MAX_RETRIES attempts"
220+
exit 1
221+
fi
222+
fi
223+
done
224+
fi
163225
164-
# Login in AWs
226+
# Login in AWS
165227
- name: Configure AWS credentials using OIDC
166228
uses: aws-actions/configure-aws-credentials@v3
167229
with:

0 commit comments

Comments
 (0)