Skip to content

Commit 1ccf400

Browse files
carlospolopcarlospolop
authored
Merge branch 'master' into update_TOTOLINK_X6000R__Three_New_Vulnerabilities_Uncover_20251001_124438
2 parents 90afd5f + 1951fd1 commit 1ccf400

File tree

7 files changed

+642
-275
lines changed

7 files changed

+642
-275
lines changed

‎.github/workflows/build_master.yml‎

Lines changed: 81 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -43,51 +43,107 @@ jobs:
4343
&& sudo apt update \
4444
&& sudo apt install gh -y
4545
46-
- name: Publish search index release asset
46+
- name: Push search index to hacktricks-searchindex repo
4747
shell: bash
4848
env:
4949
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
5050
run: |
5151
set -euo pipefail
5252
5353
ASSET="book/searchindex.js"
54-
TAG="searchindex-en"
55-
TITLE="Search Index (en)"
54+
TARGET_REPO="HackTricks-wiki/hacktricks-searchindex"
55+
FILENAME="searchindex-en.js"
5656
5757
if [ ! -f "$ASSET" ]; then
5858
echo "Expected $ASSET to exist after build" >&2
5959
exit 1
6060
fi
6161
62-
TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
62+
TOKEN="${PAT_TOKEN}"
6363
if [ -z "$TOKEN" ]; then
64-
echo "No token available for GitHub CLI" >&2
64+
echo "No PAT_TOKEN available" >&2
6565
exit 1
6666
fi
67-
export GH_TOKEN="$TOKEN"
6867
69-
# Delete the release if it exists
70-
echo "Checking if release $TAG exists..."
71-
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
72-
echo "Release $TAG already exists, deleting it..."
73-
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag || {
74-
echo "Failed to delete release, trying without cleanup-tag..."
75-
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" || {
76-
echo "Warning: Could not delete existing release, will try to recreate..."
77-
}
78-
}
79-
sleep 2 # Give GitHub API a moment to process the deletion
68+
# Clone the searchindex repo
69+
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo
70+
71+
cd /tmp/searchindex-repo
72+
git config user.name "GitHub Actions"
73+
git config user.email "github-actions@github.com"
74+
75+
# Compress the searchindex file
76+
cd "${GITHUB_WORKSPACE}"
77+
gzip -9 -k -f "$ASSET"
78+
79+
# Show compression stats
80+
ORIGINAL_SIZE=$(wc -c < "$ASSET")
81+
COMPRESSED_SIZE=$(wc -c < "${ASSET}.gz")
82+
RATIO=$(awk "BEGIN {printf \"%.1f\", ($COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100}")
83+
echo "Compression: ${ORIGINAL_SIZE} bytes -> ${COMPRESSED_SIZE} bytes (${RATIO}%)"
84+
85+
# Copy the .gz version to the searchindex repo
86+
cd /tmp/searchindex-repo
87+
cp "${GITHUB_WORKSPACE}/${ASSET}.gz" "${FILENAME}.gz"
88+
89+
# Stage the updated file
90+
git add "${FILENAME}.gz"
91+
92+
# Commit and push with retry logic
93+
if git diff --staged --quiet; then
94+
echo "No changes to commit"
8095
else
81-
echo "Release $TAG does not exist, proceeding with creation..."
96+
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
97+
git commit -m "Update searchindex files - ${TIMESTAMP}"
98+
99+
# Retry push up to 20 times with pull --rebase between attempts
100+
MAX_RETRIES=20
101+
RETRY_COUNT=0
102+
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
103+
if git push origin master; then
104+
echo "Successfully pushed on attempt $((RETRY_COUNT + 1))"
105+
break
106+
else
107+
RETRY_COUNT=$((RETRY_COUNT + 1))
108+
if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
109+
echo "Push failed, attempt $RETRY_COUNT/$MAX_RETRIES. Pulling and retrying..."
110+
111+
# Try normal rebase first
112+
if git pull --rebase origin master 2>&1 | tee /tmp/pull_output.txt; then
113+
echo "Rebase successful, retrying push..."
114+
else
115+
# If rebase fails due to divergent histories (orphan branch reset), re-clone
116+
if grep -q "unrelated histories\|refusing to merge\|fatal: invalid upstream\|couldn't find remote ref" /tmp/pull_output.txt; then
117+
echo "Detected history rewrite, re-cloning repository..."
118+
cd /tmp
119+
rm -rf searchindex-repo
120+
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git searchindex-repo
121+
cd searchindex-repo
122+
git config user.name "GitHub Actions"
123+
git config user.email "github-actions@github.com"
124+
125+
# Re-copy the .gz version
126+
cp "${GITHUB_WORKSPACE}/${ASSET}.gz" "${FILENAME}.gz"
127+
128+
git add "${FILENAME}.gz"
129+
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
130+
git commit -m "Update searchindex files - ${TIMESTAMP}"
131+
echo "Re-cloned and re-committed, will retry push..."
132+
else
133+
echo "Rebase failed for unknown reason, retrying anyway..."
134+
fi
135+
fi
136+
137+
sleep 1
138+
else
139+
echo "Failed to push after $MAX_RETRIES attempts"
140+
exit 1
141+
fi
142+
fi
143+
done
82144
fi
83145
84-
# Create new release (with force flag to overwrite if deletion failed)
85-
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY" || {
86-
echo "Failed to create release, trying with force flag..."
87-
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag >/dev/null 2>&1 || true
88-
sleep 2
89-
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY"
90-
}
146+
echo "Successfully pushed searchindex files"
91147
92148
93149
# Login in AWs

‎.github/workflows/translate_all.yml‎

Lines changed: 76 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -129,39 +129,101 @@ jobs:
129129
git pull
130130
MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
131131
132-
- name: Publish search index release asset
132+
- name: Push search index to hacktricks-searchindex repo
133133
shell: bash
134134
env:
135135
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
136136
run: |
137137
set -euo pipefail
138138
139139
ASSET="book/searchindex.js"
140-
TAG="searchindex-${BRANCH}"
141-
TITLE="Search Index (${BRANCH})"
140+
TARGET_REPO="HackTricks-wiki/hacktricks-searchindex"
141+
FILENAME="searchindex-${BRANCH}.js"
142142
143143
if [ ! -f "$ASSET" ]; then
144144
echo "Expected $ASSET to exist after build" >&2
145145
exit 1
146146
fi
147147
148-
TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
148+
TOKEN="${PAT_TOKEN}"
149149
if [ -z "$TOKEN" ]; then
150-
echo "No token available for GitHub CLI" >&2
150+
echo "No PAT_TOKEN available" >&2
151151
exit 1
152152
fi
153-
export GH_TOKEN="$TOKEN"
154153
155-
# Delete the release if it exists
156-
if gh release view "$TAG" >/dev/null 2>&1; then
157-
echo "Release $TAG already exists, deleting it..."
158-
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY"
159-
fi
154+
# Clone the searchindex repo
155+
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo
156+
157+
# Compress the searchindex file
158+
gzip -9 -k -f "$ASSET"
159+
160+
# Show compression stats
161+
ORIGINAL_SIZE=$(wc -c < "$ASSET")
162+
COMPRESSED_SIZE=$(wc -c < "${ASSET}.gz")
163+
RATIO=$(awk "BEGIN {printf \"%.1f\", ($COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100}")
164+
echo "Compression: ${ORIGINAL_SIZE} bytes -> ${COMPRESSED_SIZE} bytes (${RATIO}%)"
165+
166+
# Copy ONLY the .gz version to the searchindex repo (no uncompressed .js)
167+
cp "${ASSET}.gz" "/tmp/searchindex-repo/${FILENAME}.gz"
160168
161-
# Create new release
162-
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY"
169+
# Commit and push with retry logic
170+
cd /tmp/searchindex-repo
171+
git config user.name "GitHub Actions"
172+
git config user.email "github-actions@github.com"
173+
git add "${FILENAME}.gz"
174+
175+
if git diff --staged --quiet; then
176+
echo "No changes to commit"
177+
else
178+
git commit -m "Update ${FILENAME} from hacktricks-cloud build"
179+
180+
# Retry push up to 20 times with pull --rebase between attempts
181+
MAX_RETRIES=20
182+
RETRY_COUNT=0
183+
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
184+
if git push origin master; then
185+
echo "Successfully pushed on attempt $((RETRY_COUNT + 1))"
186+
break
187+
else
188+
RETRY_COUNT=$((RETRY_COUNT + 1))
189+
if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
190+
echo "Push failed, attempt $RETRY_COUNT/$MAX_RETRIES. Pulling and retrying..."
191+
192+
# Try normal rebase first
193+
if git pull --rebase origin master 2>&1 | tee /tmp/pull_output.txt; then
194+
echo "Rebase successful, retrying push..."
195+
else
196+
# If rebase fails due to divergent histories (orphan branch reset), re-clone
197+
if grep -q "unrelated histories\|refusing to merge\|fatal: invalid upstream\|couldn't find remote ref" /tmp/pull_output.txt; then
198+
echo "Detected history rewrite, re-cloning repository..."
199+
cd /tmp
200+
rm -rf searchindex-repo
201+
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git searchindex-repo
202+
cd searchindex-repo
203+
git config user.name "GitHub Actions"
204+
git config user.email "github-actions@github.com"
205+
206+
# Re-copy ONLY the .gz version (no uncompressed .js)
207+
cp "${ASSET}.gz" "${FILENAME}.gz"
208+
209+
git add "${FILENAME}.gz"
210+
git commit -m "Update ${FILENAME}.gz from hacktricks-cloud build"
211+
echo "Re-cloned and re-committed, will retry push..."
212+
else
213+
echo "Rebase failed for unknown reason, retrying anyway..."
214+
fi
215+
fi
216+
217+
sleep 1
218+
else
219+
echo "Failed to push after $MAX_RETRIES attempts"
220+
exit 1
221+
fi
222+
fi
223+
done
224+
fi
163225
164-
# Login in AWs
226+
# Login in AWS
165227
- name: Configure AWS credentials using OIDC
166228
uses: aws-actions/configure-aws-credentials@v3
167229
with:

0 commit comments

Comments
 (0)