Skip to content

Commit 64d2204

Browse files
jservjserv
committed
CI: Enhance workflow reliability and consistency
This improves GitHub Actions workflows: - Add reliable-download.sh wrapper with automatic retry logic and atomic file operations for all network downloads - Add concurrency control to prevent duplicate workflow runs - Increase network timeout values (30s connect, 60s read) - Adds command existence validation before execution - Includes trap handlers for cleanup on cancellation
1 parent bd0f10a commit 64d2204

File tree

6 files changed

+524
-207
lines changed

6 files changed

+524
-207
lines changed

‎.ci/reliable-download.sh‎

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
#!/usr/bin/env bash
2+
# Reliable download wrapper with retry logic and timeout handling
3+
# Prefers curl over wget for better error handling and features
4+
5+
set -euo pipefail
6+
7+
# Configuration
8+
RETRIES=3
9+
TIMEOUT=30
10+
READ_TIMEOUT=60
11+
WAIT_RETRY=1
12+
13+
# Color output
14+
RED='\033[0;31m'
15+
GREEN='\033[0;32m'
16+
YELLOW='\033[1;33m'
17+
NC='\033[0m' # No Color
18+
19+
usage()
20+
{
21+
cat << EOF
22+
Usage: $0 [OPTIONS] URL [OUTPUT_FILE]
23+
24+
Reliable download wrapper with automatic retry logic.
25+
26+
OPTIONS:
27+
-h, --help Show this help message
28+
-r, --retries N Number of retry attempts (default: $RETRIES)
29+
-t, --timeout N Timeout in seconds (default: $TIMEOUT)
30+
-o, --output FILE Output file path
31+
-q, --quiet Quiet mode (minimal output)
32+
33+
EXAMPLES:
34+
$0 https://example.com/file.tar.gz
35+
$0 -o output.zip https://example.com/archive.zip
36+
$0 --retries 5 --timeout 30 https://example.com/large-file.bin
37+
38+
ENVIRONMENT:
39+
PREFER_WGET=1 Force using wget instead of curl
40+
EOF
41+
exit 0
42+
}
43+
44+
log()
45+
{
46+
echo -e "${GREEN}[reliable-download]${NC} $*" >&2
47+
}
48+
49+
warn()
50+
{
51+
echo -e "${YELLOW}[WARNING]${NC} $*" >&2
52+
}
53+
54+
error()
55+
{
56+
echo -e "${RED}[ERROR]${NC} $*" >&2
57+
}
58+
59+
# Parse arguments
60+
QUIET=0
61+
OUTPUT=""
62+
URL=""
63+
64+
while [[ $# -gt 0 ]]; do
65+
case $1 in
66+
-h | --help)
67+
usage
68+
;;
69+
-r | --retries)
70+
RETRIES="$2"
71+
shift 2
72+
;;
73+
-t | --timeout)
74+
TIMEOUT="$2"
75+
shift 2
76+
;;
77+
-o | --output)
78+
OUTPUT="$2"
79+
shift 2
80+
;;
81+
-q | --quiet)
82+
QUIET=1
83+
shift
84+
;;
85+
-*)
86+
error "Unknown option: $1"
87+
usage
88+
;;
89+
*)
90+
if [[ -z "$URL" ]]; then
91+
URL="$1"
92+
elif [[ -z "$OUTPUT" ]]; then
93+
OUTPUT="$1"
94+
else
95+
error "Too many arguments"
96+
usage
97+
fi
98+
shift
99+
;;
100+
esac
101+
done
102+
103+
# Validate URL
104+
if [[ -z "$URL" ]]; then
105+
error "URL is required"
106+
usage
107+
fi
108+
109+
# Determine output file
110+
if [[ -z "$OUTPUT" ]]; then
111+
OUTPUT=$(basename "$URL")
112+
fi
113+
114+
# Check available tools
115+
HAS_CURL=0
116+
HAS_WGET=0
117+
118+
if command -v curl &> /dev/null; then
119+
HAS_CURL=1
120+
fi
121+
122+
if command -v wget &> /dev/null; then
123+
HAS_WGET=1
124+
fi
125+
126+
if [[ $HAS_CURL -eq 0 && $HAS_WGET -eq 0 ]]; then
127+
error "Neither curl nor wget is available"
128+
exit 1
129+
fi
130+
131+
# Prefer curl unless PREFER_WGET is set
132+
USE_CURL=0
133+
if [[ $HAS_CURL -eq 1 && -z "${PREFER_WGET:-}" ]]; then
134+
USE_CURL=1
135+
elif [[ $HAS_WGET -eq 0 ]]; then
136+
error "wget not available and PREFER_WGET is set"
137+
exit 1
138+
fi
139+
140+
# Download function
141+
download()
142+
{
143+
local attempt=$1
144+
local url=$2
145+
local output=$3
146+
147+
[[ $QUIET -eq 0 ]] && log "Attempt $attempt/$RETRIES: Downloading $url"
148+
149+
if [[ $USE_CURL -eq 1 ]]; then
150+
# Verify curl is available
151+
if ! command -v curl > /dev/null 2>&1; then
152+
error "curl binary not found in PATH"
153+
return 1
154+
fi
155+
# curl options:
156+
# -f: Fail silently on HTTP errors
157+
# -L: Follow redirects
158+
# -S: Show error even with -s
159+
# -s: Silent mode (if quiet)
160+
# --retry: Number of retries
161+
# --retry-delay: Wait time between retries
162+
# --connect-timeout: Connection timeout
163+
# --max-time: Total operation timeout
164+
# -o: Output file
165+
local curl_opts=(
166+
-f
167+
-L
168+
--retry "$RETRIES"
169+
--retry-delay "$WAIT_RETRY"
170+
--retry-connrefused
171+
--connect-timeout "$TIMEOUT"
172+
--max-time "$READ_TIMEOUT"
173+
-o "$output"
174+
)
175+
176+
if [[ $QUIET -eq 1 ]]; then
177+
curl_opts+=(-s -S)
178+
else
179+
curl_opts+=(--progress-bar)
180+
fi
181+
182+
curl "${curl_opts[@]}" "$url"
183+
else
184+
# Verify wget is available
185+
if ! command -v wget > /dev/null 2>&1; then
186+
error "wget binary not found in PATH"
187+
return 1
188+
fi
189+
# wget options:
190+
# --retry-connrefused: Retry on connection refused
191+
# --waitretry: Wait between retries
192+
# --read-timeout: Read timeout
193+
# --timeout: Connection timeout
194+
# --tries: Number of attempts
195+
# -O: Output file
196+
local wget_opts=(
197+
--retry-connrefused
198+
--waitretry="$WAIT_RETRY"
199+
--read-timeout="$READ_TIMEOUT"
200+
--timeout="$TIMEOUT"
201+
--tries="$RETRIES"
202+
-O "$output"
203+
)
204+
205+
if [[ $QUIET -eq 1 ]]; then
206+
wget_opts+=(-q)
207+
fi
208+
209+
wget "${wget_opts[@]}" "$url"
210+
fi
211+
}
212+
213+
# Main download logic with retry using atomic operations
214+
SUCCESS=0
215+
TEMP_OUTPUT="${OUTPUT}.downloading.$$"
216+
217+
# Cleanup trap for partial downloads
218+
trap 'rm -f "$TEMP_OUTPUT"' EXIT INT TERM
219+
220+
for i in $(seq 1 "$RETRIES"); do
221+
if download "$i" "$URL" "$TEMP_OUTPUT"; then
222+
# Atomic move to final location
223+
if mv -f "$TEMP_OUTPUT" "$OUTPUT"; then
224+
SUCCESS=1
225+
[[ $QUIET -eq 0 ]] && log "Download successful: $OUTPUT"
226+
break
227+
else
228+
error "Failed to move temporary file to $OUTPUT"
229+
fi
230+
else
231+
EXIT_CODE=$?
232+
warn "Download failed (attempt $i/$RETRIES, exit code: $EXIT_CODE)"
233+
234+
if [[ $i -lt $RETRIES ]]; then
235+
[[ $QUIET -eq 0 ]] && log "Retrying in ${WAIT_RETRY}s..."
236+
sleep "$WAIT_RETRY"
237+
fi
238+
fi
239+
done
240+
241+
if [[ $SUCCESS -eq 0 ]]; then
242+
error "Download failed after $RETRIES attempts"
243+
rm -f "$TEMP_OUTPUT" # Clean up partial download
244+
exit 1
245+
fi
246+
247+
# Verify file was created and is not empty
248+
if [[ ! -f "$OUTPUT" ]]; then
249+
error "Output file not created: $OUTPUT"
250+
exit 1
251+
fi
252+
253+
if [[ ! -s "$OUTPUT" ]]; then
254+
error "Output file is empty: $OUTPUT"
255+
rm -f "$OUTPUT"
256+
exit 1
257+
fi
258+
259+
exit 0

‎.github/workflows/benchmark.yml‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,19 @@ name: Benchmark
22

33
on: [push, pull_request_target, workflow_dispatch]
44

5+
concurrency:
6+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.pull_request.head.sha || github.sha }}
7+
cancel-in-progress: true
8+
59
jobs:
610
benchmark:
711
name: Performance regression check
812
if: contains(toJSON(github.event.head_commit.message), 'Merge pull request ') == false
13+
timeout-minutes: 30
914
runs-on: ubuntu-24.04
1015
steps:
1116
- uses: actions/checkout@v4
17+
1218
- name: Test changed files
1319
id: changed-files
1420
uses: tj-actions/changed-files@v46
@@ -19,24 +25,35 @@ jobs:
1925
src/emulate.c
2026
src/rv32_template.c
2127
src/rv32_constopt.c
22-
- name: install-dependencies
23-
if: ${{ steps.changed-files.outputs.any_changed == 'true' ||
24-
github.event_name == 'workflow_dispatch'}}
28+
src/cache.c
29+
src/io.c
30+
src/jit.c
31+
32+
- name: Cache benchmark artifacts
33+
if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch'
34+
uses: actions/cache@v4
35+
with:
36+
path: build/
37+
key: benchmark-artifacts-${{ runner.os }}-${{ hashFiles('mk/artifact.mk') }}
38+
restore-keys: |
39+
benchmark-artifacts-${{ runner.os }}-
40+
41+
- name: Install dependencies
42+
if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch'
2543
run: |
2644
sudo pip3 install numpy --break-system-packages
2745
shell: bash
28-
- name: default build
29-
if: ${{ steps.changed-files.outputs.any_changed == 'true' ||
30-
github.event_name == 'workflow_dispatch'}}
46+
47+
- name: Build for benchmark
48+
if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch'
3149
run: make ENABLE_SDL=0 all artifact
3250
- name: Run benchmark
33-
if: ${{ steps.changed-files.outputs.any_changed == 'true' ||
34-
github.event_name == 'workflow_dispatch'}}
51+
if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch'
3552
run: |
3653
tests/bench-aggregator.py
54+
3755
- name: Store benchmark results
38-
if: ${{ steps.changed-files.outputs.any_changed == 'true' ||
39-
github.event_name == 'workflow_dispatch'}}
56+
if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch'
4057
uses: benchmark-action/github-action-benchmark@v1
4158
with:
4259
name: Benchmarks

‎.github/workflows/build-artifact.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,13 @@ on:
66
- master
77
workflow_dispatch:
88

9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
913
jobs:
1014
detect-file-change:
11-
runs-on: ubuntu-22.04
15+
runs-on: ubuntu-24.04
1216
steps:
1317
- name: Checkout repository
1418
uses: actions/checkout@v4
@@ -41,7 +45,7 @@ jobs:
4145
build-artifact:
4246
needs: [detect-file-change]
4347
if: ${{ needs.detect-file-change.outputs.has_changed_files == 'true' || github.event_name == 'workflow_dispatch' }}
44-
runs-on: ubuntu-22.04
48+
runs-on: ubuntu-24.04
4549
steps:
4650
- name: Checkout repository
4751
uses: actions/checkout@v4

‎.github/workflows/build-linux-artifacts.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ on:
66
- master
77
workflow_dispatch:
88

9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
913
jobs:
1014
detect-file-change:
1115
runs-on: ubuntu-24.04
@@ -40,7 +44,7 @@ jobs:
4044
build-linux-image-artifact:
4145
needs: [detect-file-change]
4246
if: ${{ needs.detect-file-change.outputs.has_changed_linux_image_version == 'true' || github.event_name == 'workflow_dispatch' }}
43-
runs-on: ubuntu-22.04
47+
runs-on: ubuntu-24.04
4448
steps:
4549
- name: Checkout repository
4650
uses: actions/checkout@v4

0 commit comments

Comments
 (0)