From c15c5569ae9acaef0ae8d92a0f2070afb382ab44 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:04:58 +0000 Subject: [PATCH 1/7] Initial plan From 5a98cefc7053a9e8f57f7c830bb60528d1c91581 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:09:35 +0000 Subject: [PATCH 2/7] Add k6 HTML report generation and verification Co-authored-by: chrisgleissner <3969147+chrisgleissner@users.noreply.github.com> --- src/main/bash/benchmark-scenario.sh | 7 ++++++- src/main/bash/generate-results-markdown.sh | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/bash/benchmark-scenario.sh b/src/main/bash/benchmark-scenario.sh index 813466b5..6f687686 100755 --- a/src/main/bash/benchmark-scenario.sh +++ b/src/main/bash/benchmark-scenario.sh @@ -68,6 +68,7 @@ jvmCsvFile="$resultDir/$approach"-jvm.csv latencyCsvFile="$resultDir/$approach"-latency.csv systemCsvFile="$resultDir/$approach"-system.csv chartFile="$resultDir/$approach".png +htmlReportFile="$resultDir/$approach".html resultsCsvFile="$resultsDir/results.csv" clientErrorLogFile="$resultDir/$approach"-client-error.log serviceErrorLogFile="$resultDir/$approach"-service-error.log @@ -174,6 +175,10 @@ verify_chart_results() { log "Results file $resultsCsvFile does not exist; terminating" exit 1 fi + if [ ! -f "$htmlReportFile" ]; then + log "k6 HTML report file $htmlReportFile does not exist; terminating" + exit 1 + fi } load() { @@ -181,7 +186,7 @@ load() { k6ConfigFile=src/main/resources/scenarios/"$k6Config" log "Issuing requests for ${_durationInSeconds}s using ${k6ConfigFile}..." - k6 run --env DURATION_IN_SECONDS="${_durationInSeconds}" --out csv="$k6OutputTmpFile" --env K6_CSV_TIME_FORMAT="unix_milli" --env DELAY_CALL_DEPTH="$delayCallDepth" --env DELAY_IN_MILLIS="$delayInMillis" --env SERVICE_API_BASE_URL="$serviceApiBaseUrl" --env VUS="$connections" --env RPS="$requestsPerSecond" "$k6ConfigFile" 2>&1 | tee "$k6LogTmpFile" + K6_WEB_DASHBOARD=true K6_WEB_DASHBOARD_EXPORT="$htmlReportFile" k6 run --env DURATION_IN_SECONDS="${_durationInSeconds}" --out csv="$k6OutputTmpFile" --env K6_CSV_TIME_FORMAT="unix_milli" --env DELAY_CALL_DEPTH="$delayCallDepth" --env DELAY_IN_MILLIS="$delayInMillis" --env SERVICE_API_BASE_URL="$serviceApiBaseUrl" --env VUS="$connections" --env RPS="$requestsPerSecond" "$k6ConfigFile" 2>&1 | tee "$k6LogTmpFile" # csv: metric_name,timestamp,metric_value,check,error,error_code,expected_response,group,method,name,proto,scenario,service,status # shellcheck disable=SC2002 diff --git a/src/main/bash/generate-results-markdown.sh b/src/main/bash/generate-results-markdown.sh index a7c96a97..ef9d89e5 100755 --- a/src/main/bash/generate-results-markdown.sh +++ b/src/main/bash/generate-results-markdown.sh @@ -80,8 +80,12 @@ while IFS=',' read -r scenario k6Config serverProfiles delayCallDepth delayInMil IFS=',' read -ra approach_array <<< "$approaches" for approach in "${approach_array[@]}"; do image_path="$results_dir/$scenario/$approach.png" + html_report_path="$results_dir/$scenario/$approach.html" if [[ -f "$image_path" ]]; then markdown+="#### $approach\n\n" + if [[ -f "$html_report_path" ]]; then + markdown+="[k6 Report](./$scenario/$approach.html)\n\n" + fi markdown+="![$approach](./$scenario/$approach.png)\n\n" else markdown+="#### $approach (No image available)\n\n" From 8a06f5e7c997f3fde5a636ff1d65d12213362529 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:14:32 +0000 Subject: [PATCH 3/7] Add CI workflow verification for k6 HTML reports Co-authored-by: chrisgleissner <3969147+chrisgleissner@users.noreply.github.com> --- .github/workflows/reusable-build.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/reusable-build.yaml b/.github/workflows/reusable-build.yaml index 1713a664..66155b01 100644 --- a/.github/workflows/reusable-build.yaml +++ b/.github/workflows/reusable-build.yaml @@ -84,6 +84,13 @@ jobs: else echo "Chart $chartFilename exists" fi + htmlReportFilename=$scenarioPath$approach.html + if [ ! -f "$htmlReportFilename" ]; then + echo "Verification error: k6 HTML report $htmlReportFilename missing" + exit 1 + else + echo "k6 HTML report $htmlReportFilename exists" + fi done done resultsCsvFilename=build/results/results.csv From a28d045ced8ae78dfbd57a882a061346859d191a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:17:35 +0000 Subject: [PATCH 4/7] Improve readability of k6 command with line breaks Co-authored-by: chrisgleissner <3969147+chrisgleissner@users.noreply.github.com> --- src/main/bash/benchmark-scenario.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/bash/benchmark-scenario.sh b/src/main/bash/benchmark-scenario.sh index 6f687686..19256997 100755 --- a/src/main/bash/benchmark-scenario.sh +++ b/src/main/bash/benchmark-scenario.sh @@ -186,7 +186,17 @@ load() { k6ConfigFile=src/main/resources/scenarios/"$k6Config" log "Issuing requests for ${_durationInSeconds}s using ${k6ConfigFile}..." - K6_WEB_DASHBOARD=true K6_WEB_DASHBOARD_EXPORT="$htmlReportFile" k6 run --env DURATION_IN_SECONDS="${_durationInSeconds}" --out csv="$k6OutputTmpFile" --env K6_CSV_TIME_FORMAT="unix_milli" --env DELAY_CALL_DEPTH="$delayCallDepth" --env DELAY_IN_MILLIS="$delayInMillis" --env SERVICE_API_BASE_URL="$serviceApiBaseUrl" --env VUS="$connections" --env RPS="$requestsPerSecond" "$k6ConfigFile" 2>&1 | tee "$k6LogTmpFile" + K6_WEB_DASHBOARD=true K6_WEB_DASHBOARD_EXPORT="$htmlReportFile" \ + k6 run \ + --env DURATION_IN_SECONDS="${_durationInSeconds}" \ + --out csv="$k6OutputTmpFile" \ + --env K6_CSV_TIME_FORMAT="unix_milli" \ + --env DELAY_CALL_DEPTH="$delayCallDepth" \ + --env DELAY_IN_MILLIS="$delayInMillis" \ + --env SERVICE_API_BASE_URL="$serviceApiBaseUrl" \ + --env VUS="$connections" \ + --env RPS="$requestsPerSecond" \ + "$k6ConfigFile" 2>&1 | tee "$k6LogTmpFile" # csv: metric_name,timestamp,metric_value,check,error,error_code,expected_response,group,method,name,proto,scenario,service,status # shellcheck disable=SC2002 From dbaf3e9f01f762e58507adbc7b6ba31e0c48a473 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:37:51 +0000 Subject: [PATCH 5/7] Extend smoketest duration to 15s and copy all result files including HTML reports Co-authored-by: chrisgleissner <3969147+chrisgleissner@users.noreply.github.com> --- .github/workflows/reusable-build.yaml | 2 +- src/main/resources/scenarios/scenarios-smoketest.csv | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable-build.yaml b/.github/workflows/reusable-build.yaml index 66155b01..f9b4ad0b 100644 --- a/.github/workflows/reusable-build.yaml +++ b/.github/workflows/reusable-build.yaml @@ -108,7 +108,7 @@ jobs: scenarios=${scenariosPath%%.*} resultsDir=results/ci/$scenarios/${{ matrix.os }}/java-${{ matrix.java-version }}/ mkdir -p $resultsDir - cp build/results/results* $resultsDir + cp -r build/results/* $resultsDir git pull - name: Commit results diff --git a/src/main/resources/scenarios/scenarios-smoketest.csv b/src/main/resources/scenarios/scenarios-smoketest.csv index 7ab57371..40738326 100644 --- a/src/main/resources/scenarios/scenarios-smoketest.csv +++ b/src/main/resources/scenarios/scenarios-smoketest.csv @@ -1,5 +1,5 @@ scenario,k6Config,serverProfiles,delayCallDepth,delayInMillis,connections,requestsPerSecond,warmupDurationInSeconds,testDurationInSeconds -smoketest-get-time,get-time.js,,0,0,100,200,2,6 -smoketest-get-movies-h2,get-movies.js,,1,100,10,10,0,6 -smoketest-get-movies-postgres,get-movies.js,postgres,1,100,10,10,0,6 -smoketest-get-movies-postgres-no-cache,get-movies.js,postgres|no-cache,1,100,10,10,0,6 +smoketest-get-time,get-time.js,,0,0,100,200,2,15 +smoketest-get-movies-h2,get-movies.js,,1,100,10,10,0,15 +smoketest-get-movies-postgres,get-movies.js,postgres,1,100,10,10,0,15 +smoketest-get-movies-postgres-no-cache,get-movies.js,postgres|no-cache,1,100,10,10,0,15 From 61be0fe3b4f122a3f125d81387c405b8acc6a991 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:54:12 +0000 Subject: [PATCH 6/7] Only generate k6 HTML reports during test phase, not warmup Co-authored-by: chrisgleissner <3969147+chrisgleissner@users.noreply.github.com> --- src/main/bash/benchmark-scenario.sh | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/main/bash/benchmark-scenario.sh b/src/main/bash/benchmark-scenario.sh index 19256997..06636ddd 100755 --- a/src/main/bash/benchmark-scenario.sh +++ b/src/main/bash/benchmark-scenario.sh @@ -141,7 +141,7 @@ load_and_measure_system() { (sleep 2 && ./src/main/bash/system-measure.sh "$systemCsvFile" "$durationInSeconds") & systemMeasurePid=$! - load "$durationInSeconds" + load "$phase" "$durationInSeconds" mv "$jvmCsvTmpFile" "$jvmCsvFile" && log "Saved $jvmCsvFile" wait_for_system_csv_file @@ -182,11 +182,26 @@ verify_chart_results() { } load() { - _durationInSeconds=$1 + phase=$1 + _durationInSeconds=$2 k6ConfigFile=src/main/resources/scenarios/"$k6Config" log "Issuing requests for ${_durationInSeconds}s using ${k6ConfigFile}..." - K6_WEB_DASHBOARD=true K6_WEB_DASHBOARD_EXPORT="$htmlReportFile" \ + + # Only generate HTML report during test phase, not during warmup + if [ "$phase" == "test" ]; then + K6_WEB_DASHBOARD=true K6_WEB_DASHBOARD_EXPORT="$htmlReportFile" \ + k6 run \ + --env DURATION_IN_SECONDS="${_durationInSeconds}" \ + --out csv="$k6OutputTmpFile" \ + --env K6_CSV_TIME_FORMAT="unix_milli" \ + --env DELAY_CALL_DEPTH="$delayCallDepth" \ + --env DELAY_IN_MILLIS="$delayInMillis" \ + --env SERVICE_API_BASE_URL="$serviceApiBaseUrl" \ + --env VUS="$connections" \ + --env RPS="$requestsPerSecond" \ + "$k6ConfigFile" 2>&1 | tee "$k6LogTmpFile" + else k6 run \ --env DURATION_IN_SECONDS="${_durationInSeconds}" \ --out csv="$k6OutputTmpFile" \ @@ -197,6 +212,7 @@ load() { --env VUS="$connections" \ --env RPS="$requestsPerSecond" \ "$k6ConfigFile" 2>&1 | tee "$k6LogTmpFile" + fi # csv: metric_name,timestamp,metric_value,check,error,error_code,expected_response,group,method,name,proto,scenario,service,status # shellcheck disable=SC2002 From fbfd49dea3635318be3148b17d9a7d1920db191e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:59:28 +0000 Subject: [PATCH 7/7] Extend scenarios-default.csv smoketest duration to 15s Co-authored-by: chrisgleissner <3969147+chrisgleissner@users.noreply.github.com> --- src/main/resources/scenarios/scenarios-default.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/scenarios/scenarios-default.csv b/src/main/resources/scenarios/scenarios-default.csv index e3ae5e5c..76607783 100644 --- a/src/main/resources/scenarios/scenarios-default.csv +++ b/src/main/resources/scenarios/scenarios-default.csv @@ -1,5 +1,5 @@ scenario,k6Config,serverProfiles,delayCallDepth,delayInMillis,connections,requestsPerSecond,warmupDurationInSeconds,testDurationInSeconds -smoketest,get-time.js,,1,100,5,5,0,5 +smoketest,get-time.js,,1,100,5,5,0,15 1k-vus-and-rps-get-time-no-delay,get-time.js,,0,0,1000,1000,10,180 5k-vus-and-rps-get-time,get-time.js,,0,100,5000,5000,10,180 5k-vus-and-rps-get-movies,get-movies.js,,0,100,5000,5000,10,180