add default feature in generate support matrices

Signed-off-by: Teresa Chen <boe20211@gmail.com>

Author: boe20211

.buildkite/features/Chunked_Prefill.yml

@@ -1,27 +1,27 @@
 # JAX-Path Qxix Quantization
 steps:
-  - label: "Correctness tests for JAX-Path Qxix Quantization"
-    key: "JAX-Path_Qxix_Quantization_CorrectnessTest"
-    soft_fail: true
-    agents:
-      queue: tpu_v6e_queue
-    commands:
-      - echo "covered by performance tests"
-  - label: "Record correctness test result for JAX-Path Qxix Quantization"
-    key: "record_JAX-Path_Qxix_Quantization_CorrectnessTest"
-    depends_on: "JAX-Path_Qxix_Quantization_CorrectnessTest"
-    env:
-      CI_TARGET: "JAX-Path Qxix Quantization"
-      CI_STAGE: "CorrectnessTest"
-    agents:
-      queue: cpu
-    commands:
-      - |
-        .buildkite/scripts/record_step_result.sh JAX-Path_Qxix_Quantization_CorrectnessTest
+  # - label: "Correctness tests for JAX-Path Qxix Quantization"
+  #   key: "JAX-Path_Qxix_Quantization_CorrectnessTest"
+  #   soft_fail: true
+  #   agents:
+  #     queue: tpu_v6e_queue
+  #   commands:
+  #     - echo "covered by performance tests"
+  # - label: "Record correctness test result for JAX-Path Qxix Quantization"
+  #   key: "record_JAX-Path_Qxix_Quantization_CorrectnessTest"
+  #   depends_on: "JAX-Path_Qxix_Quantization_CorrectnessTest"
+  #   env:
+  #     CI_TARGET: "JAX-Path Qxix Quantization"
+  #     CI_STAGE: "CorrectnessTest"
+  #   agents:
+  #     queue: cpu
+  #   commands:
+  #     - |
+  #       .buildkite/scripts/record_step_result.sh JAX-Path_Qxix_Quantization_CorrectnessTest
 
   - label: "Performance tests for JAX-Path Qxix Quantization"
     key: "JAX-Path_Qxix_Quantization_PerformanceTest"
-    depends_on: "record_JAX-Path_Qxix_Quantization_CorrectnessTest"
+    # depends_on: "record_JAX-Path_Qxix_Quantization_CorrectnessTest"
     soft_fail: true
     env:
       QUANTIZATION: "True"

.buildkite/features/JAX-Path_Qxix_Quantization.yml

@@ -1,27 +1,27 @@
 # Quantized Matmul Attention and KV Cache
 steps:
-  - label: "Correctness tests for Quantized Matmul Attention and KV Cache"
-    key: "Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest"
-    soft_fail: true
-    agents:
-      queue: cpu
-    commands:
-      - echo "covered by performance test"
-  - label: "Record correctness test result for Quantized Matmul Attention and KV Cache"
-    key: "record_Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest"
-    depends_on: "Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest"
-    env:
-      CI_TARGET: "Quantized Matmul Attention and KV Cache"
-      CI_STAGE: "CorrectnessTest"
-    agents:
-      queue: cpu
-    commands:
-      - |
-        .buildkite/scripts/record_step_result.sh Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest
+  # - label: "Correctness tests for Quantized Matmul Attention and KV Cache"
+  #   key: "Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest"
+  #   soft_fail: true
+  #   agents:
+  #     queue: cpu
+  #   commands:
+  #     - echo "covered by performance test"
+  # - label: "Record correctness test result for Quantized Matmul Attention and KV Cache"
+  #   key: "record_Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest"
+  #   depends_on: "Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest"
+  #   env:
+  #     CI_TARGET: "Quantized Matmul Attention and KV Cache"
+  #     CI_STAGE: "CorrectnessTest"
+  #   agents:
+  #     queue: cpu
+  #   commands:
+  #     - |
+  #       .buildkite/scripts/record_step_result.sh Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest
 
   - label: "Performance tests for Quantized Matmul Attention and KV Cache"
     key: "Quantized_Matmul_Attention_and_KV_Cache_PerformanceTest"
-    depends_on: "record_Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest"
+    # depends_on: "record_Quantized_Matmul_Attention_and_KV_Cache_CorrectnessTest"
     soft_fail: true
     agents:
       queue: tpu_v6e_8_queue

.buildkite/features/Prefix_Caching.yml

@@ -0,0 +1,4 @@
+Chunked Prefill
+Prefix Caching
+Ragged Paged Attention V3
+Single Program Multi Data

.buildkite/features/Quantized_Matmul_Attention_and_KV_Cache.yml

@@ -5,11 +5,12 @@ ANY_FAILED=false
 
 MODEL_LIST_KEY="model-list"
 FEATURE_LIST_KEY="feature-list"
+DEFAULT_FEATURES_FILE=".buildkite/features/default_features.txt"
 
 # Note: This script assumes the metadata keys contain newline-separated lists.
 # The `mapfile` command reads these lists into arrays, correctly handling spaces.
 mapfile -t model_list < <(buildkite-agent meta-data get "${MODEL_LIST_KEY}" --default "")
-mapfile -t feature_list < <(buildkite-agent meta-data get "${FEATURE_LIST_KEY}" --default "")
+mapfile -t metadata_feature_list < <(buildkite-agent meta-data get "${FEATURE_LIST_KEY}" --default "")
 MODEL_STAGES=("UnitTest" "IntegrationTest" "Benchmark")
 FEATURE_STAGES=("CorrectnessTest" "PerformanceTest")
 
@@ -20,8 +21,24 @@ echo "Model,UnitTest,IntegrationTest,Benchmark" > "$model_support_matrix_csv"
 feature_support_matrix_csv="feature_support_matrix.csv"
 echo "Feature,CorrectnessTest,PerformanceTest" > "$feature_support_matrix_csv"
 
+# Read the list of default features from the specified file
+if [[ -f "${DEFAULT_FEATURES_FILE}" ]]; then
+    mapfile -t default_feature_list < <(sed 's/\r$//; /^$/d' "${DEFAULT_FEATURES_FILE}")
+else
+    default_feature_list=()
+    echo "Warning: Default features file not found at ${DEFAULT_FEATURES_FILE}"
+fi
+
+# Combine metadata features and default features into the final unique list
+mapfile -t feature_list < <(
+    (
+        printf "%s\n" "${metadata_feature_list[@]:-}"
+        printf "%s\n" "${default_feature_list[@]:-}"
+    ) | sort -u | sed '/^$/d'
+)
+
 process_models() {
-    for model in "$@"; do
+    for model in "${model_list[@]:-}"; do
         row="\"$model\""
         for stage in "${MODEL_STAGES[@]}"; do
             result=$(buildkite-agent meta-data get "${model}:${stage}" --default "N/A")
@@ -35,10 +52,23 @@ process_models() {
 }
 
 process_features() {
-    for feature in "$@"; do
+    for feature in "${feature_list[@]:-}"; do
         row="\"$feature\""
+        # Check if the current feature is in the default_feature_list
+        is_default_feature=false
+        for default_feat in "${default_feature_list[@]:-}"; do
+            if [[ "${feature}" == "${default_feat}" ]]; then
+                is_default_feature=true
+                break
+            fi
+        done
         for stage in "${FEATURE_STAGES[@]}"; do
-            result=$(buildkite-agent meta-data get "${feature}:${stage}" --default "N/A")
+            # Set the default value based on whether it's a default feature
+            local default_value="N/A"
+            if [ "$is_default_feature" = true ]; then
+                default_value="✅"
+            fi
+            result=$(buildkite-agent meta-data get "${feature}:${stage}" --default "${default_value}")
             row="$row,$result"
             if [ "${result}" != "✅" ] && [ "${result}" != "N/A" ] ; then
                 ANY_FAILED=true
@@ -49,11 +79,11 @@ process_features() {
 }
 
 if [ ${#model_list[@]} -gt 0 ]; then
-    process_models "${model_list[@]}"
+    process_models
 fi
 
 if [ ${#feature_list[@]} -gt 0 ]; then
-    process_features "${feature_list[@]}"
+    process_features
 fi
 
 buildkite-agent meta-data set "CI_TESTS_FAILED" "${ANY_FAILED}"