Merge branch 'master' into rmt_loop_count

Author: SuGlider

.github/scripts/get_affected.py

@@ -7,27 +7,33 @@ CHECK_REQUIREMENTS="./components/arduino-esp32/.github/scripts/sketch_utils.sh c
 # Export IDF environment
 . ${IDF_PATH}/export.sh
 
-# Find all examples in ./components/arduino-esp32/idf_component_examples
-idf_component_examples=$(find ./components/arduino-esp32/idf_component_examples -mindepth 1 -maxdepth 1 -type d)
-
-for example in $idf_component_examples; do
-    if [ -f "$example"/ci.json ]; then
+if [ -n "$1" ]; then
+    # If a file is provided, use it to get the affected examples.
+    affected_examples=$(cat "$1" | sort)
+else
+    # Otherwise, find all examples in ./components/arduino-esp32/idf_component_examples
+    affected_examples=$(find ./components/arduino-esp32/idf_component_examples -mindepth 1 -maxdepth 1 -type d | sed 's|^\./components/arduino-esp32/||' | sort)
+fi
+
+for example in $affected_examples; do
+    example_path="$PWD/components/arduino-esp32/$example"
+    if [ -f "$example_path/ci.json" ]; then
         # If the target is listed as false, skip the sketch. Otherwise, include it.
-        is_target=$(jq -r --arg target "$IDF_TARGET" '.targets[$target]' "$example"/ci.json)
+        is_target=$(jq -r --arg target "$IDF_TARGET" '.targets[$target]' "$example_path/ci.json")
         if [[ "$is_target" == "false" ]]; then
             printf "\n\033[93mSkipping %s for target %s\033[0m\n\n" "$example" "$IDF_TARGET"
             continue
         fi
     fi
 
-    idf.py -C "$example" set-target "$IDF_TARGET"
+    idf.py --preview -C "$example_path" set-target "$IDF_TARGET"
 
-    has_requirements=$(${CHECK_REQUIREMENTS} "$example" "$example/sdkconfig")
+    has_requirements=$(${CHECK_REQUIREMENTS} "$example_path" "$example_path/sdkconfig")
     if [ "$has_requirements" -eq 0 ]; then
         printf "\n\033[93m%s does not meet the requirements for %s. Skipping...\033[0m\n\n" "$example" "$IDF_TARGET"
         continue
     fi
 
     printf "\n\033[95mBuilding %s\033[0m\n\n" "$example"
-    idf.py -C "$example" -DEXTRA_COMPONENT_DIRS="$PWD/components" build
+    idf.py --preview -C "$example_path" -DEXTRA_COMPONENT_DIRS="$PWD/components" build
 done

.github/scripts/on-push-idf.sh

@@ -244,6 +244,14 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
         fi
     fi
 
+    # Install libraries from ci.json if they exist
+    install_libs -ai "$ide_path" -s "$sketchdir"
+    install_result=$?
+    if [ $install_result -ne 0 ]; then
+        echo "ERROR: Library installation failed for $sketchname" >&2
+        exit $install_result
+    fi
+
     ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
     if [ -n "$ARDUINO_BUILD_DIR" ]; then
         build_dir="$ARDUINO_BUILD_DIR"
@@ -349,7 +357,7 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
     unset options
 }
 
-function count_sketches { # count_sketches <path> [target] [file] [ignore-requirements]
+function count_sketches { # count_sketches <path> [target] [ignore-requirements] [file]
     local path=$1
     local target=$2
     local ignore_requirements=$3
@@ -368,7 +376,7 @@ function count_sketches { # count_sketches <path> [target] [file] [ignore-requir
     fi
 
     if [ -f "$file" ]; then
-        sketches=$(cat "$file")
+        sketches=$(cat "$file" | sort)
     else
         sketches=$(find "$path" -name '*.ino' | sort)
     fi
@@ -580,13 +588,168 @@ function build_sketches { # build_sketches <ide_path> <user_path> <target> <path
     return 0
 }
 
+# Return 0 if the string looks like a git URL we should pass to --git-url
+is_git_like_url() {
+    local u=$1
+    [[ "$u" =~ ^https?://.+ ]] || [[ "$u" =~ ^git@[^:]+:.+ ]]
+}
+
+# If status!=0, print errors/warnings from captured output (fallback to full output)
+print_err_warnings() {
+    local status=$1; shift
+    local out=$*
+    if [ "$status" -ne 0 ]; then
+        printf '%s\n' "$out" | grep -Ei "error|warning|warn" >&2 || printf '%s\n' "$out" >&2
+    fi
+}
+
+function install_libs { # install_libs <ide_path> <sketchdir> [-v]
+    local ide_path=""
+    local sketchdir=""
+    local verbose=false
+
+    while [ -n "$1" ]; do
+        case "$1" in
+        -ai ) shift; ide_path=$1 ;;
+        -s  ) shift; sketchdir=$1 ;;
+        -v  ) verbose=true ;;
+        * )
+            echo "ERROR: Unknown argument: $1" >&2
+            echo "USAGE: install_libs -ai <ide_path> -s <sketchdir> [-v]" >&2
+            return 1
+            ;;
+        esac
+        shift
+    done
+
+    if [ -z "$ide_path" ]; then
+        echo "ERROR: IDE path not provided" >&2
+        echo "USAGE: install_libs -ai <ide_path> -s <sketchdir> [-v]" >&2
+        return 1
+    fi
+    if [ -z "$sketchdir" ]; then
+        echo "ERROR: Sketch directory not provided" >&2
+        echo "USAGE: install_libs -ai <ide_path> -s <sketchdir> [-v]" >&2
+        return 1
+    fi
+    if [ ! -f "$ide_path/arduino-cli" ]; then
+        echo "ERROR: arduino-cli not found at $ide_path/arduino-cli" >&2
+        return 1
+    fi
+
+    if [ ! -f "$sketchdir/ci.json" ]; then
+        [ "$verbose" = true ] && echo "No ci.json found in $sketchdir, skipping library installation"
+        return 0
+    fi
+    if ! jq -e . "$sketchdir/ci.json" >/dev/null 2>&1; then
+        echo "ERROR: $sketchdir/ci.json is not valid JSON" >&2
+        return 1
+    fi
+
+    local libs_type
+    libs_type=$(jq -r '.libs | type' "$sketchdir/ci.json" 2>/dev/null)
+    if [ -z "$libs_type" ] || [ "$libs_type" = "null" ]; then
+        [ "$verbose" = true ] && echo "No libs field found in ci.json, skipping library installation"
+        return 0
+    elif [ "$libs_type" != "array" ]; then
+        echo "ERROR: libs field in ci.json must be an array, found: $libs_type" >&2
+        return 1
+    fi
+
+    local libs_count
+    libs_count=$(jq -r '.libs | length' "$sketchdir/ci.json" 2>/dev/null)
+    if [ "$libs_count" -eq 0 ]; then
+        [ "$verbose" = true ] && echo "libs array is empty in ci.json, skipping library installation"
+        return 0
+    fi
+
+    echo "Installing $libs_count libraries from $sketchdir/ci.json"
+
+    local needs_unsafe=false
+    local original_unsafe_setting=""
+    local libs
+    libs=$(jq -r '.libs[]? // empty' "$sketchdir/ci.json")
+
+    # Detect any git-like URL (GitHub/GitLab/Bitbucket/self-hosted/ssh)
+    for lib in $libs; do
+        if is_git_like_url "$lib"; then
+            needs_unsafe=true
+            break
+        fi
+    done
+
+    if [ "$needs_unsafe" = true ]; then
+        [ "$verbose" = true ] && echo "Checking current unsafe install setting..."
+        original_unsafe_setting=$("$ide_path/arduino-cli" config get library.enable_unsafe_install 2>/dev/null || echo "false")
+        if [ "$original_unsafe_setting" = "false" ]; then
+            [ "$verbose" = true ] && echo "Enabling unsafe installs for Git URLs..."
+            "$ide_path/arduino-cli" config set library.enable_unsafe_install true >/dev/null 2>&1 || \
+                echo "WARNING: Failed to enable unsafe installs, Git URL installs may fail" >&2
+        else
+            [ "$verbose" = true ] && echo "Unsafe installs already enabled"
+        fi
+    fi
+
+    local rc=0 install_status=0 output=""
+    for lib in $libs; do
+        [ "$verbose" = true ] && echo "Processing library: $lib"
+
+        if is_git_like_url "$lib"; then
+            [ "$verbose" = true ] && echo "Installing library from git URL: $lib"
+            if [ "$verbose" = true ]; then
+                "$ide_path/arduino-cli" lib install --git-url "$lib"
+                install_status=$?
+            else
+                output=$("$ide_path/arduino-cli" lib install --git-url "$lib" 2>&1)
+                install_status=$?
+            fi
+        else
+            [ "$verbose" = true ] && echo "Installing library by name: $lib"
+            if [ "$verbose" = true ]; then
+                "$ide_path/arduino-cli" lib install "$lib"
+                install_status=$?
+            else
+                output=$("$ide_path/arduino-cli" lib install "$lib" 2>&1)
+                install_status=$?
+            fi
+        fi
+
+        # Treat "already installed"/"up to date" as success (idempotent)
+        if [ $install_status -ne 0 ] && echo "$output" | grep -qiE 'already installed|up to date'; then
+            install_status=0
+        fi
+
+        if [ "$verbose" != true ]; then
+            print_err_warnings "$install_status" "$output"
+        fi
+
+        if [ $install_status -ne 0 ]; then
+            echo "ERROR: Failed to install library: $lib" >&2
+            rc=$install_status
+            break
+        else
+            [ "$verbose" = true ] && echo "Successfully installed library: $lib"
+        fi
+    done
+
+    if [ "$needs_unsafe" = true ] && [ "$original_unsafe_setting" = "false" ]; then
+        [ "$verbose" = true ] && echo "Restoring original unsafe install setting..."
+        "$ide_path/arduino-cli" config set library.enable_unsafe_install false >/dev/null 2>&1 || true
+    fi
+
+    [ $rc -eq 0 ] && echo "Library installation completed"
+    return $rc
+}
+
+
 USAGE="
 USAGE: ${0} [command] [options]
 Available commands:
     count: Count sketches.
     build: Build a sketch.
     chunk_build: Build a chunk of sketches.
     check_requirements: Check if target meets sketch requirements.
+    install_libs: Install libraries from ci.json file.
 "
 
 cmd=$1
@@ -606,6 +769,8 @@ case "$cmd" in
     ;;
     "check_requirements") check_requirements "$@"
     ;;
+    "install_libs") install_libs "$@"
+    ;;
     *)
         echo "ERROR: Unrecognized command"
         echo "$USAGE"

.github/scripts/set_push_chunks.sh

@@ -115,14 +115,14 @@ function run_test {
         rm "$sketchdir"/diagram.json 2>/dev/null || true
 
         result=0
-        printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
-        bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
+        printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
+        bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
         printf "\n"
         if [ $result -ne 0 ]; then
             result=0
             printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i"
-            printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
-            bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
+            printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
+            bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
             printf "\n"
             if [ $result -ne 0 ]; then
                 printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i"

.github/scripts/sketch_utils.sh

@@ -5,6 +5,9 @@ on:
   repository_dispatch:
     types: [test-boards]
 
+permissions:
+  contents: read
+
 jobs:
   find-boards:
     runs-on: ubuntu-latest
@@ -36,10 +39,6 @@ jobs:
         with:
           ref: ${{ github.event.client_payload.branch }}
 
-      - run: npm install
-      - name: Setup jq
-        uses: dcarbone/install-jq-action@e397bd87438d72198f81efd21f876461183d383a # v3.0.1
-
       - id: set-test-chunks
         name: Set Chunks
         run: echo "test-chunks<<EOF" >> $GITHUB_OUTPUT

.github/scripts/tests_run.sh

@@ -8,6 +8,9 @@ on:
       - "libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino"
       - ".github/workflows/boards.yml"
 
+permissions:
+  contents: read
+
 env:
   # It's convenient to set variables for values used multiple times in the workflow
   GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
@@ -24,9 +27,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - name: Setup jq
-        uses: dcarbone/install-jq-action@e397bd87438d72198f81efd21f876461183d383a # v3.0.1
-
       - name: Get board name
         run: bash .github/scripts/find_new_boards.sh ${{ github.repository }} ${{github.base_ref}}