feat(ci): implement caching mechanism for documentation binaries and streamline preparation process in CI workflows

Author: JakubAndrysek

.github/PULL_REQUEST_TEMPLATE.md

@@ -118,6 +118,12 @@ def parse_args(argv):
         action="store_true",
         help="Clean up docs binaries directory and exit",
     )
+    p.add_argument(
+        "-p",
+        dest="prepare_cached",
+        action="store_true",
+        help="Prepare cached binaries from docs_binaries/ to docs/_static/binaries/",
+    )
     p.add_argument(
         "-d",
         dest="generate_diagrams",
@@ -182,6 +188,55 @@ def cleanup_binaries():
     print("Cleanup completed")
 
 
+def prepare_cached_binaries():
+    """Copy cached binaries from docs_binaries/ to docs/_static/binaries/"""
+    import re
+    
+    docs_binaries = Path("docs_binaries")
+    if not docs_binaries.exists():
+        print("WARNING: No docs_binaries directory found, continuing without binaries.")
+        return 0
+    
+    DOCS_BINARIES_DIR.mkdir(parents=True, exist_ok=True)
+    
+    bin_count = 0
+    # Find all .merged.bin files
+    for bin_file in docs_binaries.rglob("*.merged.bin"):
+        bin_name = bin_file.name
+        # Extract sketch name (remove .ino.merged.bin extension)
+        sketch_name = bin_name.replace(".ino.merged.bin", "")
+        
+        # Extract target from path
+        target_match = re.search(r'/(esp32[a-z0-9]*)/', str(bin_file))
+        if not target_match:
+            continue
+        target = target_match.group(1)
+        
+        # Search for the sketch in libraries directory
+        sketch_ino_files = list(Path('libraries').rglob(f'{sketch_name}.ino'))
+        if not sketch_ino_files:
+            continue
+        
+        sketch_ino_file = sketch_ino_files[0]
+        sketch_dir = sketch_ino_file.parent
+        relative_path = str(sketch_dir).replace('libraries/', '', 1)
+        
+        # Create output directory
+        output_dir = DOCS_BINARIES_DIR / "libraries" / relative_path / target
+        output_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Copy the .merged.bin file
+        shutil.copy(bin_file, output_dir / bin_name)
+        bin_count += 1
+        
+        # Also copy ci.json if it exists in the same directory
+        ci_json_file = bin_file.parent / "ci.json"
+        if ci_json_file.exists():
+            shutil.copy(ci_json_file, output_dir / "ci.json")
+    
+    return bin_count
+
+
 def find_examples_with_upload_binary():
     res = []
     for ino in Path('.').rglob('*.ino'):
@@ -311,6 +366,16 @@ def main(argv):
     if args.cleanup:
         cleanup_binaries()
         return
+    if args.prepare_cached:
+        bin_count = prepare_cached_binaries()
+        print(f'\nPrepared {bin_count} binaries for documentation')
+        # Show project names
+        print('\nProjects ready for documentation:')
+        merged_bins = list(DOCS_BINARIES_DIR.rglob('*.ino.merged.bin'))
+        project_names = sorted(set(f.name.replace('.ino.merged.bin', '') for f in merged_bins))
+        for name in project_names:
+            print(f'  {name}')
+        return
     validate_prerequisites(args)
     GENERATE_DIAGRAMS = args.generate_diagrams
     GENERATE_LAUNCHPAD_CONFIG = args.generate_launchpad_config

.github/scripts/docs_build_examples.py

@@ -1,15 +1,15 @@
 name: Documentation Build and Deploy CI
 
 on:
-  # # Wait for Compilation Tests workflow to complete (uses macOS binaries)
-  # workflow_run:
-  #   workflows: ["Compilation Tests"]
-  #   types:
-  #     - completed
-  #   branches:
-  #     - master
-  #     - release/v2.x
-  #     - wokwi-embed-launchpad
+  # Wait for Compilation Tests workflow to complete (uses macOS binaries)
+  workflow_run:
+    workflows: ["Compilation Tests"]
+    types:
+      - completed
+    branches:
+      - master
+      - release/v2.x
+      - wokwi-embed-launchpad
   # Manual trigger for testing docs changes without waiting for compilation
   workflow_dispatch:
   # Also run on direct pushes to docs or workflow files (for testing)
@@ -57,57 +57,11 @@ jobs:
           path: docs_binaries
           lookup-only: false
 
-      - name: Copy cached binaries to docs/_static/binaries/
-        run: |
-          mkdir -p docs/_static/binaries
-
-          if [ ! -d "docs_binaries" ]; then
-            echo "WARNING: No docs_binaries directory found, continuing without binaries."
-            exit 0
-          fi
-
-          bin_count=0
-          
-          while IFS= read -r bin_file; do
-            bin_name=$(basename "$bin_file")
-            sketch_name="${bin_name%.ino.merged.bin}"
-
-            if [[ "$bin_file" =~ /(esp32[a-z0-9]*)/ ]]; then
-              target="${BASH_REMATCH[1]}"
-            else
-              continue
-            fi
-
-            sketch_ino_file=$(find libraries -type f -name "${sketch_name}.ino" 2>/dev/null | head -1)
-            if [ -z "$sketch_ino_file" ]; then
-              continue
-            fi
-
-            sketch_dir=$(dirname "$sketch_ino_file")
-            relative_path="${sketch_dir#libraries/}"
-            output_dir="docs/_static/binaries/libraries/${relative_path}/${target}"
-            mkdir -p "$output_dir"
-
-            cp "$bin_file" "$output_dir/"
-            bin_count=$((bin_count + 1))
-
-            ci_json_file=$(dirname "$bin_file")/ci.json
-            if [ -f "$ci_json_file" ]; then
-              cp "$ci_json_file" "$output_dir/"
-            fi
-          done < <(find docs_binaries -type f -name "*.merged.bin")
-
-      - name: Cleanup Binaries
+      - name: Prepare cached binaries for documentation
         run: |
+          python3 .github/scripts/docs_build_examples.py -p
           python3 .github/scripts/docs_build_examples.py -c
 
-      - name: Show processed projects
-        run: |
-          echo "Projects ready for documentation:"
-          find docs/_static/binaries/libraries -type f -name "*.ino.merged.bin" 2>/dev/null | while read f; do
-            basename "$f" .ino.merged.bin
-          done | sort -u
-
       - name: Build
         run: |
           sudo apt update

.github/workflows/docs_build.yml

@@ -2,9 +2,10 @@ name: Documentation Build and Production Deploy CI
 
 on:
   workflow_run:
-    workflows: ["ESP32 Arduino Release"]
+    workflows: ["ESP32 Arduino Release", "Compilation Tests"]
     types:
       - completed
+  workflow_dispatch:
   push:
     branches:
       - release/v2.x
@@ -40,6 +41,21 @@ jobs:
           cache: "pip"
           python-version: "3.10"
 
+      - name: Restore docs binaries from cache (macOS builds)
+        id: cache-restore
+        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          key: docs-binaries-macos-${{ github.run_id }}
+          restore-keys: |
+            docs-binaries-macos-
+          path: docs_binaries
+          lookup-only: false
+
+      - name: Prepare cached binaries for documentation
+        run: |
+          python3 .github/scripts/docs_build_examples.py -p
+          python3 .github/scripts/docs_build_examples.py -c
+
       - name: Deploy Documentation
         env:
           # Deploy to production server