chore(ci): Combine code coverage in CI before uploading to codecov

.github/workflows/ci.yml

@@ -338,33 +338,46 @@ jobs:
         env:
           TOOLKIT_VERSION: ${{ steps.version.outputs.VERSION }}
           PYTHON_VERSION: ${{ matrix.python-version }}
-          COVERAGE_FILE: coverage/.coverage.${{ matrix.python-version }}
         run: |
           set -euo pipefail
 
-          # Create coverage directory
-          mkdir -p coverage
-
-          # Run tests with coverage
           poetry run pytest tests/unit \
             --cov=deepnote_toolkit \
             --cov=installer \
             --cov=deepnote_core \
             --cov-branch \
             --cov-report=term-missing:skip-covered \
-            --cov-report=xml:coverage/coverage-${PYTHON_VERSION}.xml \
-            --cov-report=json:coverage/coverage-${PYTHON_VERSION}.json \
             --junitxml=junit.xml \
             -o junit_family=legacy
 
-          # Check if coverage file was generated
-          if [ -f "coverage/.coverage.${PYTHON_VERSION}" ]; then
+          # Debug: Check what coverage files exist
+          echo "=== Coverage files after pytest ==="
+          ls -la .coverage* 2>/dev/null || echo "No .coverage* files found"
+          ls -la .coverage.* 2>/dev/null || echo "No .coverage.* files found"
+
+          # Coverage parallel mode may create .coverage.machine.pid files
+          # Check for both .coverage and .coverage.* patterns
+          if [ -f ".coverage" ]; then
+            # Single .coverage file exists (parallel mode auto-combined or not used)
+            mv .coverage .coverage.${PYTHON_VERSION}
+            echo "Renamed .coverage to .coverage.${PYTHON_VERSION}"
             echo "coverage_generated=true" >> $GITHUB_OUTPUT
-            echo "Coverage files found:"
-            ls -la coverage/
           else
-            echo "coverage_generated=false" >> $GITHUB_OUTPUT
-            echo "Warning: No coverage file generated"
+            # Check for parallel coverage files
+            shopt -s nullglob
+            coverage_files=(.coverage.*)
+
+            if [ ${#coverage_files[@]} -gt 0 ]; then
+              echo "Found ${#coverage_files[@]} parallel coverage files, combining..."
+              poetry run coverage combine
+              mv .coverage .coverage.${PYTHON_VERSION}
+              echo "Combined and saved as .coverage.${PYTHON_VERSION}"
+              echo "coverage_generated=true" >> $GITHUB_OUTPUT
+            else
+              echo "Error: No coverage data files found"
+              echo "coverage_generated=false" >> $GITHUB_OUTPUT
+              exit 1
+            fi
           fi
 
       - name: Per-version coverage summary
@@ -373,7 +386,7 @@ jobs:
           PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
           echo "## Python ${PYTHON_VERSION} Coverage" >> $GITHUB_STEP_SUMMARY
-          poetry run coverage report --data-file=coverage/.coverage.${PYTHON_VERSION} --format=markdown >> $GITHUB_STEP_SUMMARY
+          poetry run coverage report --data-file=.coverage.${PYTHON_VERSION} --format=markdown >> $GITHUB_STEP_SUMMARY
 
       - name: Upload test results to Codecov (these are results not coverage reports)
         if: ${{ !cancelled() }}
@@ -382,14 +395,84 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: python-${{ matrix.python-version }}
 
-      - name: Upload coverage to Codecov
+      - name: Upload coverage artifacts
+        if: steps.test-unit.outputs.coverage_generated == 'true'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: coverage-${{ matrix.python-version }}
+          path: .coverage.${{ matrix.python-version }}
+          retention-days: 1
+
+  coverage-combine:
+    name: Combine and Upload Coverage
+    runs-on: ubuntu-latest
+    needs: tests-unit
+    if: always()
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: '3.11'
+
+      - name: Install coverage
+        run: pip install coverage[toml]
+
+      - name: Download all coverage artifacts
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+        continue-on-error: true
+        with:
+          pattern: coverage-*
+          path: coverage-data/
+          merge-multiple: true
+
+      - name: Combine coverage files
+        id: combine
+        run: |
+          set -euo pipefail
+
+          shopt -s nullglob
+          coverage_files=(coverage-data/.coverage.*)
+
+          if [ ${#coverage_files[@]} -eq 0 ]; then
+            echo "No coverage files to combine (tests may have failed)"
+            echo "success=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          echo "Combining ${#coverage_files[@]} coverage file(s)"
+          cd coverage-data
+
+          coverage combine
+          coverage xml -o coverage.xml
+          coverage report
+
+          echo "## Combined Coverage Report" >> $GITHUB_STEP_SUMMARY
+          coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+          echo "success=true" >> $GITHUB_OUTPUT
+
+      - name: Upload combined coverage to Codecov
+        if: steps.combine.outputs.success == 'true'
         uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          slug: deepnote/deepnote-toolkit
-          files: ./coverage/coverage-${{ matrix.python-version }}.xml
+          slug: ${{ github.repository }}
+          files: ./coverage-data/coverage.xml
+          flags: combined
           fail_ci_if_error: ${{ github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push' }}
 
+      - name: Upload combined coverage report
+        if: steps.combine.outputs.success == 'true'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: coverage-combined-report
+          path: coverage-data/coverage.xml
+          retention-days: 30
+
   audit-prod:
     name: Audit - Production
     runs-on: ubuntu-latest