Verify bundled sources using CI - XSSE, timelib, xxHash, UCD

.github/actions/verify-generated-files/action.yml

@@ -13,4 +13,5 @@ runs:
         ext/tokenizer/tokenizer_data_gen.php
         build/gen_stub.php -f --generate-optimizer-info --verify
         ext/phar/makestub.php
+        .github/scripts/download-bundled/make-workflow-file.php
         .github/scripts/test-directory-unchanged.sh .

.github/scripts/download-bundled/.gitignore

@@ -0,0 +1 @@
+!*.patch

.github/scripts/download-bundled/boost-context.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+set -ex
+cd "$(dirname "$0")/../../.."
+
+tmp_dir=/tmp/php-src-download-bundled/boost-context
+rm -rf "$tmp_dir"
+
+revision=refs/tags/boost-1.86.0
+
+git clone --depth 1 --revision="$revision" https://github.com/boostorg/context.git "$tmp_dir"
+
+rm -rf Zend/asm
+cp -R "$tmp_dir"/src/asm Zend/asm
+
+cd Zend/asm
+
+# remove unneeded files
+rm jump_arm_aapcs_pe_armasm.asm
+rm jump_i386_ms_pe_clang_gas.S
+rm jump_i386_ms_pe_gas.asm
+rm jump_i386_x86_64_sysv_macho_gas.S
+rm jump_ppc32_ppc64_sysv_macho_gas.S
+rm jump_x86_64_ms_pe_clang_gas.S
+rm make_arm_aapcs_pe_armasm.asm
+rm make_i386_ms_pe_clang_gas.S
+rm make_i386_ms_pe_gas.asm
+rm make_i386_x86_64_sysv_macho_gas.S
+rm make_ppc32_ppc64_sysv_macho_gas.S
+rm make_x86_64_ms_pe_clang_gas.S
+rm ontop_*.S
+rm ontop_*.asm
+rm tail_ontop_ppc32_sysv.cpp
+
+# move renamed files
+# GH-13896 introduced these 2 files named as .S but since https://github.com/boostorg/context/pull/265 they are named as .asm
+mv jump_x86_64_ms_pe_gas.asm jump_x86_64_ms_pe_gas.S
+mv make_x86_64_ms_pe_gas.asm make_x86_64_ms_pe_gas.S
+
+# add extra files
+git restore LICENSE
+git restore save_xmm_x86_64_ms_masm.asm # added in GH-18352, not an upstream boost.context file

.github/scripts/download-bundled/ir.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+set -ex
+cd "$(dirname "$0")/../../.."
+
+tmp_dir=/tmp/php-src-download-bundled/ir
+rm -rf "$tmp_dir"
+
+revision=5a81104e650ebd7ac24eb63d4dff67db723a5278
+
+git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir"
+
+rm -rf ext/opcache/jit/ir
+cp -R "$tmp_dir" ext/opcache/jit/ir
+
+cd ext/opcache/jit/ir
+
+# remove unneeded files
+rm -r .git
+rm -r .github
+rm -r bench
+rm -r examples
+rm -r tests
+rm -r tools
+rm README.md
+rm TODO
+rm ir.g
+rm ir_cpuinfo.c
+rm ir_emit_c.c
+rm ir_emit_llvm.c
+rm ir_load.c
+rm ir_load_llvm.c
+rm ir_main.c
+rm ir_mem2ssa.c
+
+# add extra files
+git restore README

.github/scripts/download-bundled/make-workflow-file.php

@@ -0,0 +1,212 @@
+#!/usr/bin/env php
+<?php
+
+declare(strict_types=1);
+
+namespace Phpsrc\Ci\DownloadBundled;
+
+$bundles = [
+    new Bundle('boost.context', ['Zend/asm']),
+    new Bundle('XSSE', ['Zend/zend_simd.h']),
+    new Bundle('timelib', ['ext/date/lib']),
+    new Bundle('xxHash', ['ext/hash/xxhash']),
+    // new Bundle('Unicode Character Database', ['ext/mbstring']),
+    new Bundle('IR', ['ext/opcache/jit/ir']),
+    new Bundle('PCRE2', ['ext/pcre/pcre2lib']),
+    new Bundle('uriparser', ['ext/uri/uriparser']),
+];
+
+class Bundle
+{
+    /**
+     * @param list<string> $directories
+     */
+    public function __construct(
+        public string $name,
+        public array $directories
+    ) {}
+
+    public function getNameForPath(): string
+    {
+        return preg_replace('~\W+~', '-', strtolower($this->name));
+    }
+}
+
+class Generator
+{
+    /**
+     * @param list<Bundle> $bundles
+     */
+    public function __construct(
+        public array $bundles
+    ) {}
+
+    protected function getRepoDirectory(): string
+    {
+        return dirname(__DIR__, 3);
+    }
+
+    protected function indentString(string $value, int $levels, bool $inclFirstLine): string
+    {
+        return preg_replace(
+            '~' . ($inclFirstLine ? '^|' : '') . '(?<=\n)~',
+            str_repeat('  ', $levels),
+            $value
+        );
+    }
+
+    /**
+     * @param mixed $data
+     */
+    protected function encodeYml($data): string
+    {
+        if (is_array($data)) {
+            $isList = array_is_list($data);
+            $resParts = [];
+            foreach ($data as $k => $v) {
+                $kEncoded = $isList
+                    ? '-'
+                    : $this->encodeYml($k) . ':';
+                $vEncoded = $this->encodeYml($v);
+
+                $resParts[] = $kEncoded
+                    . (!$isList && is_array($v) && $v !== [] ? "\n  " : ' ')
+                    . (is_array($v) ? $this->indentString($vEncoded, 1, false) : $vEncoded);
+            }
+
+            return implode("\n", $resParts);
+        }
+
+        if (preg_match('~^(\w+|\$\{\{[^\}]+\}\})$~', $data)) {
+            return $data;
+        }
+
+        return strpos($data, "\n") !== false
+            ? '|' . "\n" . $this->indentString($data, 1, true)
+            : '\'' . str_replace('\'', '\'\'', $data) . '\'';
+    }
+
+    public function makeWorkflowFile(): void
+    {
+        $content = <<<'EOD'
+            name: Verify Bundled Files
+
+            on:
+              push:
+                paths: &paths
+                  %paths%
+              pull_request:
+                paths: *paths
+              schedule:
+                - cron: "0 1 * * *"
+              workflow_dispatch: ~
+
+            permissions:
+              contents: read
+
+            jobs:
+              VERIFY_BUNDLED_FILES:
+                name: Verify Bundled Files
+                runs-on: ubuntu-24.04
+                steps:
+                  - name: git checkout
+                    uses: actions/checkout@v5
+
+                  - name: Detect changed files
+                    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}
+                    uses: dorny/paths-filter@v3
+                    id: changes
+                    with:
+                      filters: %filters%
+
+                  %steps%
+
+            EOD;
+
+        $paths = [
+            '.github/scripts/download-bundled/**',
+        ];
+        foreach ($this->bundles as $bundle) {
+            foreach ($this->makeDornyPathsFilterFilters($bundle) as $p) {
+                if (str_starts_with($p, '.github/scripts/download-bundled/')) {
+                    continue;
+                }
+
+                $paths[] = $p;
+            }
+        }
+        $content = str_replace('%paths%', $this->indentString($this->encodeYml($paths), 3, false), $content);
+
+        $filters = [];
+        foreach ($this->bundles as $bundle) {
+            $filters[$bundle->getNameForPath()] = $this->makeDornyPathsFilterFilters($bundle);
+        }
+        $content = str_replace('%filters%', $this->indentString($this->encodeYml($this->encodeYml($filters)), 5, false), $content);
+
+        $steps = [];
+        foreach ($this->bundles as $bundle) {
+            $steps[] = [
+                'name' => $bundle->name,
+                'if' => '${{ !cancelled() && (steps.changes.outputs.pcre2 == \'true\' || github.event_name == \'schedule\' || github.event_name == \'workflow_dispatch\') }}',
+                'run' => implode("\n", [
+                    'echo "::group::Download"',
+                    '.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.sh',
+                    'echo "::endgroup::"',
+                    'echo "::group::Verify files"',
+                    ...array_map(static fn ($v) => '.github/scripts/test-directory-unchanged.sh \'' . $v . '\'', $bundle->directories),
+                    'echo "::endgroup::"',
+                ]),
+            ];
+        }
+        $content = str_replace('%steps%', $this->indentString($this->encodeYml($steps), 3, false), $content);
+
+        file_put_contents($this->getRepoDirectory() . '/.github/workflows/verify-bundled-files.yml', $content);
+    }
+
+    protected function makeDornyPathsFilterFilters(Bundle $bundle): array
+    {
+        return [
+            '.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.*',
+            ...array_map(fn ($v) => is_file($this->getRepoDirectory() . '/' . $v) ? $v : $v . '/**', $bundle->directories),
+        ];
+    }
+
+    public function makeDownloadScriptHeaders(): void
+    {
+        foreach ($this->bundles as $bundle) {
+            $this->makeDownloadScriptHeader($bundle);
+        }
+    }
+
+    protected function makeDownloadScriptHeader(Bundle $bundle): void
+    {
+        $scriptPath = $this->getRepoDirectory() . '/.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.sh';
+
+        $content = !file_exists($scriptPath)
+            ? "# TODO\n"
+            : file_get_contents($scriptPath);
+
+        $header = <<<'EOD'
+            #!/bin/sh
+            set -ex
+            cd "$(dirname "$0")/../../.."
+
+            tmp_dir=%tmp_dir%
+            rm -rf "$tmp_dir"
+
+
+            EOD;
+
+        $header = str_replace('%tmp_dir%', '/tmp/php-src-download-bundled/' . $bundle->getNameForPath(), $header);
+
+        if (!str_starts_with($content, $header)) {
+            $content = $header . $content;
+        }
+
+        file_put_contents($scriptPath, $content);
+    }
+}
+
+$generator = new Generator($bundles);
+$generator->makeWorkflowFile();
+$generator->makeDownloadScriptHeaders();

.github/scripts/download-bundled/pcre2.sh

@@ -2,12 +2,15 @@
 set -ex
 cd "$(dirname "$0")/../../.."
 
+tmp_dir=/tmp/php-src-download-bundled/pcre2
+rm -rf "$tmp_dir"
+
 revision=refs/tags/pcre2-10.44
 
-git clone --depth 1 --recurse-submodules --revision="$revision" https://github.com/PCRE2Project/pcre2.git /tmp/php-src-bundled/pcre2
+git clone --depth 1 --recurse-submodules --revision="$revision" https://github.com/PCRE2Project/pcre2.git "$tmp_dir"
 
 rm -rf ext/pcre/pcre2lib
-cp -R /tmp/php-src-bundled/pcre2/src ext/pcre/pcre2lib
+cp -R "$tmp_dir"/src ext/pcre/pcre2lib
 
 cd ext/pcre/pcre2lib
 

.github/scripts/download-bundled/timelib.parse_date.patch

@@ -0,0 +1,12 @@
+diff --git a/ext/date/lib/parse_date.re b/ext/date/lib/parse_date.re
+index 2f05f03..893ce16 100644
+--- a/ext/date/lib/parse_date.re
++++ b/ext/date/lib/parse_date.re
+@@ -1281,7 +1281,6 @@ weekdayof        = (reltextnumber|reltexttext) space (dayfulls|dayfull|dayabbr) 
+ 		DEBUG_OUTPUT("firstdayof | lastdayof");
+ 		TIMELIB_INIT;
+ 		TIMELIB_HAVE_RELATIVE();
+-		TIMELIB_UNHAVE_TIME(); // Don't merge into PHP
+
+ 		/* skip "last day of" or "first day of" */
+ 		if (*ptr == 'l' || *ptr == 'L') {

.github/scripts/download-bundled/timelib.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+set -ex
+cd "$(dirname "$0")/../../.."
+
+tmp_dir=/tmp/php-src-download-bundled/timelib
+rm -rf "$tmp_dir"
+
+revision=refs/tags/2022.14
+
+git clone --depth 1 --revision="$revision" https://github.com/derickr/timelib.git "$tmp_dir"
+
+rm -rf ext/date/lib
+cp -R "$tmp_dir" ext/date/lib
+
+cd ext/date/lib
+
+# remove unneeded files
+rm -r docs
+rm -r tests
+rm -r zones
+rm .gitignore
+rm gettzmapping.php
+rm parse_zoneinfo.c
+rm win_dirent.h
+
+# add extra files
+rm -r .git
+git restore parse_date.c
+git restore parse_iso_intervals.c
+
+# patch customized files
+git apply -v ../../../.github/scripts/download-bundled/timelib.parse_date.patch

.github/scripts/download-bundled/unicode-character-database.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -ex
+cd "$(dirname "$0")/../../.."
+
+tmp_dir=/tmp/php-src-download-bundled/unicode-character-database
+rm -rf "$tmp_dir"
+
+version=17.0.0
+
+mkdir -p "$tmp_dir"
+curl --fail https://www.unicode.org/Public/$version/ucd/UCD.zip -o "$tmp_dir/data.zip"
+unzip "$tmp_dir/data.zip" -d "$tmp_dir"
+
+cd ext/mbstring
+
+rm libmbfl/mbfl/eaw_table.h
+rm unicode_data.h
+
+./ucgendat/ucgendat.php "$tmp_dir"

.github/scripts/download-bundled/uriparser.config.patch

@@ -0,0 +1,14 @@
+diff --git a/ext/uri/uriparser/src/UriConfig.h b/ext/uri/uriparser/src/UriConfig.h
+index b9a85a8..ab78b96 100644
+--- a/ext/uri/uriparser/src/UriConfig.h
++++ b/ext/uri/uriparser/src/UriConfig.h
+@@ -41,7 +41,9 @@
+
+ #  define PACKAGE_VERSION "@PROJECT_VERSION@"
+
++/*
+ #cmakedefine HAVE_WPRINTF
+ #cmakedefine HAVE_REALLOCARRAY
++*/
+
+ #endif /* !defined(URI_CONFIG_H) */