Skip to content

Commit 1788f9a

Browse files
amisskiijeffmaury
amisskii
and
jeffmaury
authored
chore(test): run ai-lab e2e tests on testing farm ci (#3872)
* chore(test): run ai-lab e2e tests on testing farm ci Signed-off-by: Anton Misskii <amisskii@redhat.com> * Apply suggestions from code review Co-authored-by: Jeff MAURY <jmaury@redhat.com> Signed-off-by: Anton Misskii <124462506+amisskii@users.noreply.github.com> * fix: use commit id Signed-off-by: Anton Misskii <amisskii@redhat.com> * fix: addressing review comments Signed-off-by: Anton Misskii <amisskii@redhat.com> --------- Signed-off-by: Anton Misskii <amisskii@redhat.com> Signed-off-by: Anton Misskii <124462506+amisskii@users.noreply.github.com> Co-authored-by: Jeff MAURY <jmaury@redhat.com>
1 parent 29dbe0a commit 1788f9a

File tree

8 files changed

+504
-0
lines changed

8 files changed

+504
-0
lines changed

‎.fmf/version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1

‎.github/workflows/e2e-main-tf.yaml‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# Copyright (C) 2025 Red Hat, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
# SPDX-License-Identifier: Apache-2.0
16+
17+
name: PD AI Lab E2E Nightly Testing Farm
18+
19+
on:
20+
schedule:
21+
- cron: '0 0 * * *'
22+
23+
workflow_dispatch:
24+
inputs:
25+
podman_version:
26+
default: 'latest'
27+
description: 'Podman version to install (e.g., "5.5.2", "5.6.0~rc1"). Use "latest" for stable or "nightly" for the latest development build.'
28+
type: string
29+
required: true
30+
npm_target:
31+
description: npm tests target
32+
type: choice
33+
default: 'e2e'
34+
options:
35+
- e2e
36+
- smoke
37+
- instructlab
38+
39+
jobs:
40+
pd-ai-lab-e2e-testing-farm:
41+
timeout-minutes: 90
42+
name: pd-e2e-testing-farm-ci
43+
runs-on: ubuntu-latest
44+
strategy:
45+
fail-fast: false
46+
matrix:
47+
fedora-version: ['Fedora-42', 'Fedora-43']
48+
steps:
49+
- name: Set the default env. variables
50+
env:
51+
DEFAULT_NPM_TARGET: 'smoke'
52+
DEFAULT_PODMAN_VERSION: 'latest'
53+
run: |
54+
echo "NPM_TARGET=${{ github.event.inputs.npm_target || env.DEFAULT_NPM_TARGET }}" >> $GITHUB_ENV
55+
echo "PODMAN_VERSION=${{ github.event.inputs.podman_version || env.DEFAULT_PODMAN_VERSION }}" >> $GITHUB_ENV
56+
57+
- name: Run Podman Desktop Playwright E2E tests on Testing Farm CI
58+
id: run-e2e-tf
59+
uses: sclorg/testing-farm-as-github-action@f3a39272a197ed060d5f786186b0e7a3ee7b0b4f #v4.1.0
60+
with:
61+
api_key: ${{ secrets.TF_TOKEN }}
62+
create_github_summary: "false"
63+
compose: ${{ matrix.fedora-version }}
64+
tmt_plan_filter: 'name:/tests/tmt/plans/ai-lab-e2e-plan/${{ env.NPM_TARGET }}'
65+
variables: COMPOSE=${{ matrix.fedora-version }};ARCH=x86_64;PODMAN_VERSION=${{ env.PODMAN_VERSION }}
66+
67+
- name: Extract Testing Farm work ID and base URL
68+
if: always()
69+
run: |
70+
TF_ARTIFACTS_URL="${{ steps.run-e2e-tf.outputs.test_log_url }}"
71+
TF_DEFAULT_JUNIT_DEFAULT="${TF_ARTIFACTS_URL}/results-junit.xml"
72+
curl -o results-junit.xml "$TF_DEFAULT_JUNIT_DEFAULT"
73+
74+
TF_WORK_ID=$(grep -o 'work-${{ env.NPM_TARGET }}[^/"]*' results-junit.xml | head -1)
75+
76+
echo "TF_WORK_ID=$TF_WORK_ID" >> $GITHUB_ENV
77+
echo "TF_ARTIFACTS_URL=$TF_ARTIFACTS_URL" >> $GITHUB_ENV
78+
79+
- name: Download Playwright JUnit report from Testing Farm
80+
if: always()
81+
run: |
82+
TF_PLAYWRIGHT_JUNIT_URL="${{ env.TF_ARTIFACTS_URL }}/${{ env.TF_WORK_ID }}/tests/tmt/plans/ai-lab-e2e-plan/${{ env.NPM_TARGET }}/execute/data/guest/default-0/tests/tmt/tests/${{ env.NPM_TARGET }}-test-1/data/junit-results.xml"
83+
curl -o junit-playwright-results.xml "$TF_PLAYWRIGHT_JUNIT_URL"
84+
85+
- name: Publish test report to PR
86+
if: always()
87+
uses: mikepenz/action-junit-report@5b7ee5a21e8674b695313d769f3cbdfd5d4d53a4 #v6.0.0
88+
with:
89+
fail_on_failure: true
90+
include_passed: true
91+
detailed_summary: true
92+
annotate_only: true
93+
require_tests: true
94+
report_paths: '**/junit-playwright-results.xml'
95+
96+
- name: Download test artifacts from Testing Farm
97+
if: failure()
98+
run: |
99+
mkdir -p results
100+
101+
TF_TEST_DATA_URL="${{ env.TF_ARTIFACTS_URL }}/${{ env.TF_WORK_ID }}/tests/tmt/plans/ai-lab-e2e-plan/${{ env.NPM_TARGET }}/execute/data/guest/default-0/tests/tmt/tests/${{ env.NPM_TARGET }}-test-1/data"
102+
TF_TRACES_URL="${TF_TEST_DATA_URL}/traces/"
103+
TF_VIDEOS_URL="${TF_TEST_DATA_URL}/videos/"
104+
105+
echo "Downloading traces"
106+
wget \
107+
--recursive \
108+
--no-parent \
109+
--no-host-directories \
110+
--cut-dirs=10 \
111+
--reject "index.html*" \
112+
--directory-prefix=results \
113+
"$TF_TRACES_URL"
114+
115+
echo "Downloading videos"
116+
wget \
117+
--recursive \
118+
--no-parent \
119+
--no-host-directories \
120+
--cut-dirs=10 \
121+
--reject "index.html*" \
122+
--directory-prefix=results \
123+
"$TF_VIDEOS_URL"
124+
125+
- name: Upload test artifacts
126+
if: always()
127+
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
128+
with:
129+
name: ai-lab-testing-farm-artifacts-${{ matrix.fedora-version }}
130+
path: |
131+
results/*
132+
**/junit-playwright-results.xml

‎tests/tmt/plans/ai-lab-e2e-plan.fmf‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# /**********************************************************************
2+
# Copyright (C) 2025 Red Hat, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
# ***********************************************************************/
18+
19+
summary: |
20+
This plan provisions a test environment for running Playwright E2E tests for the Podman Desktop AI Lab extension.
21+
It installs all required system dependencies, including GUI libraries,
22+
sets up Node.js with pnpm, and starts Podman rootless service.
23+
24+
discover:
25+
how: fmf
26+
27+
execute:
28+
how: tmt
29+
30+
provision:
31+
hardware:
32+
memory: ">= 16 GB"
33+
cpu:
34+
cores: ">= 4"
35+
disk:
36+
- size: ">= 20 GB"
37+
38+
prepare:
39+
- name: Install required packages
40+
how: shell
41+
script: |
42+
sudo dnf update -y
43+
sudo dnf install -y \
44+
git nodejs npm xorg-x11-server-Xvfb \
45+
nss nspr atk at-spi2-atk cups libXcomposite libXdamage libXfixes libXrandr cairo pango alsa-lib \
46+
gcc-c++ gtk3 passt jq
47+
sudo npm install -g pnpm
48+
echo "NodeJS version: $(node -v)"
49+
echo "npm version: $(npm -v)"
50+
echo "pnpm Version: $(pnpm --version)"
51+
52+
- name: Install podman
53+
how: shell
54+
script: |
55+
bash $TMT_TREE/tests/tmt/scripts/install-podman.sh
56+
57+
- name: Enable and start Podman rootless service
58+
how: shell
59+
script: |
60+
systemctl --user enable podman.socket || exit 1
61+
systemctl --user start podman.socket || exit 1
62+
systemctl --user is-active podman.socket || exit 1
63+
64+
/e2e:
65+
summary: Execute Playwright E2E tests.
66+
discover+:
67+
filter: 'tag:e2e'
68+
69+
/smoke:
70+
summary: Execute smoke Playwright E2E tests.
71+
discover+:
72+
filter: 'tag:smoke'
73+
74+
/instructlab:
75+
summary: Execute instructlab Playwright E2E tests.
76+
discover+:
77+
filter: 'tag:instructlab'

‎tests/tmt/scripts/create-results.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# /**********************************************************************
2+
# Copyright (C) 2025 Red Hat, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
# ***********************************************************************/
18+
19+
#!/bin/bash
20+
set -euo pipefail
21+
22+
PLAYWRIGHT_JUNIT_DIR="$TMT_TREE/tests/playwright/output/"
23+
PLAYWRIGHT_TRACE_VIDEOS_DIR="$TMT_TREE/tests/playwright/tests/playwright/output/"
24+
25+
cd "$TMT_TEST_DATA"
26+
27+
if [ -f "$PLAYWRIGHT_JUNIT_DIR/junit-results.xml" ]; then
28+
cp "$PLAYWRIGHT_JUNIT_DIR/junit-results.xml" .
29+
else
30+
echo "Error: junit-results.xml not found"
31+
exit 1
32+
fi
33+
34+
if [ "$1" -eq 0 ]; then
35+
cat <<EOF > ./results.yaml
36+
- name: /tests/$2
37+
result: pass
38+
note:
39+
- "Playwright end-to-end tests completed successfully."
40+
log:
41+
- ../output.txt
42+
- junit-results.xml
43+
EOF
44+
45+
elif [ "$1" -eq 255 ]; then
46+
47+
if [ -d "$PLAYWRIGHT_TRACE_VIDEOS_DIR/traces" ]; then
48+
cp -r "$PLAYWRIGHT_TRACE_VIDEOS_DIR/traces" .
49+
else
50+
echo "Warning: traces directory does not exist" >&2
51+
fi
52+
53+
if [ -d "$PLAYWRIGHT_TRACE_VIDEOS_DIR/videos" ]; then
54+
cp -r "$PLAYWRIGHT_TRACE_VIDEOS_DIR/videos" .
55+
else
56+
echo "Warning: videos directory does not exist" >&2
57+
fi
58+
59+
cat <<EOF > ./results.yaml
60+
- name: /tests/$2
61+
result: fail
62+
note:
63+
- "Playwright tests failed."
64+
log:
65+
- ../output.txt
66+
- junit-results.xml
67+
- videos
68+
- traces
69+
EOF
70+
71+
else
72+
echo "Warning: Unexpected exit code: $1, treating as failure" >&2
73+
cat <<EOF > ./results.yaml
74+
- name: /tests/$2
75+
result: fail
76+
note:
77+
- "Tests failed with unexpected exit code: $1"
78+
log:
79+
- ../output.txt
80+
EOF
81+
fi
82+
exit 0

‎tests/tmt/scripts/install-podman.sh‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# /**********************************************************************
2+
# Copyright (C) 2025 Red Hat, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# SPDX-License-Identifier: Apache-2.0
17+
# ***********************************************************************/
18+
19+
#!/bin/bash
20+
set -euo pipefail
21+
22+
# Uninstall a preinstalled Podman version to ensure the desired version will be installed.
23+
sudo dnf remove -y podman
24+
25+
# Construct the download URL for the specific Podman version.
26+
COMPOSE_VERSION="fc$(echo "$COMPOSE" | cut -d'-' -f2)"
27+
28+
# Install Podman based on the requested version:
29+
# "nightly": latest nightly build from rhcontainerbot/podman-next COPR repository
30+
if [[ "$PODMAN_VERSION" == "nightly" ]]; then
31+
sudo dnf copr enable -y rhcontainerbot/podman-next
32+
sudo dnf install -y podman --disablerepo=testing-farm-tag-repository
33+
PODMAN_VERSION="$(dnf --quiet \
34+
--repofrompath=podman-next,https://download.copr.fedorainfracloud.org/results/rhcontainerbot/podman-next/fedora-$(rpm -E %fedora)/${ARCH}/ \
35+
list --showduplicates podman 2>/dev/null | grep dev | tail -n1 | cut -d':' -f2 | cut -d'-' -f1 )"
36+
else
37+
# For "latest" or specific version, fetch version if needed and install from RPM
38+
if [[ "$PODMAN_VERSION" == "latest" ]]; then
39+
PODMAN_VERSION="$(curl -s https://api.github.com/repos/containers/podman/releases/latest | jq -r .tag_name | sed 's/^v//')"
40+
fi
41+
CUSTOM_PODMAN_URL="https://kojipkgs.fedoraproject.org//packages/podman/${PODMAN_VERSION}/1.${COMPOSE_VERSION}/${ARCH}/podman-${PODMAN_VERSION}-1.${COMPOSE_VERSION}.${ARCH}.rpm"
42+
curl -Lo podman.rpm "$CUSTOM_PODMAN_URL"
43+
if [[ $? -ne 0 ]]; then
44+
echo "Error: Failed to download Podman RPM from $CUSTOM_PODMAN_URL"
45+
exit 1
46+
fi
47+
if [[ ! -s podman.rpm ]]; then
48+
echo "Error: Downloaded Podman RPM file is missing or empty."
49+
rm -f podman.rpm
50+
exit 1
51+
fi
52+
sudo dnf install -y ./podman.rpm
53+
rm -f podman.rpm
54+
fi
55+
56+
# Verify that the installed Podman version matches the expected version.
57+
INSTALLED_PODMAN_VERSION="$(podman --version | cut -d' ' -f3)"
58+
NORMALIZED_PODMAN_VERSION="${PODMAN_VERSION//\~/-}"
59+
60+
if [[ "$INSTALLED_PODMAN_VERSION" != "$NORMALIZED_PODMAN_VERSION" ]]; then
61+
echo "Podman version mismatch: expected $NORMALIZED_PODMAN_VERSION but got $INSTALLED_PODMAN_VERSION"
62+
exit 1
63+
fi
64+
65+
echo "Podman installed successfully: $INSTALLED_PODMAN_VERSION"

0 commit comments

Comments
 (0)