Skip to content

Commit 9fb036e

Browse files
PhaowPhaow
committed
Enable the VolumeGroupSnapshot tests
Signed-off-by: Penghao <pewang@redhat.com>
1 parent 5334430 commit 9fb036e

File tree

1 file changed

+170
-58
lines changed

1 file changed

+170
-58
lines changed

‎prow.sh‎

Lines changed: 170 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,8 @@ csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr
140140
# for that, otherwise the latest stable release for which we then
141141
# list the officially supported images below.
142142
kind_version_default () {
143-
case "${CSI_PROW_KUBERNETES_VERSION}" in
144-
latest|master)
143+
case "${CSI_PROW_KUBERNETES_VERSION},$(basename "${REPO_DIR}")" in
144+
latest,*|master,*|*,kubernetes)
145145
echo main;;
146146
*)
147147
echo v0.25.0;;
@@ -381,6 +381,9 @@ default_csi_snapshotter_version () {
381381
}
382382
configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external-snapshotter version tag"
383383

384+
# Enable installing VolumeGroupSnapshot CRDs (off by default, can be set to true in prow jobs)
385+
configvar CSI_PROW_ENABLE_GROUP_SNAPSHOT "false" "Enable the VolumeGroupSnapshot tests"
386+
384387
# Some tests are known to be unusable in a KinD cluster. For example,
385388
# stopping kubelet with "ssh <node IP> systemctl stop kubelet" simply
386389
# doesn't work. Such tests should be written in a way that they verify
@@ -552,6 +555,14 @@ list_gates () (
552555
# with https://kind.sigs.k8s.io/docs/user/configuration/#runtime-config
553556
list_api_groups () (
554557
set -f; IFS=','
558+
559+
# If the volumegroupsnapshot gate is enabled, output required API groups
560+
if ${CSI_PROW_ENABLE_GROUP_SNAPSHOT}; then
561+
echo ' "api/ga": "true"'
562+
echo ' "storage.k8s.io/v1alpha1": "true"'
563+
echo ' "storage.k8s.io/v1beta1": "true"'
564+
fi
565+
555566
# Ignore: Double quote to prevent globbing and word splitting.
556567
# shellcheck disable=SC2086
557568
set -- $1
@@ -617,19 +628,29 @@ start_cluster () {
617628
if ! [ "$image" ]; then
618629
if ! ${csi_prow_kind_have_kubernetes}; then
619630
local version="${CSI_PROW_KUBERNETES_VERSION}"
620-
if [ "$version" = "latest" ]; then
621-
version=master
631+
# Detect if running inside k/k repo
632+
if [[ $(basename "${REPO_DIR}") == "kubernetes" ]]; then
633+
echo "Using Kubernetes source from CI checkout ..."
634+
ksrc="${REPO_DIR}"
635+
version="$(git -C "$ksrc" rev-parse HEAD)"
636+
else
637+
echo "Cloning Kubernetes..."
638+
[ "$version" = "latest" ] && version=master
639+
ksrc="${CSI_PROW_WORK}/src/kubernetes"
640+
git_clone https://github.com/kubernetes/kubernetes "$ksrc" "$(version_to_git "$version")" \
641+
|| die "checking out Kubernetes $version failed"
622642
fi
623-
git_clone https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$(version_to_git "$version")" || die "checking out Kubernetes $version failed"
624643

625-
go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
644+
645+
go_version="$(go_version_for_kubernetes "$ksrc" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
626646
# Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910
627647
# shellcheck disable=SC2046
628-
(cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image "${CSI_PROW_WORK}/src/kubernetes" --image csiprow/node:latest) || die "'kind build node-image' failed"
648+
(cd "$ksrc" && run_with_go "$go_version" kind build node-image "$ksrc" --image csiprow/node:latest) || die "'kind build node-image' failed"
629649
csi_prow_kind_have_kubernetes=true
630650
fi
631651
image="csiprow/node:latest"
632652
fi
653+
633654
cat >"${CSI_PROW_WORK}/kind-config.yaml" <<EOF
634655
kind: Cluster
635656
apiVersion: kind.x-k8s.io/v1alpha4
@@ -792,6 +813,82 @@ install_snapshot_crds() {
792813
cnt=$((cnt + 1))
793814
sleep 2
794815
done
816+
817+
if ${CSI_PROW_ENABLE_GROUP_SNAPSHOT}; then
818+
echo "Installing VolumeGroupSnapshot CRDs from ${CRD_BASE_DIR}"
819+
kubectl apply -f "${CRD_BASE_DIR}/groupsnapshot.storage.k8s.io_volumegroupsnapshotclasses.yaml" --validate=false
820+
kubectl apply -f "${CRD_BASE_DIR}/groupsnapshot.storage.k8s.io_volumegroupsnapshotcontents.yaml" --validate=false
821+
kubectl apply -f "${CRD_BASE_DIR}/groupsnapshot.storage.k8s.io_volumegroupsnapshots.yaml" --validate=false
822+
823+
local cnt=0
824+
until kubectl get volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io \
825+
&& kubectl get volumegroupsnapshots.groupsnapshot.storage.k8s.io \
826+
&& kubectl get volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io; do
827+
if [ $cnt -gt 30 ]; then
828+
echo >&2 "ERROR: VolumeGroupSnapshot CRDs not ready after 60s"
829+
exit 1
830+
fi
831+
echo "$(date +%H:%M:%S)" "waiting for VolumeGroupSnapshot CRDs, attempt #$cnt"
832+
cnt=$((cnt + 1))
833+
sleep 2
834+
done
835+
echo "VolumeGroupSnapshot CRDs installed and ready"
836+
fi
837+
}
838+
839+
# Inject the CSIVolumeGroupSnapshot feature-gate for snapshot controller
840+
# Arguments:
841+
# $1: file path (optional) - if not provided, reads from stdin
842+
inject_vgs_feature_gate() {
843+
if [ "${CSI_PROW_ENABLE_GROUP_SNAPSHOT}" = "true" ]; then
844+
awk '
845+
BEGIN { inserted=0 }
846+
{
847+
print
848+
if ($0 ~ /--leader-election=true/ && !inserted) {
849+
print " - \"--feature-gates=CSIVolumeGroupSnapshot=true\""
850+
inserted=1
851+
}
852+
}
853+
'
854+
else
855+
cat # just pass the file through unchanged
856+
fi
857+
}
858+
859+
# Replace controller image based on the deployment mode
860+
# Arguments:
861+
# $1: yaml content
862+
# $2: mode ("external-snapshotter", "canary", or "default")
863+
# $3: new_tag (for external-snapshotter mode)
864+
# $4: registry (for canary mode)
865+
replace_controller_image() {
866+
local yaml="$1"
867+
local mode="$2"
868+
local new_tag="${3:-}"
869+
local registry="${4:-}"
870+
871+
case "$mode" in
872+
external-snapshotter)
873+
# Replace with snapshot-controller:new_tag
874+
echo "$yaml" | sed -E \
875+
"s|^([[:space:]]*image: )(.*snapshot-controller):[^[:space:]]*|\1snapshot-controller:${new_tag}|"
876+
;;
877+
canary)
878+
# Replace with registry/snapshot-controller:canary
879+
echo "$yaml" | sed -E \
880+
"s|^([[:space:]]*image: ).*/([^/:]+):[^[:space:]]*|\1${registry}/\2:canary|"
881+
;;
882+
default)
883+
# Replace only the snapshot-controller container tag with CSI_SNAPSHOTTER_VERSION
884+
echo "$yaml" | sed -E \
885+
"s|^([[:space:]]*image: .*/[^:]+):[^[:space:]]*|\1:${CSI_SNAPSHOTTER_VERSION}|"
886+
;;
887+
*)
888+
echo "Error: Unknown mode '$mode' for replace_controller_image" >&2
889+
echo "$yaml"
890+
;;
891+
esac
795892
}
796893

797894
# Install snapshot controller and associated RBAC, retrying until the pod is running.
@@ -820,6 +917,12 @@ install_snapshot_controller() {
820917
done
821918

822919
SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
920+
if [[ "$SNAPSHOT_CONTROLLER_YAML" =~ ^https?:// ]]; then
921+
yaml="$(curl -sL "$SNAPSHOT_CONTROLLER_YAML")"
922+
else
923+
yaml="$(cat "$SNAPSHOT_CONTROLLER_YAML")"
924+
fi
925+
823926
if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then
824927
# snapshot-controller image built from the PR will get a "csiprow" tag.
825928
# Load it into the "kind" cluster so that we can deploy it.
@@ -833,55 +936,32 @@ install_snapshot_controller() {
833936
# Replace image in SNAPSHOT_CONTROLLER_YAML with snapshot-controller:csiprow and deploy
834937
# NOTE: This logic is similar to the logic here:
835938
# https://github.com/kubernetes-csi/csi-driver-host-path/blob/v1.4.0/deploy/util/deploy-hostpath.sh#L155
836-
# Ignore: Double quote to prevent globbing and word splitting.
837-
# shellcheck disable=SC2086
838-
# Ignore: Use find instead of ls to better handle non-alphanumeric filenames.
839-
# shellcheck disable=SC2012
840-
for i in $(ls ${SNAPSHOT_CONTROLLER_YAML} | sort); do
841-
echo " $i"
842-
# Ignore: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
843-
# shellcheck disable=SC2002
844-
# Ignore: See if you can use ${variable//search/replace} instead.
845-
# shellcheck disable=SC2001
846-
modified="$(cat "$i" | while IFS= read -r line; do
847-
nocomments="$(echo "$line" | sed -e 's/ *#.*$//')"
848-
if echo "$nocomments" | grep -q '^[[:space:]]*image:[[:space:]]*'; then
849-
# Split 'image: registry.k8s.io/sig-storage/snapshot-controller:v3.0.0'
850-
# into image (snapshot-controller:v3.0.0),
851-
# name (snapshot-controller),
852-
# tag (v3.0.0).
853-
image=$(echo "$nocomments" | sed -e 's;.*image:[[:space:]]*;;')
854-
name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;')
855-
tag=$(echo "$image" | sed -e 's;.*:;;')
856-
857-
# Now replace registry and/or tag
858-
NEW_TAG="csiprow"
859-
line="$(echo "$nocomments" | sed -e "s;$image;${name}:${NEW_TAG};")"
860-
echo " using $line" >&2
861-
fi
862-
echo "$line"
863-
done)"
864-
if ! echo "$modified" | kubectl apply -f -; then
865-
echo "modified version of $i:"
866-
echo "$modified"
867-
exit 1
868-
fi
869-
done
939+
modified="$(replace_controller_image "$yaml" "external-snapshotter" "$NEW_TAG" | inject_vgs_feature_gate)"
940+
diff <(echo "$yaml") <(echo "$modified")
941+
if ! echo "$modified" | kubectl apply -f -; then
942+
echo "modified version of ${SNAPSHOT_CONTROLLER_YAML}:"
943+
echo "$modified"
944+
exit 1
945+
fi
870946
elif [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then
871947
echo "Deploying snapshot-controller from ${SNAPSHOT_CONTROLLER_YAML} with canary images."
872-
yaml="$(kubectl apply --dry-run=client -o yaml -f "$SNAPSHOT_CONTROLLER_YAML")"
873-
# Ignore: See if you can use ${variable//search/replace} instead.
874-
# shellcheck disable=SC2001
875-
modified="$(echo "$yaml" | sed -e "s;image: .*/\([^/:]*\):.*;image: ${CSI_PROW_DRIVER_CANARY_REGISTRY}/\1:canary;")"
948+
modified="$(replace_controller_image "$yaml" "canary" "" "${CSI_PROW_DRIVER_CANARY_REGISTRY}" | inject_vgs_feature_gate)"
876949
diff <(echo "$yaml") <(echo "$modified")
877950
if ! echo "$modified" | kubectl apply -f -; then
878-
echo "modified version of $SNAPSHOT_CONTROLLER_YAML:"
951+
echo "modified version of ${SNAPSHOT_CONTROLLER_YAML}:"
879952
echo "$modified"
880953
exit 1
881954
fi
882955
else
883956
echo "kubectl apply -f $SNAPSHOT_CONTROLLER_YAML"
884-
kubectl apply -f "$SNAPSHOT_CONTROLLER_YAML"
957+
# Replace snapshot-controller container tag to make it consistent with CSI_SNAPSHOTTER_VERSION
958+
modified="$(replace_controller_image "$yaml" "default" | inject_vgs_feature_gate)"
959+
diff <(echo "$yaml") <(echo "$modified")
960+
if ! echo "$modified" | kubectl apply -f -; then
961+
echo "modified version of ${SNAPSHOT_CONTROLLER_YAML}:"
962+
echo "$modified"
963+
exit 1
964+
fi
885965
fi
886966

887967
cnt=0
@@ -980,14 +1060,23 @@ install_e2e () {
9801060
if sidecar_tests_enabled; then
9811061
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" go test -c -o "${CSI_PROW_WORK}/e2e-local.test" "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}"
9821062
fi
983-
git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 &&
1063+
1064+
# In kubernetes presubmit do not need clone the src
1065+
if [[ $(basename "${REPO_DIR}") == "kubernetes" ]]; then
1066+
echo "Using existing repo at ${REPO_DIR}"
1067+
E2E_SRC_DIR="${REPO_DIR}"
1068+
else
1069+
git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1
1070+
E2E_SRC_DIR="${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}"
1071+
fi
1072+
9841073
if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then
9851074
patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" &&
986-
go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" &&
987-
run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
988-
ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" &&
989-
run_with_go "$go_version" make WHAT=vendor/github.com/onsi/ginkgo/ginkgo "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
990-
ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/ginkgo" "${CSI_PROW_BIN}"
1075+
go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${E2E_SRC_DIR}" "${CSI_PROW_E2E_VERSION}")}" &&
1076+
run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${E2E_SRC_DIR}" &&
1077+
ln -s "${E2E_SRC_DIR}/_output/bin/e2e.test" "${CSI_PROW_WORK}" &&
1078+
run_with_go "$go_version" make WHAT=vendor/github.com/onsi/ginkgo/ginkgo "-C${E2E_SRC_DIR}" &&
1079+
ln -s "${E2E_SRC_DIR}/_output/bin/ginkgo" "${CSI_PROW_BIN}"
9911080
else
9921081
run_with_go "${CSI_PROW_GO_VERSION_E2E}" go test -c -o "${CSI_PROW_WORK}/e2e.test" "${CSI_PROW_E2E_IMPORT_PATH}/test/e2e"
9931082
fi
@@ -1028,11 +1117,20 @@ run_e2e () (
10281117
# Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again
10291118
# and to avoid the large number of "skipped" tests that we get from using
10301119
# the full Kubernetes E2E testsuite while only running a few tests.
1120+
# shellcheck disable=SC2329
10311121
move_junit () {
1122+
# shellcheck disable=SC2317
1123+
FILTER='External.Storage|CSI.mock.volume'
1124+
# shellcheck disable=SC2317
1125+
if [ "${CSI_PROW_ENABLE_GROUP_SNAPSHOT}" = "true" ]; then
1126+
FILTER="${FILTER}|\[Feature:volumegroupsnapshot\]"
1127+
fi
1128+
1129+
# shellcheck disable=SC2317
10321130
if ls "${ARTIFACTS}"/junit_[0-9]*.xml 2>/dev/null >/dev/null; then
10331131
mkdir -p "${ARTIFACTS}/junit/${name}" &&
10341132
mkdir -p "${ARTIFACTS}/junit/steps" &&
1035-
run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit/steps/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml &&
1133+
run_filter_junit -t="${FILTER}" -o "${ARTIFACTS}/junit/steps/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml &&
10361134
mv "${ARTIFACTS}"/junit_[0-9]*.xml "${ARTIFACTS}/junit/${name}/"
10371135
fi
10381136
}
@@ -1041,6 +1139,9 @@ run_e2e () (
10411139
if [ "${name}" == "local" ]; then
10421140
cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_PATH}" &&
10431141
run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local
1142+
elif [ "${CSI_PROW_ENABLE_GROUP_SNAPSHOT}" = "true" ] && [ "${name}" == "parallel-features" ]; then
1143+
cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
1144+
run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -report-prefix vgs
10441145
else
10451146
cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
10461147
run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
@@ -1248,7 +1349,10 @@ main () {
12481349
images=
12491350
if ${CSI_PROW_BUILD_JOB}; then
12501351
# A successful build is required for testing.
1251-
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" "BUILD_PLATFORMS=${CSI_PROW_BUILD_PLATFORMS}" || die "'make all' failed"
1352+
# In kubernetes presubmit the needed test packages are already built in kind node image
1353+
if [[ $(basename "${REPO_DIR}") != "kubernetes" ]]; then
1354+
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" "BUILD_PLATFORMS=${CSI_PROW_BUILD_PLATFORMS}" || die "'make all' failed"
1355+
fi
12521356
# We don't want test failures to prevent E2E testing below, because the failure
12531357
# might have been minor or unavoidable, for example when experimenting with
12541358
# changes in "release-tools" in a PR (that fails the "is release-tools unmodified"
@@ -1264,7 +1368,9 @@ main () {
12641368
fi
12651369
fi
12661370
# Required for E2E testing.
1267-
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make container' failed"
1371+
if [[ $(basename "${REPO_DIR}") != "kubernetes" ]]; then
1372+
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make container' failed"
1373+
fi
12681374
fi
12691375

12701376
if tests_need_kind; then
@@ -1318,7 +1424,6 @@ main () {
13181424
install_snapshot_crds
13191425
install_snapshot_controller
13201426

1321-
13221427
# Installing the driver might be disabled.
13231428
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
13241429
collect_cluster_info
@@ -1339,11 +1444,18 @@ main () {
13391444
ret=1
13401445
fi
13411446

1447+
# Only add feature regex if groupsnapshot is not enabled
1448+
if [ "${CSI_PROW_ENABLE_GROUP_SNAPSHOT}" = "true" ]; then
1449+
combined_focus="${CSI_PROW_E2E_FOCUS}"
1450+
else
1451+
combined_focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))"
1452+
fi
1453+
13421454
# Run tests that are feature tagged, but non-alpha
13431455
# Ignore: Double quote to prevent globbing and word splitting.
13441456
# shellcheck disable=SC2086
13451457
if ! run_e2e parallel-features ${CSI_PROW_GINKGO_PARALLEL} \
1346-
-focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
1458+
-focus="$combined_focus" \
13471459
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then
13481460
warn "E2E parallel features failed"
13491461
ret=1

0 commit comments

Comments
 (0)