Skip to content

Commit 429d928

Browse files
bochengchubochengchu
committed
add a e2e test for internal load balancer only
1 parent 32a907c commit 429d928

File tree

8 files changed

+300
-27
lines changed

8 files changed

+300
-27
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ SKIP_CLEANUP ?= false
176176
SKIP_CREATE_MGMT_CLUSTER ?= false
177177

178178
.PHONY: test-e2e-run
179-
test-e2e-run: $(ENVSUBST) $(KUBECTL) $(GINKGO) e2e-image ## Run the end-to-end tests
179+
test-e2e-run: $(ENVSUBST) $(KUBECTL) $(GINKGO) ## Run the end-to-end tests
180180
$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
181181
time $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) \
182182
--tags=e2e --focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) \

‎config/default/manager_image_patch.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@ spec:
88
spec:
99
containers:
1010
# Change the value of image field below to your controller image URL
11-
- image: gcr.io/k8s-staging-cluster-api-gcp/cluster-api-gcp-controller:e2e
11+
- image: ${CONTROLLER_IMAGE}
1212
name: manager

‎scripts/ci-e2e.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ export KUBERNETES_MINOR_VERSION="${VERSION_PARTS[1]}"
5555
export KUBERNETES_PATCH_VERSION="${VERSION_PARTS[2]}"
5656
# using prebuilt image from image-builder project the image is built everyday and the job is available here https://prow.k8s.io/?job=periodic-image-builder-gcp-all-nightly
5757
export IMAGE_ID="projects/k8s-staging-cluster-api-gcp/global/images/cluster-api-ubuntu-2204-${KUBERNETES_VERSION//[.+]/-}-nightly"
58+
export CONTROLLER_IMAGE="gcr.io/${GCP_PROJECT}/cluster-api-gcp-controller:${TEST_NAME}"
5859

5960
init_image() {
6061
if [[ "${REUSE_OLD_IMAGES:-false}" == "true" ]]; then
@@ -101,6 +102,11 @@ EOF
101102
export IMAGE_ID="projects/${GCP_PROJECT}/global/images/${image_id}"
102103
}
103104

105+
init_controller_image() {
106+
echo "Tagging and pushing controller image to ${CONTROLLER_IMAGE}"
107+
docker tag gcr.io/k8s-staging-cluster-api-gcp/cluster-api-gcp-controller:e2e "${CONTROLLER_IMAGE}"
108+
docker push "${CONTROLLER_IMAGE}"
109+
}
104110

105111
# initialize a router and cloud NAT
106112
init_networks() {
@@ -127,6 +133,14 @@ init_networks() {
127133
--nat-all-subnet-ip-ranges --auto-allocate-nat-external-ips
128134
}
129135

136+
# create a GKE cluster to be used as a bootstrap cluster
137+
create_gke_bootstrap_cluster() {
138+
gcloud container clusters create "${TEST_NAME}-gke-bootstrap" --project "$GCP_PROJECT" \
139+
--region "$GCP_REGION" --num-nodes 1 --machine-type e2-medium --release-channel regular \
140+
--network "${GCP_NETWORK_NAME}" --quiet
141+
export GKE_BOOTSTRAP_KUBECONFIG="${ARTIFACTS}/gke_bootstrap_kubeconfig"
142+
KUBECONFIG="${GKE_BOOTSTRAP_KUBECONFIG}" gcloud container clusters get-credentials "${TEST_NAME}-gke-bootstrap" --region "${GCP_REGION}" --project "${GCP_PROJECT}"
143+
}
130144

131145
cleanup() {
132146
# Force a cleanup of cluster api created resources using gcloud commands
@@ -152,6 +166,9 @@ cleanup() {
152166
| awk '{print "gcloud compute firewall-rules delete --project '"$GCP_PROJECT"' --quiet " $1 "\n"}' \
153167
| bash) || true
154168

169+
gcloud container clusters delete "${TEST_NAME}-gke-bootstrap" --project "$GCP_PROJECT" \
170+
--region "$GCP_REGION" --quiet || true
171+
155172
# cleanup the networks
156173
gcloud compute routers nats delete "${TEST_NAME}-mynat" --project="${GCP_PROJECT}" \
157174
--router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" --quiet || true
@@ -275,6 +292,16 @@ EOF
275292
init_networks
276293
fi
277294

295+
# Initialize the GKE bootstrap cluster
296+
if [[ -n "${SKIP_INIT_GKE_BOOTSTRAP:-}" ]]; then
297+
echo "Skipping GKE bootstrap cluster initialization..."
298+
else
299+
create_gke_bootstrap_cluster
300+
fi
301+
302+
make e2e-image
303+
init_controller_image
304+
278305
make test-e2e
279306
test_status="${?}"
280307
echo TESTSTATUS

‎test/e2e/config/gcp-ci.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ managementClusterName: capg-e2e
88

99
images:
1010
# Use local dev images built source tree;
11-
- name: gcr.io/k8s-staging-cluster-api-gcp/cluster-api-gcp-controller:e2e
11+
- name: ${CONTROLLER_IMAGE}
1212
loadBehavior: mustLoad
1313

1414
providers:

‎test/e2e/data/infrastructure-gcp/cluster-template-ci-with-external-and-internal-lb.yaml‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: Cluster
4+
metadata:
5+
name: "${CLUSTER_NAME}"
6+
labels:
7+
cni: "${CLUSTER_NAME}-crs-cni"
8+
ccm: "${CLUSTER_NAME}-crs-ccm"
9+
spec:
10+
clusterNetwork:
11+
pods:
12+
cidrBlocks: ["192.168.0.0/16"]
13+
infrastructureRef:
14+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
15+
kind: GCPCluster
16+
name: "${CLUSTER_NAME}"
17+
controlPlaneRef:
18+
kind: KubeadmControlPlane
19+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
20+
name: "${CLUSTER_NAME}-control-plane"
21+
---
22+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
23+
kind: GCPCluster
24+
metadata:
25+
name: "${CLUSTER_NAME}"
26+
spec:
27+
project: "${GCP_PROJECT}"
28+
region: "${GCP_REGION}"
29+
network:
30+
name: "${GCP_NETWORK_NAME}"
31+
subnets:
32+
- name: control-plane-subnet
33+
cidrBlock: "10.0.0.0/17"
34+
purpose: PRIVATE
35+
region: us-east4
36+
loadBalancer:
37+
loadBalancerType: InternalExternal
38+
---
39+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
40+
kind: KubeadmControlPlane
41+
metadata:
42+
name: "${CLUSTER_NAME}-control-plane"
43+
spec:
44+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
45+
machineTemplate:
46+
infrastructureRef:
47+
kind: GCPMachineTemplate
48+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
49+
name: "${CLUSTER_NAME}-control-plane"
50+
kubeadmConfigSpec:
51+
initConfiguration:
52+
nodeRegistration:
53+
name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
54+
kubeletExtraArgs:
55+
cloud-provider: external
56+
clusterConfiguration:
57+
apiServer:
58+
timeoutForControlPlane: 20m
59+
controllerManager:
60+
extraArgs:
61+
cloud-provider: external
62+
allocate-node-cidrs: "false"
63+
kubernetesVersion: "${KUBERNETES_VERSION}"
64+
files:
65+
- content: |
66+
[Global]
67+
68+
project-id = "${GCP_PROJECT}"
69+
network-name = "${GCP_NETWORK_NAME}"
70+
multizone = true
71+
owner: root:root
72+
path: /etc/kubernetes/cloud.config
73+
permissions: "0744"
74+
joinConfiguration:
75+
nodeRegistration:
76+
name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
77+
kubeletExtraArgs:
78+
cloud-provider: external
79+
version: "${KUBERNETES_VERSION}"
80+
---
81+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
82+
kind: GCPMachineTemplate
83+
metadata:
84+
name: "${CLUSTER_NAME}-control-plane"
85+
spec:
86+
template:
87+
spec:
88+
instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}"
89+
image: "${IMAGE_ID}"
90+
---
91+
apiVersion: cluster.x-k8s.io/v1beta1
92+
kind: MachineDeployment
93+
metadata:
94+
name: "${CLUSTER_NAME}-md-0"
95+
spec:
96+
clusterName: "${CLUSTER_NAME}"
97+
replicas: ${WORKER_MACHINE_COUNT}
98+
selector:
99+
matchLabels:
100+
template:
101+
spec:
102+
clusterName: "${CLUSTER_NAME}"
103+
version: "${KUBERNETES_VERSION}"
104+
bootstrap:
105+
configRef:
106+
name: "${CLUSTER_NAME}-md-0"
107+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
108+
kind: KubeadmConfigTemplate
109+
infrastructureRef:
110+
name: "${CLUSTER_NAME}-md-0"
111+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
112+
kind: GCPMachineTemplate
113+
---
114+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
115+
kind: GCPMachineTemplate
116+
metadata:
117+
name: "${CLUSTER_NAME}-md-0"
118+
spec:
119+
template:
120+
spec:
121+
instanceType: "${GCP_NODE_MACHINE_TYPE}"
122+
image: "${IMAGE_ID}"
123+
---
124+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
125+
kind: KubeadmConfigTemplate
126+
metadata:
127+
name: "${CLUSTER_NAME}-md-0"
128+
spec:
129+
template:
130+
spec:
131+
joinConfiguration:
132+
nodeRegistration:
133+
name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
134+
kubeletExtraArgs:
135+
cloud-provider: external
136+
---
137+
apiVersion: v1
138+
kind: ConfigMap
139+
metadata:
140+
name: "${CLUSTER_NAME}-crs-cni"
141+
data: ${CNI_RESOURCES}
142+
---
143+
apiVersion: addons.cluster.x-k8s.io/v1beta1
144+
kind: ClusterResourceSet
145+
metadata:
146+
name: "${CLUSTER_NAME}-crs-cni"
147+
spec:
148+
strategy: ApplyOnce
149+
clusterSelector:
150+
matchLabels:
151+
cni: "${CLUSTER_NAME}-crs-cni"
152+
resources:
153+
- name: "${CLUSTER_NAME}-crs-cni"
154+
kind: ConfigMap
155+
---
156+
apiVersion: v1
157+
kind: ConfigMap
158+
metadata:
159+
name: "${CLUSTER_NAME}-crs-ccm"
160+
data: ${CCM_RESOURCES}
161+
---
162+
apiVersion: addons.cluster.x-k8s.io/v1beta1
163+
kind: ClusterResourceSet
164+
metadata:
165+
name: "${CLUSTER_NAME}-crs-ccm"
166+
spec:
167+
strategy: ApplyOnce
168+
clusterSelector:
169+
matchLabels:
170+
ccm: "${CLUSTER_NAME}-crs-ccm"
171+
resources:
172+
- name: "${CLUSTER_NAME}-crs-ccm"
173+
kind: ConfigMap

‎test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ metadata:
88
ccm: "${CLUSTER_NAME}-crs-ccm"
99
spec:
1010
clusterNetwork:
11+
apiServerPort: 6443
1112
pods:
1213
cidrBlocks: ["192.168.0.0/16"]
1314
infrastructureRef:
@@ -33,8 +34,10 @@ spec:
3334
cidrBlock: "10.0.0.0/17"
3435
purpose: PRIVATE
3536
region: us-east4
37+
controlPlaneEndpoint:
38+
port: 6443
3639
loadBalancer:
37-
loadBalancerType: InternalExternal
40+
loadBalancerType: Internal
3841
---
3942
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
4043
kind: KubeadmControlPlane

0 commit comments

Comments
 (0)