Skip to content

Commit 8aa8858

Browse files
yitzhaktalyitzhaktal
committed
DEVOPS-2694 - Update the lightrun-k8s-operator deployment to mount Secrets as files via volumes instead of exposing them as environment variables in containers.
1 parent 2be38da commit 8aa8858

File tree

4 files changed

+148
-99
lines changed

4 files changed

+148
-99
lines changed

‎internal/controller/lightrunjavaagent_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ func (r *LightrunJavaAgentReconciler) reconcileDeployment(ctx context.Context, l
252252

253253
// Create config map
254254
log.V(2).Info("Reconciling config map with agent configuration")
255-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
255+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
256256
if err != nil {
257257
log.Error(err, "unable to create configMap")
258258
return r.errorStatus(ctx, lightrunJavaAgent, err)
@@ -493,7 +493,7 @@ func (r *LightrunJavaAgentReconciler) reconcileStatefulSet(ctx context.Context,
493493

494494
// Create config map
495495
log.V(2).Info("Reconciling config map with agent configuration")
496-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
496+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
497497
if err != nil {
498498
log.Error(err, "unable to create configMap")
499499
return r.errorStatus(ctx, lightrunJavaAgent, err)

‎internal/controller/patch_funcs.go‎

Lines changed: 75 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ const (
2727
annotationAgentName = "lightrun.com/lightrunjavaagent"
2828
)
2929

30-
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent) (corev1.ConfigMap, error) {
30+
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) (corev1.ConfigMap, error) {
3131
populateTags(lightrunJavaAgent.Spec.AgentTags, lightrunJavaAgent.Spec.AgentName, &metadata)
3232
jsonString, err := json.Marshal(metadata)
3333
if err != nil {
@@ -52,26 +52,28 @@ func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agent
5252
}
5353

5454
func (r *LightrunJavaAgentReconciler) patchDeployment(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, cmDataHash uint64) error {
55-
5655
// init spec.template.spec
5756
deploymentApplyConfig.WithSpec(
5857
appsv1ac.DeploymentSpec().WithTemplate(
5958
corev1ac.PodTemplateSpec().WithSpec(
6059
corev1ac.PodSpec(),
6160
).WithAnnotations(map[string]string{
6261
annotationConfigMapHash: fmt.Sprint(cmDataHash),
63-
},
64-
),
62+
}),
6563
),
6664
).WithAnnotations(map[string]string{
6765
annotationAgentName: lightrunJavaAgent.Name,
6866
})
6967
r.addVolume(deploymentApplyConfig, lightrunJavaAgent)
7068
r.addInitContainer(deploymentApplyConfig, lightrunJavaAgent, secret)
71-
err = r.patchAppContainers(lightrunJavaAgent, origDeployment, deploymentApplyConfig)
69+
err := r.patchAppContainers(lightrunJavaAgent, origDeployment, deploymentApplyConfig)
7270
if err != nil {
7371
return err
7472
}
73+
deploymentApplyConfig.Spec.Template.Spec.WithSecurityContext(
74+
corev1ac.PodSecurityContext().
75+
WithFSGroup(1000),
76+
)
7577
return nil
7678
}
7779

@@ -98,55 +100,55 @@ func (r *LightrunJavaAgentReconciler) addVolume(deploymentApplyConfig *appsv1ac.
98100
)
99101
}
100102

103+
// Client side patch, as we can't update value from 2 sources
101104
func (r *LightrunJavaAgentReconciler) addInitContainer(deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) {
102-
103105
deploymentApplyConfig.Spec.Template.Spec.WithInitContainers(
104106
corev1ac.Container().
105107
WithName(initContainerName).
106108
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
107109
WithVolumeMounts(
108-
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
110+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath),
109111
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
110-
).WithEnv(
111-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
112-
corev1ac.EnvVarSource().WithSecretKeyRef(
113-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
114-
),
115-
),
116-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
117-
corev1ac.EnvVarSource().WithSecretKeyRef(
118-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
119-
),
120-
),
121-
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
122-
).
112+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
113+
).
114+
WithEnv(
115+
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
116+
).
117+
WithSecurityContext(
118+
corev1ac.SecurityContext().
119+
WithReadOnlyRootFilesystem(true).
120+
WithAllowPrivilegeEscalation(false).
121+
WithRunAsNonRoot(true).
122+
WithRunAsUser(1000),
123+
).
123124
WithResources(
124125
corev1ac.ResourceRequirements().
125126
WithLimits(
126127
corev1.ResourceList{
127128
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
128-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 500 * 10^6 = 500M
129+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
129130
},
130-
).WithRequests(
131-
corev1.ResourceList{
132-
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
133-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
134-
},
135-
),
136-
).
137-
WithSecurityContext(
138-
corev1ac.SecurityContext().
139-
WithCapabilities(
140-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
141131
).
142-
WithAllowPrivilegeEscalation(false).
143-
WithRunAsNonRoot(true).
144-
WithSeccompProfile(
145-
corev1ac.SeccompProfile().
146-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
132+
WithRequests(
133+
corev1.ResourceList{
134+
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
135+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
136+
},
147137
),
148138
),
149139
)
140+
141+
// Add volume for secret with proper permissions
142+
deploymentApplyConfig.Spec.Template.Spec.WithVolumes(
143+
corev1ac.Volume().WithName("lightrun-secret").
144+
WithSecret(corev1ac.SecretVolumeSource().
145+
WithSecretName(secret.Name).
146+
WithItems(
147+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
148+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
149+
).
150+
WithDefaultMode(0440)),
151+
)
150152
}
151153

152154
func (r *LightrunJavaAgentReconciler) patchAppContainers(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration) error {
@@ -167,8 +169,7 @@ func (r *LightrunJavaAgentReconciler) patchAppContainers(lightrunJavaAgent *agen
167169
}
168170
}
169171
if !found {
170-
err = errors.New("unable to find matching container to patch")
171-
return err
172+
return errors.New("unable to find matching container to patch")
172173
}
173174
return nil
174175
}
@@ -179,14 +180,14 @@ func (r *LightrunJavaAgentReconciler) patchJavaToolEnv(deplAnnotations map[strin
179180
patchedEnv := deplAnnotations[annotationPatchedEnvName]
180181
patchedEnvValue := deplAnnotations[annotationPatchedEnvValue]
181182

183+
// If different env was patched before - unpatch it
182184
if patchedEnv != targetEnvVar || patchedEnvValue != agentArg {
183-
// If different env was patched before - unpatch it
184185
r.unpatchJavaToolEnv(deplAnnotations, container)
185186
}
186187

188+
// No such env - add new
187189
targetEnvVarIndex := findEnvVarIndex(targetEnvVar, container.Env)
188190
if targetEnvVarIndex == -1 {
189-
// No such env - add new
190191
container.Env = append(container.Env, corev1.EnvVar{
191192
Name: targetEnvVar,
192193
Value: agentArg,
@@ -230,21 +231,18 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSet(lightrunJavaAgent *agentv
230231
corev1ac.PodSpec(),
231232
).WithAnnotations(map[string]string{
232233
annotationConfigMapHash: fmt.Sprint(cmDataHash),
233-
},
234-
),
234+
}),
235235
),
236236
).WithAnnotations(map[string]string{
237237
annotationAgentName: lightrunJavaAgent.Name,
238238
})
239239

240240
// Add volumes to the StatefulSet
241241
r.addVolumeToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent)
242-
243242
// Add init container to the StatefulSet
244243
r.addInitContainerToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent, secret)
245-
246244
// Patch app containers in the StatefulSet
247-
err = r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
245+
err := r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
248246
if err != nil {
249247
return err
250248
}
@@ -271,6 +269,15 @@ func (r *LightrunJavaAgentReconciler) addVolumeToStatefulSet(statefulSetApplyCon
271269
corev1ac.KeyToPath().WithKey("metadata").WithPath("agent.metadata.json"),
272270
),
273271
),
272+
).WithVolumes(
273+
corev1ac.Volume().WithName("lightrun-secret").
274+
WithSecret(corev1ac.SecretVolumeSource().
275+
WithSecretName(secret.Name).
276+
WithItems(
277+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
278+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
279+
).
280+
WithDefaultMode(0440)),
274281
)
275282
}
276283

@@ -280,48 +287,45 @@ func (r *LightrunJavaAgentReconciler) addInitContainerToStatefulSet(statefulSetA
280287
WithName(initContainerName).
281288
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
282289
WithVolumeMounts(
283-
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
290+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath),
284291
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
292+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
285293
).WithEnv(
286-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
287-
corev1ac.EnvVarSource().WithSecretKeyRef(
288-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
289-
),
290-
),
291-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
292-
corev1ac.EnvVarSource().WithSecretKeyRef(
293-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
294-
),
295-
),
296294
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
297295
).
296+
WithSecurityContext(
297+
corev1ac.SecurityContext().
298+
WithReadOnlyRootFilesystem(true).
299+
WithAllowPrivilegeEscalation(false).
300+
WithRunAsNonRoot(true).
301+
WithRunAsUser(1000),
302+
).
298303
WithResources(
299304
corev1ac.ResourceRequirements().
300305
WithLimits(
301306
corev1.ResourceList{
302307
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
303-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 64M
308+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
304309
},
305310
).WithRequests(
306311
corev1.ResourceList{
307312
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
308313
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
309314
},
310315
),
311-
).
312-
WithSecurityContext(
313-
corev1ac.SecurityContext().
314-
WithCapabilities(
315-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
316-
).
317-
WithAllowPrivilegeEscalation(false).
318-
WithRunAsNonRoot(true).
319-
WithSeccompProfile(
320-
corev1ac.SeccompProfile().
321-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
322-
),
323316
),
324317
)
318+
319+
statefulSetApplyConfig.Spec.Template.Spec.WithVolumes(
320+
corev1ac.Volume().WithName("lightrun-secret").
321+
WithSecret(corev1ac.SecretVolumeSource().
322+
WithSecretName(secret.Name).
323+
WithItems(
324+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
325+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
326+
).
327+
WithDefaultMode(0440)),
328+
)
325329
}
326330

327331
func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, origStatefulSet *appsv1.StatefulSet, statefulSetApplyConfig *appsv1ac.StatefulSetApplyConfiguration) error {
@@ -342,15 +346,13 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJava
342346
}
343347
}
344348
if !found {
345-
err = errors.New("unable to find matching container to patch")
346-
return err
349+
return errors.New("unable to find matching container to patch")
347350
}
348351
return nil
349352
}
350353

351354
// configMapDataHash calculates a hash of the ConfigMap data to detect changes
352355
func configMapDataHash(cmData map[string]string) uint64 {
353-
// Combine all data values into a single string for hashing
354356
var hashString string
355357
for _, v := range cmData {
356358
hashString += v

‎lightrun-init-agent/Dockerfile‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,29 @@
11
ARG base_image_tag=alpine-3.20.0-r1
22

33
FROM lightruncom/prod-base:${base_image_tag}
4-
ARG FILE
4+
ARG FILE
55

6-
COPY lightrun-init-agent/$FILE /tmp/$FILE
6+
# Copy the agent zip file
7+
COPY $FILE /tmp/$FILE
78

89
RUN unzip -o /tmp/$FILE -d /agent ;\
910
rm -rf /tmp/$FILE && \
1011
# Erase default values
1112
sed -i.bak "s|com.lightrun.secret=.*|com.lightrun.secret=|" /agent/agent.config && rm /agent/agent.config.bak && \
1213
sed -i.bak "s|pinned_certs=.*|pinned_certs=|" /agent/agent.config && rm /agent/agent.config.bak && \
13-
# In openshift UID will be dynamic per project, hence procide permissions to root group (defualt in k8s)
14-
chgrp -R 0 /agent && \
15-
chmod -R g=u /agent
14+
# Set proper permissions for the agent directory
15+
chown -R 1000:1000 /agent && \
16+
chmod -R 750 /agent && \
17+
# Create secret directory with proper permissions
18+
mkdir -p /etc/lightrun/secret && \
19+
chown -R 1000:1000 /etc/lightrun/secret && \
20+
chmod -R 700 /etc/lightrun/secret
21+
22+
# Copy and set permissions for update_config.sh before switching user
23+
COPY update_config.sh /update_config.sh
24+
RUN chmod 750 /update_config.sh && \
25+
chown 1000:1000 /update_config.sh
1626

1727
USER 1000
18-
COPY lightrun-init-agent/update_config.sh /update_config.sh
1928

2029
CMD [ "/bin/sh", "/update_config.sh" ]

0 commit comments

Comments
 (0)