Skip to content

Commit ad2d821

Browse files
yitzhaktalyitzhaktal
committed
DEVOPS-2694 - Update the lightrun-k8s-operator deployment to mount Secrets as files via volumes instead of exposing them as environment variables in containers.
1 parent 2be38da commit ad2d821

File tree

6 files changed

+218
-97
lines changed

6 files changed

+218
-97
lines changed

‎api/v1beta/lightrunjavaagent_types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@ type LightrunJavaAgentSpec struct {
9090
// +optional
9191
// Agent name for registration to the server
9292
AgentName string `json:"agentName,omitempty"`
93+
94+
// UseSecretAsEnvVars determines whether to use secret values as environment variables (true) or as mounted files (false)
95+
// +kubebuilder:default=true
96+
UseSecretAsEnvVars bool `json:"useSecretAsEnvVars,omitempty"`
9397
}
9498

9599
// LightrunJavaAgentStatus defines the observed state of LightrunJavaAgent

‎config/crd/bases/agents.lightrun.com_lightrunjavaagents.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,11 @@ spec:
120120
Lightrun server hostname that will be used for downloading an agent
121121
Key and company id in the secret has to be taken from this server as well
122122
type: string
123+
useSecretAsEnvVars:
124+
default: true
125+
description: UseSecretAsEnvVars determines whether to use secret values
126+
as environment variables (true) or as mounted files (false)
127+
type: boolean
123128
workloadName:
124129
description: Name of the Workload that will be patched. workload can
125130
be either Deployment or StatefulSet e.g. my-deployment, my-statefulset

‎internal/controller/lightrunjavaagent_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ func (r *LightrunJavaAgentReconciler) reconcileDeployment(ctx context.Context, l
252252

253253
// Create config map
254254
log.V(2).Info("Reconciling config map with agent configuration")
255-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
255+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
256256
if err != nil {
257257
log.Error(err, "unable to create configMap")
258258
return r.errorStatus(ctx, lightrunJavaAgent, err)
@@ -493,7 +493,7 @@ func (r *LightrunJavaAgentReconciler) reconcileStatefulSet(ctx context.Context,
493493

494494
// Create config map
495495
log.V(2).Info("Reconciling config map with agent configuration")
496-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
496+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
497497
if err != nil {
498498
log.Error(err, "unable to create configMap")
499499
return r.errorStatus(ctx, lightrunJavaAgent, err)

‎internal/controller/patch_funcs.go‎

Lines changed: 130 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ const (
2727
annotationAgentName = "lightrun.com/lightrunjavaagent"
2828
)
2929

30-
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent) (corev1.ConfigMap, error) {
30+
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) (corev1.ConfigMap, error) {
3131
populateTags(lightrunJavaAgent.Spec.AgentTags, lightrunJavaAgent.Spec.AgentName, &metadata)
3232
jsonString, err := json.Marshal(metadata)
3333
if err != nil {
@@ -52,16 +52,14 @@ func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agent
5252
}
5353

5454
func (r *LightrunJavaAgentReconciler) patchDeployment(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, cmDataHash uint64) error {
55-
5655
// init spec.template.spec
5756
deploymentApplyConfig.WithSpec(
5857
appsv1ac.DeploymentSpec().WithTemplate(
5958
corev1ac.PodTemplateSpec().WithSpec(
6059
corev1ac.PodSpec(),
6160
).WithAnnotations(map[string]string{
6261
annotationConfigMapHash: fmt.Sprint(cmDataHash),
63-
},
64-
),
62+
}),
6563
),
6664
).WithAnnotations(map[string]string{
6765
annotationAgentName: lightrunJavaAgent.Name,
@@ -72,19 +70,21 @@ func (r *LightrunJavaAgentReconciler) patchDeployment(lightrunJavaAgent *agentv1
7270
if err != nil {
7371
return err
7472
}
73+
deploymentApplyConfig.Spec.Template.Spec.WithSecurityContext(
74+
corev1ac.PodSecurityContext().
75+
WithFSGroup(1000),
76+
)
7577
return nil
7678
}
7779

7880
func (r *LightrunJavaAgentReconciler) addVolume(deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent) {
79-
80-
deploymentApplyConfig.Spec.Template.Spec.
81-
WithVolumes(
82-
corev1ac.Volume().
83-
WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).
84-
WithEmptyDir(
85-
corev1ac.EmptyDirVolumeSource(),
86-
),
87-
).WithVolumes(
81+
// Start with base volumes
82+
volumes := []*corev1ac.VolumeApplyConfiguration{
83+
corev1ac.Volume().
84+
WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).
85+
WithEmptyDir(
86+
corev1ac.EmptyDirVolumeSource(),
87+
),
8888
corev1ac.Volume().
8989
WithName(cmVolumeName).
9090
WithConfigMap(
@@ -95,19 +95,34 @@ func (r *LightrunJavaAgentReconciler) addVolume(deploymentApplyConfig *appsv1ac.
9595
corev1ac.KeyToPath().WithKey("metadata").WithPath("agent.metadata.json"),
9696
),
9797
),
98-
)
98+
}
99+
100+
// Add secret volume if UseSecretAsEnvVars is false
101+
if !lightrunJavaAgent.Spec.UseSecretAsEnvVars {
102+
volumes = append(volumes,
103+
corev1ac.Volume().WithName("lightrun-secret").
104+
WithSecret(corev1ac.SecretVolumeSource().
105+
WithSecretName(secret.Name).
106+
WithItems(
107+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
108+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
109+
).
110+
WithDefaultMode(0440)),
111+
)
112+
}
113+
114+
deploymentApplyConfig.Spec.Template.Spec.WithVolumes(volumes...)
99115
}
100116

101117
func (r *LightrunJavaAgentReconciler) addInitContainer(deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) {
118+
// Start with base environment variables
119+
envVars := []*corev1ac.EnvVarApplyConfiguration{
120+
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
121+
}
102122

103-
deploymentApplyConfig.Spec.Template.Spec.WithInitContainers(
104-
corev1ac.Container().
105-
WithName(initContainerName).
106-
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
107-
WithVolumeMounts(
108-
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
109-
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
110-
).WithEnv(
123+
// Add secret environment variables if UseSecretAsEnvVars is true
124+
if lightrunJavaAgent.Spec.UseSecretAsEnvVars {
125+
envVars = append(envVars,
111126
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
112127
corev1ac.EnvVarSource().WithSecretKeyRef(
113128
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
@@ -118,33 +133,48 @@ func (r *LightrunJavaAgentReconciler) addInitContainer(deploymentApplyConfig *ap
118133
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
119134
),
120135
),
121-
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
122-
).
136+
)
137+
}
138+
139+
// Start with base volume mounts
140+
volumeMounts := []*corev1ac.VolumeMountApplyConfiguration{
141+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
142+
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
143+
}
144+
145+
// Add secret volume mount if UseSecretAsEnvVars is false
146+
if !lightrunJavaAgent.Spec.UseSecretAsEnvVars {
147+
volumeMounts = append(volumeMounts,
148+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
149+
)
150+
}
151+
152+
deploymentApplyConfig.Spec.Template.Spec.WithInitContainers(
153+
corev1ac.Container().
154+
WithName(initContainerName).
155+
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
156+
WithVolumeMounts(volumeMounts...).
157+
WithEnv(envVars...).
158+
WithSecurityContext(
159+
corev1ac.SecurityContext().
160+
WithReadOnlyRootFilesystem(true).
161+
WithAllowPrivilegeEscalation(false).
162+
WithRunAsNonRoot(true).
163+
WithRunAsUser(1000),
164+
).
123165
WithResources(
124166
corev1ac.ResourceRequirements().
125167
WithLimits(
126168
corev1.ResourceList{
127169
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
128-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 500 * 10^6 = 500M
170+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 64M
129171
},
130172
).WithRequests(
131173
corev1.ResourceList{
132174
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
133175
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
134176
},
135177
),
136-
).
137-
WithSecurityContext(
138-
corev1ac.SecurityContext().
139-
WithCapabilities(
140-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
141-
).
142-
WithAllowPrivilegeEscalation(false).
143-
WithRunAsNonRoot(true).
144-
WithSeccompProfile(
145-
corev1ac.SeccompProfile().
146-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
147-
),
148178
),
149179
)
150180
}
@@ -230,19 +260,16 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSet(lightrunJavaAgent *agentv
230260
corev1ac.PodSpec(),
231261
).WithAnnotations(map[string]string{
232262
annotationConfigMapHash: fmt.Sprint(cmDataHash),
233-
},
234-
),
263+
}),
235264
),
236265
).WithAnnotations(map[string]string{
237266
annotationAgentName: lightrunJavaAgent.Name,
238267
})
239268

240269
// Add volumes to the StatefulSet
241270
r.addVolumeToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent)
242-
243271
// Add init container to the StatefulSet
244272
r.addInitContainerToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent, secret)
245-
246273
// Patch app containers in the StatefulSet
247274
err = r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
248275
if err != nil {
@@ -253,14 +280,13 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSet(lightrunJavaAgent *agentv
253280
}
254281

255282
func (r *LightrunJavaAgentReconciler) addVolumeToStatefulSet(statefulSetApplyConfig *appsv1ac.StatefulSetApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent) {
256-
statefulSetApplyConfig.Spec.Template.Spec.
257-
WithVolumes(
258-
corev1ac.Volume().
259-
WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).
260-
WithEmptyDir(
261-
corev1ac.EmptyDirVolumeSource(),
262-
),
263-
).WithVolumes(
283+
// Start with base volumes
284+
volumes := []*corev1ac.VolumeApplyConfiguration{
285+
corev1ac.Volume().
286+
WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).
287+
WithEmptyDir(
288+
corev1ac.EmptyDirVolumeSource(),
289+
),
264290
corev1ac.Volume().
265291
WithName(cmVolumeName).
266292
WithConfigMap(
@@ -271,18 +297,34 @@ func (r *LightrunJavaAgentReconciler) addVolumeToStatefulSet(statefulSetApplyCon
271297
corev1ac.KeyToPath().WithKey("metadata").WithPath("agent.metadata.json"),
272298
),
273299
),
274-
)
300+
}
301+
302+
// Add secret volume if UseSecretAsEnvVars is false
303+
if !lightrunJavaAgent.Spec.UseSecretAsEnvVars {
304+
volumes = append(volumes,
305+
corev1ac.Volume().WithName("lightrun-secret").
306+
WithSecret(corev1ac.SecretVolumeSource().
307+
WithSecretName(secret.Name).
308+
WithItems(
309+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
310+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
311+
).
312+
WithDefaultMode(0440)),
313+
)
314+
}
315+
316+
statefulSetApplyConfig.Spec.Template.Spec.WithVolumes(volumes...)
275317
}
276318

277319
func (r *LightrunJavaAgentReconciler) addInitContainerToStatefulSet(statefulSetApplyConfig *appsv1ac.StatefulSetApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) {
278-
statefulSetApplyConfig.Spec.Template.Spec.WithInitContainers(
279-
corev1ac.Container().
280-
WithName(initContainerName).
281-
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
282-
WithVolumeMounts(
283-
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
284-
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
285-
).WithEnv(
320+
// Start with base environment variables
321+
envVars := []*corev1ac.EnvVarApplyConfiguration{
322+
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
323+
}
324+
325+
// Add secret environment variables if UseSecretAsEnvVars is true
326+
if lightrunJavaAgent.Spec.UseSecretAsEnvVars {
327+
envVars = append(envVars,
286328
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
287329
corev1ac.EnvVarSource().WithSecretKeyRef(
288330
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
@@ -293,8 +335,35 @@ func (r *LightrunJavaAgentReconciler) addInitContainerToStatefulSet(statefulSetA
293335
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
294336
),
295337
),
296-
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
297-
).
338+
)
339+
}
340+
341+
// Start with base volume mounts
342+
volumeMounts := []*corev1ac.VolumeMountApplyConfiguration{
343+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
344+
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
345+
}
346+
347+
// Add secret volume mount if UseSecretAsEnvVars is false
348+
if !lightrunJavaAgent.Spec.UseSecretAsEnvVars {
349+
volumeMounts = append(volumeMounts,
350+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
351+
)
352+
}
353+
354+
statefulSetApplyConfig.Spec.Template.Spec.WithInitContainers(
355+
corev1ac.Container().
356+
WithName(initContainerName).
357+
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
358+
WithVolumeMounts(volumeMounts...).
359+
WithEnv(envVars...).
360+
WithSecurityContext(
361+
corev1ac.SecurityContext().
362+
WithReadOnlyRootFilesystem(true).
363+
WithAllowPrivilegeEscalation(false).
364+
WithRunAsNonRoot(true).
365+
WithRunAsUser(1000),
366+
).
298367
WithResources(
299368
corev1ac.ResourceRequirements().
300369
WithLimits(
@@ -308,18 +377,6 @@ func (r *LightrunJavaAgentReconciler) addInitContainerToStatefulSet(statefulSetA
308377
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
309378
},
310379
),
311-
).
312-
WithSecurityContext(
313-
corev1ac.SecurityContext().
314-
WithCapabilities(
315-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
316-
).
317-
WithAllowPrivilegeEscalation(false).
318-
WithRunAsNonRoot(true).
319-
WithSeccompProfile(
320-
corev1ac.SeccompProfile().
321-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
322-
),
323380
),
324381
)
325382
}
@@ -335,7 +392,7 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJava
335392
WithName(container.Name).
336393
WithImage(container.Image).
337394
WithVolumeMounts(
338-
corev1ac.VolumeMount().WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath).WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName),
395+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath),
339396
),
340397
)
341398
}

‎lightrun-init-agent/Dockerfile‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
ARG base_image_tag=alpine-3.20.0-r1
22

33
FROM lightruncom/prod-base:${base_image_tag}
4-
ARG FILE
4+
ARG FILE
55

6-
COPY lightrun-init-agent/$FILE /tmp/$FILE
6+
COPY $FILE /tmp/$FILE
77

88
RUN unzip -o /tmp/$FILE -d /agent ;\
99
rm -rf /tmp/$FILE && \
@@ -12,9 +12,20 @@ RUN unzip -o /tmp/$FILE -d /agent ;\
1212
sed -i.bak "s|pinned_certs=.*|pinned_certs=|" /agent/agent.config && rm /agent/agent.config.bak && \
1313
# In openshift UID will be dynamic per project, hence procide permissions to root group (defualt in k8s)
1414
chgrp -R 0 /agent && \
15-
chmod -R g=u /agent
15+
chmod -R g=u /agent && \
16+
# Set proper permissions for the agent directory
17+
chown -R 1000:1000 /agent && \
18+
chmod -R 750 /agent && \
19+
# Create secret directory with proper permissions
20+
mkdir -p /etc/lightrun/secret && \
21+
chown -R 1000:1000 /etc/lightrun/secret && \
22+
chmod -R 700 /etc/lightrun/secret
23+
24+
# Copy and set permissions for update_config.sh before switching user
25+
COPY update_config.sh /update_config.sh
26+
RUN chmod 750 /update_config.sh && \
27+
chown 1000:1000 /update_config.sh
1628

1729
USER 1000
18-
COPY lightrun-init-agent/update_config.sh /update_config.sh
1930

2031
CMD [ "/bin/sh", "/update_config.sh" ]

0 commit comments

Comments
 (0)