Skip to content

Commit 5a37bcc

Browse files
yitzhaktalyitzhaktal
committed
DEVOPS-2694 - Update the lightrun-k8s-operator deployment to mount Secrets as files via volumes instead of exposing them as environment variables in containers.
1 parent 2be38da commit 5a37bcc

File tree

4 files changed

+133
-85
lines changed

4 files changed

+133
-85
lines changed

‎internal/controller/lightrunjavaagent_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ func (r *LightrunJavaAgentReconciler) reconcileDeployment(ctx context.Context, l
252252

253253
// Create config map
254254
log.V(2).Info("Reconciling config map with agent configuration")
255-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
255+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
256256
if err != nil {
257257
log.Error(err, "unable to create configMap")
258258
return r.errorStatus(ctx, lightrunJavaAgent, err)
@@ -493,7 +493,7 @@ func (r *LightrunJavaAgentReconciler) reconcileStatefulSet(ctx context.Context,
493493

494494
// Create config map
495495
log.V(2).Info("Reconciling config map with agent configuration")
496-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
496+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
497497
if err != nil {
498498
log.Error(err, "unable to create configMap")
499499
return r.errorStatus(ctx, lightrunJavaAgent, err)

‎internal/controller/patch_funcs.go‎

Lines changed: 54 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ const (
2727
annotationAgentName = "lightrun.com/lightrunjavaagent"
2828
)
2929

30-
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent) (corev1.ConfigMap, error) {
30+
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) (corev1.ConfigMap, error) {
3131
populateTags(lightrunJavaAgent.Spec.AgentTags, lightrunJavaAgent.Spec.AgentName, &metadata)
3232
jsonString, err := json.Marshal(metadata)
3333
if err != nil {
@@ -52,16 +52,14 @@ func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agent
5252
}
5353

5454
func (r *LightrunJavaAgentReconciler) patchDeployment(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, cmDataHash uint64) error {
55-
5655
// init spec.template.spec
5756
deploymentApplyConfig.WithSpec(
5857
appsv1ac.DeploymentSpec().WithTemplate(
5958
corev1ac.PodTemplateSpec().WithSpec(
6059
corev1ac.PodSpec(),
6160
).WithAnnotations(map[string]string{
6261
annotationConfigMapHash: fmt.Sprint(cmDataHash),
63-
},
64-
),
62+
}),
6563
),
6664
).WithAnnotations(map[string]string{
6765
annotationAgentName: lightrunJavaAgent.Name,
@@ -72,6 +70,10 @@ func (r *LightrunJavaAgentReconciler) patchDeployment(lightrunJavaAgent *agentv1
7270
if err != nil {
7371
return err
7472
}
73+
deploymentApplyConfig.Spec.Template.Spec.WithSecurityContext(
74+
corev1ac.PodSecurityContext().
75+
WithFSGroup(1000),
76+
)
7577
return nil
7678
}
7779

@@ -99,54 +101,53 @@ func (r *LightrunJavaAgentReconciler) addVolume(deploymentApplyConfig *appsv1ac.
99101
}
100102

101103
func (r *LightrunJavaAgentReconciler) addInitContainer(deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) {
102-
103104
deploymentApplyConfig.Spec.Template.Spec.WithInitContainers(
104105
corev1ac.Container().
105106
WithName(initContainerName).
106107
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
107108
WithVolumeMounts(
108109
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
109110
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
110-
).WithEnv(
111-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
112-
corev1ac.EnvVarSource().WithSecretKeyRef(
113-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
114-
),
115-
),
116-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
117-
corev1ac.EnvVarSource().WithSecretKeyRef(
118-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
119-
),
120-
),
121-
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
122-
).
111+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
112+
).
113+
WithEnv(
114+
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
115+
).
116+
WithSecurityContext(
117+
corev1ac.SecurityContext().
118+
WithReadOnlyRootFilesystem(true).
119+
WithAllowPrivilegeEscalation(false).
120+
WithRunAsNonRoot(true).
121+
WithRunAsUser(1000),
122+
).
123123
WithResources(
124124
corev1ac.ResourceRequirements().
125125
WithLimits(
126126
corev1.ResourceList{
127127
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
128128
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 500 * 10^6 = 500M
129129
},
130-
).WithRequests(
131-
corev1.ResourceList{
132-
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
133-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
134-
},
135-
),
136-
).
137-
WithSecurityContext(
138-
corev1ac.SecurityContext().
139-
WithCapabilities(
140-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
141130
).
142-
WithAllowPrivilegeEscalation(false).
143-
WithRunAsNonRoot(true).
144-
WithSeccompProfile(
145-
corev1ac.SeccompProfile().
146-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
131+
WithRequests(
132+
corev1.ResourceList{
133+
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
134+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
135+
},
147136
),
148137
),
149138
)
139+
140+
// Add volume for secret with proper permissions
141+
deploymentApplyConfig.Spec.Template.Spec.WithVolumes(
142+
corev1ac.Volume().WithName("lightrun-secret").
143+
WithSecret(corev1ac.SecretVolumeSource().
144+
WithSecretName(secret.Name).
145+
WithItems(
146+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
147+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
148+
).
149+
WithDefaultMode(0440)),
150+
)
150151
}
151152

152153
func (r *LightrunJavaAgentReconciler) patchAppContainers(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration) error {
@@ -230,19 +231,16 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSet(lightrunJavaAgent *agentv
230231
corev1ac.PodSpec(),
231232
).WithAnnotations(map[string]string{
232233
annotationConfigMapHash: fmt.Sprint(cmDataHash),
233-
},
234-
),
234+
}),
235235
),
236236
).WithAnnotations(map[string]string{
237237
annotationAgentName: lightrunJavaAgent.Name,
238238
})
239239

240240
// Add volumes to the StatefulSet
241241
r.addVolumeToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent)
242-
243242
// Add init container to the StatefulSet
244243
r.addInitContainerToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent, secret)
245-
246244
// Patch app containers in the StatefulSet
247245
err = r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
248246
if err != nil {
@@ -271,6 +269,15 @@ func (r *LightrunJavaAgentReconciler) addVolumeToStatefulSet(statefulSetApplyCon
271269
corev1ac.KeyToPath().WithKey("metadata").WithPath("agent.metadata.json"),
272270
),
273271
),
272+
).WithVolumes(
273+
corev1ac.Volume().WithName("lightrun-secret").
274+
WithSecret(corev1ac.SecretVolumeSource().
275+
WithSecretName(secret.Name).
276+
WithItems(
277+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
278+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
279+
).
280+
WithDefaultMode(0440)),
274281
)
275282
}
276283

@@ -282,19 +289,17 @@ func (r *LightrunJavaAgentReconciler) addInitContainerToStatefulSet(statefulSetA
282289
WithVolumeMounts(
283290
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
284291
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
292+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
285293
).WithEnv(
286-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
287-
corev1ac.EnvVarSource().WithSecretKeyRef(
288-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
289-
),
290-
),
291-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
292-
corev1ac.EnvVarSource().WithSecretKeyRef(
293-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
294-
),
295-
),
296294
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
297295
).
296+
WithSecurityContext(
297+
corev1ac.SecurityContext().
298+
WithReadOnlyRootFilesystem(true).
299+
WithAllowPrivilegeEscalation(false).
300+
WithRunAsNonRoot(true).
301+
WithRunAsUser(1000),
302+
).
298303
WithResources(
299304
corev1ac.ResourceRequirements().
300305
WithLimits(
@@ -308,18 +313,6 @@ func (r *LightrunJavaAgentReconciler) addInitContainerToStatefulSet(statefulSetA
308313
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
309314
},
310315
),
311-
).
312-
WithSecurityContext(
313-
corev1ac.SecurityContext().
314-
WithCapabilities(
315-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
316-
).
317-
WithAllowPrivilegeEscalation(false).
318-
WithRunAsNonRoot(true).
319-
WithSeccompProfile(
320-
corev1ac.SeccompProfile().
321-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
322-
),
323316
),
324317
)
325318
}
@@ -335,7 +328,7 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJava
335328
WithName(container.Name).
336329
WithImage(container.Image).
337330
WithVolumeMounts(
338-
corev1ac.VolumeMount().WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath).WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName),
331+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath),
339332
),
340333
)
341334
}

‎lightrun-init-agent/Dockerfile‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
ARG base_image_tag=alpine-3.20.0-r1
22

33
FROM lightruncom/prod-base:${base_image_tag}
4-
ARG FILE
4+
ARG FILE
55

6-
COPY lightrun-init-agent/$FILE /tmp/$FILE
6+
COPY $FILE /tmp/$FILE
77

88
RUN unzip -o /tmp/$FILE -d /agent ;\
99
rm -rf /tmp/$FILE && \
@@ -12,9 +12,20 @@ RUN unzip -o /tmp/$FILE -d /agent ;\
1212
sed -i.bak "s|pinned_certs=.*|pinned_certs=|" /agent/agent.config && rm /agent/agent.config.bak && \
1313
# In openshift UID will be dynamic per project, hence procide permissions to root group (defualt in k8s)
1414
chgrp -R 0 /agent && \
15-
chmod -R g=u /agent
15+
chmod -R g=u /agent && \
16+
# Set proper permissions for the agent directory
17+
chown -R 1000:1000 /agent && \
18+
chmod -R 750 /agent && \
19+
# Create secret directory with proper permissions
20+
mkdir -p /etc/lightrun/secret && \
21+
chown -R 1000:1000 /etc/lightrun/secret && \
22+
chmod -R 700 /etc/lightrun/secret
23+
24+
# Copy and set permissions for update_config.sh before switching user
25+
COPY update_config.sh /update_config.sh
26+
RUN chmod 750 /update_config.sh && \
27+
chown 1000:1000 /update_config.sh
1628

1729
USER 1000
18-
COPY lightrun-init-agent/update_config.sh /update_config.sh
1930

2031
CMD [ "/bin/sh", "/update_config.sh" ]

‎lightrun-init-agent/update_config.sh‎

Lines changed: 62 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
#!/bin/sh
22
# Script to initialize and configure the Lightrun agent
33
# This script:
4-
# 1. Validates required environment variables
4+
# 1. Validates required environment variables and files
55
# 2. Sets up a working directory
66
# 3. Merges configuration files
7-
# 4. Updates configuration with environment variables
7+
# 4. Updates configuration with values from files
88
# 5. Copies the final configuration to destination
99

1010
set -e
@@ -14,23 +14,58 @@ TMP_DIR="/tmp"
1414
WORK_DIR="${TMP_DIR}/agent-workdir"
1515
FINAL_DEST="${TMP_DIR}/agent"
1616
CONFIG_MAP_DIR="${TMP_DIR}/cm"
17+
SECRET_DIR="/etc/lightrun/secret"
18+
19+
# Function to get value from either environment variable or file
20+
get_value() {
21+
local env_var=$1
22+
local file_path=$2
23+
local value=""
24+
25+
# First try environment variable
26+
eval "value=\$${env_var}"
27+
if [ -n "${value}" ]; then
28+
echo "[WARNING] Using environment variable ${env_var}. Please upgrade to the latest operator version for better security and management." >&2
29+
echo "[INFO] Using value from environment variable ${env_var}" >&2
30+
echo "${value}"
31+
return 0
32+
fi
33+
34+
# Then try file
35+
if [ -f "${file_path}" ]; then
36+
value=$(cat "${file_path}")
37+
echo "[INFO] Using value from file ${file_path}" >&2
38+
echo "${value}"
39+
return 0
40+
fi
1741

18-
# Function to validate required environment variables
42+
echo ""
43+
return 1
44+
}
45+
46+
# Function to validate required files and environment variables
1947
validate_env_vars() {
20-
local missing_vars=""
48+
local missing_requirements=""
2149

22-
if [ -z "${LIGHTRUN_KEY}" ]; then
23-
missing_vars="${missing_vars} LIGHTRUN_KEY"
50+
# Check for LIGHTRUN_SERVER (required in both old and new versions)
51+
if [ -z "${LIGHTRUN_SERVER}" ]; then
52+
missing_requirements="${missing_requirements} LIGHTRUN_SERVER"
2453
fi
25-
if [ -z "${PINNED_CERT}" ]; then
26-
missing_vars="${missing_vars} PINNED_CERT"
54+
55+
# Check for lightrun_key (either env var or file)
56+
local lightrun_key=$(get_value "LIGHTRUN_KEY" "${SECRET_DIR}/lightrun_key")
57+
if [ -z "${lightrun_key}" ]; then
58+
missing_requirements="${missing_requirements} LIGHTRUN_KEY"
2759
fi
28-
if [ -z "${LIGHTRUN_SERVER}" ]; then
29-
missing_vars="${missing_vars} LIGHTRUN_SERVER"
60+
61+
# Check for pinned_cert (either env var or file)
62+
local pinned_cert=$(get_value "PINNED_CERT" "${SECRET_DIR}/pinned_cert_hash")
63+
if [ -z "${pinned_cert}" ]; then
64+
missing_requirements="${missing_requirements} PINNED_CERT"
3065
fi
3166

32-
if [ -n "${missing_vars}" ]; then
33-
echo "Error: Missing required environment variables:${missing_vars}"
67+
if [ -n "${missing_requirements}" ]; then
68+
echo "Error: Missing required environment variables or files:${missing_requirements}"
3469
exit 1
3570
fi
3671
}
@@ -64,27 +99,36 @@ merge_configs() {
6499
rm "${temp_conf}"
65100
}
66101

67-
# Function to update configuration with environment variables
102+
# Function to update configuration with values from files
68103
update_config() {
69-
echo "Updating configuration with environment variables"
104+
echo "Updating configuration with values from files"
70105
local config_file="${WORK_DIR}/agent.config"
71106
local missing_configuration_params=""
72107

108+
if [ ! -f "${config_file}" ]; then
109+
echo "[ERROR] Config file not found at ${config_file}"
110+
exit 1
111+
fi
112+
113+
# Get values from either environment variables or files
114+
local lightrun_key=$(get_value "LIGHTRUN_KEY" "${SECRET_DIR}/lightrun_key")
115+
local pinned_cert=$(get_value "PINNED_CERT" "${SECRET_DIR}/pinned_cert_hash")
116+
73117
if sed -n "s|com.lightrun.server=.*|com.lightrun.server=https://${LIGHTRUN_SERVER}|p" "${config_file}" | grep -q .; then
74118
# Perform actual in-place change
75119
sed -i "s|com.lightrun.server=.*|com.lightrun.server=https://${LIGHTRUN_SERVER}|" "${config_file}"
76120
else
77121
missing_configuration_params="${missing_configuration_params} com.lightrun.server"
78122
fi
79-
if sed -n "s|com.lightrun.secret=.*|com.lightrun.secret=${LIGHTRUN_KEY}|p" "${config_file}" | grep -q .; then
123+
if sed -n "s|com.lightrun.secret=.*|com.lightrun.secret=${lightrun_key}|p" "${config_file}" | grep -q .; then
80124
# Perform actual in-place change
81-
sed -i "s|com.lightrun.secret=.*|com.lightrun.secret=${LIGHTRUN_KEY}|" "${config_file}"
125+
sed -i "s|com.lightrun.secret=.*|com.lightrun.secret=${lightrun_key}|" "${config_file}"
82126
else
83127
missing_configuration_params="${missing_configuration_params} com.lightrun.secret"
84128
fi
85-
if sed -n "s|pinned_certs=.*|pinned_certs=${PINNED_CERT}|p" "${config_file}" | grep -q .; then
129+
if sed -n "s|pinned_certs=.*|pinned_certs=${pinned_cert}|p" "${config_file}" | grep -q .; then
86130
# Perform actual in-place change
87-
sed -i "s|pinned_certs=.*|pinned_certs=${PINNED_CERT}|" "${config_file}"
131+
sed -i "s|pinned_certs=.*|pinned_certs=${pinned_cert}|" "${config_file}"
88132
else
89133
missing_configuration_params="${missing_configuration_params} pinned_certs"
90134
fi

0 commit comments

Comments
 (0)