roachprod: auto restart option for cockroach processes

Previously, if a cockroach process died, it needed to be manually restarted.
This is expected in most circumstances. But this behavior might not always be
practical, for instance, when running a long unattended benchmark.

To support this, a new start option has been added that automatically restarts
the service if a process fails (e.g., due to a disk stall). This ensures the
process comes back online quickly without manual intervention.

Epic: None
Release note: None

Author: herkolategan

pkg/roachprod/install/cockroach.go

@@ -152,6 +152,9 @@ type StartOpts struct {
 	// initialization and sequential node starts and also reuses the previous start script.
 	IsRestart bool
 
+	// AutoRestart enables automatically restarting a process if it died.
+	AutoRestart bool
+
 	// EnableFluentSink determines whether to enable the fluent-servers attribute
 	// in the CockroachDB logging configuration.
 	EnableFluentSink bool
@@ -1004,6 +1007,7 @@ func (c *SyncedCluster) generateStartCmd(
 		NumFilesLimit:       startOpts.NumFilesLimit,
 		VirtualClusterLabel: VirtualClusterLabel(startOpts.VirtualClusterName, startOpts.SQLInstance),
 		Local:               c.IsLocal(),
+		AutoRestart:         startOpts.AutoRestart,
 	})
 }
 
@@ -1017,6 +1021,7 @@ type startTemplateData struct {
 	VirtualClusterLabel string
 	Args                []string
 	EnvVars             []string
+	AutoRestart         bool
 }
 
 type loggingTemplateData struct {

pkg/roachprod/install/scripts/start.sh

@@ -16,6 +16,10 @@ KEY_CMD=#{.KeyCmd#}
 MEMORY_MAX=#{.MemoryMax#}
 NUM_FILES_LIMIT=#{.NumFilesLimit#}
 VIRTUAL_CLUSTER_LABEL=#{.VirtualClusterLabel#}
+#{if .AutoRestart#}
+AUTO_RESTART=1
+#{end#}
+
 ARGS=(
 #{range .Args -#}
 #{shesc .#}
@@ -99,4 +103,5 @@ sudo systemd-run --unit "${VIRTUAL_CLUSTER_LABEL}" \
   -p "MemoryMax=${MEMORY_MAX}" \
   -p LimitCORE=infinity \
   -p "LimitNOFILE=${NUM_FILES_LIMIT}" \
+  ${AUTO_RESTART:+-p Restart=always -p RestartSec=5s} \
   bash "${0}" run