project-runner: quick proof of concept of rsync first

Author: williamstein

src/packages/backend/data.ts

@@ -288,7 +288,7 @@ export const sshServer: { name: string; host: string; port: number } = (() => {
     process.env.COCALC_SSH_SERVER ?? "host.containers.internal"
   ).split(":");
   return {
-    name: "cocalc-core",
+    name: "file-server",
     host: host ? host : "host.containers.internal",
     port: parseInt(port),
   };

src/packages/backend/mutagen/ssh-keys.ts

@@ -48,7 +48,7 @@ Host ${name}
     }
     if (!config.includes(hostConfig)) {
       // put at front since only the first with a given name is used by ssh
-      await writeFile(configPath, hostConfig + "\n" + config);
+      await writeFile(configPath, hostConfig + "\n" + config, { mode: 0o700 });
     }
   }
 }

src/packages/conat/project/runner/run.ts

@@ -39,10 +39,18 @@ export interface Options {
     sshServers?: SshServersFunction;
   }) => Promise<void>;
   // ensure a specific project is not running on this runner
-  stop: (opts: { project_id: string }) => Promise<void>;
+  stop: (opts: {
+    project_id: string;
+    localPath: LocalPathFunction;
+    sshServers?: SshServersFunction;
+  }) => Promise<void>;
   // get the status of a project here.
 
-  status: (opts: { project_id: string }) => Promise<ProjectStatus>;
+  status: (opts: {
+    project_id: string;
+    localPath: LocalPathFunction;
+    sshServers?: SshServersFunction;
+  }) => Promise<ProjectStatus>;
   // local -- the absolute path on the filesystem where the home directory of this
   // project is hosted.  In case of a single server setup it could be the exact
   // same path as the remote files and no sync is involved.
@@ -110,14 +118,25 @@ export async function server(options: Options) {
     async stop(opts: { project_id: string }) {
       logger.debug("stop", opts.project_id);
       projects.set(opts.project_id, { server: id, state: "stopping" } as const);
-      await stop(opts);
+      await stop({
+        ...opts,
+        localPath: options.localPath,
+        sshServers: options.sshServers,
+      });
       const s = { server: id, state: "opened" } as const;
       projects.set(opts.project_id, s);
       return s;
     },
     async status(opts: { project_id: string }) {
       logger.debug("status", opts.project_id);
-      const s = { ...(await status(opts)), server: id };
+      const s = {
+        ...(await status({
+          ...opts,
+          localPath: options.localPath,
+          sshServers: options.sshServers,
+        })),
+        server: id,
+      };
       projects.set(opts.project_id, s);
       return s;
     },
@@ -134,11 +153,18 @@ export async function server(options: Options) {
 
 export function client({
   client,
+  project_id,
   subject,
+  timeout,
+  waitForInterest = true,
 }: {
   client?: Client;
-  subject: string;
+  project_id?: string;
+  subject?: string;
+  timeout?: number;
+  waitForInterest?: boolean;
 }): API {
+  subject ??= `project.${project_id}.run`;
   client ??= conat();
-  return client.call<API>(subject, { waitForInterest: true });
+  return client.call<API>(subject, { waitForInterest, timeout });
 }

src/packages/conat/project/runner/state.ts

@@ -10,7 +10,7 @@ export type ProjectState = "running" | "opened" | "stopping" | "starting";
 export interface ProjectStatus {
   server?: string;
   state: ProjectState;
-  ip?: string; // the ip address when running
+  publicKey?: string; // ed25519 ssh public key
 }
 
 export default async function state({ client }) {

src/packages/file-server/ssh/auth.ts

@@ -12,6 +12,7 @@ import {
   client as createFileClient,
   type Fileserver,
 } from "@cocalc/conat/files/file-server";
+import { client as projectRunnerClient } from "@cocalc/conat/project/runner/run";
 
 const logger = getLogger("file-server:ssh:auth");
 
@@ -98,13 +99,28 @@ async function handleRequest(
     const volume = `project-${project_id}`;
     const id = user.slice("project-".length + 37);
     const compute_server_id = parseInt(id ? id : "0");
-    const api = projectApiClient({
-      project_id,
-      compute_server_id,
-      client,
-      timeout: 5000,
-    });
-    const authorizedKeys = await api.system.sshPublicKey();
+    let authorizedKeys;
+    if (!compute_server_id) {
+      const runner = projectRunnerClient({
+        client,
+        project_id,
+        timeout: 5000,
+        waitForInterest: false,
+      });
+      const s = await runner.status({ project_id });
+      authorizedKeys = s.publicKey;
+      if (!authorizedKeys) {
+        throw Error("no ssh key known");
+      }
+    } else {
+      const api = projectApiClient({
+        project_id,
+        compute_server_id,
+        client,
+        timeout: 5000,
+      });
+      authorizedKeys = await api.system.sshPublicKey();
+    }
 
     // NOTE/TODO: we could have a special username that maps to a
     // specific path in a project, which would change this path here,

src/packages/frontend/conat/client.ts

@@ -473,7 +473,7 @@ export class ConatClient extends EventEmitter {
 
   projectRunner = (project_id: string) => {
     return projectRunnerClient({
-      subject: `project.${project_id}.run`,
+      project_id,
       client: this.conat(),
     });
   };

src/packages/project-runner/run/filesystem.ts

@@ -4,6 +4,8 @@ import {
 } from "@cocalc/conat/files/file-server";
 import { type Client as ConatClient } from "@cocalc/conat/core/client";
 import { sshServer as defaultSshServer } from "@cocalc/backend/data";
+import { join } from "node:path";
+import { mkdir } from "node:fs/promises";
 
 //import getLogger from "@cocalc/backend/logger";
 
@@ -37,6 +39,11 @@ export async function localPath({
 }: {
   project_id: string;
 }): Promise<string> {
+  if (process.env.COCALC_PROJECT_PATH) {
+    const path = join(process.env.COCALC_PROJECT_PATH, project_id);
+    await mkdir(path, { recursive: true });
+    return path;
+  }
   const c = getFsClient();
   const { path } = await c.mount({ project_id });
   return path;

src/packages/project-runner/run/podman.ts

@@ -20,7 +20,7 @@ import { nodePath } from "./mounts";
 import { isValidUUID } from "@cocalc/util/misc";
 import { ensureConfFilesExists, setupDataPath, writeSecretToken } from "./util";
 import { getEnvironment } from "./env";
-import { mkdir } from "fs/promises";
+import { mkdir, readFile } from "node:fs/promises";
 import { spawn } from "node:child_process";
 import { getCoCalcMounts, COCALC_SRC } from "./mounts";
 import { setQuota } from "./filesystem";
@@ -76,27 +76,31 @@ export async function start({
 
   const home = await localPath({ project_id });
   logger.debug("start: got home", { project_id, home });
+  const mounts = getCoCalcMounts();
+  const image = getImage(config);
+  await initSshKeys({ home, sshServers: await sshServers?.({ project_id }) });
+
+  const env = await getEnvironment({
+    project_id,
+    env: config?.env,
+    HOME: "/root",
+    image,
+  });
+  await startSidecar({ project_id, home, mounts, env, pod });
+
   const rootfs = await rootFilesystem.mount({ project_id, home, config });
   logger.debug("start: got rootfs", { project_id, rootfs });
   await mkdir(home, { recursive: true });
   logger.debug("start: created home", { project_id });
   await ensureConfFilesExists(home);
   logger.debug("start: created conf files", { project_id });
-  const image = getImage(config);
 
   await writeMutagenConfig({
     home,
     sync: config?.sync,
     forward: config?.forward,
   });
-  await initSshKeys({ home, sshServers: await sshServers?.({ project_id }) });
 
-  const env = await getEnvironment({
-    project_id,
-    env: config?.env,
-    HOME: "/root",
-    image,
-  });
   await setupDataPath(home);
   logger.debug("start: setup data path", { project_id });
   if (config?.secret) {
@@ -124,7 +128,6 @@ export async function start({
   args.push("--hostname", `project-${project_id}`);
   args.push("--name", `project-${project_id}`);
 
-  const mounts = getCoCalcMounts();
   for (const path in mounts) {
     args.push("-v", `${path}:${mounts[path]}:ro`);
   }
@@ -154,10 +157,14 @@ export async function start({
   child.stderr.on("data", (chunk: Buffer) => {
     logger.debug(`project_id=${project_id}.stderr: `, chunk.toString());
   });
+}
 
+async function startSidecar({ project_id, mounts, env, pod, home }) {
+  // sidecar: refactor
+  const sidecarPodName = `sidecar-${project_id}`;
   const args2 = [
     "run",
-    `--name=sidecar-${project_id}`,
+    `--name=${sidecarPodName}`,
     "--detach",
     "--rm",
     "--pod",
@@ -172,11 +179,21 @@ export async function start({
     args2.push("-e", `${name}=${env[name]}`);
   }
   args2.push(sidecarImageName, "mutagen", "daemon", "run");
-  // TODO: we don't block the startup for this, but we do
-  // need to check and provide info to the user about what happened.
-  (async () => {
-    await podman(args2);
-  })();
+  await podman(args2);
+  try {
+    await podman([
+      "exec",
+      sidecarPodName,
+      "rsync",
+      "-axH",
+      "--exclude=.mutagen",
+      "--delete",
+      "file-server:/root/",
+      "/root/",
+    ]);
+  } catch (err) {
+    console.log(err);
+  }
 }
 
 export async function stop({
@@ -265,13 +282,18 @@ async function state(project_id): Promise<ProjectState> {
   return status == "running" ? "running" : "opened";
 }
 
-export async function status({ project_id }) {
+export async function status({ project_id, localPath }) {
   if (!isValidUUID(project_id)) {
     throw Error("status: project_id must be valid");
   }
   logger.debug("status", { project_id });
-  // TODO
-  return { state: await state(project_id), ip: "127.0.0.1" };
+  const s = await state(project_id);
+  let publicKey: string | undefined = undefined;
+  const home = await localPath({ project_id });
+  try {
+    publicKey = await readFile(join(home, ".ssh", "id_ed25519.pub"), "utf8");
+  } catch {}
+  return { state: s, ip: "127.0.0.1", publicKey };
 }
 
 export async function close() {

src/packages/project-runner/run/sidecar.ts

@@ -32,10 +32,10 @@ import { build } from "@cocalc/backend/podman/build-container";
 
 const Dockerfile = `
 FROM docker.io/alpine:latest
-RUN apk update && apk add --no-cache openssh-client
+RUN apk update && apk add --no-cache openssh-client rsync
 `;
 
-export const sidecarImageName = "localhost/sidecar:0.2";
+export const sidecarImageName = "localhost/sidecar:0.3";
 
 export async function init() {
   await build({ name: sidecarImageName, Dockerfile });

src/packages/server/conat/project/load-balancer.ts

@@ -11,7 +11,7 @@ import { loadConatConfiguration } from "../configuration";
 import getLogger from "@cocalc/backend/logger";
 import getPool from "@cocalc/database/pool";
 import { getProject } from "@cocalc/server/projects/control";
-import { type Configuration } from "@cocalc/project-runner/run";
+import { type Configuration } from "@cocalc/conat/project/runner/types";
 import { getProjectSecretToken } from "@cocalc/server/projects/control/secret-token";
 
 const logger = getLogger("server:conat:project:load-balancer");