initial argo commit

2025-09-12 11:20:18 -04:00
parent 9153c3b1c6
commit e5cae3dc52
24 changed files with 642 additions and 79 deletions
@@ -1,5 +1,6 @@
 import http from "node:http";
-import { spawn } from "node:child_process";
+import * as k8s from "@kubernetes/client-node";
+

 const PORT = Number(process.env.PORT || 8787);
 const HOST = "0.0.0.0";
@@ -41,8 +42,14 @@ type fileType = {
 	content: string
 }

-// docker binary (or set DOCKER_BIN=podman)
-const DOCKER_BIN = process.env.DOCKER_BIN || "docker";
+const NS = process.env.NAMESPACE || "ai";
+
+const kc = new k8s.KubeConfig();
+kc.loadFromDefault(); // in-cluster uses serviceaccount
+
+const batch = kc.makeApiClient(k8s.BatchV1Api),
+	core = kc.makeApiClient(k8s.CoreV1Api);
+

 // basic openapi for open webui
 const OPENAPI = {
@@ -123,7 +130,7 @@ const OPENAPI = {
 };


-function sendJson(res, status, obj) {
+function sendJson(res: any, status: number, obj: any) {
 	const body = JSON.stringify(obj);
 	res.writeHead(status, { "content-type": "application/json; charset=utf-8" });
 	res.end(body);
@@ -175,63 +182,110 @@ async function ensureImage(spec: langObj) {
 	// 	}
 }

+async function waitForJobPod(core: k8s.CoreV1Api, jobName: string): Promise<string> {
+	const labelSelector = `job-name=${jobName}`;
+	for (; ;) {
+		const pods = await core.listNamespacedPod({ namespace: NS, labelSelector });
+		const pod = pods.items.find((p) => p.status?.phase === "Running" || p.status?.phase === "Succeeded" || p.status?.phase === "Failed");
+		if (pod?.metadata?.name) return pod.metadata.name;
+		await new Promise((r) => setTimeout(r, 400));
+	}
+};
+
+async function waitForCompletionAndLogs(core: k8s.CoreV1Api, podName: string): Promise<{ status: string; stdout: string; stderr: string; }> {
+	for (; ;) {
+		const readReq = {
+			name: podName,
+			namespace: NS
+		},
+			p = await core.readNamespacedPod(readReq),
+			phase = p.status?.phase ?? "Pending";
+
+		if (phase === "Succeeded" || phase === "Failed") {
+			const logs = await core.readNamespacedPodLog(readReq);
+			// stderr is not separated by the api; you can split by stream if needed
+			return { status: phase, stdout: logs, stderr: "" };
+		};
+		await new Promise((r) => setTimeout(r, 500));
+	}
+};
+
+
 async function runInContainer({ language, code, args = [], files = [] }: {
-	language: string,
-	code: string,
-	args: string[],
-	files: fileType[]
+	language: string, code: string, args: string[], files: fileType[]
 }) {
-	if (!LANGS[language]) throw new Error(`language not allowed: ${language}`);
-	const spec = LANGS[language];
+	if (!(language in LANGS)) throw new Error(`language not allowed: ${language}`);
+	const spec = LANGS[language as keyof typeof LANGS];

-	await ensureImage(spec);
-
-	// build the Docker args
-	const dockerArgs = [
-		"run", "--rm",
-		"--network=none", "--read-only",
-		"--pids-limit=256",
-		"--cpus=1", "--memory=512m",
-		"--cap-drop=ALL", "--security-opt", "no-new-privileges",
-		"--tmpfs", "/work:rw,exec,size=64m",
-		"-w", "/work",
-		"--pull=never",
-		spec.image
-	];
-
-	// inside the container, write the files and run code
+	// build the same shell script you already use
 	const script = [
-		// write the main file using base64
 		`echo ${JSON.stringify(Buffer.from(code, "utf8").toString("base64"))} | base64 -d > ${spec.filename}`,
-		// write any extra files using base64
 		...files.flatMap((f) => [
 			`mkdir -p "$(dirname "${f.path}")"`,
 			`echo ${JSON.stringify(Buffer.from(f.content, "utf8").toString("base64"))} | base64 -d > "${f.path}"`,
 		]),
-		// run it
 		`${spec.run.join(' ')} ${args.map((a) => JSON.stringify(a)).join(' ')}`
 	].join('\n');

-	dockerArgs.push("sh", "-lc", script);
+	const uid = crypto.randomUUID().slice(0, 8);
+	const jobName = `coderun-${uid}`;

-	const result = await new Promise((resolve) => {
-		const child = spawn(DOCKER_BIN, dockerArgs, { stdio: ["ignore", "pipe", "pipe"] });
-		let stdout = "", stderr = "";
+	// create a short-lived Job with tight security and resource caps
+	const job: k8s.V1Job = {
+		apiVersion: "batch/v1",
+		kind: "Job",
+		metadata: { name: jobName, namespace: NS },
+		spec: {
+			ttlSecondsAfterFinished: 300,         // auto-clean once done
+			backoffLimit: 0,                      // no retries
+			activeDeadlineSeconds: 25,            // mirrors your 25s timeout
+			template: {
+				metadata: { labels: { app: "coderunner-task" } },
+				spec: {
+					restartPolicy: "Never",
+					securityContext: {
+						runAsNonRoot: true,
+						seccompProfile: { type: "RuntimeDefault" }
+					},
+					containers: [{
+						name: "task",
+						image: spec.image,
+						command: ["sh", "-lc", script],
+						resources: {
+							requests: { cpu: "1", memory: "512Mi" },
+							limits: { cpu: "1", memory: "512Mi" }
+						},
+						securityContext: {
+							allowPrivilegeEscalation: false,
+							readOnlyRootFilesystem: true,
+							capabilities: { drop: ["ALL"] }
+						}
+					}]
+				}
+			}
+		}
+	};

-		const timer = setTimeout(() => {
-			child.kill("SIGKILL");
-			resolve({ stdout, stderr: stderr + "\n[killed: timeout]", exitCode: 137, timedOut: true });
-		}, 25_000);
-
-		child.stdout.on("data", (d) => { stdout += d.toString(); });
-		child.stderr.on("data", (d) => { stderr += d.toString(); });
-		child.on("close", (code) => {
-			clearTimeout(timer);
-			resolve({ stdout, stderr, exitCode: code ?? 1, timedOut: false });
-		});
+	await batch.createNamespacedJob({
+		namespace: NS,
+		body: job
 	});

-	return result;
+	// wait for pod to complete, then get logs
+	const podName = await waitForJobPod(core, jobName);
+	const { status, stdout, stderr } = await waitForCompletionAndLogs(core, podName);
+
+	// delete job for hygiene (ttl also cleans it eventually)
+	try {
+		await batch.deleteNamespacedJob({ namespace: NS, propagationPolicy: "Background", name: jobName });
+	} catch (err) { console.error(err); };
+
+	return {
+		stdout,
+		stderr,
+		exitCode: status === "Succeeded" ? 0 : 1,
+		timedOut: status === "Failed"
+	};
 }

 const server = http.createServer(async (req, res) => {
@@ -257,7 +311,7 @@ const server = http.createServer(async (req, res) => {
 					const payload = JSON.parse(body || "{}");
 					const out = await runInContainer(payload);
 					sendJson(res, 200, out);
-				} catch (e) {
+				} catch (e: any) {
 					sendJson(res, 400, { error: String(e?.message || e) });
 				}
 			});
@@ -266,7 +320,7 @@ const server = http.createServer(async (req, res) => {

 		res.writeHead(404);
 		res.end("not found");
-	} catch (e) {
+	} catch (e: any) {
 		sendJson(res, 500, { error: String(e?.message || e) });
 	}
 });