fixed some more bugs

2025-10-11 11:29:11 -04:00
parent b915ae88a4
commit 2659c51b02
6 changed files with 298 additions and 81 deletions
@@ -149,8 +149,8 @@ services:
    ollama-nextcloud:
        build: ./nextcloud
        restart: unless-stopped
-        ports:
-            - "13284:1111"
+        # ports:
+        #     - "13284:1111"
        env_file: .env
        networks:
            - internal
@@ -13,7 +13,6 @@ type CacheRow = {
 	updated_at: number
 };

-// TODO: CHANGEME WHEN RUNNING IN DOCKER
 export const CACHE_DIR = "/tmp",
 	db = new Database(path.resolve(`${CACHE_DIR}/index.sqlite`));

@@ -1,15 +1,26 @@
 import { Request } from "express";
 import { WebDAVClient } from "webdav";
 import { DIRS } from "./server";
+import fetch from 'node-fetch'
+import { XMLParser } from 'fast-xml-parser';


 export async function statOne(fpath: string, client: WebDAVClient) {
 	const parent = fpath.replace(/\/[^/]+$/, '') || '/',
 		base = fpath.split('/').filter(Boolean).pop(),
-		list = await client.getDirectoryContents(parent, { deep: false }),
+		list = await client.getDirectoryContents(parent, { deep: false }).catch(err => {
+			console.error(`error for "${parent}":`);
+			console.error(err);
+
+			if (err.response?.status === 404) {
+				throw `Path "${fpath}" not found`;
+			}
+
+			throw err;
+		}),
 		entry = Array.isArray(list) ? list.find((e: any) => e.basename === base) : undefined;

-	if (!entry) throw `not found: ${fpath}`;
+	if (!entry) throw `Path "${fpath}" not found`;
 	return { etag: entry.etag as string, size: Number(entry.size), mime: entry.mime as string | undefined };
 }

@@ -29,17 +40,88 @@ export const toRegExp = (spec: string): RegExp | null => {
 	}
 }

-export function checkIfHasPerms(req: Request) {
-	if (!req.body) {
+export function checkIfHasPerms(req: Request | string) {
+	try {
+		if (typeof req === 'string') {
+			if (req.includes('..')) return false;
+
+			return !DIRS || DIRS.find(r => req.match(r));
+		}
+		else if (!req.body) {
+			return false;
+		}
+
+		const { fpath, obj_type, deep, usecache } = req.body,
+			o = { fpath, obj_type, deep, usecache };
+
+		if (!fpath) return false;
+		if (!DIRS) return o;
+		if (fpath.includes('..')) return false;
+
+		return DIRS.find(r => fpath.match(r)) ? o : false;
+	}
+	catch (err) {
+		console.error(err);
 		return false;
 	}
-	const { fpath, obj_type, deep, usecache } = req.body,
-		o = { fpath, obj_type, deep, usecache };
-
-	if (!fpath) return false;
-	if (!DIRS) return o;
-	if (fpath.includes('..')) return false;
-
-	return DIRS.find(r => fpath.match(r)) ? o : false;
 }

+function buildCountPropfindXml() {
+	return `<?xml version="1.0" encoding="UTF-8"?>
+<d:propfind xmlns:d="DAV:"
+           xmlns:oc="http://owncloud.org/ns"
+           xmlns:nc="http://nextcloud.org/ns">
+  <d:prop>
+    <nc:contained-file-count/>
+    <nc:contained-folder-count/>
+    <oc:contained-file-count/>
+    <oc:contained-folder-count/>
+  </d:prop>
+</d:propfind>`;
+}
+
+const authHeaders = (() => {
+	const u = process.env.NEXTCLOUD_APP_ID!,
+		p = process.env.NEXTCLOUD_APP_PASS!,
+		token = Buffer.from(`${u}:${p}`).toString("base64");
+	return `Basic ${token}`;
+})();
+
+export async function getDirectoryFileCount(dirPath: string, client: WebDAVClient) {
+	const url = client.getFileDownloadLink(dirPath),
+		xml = buildCountPropfindXml();
+
+	const resp = await fetch(url, {
+		method: "PROPFIND",
+		headers: {
+			"Content-Type": "text/xml",
+			"Depth": "0",
+			"Authorization": authHeaders
+		},
+		body: xml
+	});
+
+	if (!resp.ok) {
+		throw resp;
+	}
+
+	// parse XML
+	const parser = new XMLParser({
+		ignoreAttributes: false,
+		attributeNamePrefix: "@_",
+		allowBooleanAttributes: true,
+		numberParseOptions: {
+			hex: false,
+			leadingZeros: false
+		}
+	}),
+		text = await resp.text(),
+		doc = parser.parse(text),
+		objs = doc['d:multistatus']['d:response']['d:propstat'],
+		obj = objs.find((o: any) => o['d:status'].includes("200"))['d:prop'];
+
+	return {
+		file_count: obj['nc:contained-file-count'],
+		folder: obj['nc:contained-folder-count']
+	};
+}
@@ -141,34 +141,29 @@
 			},
 			"put": {
 				"operationId": "nextcloudUploadFile",
-				"summary": "Upload a file",
-				"description": "uploads a file into a target directory in nextcloud via webdav. requires a multipart/form-data body with the file part and the destination directory. the server will not overwrite existing files.",
+				"summary": "Upload a file (application/json, base64-encoded)",
+				"description": "Uploads a file into a target directory in nextcloud via webdav. the server will not overwrite existing files. the body must be application/json and include `fdir` and `file.buffer` (base64).",
 				"requestBody": {
 					"required": true,
 					"content": {
-						"multipart/form-data": {
+						"application/json": {
 							"schema": {
-								"type": "object",
-								"properties": {
-									"fdir": {
-										"type": "string",
-										"description": "destination directory path (e.g., /Documents)"
-									},
-									"createnewdirs": {
-										"type": "boolean",
-										"default": false,
-										"description": "if true and the directory does not exist, create it recursively before uploading"
-									},
-									"file": {
-										"type": "string",
-										"format": "binary",
-										"description": "the file contents"
+								"$ref": "#/components/schemas/NewFileMeta"
+							},
+							"examples": {
+								"json-base64": {
+									"value": {
+										"fdir": "/Documents",
+										"createnewdirs": false,
+										"file": {
+											"filename": "file",
+											"originalname": "photo.jpg",
+											"mimetype": "image/jpeg",
+											"size": 52345,
+											"buffer": "/9j/4AAQSkZJRgABAQAAAQABAAD... (truncated base64)"
+										}
 									}
-								},
-								"required": [
-									"fdir",
-									"file"
-								]
+								}
 							}
 						}
 					}
@@ -204,7 +199,7 @@
 						}
 					},
 					"400": {
-						"description": "missing file upload field",
+						"description": "missing file upload field or invalid base64",
 						"content": {
 							"application/json": {
 								"schema": {
@@ -216,7 +211,7 @@
 										},
 										"error": {
 											"type": "string",
-											"example": "missing file"
+											"example": "missing file or invalid buffer"
 										}
 									}
 								}
@@ -268,7 +263,7 @@
 			"post": {
 				"operationId": "nextcloudListDirectory",
 				"summary": "List a directory",
-				"description": "lists directory entries from nextcloud webdav. supports shallow or deep listing.",
+				"description": "Lists directory entries from nextcloud webdav. Supports shallow or deep listing.",
 				"requestBody": {
 					"required": true,
 					"content": {
@@ -284,6 +279,16 @@
 										"type": "boolean",
 										"default": false,
 										"description": "whether to recurse into subdirectories"
+									},
+									"startInd": {
+										"type": "number",
+										"default": 0,
+										"description": "start listing from this index (for pagination)"
+									},
+									"limit": {
+										"type": "number",
+										"default": 250,
+										"description": "maximum number of entries to return"
 									}
 								},
 								"required": [
@@ -331,7 +336,7 @@
 			"get": {
 				"operationId": "nextcloudGetOpenapi",
 				"summary": "Serve OpenAPI schema",
-				"description": "serves this specification (used by open webui when registering a tool server).",
+				"description": "Serves this specification (used by open webui when registering a tool server).",
 				"responses": {
 					"200": {
 						"description": "openapi document",
@@ -352,7 +357,7 @@
 		"schemas": {
 			"DirEntry": {
 				"type": "object",
-				"description": "entry returned by webdav client",
+				"description": "Entry returned by webdav client",
 				"properties": {
 					"filename": {
 						"type": "string",
@@ -395,7 +400,7 @@
 			},
 			"FileMeta": {
 				"type": "object",
-				"description": "metadata about a file (json-friendly alternative to raw bytes)",
+				"description": "Metadata about a file (json-friendly alternative to raw bytes)",
 				"properties": {
 					"filename": {
 						"type": "string"
@@ -438,6 +443,62 @@
 					"basename",
 					"type"
 				]
+			},
+			"NewFileMeta": {
+				"type": "object",
+				"properties": {
+					"fdir": {
+						"type": "string",
+						"description": "destination directory path (e.g., /Documents)"
+					},
+					"createnewdirs": {
+						"type": "boolean",
+						"default": false,
+						"description": "if true and the directory does not exist, create it recursively before uploading"
+					},
+					"file": {
+						"type": "object",
+						"description": "uploaded file metadata + contents (base64 if image or misc formats otherwise)",
+						"properties": {
+							"filename": {
+								"type": "string",
+								"description": "name of the form field associated with this file or client-side identifier"
+							},
+							"originalname": {
+								"type": "string",
+								"default": "",
+								"description": "name of the file on the uploader's computer"
+							},
+							"mimetype": {
+								"type": "string",
+								"default": "text/plain",
+								"description": "value of the Content-Type for this file, e.g. image/jpeg"
+							},
+							"size": {
+								"type": "integer",
+								"format": "int64",
+								"default": 0,
+								"description": "size of the file in bytes"
+							},
+							"buffer": {
+								"type": "string",
+								"format": "byte",
+								"contentEncoding": "any",
+								"contentMediaType": "application/octet-stream",
+								"description": "the contents of the file"
+							}
+						},
+						"required": [
+							"filename",
+							"mimetype",
+							"buffer"
+						]
+					}
+				},
+				"required": [
+					"fdir",
+					"file"
+				]
 			}
 		},
 		"responses": {
@@ -500,4 +561,4 @@
 			}
 		}
 	}
-}
+}
@@ -2,7 +2,8 @@
  "name": "nextcloud",
  "private": true,
  "devDependencies": {
-    "@types/bun": "latest"
+    "@types/bun": "latest",
+    "@types/xmldom": "^0.1.34"
  },
  "peerDependencies": {
    "typescript": "^5"
@@ -10,10 +11,8 @@
  "dependencies": {
    "@types/cors": "^2.8.19",
    "@types/express": "^5.0.3",
-    "@types/multer": "^2.0.0",
    "cors": "^2.8.5",
    "express": "^5.1.0",
-    "multer": "^2.0.2",
    "webdav": "^5.8.0"
  }
 }
@@ -1,10 +1,10 @@
 import express from 'express';
-import * as multer from 'multer';
 import fs from 'fs/promises';
 import fsSync from 'fs';
+import { randomUUID } from 'crypto';
 import { createClient } from 'webdav';
 import { pathKey, fanoutPath, ensureDir, getRow, upsertRow, CACHE_DIR } from './cache.ts';
-import { checkIfHasPerms, statOne, toRegExp } from './helpers.ts';
+import { checkIfHasPerms, getDirectoryFileCount, statOne, toRegExp } from './helpers.ts';
 import path from 'path';

 type statRetType = {
@@ -17,14 +17,26 @@ type statRetType = {
 	"mime"?: string
 };

+type ReqFileType = {
+	filename: string; // Name of the form field associated with this file.
+	originalname?: string; // Name of the file on the uploader's computer.
+	mimetype: string; // Value of the Content-Type header for this file.
+	size?: number; // Size of the file in bytes.
+	buffer: Buffer; // A Buffer containing the file content.
+	localFilename: string; // the temporary ID assigned to the file
+};
+
 declare global {
 	namespace Express {
 		interface Request {
 			fobj?: {
 				fpath: any,
 				deep?: boolean,
-				bypasscache?: boolean
-			}
+				bypasscache?: boolean,
+				startInd?: number,
+				limit?: number
+			},
+			file: ReqFileType
 		}
 	}
 }
@@ -54,28 +66,25 @@ app.use((req, res, next) => {
 		return res.sendFile('openapi.json', { root: '.' });
 	}

-	// GET has no body (independant handlers)
-	else if (req.method === 'GET') return next();
-
+	// GET/PUT has independant handlers
+	else if (['GET', 'PUT'].includes(req.method)) return next();
 	const pth = checkIfHasPerms(req);
+
 	if (!pth && req.body?.fpath) {
 		return res.status(401).send(`Not allowed to access "${req.body.fpath}"`);
 	}
 	else if (!pth) {
 		return res.status(404).send("Unknown path");
 	}
+	else if (typeof pth === 'string') {
+		console.warn(`somehow got string out of checkIfHasPerms:\npth: ${pth}\nreq.body: ${req.body}`);
+		return res.sendStatus(400);
+	}

 	req.fobj = pth;
 	next();
 });

-//@ts-ignore stupid
-const multerInstance = multer.default({
-	storage: multer.diskStorage({
-		destination: CACHE_DIR
-	})
-});
-
 export const DIRS: string[] = (() => {
 	try {
 		if (!_DIRS) return null;
@@ -91,7 +100,7 @@ if (!DIRS) {

 const client = createClient(process.env.NEXTCLOUD_WEBDAV_ADDR!, {
 	username: process.env.NEXTCLOUD_APP_ID!,
-	password: process.env.NEXTCLOUD_APP_PASS!,
+	password: process.env.NEXTCLOUD_APP_PASS!
 });

 app.post('/file', async (req, res) => {
@@ -103,7 +112,8 @@ app.post('/file', async (req, res) => {
 		const { etag, size, mime } = await statOne(fpath, client).catch((err) => {
 			if (typeof err === 'string') {
 				console.error(err);
-				res.sendStatus(404).send(err);
+
+				if (!res.headersSent) res.sendStatus(404).send(err);
 				return { etag: undefined, size: -1, mime: -1 };
 			}

@@ -145,55 +155,121 @@ app.post('/file', async (req, res) => {
 		res.send(buf);
 	} catch (err) {
 		console.error(err);
-		res.sendStatus(500);
+		if (!res.headersSent) res.sendStatus(500);
 	}
 });

 app.post('/dir', async (req, res) => {
 	try {
 		if (!req.fobj) return res.sendStatus(401);
+		const { fpath, deep } = req.fobj,
+			{ startInd = 0, limit = 250 } = req.body;

-		const { fpath, deep } = req.fobj;
-    if (!fpath) return res.sendStatus(400);
-    else if (!(await client.exists(fpath)) {
-        return res.status(404).send("Directory not found");
-    }
+		if (!fpath) return res.sendStatus(400);

-		const filesInDir = await client.getDirectoryContents(fpath, {
-				deep
-			});
+		if (!(await client.exists(fpath))) {
+			return res.status(404).send("Directory not found");
+		}

-		res.json(filesInDir);
+		const getFilesInDir = async () => {
+			for (let i = 0; i < 3; i++) {
+				try {
+					return await client.getDirectoryContents(fpath, {
+						deep
+					});
+				}
+				catch (err: any) {
+					if (err.response?.status === 500) {
+						await new Promise(resolve => setTimeout(resolve, 2000));
+						continue;
+					}
+
+					console.error(`error for "${fpath}":`);
+					throw err;
+				}
+			}
+
+			const counts = await getDirectoryFileCount('/Photos', client);
+
+			let err = "failed to find dir contents (perhaps the server returned 500 too many times?)";
+			err += `\n"${fpath}" contains the following, consider listing directories individually (instead of using 'deep' if too many are present: ${JSON.stringify(counts)}`;
+
+			res.status(500).send(err);
+		}
+
+		const ret = await getFilesInDir();
+		if (!ret) return;
+
+		const filesInDir = Array.isArray(ret) ? ret : ret.data;
+		res.json(filesInDir.slice(startInd, limit));
 	}
 	catch (err) {
 		console.error(err);
-    res.sendStatus(500);
+		res.sendStatus(500);
 	}
 });

-app.put('/file', multerInstance.single('file'), async (req, res) => {
-	if (!req.fobj) return res.sendStatus(401);
-	if (!req.file) return res.status(400).send("missing file");
+// openwebui doesn't really help with `multerInstance.single('file'),`
+const ensureMockFile = async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+	try {
+		const { file, fdir } = req.body as { fdir: any, file: { filename: any, buffer: any, mimetype: any } };
+		if (!file) return res.status(400).send("missing file");
+		if (typeof fdir !== 'string') {
+			return res.status(400).send("Missing fdir in request");
+		}
+		if (typeof file.filename !== 'string') {
+			return res.status(400).send("Invalid filename");
+		}
+		if (typeof file.buffer === 'undefined') {
+			return res.status(400).send("Invalid file upload payload");
+		}
+
+		if (!checkIfHasPerms(fdir)) return res.sendStatus(401);
+
+		const fname = `${randomUUID().replaceAll('-', '')}_${file.filename}`,
+			fpath = path.join(CACHE_DIR, fname);
+
+		await fs.writeFile(fpath, file.buffer).catch((err) => {
+			console.error(err);
+			res.sendStatus(500);
+		});
+
+		if (res.headersSent) return;
+		req.file = { ...file, localFilename: fname, mimetype: req.body.mimetype || '' };
+
+		next();
+	}
+	catch (err) {
+		console.error(err);
+		if (!res.headersSent) res.sendStatus(500);
+	}
+};
+
+app.put('/file', ensureMockFile, async (req, res) => {
+	const file = req.file;
+	if (!file) return;

 	const { fdir, createnewdirs } = req.body as {
 		fdir: string,
 		createnewdirs: boolean
 	};

+	if (!fdir) return (res.headersSent || res.status(400).send("missing fdir in body"));
+
 	if (createnewdirs && !(await client.exists(fdir))) {
 		await client.createDirectory(fdir, {
 			recursive: true
 		});
 	}

-	const fname = req.file.filename || req.file.originalname,
-		fpath = path.join(fdir, fname);
+	const fname = file.filename || file.originalname,
+		fpath = path.join(fdir, fname!);

 	if (await client.exists(fpath)) {
 		return res.status(503).send('file already exists, you do not have permissions to overwrite files');
 	}

-	const sstr = fsSync.createReadStream(req.file.path).pipe(client.createWriteStream(fpath, {
+	const sstr = fsSync.createReadStream(path.join(CACHE_DIR, file.localFilename)).pipe(client.createWriteStream(fpath, {
 		overwrite: false
 	}, (r) => {
 		if (res.headersSent) return;