From 2659c51b025a4e578b977b20d665d491aa8cfb16 Mon Sep 17 00:00:00 2001 From: ION606 Date: Sat, 11 Oct 2025 11:29:11 -0400 Subject: [PATCH] fixed some more bugs --- docker-compose.yml | 4 +- nextcloud/cache.ts | 1 - nextcloud/helpers.ts | 106 ++++++++++++++++++++++++++---- nextcloud/openapi.json | 121 ++++++++++++++++++++++++++--------- nextcloud/package.json | 5 +- nextcloud/server.ts | 142 +++++++++++++++++++++++++++++++---------- 6 files changed, 298 insertions(+), 81 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 360719a..90f17ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -149,8 +149,8 @@ services: ollama-nextcloud: build: ./nextcloud restart: unless-stopped - ports: - - "13284:1111" + # ports: + # - "13284:1111" env_file: .env networks: - internal diff --git a/nextcloud/cache.ts b/nextcloud/cache.ts index ddd4d4a..06a6e02 100644 --- a/nextcloud/cache.ts +++ b/nextcloud/cache.ts @@ -13,7 +13,6 @@ type CacheRow = { updated_at: number }; -// TODO: CHANGEME WHEN RUNNING IN DOCKER export const CACHE_DIR = "/tmp", db = new Database(path.resolve(`${CACHE_DIR}/index.sqlite`)); diff --git a/nextcloud/helpers.ts b/nextcloud/helpers.ts index b39a81b..b8b122f 100644 --- a/nextcloud/helpers.ts +++ b/nextcloud/helpers.ts @@ -1,15 +1,26 @@ import { Request } from "express"; import { WebDAVClient } from "webdav"; import { DIRS } from "./server"; +import fetch from 'node-fetch' +import { XMLParser } from 'fast-xml-parser'; export async function statOne(fpath: string, client: WebDAVClient) { const parent = fpath.replace(/\/[^/]+$/, '') || '/', base = fpath.split('/').filter(Boolean).pop(), - list = await client.getDirectoryContents(parent, { deep: false }), + list = await client.getDirectoryContents(parent, { deep: false }).catch(err => { + console.error(`error for "${parent}":`); + console.error(err); + + if (err.response?.status === 404) { + throw `Path "${fpath}" not found`; + } + + throw err; + }), entry = Array.isArray(list) ? list.find((e: any) => e.basename === base) : undefined; - if (!entry) throw `not found: ${fpath}`; + if (!entry) throw `Path "${fpath}" not found`; return { etag: entry.etag as string, size: Number(entry.size), mime: entry.mime as string | undefined }; } @@ -29,17 +40,88 @@ export const toRegExp = (spec: string): RegExp | null => { } } -export function checkIfHasPerms(req: Request) { - if (!req.body) { +export function checkIfHasPerms(req: Request | string) { + try { + if (typeof req === 'string') { + if (req.includes('..')) return false; + + return !DIRS || DIRS.find(r => req.match(r)); + } + else if (!req.body) { + return false; + } + + const { fpath, obj_type, deep, usecache } = req.body, + o = { fpath, obj_type, deep, usecache }; + + if (!fpath) return false; + if (!DIRS) return o; + if (fpath.includes('..')) return false; + + return DIRS.find(r => fpath.match(r)) ? o : false; + } + catch (err) { + console.error(err); return false; } - const { fpath, obj_type, deep, usecache } = req.body, - o = { fpath, obj_type, deep, usecache }; - - if (!fpath) return false; - if (!DIRS) return o; - if (fpath.includes('..')) return false; - - return DIRS.find(r => fpath.match(r)) ? o : false; } +function buildCountPropfindXml() { + return ` + + + + + + + +`; +} + +const authHeaders = (() => { + const u = process.env.NEXTCLOUD_APP_ID!, + p = process.env.NEXTCLOUD_APP_PASS!, + token = Buffer.from(`${u}:${p}`).toString("base64"); + return `Basic ${token}`; +})(); + +export async function getDirectoryFileCount(dirPath: string, client: WebDAVClient) { + const url = client.getFileDownloadLink(dirPath), + xml = buildCountPropfindXml(); + + const resp = await fetch(url, { + method: "PROPFIND", + headers: { + "Content-Type": "text/xml", + "Depth": "0", + "Authorization": authHeaders + }, + body: xml + }); + + if (!resp.ok) { + throw resp; + } + + // parse XML + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + allowBooleanAttributes: true, + numberParseOptions: { + hex: false, + leadingZeros: false + } + }), + text = await resp.text(), + doc = parser.parse(text), + objs = doc['d:multistatus']['d:response']['d:propstat'], + obj = objs.find((o: any) => o['d:status'].includes("200"))['d:prop']; + + return { + file_count: obj['nc:contained-file-count'], + folder: obj['nc:contained-folder-count'] + }; +} \ No newline at end of file diff --git a/nextcloud/openapi.json b/nextcloud/openapi.json index faaa6e6..568ed3b 100644 --- a/nextcloud/openapi.json +++ b/nextcloud/openapi.json @@ -141,34 +141,29 @@ }, "put": { "operationId": "nextcloudUploadFile", - "summary": "Upload a file", - "description": "uploads a file into a target directory in nextcloud via webdav. requires a multipart/form-data body with the file part and the destination directory. the server will not overwrite existing files.", + "summary": "Upload a file (application/json, base64-encoded)", + "description": "Uploads a file into a target directory in nextcloud via webdav. the server will not overwrite existing files. the body must be application/json and include `fdir` and `file.buffer` (base64).", "requestBody": { "required": true, "content": { - "multipart/form-data": { + "application/json": { "schema": { - "type": "object", - "properties": { - "fdir": { - "type": "string", - "description": "destination directory path (e.g., /Documents)" - }, - "createnewdirs": { - "type": "boolean", - "default": false, - "description": "if true and the directory does not exist, create it recursively before uploading" - }, - "file": { - "type": "string", - "format": "binary", - "description": "the file contents" + "$ref": "#/components/schemas/NewFileMeta" + }, + "examples": { + "json-base64": { + "value": { + "fdir": "/Documents", + "createnewdirs": false, + "file": { + "filename": "file", + "originalname": "photo.jpg", + "mimetype": "image/jpeg", + "size": 52345, + "buffer": "/9j/4AAQSkZJRgABAQAAAQABAAD... (truncated base64)" + } } - }, - "required": [ - "fdir", - "file" - ] + } } } } @@ -204,7 +199,7 @@ } }, "400": { - "description": "missing file upload field", + "description": "missing file upload field or invalid base64", "content": { "application/json": { "schema": { @@ -216,7 +211,7 @@ }, "error": { "type": "string", - "example": "missing file" + "example": "missing file or invalid buffer" } } } @@ -268,7 +263,7 @@ "post": { "operationId": "nextcloudListDirectory", "summary": "List a directory", - "description": "lists directory entries from nextcloud webdav. supports shallow or deep listing.", + "description": "Lists directory entries from nextcloud webdav. Supports shallow or deep listing.", "requestBody": { "required": true, "content": { @@ -284,6 +279,16 @@ "type": "boolean", "default": false, "description": "whether to recurse into subdirectories" + }, + "startInd": { + "type": "number", + "default": 0, + "description": "start listing from this index (for pagination)" + }, + "limit": { + "type": "number", + "default": 250, + "description": "maximum number of entries to return" } }, "required": [ @@ -331,7 +336,7 @@ "get": { "operationId": "nextcloudGetOpenapi", "summary": "Serve OpenAPI schema", - "description": "serves this specification (used by open webui when registering a tool server).", + "description": "Serves this specification (used by open webui when registering a tool server).", "responses": { "200": { "description": "openapi document", @@ -352,7 +357,7 @@ "schemas": { "DirEntry": { "type": "object", - "description": "entry returned by webdav client", + "description": "Entry returned by webdav client", "properties": { "filename": { "type": "string", @@ -395,7 +400,7 @@ }, "FileMeta": { "type": "object", - "description": "metadata about a file (json-friendly alternative to raw bytes)", + "description": "Metadata about a file (json-friendly alternative to raw bytes)", "properties": { "filename": { "type": "string" @@ -438,6 +443,62 @@ "basename", "type" ] + }, + "NewFileMeta": { + "type": "object", + "properties": { + "fdir": { + "type": "string", + "description": "destination directory path (e.g., /Documents)" + }, + "createnewdirs": { + "type": "boolean", + "default": false, + "description": "if true and the directory does not exist, create it recursively before uploading" + }, + "file": { + "type": "object", + "description": "uploaded file metadata + contents (base64 if image or misc formats otherwise)", + "properties": { + "filename": { + "type": "string", + "description": "name of the form field associated with this file or client-side identifier" + }, + "originalname": { + "type": "string", + "default": "", + "description": "name of the file on the uploader's computer" + }, + "mimetype": { + "type": "string", + "default": "text/plain", + "description": "value of the Content-Type for this file, e.g. image/jpeg" + }, + "size": { + "type": "integer", + "format": "int64", + "default": 0, + "description": "size of the file in bytes" + }, + "buffer": { + "type": "string", + "format": "byte", + "contentEncoding": "any", + "contentMediaType": "application/octet-stream", + "description": "the contents of the file" + } + }, + "required": [ + "filename", + "mimetype", + "buffer" + ] + } + }, + "required": [ + "fdir", + "file" + ] } }, "responses": { @@ -500,4 +561,4 @@ } } } -} +} \ No newline at end of file diff --git a/nextcloud/package.json b/nextcloud/package.json index fe6792c..2ea7e6e 100644 --- a/nextcloud/package.json +++ b/nextcloud/package.json @@ -2,7 +2,8 @@ "name": "nextcloud", "private": true, "devDependencies": { - "@types/bun": "latest" + "@types/bun": "latest", + "@types/xmldom": "^0.1.34" }, "peerDependencies": { "typescript": "^5" @@ -10,10 +11,8 @@ "dependencies": { "@types/cors": "^2.8.19", "@types/express": "^5.0.3", - "@types/multer": "^2.0.0", "cors": "^2.8.5", "express": "^5.1.0", - "multer": "^2.0.2", "webdav": "^5.8.0" } } diff --git a/nextcloud/server.ts b/nextcloud/server.ts index 6194b6f..49d51c2 100644 --- a/nextcloud/server.ts +++ b/nextcloud/server.ts @@ -1,10 +1,10 @@ import express from 'express'; -import * as multer from 'multer'; import fs from 'fs/promises'; import fsSync from 'fs'; +import { randomUUID } from 'crypto'; import { createClient } from 'webdav'; import { pathKey, fanoutPath, ensureDir, getRow, upsertRow, CACHE_DIR } from './cache.ts'; -import { checkIfHasPerms, statOne, toRegExp } from './helpers.ts'; +import { checkIfHasPerms, getDirectoryFileCount, statOne, toRegExp } from './helpers.ts'; import path from 'path'; type statRetType = { @@ -17,14 +17,26 @@ type statRetType = { "mime"?: string }; +type ReqFileType = { + filename: string; // Name of the form field associated with this file. + originalname?: string; // Name of the file on the uploader's computer. + mimetype: string; // Value of the Content-Type header for this file. + size?: number; // Size of the file in bytes. + buffer: Buffer; // A Buffer containing the file content. + localFilename: string; // the temporary ID assigned to the file +}; + declare global { namespace Express { interface Request { fobj?: { fpath: any, deep?: boolean, - bypasscache?: boolean - } + bypasscache?: boolean, + startInd?: number, + limit?: number + }, + file: ReqFileType } } } @@ -54,28 +66,25 @@ app.use((req, res, next) => { return res.sendFile('openapi.json', { root: '.' }); } - // GET has no body (independant handlers) - else if (req.method === 'GET') return next(); - + // GET/PUT has independant handlers + else if (['GET', 'PUT'].includes(req.method)) return next(); const pth = checkIfHasPerms(req); + if (!pth && req.body?.fpath) { return res.status(401).send(`Not allowed to access "${req.body.fpath}"`); } else if (!pth) { return res.status(404).send("Unknown path"); } + else if (typeof pth === 'string') { + console.warn(`somehow got string out of checkIfHasPerms:\npth: ${pth}\nreq.body: ${req.body}`); + return res.sendStatus(400); + } req.fobj = pth; next(); }); -//@ts-ignore stupid -const multerInstance = multer.default({ - storage: multer.diskStorage({ - destination: CACHE_DIR - }) -}); - export const DIRS: string[] = (() => { try { if (!_DIRS) return null; @@ -91,7 +100,7 @@ if (!DIRS) { const client = createClient(process.env.NEXTCLOUD_WEBDAV_ADDR!, { username: process.env.NEXTCLOUD_APP_ID!, - password: process.env.NEXTCLOUD_APP_PASS!, + password: process.env.NEXTCLOUD_APP_PASS! }); app.post('/file', async (req, res) => { @@ -103,7 +112,8 @@ app.post('/file', async (req, res) => { const { etag, size, mime } = await statOne(fpath, client).catch((err) => { if (typeof err === 'string') { console.error(err); - res.sendStatus(404).send(err); + + if (!res.headersSent) res.sendStatus(404).send(err); return { etag: undefined, size: -1, mime: -1 }; } @@ -145,55 +155,121 @@ app.post('/file', async (req, res) => { res.send(buf); } catch (err) { console.error(err); - res.sendStatus(500); + if (!res.headersSent) res.sendStatus(500); } }); app.post('/dir', async (req, res) => { try { if (!req.fobj) return res.sendStatus(401); + const { fpath, deep } = req.fobj, + { startInd = 0, limit = 250 } = req.body; - const { fpath, deep } = req.fobj; - if (!fpath) return res.sendStatus(400); - else if (!(await client.exists(fpath)) { - return res.status(404).send("Directory not found"); - } + if (!fpath) return res.sendStatus(400); - const filesInDir = await client.getDirectoryContents(fpath, { - deep - }); + if (!(await client.exists(fpath))) { + return res.status(404).send("Directory not found"); + } - res.json(filesInDir); + const getFilesInDir = async () => { + for (let i = 0; i < 3; i++) { + try { + return await client.getDirectoryContents(fpath, { + deep + }); + } + catch (err: any) { + if (err.response?.status === 500) { + await new Promise(resolve => setTimeout(resolve, 2000)); + continue; + } + + console.error(`error for "${fpath}":`); + throw err; + } + } + + const counts = await getDirectoryFileCount('/Photos', client); + + let err = "failed to find dir contents (perhaps the server returned 500 too many times?)"; + err += `\n"${fpath}" contains the following, consider listing directories individually (instead of using 'deep' if too many are present: ${JSON.stringify(counts)}`; + + res.status(500).send(err); + } + + const ret = await getFilesInDir(); + if (!ret) return; + + const filesInDir = Array.isArray(ret) ? ret : ret.data; + res.json(filesInDir.slice(startInd, limit)); } catch (err) { console.error(err); - res.sendStatus(500); + res.sendStatus(500); } }); -app.put('/file', multerInstance.single('file'), async (req, res) => { - if (!req.fobj) return res.sendStatus(401); - if (!req.file) return res.status(400).send("missing file"); +// openwebui doesn't really help with `multerInstance.single('file'),` +const ensureMockFile = async (req: express.Request, res: express.Response, next: express.NextFunction) => { + try { + const { file, fdir } = req.body as { fdir: any, file: { filename: any, buffer: any, mimetype: any } }; + if (!file) return res.status(400).send("missing file"); + if (typeof fdir !== 'string') { + return res.status(400).send("Missing fdir in request"); + } + if (typeof file.filename !== 'string') { + return res.status(400).send("Invalid filename"); + } + if (typeof file.buffer === 'undefined') { + return res.status(400).send("Invalid file upload payload"); + } + + if (!checkIfHasPerms(fdir)) return res.sendStatus(401); + + const fname = `${randomUUID().replaceAll('-', '')}_${file.filename}`, + fpath = path.join(CACHE_DIR, fname); + + await fs.writeFile(fpath, file.buffer).catch((err) => { + console.error(err); + res.sendStatus(500); + }); + + if (res.headersSent) return; + req.file = { ...file, localFilename: fname, mimetype: req.body.mimetype || '' }; + + next(); + } + catch (err) { + console.error(err); + if (!res.headersSent) res.sendStatus(500); + } +}; + +app.put('/file', ensureMockFile, async (req, res) => { + const file = req.file; + if (!file) return; const { fdir, createnewdirs } = req.body as { fdir: string, createnewdirs: boolean }; + if (!fdir) return (res.headersSent || res.status(400).send("missing fdir in body")); + if (createnewdirs && !(await client.exists(fdir))) { await client.createDirectory(fdir, { recursive: true }); } - const fname = req.file.filename || req.file.originalname, - fpath = path.join(fdir, fname); + const fname = file.filename || file.originalname, + fpath = path.join(fdir, fname!); if (await client.exists(fpath)) { return res.status(503).send('file already exists, you do not have permissions to overwrite files'); } - const sstr = fsSync.createReadStream(req.file.path).pipe(client.createWriteStream(fpath, { + const sstr = fsSync.createReadStream(path.join(CACHE_DIR, file.localFilename)).pipe(client.createWriteStream(fpath, { overwrite: false }, (r) => { if (res.headersSent) return;