From 478095f2b796461eb8121fc5c34e1ba4898608b8 Mon Sep 17 00:00:00 2001 From: Nezumi-2711 Date: Sun, 16 Aug 2026 15:34:46 +0700 Subject: [PATCH] fix: resolve the issue read file --- src/google-drive.ts | 96 ++++++++++++++++++++++++++++++++++++++------- src/router.ts | 9 ++++- src/s3-xml.ts | 14 ++++--- test/s3.test.ts | 64 ++++++++++++++++++++++++++---- 4 files changed, 152 insertions(+), 31 deletions(-) diff --git a/src/google-drive.ts b/src/google-drive.ts index d082c94..614a682 100644 --- a/src/google-drive.ts +++ b/src/google-drive.ts @@ -12,6 +12,8 @@ interface GoogleDriveCreateResponse { } const DRIVE_FIELDS = "id,name,size,mimeType,md5Checksum"; +const FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"; +const LIST_NODE_CAP = 5000; function driveLiteral(value: string): string { return value.replace(/\\/g, "\\\\").replace(/'/g, "\\'"); @@ -56,6 +58,18 @@ export async function getAccessToken(env: Env): Promise { return data.access_token; } +/** Finds a folder by name under the given parent without creating it. Returns null if absent. */ +async function findFolderId(accessToken: string, folderName: string, parentId: string | null): Promise { + const parentQuery = parentId ? ` and '${parentId}' in parents` : ""; + const searchRes = await fetch(driveFilesUrl(`name='${driveLiteral(folderName)}' and mimeType='${FOLDER_MIME_TYPE}' and trashed=false${parentQuery}`, "files(id,name)"), { + headers: { Authorization: `Bearer ${accessToken}` }, + }); + if (!searchRes.ok) throw new Error(`Drive folder search failed: ${await searchRes.text()}`); + + const searchData: GoogleDriveSearchResponse = await searchRes.json(); + return searchData.files && searchData.files.length > 0 ? searchData.files[0].id : null; +} + /** Finds a folder by name under the given parent, creating it if it doesn't exist yet. */ async function getOrCreateFolder(accessToken: string, folderName: string, parentId: string | null, env: Env): Promise { // Include parentId in the cache key so folders with the same name in different parents don't collide. @@ -63,22 +77,15 @@ async function getOrCreateFolder(accessToken: string, folderName: string, parent const cached = await env.FOLDER_CACHE.get(cacheKey); if (cached) return cached; - const parentQuery = parentId ? ` and '${parentId}' in parents` : ""; - const searchRes = await fetch(driveFilesUrl(`name='${driveLiteral(folderName)}' and mimeType='application/vnd.google-apps.folder' and trashed=false${parentQuery}`, "files(id,name)"), { - headers: { Authorization: `Bearer ${accessToken}` }, - }); - - const searchData: GoogleDriveSearchResponse = await searchRes.json(); - - if (searchData.files && searchData.files.length > 0) { - const folderId = searchData.files[0].id; - await env.FOLDER_CACHE.put(cacheKey, folderId, { expirationTtl: 3600 }); - return folderId; + const found = await findFolderId(accessToken, folderName, parentId); + if (found) { + await env.FOLDER_CACHE.put(cacheKey, found, { expirationTtl: 3600 }); + return found; } const createBody: { name: string; mimeType: string; parents?: string[] } = { name: folderName, - mimeType: "application/vnd.google-apps.folder", + mimeType: FOLDER_MIME_TYPE, }; if (parentId) { @@ -93,6 +100,7 @@ async function getOrCreateFolder(accessToken: string, folderName: string, parent }, body: JSON.stringify(createBody), }); + if (!createRes.ok) throw new Error(`Drive folder creation failed: ${await createRes.text()}`); const createData: GoogleDriveCreateResponse = await createRes.json(); await env.FOLDER_CACHE.put(cacheKey, createData.id, { expirationTtl: 3600 }); @@ -181,6 +189,7 @@ export async function findFileInFolder(accessToken: string, folderId: string, fi const searchRes = await fetch(driveFilesUrl(`name='${driveLiteral(fileName)}' and '${driveLiteral(folderId)}' in parents and trashed=false`, `files(${DRIVE_FIELDS})`), { headers: { Authorization: `Bearer ${accessToken}` }, }); + if (!searchRes.ok) throw new Error(`Drive file search failed: ${await searchRes.text()}`); const data: GoogleDriveSearchResponse = await searchRes.json(); return data.files && data.files.length > 0 ? data.files[0] : null; @@ -258,13 +267,70 @@ export async function getFileMetadata(accessToken: string, bucket: string, objec }; } -export async function listFiles(accessToken: string, bucket: string, env: Env): Promise { - const folderId = await getOrCreateFolder(accessToken, bucket, null, env); - +async function listChildren(accessToken: string, folderId: string): Promise { const listRes = await fetch(driveFilesUrl(`'${driveLiteral(folderId)}' in parents and trashed=false`, "files(id,name,mimeType,size,modifiedTime,md5Checksum)"), { headers: { Authorization: `Bearer ${accessToken}` }, }); + if (!listRes.ok) throw new Error(`Drive list failed: ${await listRes.text()}`); const data: GoogleDriveSearchResponse = await listRes.json(); return data.files || []; } + +/** Splits an S3 prefix into the directory portion (real Drive folder path) and the partial name filter for the final segment. */ +function splitPrefix(prefix: string): { dirPrefix: string; partial: string } { + const index = prefix.lastIndexOf("/"); + return index === -1 ? { dirPrefix: "", partial: prefix } : { dirPrefix: prefix.slice(0, index + 1), partial: prefix.slice(index + 1) }; +} + +/** Walks an existing (read-only) folder path under the bucket; returns null if any segment is missing. */ +async function resolvePrefixFolder(accessToken: string, bucket: string, dirParts: string[]): Promise { + let folderId = await findFolderId(accessToken, bucket, null); + for (const part of dirParts) { + if (folderId === null) return null; + folderId = await findFolderId(accessToken, part, folderId); + } + return folderId; +} + +export interface ListedObject extends GoogleDriveFile { + key: string; +} + +/** Lists objects under a bucket, honoring an S3-style prefix and an optional single-level delimiter. */ +export async function listObjects(accessToken: string, bucket: string, prefix: string, delimiter?: string): Promise<{ contents: ListedObject[]; commonPrefixes: string[]; truncated: boolean }> { + const { dirPrefix, partial } = splitPrefix(prefix); + const dirParts = dirPrefix.split("/").filter((part) => part !== ""); + const folderId = await resolvePrefixFolder(accessToken, bucket, dirParts); + if (folderId === null) return { contents: [], commonPrefixes: [], truncated: false }; + + const contents: ListedObject[] = []; + const commonPrefixes = new Set(); + let truncated = false; + let scanned = 0; + + async function walk(currentFolderId: string, keyPrefix: string, applyPartialFilter: boolean): Promise { + const children = await listChildren(accessToken, currentFolderId); + for (const child of children) { + if (applyPartialFilter && !child.name.startsWith(partial)) continue; + if (++scanned > LIST_NODE_CAP) { + truncated = true; + return; + } + const childKey = `${keyPrefix}${child.name}`; + if (child.mimeType === FOLDER_MIME_TYPE) { + if (delimiter) { + commonPrefixes.add(`${childKey}${delimiter}`); + } else { + await walk(child.id, `${childKey}/`, false); + } + } else { + contents.push({ ...child, key: childKey }); + } + if (truncated) return; + } + } + + await walk(folderId, dirPrefix, true); + return { contents, commonPrefixes: [...commonPrefixes].sort(), truncated }; +} diff --git a/src/router.ts b/src/router.ts index 2ebb3c8..da81f69 100644 --- a/src/router.ts +++ b/src/router.ts @@ -1,6 +1,6 @@ import { decodedContentLength, isAwsChunked, pumpBody } from "./aws-chunked"; import { createSession, nextDriveOffset } from "./drive-resumable"; -import { deleteFromDrive, findFileInFolder, getFileMetadata, listFiles, resolvePathToFolderAndFile, streamDownloadFromDrive, streamUploadToDrive } from "./google-drive"; +import { deleteFromDrive, findFileInFolder, getFileMetadata, listObjects, resolvePathToFolderAndFile, streamDownloadFromDrive, streamUploadToDrive } from "./google-drive"; import { S3Exception, s3Error } from "./s3-errors"; import { completeMultipartUploadResult, generateListBucketResult, initiateMultipartUploadResult, listMultipartUploadsResult, listPartsResult, parseCompleteMultipartUpload } from "./s3-xml"; import type { DriveUploadResult, Env } from "./types"; @@ -191,7 +191,12 @@ export async function dispatch(request: Request, env: Env, accessToken: string, if (method === "PUT" && key) return putObject(request, env, accessToken, bucket, key); if (method === "GET") { - if (!key) return xmlResponse(generateListBucketResult(await listFiles(accessToken, bucket, env), bucket)); + if (!key) { + const prefix = url.searchParams.get("prefix") ?? ""; + const delimiter = url.searchParams.get("delimiter") ?? undefined; + const { contents, commonPrefixes, truncated } = await listObjects(accessToken, bucket, prefix, delimiter); + return xmlResponse(generateListBucketResult(bucket, prefix, delimiter, contents, commonPrefixes, truncated)); + } try { const file = await streamDownloadFromDrive(accessToken, bucket, key, env, request.headers.get("Range") ?? undefined); const headers = new Headers({ diff --git a/src/s3-xml.ts b/src/s3-xml.ts index 2535b7c..7dedfb6 100644 --- a/src/s3-xml.ts +++ b/src/s3-xml.ts @@ -1,15 +1,15 @@ -import type { GoogleDriveFile } from "./types"; +import type { ListedObject } from "./google-drive"; export function escapeXml(str: string): string { return str.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """).replace(/'/g, "'"); } -export function generateListBucketResult(files: GoogleDriveFile[], bucket: string): string { +export function generateListBucketResult(bucket: string, prefix: string, delimiter: string | undefined, files: ListedObject[], commonPrefixes: string[], isTruncated: boolean): string { const contents = files .map( (f) => ` - ${escapeXml(f.name)} + ${escapeXml(f.key)} ${f.modifiedTime || new Date().toISOString()} "${f.md5Checksum || f.id}" ${f.size || 0} @@ -18,13 +18,15 @@ export function generateListBucketResult(files: GoogleDriveFile[], bucket: strin ) .join(""); + const commonPrefixesXml = commonPrefixes.map((p) => `\n ${escapeXml(p)}`).join(""); + return ` ${escapeXml(bucket)} - + ${escapeXml(prefix)} + ${delimiter ? `${escapeXml(delimiter)}` : ""} 1000 - false - ${contents} + ${isTruncated}${contents}${commonPrefixesXml} `; } diff --git a/test/s3.test.ts b/test/s3.test.ts index 559a16f..36905d3 100644 --- a/test/s3.test.ts +++ b/test/s3.test.ts @@ -20,6 +20,14 @@ interface StoredFile { md5Checksum: string; } +interface StoredFolder { + id: string; + name: string; + parent: string; +} + +const FOLDER_MIME = "application/vnd.google-apps.folder"; + interface UploadSession { id: string; fileId?: string; @@ -31,6 +39,7 @@ interface UploadSession { class FakeDrive { readonly files = new Map(); + readonly folders = new Map(); readonly sessions = new Map(); private nextId = 1; @@ -54,23 +63,36 @@ class FakeDrive { private search(url: URL): Response { const q = url.searchParams.get("q") ?? ""; + const hasNameFilter = /name='/.test(q); const name = /name='((?:\\.|[^'])*)'/.exec(q)?.[1]?.replace(/\\'/g, "'").replace(/\\\\/g, "\\"); - const parent = /'([^']+)' in parents/.exec(q)?.[1]; - if (q.includes("application/vnd.google-apps.folder")) { - const id = parent ? `folder:${parent}:${name}` : `folder:${name}`; - return Response.json({ files: [{ id, name, mimeType: "application/vnd.google-apps.folder" }] }); + const parent = /'([^']+)' in parents/.exec(q)?.[1] ?? "root"; + + if (q.includes(FOLDER_MIME)) { + const match = [...this.folders.values()].find((folder) => folder.name === name && folder.parent === parent); + return Response.json({ files: match ? [{ id: match.id, name: match.name, mimeType: FOLDER_MIME }] : [] }); } - const files = [...this.files.values()].filter((file) => file.name === name && file.parent === parent); - return Response.json({ files: files.map((file) => ({ ...file, size: String(file.data.byteLength), data: undefined })) }); + + const files = [...this.files.values()].filter((file) => (!hasNameFilter || file.name === name) && file.parent === parent); + const folders = [...this.folders.values()].filter((folder) => (!hasNameFilter || folder.name === name) && folder.parent === parent); + return Response.json({ + files: [...files.map((file) => ({ ...file, size: String(file.data.byteLength), data: undefined })), ...folders.map((folder) => ({ id: folder.id, name: folder.name, mimeType: FOLDER_MIME }))], + }); } private createMetadata(metadata: Record): Response { + const mimeType = String(metadata.mimeType ?? "application/octet-stream"); + const parent = String((metadata.parents as string[] | undefined)?.[0] ?? "root"); + if (mimeType === FOLDER_MIME) { + const id = `folder-${this.nextId++}`; + this.folders.set(id, { id, name: String(metadata.name), parent }); + return Response.json({ id, name: metadata.name, mimeType }); + } const id = `file-${this.nextId++}`; const file: StoredFile = { id, name: String(metadata.name), - parent: String((metadata.parents as string[] | undefined)?.[0] ?? ""), - mimeType: String(metadata.mimeType ?? "application/octet-stream"), + parent, + mimeType, data: new Uint8Array(), md5Checksum: "d41d8cd98f00b204e9800998ecf8427e", }; @@ -187,6 +209,7 @@ beforeEach(async () => { vi.fn((input, init) => drive.handle(input, init)), ); await ENV.AUTH_KV.delete("google_access_token"); + for (const { name } of (await ENV.FOLDER_CACHE.list()).keys) await ENV.FOLDER_CACHE.delete(name); }); describe("S3 compatibility", () => { @@ -286,6 +309,31 @@ describe("S3 compatibility", () => { expect(response.status).toBe(200); expect([...drive.files.values()].find((file) => file.name === "empty")!.data.byteLength).toBe(0); }); + + it("lists nested keys under a prefix, as CommonPrefixes with a delimiter and recursively without one", async () => { + await worker.fetch(await signed("/test-bucket/dir1/a.txt", { method: "PUT", body: "a" }), ENV, CTX); + await worker.fetch(await signed("/test-bucket/dir1/sub/b.txt", { method: "PUT", body: "b" }), ENV, CTX); + await worker.fetch(await signed("/test-bucket/dir2/c.txt", { method: "PUT", body: "c" }), ENV, CTX); + + const root = await worker.fetch(await signed(`/test-bucket?prefix=&delimiter=${encodeURIComponent("/")}`, { method: "GET" }), ENV, CTX); + expect(root.status).toBe(200); + const rootXml = await root.text(); + expect(rootXml).toContain("dir1/"); + expect(rootXml).toContain("dir2/"); + expect(rootXml).not.toContain(""); + + const dir1Delimited = await worker.fetch(await signed(`/test-bucket?prefix=${encodeURIComponent("dir1/")}&delimiter=${encodeURIComponent("/")}`, { method: "GET" }), ENV, CTX); + const dir1Xml = await dir1Delimited.text(); + expect(dir1Xml).toContain("dir1/a.txt"); + expect(dir1Xml).toContain("dir1/sub/"); + expect(dir1Xml).not.toContain("dir1/sub/b.txt"); + + const dir1Recursive = await worker.fetch(await signed(`/test-bucket?prefix=${encodeURIComponent("dir1/")}`, { method: "GET" }), ENV, CTX); + const dir1RecursiveXml = await dir1Recursive.text(); + expect(dir1RecursiveXml).toContain("dir1/a.txt"); + expect(dir1RecursiveXml).toContain("dir1/sub/b.txt"); + expect(dir1RecursiveXml).not.toContain(""); + }); }); function streamOf(chunks: Uint8Array[]): ReadableStream {