From 4d0856eb76def5bd0b6eb46a40e3ce2a5107654c Mon Sep 17 00:00:00 2001 From: Nezumi-2711 Date: Wed, 19 Aug 2026 17:48:55 +0700 Subject: [PATCH] feat: add api for manage s3 storage --- .dev.vars.example | 6 +- .env.example | 6 +- README.md | 3 +- docs/authentication.md | 2 +- docs/limitations.md | 2 +- docs/openapi.yaml | 217 +++++++++++++++++++++++- src/bucket-access.ts | 46 ------ src/bucket-api.ts | 117 +++++++++++++ src/bucket-registry.ts | 292 +++++++++++++++++++++++++++++++++ src/cors.ts | 2 +- src/google-drive.ts | 83 +++++++++- src/index.ts | 16 +- src/router.ts | 2 +- src/status-api.ts | 86 ++++++++-- src/types.ts | 5 +- test/auth.test.ts | 7 - test/buckets.test.ts | 365 +++++++++++++++++++++++++++++++++++++++++ test/cors.test.ts | 14 +- test/docs.test.ts | 27 +-- test/s3.test.ts | 8 + test/status.test.ts | 55 +++++-- vitest.config.mts | 2 +- wrangler.jsonc | 3 +- 23 files changed, 1241 insertions(+), 125 deletions(-) delete mode 100644 src/bucket-access.ts create mode 100644 src/bucket-api.ts create mode 100644 src/bucket-registry.ts create mode 100644 test/buckets.test.ts diff --git a/.dev.vars.example b/.dev.vars.example index 8018200..7352b77 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -12,10 +12,8 @@ GOOGLE_CLIENT_ID=replace-with-google-oauth-client-id GOOGLE_CLIENT_SECRET=replace-with-google-oauth-client-secret GOOGLE_REFRESH_TOKEN=replace-with-google-refresh-token -# Comma-separated, exact bucket names. Unset denies every bucket. -ALLOWED_BUCKETS=assets -# Optional comma-separated subset of ALLOWED_BUCKETS that permits unsigned GET/HEAD. -PUBLIC_READ_BUCKETS= +# Google Drive storage root folder name +DRIVE_ROOT_FOLDER=s3-storage # Plaintext password for dashboard management API authentication. DASHBOARD_PASSWORD=replace-with-dashboard-password diff --git a/.env.example b/.env.example index 8aea324..18c1124 100644 --- a/.env.example +++ b/.env.example @@ -8,10 +8,8 @@ GOOGLE_CLIENT_ID=replace-with-google-oauth-client-id GOOGLE_CLIENT_SECRET=replace-with-google-oauth-client-secret GOOGLE_REFRESH_TOKEN=replace-with-google-refresh-token -# Comma-separated, exact bucket names. Unset denies every bucket. -ALLOWED_BUCKETS=assets -# Optional comma-separated subset of ALLOWED_BUCKETS that permits unsigned GET/HEAD. -PUBLIC_READ_BUCKETS= +# Google Drive storage root folder name +DRIVE_ROOT_FOLDER=s3-storage # Plaintext password for dashboard management API authentication DASHBOARD_PASSWORD=replace-with-dashboard-password diff --git a/README.md b/README.md index 21d69e2..154fca5 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,7 @@ https://developers.cloudflare.com/workers/configuration/secrets/#via-the-dashboa | `REGION` | The region used by the S3 client. | | `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, `GOOGLE_REFRESH_TOKEN` | Google API credentials obtained from rclone. | | `DASHBOARD_PASSWORD` | *(Optional)* Plaintext password for dashboard management API authentication. | -| `ALLOWED_BUCKETS` | Set the buckets allowed, separated by `,`. A directory with the bucket name will be created directly under Google Drive. | -| `PUBLIC_READ_BUCKETS` | *(Optional)* Buckets that allow unauthenticated GET/HEAD access without signature, separated by `,`. Write operations (PUT/POST/DELETE) still require authentication. Must be a subset of `ALLOWED_BUCKETS`. | +| `DRIVE_ROOT_FOLDER` | Root folder name in Google Drive where all buckets reside (e.g. `s3-storage`). Set in `wrangler.jsonc` vars or as a secret. | | `CORS_ALLOWED_ORIGINS` | *(Optional)* Comma-separated exact browser origins, or `*`. Unset emits no CORS headers. | | `ENABLE_DOCS` | *(Optional)* Set to `false` to disable `/docs` and `/openapi.yaml`; enabled by default. | diff --git a/docs/authentication.md b/docs/authentication.md index 963ca83..983e37a 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -34,7 +34,7 @@ Payload hashes are **not** verified. Browser and BFF clients should use `x-amz-c ## Public-read buckets -Buckets listed in `PUBLIC_READ_BUCKETS` permit unsigned `GET` and `HEAD` requests. All write operations still require valid Signature V4 authentication. `PUBLIC_READ_BUCKETS` must be a subset of `ALLOWED_BUCKETS`. +Buckets configured with `publicRead: true` (managed via the dashboard or API) permit unsigned `GET` and `HEAD` requests. All write operations still require valid Signature V4 authentication. Bucket metadata is tracked via Drive `appProperties`. ## Dashboard login API diff --git a/docs/limitations.md b/docs/limitations.md index df53fa0..3ef8106 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -14,7 +14,7 @@ Use narrow prefixes and `delimiter=/` for a file browser. Do not build an infini ## Unsupported S3 features -- `ListBuckets`, `CreateBucket`, and `DeleteBucket` +- `ListBuckets`, `CreateBucket`, and `DeleteBucket` via S3 API (Bucket management and creation/deletion are handled via the dashboard or `/api/buckets` REST API). - batch `DeleteObjects` - object versioning, ACLs, object tags, lifecycle rules, and `x-amz-meta-*` user metadata - CopyObject and UploadPartCopy diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 7373b42..dcf3e5b 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -18,6 +18,7 @@ tags: - name: Buckets - name: Multipart uploads - name: Dashboard Auth + - name: Dashboard Status paths: /auth/login: post: @@ -146,7 +147,157 @@ paths: application/json: schema: { $ref: '#/components/schemas/AuthError' } '503': - description: Dashboard authentication not configured. + description: Storage root folder or authentication not configured. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + post: + tags: [Dashboard Status] + operationId: createBucket + summary: Create a new bucket + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/CreateBucketRequest' } + responses: + '201': + description: Bucket created. + content: + application/json: + schema: { $ref: '#/components/schemas/BucketRecord' } + '400': + description: Invalid bucket name. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + '401': + description: Invalid or expired session. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + '409': + description: Bucket already exists. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + '503': + description: Storage root folder or authentication not configured. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + /api/buckets/{name}: + parameters: + - name: name + in: path + required: true + schema: { type: string } + patch: + tags: [Dashboard Status] + operationId: updateBucket + summary: Update bucket settings (toggle public read or rename) + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/UpdateBucketRequest' } + responses: + '200': + description: Bucket updated. + content: + application/json: + schema: { $ref: '#/components/schemas/BucketRecord' } + '400': + description: Invalid request. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + '401': + description: Invalid or expired session. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + '404': + description: Bucket not found. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + '409': + description: Target bucket name conflict. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + delete: + tags: [Dashboard Status] + operationId: deleteBucket + summary: Delete an empty bucket + security: + - bearerAuth: [] + responses: + '204': + description: Bucket deleted. + '401': + description: Invalid or expired session. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + '404': + description: Bucket not found. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + '409': + description: Bucket is not empty. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + /api/import-candidates: + get: + tags: [Dashboard Status] + operationId: listImportCandidates + summary: List Drive folders available for import + security: + - bearerAuth: [] + responses: + '200': + description: List of folders under Drive root. + content: + application/json: + schema: { $ref: '#/components/schemas/ImportCandidatesResponse' } + '401': + description: Invalid or expired session. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + /api/import: + post: + tags: [Dashboard Status] + operationId: importBuckets + summary: Import folders from Drive root into storage root folder + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/ImportBucketsRequest' } + responses: + '200': + description: Import results. + content: + application/json: + schema: { $ref: '#/components/schemas/ImportResult' } + '400': + description: Invalid body. + content: + application/json: + schema: { $ref: '#/components/schemas/AuthError' } + '401': + description: Invalid or expired session. content: application/json: schema: { $ref: '#/components/schemas/AuthError' } @@ -460,7 +611,7 @@ components: properties: gateway: type: object - required: [status, region, multipartEnabled, etagStyle, docsEnabled, buckets, publicReadBuckets, corsOrigins, credentials] + required: [status, region, multipartEnabled, etagStyle, docsEnabled, buckets, publicReadBuckets, rootFolder, corsOrigins, credentials] properties: status: { type: string, enum: [ok, degraded] } region: { type: string, example: auto } @@ -469,6 +620,13 @@ components: docsEnabled: { type: boolean } buckets: { type: array, items: { type: string } } publicReadBuckets: { type: array, items: { type: string } } + rootFolder: + type: object + required: [name, id, configured] + properties: + name: { type: string, nullable: true } + id: { type: string, nullable: true } + configured: { type: boolean } corsOrigins: { type: array, items: { type: string } } credentials: type: object @@ -529,6 +687,61 @@ components: cachedAt: type: string format: date-time + BucketRecord: + type: object + required: [name, folderId, publicRead, createdTime] + properties: + name: { type: string, example: assets } + folderId: { type: string, example: 1A2b3C4d5E6f } + publicRead: { type: boolean, example: false } + createdTime: { type: string, format: date-time, nullable: true } + CreateBucketRequest: + type: object + required: [name] + properties: + name: { type: string, example: assets } + publicRead: { type: boolean, default: false } + UpdateBucketRequest: + type: object + properties: + publicRead: { type: boolean } + name: { type: string } + ImportCandidate: + type: object + required: [name, folderId, objectCount] + properties: + name: { type: string, example: legacy-photos } + folderId: { type: string, example: 1A2b3C4d5E6f } + objectCount: { type: integer, example: 42 } + ImportCandidatesResponse: + type: object + required: [candidates] + properties: + candidates: + type: array + items: { $ref: '#/components/schemas/ImportCandidate' } + ImportBucketsRequest: + type: object + required: [names] + properties: + names: + type: array + items: { type: string } + ImportResult: + type: object + required: [imported, failed] + properties: + imported: + type: array + items: { type: string } + failed: + type: array + items: + type: object + required: [name, error] + properties: + name: { type: string } + error: { type: string } AuthError: type: object required: [message] diff --git a/src/bucket-access.ts b/src/bucket-access.ts deleted file mode 100644 index c6466a6..0000000 --- a/src/bucket-access.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { Env } from "./types"; - -/** - * Returns the list of configured allowed buckets. - */ -export function allowedBuckets(env: Env): string[] { - if (!env.ALLOWED_BUCKETS) { - return []; - } - - return env.ALLOWED_BUCKETS.split(",") - .map((b) => b.trim()) - .filter((b) => b.length > 0); -} - -/** - * Returns the list of configured public read buckets. - */ -export function publicReadBuckets(env: Env): string[] { - if (!env.PUBLIC_READ_BUCKETS) { - return []; - } - - return env.PUBLIC_READ_BUCKETS.split(",") - .map((b) => b.trim()) - .filter((b) => b.length > 0); -} - -/** - * Checks whether the bucket is present in the ALLOWED_BUCKETS allowlist. - * Access is denied by default when the allowlist is missing or empty. - */ -export function isAllowedBucket(bucket: string, env: Env): boolean { - const buckets = allowedBuckets(env); - if (buckets.length === 0) { - return false; - } - - return buckets.includes(bucket); -} - -/** Checks whether the bucket allows unauthenticated read access. */ -export function isPublicReadBucket(bucket: string, env: Env): boolean { - const buckets = publicReadBuckets(env); - return buckets.includes(bucket); -} diff --git a/src/bucket-api.ts b/src/bucket-api.ts new file mode 100644 index 0000000..152900c --- /dev/null +++ b/src/bucket-api.ts @@ -0,0 +1,117 @@ +import { jsonResponse } from "./auth-api"; +import { createBucket, deleteBucket, importBuckets, listImportCandidates, updateBucket } from "./bucket-registry"; +import type { Env } from "./types"; + +interface CreateBucketBody { + name?: string; + publicRead?: boolean; +} + +interface UpdateBucketBody { + publicRead?: boolean; + name?: string; +} + +interface ImportBucketsBody { + names?: string[]; +} + +export async function handleBucketRoutes(request: Request, env: Env, subSegments: string[]): Promise { + const method = request.method; + const bucketName = subSegments[0]; + + // POST /api/buckets + if (method === "POST" && !bucketName) { + let body: CreateBucketBody; + try { + body = (await request.json()) as CreateBucketBody; + } catch { + return jsonResponse({ message: "Invalid JSON body" }, 400); + } + + if (!body.name) { + return jsonResponse({ message: "Bucket name is required" }, 400); + } + + try { + const record = await createBucket(env, body.name, Boolean(body.publicRead)); + return jsonResponse(record, 201); + } catch (err: unknown) { + const status = (err as { status?: number }).status || 500; + const message = err instanceof Error ? err.message : String(err); + return jsonResponse({ message }, status); + } + } + + // PATCH /api/buckets/:name + if (method === "PATCH" && bucketName) { + let body: UpdateBucketBody; + try { + body = (await request.json()) as UpdateBucketBody; + } catch { + return jsonResponse({ message: "Invalid JSON body" }, 400); + } + + try { + const record = await updateBucket(env, bucketName, body); + return jsonResponse(record, 200); + } catch (err: unknown) { + const status = (err as { status?: number }).status || 500; + const message = err instanceof Error ? err.message : String(err); + return jsonResponse({ message }, status); + } + } + + // DELETE /api/buckets/:name + if (method === "DELETE" && bucketName) { + try { + await deleteBucket(env, bucketName); + return new Response(null, { status: 204 }); + } catch (err: unknown) { + const status = (err as { status?: number }).status || 500; + const message = err instanceof Error ? err.message : String(err); + return jsonResponse({ message }, status); + } + } + + return jsonResponse({ message: "Method Not Allowed" }, 405); +} + +export async function handleImportCandidatesRoute(request: Request, env: Env): Promise { + if (request.method !== "GET") { + return jsonResponse({ message: "Method Not Allowed" }, 405); + } + try { + const candidates = await listImportCandidates(env); + return jsonResponse({ candidates }, 200); + } catch (err: unknown) { + const status = (err as { status?: number }).status || 500; + const message = err instanceof Error ? err.message : String(err); + return jsonResponse({ message }, status); + } +} + +export async function handleImportRoute(request: Request, env: Env): Promise { + if (request.method !== "POST") { + return jsonResponse({ message: "Method Not Allowed" }, 405); + } + let body: ImportBucketsBody; + try { + body = (await request.json()) as ImportBucketsBody; + } catch { + return jsonResponse({ message: "Invalid JSON body" }, 400); + } + + if (!Array.isArray(body.names)) { + return jsonResponse({ message: "'names' array is required" }, 400); + } + + try { + const result = await importBuckets(env, body.names); + return jsonResponse(result, 200); + } catch (err: unknown) { + const status = (err as { status?: number }).status || 500; + const message = err instanceof Error ? err.message : String(err); + return jsonResponse({ message }, status); + } +} diff --git a/src/bucket-registry.ts b/src/bucket-registry.ts new file mode 100644 index 0000000..6d6fbaa --- /dev/null +++ b/src/bucket-registry.ts @@ -0,0 +1,292 @@ +import { folderHasChildren, getAccessToken, getOrCreateFolder, getRootFolderId, listFolderChildren, updateDriveFile } from "./google-drive"; +import type { Env } from "./types"; + +export interface BucketRecord { + name: string; + folderId: string; + publicRead: boolean; + createdTime: string | null; +} + +export const RESERVED_BUCKET_NAMES = ["auth", "api", "docs"] as const; + +const BUCKET_REGISTRY_CACHE_KEY = "bucket-registry"; +const BUCKET_REGISTRY_TTL = 60; // 60 seconds + +/** + * Validates bucket name format. + * 3-63 chars, lowercase alphanumeric or hyphens, no adjacent hyphens, not formatted as IP. + * Returns null if valid, error message otherwise. + */ +export function validateBucketName(name: string): string | null { + if (!name || typeof name !== "string") { + return "Bucket name is required"; + } + if (name.length < 3 || name.length > 63) { + return "Bucket name must be between 3 and 63 characters long"; + } + if (RESERVED_BUCKET_NAMES.includes(name.toLowerCase() as (typeof RESERVED_BUCKET_NAMES)[number])) { + return `Bucket name '${name}' is reserved`; + } + const regex = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; + if (!regex.test(name)) { + return "Bucket name must contain only lowercase letters, numbers, and hyphens, and start/end with a letter or number"; + } + if (name.includes("--")) { + return "Bucket name must not contain consecutive hyphens"; + } + const ipRegex = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/; + if (ipRegex.test(name)) { + return "Bucket name must not be formatted as an IP address"; + } + return null; +} + +async function invalidateBucketCache(env: Env, name?: string, rootFolderId?: string): Promise { + await env.FOLDER_CACHE.delete(BUCKET_REGISTRY_CACHE_KEY); + if (name) { + await env.FOLDER_CACHE.delete(`bucket-stats:${name}`); + if (rootFolderId) { + await env.FOLDER_CACHE.delete(`${rootFolderId}/${name}`); + } + } +} + +/** Lists all buckets from KV cache or fetches from Google Drive */ +export async function getBucketRegistry(env: Env): Promise { + const cached = await env.FOLDER_CACHE.get(BUCKET_REGISTRY_CACHE_KEY); + if (cached) { + try { + return JSON.parse(cached) as BucketRecord[]; + } catch { + // cache corrupt, fallback to fetching + } + } + + const accessToken = await getAccessToken(env); + const rootFolderId = await getRootFolderId(accessToken, env); + const files = await listFolderChildren(accessToken, rootFolderId, "files(id,name,mimeType,createdTime,appProperties)"); + + const records: BucketRecord[] = files + .filter((file) => file.mimeType === "application/vnd.google-apps.folder" && !RESERVED_BUCKET_NAMES.includes(file.name.toLowerCase() as (typeof RESERVED_BUCKET_NAMES)[number])) + .map((folder) => ({ + name: folder.name, + folderId: folder.id, + publicRead: folder.appProperties?.s3PublicRead === "true", + createdTime: folder.createdTime ?? null, + })); + + await env.FOLDER_CACHE.put(BUCKET_REGISTRY_CACHE_KEY, JSON.stringify(records), { + expirationTtl: BUCKET_REGISTRY_TTL, + }); + + return records; +} + +/** Looks up a single bucket record by name */ +export async function findBucketRecord(env: Env, bucket: string): Promise { + if (!bucket) return null; + const registry = await getBucketRegistry(env); + return registry.find((b) => b.name === bucket) ?? null; +} + +/** Creates a new bucket folder under root folder */ +export async function createBucket(env: Env, name: string, publicRead: boolean): Promise { + const validationError = validateBucketName(name); + if (validationError) { + const error = new Error(validationError); + (error as { status?: number }).status = 400; + throw error; + } + + const existing = await findBucketRecord(env, name); + if (existing) { + const error = new Error(`Bucket '${name}' already exists`); + (error as { status?: number }).status = 409; + throw error; + } + + const accessToken = await getAccessToken(env); + const rootFolderId = await getRootFolderId(accessToken, env); + + const folderId = await getOrCreateFolder(accessToken, name, rootFolderId, env); + + if (publicRead) { + await updateDriveFile(accessToken, folderId, { + appProperties: { s3PublicRead: "true" }, + }); + } + + await invalidateBucketCache(env, name, rootFolderId); + + return { + name, + folderId, + publicRead, + createdTime: new Date().toISOString(), + }; +} + +/** Updates an existing bucket (publicRead toggle or rename) */ +export async function updateBucket(env: Env, name: string, patch: { publicRead?: boolean; name?: string }): Promise { + const record = await findBucketRecord(env, name); + if (!record) { + const error = new Error(`Bucket '${name}' not found`); + (error as { status?: number }).status = 404; + throw error; + } + + const accessToken = await getAccessToken(env); + const rootFolderId = await getRootFolderId(accessToken, env); + + const updateBody: Record = {}; + let newPublicRead = record.publicRead; + let newName = record.name; + + if (patch.publicRead !== undefined) { + newPublicRead = patch.publicRead; + updateBody.appProperties = { + s3PublicRead: newPublicRead ? "true" : "false", + }; + } + + if (patch.name !== undefined && patch.name !== name) { + const validationError = validateBucketName(patch.name); + if (validationError) { + const error = new Error(validationError); + (error as { status?: number }).status = 400; + throw error; + } + + const conflict = await findBucketRecord(env, patch.name); + if (conflict) { + const error = new Error(`Bucket '${patch.name}' already exists`); + (error as { status?: number }).status = 409; + throw error; + } + + newName = patch.name; + updateBody.name = newName; + } + + await updateDriveFile(accessToken, record.folderId, updateBody); + + await invalidateBucketCache(env, name, rootFolderId); + if (newName !== name) { + await invalidateBucketCache(env, newName, rootFolderId); + } + + return { + name: newName, + folderId: record.folderId, + publicRead: newPublicRead, + createdTime: record.createdTime, + }; +} + +/** Deletes a bucket if empty (moves folder to Drive trash) */ +export async function deleteBucket(env: Env, name: string): Promise { + const record = await findBucketRecord(env, name); + if (!record) { + const error = new Error(`Bucket '${name}' not found`); + (error as { status?: number }).status = 404; + throw error; + } + + const accessToken = await getAccessToken(env); + const rootFolderId = await getRootFolderId(accessToken, env); + + const hasChildren = await folderHasChildren(accessToken, record.folderId); + if (hasChildren) { + const error = new Error(`Bucket '${name}' is not empty`); + (error as { status?: number }).status = 409; + throw error; + } + + // Invalidate cache BEFORE moving to trash to prevent race reading stale state + await invalidateBucketCache(env, name, rootFolderId); + + await updateDriveFile(accessToken, record.folderId, { + trashed: true, + }); +} + +export interface ImportCandidate { + name: string; + folderId: string; + objectCount: number; +} + +/** Lists folders in Drive root eligible for import into storage root folder */ +export async function listImportCandidates(env: Env): Promise { + const accessToken = await getAccessToken(env); + const rootFolderId = await getRootFolderId(accessToken, env); + + const rootChildren = await listFolderChildren(accessToken, "root", "files(id,name,mimeType)"); + const existingRegistry = await getBucketRegistry(env); + const existingNames = new Set(existingRegistry.map((b) => b.name)); + + const folders = rootChildren.filter((f) => f.mimeType === "application/vnd.google-apps.folder" && f.id !== rootFolderId && !existingNames.has(f.name) && !RESERVED_BUCKET_NAMES.includes(f.name.toLowerCase() as (typeof RESERVED_BUCKET_NAMES)[number])); + + const candidates: ImportCandidate[] = []; + for (const folder of folders) { + let objectCount = 0; + try { + const children = await listFolderChildren(accessToken, folder.id, "files(id,mimeType)"); + objectCount = children.filter((c) => c.mimeType !== "application/vnd.google-apps.folder").length; + } catch { + objectCount = 0; + } + candidates.push({ + name: folder.name, + folderId: folder.id, + objectCount, + }); + } + + return candidates; +} + +/** Moves selected folders from Drive root into storage root folder */ +export async function importBuckets(env: Env, names: string[]): Promise<{ imported: string[]; failed: { name: string; error: string }[] }> { + const accessToken = await getAccessToken(env); + const rootFolderId = await getRootFolderId(accessToken, env); + const rootChildren = await listFolderChildren(accessToken, "root", "files(id,name,mimeType)"); + + const imported: string[] = []; + const failed: { name: string; error: string }[] = []; + + for (const name of names) { + const validationError = validateBucketName(name); + if (validationError) { + failed.push({ name, error: validationError }); + continue; + } + + const match = rootChildren.find((f) => f.mimeType === "application/vnd.google-apps.folder" && f.name === name); + if (!match) { + failed.push({ name, error: `Folder '${name}' not found under My Drive root` }); + continue; + } + + try { + await updateDriveFile( + accessToken, + match.id, + {}, + { + addParents: rootFolderId, + removeParents: "root", + }, + ); + await invalidateBucketCache(env, name, rootFolderId); + imported.push(name); + } catch (err) { + failed.push({ name, error: err instanceof Error ? err.message : String(err) }); + } + } + + await env.FOLDER_CACHE.delete(BUCKET_REGISTRY_CACHE_KEY); + + return { imported, failed }; +} diff --git a/src/cors.ts b/src/cors.ts index 091d63a..1c4e357 100644 --- a/src/cors.ts +++ b/src/cors.ts @@ -1,6 +1,6 @@ import type { Env } from "./types"; -const ALLOWED_METHODS = "GET, HEAD, PUT, POST, DELETE, OPTIONS"; +const ALLOWED_METHODS = "GET, HEAD, PUT, POST, PATCH, DELETE, OPTIONS"; const DEFAULT_ALLOWED_HEADERS = "Authorization, Content-Type, Content-Length, Content-MD5, Content-Encoding, Range, x-amz-content-sha256, x-amz-copy-source, x-amz-date, x-amz-decoded-content-length, x-amz-mp-object-size, x-amz-security-token"; const EXPOSED_HEADERS = "ETag, Content-Range, Content-Length, Last-Modified, Accept-Ranges, x-amz-request-id"; diff --git a/src/google-drive.ts b/src/google-drive.ts index a1bdc15..022b9b3 100644 --- a/src/google-drive.ts +++ b/src/google-drive.ts @@ -14,6 +14,7 @@ interface GoogleDriveCreateResponse { const DRIVE_FIELDS = "id,name,size,mimeType,md5Checksum,modifiedTime"; const FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"; const LIST_NODE_CAP = 5000; +const ROOT_PARENT = "root"; function driveLiteral(value: string): string { return value.replace(/\\/g, "\\\\").replace(/'/g, "\\'"); @@ -58,8 +59,35 @@ export async function getAccessToken(env: Env): Promise { return data.access_token; } +/** Resolve (tạo nếu chưa có) folder gốc chứa toàn bộ bucket. Throw nếu chưa cấu hình. */ +export async function getRootFolderId(accessToken: string, env: Env): Promise { + if (!env.DRIVE_ROOT_FOLDER || env.DRIVE_ROOT_FOLDER.trim() === "") { + throw new Error("Storage root folder is not configured"); + } + const rootName = env.DRIVE_ROOT_FOLDER.trim(); + const cacheKey = `root:${rootName}`; + const cached = await env.FOLDER_CACHE.get(cacheKey); + if (cached) return cached; + + const id = await getOrCreateFolder(accessToken, rootName, ROOT_PARENT, env); + await env.FOLDER_CACHE.put(cacheKey, id, { expirationTtl: 3600 }); + return id; +} + +/** Folder của bucket, tạo nếu chưa có — dùng cho write path. */ +export async function getBucketFolderId(accessToken: string, bucket: string, env: Env): Promise { + const rootFolderId = await getRootFolderId(accessToken, env); + return await getOrCreateFolder(accessToken, bucket, rootFolderId, env); +} + +/** Folder của bucket, null nếu chưa tồn tại — dùng cho list/stat path. */ +export async function findBucketFolderId(accessToken: string, bucket: string, env: Env): Promise { + const rootFolderId = await getRootFolderId(accessToken, env); + return await findFolderId(accessToken, bucket, rootFolderId); +} + /** Finds a folder by name under the given parent without creating it. Returns null if absent. */ -async function findFolderId(accessToken: string, folderName: string, parentId: string | null): Promise { +export async function findFolderId(accessToken: string, folderName: string, parentId: string | null): Promise { const parentQuery = parentId ? ` and '${parentId}' in parents` : ""; const searchRes = await fetch(driveFilesUrl(`name='${driveLiteral(folderName)}' and mimeType='${FOLDER_MIME_TYPE}' and trashed=false${parentQuery}`, "files(id,name)"), { headers: { Authorization: `Bearer ${accessToken}` }, @@ -71,7 +99,7 @@ async function findFolderId(accessToken: string, folderName: string, parentId: s } /** Finds a folder by name under the given parent, creating it if it doesn't exist yet. */ -async function getOrCreateFolder(accessToken: string, folderName: string, parentId: string | null, env: Env): Promise { +export async function getOrCreateFolder(accessToken: string, folderName: string, parentId: string | null, env: Env): Promise { // Include parentId in the cache key so folders with the same name in different parents don't collide. const cacheKey = parentId ? `${parentId}/${folderName}` : folderName; const cached = await env.FOLDER_CACHE.get(cacheKey); @@ -107,9 +135,50 @@ async function getOrCreateFolder(accessToken: string, folderName: string, parent return createData.id; } +export async function listFolderChildren(accessToken: string, folderId: string, fields = "files(id,name,mimeType,size,modifiedTime,createdTime,appProperties)"): Promise { + const listRes = await fetch(driveFilesUrl(`'${driveLiteral(folderId)}' in parents and trashed=false`, fields), { + headers: { Authorization: `Bearer ${accessToken}` }, + }); + if (!listRes.ok) throw new Error(`Drive list failed: ${await listRes.text()}`); + + const data: GoogleDriveSearchResponse = await listRes.json(); + return data.files || []; +} + +export async function folderHasChildren(accessToken: string, folderId: string): Promise { + const url = new URL("https://www.googleapis.com/drive/v3/files"); + url.searchParams.set("q", `'${driveLiteral(folderId)}' in parents and trashed=false`); + url.searchParams.set("pageSize", "1"); + url.searchParams.set("fields", "files(id)"); + const res = await fetch(url.toString(), { + headers: { Authorization: `Bearer ${accessToken}` }, + }); + if (!res.ok) throw new Error(`Drive folder child check failed: ${await res.text()}`); + const data: GoogleDriveSearchResponse = await res.json(); + return Boolean(data.files && data.files.length > 0); +} + +export async function updateDriveFile(accessToken: string, fileId: string, body: Record, params?: { addParents?: string; removeParents?: string }): Promise { + const url = new URL(`https://www.googleapis.com/drive/v3/files/${fileId}`); + url.searchParams.set("fields", "id,name,mimeType,size,modifiedTime,createdTime,appProperties,trashed"); + if (params?.addParents) url.searchParams.set("addParents", params.addParents); + if (params?.removeParents) url.searchParams.set("removeParents", params.removeParents); + + const res = await fetch(url.toString(), { + method: "PATCH", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + if (!res.ok) throw new Error(`Drive file update failed: ${await res.text()}`); + return (await res.json()) as GoogleDriveFile; +} + /** Resolves an S3 object key to its parent folder ID, creating the directory hierarchy as needed. */ export async function resolvePathToFolderAndFile(accessToken: string, bucket: string, objectKey: string, env: Env): Promise<{ parentFolderId: string; fileName: string }> { - let currentFolderId = await getOrCreateFolder(accessToken, bucket, null, env); + let currentFolderId = await getBucketFolderId(accessToken, bucket, env); const parts = objectKey.split("/").filter((p) => p); @@ -287,8 +356,8 @@ function splitPrefix(prefix: string): { dirPrefix: string; partial: string } { } /** Walks an existing (read-only) folder path under the bucket; returns null if any segment is missing. */ -async function resolvePrefixFolder(accessToken: string, bucket: string, dirParts: string[]): Promise { - let folderId = await findFolderId(accessToken, bucket, null); +async function resolvePrefixFolder(accessToken: string, bucket: string, dirParts: string[], env: Env): Promise { + let folderId = await findBucketFolderId(accessToken, bucket, env); for (const part of dirParts) { if (folderId === null) return null; folderId = await findFolderId(accessToken, part, folderId); @@ -301,10 +370,10 @@ export interface ListedObject extends GoogleDriveFile { } /** Lists objects under a bucket, honoring an S3-style prefix and an optional single-level delimiter. */ -export async function listObjects(accessToken: string, bucket: string, prefix: string, delimiter?: string): Promise<{ contents: ListedObject[]; commonPrefixes: string[]; truncated: boolean }> { +export async function listObjects(accessToken: string, bucket: string, prefix: string, env: Env, delimiter?: string): Promise<{ contents: ListedObject[]; commonPrefixes: string[]; truncated: boolean }> { const { dirPrefix, partial } = splitPrefix(prefix); const dirParts = dirPrefix.split("/").filter((part) => part !== ""); - const folderId = await resolvePrefixFolder(accessToken, bucket, dirParts); + const folderId = await resolvePrefixFolder(accessToken, bucket, dirParts, env); if (folderId === null) return { contents: [], commonPrefixes: [], truncated: false }; const contents: ListedObject[] = []; diff --git a/src/index.ts b/src/index.ts index d7a8e2e..ecaff40 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ import { AUTH_PATH_PREFIX, handleAuth } from "./auth-api"; import { verifySignature } from "./aws-signature"; -import { isAllowedBucket, isPublicReadBucket } from "./bucket-access"; +import { findBucketRecord } from "./bucket-registry"; import { preflightResponse, withCors } from "./cors"; import * as docs from "./docs"; import { getAccessToken } from "./google-drive"; @@ -19,14 +19,14 @@ export default { const url = new URL(request.url); if (env.ENABLE_DOCS !== "false") { if (request.method === "GET" && url.pathname === docs.OPENAPI_PATH) return withCors(docs.openApiResponse(), request, env); - if (request.method === "GET" && url.pathname === docs.DOCS_PATH && !isAllowedBucket("docs", env)) return withCors(docs.docsResponse(), request, env); + if (request.method === "GET" && url.pathname === docs.DOCS_PATH) return withCors(docs.docsResponse(), request, env); } const pathParts = url.pathname.split("/").filter(Boolean); - if (pathParts[0] === AUTH_PATH_PREFIX && !isAllowedBucket(AUTH_PATH_PREFIX, env)) { + if (pathParts[0] === AUTH_PATH_PREFIX) { return withCors(await handleAuth(request, env, pathParts.slice(1).join("/")), request, env); } - if (pathParts[0] === API_PATH_PREFIX && !isAllowedBucket(API_PATH_PREFIX, env)) { + if (pathParts[0] === API_PATH_PREFIX) { return withCors(await handleApi(request, env, pathParts.slice(1).join("/")), request, env); } @@ -35,9 +35,10 @@ export default { const resource = url.pathname || "/"; try { - if (!isAllowedBucket(bucket, env)) return withCors(s3Error("AccessDenied", 403, undefined, resource, request.method === "HEAD"), request, env); + const record = await findBucketRecord(env, bucket); + if (!record) return withCors(s3Error("AccessDenied", 403, undefined, resource, request.method === "HEAD"), request, env); - const isPublicRead = isPublicReadBucket(bucket, env) && (request.method === "GET" || request.method === "HEAD"); + const isPublicRead = record.publicRead && (request.method === "GET" || request.method === "HEAD"); const signature = isPublicRead ? { ok: true as const } : await verifySignature(request, env); if (!signature.ok) { return withCors(s3Error(signature.code, 403, signature.message, resource, request.method === "HEAD"), request, env); @@ -46,6 +47,9 @@ export default { return withCors(await dispatch(request, env, await getAccessToken(env), bucket, objectKey), request, env); } catch (error) { if (error instanceof S3Exception) return withCors(s3Error(error.code, error.status, error.message, resource, request.method === "HEAD", error.headers), request, env); + if (error instanceof Error && error.message === "Storage root folder is not configured") { + return withCors(s3Error("AccessDenied", 403, "Access Denied", resource, request.method === "HEAD"), request, env); + } console.error(JSON.stringify({ message: "request failed", error: error instanceof Error ? error.message : String(error), method: request.method, path: url.pathname })); return withCors(s3Error("InternalError", 500, undefined, resource, request.method === "HEAD"), request, env); } diff --git a/src/router.ts b/src/router.ts index 24d1e58..bc0e0ca 100644 --- a/src/router.ts +++ b/src/router.ts @@ -194,7 +194,7 @@ export async function dispatch(request: Request, env: Env, accessToken: string, if (!key) { const prefix = url.searchParams.get("prefix") ?? ""; const delimiter = url.searchParams.get("delimiter") ?? undefined; - const { contents, commonPrefixes, truncated } = await listObjects(accessToken, bucket, prefix, delimiter); + const { contents, commonPrefixes, truncated } = await listObjects(accessToken, bucket, prefix, env, delimiter); return xmlResponse(generateListBucketResult(bucket, prefix, delimiter, contents, commonPrefixes, truncated)); } try { diff --git a/src/status-api.ts b/src/status-api.ts index d4089f7..b3a30df 100644 --- a/src/status-api.ts +++ b/src/status-api.ts @@ -1,6 +1,7 @@ import { jsonResponse, verifySessionToken } from "./auth-api"; -import { allowedBuckets, publicReadBuckets } from "./bucket-access"; -import { getAccessToken, getDriveAbout, listObjects } from "./google-drive"; +import { handleBucketRoutes, handleImportCandidatesRoute, handleImportRoute } from "./bucket-api"; +import { getBucketRegistry } from "./bucket-registry"; +import { getAccessToken, getDriveAbout, getRootFolderId, listObjects } from "./google-drive"; import type { DriveAbout, Env } from "./types"; export const API_PATH_PREFIX = "api"; @@ -17,6 +18,11 @@ export interface GatewayStatusResponse { docsEnabled: boolean; buckets: string[]; publicReadBuckets: string[]; + rootFolder: { + name: string | null; + id: string | null; + configured: boolean; + }; corsOrigins: string[]; credentials: { s3Keys: boolean; @@ -74,19 +80,35 @@ export async function handleApi(request: Request, env: Env, subPath: string): Pr return jsonResponse({ message: "Session expired or invalid" }, 401); } - if (request.method !== "GET") { - return jsonResponse({ message: "Method Not Allowed" }, 405); + if (!env.DRIVE_ROOT_FOLDER || env.DRIVE_ROOT_FOLDER.trim() === "") { + return jsonResponse({ message: "Storage root folder is not configured" }, 503); } + const segments = subPath.split("/").filter(Boolean); + const firstSegment = segments[0] || ""; + const remainingSegments = segments.slice(1); + const url = new URL(request.url); - if (subPath === "status") { + if (firstSegment === "status") { + if (request.method !== "GET") return jsonResponse({ message: "Method Not Allowed" }, 405); return await handleStatus(env); } - if (subPath === "buckets") { - const forceRefresh = url.searchParams.get("refresh") === "1"; - return await handleBuckets(env, forceRefresh); + if (firstSegment === "buckets") { + if (request.method === "GET" && remainingSegments.length === 0) { + const forceRefresh = url.searchParams.get("refresh") === "1"; + return await handleBuckets(env, forceRefresh); + } + return await handleBucketRoutes(request, env, remainingSegments); + } + + if (firstSegment === "import-candidates") { + return await handleImportCandidatesRoute(request, env); + } + + if (firstSegment === "import") { + return await handleImportRoute(request, env); } return jsonResponse({ message: "Not Found" }, 404); @@ -104,8 +126,18 @@ export async function handleApi(request: Request, env: Env, subPath: string): Pr } async function handleStatus(env: Env): Promise { - const buckets = allowedBuckets(env); - const pubBuckets = publicReadBuckets(env); + let buckets: string[] = []; + let pubBuckets: string[] = []; + let rootFolderId: string | null = null; + + try { + const registry = await getBucketRegistry(env); + buckets = registry.map((b) => b.name); + pubBuckets = registry.filter((b) => b.publicRead).map((b) => b.name); + } catch (err) { + console.error("Failed to load bucket registry for status", err); + } + const corsOrigins = env.CORS_ALLOWED_ORIGINS ? env.CORS_ALLOWED_ORIGINS.split(",") .map((o) => o.trim()) @@ -116,11 +148,19 @@ async function handleStatus(env: Env): Promise { let driveError: string | null = null; try { + const accessToken = await getAccessToken(env); + if (env.DRIVE_ROOT_FOLDER) { + try { + rootFolderId = await getRootFolderId(accessToken, env); + } catch (err) { + console.error("Failed to get root folder id", err); + } + } + const cached = await env.AUTH_KV.get(DRIVE_ABOUT_CACHE_KEY); if (cached) { driveAbout = JSON.parse(cached) as DriveAbout; } else { - const accessToken = await getAccessToken(env); driveAbout = await getDriveAbout(accessToken); await env.AUTH_KV.put(DRIVE_ABOUT_CACHE_KEY, JSON.stringify(driveAbout), { expirationTtl: DRIVE_ABOUT_CACHE_TTL, @@ -141,6 +181,11 @@ async function handleStatus(env: Env): Promise { docsEnabled: env.ENABLE_DOCS !== "false", buckets, publicReadBuckets: pubBuckets, + rootFolder: { + name: env.DRIVE_ROOT_FOLDER || null, + id: rootFolderId, + configured: Boolean(env.DRIVE_ROOT_FOLDER && env.DRIVE_ROOT_FOLDER.trim() !== ""), + }, corsOrigins, credentials: { s3Keys: Boolean(env.ACCESS_KEY && env.SECRET_KEY), @@ -166,8 +211,12 @@ async function handleStatus(env: Env): Promise { } async function handleBuckets(env: Env, forceRefresh: boolean): Promise { - const buckets = allowedBuckets(env); - const pubBuckets = new Set(publicReadBuckets(env)); + let records: { name: string; publicRead: boolean }[] = []; + try { + records = await getBucketRegistry(env); + } catch (err) { + console.error("Failed to acquire bucket registry", err); + } let accessToken: string | null = null; try { @@ -178,7 +227,8 @@ async function handleBuckets(env: Env, forceRefresh: boolean): Promise const bucketStats: BucketStatItem[] = []; - for (const bucket of buckets) { + for (const record of records) { + const bucket = record.name; const cacheKey = `bucket-stats:${bucket}`; if (!forceRefresh) { const cached = await env.FOLDER_CACHE.get(cacheKey); @@ -199,14 +249,14 @@ async function handleBuckets(env: Env, forceRefresh: boolean): Promise totalSize: 0, lastModified: null, truncated: false, - publicRead: pubBuckets.has(bucket), + publicRead: record.publicRead, error: "Google Drive access unavailable", }); continue; } try { - const { contents, truncated } = await listObjects(accessToken, bucket, ""); + const { contents, truncated } = await listObjects(accessToken, bucket, "", env); let totalSize = 0; let latestModified: number | null = null; @@ -227,7 +277,7 @@ async function handleBuckets(env: Env, forceRefresh: boolean): Promise totalSize, lastModified: latestModified ? new Date(latestModified).toISOString() : null, truncated, - publicRead: pubBuckets.has(bucket), + publicRead: record.publicRead, error: null, }; @@ -243,7 +293,7 @@ async function handleBuckets(env: Env, forceRefresh: boolean): Promise totalSize: 0, lastModified: null, truncated: false, - publicRead: pubBuckets.has(bucket), + publicRead: record.publicRead, error: err instanceof Error ? err.message : String(err), }); } diff --git a/src/types.ts b/src/types.ts index 7a87547..fcebe51 100644 --- a/src/types.ts +++ b/src/types.ts @@ -9,8 +9,7 @@ export interface Env { FOLDER_CACHE: KVNamespace; MPU: DurableObjectNamespace; DASHBOARD_PASSWORD?: string; - ALLOWED_BUCKETS?: string; - PUBLIC_READ_BUCKETS?: string; + DRIVE_ROOT_FOLDER?: string; ALLOW_MULTIPART?: string; ETAG_STYLE?: "md5" | "multipart"; CORS_ALLOWED_ORIGINS?: string; @@ -23,7 +22,9 @@ export interface GoogleDriveFile { mimeType: string; size: string; modifiedTime?: string; + createdTime?: string; md5Checksum?: string; + appProperties?: Record; } export interface GoogleDriveSearchResponse { diff --git a/test/auth.test.ts b/test/auth.test.ts index c0aef60..12333de 100644 --- a/test/auth.test.ts +++ b/test/auth.test.ts @@ -194,13 +194,6 @@ describe("Dashboard authentication API routes", () => { expect(blockedRes.headers.get("Retry-After")).toBe("900"); }); - it("does not claim /auth when auth is a configured bucket", async () => { - const withAuthBucket = { ...ENV, ALLOWED_BUCKETS: "test-bucket,auth" }; - const response = await worker.fetch(new Request(`${ENDPOINT}/auth/login`), withAuthBucket, CTX); - expect(response.status).toBe(403); - expect(await response.text()).toContain("SignatureDoesNotMatch"); - }); - it("emits CORS headers for allowed origin", async () => { const res = await worker.fetch( new Request(`${ENDPOINT}/auth/session`, { diff --git a/test/buckets.test.ts b/test/buckets.test.ts new file mode 100644 index 0000000..7b39339 --- /dev/null +++ b/test/buckets.test.ts @@ -0,0 +1,365 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { sha256 } from "../src/aws-signature"; +import worker from "../src/index"; +import type { Env } from "../src/types"; + +import { env } from "cloudflare:test"; + +const ENV = env as unknown as Env; +const ENDPOINT = "https://s3-api.example.com"; +const CTX = { waitUntil: vi.fn(), passThroughOnException: vi.fn() } as unknown as ExecutionContext; + +let testIpCounter = 1; +function getUniqueIp(): string { + return `10.0.0.${testIpCounter++}`; +} + +async function getValidToken(ip = getUniqueIp()): Promise { + const passwordHash = await sha256("test-dashboard-password"); + const loginRes = await worker.fetch( + new Request(`${ENDPOINT}/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json", "CF-Connecting-IP": ip }, + body: JSON.stringify({ passwordHash }), + }), + ENV, + CTX, + ); + const data = (await loginRes.json()) as { token: string }; + return data.token; +} + +beforeEach(async () => { + await (ENV.AUTH_KV as KVNamespace).delete("drive-about"); + await (ENV.FOLDER_CACHE as KVNamespace).delete("bucket-registry"); + for (const { name } of (await (ENV.FOLDER_CACHE as KVNamespace).list()).keys) { + await (ENV.FOLDER_CACHE as KVNamespace).delete(name); + } +}); + +describe("Bucket management CRUD API routes (/api/buckets, /api/import*)", () => { + it("returns 503 on /api/buckets when DRIVE_ROOT_FOLDER is unset", async () => { + const token = await getValidToken(); + const customEnv = { ...ENV, DRIVE_ROOT_FOLDER: undefined }; + const res = await worker.fetch( + new Request(`${ENDPOINT}/api/buckets`, { + headers: { Authorization: `Bearer ${token}` }, + }), + customEnv, + CTX, + ); + expect(res.status).toBe(503); + const data = (await res.json()) as { message: string }; + expect(data.message).toBe("Storage root folder is not configured"); + }); + + it("rejects reserved bucket names on POST /api/buckets with 400", async () => { + const token = await getValidToken(); + for (const reserved of ["auth", "api", "docs"]) { + const res = await worker.fetch( + new Request(`${ENDPOINT}/api/buckets`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, + body: JSON.stringify({ name: reserved }), + }), + ENV, + CTX, + ); + expect(res.status).toBe(400); + const data = (await res.json()) as { message: string }; + expect(data.message).toContain("reserved"); + } + }); + + it("rejects invalid bucket names on POST /api/buckets with 400", async () => { + const token = await getValidToken(); + const invalidNames = ["ab", "Abc", "bucket--name", "-bucket", "bucket-", "192.168.1.1"]; + for (const name of invalidNames) { + const res = await worker.fetch( + new Request(`${ENDPOINT}/api/buckets`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, + body: JSON.stringify({ name }), + }), + ENV, + CTX, + ); + expect(res.status).toBe(400); + } + }); + + it("creates a bucket with POST /api/buckets and handles conflict with 409", async () => { + const token = await getValidToken(); + + const fakeFetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(typeof input === "string" ? input : input instanceof Request ? input.url : input.toString()); + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); + + if (url.origin === "https://oauth2.googleapis.com") { + return Response.json({ access_token: "mock-access-token", expires_in: 3600 }); + } + + if (url.pathname === "/drive/v3/files") { + const q = url.searchParams.get("q") ?? ""; + if (method === "GET") { + if (q.includes("name='s3-storage'") && q.includes("'root' in parents")) { + return Response.json({ files: [{ id: "root-folder-id", name: "s3-storage" }] }); + } + if (q.includes("'root-folder-id' in parents") && q.includes("mimeType='application/vnd.google-apps.folder'")) { + return Response.json({ files: [] }); + } + if (q.includes("name='new-bucket'")) { + return Response.json({ files: [] }); + } + return Response.json({ files: [] }); + } + if (method === "POST") { + return Response.json({ id: "new-bucket-folder-id" }); + } + } + if (url.pathname === "/drive/v3/files/new-bucket-folder-id" && method === "PATCH") { + return Response.json({ id: "new-bucket-folder-id", name: "new-bucket" }); + } + return new Response("Not found", { status: 404 }); + }); + + vi.stubGlobal("fetch", fakeFetch); + + try { + const res = await worker.fetch( + new Request(`${ENDPOINT}/api/buckets`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, + body: JSON.stringify({ name: "new-bucket", publicRead: true }), + }), + ENV, + CTX, + ); + expect(res.status).toBe(201); + const data = (await res.json()) as { name: string; folderId: string; publicRead: boolean }; + expect(data.name).toBe("new-bucket"); + expect(data.folderId).toBe("new-bucket-folder-id"); + expect(data.publicRead).toBe(true); + } finally { + vi.unstubAllGlobals(); + } + }); + + it("updates bucket publicRead with PATCH /api/buckets/:name", async () => { + const token = await getValidToken(); + + const fakeFetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(typeof input === "string" ? input : input instanceof Request ? input.url : input.toString()); + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); + + if (url.origin === "https://oauth2.googleapis.com") { + return Response.json({ access_token: "mock-access-token", expires_in: 3600 }); + } + if (url.pathname === "/drive/v3/files") { + const q = url.searchParams.get("q") ?? ""; + if (q.includes("name='s3-storage'") && q.includes("'root' in parents")) { + return Response.json({ files: [{ id: "root-folder-id", name: "s3-storage" }] }); + } + if (q.includes("'root-folder-id' in parents")) { + return Response.json({ + files: [{ id: "folder-target-bucket", name: "target-bucket", mimeType: "application/vnd.google-apps.folder", appProperties: { s3PublicRead: "false" } }], + }); + } + } + if (url.pathname === "/drive/v3/files/folder-target-bucket" && method === "PATCH") { + return Response.json({ id: "folder-target-bucket", name: "target-bucket", appProperties: { s3PublicRead: "true" } }); + } + return new Response("Not found", { status: 404 }); + }); + + vi.stubGlobal("fetch", fakeFetch); + + try { + const res = await worker.fetch( + new Request(`${ENDPOINT}/api/buckets/target-bucket`, { + method: "PATCH", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, + body: JSON.stringify({ publicRead: true }), + }), + ENV, + CTX, + ); + expect(res.status).toBe(200); + const data = (await res.json()) as { name: string; publicRead: boolean }; + expect(data.name).toBe("target-bucket"); + expect(data.publicRead).toBe(true); + } finally { + vi.unstubAllGlobals(); + } + }); + + it("rejects DELETE /api/buckets/:name when bucket has children with 409", async () => { + const token = await getValidToken(); + + const fakeFetch = vi.fn(async (input: RequestInfo | URL) => { + const url = new URL(typeof input === "string" ? input : input instanceof Request ? input.url : input.toString()); + + if (url.origin === "https://oauth2.googleapis.com") { + return Response.json({ access_token: "mock-access-token", expires_in: 3600 }); + } + if (url.pathname === "/drive/v3/files") { + const q = url.searchParams.get("q") ?? ""; + if (q.includes("name='s3-storage'") && q.includes("'root' in parents")) { + return Response.json({ files: [{ id: "root-folder-id", name: "s3-storage" }] }); + } + if (q.includes("'root-folder-id' in parents")) { + return Response.json({ + files: [{ id: "folder-non-empty", name: "non-empty", mimeType: "application/vnd.google-apps.folder" }], + }); + } + if (q.includes("'folder-non-empty' in parents")) { + return Response.json({ files: [{ id: "child-file-1" }] }); + } + } + return new Response("Not found", { status: 404 }); + }); + + vi.stubGlobal("fetch", fakeFetch); + + try { + const res = await worker.fetch( + new Request(`${ENDPOINT}/api/buckets/non-empty`, { + method: "DELETE", + headers: { Authorization: `Bearer ${token}` }, + }), + ENV, + CTX, + ); + expect(res.status).toBe(409); + const data = (await res.json()) as { message: string }; + expect(data.message).toContain("not empty"); + } finally { + vi.unstubAllGlobals(); + } + }); + + it("deletes empty bucket with DELETE /api/buckets/:name and returns 204", async () => { + const token = await getValidToken(); + + const fakeFetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(typeof input === "string" ? input : input instanceof Request ? input.url : input.toString()); + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); + + if (url.origin === "https://oauth2.googleapis.com") { + return Response.json({ access_token: "mock-access-token", expires_in: 3600 }); + } + if (url.pathname === "/drive/v3/files") { + const q = url.searchParams.get("q") ?? ""; + if (q.includes("name='s3-storage'") && q.includes("'root' in parents")) { + return Response.json({ files: [{ id: "root-folder-id", name: "s3-storage" }] }); + } + if (q.includes("'root-folder-id' in parents")) { + return Response.json({ + files: [{ id: "folder-empty", name: "empty", mimeType: "application/vnd.google-apps.folder" }], + }); + } + if (q.includes("'folder-empty' in parents")) { + return Response.json({ files: [] }); + } + } + if (url.pathname === "/drive/v3/files/folder-empty" && method === "PATCH") { + return Response.json({ id: "folder-empty", trashed: true }); + } + return new Response("Not found", { status: 404 }); + }); + + vi.stubGlobal("fetch", fakeFetch); + + try { + const res = await worker.fetch( + new Request(`${ENDPOINT}/api/buckets/empty`, { + method: "DELETE", + headers: { Authorization: `Bearer ${token}` }, + }), + ENV, + CTX, + ); + expect(res.status).toBe(204); + } finally { + vi.unstubAllGlobals(); + } + }); + + it("lists import candidates and imports selected buckets", async () => { + const token = await getValidToken(); + + const fakeFetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(typeof input === "string" ? input : input instanceof Request ? input.url : input.toString()); + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); + + if (url.origin === "https://oauth2.googleapis.com") { + return Response.json({ access_token: "mock-access-token", expires_in: 3600 }); + } + if (url.pathname === "/drive/v3/files") { + const q = url.searchParams.get("q") ?? ""; + if (q.includes("name='s3-storage'") && q.includes("'root' in parents")) { + return Response.json({ files: [{ id: "root-folder-id", name: "s3-storage" }] }); + } + if (q.includes("'root-folder-id' in parents")) { + return Response.json({ files: [] }); + } + if (q.includes("'root' in parents")) { + return Response.json({ + files: [ + { id: "root-folder-id", name: "s3-storage", mimeType: "application/vnd.google-apps.folder" }, + { id: "import-folder-1", name: "legacy-bucket", mimeType: "application/vnd.google-apps.folder" }, + ], + }); + } + if (q.includes("'import-folder-1' in parents")) { + return Response.json({ + files: [ + { id: "f1", mimeType: "text/plain" }, + { id: "f2", mimeType: "text/plain" }, + ], + }); + } + } + if (url.pathname === "/drive/v3/files/import-folder-1" && method === "PATCH") { + return Response.json({ id: "import-folder-1", name: "legacy-bucket" }); + } + return new Response("Not found", { status: 404 }); + }); + + vi.stubGlobal("fetch", fakeFetch); + + try { + // GET /api/import-candidates + const listRes = await worker.fetch( + new Request(`${ENDPOINT}/api/import-candidates`, { + headers: { Authorization: `Bearer ${token}` }, + }), + ENV, + CTX, + ); + expect(listRes.status).toBe(200); + const listData = (await listRes.json()) as { candidates: Array<{ name: string; folderId: string; objectCount: number }> }; + expect(listData.candidates).toHaveLength(1); + expect(listData.candidates[0].name).toBe("legacy-bucket"); + expect(listData.candidates[0].objectCount).toBe(2); + + // POST /api/import + const importRes = await worker.fetch( + new Request(`${ENDPOINT}/api/import`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, + body: JSON.stringify({ names: ["legacy-bucket"] }), + }), + ENV, + CTX, + ); + expect(importRes.status).toBe(200); + const importData = (await importRes.json()) as { imported: string[]; failed: unknown[] }; + expect(importData.imported).toEqual(["legacy-bucket"]); + expect(importData.failed).toHaveLength(0); + } finally { + vi.unstubAllGlobals(); + } + }); +}); diff --git a/test/cors.test.ts b/test/cors.test.ts index aee85c7..8569e28 100644 --- a/test/cors.test.ts +++ b/test/cors.test.ts @@ -25,7 +25,19 @@ function fakeGoogleFetch(input: string | URL | Request, init?: RequestInit): Pro const request = input instanceof Request ? input : new Request(input, init); const url = new URL(request.url); if (url.hostname === "oauth2.googleapis.com") return Promise.resolve(Response.json({ access_token: "token", expires_in: 3600 })); - if (url.pathname === "/drive/v3/files" && request.method === "GET") return Promise.resolve(Response.json({ files: [] })); + if (url.pathname === "/drive/v3/files" && request.method === "GET") { + const q = url.searchParams.get("q") ?? ""; + if (q.includes("name='s3-storage'") && q.includes("'root' in parents")) { + return Promise.resolve(Response.json({ files: [{ id: "root-folder-id", name: "s3-storage" }] })); + } + if (q.includes("'root-folder-id' in parents")) { + return Promise.resolve(Response.json({ files: [{ id: "folder-test-bucket", name: "test-bucket", mimeType: "application/vnd.google-apps.folder" }] })); + } + if (q.includes("name='test-bucket'")) { + return Promise.resolve(Response.json({ files: [{ id: "folder-test-bucket", name: "test-bucket", mimeType: "application/vnd.google-apps.folder" }] })); + } + return Promise.resolve(Response.json({ files: [] })); + } if (url.pathname === "/drive/v3/files" && request.method === "POST") return Promise.resolve(Response.json({ id: "folder-1" })); if (url.pathname.startsWith("/upload/drive/v3/files")) return Promise.resolve(new Response(null, { headers: { Location: "https://www.googleapis.com/upload/session/test" } })); if (url.pathname === "/upload/session/test") return Promise.resolve(Response.json({ id: "file-1", name: "file.txt", md5Checksum: "d41d8cd98f00b204e9800998ecf8427e" })); diff --git a/test/docs.test.ts b/test/docs.test.ts index d6a20e7..6f75464 100644 --- a/test/docs.test.ts +++ b/test/docs.test.ts @@ -24,16 +24,23 @@ describe("API documentation routes", () => { it("bypasses documentation routes when disabled", async () => { const disabled = { ...ENV, ENABLE_DOCS: "false" }; - for (const path of ["/docs", "/openapi.yaml"]) { - const response = await worker.fetch(new Request(`${ENDPOINT}${path}`), disabled, CTX); - expect(response.status).toBe(403); + const fakeFetch = vi.fn(async (input: RequestInfo | URL) => { + const url = new URL(typeof input === "string" ? input : input instanceof Request ? input.url : input.toString()); + if (url.origin === "https://oauth2.googleapis.com") return Response.json({ access_token: "token", expires_in: 3600 }); + const q = url.searchParams.get("q") ?? ""; + if (q.includes("name='s3-storage'") && q.includes("'root' in parents")) { + return Response.json({ files: [{ id: "root-folder-id", name: "s3-storage" }] }); + } + return Response.json({ files: [] }); + }); + vi.stubGlobal("fetch", fakeFetch); + try { + for (const path of ["/docs", "/openapi.yaml"]) { + const response = await worker.fetch(new Request(`${ENDPOINT}${path}`), disabled, CTX); + expect(response.status).toBe(403); + } + } finally { + vi.unstubAllGlobals(); } }); - - it("does not claim /docs when docs is a configured bucket", async () => { - const withDocsBucket = { ...ENV, ALLOWED_BUCKETS: "test-bucket,docs" }; - const response = await worker.fetch(new Request(`${ENDPOINT}/docs`), withDocsBucket, CTX); - expect(response.status).toBe(403); - expect(await response.text()).toContain("SignatureDoesNotMatch"); - }); }); diff --git a/test/s3.test.ts b/test/s3.test.ts index af259a0..0dd2d21 100644 --- a/test/s3.test.ts +++ b/test/s3.test.ts @@ -225,6 +225,14 @@ let drive: FakeDrive; beforeEach(async () => { drive = new FakeDrive(); + // Pre-create root folder "s3-storage" under "root" and bucket folders under root folder + const rootFolderId = "folder-root"; + drive.folders.set(rootFolderId, { id: rootFolderId, name: "s3-storage", parent: "root" }); + const testBucketId = "folder-test-bucket"; + drive.folders.set(testBucketId, { id: testBucketId, name: "test-bucket", parent: rootFolderId }); + const emptyBucketId = "folder-empty-bucket"; + drive.folders.set(emptyBucketId, { id: emptyBucketId, name: "empty-bucket", parent: rootFolderId }); + vi.stubGlobal( "fetch", vi.fn((input, init) => drive.handle(input, init)), diff --git a/test/status.test.ts b/test/status.test.ts index d84dd83..fc0a57d 100644 --- a/test/status.test.ts +++ b/test/status.test.ts @@ -92,6 +92,23 @@ describe("Dashboard status API routes (/api/*)", () => { }); } + if (url.pathname === "/drive/v3/files") { + const q = url.searchParams.get("q") ?? ""; + if (q.includes("name='s3-storage'") && q.includes("'root' in parents")) { + return Response.json({ files: [{ id: "root-folder-id", name: "s3-storage" }] }); + } + if (q.includes("'root-folder-id' in parents")) { + return Response.json({ + files: [ + { id: "folder-test-bucket", name: "test-bucket", mimeType: "application/vnd.google-apps.folder" }, + { id: "folder-empty-bucket", name: "empty-bucket", mimeType: "application/vnd.google-apps.folder" }, + { id: "folder-my-bucket", name: "my-bucket", mimeType: "application/vnd.google-apps.folder" }, + ], + }); + } + return Response.json({ files: [] }); + } + return new Response("Not found", { status: 404 }); }); @@ -113,6 +130,11 @@ describe("Dashboard status API routes (/api/*)", () => { expect(data.gateway.region).toBe("auto"); expect(data.gateway.multipartEnabled).toBe(true); expect(data.gateway.buckets).toEqual(["test-bucket", "empty-bucket", "my-bucket"]); + expect(data.gateway.rootFolder).toEqual({ + name: "s3-storage", + id: "root-folder-id", + configured: true, + }); expect(data.gateway.credentials).toEqual({ s3Keys: true, googleOAuth: true, @@ -210,6 +232,7 @@ describe("Dashboard status API routes (/api/*)", () => { it("returns bucket statistics on /api/buckets", async () => { await (ENV.AUTH_KV as KVNamespace).delete("drive-about"); + await (ENV.FOLDER_CACHE as KVNamespace).delete("bucket-registry"); const token = await getValidToken(); const fakeFetch = vi.fn(async (input: RequestInfo | URL) => { @@ -219,9 +242,27 @@ describe("Dashboard status API routes (/api/*)", () => { } if (url.pathname === "/drive/v3/files") { const q = url.searchParams.get("q") ?? ""; - if (q.includes("name='test-bucket'")) { + if (q.includes("name='s3-storage'") && q.includes("'root' in parents")) { + return Response.json({ files: [{ id: "root-folder-id", name: "s3-storage" }] }); + } + if (q.includes("name='test-bucket'") && q.includes("'root-folder-id' in parents")) { return Response.json({ files: [{ id: "folder-test-bucket", name: "test-bucket" }] }); } + if (q.includes("name='empty-bucket'") && q.includes("'root-folder-id' in parents")) { + return Response.json({ files: [{ id: "folder-empty-bucket", name: "empty-bucket" }] }); + } + if (q.includes("name='my-bucket'") && q.includes("'root-folder-id' in parents")) { + return Response.json({ files: [{ id: "folder-my-bucket", name: "my-bucket" }] }); + } + if (q.includes("'root-folder-id' in parents")) { + return Response.json({ + files: [ + { id: "folder-test-bucket", name: "test-bucket", mimeType: "application/vnd.google-apps.folder" }, + { id: "folder-empty-bucket", name: "empty-bucket", mimeType: "application/vnd.google-apps.folder" }, + { id: "folder-my-bucket", name: "my-bucket", mimeType: "application/vnd.google-apps.folder" }, + ], + }); + } if (q.includes("'folder-test-bucket' in parents")) { return Response.json({ files: [ @@ -235,6 +276,9 @@ describe("Dashboard status API routes (/api/*)", () => { ], }); } + if (q.includes("'folder-empty-bucket' in parents") || q.includes("'folder-my-bucket' in parents")) { + return Response.json({ files: [] }); + } return Response.json({ files: [] }); } return new Response("Not found", { status: 404 }); @@ -265,13 +309,4 @@ describe("Dashboard status API routes (/api/*)", () => { vi.unstubAllGlobals(); } }); - - it("routes /api to S3 handler when 'api' is configured as an allowed bucket", async () => { - const customEnv = { ...ENV, ALLOWED_BUCKETS: "api,test-bucket" }; - const res = await worker.fetch(new Request(`${ENDPOINT}/api/status`), customEnv, CTX); - // S3 router checks signature or returns SignatureDoesNotMatch / AccessDenied etc. - expect(res.status).toBe(403); - const text = await res.text(); - expect(text).toContain(""); - }); }); diff --git a/vitest.config.mts b/vitest.config.mts index 11e2fc4..2ff3f25 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -14,7 +14,7 @@ export default defineConfig({ GOOGLE_CLIENT_SECRET: "test-client-secret", GOOGLE_REFRESH_TOKEN: "test-refresh-token", DASHBOARD_PASSWORD: "test-dashboard-password", - ALLOWED_BUCKETS: "test-bucket,empty-bucket,my-bucket", + DRIVE_ROOT_FOLDER: "s3-storage", ALLOW_MULTIPART: "true", ETAG_STYLE: "md5", CORS_ALLOWED_ORIGINS: "http://localhost:5173", diff --git a/wrangler.jsonc b/wrangler.jsonc index 2c708da..0637b49 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -8,9 +8,10 @@ "main": "src/index.ts", "compatibility_date": "2025-09-27", "vars": { + "DRIVE_ROOT_FOLDER": "s3-storage", "ALLOW_MULTIPART": "true", "ETAG_STYLE": "md5", - "CORS_ALLOWED_ORIGINS": "https://s3-drive-storage-manage.nezumi.workers.dev", + "CORS_ALLOWED_ORIGINS": "https://s3-drive-storage-manage.nezumi.workers.dev,http://localhost:5173", "ENABLE_DOCS": "true" }, "rules": [