diff --git a/README.md b/README.md index ec5415f..8bf46ea 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,16 @@ // LOGO OR BANNER GOES HERE ## Status -Currently still in development, ~~but main features are already working as intended.~~ Preview and download files are not working because of Vercel's limit for serverless function. Working on a workaround for this. +### Project halted at the moment. +Found out it's not cfworker fault, but Google Drive API is the one that slow. +It took 500ms to 1s response time to fetch files, took longer if it's inside protected folder. +It can be faster if I'm using `@upstash/redis` to cache the response, but for free tier it capped on 10k request per day. (It should've enough for most people) -You can check the demo [here](https://drive.mbaharip.com). +I'm currently working on a workaround for this, but it's not a priority for me at the moment. + +~~Currently still in development, ~~but main features are already working as intended.~~ Preview and download files are not working because of Vercel's limit for serverless function. Working on a workaround for this.~~ + +~~You can check the demo [here](https://drive.mbaharip.com).~~ ## What is this? gudora-index is an indexer for Google Drive, it's a simple project that I made to index my files in Google Drive. diff --git a/package.json b/package.json index 1a3aea2..d3d3a15 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "@react-three/fiber": "^8.13.0", "@types/react": "18.0.35", "@types/react-dom": "18.0.11", + "@upstash/redis": "^1.20.4", "autoprefixer": "10.4.14", "axios": "^1.3.5", "cors": "^2.8.5", diff --git a/src/pages/api/experimental/[...path].ts b/src/pages/api/experimental/[...path].ts index 1c29eff..36c9396 100644 --- a/src/pages/api/experimental/[...path].ts +++ b/src/pages/api/experimental/[...path].ts @@ -31,10 +31,12 @@ export default async function handler( let isReadmeFile = false; let payload: FileResponse | FilesResponse; + let description = ""; if (findPath.mimeType === "application/vnd.google-apps.folder") { const fetchFolder = await drive.files.list({ q: `'${findPath.id}' in parents and trashed = false and 'me' in owners`, - fields: "files(id, name, mimeType, parents), nextPageToken", + fields: + "files(id, name, mimeType, parents, description), nextPageToken", }); isReadmeFile = !!fetchFolder.data.files?.some( (file) => file.name === ".readme.md", @@ -85,29 +87,43 @@ export default async function handler( } } - if (isProtected) { - const findPassword = await drive.files.list({ - q: `name = '.password' and '${protectedId}' in parents and trashed = false and 'me' in owners`, - fields: "files(id)", + if (protectedId) { + const findPassword = await drive.files.get({ + fileId: protectedId, + fields: "description", }); - if (findPassword.data.files?.length) { - const passwordFile = await drive.files.get({ - fileId: findPassword.data.files[0].id as string, - alt: "media", - }); - const password = passwordFile.data as unknown as string; + if (findPassword.data.description) { + description = findPassword.data.description; const { password: passwordQuery } = request.query; - if (password === passwordQuery) { + if (findPassword.data.description === passwordQuery) { isValidated = true; } } } - if (isProtected && !isValidated) { - const error = new Error("Protected folder") as ExtendedError; - error.cause = "protected"; - error.code = 403; - throw error; - } + + // if (isProtected) { + // const findPassword = await drive.files.list({ + // q: `name = '.password' and '${protectedId}' in parents and trashed = false and 'me' in owners`, + // fields: "files(id)", + // }); + // if (findPassword.data.files?.length) { + // const passwordFile = await drive.files.get({ + // fileId: findPassword.data.files[0].id as string, + // alt: "media", + // }); + // const password = passwordFile.data as unknown as string; + // const { password: passwordQuery } = request.query; + // if (password === passwordQuery) { + // isValidated = true; + // } + // } + // } + // if (isProtected && !isValidated) { + // const error = new Error("Protected folder") as ExtendedError; + // error.cause = "protected"; + // error.code = 403; + // throw error; + // } payload.passwordRequired = isProtected; payload.passwordValidated = isValidated; @@ -116,7 +132,7 @@ export default async function handler( const _end = Date.now(); payload.durationMs = _end - _start; - return response.status(200).json(payload); + return response.status(200).json({ payload, description }); } catch (error: any) { console.error(error); const _end = Date.now(); diff --git a/src/pages/api/experimental/files/index.ts b/src/pages/api/experimental/files/index.ts new file mode 100644 index 0000000..e20fc62 --- /dev/null +++ b/src/pages/api/experimental/files/index.ts @@ -0,0 +1,130 @@ +import { NextApiRequest, NextApiResponse } from "next"; +import drive, { redis } from "@utils/driveClient"; +import config from "@/config/site.config"; +import { ErrorResponse, FilesResponse } from "@/types/googleapis"; +import { urlEncrypt } from "@/utils/encryptionHelper"; +import { ExtendedError } from "@/types/default"; + +export default async function handler( + request: NextApiRequest, + response: NextApiResponse, +) { + const _start = Date.now(); + + try { + const cacheKey = request.url as string; + const { isRefresh } = request.query; + + if (!isRefresh) { + const cachedResponse = await redis.get(cacheKey); + if (cachedResponse) { + const _end = Date.now(); + return response + .status(200) + .setHeader("Cache-Control", "max-age=60") + .json({ ...cachedResponse, durationMs: _end - _start }); + } + } + + const { pageToken } = request.query; + const authorization = + process.env.NODE_ENV === "development" + ? request.query.hash + : request.headers.authorization?.split(" ")[1] || null; + + const query = [ + `'${config.files.rootFolder}' in parents`, + "trashed = false", + "'me' in owners", + ]; + const promiseFolderContents = await drive.files.list({ + q: `${query.join(" and ")}`, + fields: + "files(id, name, mimeType, thumbnailLink, fileExtension, createdTime, modifiedTime, size, videoMediaMetadata), nextPageToken", + orderBy: "folder, name asc", + pageSize: config.files.itemsPerPage, + pageToken: (pageToken as string) || undefined, + }); + + const passwordFile = promiseFolderContents.data.files?.find( + (file) => file.name === ".password", + ); + const readmeFile = promiseFolderContents.data.files?.find( + (file) => file.name === ".readme.md", + ); + const folderList = + promiseFolderContents.data.files?.filter( + (item) => item.mimeType === "application/vnd.google-apps.folder", + ) || []; + const fileList = + promiseFolderContents.data.files?.filter( + (item) => item.mimeType !== "application/vnd.google-apps.folder", + ) || []; + + // if (passwordFile && !authorization) { + // const error: ExtendedError = new Error("Unauthorized"); + // error.code = 401; + // error.cause = "unauthorized"; + // throw error; + // } + + if (readmeFile && authorization) { + const validatePassword = await drive.files.get( + { + fileId: promiseFolderContents.data.files?.find( + (file) => file.name === ".password", + )?.id as string, + alt: "media", + }, + { responseType: "text" }, + ); + if (validatePassword.data !== authorization) { + const error: ExtendedError = new Error("Unauthorized"); + error.code = 401; + error.cause = "unauthorized"; + throw error; + } + } + + const _end = Date.now(); + const payload: FilesResponse = { + success: true, + timestamp: new Date().toISOString(), + durationMs: _end - _start, + passwordRequired: passwordFile ? true : false, + passwordValidated: true, + protectedId: "", + parents: [], + files: fileList.map((file) => ({ + ...file, + id: urlEncrypt(file.id as string), + })), + folders: folderList, + readmeExists: readmeFile ? true : false, + nextPageToken: promiseFolderContents.data.nextPageToken || undefined, + }; + + await redis.set(cacheKey as string, JSON.stringify(payload), { + ex: 60, + }); + return response + .status(200) + .setHeader("Cache-Control", "s-maxage=60, stale-while-revalidate") + .json(payload); + } catch (error: any) { + const _end = Date.now(); + console.log(error); + const payload: ErrorResponse = { + success: false, + timestamp: new Date().toISOString(), + durationMs: _end - _start, + code: error.code || 500, + errors: { + message: error.errors?.[0].message || error.message || "Unknown error", + reason: error.errors?.[0].reason || error.cause || "internalError", + }, + }; + + return response.status(payload.code || 500).json(payload); + } +} diff --git a/src/utils/driveClient.ts b/src/utils/driveClient.ts index d13429f..70857fa 100644 --- a/src/utils/driveClient.ts +++ b/src/utils/driveClient.ts @@ -1,6 +1,7 @@ import apiConfig from "@config/api.config"; import { google, drive_v3 } from "googleapis"; import { decrypt } from "@utils/encryptionHelper"; +import { Redis } from "@upstash/redis"; const decryptedSecret: string = decrypt( apiConfig.client_secret, @@ -11,6 +12,11 @@ const decryptedRefreshToken: string = decrypt( process.env.ENCRYPTION_KEY as string, ); +export const redis = new Redis({ + url: process.env.REDIS_HOST as string, + token: process.env.REDIS_TOKEN as string, +}); + const oauth2Client = new google.auth.OAuth2( apiConfig.client_id, decryptedSecret, diff --git a/yarn.lock b/yarn.lock index 27da3b9..d28a886 100644 --- a/yarn.lock +++ b/yarn.lock @@ -595,6 +595,13 @@ "@typescript-eslint/types" "5.58.0" eslint-visitor-keys "^3.3.0" +"@upstash/redis@^1.20.4": + version "1.20.4" + resolved "https://registry.yarnpkg.com/@upstash/redis/-/redis-1.20.4.tgz#6c02665bc0e9b7c56f37342003fc58139c994d22" + integrity sha512-U7j7py+yPvafB5KS7o+F19j2CWzZCwmQ4Tvs+n2lpCWuw/8CeFtWrFWPtQa5dgrVu6tu+Ki9DmhDiAbgMS5fGA== + dependencies: + isomorphic-fetch "^3.0.0" + "@use-gesture/core@10.2.26": version "10.2.26" resolved "https://registry.yarnpkg.com/@use-gesture/core/-/core-10.2.26.tgz#c2fc4aa7d36cee7319a98a898b0698c66b01663e" @@ -2681,6 +2688,14 @@ isexe@^2.0.0: resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== +isomorphic-fetch@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz#0267b005049046d2421207215d45d6a262b8b8b4" + integrity sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA== + dependencies: + node-fetch "^2.6.1" + whatwg-fetch "^3.4.1" + isstream@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" @@ -3582,7 +3597,7 @@ nextjs-progressbar@^0.0.16: nprogress "^0.2.0" prop-types "^15.8.1" -node-fetch@^2.6.7: +node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.9" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6" integrity sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg== @@ -5115,6 +5130,11 @@ webidl-conversions@^3.0.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== +whatwg-fetch@^3.4.1: + version "3.6.2" + resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.6.2.tgz#dced24f37f2624ed0281725d51d0e2e3fe677f8c" + integrity sha512-bJlen0FcuU/0EMLrdbJ7zOnW6ITZLrZMIarMUVmdKtsGvZna8vxKYaexICWPfZ8qwf9fzNq+UEIZrnSaApt6RA== + whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"