From da02a966fb7e66dbb70e555c1c11b2de89bbe508 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Fri, 29 Sep 2023 06:18:30 +0700 Subject: [PATCH 1/6] add npm scripts --- auto_update_github_action.yml | 16 ++++++---------- package.json | 13 +++++++++++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index cd36700..525549b 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -33,20 +33,18 @@ jobs: - name: Install npm dependencies run: npm ci - - name: Download recommended whitelist - run: bash ./get_recommended_whitelist.sh + - name: Download allowlists + run: npm run download:allowlist env: ALLOWLIST_URLS: ${{ vars.ALLOWLIST_URLS }} - - name: Download recommended filters - run: bash ./get_recommended_filters.sh + - name: Download blocklists + run: npm run download:blocklist env: BLOCKLIST_URLS: ${{ vars.BLOCKLIST_URLS }} - name: Delete old rules and lists - run: | - node cf_gateway_rule_delete.js - node cf_list_delete.js + run: npm run cloudflare-delete env: CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} @@ -55,9 +53,7 @@ jobs: FAST_MODE: ${{ vars.FAST_MODE }} - name: Create new rules and lists - run: | - node cf_list_create.js - node cf_gateway_rule_create.js + run: npm run cloudflare-create env: CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} diff --git a/package.json b/package.json index 4b98eef..03e3d8c 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,17 @@ { + "scripts": { + "start": "npm run download && npm run cloudflare-delete && npm run cloudflare-create", + "dry": "npm run download && DRY_RUN=1 npm run cloudflare-create:list", + "download": "node download_lists.js", + "download:allowlist": "node download_lists.js allowlist", + "download:blocklist": "node download_lists.js blocklist", + "cloudflare-create": "npm run create:list && npm run create:rule", + "cloudflare-delete": "npm run delete:rule && npm run delete:list", + "cloudflare-create:rule": "node cf_gateway_rule_create.js", + "cloudflare-create:list": "node cf_list_create.js", + "cloudflare-delete:rule": "node cf_gateway_rule_delete.js", + "cloudflare-delete:list": "node cf_list_delete.js" + }, "type": "module", "dependencies": { "dotenv": "^16.0.3", From 190a97bc0f5a2fa3fdaa11261cc5ba122ee5f821 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Fri, 29 Sep 2023 06:22:14 +0700 Subject: [PATCH 2/6] fix command --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 03e3d8c..e2ffefe 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,8 @@ "download": "node download_lists.js", "download:allowlist": "node download_lists.js allowlist", "download:blocklist": "node download_lists.js blocklist", - "cloudflare-create": "npm run create:list && npm run create:rule", - "cloudflare-delete": "npm run delete:rule && npm run delete:list", + "cloudflare-create": "npm run cloudflare-create:list && npm run cloudflare-create:rule", + "cloudflare-delete": "npm run cloudflare-delete:rule && npm run cloudflare-delete:list", "cloudflare-create:rule": "node cf_gateway_rule_create.js", "cloudflare-create:list": "node cf_list_create.js", "cloudflare-delete:rule": "node cf_gateway_rule_delete.js", From e03695beb903e2c213a1c8f185c69f07ef5f8797 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Fri, 29 Sep 2023 06:32:17 +0700 Subject: [PATCH 3/6] wording --- cf_list_create.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cf_list_create.js b/cf_list_create.js index ca2c0df..61e906c 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -125,9 +125,7 @@ console.log(`Number of unnecessary domains: ${unnecessaryDomainCount}`); console.log(`Number of blocked domains: ${domains.length}`); console.log(`Number of allowed domains: ${allowedDomainCount}`); console.log( - `Number of lists which will be created: ${Math.ceil( - domains.length / LIST_ITEM_SIZE - )}` + `Number of lists to be created: ${Math.ceil(domains.length / LIST_ITEM_SIZE)}` ); console.log("\n\n"); From c2b0510cfc009ca3b5dfb7bd683bf0b21963833e Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Fri, 29 Sep 2023 06:44:19 +0700 Subject: [PATCH 4/6] remove existing lists before downloading --- download_lists.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/download_lists.js b/download_lists.js index 3f04054..fb0d952 100644 --- a/download_lists.js +++ b/download_lists.js @@ -1,3 +1,5 @@ +import { existsSync } from "fs"; +import { unlink } from "fs/promises"; import { resolve } from "path"; import { @@ -15,7 +17,13 @@ const blocklistUrls = USER_DEFINED_BLOCKLIST_URLS || RECOMMENDED_BLOCKLIST_URLS; const listType = process.argv[2]; const downloadLists = async (filename, urls) => { - await downloadFiles(resolve(`./${filename}`), urls); + const filePath = resolve(`./${filename}`); + + if (existsSync(filePath)) { + await unlink(filePath); + } + + await downloadFiles(filePath, urls); console.log( `Done. The ${filename} file contains merged data from the following list(s):` ); From e4f44e3e2b7b5de97b01c67e352aa5af6c56549c Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Fri, 29 Sep 2023 05:43:38 +0700 Subject: [PATCH 5/6] add error logging for list download --- download_lists.js | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/download_lists.js b/download_lists.js index fb0d952..9087e0f 100644 --- a/download_lists.js +++ b/download_lists.js @@ -23,16 +23,22 @@ const downloadLists = async (filename, urls) => { await unlink(filePath); } - await downloadFiles(filePath, urls); - console.log( - `Done. The ${filename} file contains merged data from the following list(s):` - ); - console.log( - urls.reduce( - (previous, current, index) => previous + `${index + 1}. ${current}\n`, - "" - ) - ); + try { + await downloadFiles(filePath, urls); + + console.log( + `Done. The ${filename} file contains merged data from the following list(s):` + ); + console.log( + urls.reduce( + (previous, current, index) => previous + `${index + 1}. ${current}\n`, + "" + ) + ); + } catch (err) { + console.error(`An error occurred while processing ${filename}:\n`, err); + console.error("URLs:\n", urls); + } }; switch (listType) { From af24c152a8bc9fb7fb6b9b53c63d7fb3ed77c73f Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Fri, 29 Sep 2023 05:44:00 +0700 Subject: [PATCH 6/6] throw error if required secrets have not been added --- lib/helpers.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/helpers.js b/lib/helpers.js index 702b4f3..84b4e8f 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -20,6 +20,12 @@ if (!globalThis.fetch) { * @returns {Promise} */ const request = async (url, options) => { + if (!API_TOKEN || !ACCOUNT_ID || !ACCOUNT_EMAIL) { + throw new Error( + "One or more required secrets have not been added: CLOUDFLARE_API_KEY, CLOUDFLARE_ACCOUNT_ID, and CLOUDFLARE_ACCOUNT_EMAIL" + ); + } + const response = await fetch(url, { headers: { Authorization: `Bearer ${API_TOKEN}`,