diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index cd36700..525549b 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -33,20 +33,18 @@ jobs: - name: Install npm dependencies run: npm ci - - name: Download recommended whitelist - run: bash ./get_recommended_whitelist.sh + - name: Download allowlists + run: npm run download:allowlist env: ALLOWLIST_URLS: ${{ vars.ALLOWLIST_URLS }} - - name: Download recommended filters - run: bash ./get_recommended_filters.sh + - name: Download blocklists + run: npm run download:blocklist env: BLOCKLIST_URLS: ${{ vars.BLOCKLIST_URLS }} - name: Delete old rules and lists - run: | - node cf_gateway_rule_delete.js - node cf_list_delete.js + run: npm run cloudflare-delete env: CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} @@ -55,9 +53,7 @@ jobs: FAST_MODE: ${{ vars.FAST_MODE }} - name: Create new rules and lists - run: | - node cf_list_create.js - node cf_gateway_rule_create.js + run: npm run cloudflare-create env: CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} diff --git a/cf_list_create.js b/cf_list_create.js index ca2c0df..61e906c 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -125,9 +125,7 @@ console.log(`Number of unnecessary domains: ${unnecessaryDomainCount}`); console.log(`Number of blocked domains: ${domains.length}`); console.log(`Number of allowed domains: ${allowedDomainCount}`); console.log( - `Number of lists which will be created: ${Math.ceil( - domains.length / LIST_ITEM_SIZE - )}` + `Number of lists to be created: ${Math.ceil(domains.length / LIST_ITEM_SIZE)}` ); console.log("\n\n"); diff --git a/download_lists.js b/download_lists.js index 3f04054..9087e0f 100644 --- a/download_lists.js +++ b/download_lists.js @@ -1,3 +1,5 @@ +import { existsSync } from "fs"; +import { unlink } from "fs/promises"; import { resolve } from "path"; import { @@ -15,16 +17,28 @@ const blocklistUrls = USER_DEFINED_BLOCKLIST_URLS || RECOMMENDED_BLOCKLIST_URLS; const listType = process.argv[2]; const downloadLists = async (filename, urls) => { - await downloadFiles(resolve(`./${filename}`), urls); - console.log( - `Done. The ${filename} file contains merged data from the following list(s):` - ); - console.log( - urls.reduce( - (previous, current, index) => previous + `${index + 1}. ${current}\n`, - "" - ) - ); + const filePath = resolve(`./${filename}`); + + if (existsSync(filePath)) { + await unlink(filePath); + } + + try { + await downloadFiles(filePath, urls); + + console.log( + `Done. The ${filename} file contains merged data from the following list(s):` + ); + console.log( + urls.reduce( + (previous, current, index) => previous + `${index + 1}. ${current}\n`, + "" + ) + ); + } catch (err) { + console.error(`An error occurred while processing ${filename}:\n`, err); + console.error("URLs:\n", urls); + } }; switch (listType) { diff --git a/lib/helpers.js b/lib/helpers.js index 702b4f3..84b4e8f 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -20,6 +20,12 @@ if (!globalThis.fetch) { * @returns {Promise} */ const request = async (url, options) => { + if (!API_TOKEN || !ACCOUNT_ID || !ACCOUNT_EMAIL) { + throw new Error( + "One or more required secrets have not been added: CLOUDFLARE_API_KEY, CLOUDFLARE_ACCOUNT_ID, and CLOUDFLARE_ACCOUNT_EMAIL" + ); + } + const response = await fetch(url, { headers: { Authorization: `Bearer ${API_TOKEN}`, diff --git a/package.json b/package.json index 4b98eef..e2ffefe 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,17 @@ { + "scripts": { + "start": "npm run download && npm run cloudflare-delete && npm run cloudflare-create", + "dry": "npm run download && DRY_RUN=1 npm run cloudflare-create:list", + "download": "node download_lists.js", + "download:allowlist": "node download_lists.js allowlist", + "download:blocklist": "node download_lists.js blocklist", + "cloudflare-create": "npm run cloudflare-create:list && npm run cloudflare-create:rule", + "cloudflare-delete": "npm run cloudflare-delete:rule && npm run cloudflare-delete:list", + "cloudflare-create:rule": "node cf_gateway_rule_create.js", + "cloudflare-create:list": "node cf_list_create.js", + "cloudflare-delete:rule": "node cf_gateway_rule_delete.js", + "cloudflare-delete:list": "node cf_list_delete.js" + }, "type": "module", "dependencies": { "dotenv": "^16.0.3",