From 6cf50bddcf73314c7a976b2f1d416a050366336d Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Fri, 8 Sep 2023 05:41:30 +0700 Subject: [PATCH 01/27] updated workflow so it works correctly on the forked repo --- auto_update_github_action.yml | 93 ++++++++++++++++------------------- 1 file changed, 42 insertions(+), 51 deletions(-) diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index c86cc77..7685df1 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -2,65 +2,56 @@ name: Update Filter Lists on: schedule: - - cron: '0 3 * * 1' + - cron: "0 3 * * 1" push: branches: - - '**' + - main workflow_dispatch: jobs: cgps: runs-on: ubuntu-latest steps: - - name: Install Node.js - uses: actions/setup-node@v3 - with: - node-version: 16 - - - name: Clone repository and switch to v1 branch - run: | - git clone https://github.com/mrrfv/cloudflare-gateway-pihole-scripts.git - cd cloudflare-gateway-pihole-scripts - git checkout v1 + - name: Checkout + uses: actions/checkout@v4 - - name: Install npm dependencies - run: npm ci - working-directory: cloudflare-gateway-pihole-scripts - - - name: Download recommended whitelist - run: bash ./get_recommended_whitelist.sh - working-directory: cloudflare-gateway-pihole-scripts - - - name: Download recommended filters - run: bash ./get_recommended_filters.sh - working-directory: cloudflare-gateway-pihole-scripts - - - name: Delete old rules and lists - run: | - node cf_gateway_rule_delete.js - node cf_list_delete.js - working-directory: cloudflare-gateway-pihole-scripts - env: - CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} - CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: lts/* - - name: Create new rules and lists - run: | - node cf_list_create.js - node cf_gateway_rule_create.js - working-directory: cloudflare-gateway-pihole-scripts - env: - CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} - CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} + - name: Install npm dependencies + run: npm ci - - name: Send ping request - if: env.PING_URL != '' - working-directory: cloudflare-gateway-pihole-scripts - env: - PING_URL: ${{ secrets.PING_URL }} - run: | - curl "${{ env.PING_URL }}" + - name: Download recommended whitelist + run: bash ./get_recommended_whitelist.sh + + - name: Download recommended filters + run: bash ./get_recommended_filters.sh + + - name: Delete old rules and lists + run: | + node cf_gateway_rule_delete.js + node cf_list_delete.js + env: + CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} + CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} + + - name: Create new rules and lists + run: | + node cf_list_create.js + node cf_gateway_rule_create.js + env: + CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} + CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} + + - name: Send ping request + if: env.PING_URL != '' + env: + PING_URL: ${{ secrets.PING_URL }} + run: | + curl "${{ env.PING_URL }}" From 86a0f87f6d7d2750933cee662dd6e7f215c9b25f Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Fri, 8 Sep 2023 18:06:43 +0700 Subject: [PATCH 02/27] moved helpers to lib directory --- get_recommended_filters.sh | 2 +- get_recommended_whitelist.sh | 2 +- helpers.sh => lib/helpers.sh | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename helpers.sh => lib/helpers.sh (100%) diff --git a/get_recommended_filters.sh b/get_recommended_filters.sh index 032fba6..5e1c111 100644 --- a/get_recommended_filters.sh +++ b/get_recommended_filters.sh @@ -1,6 +1,6 @@ #!/bin/bash -source $(dirname "$0")/helpers.sh +source $(dirname "$0")/lib/helpers.sh # declare an array of urls urls=( diff --git a/get_recommended_whitelist.sh b/get_recommended_whitelist.sh index a2b0a07..4035957 100644 --- a/get_recommended_whitelist.sh +++ b/get_recommended_whitelist.sh @@ -3,7 +3,7 @@ # Use the provided lists or add your own. # There is no limit on the amount of whitelisted domains you can have. -source $(dirname "$0")/helpers.sh +source $(dirname "$0")/lib/helpers.sh # declare an array of urls urls=( diff --git a/helpers.sh b/lib/helpers.sh similarity index 100% rename from helpers.sh rename to lib/helpers.sh From 28d359e02f4c4449523fa3465317331dbec970a8 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 02:37:00 +0700 Subject: [PATCH 03/27] use original repo by default --- auto_update_github_action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index 7685df1..b2c0eda 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -14,11 +14,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + repository: "mrrfv/cloudflare-gateway-pihole-scripts" + ref: "v1" - name: Install Node.js uses: actions/setup-node@v3 with: - node-version: lts/* + node-version: "lts/*" - name: Install npm dependencies run: npm ci From 43bd87f2dd96e40896b0a610d954fc19eb4a5a03 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 03:41:15 +0700 Subject: [PATCH 04/27] use native fetch and refactored list creation --- .env.example | 8 ++--- cf_list_create.js | 81 +++--------------------------------------- lib/constants.js | 25 +++++++++++++ lib/helpers.js | 89 +++++++++++++++++++++++++++++++++++++++++++++++ lib/utils.js | 4 +++ 5 files changed, 126 insertions(+), 81 deletions(-) create mode 100644 lib/constants.js create mode 100644 lib/helpers.js create mode 100644 lib/utils.js diff --git a/.env.example b/.env.example index b54a472..ee7845c 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -CLOUDFLARE_API_KEY="" -CLOUDFLARE_ACCOUNT_ID="" -CLOUDFLARE_ACCOUNT_EMAIL="" -CLOUDFLARE_LIST_ITEM_LIMIT="300000" \ No newline at end of file +CLOUDFLARE_API_KEY= +CLOUDFLARE_ACCOUNT_ID= +CLOUDFLARE_ACCOUNT_EMAIL= +CLOUDFLARE_LIST_ITEM_LIMIT=300000 diff --git a/cf_list_create.js b/cf_list_create.js index 8d45c37..87d86db 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -1,11 +1,6 @@ -import 'dotenv/config'; -import fetch from 'node-fetch'; import fs from 'fs'; - -const API_TOKEN = process.env.CLOUDFLARE_API_KEY; -const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; -const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; -const LIST_ITEM_LIMIT = Number.isSafeInteger(Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT)) ? Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT) : 300000; +import { DRY_RUN, LIST_ITEM_LIMIT } from './lib/constants.js'; +import { createZeroTrustLists } from './lib/helpers.js'; if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`); @@ -113,79 +108,11 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { // If we are dry-running, stop here because we don't want to create lists // TODO: we should probably continue, just without making any real requests to Cloudflare - if (process.env.DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.'); + if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.'); - // Separate domains into chunks of 1000 (Cloudflare list cap) - const chunks = chunkArray(domains, 1000); - - // Create Cloudflare Zero Trust lists - for (const [index, chunk] of chunks.entries()) { - const listName = `CGPS List - Chunk ${index}`; - - let properList = []; - - chunk.forEach(domain => { - properList.push({ "value": domain }) - }); - - try { - await createZeroTrustList(listName, properList, (index+1), listsToCreate); - await sleep(350); // Sleep for 350ms between list additions - } catch (error) { - console.error(`Error creating list `, process.env.CI ? "(redacted on CI)" : `"${listName}": ${error.response.data}`); - } - } + await createZeroTrustLists(domains) }); function trimArray(arr, size) { return arr.slice(0, size); } - -// Function to check if a domain is valid -function isValidDomain(domain) { - const regex = /^((?!-)[A-Za-z0-9-]{1,63}(? setTimeout(resolve, ms)); -} diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..9b528f3 --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,25 @@ +import { isDev } from "./utils.js"; + +if (isDev()) { + const dotenv = await import("dotenv"); + + dotenv.config(); +} + +export const API_TOKEN = process.env.CLOUDFLARE_API_KEY; + +export const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; + +export const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; + +export const LIST_ITEM_LIMIT = isNaN(process.env.CLOUDFLARE_LIST_ITEM_LIMIT) + ? 300000 + : parseInt(process.env.CLOUDFLARE_LIST_ITEM_LIMIT, 10); + +export const LIST_ITEM_SIZE = 1000; + +export const API_HOST = "https://api.cloudflare.com/client/v4"; + +export const DRY_RUN = !!parseInt(process.env.DRY_RUN, 10); + +export const FAST_MODE = !!parseInt(process.env.FAST_MODE, 10); diff --git a/lib/helpers.js b/lib/helpers.js new file mode 100644 index 0000000..bd8b2b6 --- /dev/null +++ b/lib/helpers.js @@ -0,0 +1,89 @@ +import { + ACCOUNT_EMAIL, + ACCOUNT_ID, + API_HOST, + API_TOKEN, + FAST_MODE, + LIST_ITEM_SIZE, +} from "./constants.js"; +import { sleep } from "./utils.js"; + +const request = async (url, options) => { + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${API_TOKEN}`, + "Content-Type": "application/json", + "X-Auth-Email": ACCOUNT_EMAIL, + "X-Auth-Key": API_TOKEN, + }, + ...options, + }); + + return response.json(); +}; + +const createZeroTrustList = (name, items) => { + // https://developers.cloudflare.com/api/operations/zero-trust-lists-create-zero-trust-list + + return request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway/lists`, { + method: "POST", + body: JSON.stringify({ + name, + type: "DOMAIN", + items, + }), + }); +}; + +const createZeroTrustListsOneByOne = async (items) => { + let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); + + for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) { + const chunk = items + .slice(i, i + LIST_ITEM_SIZE) + .map((item) => ({ value: item })); + const listName = `CGPS List - Chunk ${listNumber}`; + + try { + const { + result: { id }, + } = await createZeroTrustList(listName, chunk); + + totalListNumber--; + listNumber++; + console.log( + `Created ${listName} with ID ${id} - ${totalListNumber} left` + ); + await sleep(); + } catch (err) { + console.error(`Could not create ${listName} - ${err.toString()}`); + } + } +}; + +const createZeroTrustListsAtOnce = async (items) => { + let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); + const requests = []; + + for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) { + const chunk = items + .slice(i, i + LIST_ITEM_SIZE) + .map((item) => ({ value: item })); + const listName = `CGPS List - Chunk ${listNumber}`; + + requests.push(createZeroTrustList(listName, chunk)); + listNumber++; + } + + await Promise.all(requests); + console.log(`Created ${totalListNumber} lists`); +}; + +export const createZeroTrustLists = (items) => { + if (FAST_MODE) { + return createZeroTrustListsAtOnce(items); + } + + return createZeroTrustListsOneByOne(items); +}; diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..f717822 --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,4 @@ +export const isDev = () => process.env.NODE_ENV !== "production"; + +export const sleep = (ms = 350) => + new Promise((resolve) => setTimeout(resolve, ms)); From bc3b16f5fa11ec9397b6fa5d41fd2ddac1258e53 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 04:35:24 +0700 Subject: [PATCH 05/27] removed list ID output --- lib/helpers.js | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/helpers.js b/lib/helpers.js index bd8b2b6..15e23d0 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -46,16 +46,11 @@ const createZeroTrustListsOneByOne = async (items) => { const listName = `CGPS List - Chunk ${listNumber}`; try { - const { - result: { id }, - } = await createZeroTrustList(listName, chunk); - + await createZeroTrustList(listName, chunk); + await sleep(); totalListNumber--; listNumber++; - console.log( - `Created ${listName} with ID ${id} - ${totalListNumber} left` - ); - await sleep(); + console.log(`Created ${listName} list - ${totalListNumber} left`); } catch (err) { console.error(`Could not create ${listName} - ${err.toString()}`); } From 53803edc6eef9051d0855b75594c264ea0806aef Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 20:46:56 +0700 Subject: [PATCH 06/27] reorganized API code --- cf_list_create.js | 2 +- lib/api.js | 81 +++++++++++++++++++++++++++++++++++++++++++++ lib/constants.js | 8 ++--- lib/helpers.js | 84 +++++++++-------------------------------------- lib/utils.js | 6 ++-- 5 files changed, 103 insertions(+), 78 deletions(-) create mode 100644 lib/api.js diff --git a/cf_list_create.js b/cf_list_create.js index 87d86db..bdef4eb 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -1,6 +1,6 @@ import fs from 'fs'; import { DRY_RUN, LIST_ITEM_LIMIT } from './lib/constants.js'; -import { createZeroTrustLists } from './lib/helpers.js'; +import { createZeroTrustLists } from './lib/api.js'; if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`); diff --git a/lib/api.js b/lib/api.js new file mode 100644 index 0000000..bd5cbfd --- /dev/null +++ b/lib/api.js @@ -0,0 +1,81 @@ +import { FAST_MODE, LIST_ITEM_SIZE } from "./constants.js"; +import { requestGateway } from "./helpers.js"; + +/** + * Creates a Zero Trust list. + * + * API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-create-zero-trust-list + * @param {string} name The name of the list. + * @param {Object[]} items The domains in the list. + * @param {string} items[].value The domain of an entry. + * @returns {Promise} + */ +const createZeroTrustList = (name, items) => + requestGateway(`/lists`, { + method: "POST", + body: JSON.stringify({ + name, + type: "DOMAIN", + items, + }), + }); + +/** + * Creates Zero Trust lists sequentially. + * @param {string[]} items The domains. + */ +const createZeroTrustListsOneByOne = async (items) => { + let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); + + for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) { + const chunk = items + .slice(i, i + LIST_ITEM_SIZE) + .map((item) => ({ value: item })); + const listName = `CGPS List - Chunk ${listNumber}`; + + try { + await createZeroTrustList(listName, chunk); + await sleep(); + totalListNumber--; + listNumber++; + console.log(`Created ${listName} list - ${totalListNumber} left`); + } catch (err) { + console.error(`Could not create ${listName} - ${err.toString()}`); + } + } +}; + +/** + * Creates all Zero Trust lists at once. + * @param {string[]} items The domains. + */ +const createZeroTrustListsAtOnce = async (items) => { + const totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); + const requests = []; + + for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) { + const chunk = items + .slice(i, i + LIST_ITEM_SIZE) + .map((item) => ({ value: item })); + const listName = `CGPS List - Chunk ${listNumber}`; + + requests.push(createZeroTrustList(listName, chunk)); + listNumber++; + } + + await Promise.all(requests); + console.log(`Created ${totalListNumber} lists`); +}; + +/** + * Create Zero Trust lists. + * @param {string[]} items The domains. + * @returns {Promise} + */ +export const createZeroTrustLists = (items) => { + if (FAST_MODE) { + return createZeroTrustListsAtOnce(items); + } + + return createZeroTrustListsOneByOne(items); +}; diff --git a/lib/constants.js b/lib/constants.js index 9b528f3..5a77df5 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -1,10 +1,6 @@ -import { isDev } from "./utils.js"; +import dotenv from "dotenv"; -if (isDev()) { - const dotenv = await import("dotenv"); - - dotenv.config(); -} +dotenv.config(); export const API_TOKEN = process.env.CLOUDFLARE_API_KEY; diff --git a/lib/helpers.js b/lib/helpers.js index 15e23d0..69ea096 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -1,16 +1,13 @@ -import { - ACCOUNT_EMAIL, - ACCOUNT_ID, - API_HOST, - API_TOKEN, - FAST_MODE, - LIST_ITEM_SIZE, -} from "./constants.js"; -import { sleep } from "./utils.js"; +import { ACCOUNT_EMAIL, ACCOUNT_ID, API_HOST, API_TOKEN } from "./constants.js"; +/** + * Fires request to the specified URL. + * @param {string} url The URL to which the request will be fired. + * @param {RequestInit} options The options to be passed to `fetch`. + * @returns {Promise} + */ const request = async (url, options) => { const response = await fetch(url, { - method: "GET", headers: { Authorization: `Bearer ${API_TOKEN}`, "Content-Type": "application/json", @@ -23,62 +20,11 @@ const request = async (url, options) => { return response.json(); }; -const createZeroTrustList = (name, items) => { - // https://developers.cloudflare.com/api/operations/zero-trust-lists-create-zero-trust-list - - return request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway/lists`, { - method: "POST", - body: JSON.stringify({ - name, - type: "DOMAIN", - items, - }), - }); -}; - -const createZeroTrustListsOneByOne = async (items) => { - let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); - - for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) { - const chunk = items - .slice(i, i + LIST_ITEM_SIZE) - .map((item) => ({ value: item })); - const listName = `CGPS List - Chunk ${listNumber}`; - - try { - await createZeroTrustList(listName, chunk); - await sleep(); - totalListNumber--; - listNumber++; - console.log(`Created ${listName} list - ${totalListNumber} left`); - } catch (err) { - console.error(`Could not create ${listName} - ${err.toString()}`); - } - } -}; - -const createZeroTrustListsAtOnce = async (items) => { - let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); - const requests = []; - - for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) { - const chunk = items - .slice(i, i + LIST_ITEM_SIZE) - .map((item) => ({ value: item })); - const listName = `CGPS List - Chunk ${listNumber}`; - - requests.push(createZeroTrustList(listName, chunk)); - listNumber++; - } - - await Promise.all(requests); - console.log(`Created ${totalListNumber} lists`); -}; - -export const createZeroTrustLists = (items) => { - if (FAST_MODE) { - return createZeroTrustListsAtOnce(items); - } - - return createZeroTrustListsOneByOne(items); -}; +/** + * Fires request to the Zero Trust gateway. + * @param {string} path The path which will be appended to the request URL. + * @param {RequestInit} options The options to be passed to `fetch`. + * @returns {Promise} + */ +export const requestGateway = (path, options) => + request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway${path}`, options); diff --git a/lib/utils.js b/lib/utils.js index f717822..9c1037f 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,4 +1,6 @@ -export const isDev = () => process.env.NODE_ENV !== "production"; - +/** + * Sleeps for a specified amount of time. + * @param {number} [ms=350] The amount of time in ms. + */ export const sleep = (ms = 350) => new Promise((resolve) => setTimeout(resolve, ms)); From 61d6c0370446c1252b2133b13cda46895b048aa4 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 21:24:02 +0700 Subject: [PATCH 07/27] handle API error --- lib/api.js | 8 ++++++-- lib/helpers.js | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/api.js b/lib/api.js index bd5cbfd..628ec9e 100644 --- a/lib/api.js +++ b/lib/api.js @@ -63,8 +63,12 @@ const createZeroTrustListsAtOnce = async (items) => { listNumber++; } - await Promise.all(requests); - console.log(`Created ${totalListNumber} lists`); + try { + await Promise.all(requests); + console.log(`Created ${totalListNumber} lists`); + } catch (err) { + console.error(`Error occurred while creating lists - ${err.toString()}`); + } }; /** diff --git a/lib/helpers.js b/lib/helpers.js index 69ea096..8fd511a 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -17,6 +17,10 @@ const request = async (url, options) => { ...options, }); + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + return response.json(); }; From ed5f24b6ec78cf6ff0dab1a1dc90218bf3d5fc8a Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 21:43:57 +0700 Subject: [PATCH 08/27] reorganized code --- cf_list_create.js | 18 ++++++++++-------- lib/api.js | 19 +++---------------- lib/utils.js | 8 ++++++++ 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/cf_list_create.js b/cf_list_create.js index bdef4eb..61260f7 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -1,6 +1,7 @@ import fs from 'fs'; -import { DRY_RUN, LIST_ITEM_LIMIT } from './lib/constants.js'; -import { createZeroTrustLists } from './lib/api.js'; +import { DRY_RUN, FAST_MODE, LIST_ITEM_LIMIT } from './lib/constants.js'; +import { createZeroTrustListsAtOnce, createZeroTrustListsOneByOne } from './lib/api.js'; +import { truncateArray } from './lib/utils.js'; if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`); @@ -99,7 +100,7 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { // Trim array to 300,000 domains if it's longer than that if (domains.length > LIST_ITEM_LIMIT) { console.warn(`${domains.length} domains found in input.csv - input has to be trimmed to ${LIST_ITEM_LIMIT} domains`); - domains = trimArray(domains, LIST_ITEM_LIMIT); + domains = truncateArray(domains, LIST_ITEM_LIMIT); } const listsToCreate = Math.ceil(domains.length / 1000); @@ -110,9 +111,10 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { // TODO: we should probably continue, just without making any real requests to Cloudflare if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.'); - await createZeroTrustLists(domains) -}); + if (FAST_MODE) { + createZeroTrustListsAtOnce(items); + return; + } -function trimArray(arr, size) { - return arr.slice(0, size); -} + createZeroTrustListsOneByOne(items); +}); diff --git a/lib/api.js b/lib/api.js index 628ec9e..ccd36c7 100644 --- a/lib/api.js +++ b/lib/api.js @@ -1,4 +1,4 @@ -import { FAST_MODE, LIST_ITEM_SIZE } from "./constants.js"; +import { LIST_ITEM_SIZE } from "./constants.js"; import { requestGateway } from "./helpers.js"; /** @@ -24,7 +24,7 @@ const createZeroTrustList = (name, items) => * Creates Zero Trust lists sequentially. * @param {string[]} items The domains. */ -const createZeroTrustListsOneByOne = async (items) => { +export const createZeroTrustListsOneByOne = async (items) => { let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) { @@ -49,7 +49,7 @@ const createZeroTrustListsOneByOne = async (items) => { * Creates all Zero Trust lists at once. * @param {string[]} items The domains. */ -const createZeroTrustListsAtOnce = async (items) => { +export const createZeroTrustListsAtOnce = async (items) => { const totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); const requests = []; @@ -70,16 +70,3 @@ const createZeroTrustListsAtOnce = async (items) => { console.error(`Error occurred while creating lists - ${err.toString()}`); } }; - -/** - * Create Zero Trust lists. - * @param {string[]} items The domains. - * @returns {Promise} - */ -export const createZeroTrustLists = (items) => { - if (FAST_MODE) { - return createZeroTrustListsAtOnce(items); - } - - return createZeroTrustListsOneByOne(items); -}; diff --git a/lib/utils.js b/lib/utils.js index 9c1037f..1d13e0c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -4,3 +4,11 @@ */ export const sleep = (ms = 350) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Truncates an array to the specified size. + * @param {any[]} arr The array to be truncated. + * @param {number} size The size to which the array will be truncated. + * @returns {any[]} + */ +export const truncateArray = (arr, size) => arr.slice(0, size); From b77da7b278f7999904d6baf5f0fbe47d84ea3d17 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 22:26:57 +0700 Subject: [PATCH 09/27] refactored list deletion --- cf_list_delete.js | 58 ++++++++--------------------------------------- lib/api.js | 56 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 49 deletions(-) diff --git a/cf_list_delete.js b/cf_list_delete.js index cf15f4c..ced83e9 100644 --- a/cf_list_delete.js +++ b/cf_list_delete.js @@ -1,58 +1,18 @@ -import 'dotenv/config'; -import fetch from 'node-fetch'; - -const API_TOKEN = process.env.CLOUDFLARE_API_KEY; -const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; -const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; - -// Function to read Cloudflare Zero Trust lists -async function getZeroTrustLists() { - const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`; - - const response = await fetch(url, { - method: 'GET', - headers: { - 'Authorization': `Bearer ${API_TOKEN}`, - 'Content-Type': 'application/json', - 'X-Auth-Email': ACCOUNT_EMAIL, - 'X-Auth-Key': API_TOKEN, - }, - }); - - const data = await response.json(); - return data.result; -} +import { deleteZeroTrustListsAtOnce, deleteZeroTrustListsOneByOne, getZeroTrustLists } from "./lib/api.js"; +import { FAST_MODE } from "./lib/constants.js"; ;(async() => { - const lists = await getZeroTrustLists(); + const { result: lists } = await getZeroTrustLists(); if (!lists) return console.warn("No file lists found - this is not an issue if it's your first time running this script. Exiting."); const cgps_lists = lists.filter(list => list.name.startsWith('CGPS List')); if (!cgps_lists.length) return console.warn("No lists with matching name found - this is not an issue if you haven't created any filter lists before. Exiting."); if (!process.env.CI) console.log(`Got ${lists.length} lists, ${cgps_lists.length} of which are CGPS lists that will be deleted.`); - let lists_processed = 0; - for (const list of cgps_lists) { - console.log(`Deleting list`, process.env.CI ? "(info redacted, running in CI)" : `${list.name} with ID ${list.id}, ${cgps_lists.length - lists_processed - 1} left`); - - const resp = await fetch(`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists/${list.id}`, { - method: 'DELETE', - headers: { - 'Authorization': `Bearer ${API_TOKEN}`, - 'Content-Type': 'application/json', - 'X-Auth-Email': ACCOUNT_EMAIL, - 'X-Auth-Key': API_TOKEN, - }, - }); - - const data = await resp.json(); - console.log('Success:', data.success); - lists_processed++; - - await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe - } -})(); + if (FAST_MODE) { + await deleteZeroTrustListsAtOnce(cgps_lists); + return; + } -async function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); -} \ No newline at end of file + await deleteZeroTrustListsOneByOne(cgps_lists); +})(); diff --git a/lib/api.js b/lib/api.js index ccd36c7..e2b0dda 100644 --- a/lib/api.js +++ b/lib/api.js @@ -1,5 +1,15 @@ import { LIST_ITEM_SIZE } from "./constants.js"; import { requestGateway } from "./helpers.js"; +import { sleep } from "./utils.js"; + +/** + * Gets Zero Trust lists. + * @returns {Promise} + */ +export const getZeroTrustLists = () => + requestGateway("/lists", { + method: "GET", + }); /** * Creates a Zero Trust list. @@ -70,3 +80,49 @@ export const createZeroTrustListsAtOnce = async (items) => { console.error(`Error occurred while creating lists - ${err.toString()}`); } }; + +/** + * Deletes a Zero Trust list. + * @param {number} id The ID of the list. + * @returns {Promise} + */ +const deleteZeroTrustList = (id) => + requestGateway(`/lists/${id}`, { method: "DELETE" }); + +/** + * Deletes Zero Trust lists sequentially. + * @param {Object[]} lists The lists to be deleted. + * @param {number} lists[].id The ID of a list. + * @param {string} lists[].name The name of a list. + */ +export const deleteZeroTrustListsOneByOne = async (lists) => { + let totalListNumber = lists.length; + + for (const { id, name } of lists) { + try { + await deleteZeroTrustList(id); + await sleep(); + totalListNumber--; + console.log(`Deleted ${name} list - ${totalListNumber} left`); + } catch (err) { + console.error(`Could not delete ${name} - ${err.toString()}`); + } + } +}; + +/** + * Deletes all Zero Trust lists at once. + * @param {Object[]} lists The lists to be deleted. + * @param {number} lists[].id The ID of a list. + * @param {string} lists[].name The name of a list. + */ +export const deleteZeroTrustListsAtOnce = async (lists) => { + const requests = lists.map(({ id }) => deleteZeroTrustList(id)); + + try { + await Promise.all(requests); + console.log(`Deleted ${lists.length} lists`); + } catch (err) { + console.error(`Error occurred while deleting lists - ${err.toString()}`); + } +}; From 603c9a087018e1434c935d9e3449e1068691775b Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 22:43:56 +0700 Subject: [PATCH 10/27] fixed list creation --- cf_list_create.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf_list_create.js b/cf_list_create.js index 61260f7..4b0ca1a 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -112,9 +112,9 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.'); if (FAST_MODE) { - createZeroTrustListsAtOnce(items); + createZeroTrustListsAtOnce(domains); return; } - createZeroTrustListsOneByOne(items); + createZeroTrustListsOneByOne(domains); }); From 8e96c87c27a9d70894905fe03337f6af17037ab1 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 23:12:22 +0700 Subject: [PATCH 11/27] refactored rule creation --- cf_gateway_rule_create.js | 62 ++------------------------------------- lib/api.js | 19 ++++++++++++ lib/helpers.js | 6 +++- 3 files changed, 27 insertions(+), 60 deletions(-) diff --git a/cf_gateway_rule_create.js b/cf_gateway_rule_create.js index 26e239c..d29ba2f 100644 --- a/cf_gateway_rule_create.js +++ b/cf_gateway_rule_create.js @@ -1,34 +1,7 @@ -import 'dotenv/config'; -import fetch from 'node-fetch'; - -const API_TOKEN = process.env.CLOUDFLARE_API_KEY; -const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; -const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; - -// Function to read Cloudflare Zero Trust lists -async function getZeroTrustLists() { - const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`; - - const response = await fetch(url, { - method: 'GET', - headers: { - 'Authorization': `Bearer ${API_TOKEN}`, - 'Content-Type': 'application/json', - 'X-Auth-Email': ACCOUNT_EMAIL, - 'X-Auth-Key': API_TOKEN, - }, - }); - - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - const data = await response.json(); - return data.result; -} +import { createZeroTrustRule, getZeroTrustLists } from './lib/api.js'; ;(async() => { - const lists = await getZeroTrustLists(); + const { result: lists } = await getZeroTrustLists(); const filtered_lists = lists.filter(list => list.name.startsWith('CGPS List')); let wirefilter_expression = ''; @@ -44,34 +17,5 @@ async function getZeroTrustLists() { wirefilter_expression = wirefilter_expression.trim().replace('\n', ''); if (!process.env.CI) console.log(`Firewall expression contains ${wirefilter_expression.length} characters, and checks against ${filtered_lists.length} filter lists.`) - const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`; - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${API_TOKEN}`, - 'Content-Type': 'application/json', - 'X-Auth-Email': ACCOUNT_EMAIL, - 'X-Auth-Key': API_TOKEN, - }, - body: JSON.stringify({ - "name": "CGPS Filter Lists", - "description": "Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.", - "enabled": true, - "action": "block", - "filters": ["dns"], - "traffic": wirefilter_expression, - }), - }); - - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - const data = await response.json(); - console.log('Success:', data.success); + await createZeroTrustRule(wirefilter_expression); })(); - -async function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); -} \ No newline at end of file diff --git a/lib/api.js b/lib/api.js index e2b0dda..91d30aa 100644 --- a/lib/api.js +++ b/lib/api.js @@ -126,3 +126,22 @@ export const deleteZeroTrustListsAtOnce = async (lists) => { console.error(`Error occurred while deleting lists - ${err.toString()}`); } }; + +/** + * Creates a Zero Trust rule. + * @param {string} wirefilterExpression The expression to be used for the rule. + * @returns {Promise} + */ +export const createZeroTrustRule = (wirefilterExpression) => + requestGateway("/rules", { + method: "POST", + body: JSON.stringify({ + name: "CGPS Filter Lists", + description: + "Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.", + enabled: true, + action: "block", + filters: ["dns"], + traffic: wirefilterExpression, + }), + }); diff --git a/lib/helpers.js b/lib/helpers.js index 8fd511a..bc724a1 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -21,7 +21,11 @@ const request = async (url, options) => { throw new Error(`HTTP error! Status: ${response.status}`); } - return response.json(); + const data = response.json(); + + console.log(data.success); + + return data; }; /** From faad7d12b308a84f0ba54fb3a16b929684df7a9c Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 23:17:32 +0700 Subject: [PATCH 12/27] refactored rule deletion --- cf_gateway_rule_delete.js | 52 +++------------------------------------ lib/api.js | 19 +++++++++++++- 2 files changed, 21 insertions(+), 50 deletions(-) diff --git a/cf_gateway_rule_delete.js b/cf_gateway_rule_delete.js index 9613d5a..c301611 100644 --- a/cf_gateway_rule_delete.js +++ b/cf_gateway_rule_delete.js @@ -1,58 +1,12 @@ -import 'dotenv/config'; -import fetch from 'node-fetch'; - -const API_TOKEN = process.env.CLOUDFLARE_API_KEY; -const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; -const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; - -// Function to read Cloudflare Zero Trust rules -async function getZeroTrustRules() { - const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`; - - const response = await fetch(url, { - method: 'GET', - headers: { - 'Authorization': `Bearer ${API_TOKEN}`, - 'Content-Type': 'application/json', - 'X-Auth-Email': ACCOUNT_EMAIL, - 'X-Auth-Key': API_TOKEN, - }, - }); - - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - const data = await response.json(); - return data.result; -} +import { deleteZeroTrustRule, getZeroTrustRules } from './lib/api.js'; ;(async() => { - const rules = await getZeroTrustRules(); + const { result: rules } = await getZeroTrustRules(); const [filtered_rule] = rules.filter(rule => rule.name === "CGPS Filter Lists"); if (!filtered_rule) return console.warn("No rule with matching name found - this is not an issue if you haven't run the create script yet. Exiting."); console.log(`Deleting rule`, process.env.CI ? "(redacted, running in CI)" : `${filtered_rule.name} with ID ${filtered_rule.id}`); - const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules/${filtered_rule.id}`; - - const resp = await fetch(url, { - method: 'DELETE', - headers: { - 'Authorization': `Bearer ${API_TOKEN}`, - 'Content-Type': 'application/json', - 'X-Auth-Email': ACCOUNT_EMAIL, - 'X-Auth-Key': API_TOKEN, - }, - }); - - const data = await resp.json(); - - console.log('Success: ', data.success); - await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe + await deleteZeroTrustRule(filtered_rule.id); })(); - -async function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); -} \ No newline at end of file diff --git a/lib/api.js b/lib/api.js index 91d30aa..f498e65 100644 --- a/lib/api.js +++ b/lib/api.js @@ -127,10 +127,17 @@ export const deleteZeroTrustListsAtOnce = async (lists) => { } }; +/** + * Gets Zero Trust rules. + * @returns {Promise} + */ +export const getZeroTrustRules = () => + requestGateway("/rules", { method: "GET" }); + /** * Creates a Zero Trust rule. * @param {string} wirefilterExpression The expression to be used for the rule. - * @returns {Promise} + * @returns {Promise} */ export const createZeroTrustRule = (wirefilterExpression) => requestGateway("/rules", { @@ -145,3 +152,13 @@ export const createZeroTrustRule = (wirefilterExpression) => traffic: wirefilterExpression, }), }); + +/** + * Deletes a Zero Trust rule. + * @param {number} id The ID of the rule to be deleted. + * @returns {Promise} + */ +export const deleteZeroTrustRule = (id) => + requestGateway(`/rules/${id}`, { + method: "DELETE", + }); From 77b6ac7dfcc73cd591ad512aa9201bf8f3333f6e Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 23:23:26 +0700 Subject: [PATCH 13/27] fixed helper log --- lib/helpers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/helpers.js b/lib/helpers.js index bc724a1..c45f99e 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -21,7 +21,7 @@ const request = async (url, options) => { throw new Error(`HTTP error! Status: ${response.status}`); } - const data = response.json(); + const data = await response.json(); console.log(data.success); From ad8488192129dc404d1d213626943ab63c790b1b Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 23:25:05 +0700 Subject: [PATCH 14/27] fixed helper log --- lib/helpers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/helpers.js b/lib/helpers.js index c45f99e..95fe9af 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -23,7 +23,7 @@ const request = async (url, options) => { const data = await response.json(); - console.log(data.success); + console.log(`Success: ${data.success}`); return data; }; From 2e21859a53c8c3e3c27af3b49701638067e5f070 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 23:38:53 +0700 Subject: [PATCH 15/27] removed node-fetch --- package-lock.json | 87 +---------------------------------------------- package.json | 3 +- 2 files changed, 2 insertions(+), 88 deletions(-) diff --git a/package-lock.json b/package-lock.json index f1bf546..16207d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,16 +5,7 @@ "packages": { "": { "dependencies": { - "dotenv": "^16.0.3", - "node-fetch": "^3.3.2" - } - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "engines": { - "node": ">= 12" + "dotenv": "^16.0.3" } }, "node_modules/dotenv": { @@ -24,82 +15,6 @@ "engines": { "node": ">=12" } - }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", - "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", - "engines": { - "node": ">= 8" - } } } } diff --git a/package.json b/package.json index 123c53c..18334ee 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,6 @@ { "type": "module", "dependencies": { - "dotenv": "^16.0.3", - "node-fetch": "^3.3.2" + "dotenv": "^16.0.3" } } From fe51d78641ee67ccacf92c94536a34087fd85e33 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 23:45:43 +0700 Subject: [PATCH 16/27] moved environment variables to workflow level --- auto_update_github_action.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index b2c0eda..e6484bf 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -8,6 +8,14 @@ on: - main workflow_dispatch: +env: + CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} + CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} + PING_URL: ${{ secrets.PING_URL }} + FAST_MODE: ${{ vars.FAST_MODE }} + jobs: cgps: runs-on: ubuntu-latest @@ -36,25 +44,13 @@ jobs: run: | node cf_gateway_rule_delete.js node cf_list_delete.js - env: - CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} - CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} - name: Create new rules and lists run: | node cf_list_create.js node cf_gateway_rule_create.js - env: - CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} - CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} - name: Send ping request if: env.PING_URL != '' - env: - PING_URL: ${{ secrets.PING_URL }} run: | curl "${{ env.PING_URL }}" From d82c0dfc37285bc8b4f634932f0e7db9b89c0db4 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 23:55:11 +0700 Subject: [PATCH 17/27] updated README for FAST_MODE --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 68e6f98..45569f0 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,12 @@ Please note that the GitHub Action downloads the recommended blocklists and whit - `CLOUDFLARE_LIST_ITEM_LIMIT`: The maximum number of blocked domains allowed for your Cloudflare Zero Trust plan. Use 300000 for the free plan or if you're unsure. - `PING_URL`: /Optional/ The HTTP(S) URL to ping (using curl) after the GitHub Action has successfully updated your filters. Useful for monitoring. -3. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property. -4. Enable GitHub Actions in your repository settings. +3. Create the following GitHub Actions variables in your repository settings if you desire: + +- `FAST_MODE`: Enable the scripts to send the requests simultaneously. Beware that there's a rate limit of 1200 requests per five minutes (https://developers.cloudflare.com/fundamentals/api/reference/limits/) so make sure you know what you are doing. + +4. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property. +5. Enable GitHub Actions in your repository settings. ### DNS setup for Cloudflare Gateway From 055ef5a548a34abbb62fa27701b848fd7ca0f97c Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sat, 9 Sep 2023 23:56:47 +0700 Subject: [PATCH 18/27] added API docs --- lib/api.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/api.js b/lib/api.js index f498e65..445dd96 100644 --- a/lib/api.js +++ b/lib/api.js @@ -4,6 +4,8 @@ import { sleep } from "./utils.js"; /** * Gets Zero Trust lists. + * + * API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-list-zero-trust-lists * @returns {Promise} */ export const getZeroTrustLists = () => @@ -83,6 +85,8 @@ export const createZeroTrustListsAtOnce = async (items) => { /** * Deletes a Zero Trust list. + * + * API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-delete-zero-trust-list * @param {number} id The ID of the list. * @returns {Promise} */ @@ -129,6 +133,8 @@ export const deleteZeroTrustListsAtOnce = async (lists) => { /** * Gets Zero Trust rules. + * + * API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-list-zero-trust-gateway-rules * @returns {Promise} */ export const getZeroTrustRules = () => @@ -136,6 +142,8 @@ export const getZeroTrustRules = () => /** * Creates a Zero Trust rule. + * + * API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-create-zero-trust-gateway-rule * @param {string} wirefilterExpression The expression to be used for the rule. * @returns {Promise} */ @@ -155,6 +163,8 @@ export const createZeroTrustRule = (wirefilterExpression) => /** * Deletes a Zero Trust rule. + * + * API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-delete-zero-trust-gateway-rule * @param {number} id The ID of the rule to be deleted. * @returns {Promise} */ From 185e30fa313803dec0f6a9395d686c56047fee29 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sun, 10 Sep 2023 03:58:31 +0700 Subject: [PATCH 19/27] updated DRY_RUN instruction --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 45569f0..c323eb8 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Alternatively, you can install the Cloudflare WARP client and log in to Zero Tru ### Dry runs -To see if e.g. your filter lists are valid without actually changing anything in your Cloudflare account, you can set the `DRY_RUN` environment variable to "true" or any value other than empty, either in `.env` or the regular way. This will only print info such as the lists that would be created or the amount of duplicate domains to the console. +To see if e.g. your filter lists are valid without actually changing anything in your Cloudflare account, you can set the `DRY_RUN` environment variable to 1, either in `.env` or the regular way. This will only print info such as the lists that would be created or the amount of duplicate domains to the console. **Warning:** This currently only works for `cf_list_create.js`. From cd91b32d4d804baff731deb8e3a2f8ee1587b39c Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Sun, 10 Sep 2023 06:47:46 +0700 Subject: [PATCH 20/27] moved environment variables in workflow to job level --- auto_update_github_action.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index e6484bf..e36c4ce 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -8,14 +8,6 @@ on: - main workflow_dispatch: -env: - CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} - CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} - PING_URL: ${{ secrets.PING_URL }} - FAST_MODE: ${{ vars.FAST_MODE }} - jobs: cgps: runs-on: ubuntu-latest @@ -44,13 +36,27 @@ jobs: run: | node cf_gateway_rule_delete.js node cf_list_delete.js + env: + CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} + CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} + FAST_MODE: ${{ vars.FAST_MODE }} - name: Create new rules and lists run: | node cf_list_create.js node cf_gateway_rule_create.js + env: + CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} + CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} + FAST_MODE: ${{ vars.FAST_MODE }} - name: Send ping request if: env.PING_URL != '' run: | curl "${{ env.PING_URL }}" + env: + PING_URL: ${{ secrets.PING_URL }} From 40a3c96a7d7e6195dfa465424b12267b1f52dd36 Mon Sep 17 00:00:00 2001 From: mrrfv Date: Sun, 10 Sep 2023 12:24:34 +0200 Subject: [PATCH 21/27] Require Node 18 or above in package.json --- package-lock.json | 3 +++ package.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/package-lock.json b/package-lock.json index 16207d7..5a1681b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,6 +6,9 @@ "": { "dependencies": { "dotenv": "^16.0.3" + }, + "engines": { + "node": ">=18" } }, "node_modules/dotenv": { diff --git a/package.json b/package.json index 18334ee..95f30e5 100644 --- a/package.json +++ b/package.json @@ -2,5 +2,8 @@ "type": "module", "dependencies": { "dotenv": "^16.0.3" + }, + "engines": { + "node" : ">=18" } } From 9299d696940f6d8b9fcdd7854a1e30b1ba7dfb11 Mon Sep 17 00:00:00 2001 From: mrrfv Date: Sun, 10 Sep 2023 12:25:10 +0200 Subject: [PATCH 22/27] "Success" -> "HTTP request succeeded" --- lib/helpers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/helpers.js b/lib/helpers.js index 95fe9af..0e4da8a 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -23,7 +23,7 @@ const request = async (url, options) => { const data = await response.json(); - console.log(`Success: ${data.success}`); + console.log(`HTTP request succeeded: ${data.success}`); return data; }; From 025d705efa68fa64839430447e7fff88d283c43f Mon Sep 17 00:00:00 2001 From: mrrfv Date: Sun, 10 Sep 2023 12:27:58 +0200 Subject: [PATCH 23/27] Use await with functions that return a promise --- cf_list_create.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf_list_create.js b/cf_list_create.js index 4b0ca1a..1ea61f6 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -112,9 +112,9 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.'); if (FAST_MODE) { - createZeroTrustListsAtOnce(domains); + await createZeroTrustListsAtOnce(domains); return; } - createZeroTrustListsOneByOne(domains); + await createZeroTrustListsOneByOne(domains); }); From ac9c6925a1c6c616cd44be5627497c4c13104375 Mon Sep 17 00:00:00 2001 From: mrrfv Date: Sun, 10 Sep 2023 12:30:00 +0200 Subject: [PATCH 24/27] Wrap rule name in quotes when deleting --- cf_gateway_rule_delete.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf_gateway_rule_delete.js b/cf_gateway_rule_delete.js index c301611..5c04aea 100644 --- a/cf_gateway_rule_delete.js +++ b/cf_gateway_rule_delete.js @@ -6,7 +6,7 @@ import { deleteZeroTrustRule, getZeroTrustRules } from './lib/api.js'; if (!filtered_rule) return console.warn("No rule with matching name found - this is not an issue if you haven't run the create script yet. Exiting."); - console.log(`Deleting rule`, process.env.CI ? "(redacted, running in CI)" : `${filtered_rule.name} with ID ${filtered_rule.id}`); + console.log(`Deleting rule`, process.env.CI ? "(redacted, running in CI)" : `"${filtered_rule.name}" with ID ${filtered_rule.id}`); await deleteZeroTrustRule(filtered_rule.id); })(); From ba2db9e3d413c8c48738495cf7d2cf3946994010 Mon Sep 17 00:00:00 2001 From: mrrfv Date: Sun, 10 Sep 2023 12:54:52 +0200 Subject: [PATCH 25/27] Wrap rule name in quotes when creating list --- lib/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api.js b/lib/api.js index 445dd96..2764a4f 100644 --- a/lib/api.js +++ b/lib/api.js @@ -50,9 +50,9 @@ export const createZeroTrustListsOneByOne = async (items) => { await sleep(); totalListNumber--; listNumber++; - console.log(`Created ${listName} list - ${totalListNumber} left`); + console.log(`Created "${listName}" list - ${totalListNumber} left`); } catch (err) { - console.error(`Could not create ${listName} - ${err.toString()}`); + console.error(`Could not create "${listName}" - ${err.toString()}`); } } }; From 6185f9e2230a5d095111b3a4db10b56431b1c2a6 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Mon, 11 Sep 2023 02:54:55 +0700 Subject: [PATCH 26/27] added back node-fetch with dynamic import --- lib/helpers.js | 4 +++ package-lock.json | 89 +++++++++++++++++++++++++++++++++++++++++++++-- package.json | 5 +-- 3 files changed, 94 insertions(+), 4 deletions(-) diff --git a/lib/helpers.js b/lib/helpers.js index 0e4da8a..40c7b14 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -1,5 +1,9 @@ import { ACCOUNT_EMAIL, ACCOUNT_ID, API_HOST, API_TOKEN } from "./constants.js"; +if (!globalThis.fetch) { + globalThis.fetch = (await import("node-fetch")).default; +} + /** * Fires request to the specified URL. * @param {string} url The URL to which the request will be fired. diff --git a/package-lock.json b/package-lock.json index 5a1681b..11f7448 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,10 +5,19 @@ "packages": { "": { "dependencies": { - "dotenv": "^16.0.3" + "dotenv": "^16.0.3", + "node-fetch": "^3.3.2" }, "engines": { - "node": ">=18" + "node": ">=16" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" } }, "node_modules/dotenv": { @@ -18,6 +27,82 @@ "engines": { "node": ">=12" } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } } } } diff --git a/package.json b/package.json index 95f30e5..4b98eef 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,10 @@ { "type": "module", "dependencies": { - "dotenv": "^16.0.3" + "dotenv": "^16.0.3", + "node-fetch": "^3.3.2" }, "engines": { - "node" : ">=18" + "node": ">=16" } } From aa7cec1e1574826e5067a253e27a29eff1006cac Mon Sep 17 00:00:00 2001 From: mrrfv Date: Mon, 11 Sep 2023 18:42:36 +0200 Subject: [PATCH 27/27] Warn if using an unsupported Node version --- lib/helpers.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/helpers.js b/lib/helpers.js index 40c7b14..5fb9e89 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -1,6 +1,10 @@ import { ACCOUNT_EMAIL, ACCOUNT_ID, API_HOST, API_TOKEN } from "./constants.js"; if (!globalThis.fetch) { + console.warn("\nIMPORTANT: Your Node.js version doesn't have native fetch support and may not be supported in the future. Please update to v18 or later.\n") + // Advise what to do if running in GitHub Actions + if (process.env.GITHUB_WORKSPACE) console.warn("Since you're running in GitHub Actions, you should update your Actions workflow configuration to use Node v18 or higher.") + // Import node-fetch since there's no native fetch in this environment globalThis.fetch = (await import("node-fetch")).default; }