Merge pull request #22 from hlqviet/refactor/api-request

API request code refactor

.env.example

@@ -1,4 +1,4 @@
-CLOUDFLARE_API_KEY=""
-CLOUDFLARE_ACCOUNT_ID=""
-CLOUDFLARE_ACCOUNT_EMAIL=""
-CLOUDFLARE_LIST_ITEM_LIMIT="300000"
+CLOUDFLARE_API_KEY=
+CLOUDFLARE_ACCOUNT_ID=
+CLOUDFLARE_ACCOUNT_EMAIL=
+CLOUDFLARE_LIST_ITEM_LIMIT=300000

README.md

@@ -56,8 +56,12 @@ Please note that the GitHub Action downloads the recommended blocklists and whit
 - `CLOUDFLARE_LIST_ITEM_LIMIT`: The maximum number of blocked domains allowed for your Cloudflare Zero Trust plan. Use 300000 for the free plan or if you're unsure.
 - `PING_URL`: /Optional/ The HTTP(S) URL to ping (using curl) after the GitHub Action has successfully updated your filters. Useful for monitoring.
 
-3. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property.
-4. Enable GitHub Actions in your repository settings.
+3. Create the following GitHub Actions variables in your repository settings if you desire:
+
+- `FAST_MODE`: Enable the scripts to send the requests simultaneously. Beware that there's a rate limit of 1200 requests per five minutes (https://developers.cloudflare.com/fundamentals/api/reference/limits/) so make sure you know what you are doing.
+
+4. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property.
+5. Enable GitHub Actions in your repository settings.
 
 ### DNS setup for Cloudflare Gateway
 
@@ -69,7 +73,7 @@ Alternatively, you can install the Cloudflare WARP client and log in to Zero Tru
 
 ### Dry runs
 
-To see if e.g. your filter lists are valid without actually changing anything in your Cloudflare account, you can set the `DRY_RUN` environment variable to "true" or any value other than empty, either in `.env` or the regular way. This will only print info such as the lists that would be created or the amount of duplicate domains to the console.
+To see if e.g. your filter lists are valid without actually changing anything in your Cloudflare account, you can set the `DRY_RUN` environment variable to 1, either in `.env` or the regular way. This will only print info such as the lists that would be created or the amount of duplicate domains to the console.
 
 **Warning:** This currently only works for `cf_list_create.js`.
 

auto_update_github_action.yml

@@ -2,65 +2,61 @@ name: Update Filter Lists
 
 on:
   schedule:
-    - cron: '0 3 * * 1'
+    - cron: "0 3 * * 1"
   push:
     branches:
-      - '**'
+      - main
   workflow_dispatch:
 
 jobs:
   cgps:
     runs-on: ubuntu-latest
     steps:
-    - name: Install Node.js
-      uses: actions/setup-node@v3
-      with:
-        node-version: 16
-    
-    - name: Clone repository and switch to v1 branch
-      run: |
-        git clone https://github.com/mrrfv/cloudflare-gateway-pihole-scripts.git
-        cd cloudflare-gateway-pihole-scripts
-        git checkout v1
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: "mrrfv/cloudflare-gateway-pihole-scripts"
+          ref: "v1"
 
-    - name: Install npm dependencies
-      run: npm ci
-      working-directory: cloudflare-gateway-pihole-scripts
-      
-    - name: Download recommended whitelist
-      run: bash ./get_recommended_whitelist.sh
-      working-directory: cloudflare-gateway-pihole-scripts
-      
-    - name: Download recommended filters
-      run: bash ./get_recommended_filters.sh
-      working-directory: cloudflare-gateway-pihole-scripts
-    
-    - name: Delete old rules and lists
-      run: |
-        node cf_gateway_rule_delete.js
-        node cf_list_delete.js
-      working-directory: cloudflare-gateway-pihole-scripts
-      env:
-        CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }}
-        CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }}
-        CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
-        CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }}
+      - name: Install Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "lts/*"
 
-    - name: Create new rules and lists
-      run: |
-        node cf_list_create.js
-        node cf_gateway_rule_create.js
-      working-directory: cloudflare-gateway-pihole-scripts
-      env:
-        CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }}
-        CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }}
-        CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
-        CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }}
+      - name: Install npm dependencies
+        run: npm ci
 
-    - name: Send ping request
-      if: env.PING_URL != ''
-      working-directory: cloudflare-gateway-pihole-scripts
-      env:
-        PING_URL: ${{ secrets.PING_URL }}
-      run: |
-        curl "${{ env.PING_URL }}"
+      - name: Download recommended whitelist
+        run: bash ./get_recommended_whitelist.sh
+
+      - name: Download recommended filters
+        run: bash ./get_recommended_filters.sh
+
+      - name: Delete old rules and lists
+        run: |
+          node cf_gateway_rule_delete.js
+          node cf_list_delete.js
+        env:
+          CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }}
+          CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }}
+          FAST_MODE: ${{ vars.FAST_MODE }}
+
+      - name: Create new rules and lists
+        run: |
+          node cf_list_create.js
+          node cf_gateway_rule_create.js
+        env:
+          CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }}
+          CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }}
+          FAST_MODE: ${{ vars.FAST_MODE }}
+
+      - name: Send ping request
+        if: env.PING_URL != ''
+        run: |
+          curl "${{ env.PING_URL }}"
+        env:
+          PING_URL: ${{ secrets.PING_URL }}

cf_gateway_rule_create.js

@@ -1,34 +1,7 @@
-import 'dotenv/config';
-import fetch from 'node-fetch';
-
-const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
-const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
-const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
-
-// Function to read Cloudflare Zero Trust lists
-async function getZeroTrustLists() {
-  const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
-
-  const response = await fetch(url, {
-    method: 'GET',
-    headers: {
-      'Authorization': `Bearer ${API_TOKEN}`,
-      'Content-Type': 'application/json',
-      'X-Auth-Email': ACCOUNT_EMAIL,
-      'X-Auth-Key': API_TOKEN,
-    },
-  });
-
-  if (!response.ok) {
-    throw new Error(`HTTP error! status: ${response.status}`);
-  }
-
-  const data = await response.json();
-  return data.result;
-}
+import { createZeroTrustRule, getZeroTrustLists } from './lib/api.js';
 
 ;(async() => {
-    const lists = await getZeroTrustLists();
+    const { result: lists } = await getZeroTrustLists();
     const filtered_lists = lists.filter(list => list.name.startsWith('CGPS List'));
 
     let wirefilter_expression = '';
@@ -44,34 +17,5 @@ async function getZeroTrustLists() {
     wirefilter_expression = wirefilter_expression.trim().replace('\n', '');
     if (!process.env.CI) console.log(`Firewall expression contains ${wirefilter_expression.length} characters, and checks against ${filtered_lists.length} filter lists.`)
 
-    const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`;
-
-    const response = await fetch(url, {
-      method: 'POST',
-      headers: {
-        'Authorization': `Bearer ${API_TOKEN}`,
-        'Content-Type': 'application/json',
-        'X-Auth-Email': ACCOUNT_EMAIL,
-        'X-Auth-Key': API_TOKEN,
-      },
-      body: JSON.stringify({
-        "name": "CGPS Filter Lists",
-        "description": "Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.",
-        "enabled": true,
-        "action": "block",
-        "filters": ["dns"],
-        "traffic": wirefilter_expression,
-      }),
-    });
-  
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
-  
-    const data = await response.json();
-    console.log('Success:', data.success);
+    await createZeroTrustRule(wirefilter_expression);
 })();
-
-async function sleep(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}

cf_gateway_rule_delete.js

@@ -1,58 +1,12 @@
-import 'dotenv/config';
-import fetch from 'node-fetch';
-
-const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
-const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
-const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
-
-// Function to read Cloudflare Zero Trust rules
-async function getZeroTrustRules() {
-  const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`;
-
-  const response = await fetch(url, {
-    method: 'GET',
-    headers: {
-      'Authorization': `Bearer ${API_TOKEN}`,
-      'Content-Type': 'application/json',
-      'X-Auth-Email': ACCOUNT_EMAIL,
-      'X-Auth-Key': API_TOKEN,
-    },
-  });
-
-  if (!response.ok) {
-    throw new Error(`HTTP error! status: ${response.status}`);
-  }
-
-  const data = await response.json();
-  return data.result;
-}
+import { deleteZeroTrustRule, getZeroTrustRules } from './lib/api.js';
 
 ;(async() => {
-    const rules = await getZeroTrustRules();
+    const { result: rules } = await getZeroTrustRules();
     const [filtered_rule] = rules.filter(rule => rule.name === "CGPS Filter Lists");
 
     if (!filtered_rule) return console.warn("No rule with matching name found - this is not an issue if you haven't run the create script yet. Exiting.");
 
-    console.log(`Deleting rule`, process.env.CI ? "(redacted, running in CI)" : `${filtered_rule.name} with ID ${filtered_rule.id}`);
+    console.log(`Deleting rule`, process.env.CI ? "(redacted, running in CI)" : `"${filtered_rule.name}" with ID ${filtered_rule.id}`);
 
-    const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules/${filtered_rule.id}`;
-
-    const resp = await fetch(url, {
-        method: 'DELETE',
-        headers: {
-            'Authorization': `Bearer ${API_TOKEN}`,
-            'Content-Type': 'application/json',
-            'X-Auth-Email': ACCOUNT_EMAIL,
-            'X-Auth-Key': API_TOKEN,
-        },
-    });
-    
-    const data = await resp.json();
-    
-    console.log('Success: ', data.success);
-    await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe
+    await deleteZeroTrustRule(filtered_rule.id);
 })();
-
-async function sleep(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}

cf_list_create.js

@@ -1,11 +1,7 @@
-import 'dotenv/config';
-import fetch from 'node-fetch';
 import fs from 'fs';
-
-const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
-const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
-const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
-const LIST_ITEM_LIMIT = Number.isSafeInteger(Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT)) ? Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT) : 300000;
+import { DRY_RUN, FAST_MODE, LIST_ITEM_LIMIT } from './lib/constants.js';
+import { createZeroTrustListsAtOnce, createZeroTrustListsOneByOne } from './lib/api.js';
+import { truncateArray } from './lib/utils.js';
 
 if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`);
 
@@ -104,7 +100,7 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
   // Trim array to 300,000 domains if it's longer than that
   if (domains.length > LIST_ITEM_LIMIT) {
     console.warn(`${domains.length} domains found in input.csv - input has to be trimmed to ${LIST_ITEM_LIMIT} domains`);
-    domains = trimArray(domains, LIST_ITEM_LIMIT);
+    domains = truncateArray(domains, LIST_ITEM_LIMIT);
   }
 
   const listsToCreate = Math.ceil(domains.length / 1000);
@@ -113,79 +109,12 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
 
   // If we are dry-running, stop here because we don't want to create lists
   // TODO: we should probably continue, just without making any real requests to Cloudflare
-  if (process.env.DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.');
+  if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.');
 
-  // Separate domains into chunks of 1000 (Cloudflare list cap)
-  const chunks = chunkArray(domains, 1000);
-
-  // Create Cloudflare Zero Trust lists
-  for (const [index, chunk] of chunks.entries()) {
-    const listName = `CGPS List - Chunk ${index}`;
-
-    let properList = [];
-
-    chunk.forEach(domain => {
-        properList.push({ "value": domain })
-    });
-
-    try {
-      await createZeroTrustList(listName, properList, (index+1), listsToCreate);
-      await sleep(350); // Sleep for 350ms between list additions
-    } catch (error) {
-      console.error(`Error creating list `, process.env.CI ? "(redacted on CI)" :  `"${listName}": ${error.response.data}`);
-    }
+  if (FAST_MODE) {
+    await createZeroTrustListsAtOnce(domains);
+    return;
   }
+
+  await createZeroTrustListsOneByOne(domains);
 });
-
-function trimArray(arr, size) {
-  return arr.slice(0, size);
-}
-
-// Function to check if a domain is valid
-function isValidDomain(domain) {
-    const regex = /^((?!-)[A-Za-z0-9-]{1,63}(?<!-)\.)+[A-Za-z]{2,6}$/;
-    return regex.test(domain);
-}
-
-// Function to split an array into chunks
-function chunkArray(array, chunkSize) {
-  const chunks = [];
-  for (let i = 0; i < array.length; i += chunkSize) {
-    chunks.push(array.slice(i, i + chunkSize));
-  }
-  return chunks;
-}
-
-// Function to create a Cloudflare Zero Trust list
-async function createZeroTrustList(name, items, currentItem, totalItems) {
-  const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
-
-  const response = await fetch(url, {
-    method: 'POST',
-    headers: {
-      'Authorization': `Bearer ${API_TOKEN}`,
-      'Content-Type': 'application/json',
-      'X-Auth-Email': ACCOUNT_EMAIL,
-      'X-Auth-Key': API_TOKEN,
-    },
-    body: JSON.stringify({
-      name,
-      type: 'DOMAIN', // Set list type to DOMAIN
-      items,
-    }),
-  });
-
-  const data = await response.json();
-  const listId = data.result.id;
-
-  console.log(`Created Zero Trust list`, process.env.CI ? "(redacted on CI)" : `"${name}" with ID ${listId} - ${totalItems - currentItem} left`);
-}
-
-function percentage(percent, total) {
-  return Math.round((percent / 100) * total);
-}
-
-// Function to sleep for a specified duration
-function sleep(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}

cf_list_delete.js

@@ -1,58 +1,18 @@
-import 'dotenv/config';
-import fetch from 'node-fetch';
-
-const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
-const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
-const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
-
-// Function to read Cloudflare Zero Trust lists
-async function getZeroTrustLists() {
-  const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
-
-  const response = await fetch(url, {
-    method: 'GET',
-    headers: {
-      'Authorization': `Bearer ${API_TOKEN}`,
-      'Content-Type': 'application/json',
-      'X-Auth-Email': ACCOUNT_EMAIL,
-      'X-Auth-Key': API_TOKEN,
-    },
-  });
-
-  const data = await response.json();
-  return data.result;
-}
+import { deleteZeroTrustListsAtOnce, deleteZeroTrustListsOneByOne, getZeroTrustLists } from "./lib/api.js";
+import { FAST_MODE } from "./lib/constants.js";
 
 ;(async() => {
-    const lists = await getZeroTrustLists();
+    const { result: lists } = await getZeroTrustLists();
     if (!lists) return console.warn("No file lists found - this is not an issue if it's your first time running this script. Exiting.");
     const cgps_lists = lists.filter(list => list.name.startsWith('CGPS List'));
     if (!cgps_lists.length) return console.warn("No lists with matching name found - this is not an issue if you haven't created any filter lists before. Exiting.");
 
     if (!process.env.CI) console.log(`Got ${lists.length} lists, ${cgps_lists.length} of which are CGPS lists that will be deleted.`);
 
-    let lists_processed = 0;
-    for (const list of cgps_lists) {
-        console.log(`Deleting list`, process.env.CI ? "(info redacted, running in CI)" : `${list.name} with ID ${list.id}, ${cgps_lists.length - lists_processed - 1} left`);
-        
-        const resp = await fetch(`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists/${list.id}`, {
-            method: 'DELETE',
-            headers: {
-                'Authorization': `Bearer ${API_TOKEN}`,
-                'Content-Type': 'application/json',
-                'X-Auth-Email': ACCOUNT_EMAIL,
-                'X-Auth-Key': API_TOKEN,
-            },
-        });
-    
-        const data = await resp.json();
-        console.log('Success:', data.success);
-        lists_processed++;
-        
-        await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe
-    }    
-})();
+    if (FAST_MODE) {
+      await deleteZeroTrustListsAtOnce(cgps_lists);
+      return;
+    }
 
-async function sleep(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
+    await deleteZeroTrustListsOneByOne(cgps_lists);
+})();

get_recommended_filters.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-source "$(dirname "$0")/helpers.sh"
+source $(dirname "$0")/lib/helpers.sh
 
 # declare an array of urls
 urls=(
@@ -16,4 +16,4 @@ urls=(
 download_lists $urls 'input.csv'
 
 # print a message when done
-echo "Done. The input.csv file contains merged data from recommended filter lists."
+echo "Done. The input.csv file contains merged data from recommended filter lists."

get_recommended_whitelist.sh

@@ -3,7 +3,7 @@
 # Use the provided lists or add your own.
 # There is no limit on the amount of whitelisted domains you can have.
 
-source "$(dirname "$0")/helpers.sh"
+source $(dirname "$0")/lib/helpers.sh
 
 # declare an array of urls
 urls=(

lib/api.js

@@ -0,0 +1,174 @@
+import { LIST_ITEM_SIZE } from "./constants.js";
+import { requestGateway } from "./helpers.js";
+import { sleep } from "./utils.js";
+
+/**
+ * Gets Zero Trust lists.
+ *
+ * API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-list-zero-trust-lists
+ * @returns {Promise<Object>}
+ */
+export const getZeroTrustLists = () =>
+  requestGateway("/lists", {
+    method: "GET",
+  });
+
+/**
+ * Creates a Zero Trust list.
+ *
+ * API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-create-zero-trust-list
+ * @param {string} name The name of the list.
+ * @param {Object[]} items The domains in the list.
+ * @param {string} items[].value The domain of an entry.
+ * @returns {Promise}
+ */
+const createZeroTrustList = (name, items) =>
+  requestGateway(`/lists`, {
+    method: "POST",
+    body: JSON.stringify({
+      name,
+      type: "DOMAIN",
+      items,
+    }),
+  });
+
+/**
+ * Creates Zero Trust lists sequentially.
+ * @param {string[]} items The domains.
+ */
+export const createZeroTrustListsOneByOne = async (items) => {
+  let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE);
+
+  for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) {
+    const chunk = items
+      .slice(i, i + LIST_ITEM_SIZE)
+      .map((item) => ({ value: item }));
+    const listName = `CGPS List - Chunk ${listNumber}`;
+
+    try {
+      await createZeroTrustList(listName, chunk);
+      await sleep();
+      totalListNumber--;
+      listNumber++;
+      console.log(`Created "${listName}" list - ${totalListNumber} left`);
+    } catch (err) {
+      console.error(`Could not create "${listName}" - ${err.toString()}`);
+    }
+  }
+};
+
+/**
+ * Creates all Zero Trust lists at once.
+ * @param {string[]} items The domains.
+ */
+export const createZeroTrustListsAtOnce = async (items) => {
+  const totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE);
+  const requests = [];
+
+  for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) {
+    const chunk = items
+      .slice(i, i + LIST_ITEM_SIZE)
+      .map((item) => ({ value: item }));
+    const listName = `CGPS List - Chunk ${listNumber}`;
+
+    requests.push(createZeroTrustList(listName, chunk));
+    listNumber++;
+  }
+
+  try {
+    await Promise.all(requests);
+    console.log(`Created ${totalListNumber} lists`);
+  } catch (err) {
+    console.error(`Error occurred while creating lists - ${err.toString()}`);
+  }
+};
+
+/**
+ * Deletes a Zero Trust list.
+ *
+ * API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-delete-zero-trust-list
+ * @param {number} id The ID of the list.
+ * @returns {Promise<any>}
+ */
+const deleteZeroTrustList = (id) =>
+  requestGateway(`/lists/${id}`, { method: "DELETE" });
+
+/**
+ * Deletes Zero Trust lists sequentially.
+ * @param {Object[]} lists The lists to be deleted.
+ * @param {number} lists[].id The ID of a list.
+ * @param {string} lists[].name The name of a list.
+ */
+export const deleteZeroTrustListsOneByOne = async (lists) => {
+  let totalListNumber = lists.length;
+
+  for (const { id, name } of lists) {
+    try {
+      await deleteZeroTrustList(id);
+      await sleep();
+      totalListNumber--;
+      console.log(`Deleted ${name} list - ${totalListNumber} left`);
+    } catch (err) {
+      console.error(`Could not delete ${name} - ${err.toString()}`);
+    }
+  }
+};
+
+/**
+ * Deletes all Zero Trust lists at once.
+ * @param {Object[]} lists The lists to be deleted.
+ * @param {number} lists[].id The ID of a list.
+ * @param {string} lists[].name The name of a list.
+ */
+export const deleteZeroTrustListsAtOnce = async (lists) => {
+  const requests = lists.map(({ id }) => deleteZeroTrustList(id));
+
+  try {
+    await Promise.all(requests);
+    console.log(`Deleted ${lists.length} lists`);
+  } catch (err) {
+    console.error(`Error occurred while deleting lists - ${err.toString()}`);
+  }
+};
+
+/**
+ * Gets Zero Trust rules.
+ *
+ * API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-list-zero-trust-gateway-rules
+ * @returns {Promise<Object>}
+ */
+export const getZeroTrustRules = () =>
+  requestGateway("/rules", { method: "GET" });
+
+/**
+ * Creates a Zero Trust rule.
+ *
+ * API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-create-zero-trust-gateway-rule
+ * @param {string} wirefilterExpression The expression to be used for the rule.
+ * @returns {Promise<Object>}
+ */
+export const createZeroTrustRule = (wirefilterExpression) =>
+  requestGateway("/rules", {
+    method: "POST",
+    body: JSON.stringify({
+      name: "CGPS Filter Lists",
+      description:
+        "Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.",
+      enabled: true,
+      action: "block",
+      filters: ["dns"],
+      traffic: wirefilterExpression,
+    }),
+  });
+
+/**
+ * Deletes a Zero Trust rule.
+ *
+ * API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-delete-zero-trust-gateway-rule
+ * @param {number} id The ID of the rule to be deleted.
+ * @returns {Promise<Object>}
+ */
+export const deleteZeroTrustRule = (id) =>
+  requestGateway(`/rules/${id}`, {
+    method: "DELETE",
+  });

lib/constants.js

@@ -0,0 +1,21 @@
+import dotenv from "dotenv";
+
+dotenv.config();
+
+export const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
+
+export const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
+
+export const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
+
+export const LIST_ITEM_LIMIT = isNaN(process.env.CLOUDFLARE_LIST_ITEM_LIMIT)
+  ? 300000
+  : parseInt(process.env.CLOUDFLARE_LIST_ITEM_LIMIT, 10);
+
+export const LIST_ITEM_SIZE = 1000;
+
+export const API_HOST = "https://api.cloudflare.com/client/v4";
+
+export const DRY_RUN = !!parseInt(process.env.DRY_RUN, 10);
+
+export const FAST_MODE = !!parseInt(process.env.FAST_MODE, 10);

lib/helpers.js

@@ -0,0 +1,46 @@
+import { ACCOUNT_EMAIL, ACCOUNT_ID, API_HOST, API_TOKEN } from "./constants.js";
+
+if (!globalThis.fetch) {
+  console.warn("\nIMPORTANT: Your Node.js version doesn't have native fetch support and may not be supported in the future. Please update to v18 or later.\n")
+  // Advise what to do if running in GitHub Actions
+  if (process.env.GITHUB_WORKSPACE) console.warn("Since you're running in GitHub Actions, you should update your Actions workflow configuration to use Node v18 or higher.") 
+  // Import node-fetch since there's no native fetch in this environment
+  globalThis.fetch = (await import("node-fetch")).default;
+}
+
+/**
+ * Fires request to the specified URL.
+ * @param {string} url The URL to which the request will be fired.
+ * @param {RequestInit} options The options to be passed to `fetch`.
+ * @returns {Promise}
+ */
+const request = async (url, options) => {
+  const response = await fetch(url, {
+    headers: {
+      Authorization: `Bearer ${API_TOKEN}`,
+      "Content-Type": "application/json",
+      "X-Auth-Email": ACCOUNT_EMAIL,
+      "X-Auth-Key": API_TOKEN,
+    },
+    ...options,
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP error! Status: ${response.status}`);
+  }
+
+  const data = await response.json();
+
+  console.log(`HTTP request succeeded: ${data.success}`);
+
+  return data;
+};
+
+/**
+ * Fires request to the Zero Trust gateway.
+ * @param {string} path The path which will be appended to the request URL.
+ * @param {RequestInit} options The options to be passed to `fetch`.
+ * @returns {Promise}
+ */
+export const requestGateway = (path, options) =>
+  request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway${path}`, options);

lib/utils.js

@@ -0,0 +1,14 @@
+/**
+ * Sleeps for a specified amount of time.
+ * @param {number} [ms=350] The amount of time in ms.
+ */
+export const sleep = (ms = 350) =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Truncates an array to the specified size.
+ * @param {any[]} arr The array to be truncated.
+ * @param {number} size The size to which the array will be truncated.
+ * @returns {any[]}
+ */
+export const truncateArray = (arr, size) => arr.slice(0, size);

package-lock.json

@@ -7,6 +7,9 @@
       "dependencies": {
         "dotenv": "^16.0.3",
         "node-fetch": "^3.3.2"
+      },
+      "engines": {
+        "node": ">=16"
       }
     },
     "node_modules/data-uri-to-buffer": {

package.json

@@ -3,5 +3,8 @@
   "dependencies": {
     "dotenv": "^16.0.3",
     "node-fetch": "^3.3.2"
+  },
+  "engines": {
+    "node": ">=16"
   }
 }