Merge branch 'main' into upstream

README.md

@@ -20,7 +20,6 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne
 - Whitelist support, allowing you to prevent false positives and breakage by forcing trusted domains to always be unblocked.
 - Optional health check: Sends a ping request ensuring continuous monitoring and alerting for the workflow execution.
 
-
 ## Usage
 
 ### Prerequisites
@@ -57,7 +56,6 @@ Please note that the GitHub Action downloads the recommended blocklists and whit
 - `CLOUDFLARE_LIST_ITEM_LIMIT`: The maximum number of blocked domains allowed for your Cloudflare Zero Trust plan. Use 300000 for the free plan or if you're unsure.
 - `PING_URL`: /Optional/ The HTTP(S) URL to ping (using curl) after the GitHub Action has successfully updated your filters. Useful for monitoring.
 
-
 3. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property.
 4. Enable GitHub Actions in your repository settings.
 
@@ -71,7 +69,7 @@ Alternatively, you can install the Cloudflare WARP client and log in to Zero Tru
 
 ### Dry runs
 
-To see if e.g. your filter lists are valid without actually changing anything in your Cloudflare account, you can set the `DRY_RUN` environment variable to "true", either in `.env` or the regular way. This will only print info such as the lists that would be created or the amount of duplicate domains to the console.
+To see if e.g. your filter lists are valid without actually changing anything in your Cloudflare account, you can set the `DRY_RUN` environment variable to "true" or any value other than empty, either in `.env` or the regular way. This will only print info such as the lists that would be created or the amount of duplicate domains to the console.
 
 **Warning:** This currently only works for `cf_list_create.js`.
 

cf_gateway_rule_create.js

@@ -1,5 +1,5 @@
-require("dotenv").config();
+import 'dotenv/config';
-const axios = require('axios');
+import fetch from 'node-fetch';
 
 const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
 const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
@@ -7,19 +7,24 @@ const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
 
 // Function to read Cloudflare Zero Trust lists
 async function getZeroTrustLists() {
-  const response = await axios.get(
+  const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
-    `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`,
-    {
-      headers: {
-        'Authorization': `Bearer ${API_TOKEN}`,
-        'Content-Type': 'application/json',
-        'X-Auth-Email': ACCOUNT_EMAIL,
-        'X-Auth-Key': API_TOKEN,
-      },
-    }
-  );
 
-  return response.data.result;
+  const response = await fetch(url, {
+    method: 'GET',
+    headers: {
+      'Authorization': `Bearer ${API_TOKEN}`,
+      'Content-Type': 'application/json',
+      'X-Auth-Email': ACCOUNT_EMAIL,
+      'X-Auth-Key': API_TOKEN,
+    },
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP error! status: ${response.status}`);
+  }
+
+  const data = await response.json();
+  return data.result;
 }
 
 ;(async() => {
@@ -39,25 +44,32 @@ async function getZeroTrustLists() {
     wirefilter_expression = wirefilter_expression.trim().replace('\n', '');
     if (!process.env.CI) console.log(`Firewall expression contains ${wirefilter_expression.length} characters, and checks against ${filtered_lists.length} filter lists.`)
 
-    const resp = await axios.request({
+    const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`;
-        method: 'POST',
+
-        url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`,
+    const response = await fetch(url, {
-        headers: {
+      method: 'POST',
-            'Authorization': `Bearer ${API_TOKEN}`,
+      headers: {
-            'Content-Type': 'application/json',
+        'Authorization': `Bearer ${API_TOKEN}`,
-            'X-Auth-Email': ACCOUNT_EMAIL,
+        'Content-Type': 'application/json',
-            'X-Auth-Key': API_TOKEN,
+        'X-Auth-Email': ACCOUNT_EMAIL,
-        },
+        'X-Auth-Key': API_TOKEN,
-        data: {
+      },
-            "name": "CGPS Filter Lists",
+      body: JSON.stringify({
-            "description": "Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.",
+        "name": "CGPS Filter Lists",
-            "enabled": true,
+        "description": "Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.",
-            "action": "block",
+        "enabled": true,
-            "filters": ["dns"],
+        "action": "block",
-            "traffic": wirefilter_expression,
+        "filters": ["dns"],
-        }
+        "traffic": wirefilter_expression,
+      }),
     });
-    console.log('Success:', resp.data.success);
+  
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+  
+    const data = await response.json();
+    console.log('Success:', data.success);
 })();
 
 async function sleep(ms) {

cf_gateway_rule_delete.js

@@ -1,5 +1,5 @@
-require("dotenv").config();
+import 'dotenv/config';
-const axios = require('axios');
+import fetch from 'node-fetch';
 
 const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
 const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
@@ -7,19 +7,24 @@ const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
 
 // Function to read Cloudflare Zero Trust rules
 async function getZeroTrustRules() {
-  const response = await axios.get(
+  const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`;
-    `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`,
-    {
-      headers: {
-        'Authorization': `Bearer ${API_TOKEN}`,
-        'Content-Type': 'application/json',
-        'X-Auth-Email': ACCOUNT_EMAIL,
-        'X-Auth-Key': API_TOKEN,
-      },
-    }
-  );
 
-  return response.data.result;
+  const response = await fetch(url, {
+    method: 'GET',
+    headers: {
+      'Authorization': `Bearer ${API_TOKEN}`,
+      'Content-Type': 'application/json',
+      'X-Auth-Email': ACCOUNT_EMAIL,
+      'X-Auth-Key': API_TOKEN,
+    },
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP error! status: ${response.status}`);
+  }
+
+  const data = await response.json();
+  return data.result;
 }
 
 ;(async() => {
@@ -30,9 +35,10 @@ async function getZeroTrustRules() {
 
     console.log(`Deleting rule`, process.env.CI ? "(redacted, running in CI)" : `${filtered_rule.name} with ID ${filtered_rule.id}`);
 
-    const resp = await axios.request({
+    const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules/${filtered_rule.id}`;
+
+    const resp = await fetch(url, {
         method: 'DELETE',
-        url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules/${filtered_rule.id}`,
         headers: {
             'Authorization': `Bearer ${API_TOKEN}`,
             'Content-Type': 'application/json',
@@ -40,8 +46,10 @@ async function getZeroTrustRules() {
             'X-Auth-Key': API_TOKEN,
         },
     });
-
+    
-    console.log('Success: ', resp.data.success);
+    const data = await resp.json();
+    
+    console.log('Success: ', data.success);
     await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe
 })();
 

cf_list_create.js

@@ -1,6 +1,6 @@
-require("dotenv").config();
+import 'dotenv/config';
-const fs = require('fs');
+import fetch from 'node-fetch';
-const axios = require('axios');
+import fs from 'fs';
 
 const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
 const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
@@ -91,13 +91,15 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
   domains = uniqueDomains;
 
   // Remove domains from the domains array that are present in the whitelist array
+  let whitelistedDomainCount = 0;
   domains = domains.filter(domain => {
     if (whitelist.includes(domain)) {
-      console.warn(`Domain found in the whitelist: ${domain} - removing`);
+      whitelistedDomainCount++;
       return false;
     }
     return true;
   });
+  if (whitelistedDomainCount > 0) console.warn(`Found ${whitelistedDomainCount} domains in input.csv that are present in the whitelist - removing them`);
 
   // Trim array to 300,000 domains if it's longer than that
   if (domains.length > LIST_ITEM_LIMIT) {
@@ -156,24 +158,26 @@ function chunkArray(array, chunkSize) {
 
 // Function to create a Cloudflare Zero Trust list
 async function createZeroTrustList(name, items, currentItem, totalItems) {
-  const response = await axios.post(
+  const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
-    `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`,
+
-    {
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${API_TOKEN}`,
+      'Content-Type': 'application/json',
+      'X-Auth-Email': ACCOUNT_EMAIL,
+      'X-Auth-Key': API_TOKEN,
+    },
+    body: JSON.stringify({
       name,
       type: 'DOMAIN', // Set list type to DOMAIN
       items,
-    },
+    }),
-    {
+  });
-      headers: {
+
-        'Authorization': `Bearer ${API_TOKEN}`,
+  const data = await response.json();
-        'Content-Type': 'application/json',
+  const listId = data.result.id;
-        'X-Auth-Email': ACCOUNT_EMAIL,
-        'X-Auth-Key': API_TOKEN,
-      },
-    }
-  );
 
-  const listId = response.data.result.id;
   console.log(`Created Zero Trust list`, process.env.CI ? "(redacted on CI)" : `"${name}" with ID ${listId} - ${totalItems - currentItem} left`);
 }
 

cf_list_delete.js

@@ -1,5 +1,5 @@
-require("dotenv").config();
+import 'dotenv/config';
-const axios = require('axios');
+import fetch from 'node-fetch';
 
 const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
 const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
@@ -7,19 +7,20 @@ const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
 
 // Function to read Cloudflare Zero Trust lists
 async function getZeroTrustLists() {
-  const response = await axios.get(
+  const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
-    `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`,
-    {
-      headers: {
-        'Authorization': `Bearer ${API_TOKEN}`,
-        'Content-Type': 'application/json',
-        'X-Auth-Email': ACCOUNT_EMAIL,
-        'X-Auth-Key': API_TOKEN,
-      },
-    }
-  );
 
-  return response.data.result;
+  const response = await fetch(url, {
+    method: 'GET',
+    headers: {
+      'Authorization': `Bearer ${API_TOKEN}`,
+      'Content-Type': 'application/json',
+      'X-Auth-Email': ACCOUNT_EMAIL,
+      'X-Auth-Key': API_TOKEN,
+    },
+  });
+
+  const data = await response.json();
+  return data.result;
 }
 
 ;(async() => {
@@ -30,21 +31,26 @@ async function getZeroTrustLists() {
 
     if (!process.env.CI) console.log(`Got ${lists.length} lists, ${cgps_lists.length} of which are CGPS lists that will be deleted.`);
 
+    let lists_processed = 0;
     for (const list of cgps_lists) {
-        console.log(`Deleting list`, process.env.CI ? "(redacted, running in CI)" : `${list.name} with ID ${list.id}`);
+        console.log(`Deleting list`, process.env.CI ? "(info redacted, running in CI)" : `${list.name} with ID ${list.id}, ${cgps_lists.length - lists_processed - 1} left`);
-        const resp = await axios.request({
+        
+        const resp = await fetch(`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists/${list.id}`, {
             method: 'DELETE',
-            url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists/${list.id}`,
             headers: {
                 'Authorization': `Bearer ${API_TOKEN}`,
                 'Content-Type': 'application/json',
                 'X-Auth-Email': ACCOUNT_EMAIL,
                 'X-Auth-Key': API_TOKEN,
-              },
+            },
         });
-        console.log('Success:', resp.data.success);
+    
+        const data = await resp.json();
+        console.log('Success:', data.success);
+        lists_processed++;
+        
         await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe
-    }
+    }    
 })();
 
 async function sleep(ms) {

get_recommended_filters.sh

@@ -16,4 +16,4 @@ urls=(
 download_lists $urls 'input.csv'
 
 # print a message when done
-echo "Done. The input.csv file contains merged data from recommended filter lists."
+echo "Done. The input.csv file contains merged data from recommended filter lists."

package-lock.json

@@ -5,42 +5,16 @@
   "packages": {
     "": {
       "dependencies": {
-        "axios": "^1.4.0",
+        "dotenv": "^16.0.3",
-        "dotenv": "^16.0.3"
+        "node-fetch": "^3.3.2"
       }
     },
-    "node_modules/asynckit": {
+    "node_modules/data-uri-to-buffer": {
-      "version": "0.4.0",
+      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+      "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
-    },
-    "node_modules/axios": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
-      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
-      "dependencies": {
-        "follow-redirects": "^1.15.0",
-        "form-data": "^4.0.0",
-        "proxy-from-env": "^1.1.0"
-      }
-    },
-    "node_modules/combined-stream": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
-      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dependencies": {
-        "delayed-stream": "~1.0.0"
-      },
       "engines": {
-        "node": ">= 0.8"
+        "node": ">= 12"
-      }
-    },
-    "node_modules/delayed-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "engines": {
-        "node": ">=0.4.0"
       }
     },
     "node_modules/dotenv": {
@@ -51,61 +25,81 @@
         "node": ">=12"
       }
     },
-    "node_modules/follow-redirects": {
+    "node_modules/fetch-blob": {
-      "version": "1.15.2",
+      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
-      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
       "funding": [
         {
-          "type": "individual",
+          "type": "github",
-          "url": "https://github.com/sponsors/RubenVerborgh"
+          "url": "https://github.com/sponsors/jimmywarting"
+        },
+        {
+          "type": "paypal",
+          "url": "https://paypal.me/jimmywarting"
+        }
+      ],
+      "dependencies": {
+        "node-domexception": "^1.0.0",
+        "web-streams-polyfill": "^3.0.3"
+      },
+      "engines": {
+        "node": "^12.20 || >= 14.13"
+      }
+    },
+    "node_modules/formdata-polyfill": {
+      "version": "4.0.10",
+      "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
+      "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
+      "dependencies": {
+        "fetch-blob": "^3.1.2"
+      },
+      "engines": {
+        "node": ">=12.20.0"
+      }
+    },
+    "node_modules/node-domexception": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/jimmywarting"
+        },
+        {
+          "type": "github",
+          "url": "https://paypal.me/jimmywarting"
         }
       ],
       "engines": {
-        "node": ">=4.0"
+        "node": ">=10.5.0"
-      },
-      "peerDependenciesMeta": {
-        "debug": {
-          "optional": true
-        }
       }
     },
-    "node_modules/form-data": {
+    "node_modules/node-fetch": {
-      "version": "4.0.0",
+      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
       "dependencies": {
-        "asynckit": "^0.4.0",
+        "data-uri-to-buffer": "^4.0.0",
-        "combined-stream": "^1.0.8",
+        "fetch-blob": "^3.1.4",
-        "mime-types": "^2.1.12"
+        "formdata-polyfill": "^4.0.10"
       },
       "engines": {
-        "node": ">= 6"
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-      }
-    },
-    "node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dependencies": {
-        "mime-db": "1.52.0"
       },
-      "engines": {
+      "funding": {
-        "node": ">= 0.6"
+        "type": "opencollective",
+        "url": "https://opencollective.com/node-fetch"
       }
     },
-    "node_modules/proxy-from-env": {
+    "node_modules/web-streams-polyfill": {
-      "version": "1.1.0",
+      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+      "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
+      "engines": {
+        "node": ">= 8"
+      }
     }
   }
 }

package.json

@@ -1,6 +1,7 @@
 {
+  "type": "module",
   "dependencies": {
-    "axios": "^1.4.0",
+    "dotenv": "^16.0.3",
-    "dotenv": "^16.0.3"
+    "node-fetch": "^3.3.2"
   }
 }