Initial version

.env.example

@@ -0,0 +1,4 @@
+CLOUDFLARE_API_KEY=""
+CLOUDFLARE_ACCOUNT_ID=""
+CLOUDFLARE_ACCOUNT_EMAIL=""
+CLOUDFLARE_LIST_ITEM_LIMIT="300000"

.gitignore

@@ -7,6 +7,10 @@ yarn-error.log*
 lerna-debug.log*
 .pnpm-debug.log*
 
+# CGPS
+output
+input.csv
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 

README.md

@@ -1,2 +1,45 @@
-# cloudflare-gateway-pihole-scripts
- Use Cloudflare Gateway VPN to block ads and tracking domains - free alternative to NextDNS, Pi-hole and Adguard
+# Cloudflare Gateway Pi-hole Scripts (CGPS)
+
+![Cloudflare Gateway Analytics screenshot](.github/images/gateway_analytics.png)
+
+Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and network traffic based on your firewall policies. This is a collection of scripts that can be used to get a similar experience as if you were using Pi-hole, but with Cloudflare Gateway - so no servers to maintain or need to buy a Raspberry Pi!
+
+## About the individual scripts
+
+- `cf_list_delete.js` - Deletes all lists created by CGPS from Cloudflare Gateway. This is useful for subsequent runs.
+- `cf_list_create.js` - Takes an input.csv file containing domains and creates lists in Cloudflare Gateway
+- `cf_gateway_rule_create.js` - Creates a Cloudflare Gateway rule to block all traffic if it matches the lists created by CGPS.
+- `cf_gateway_rule_delete.js` - Deletes the Cloudflare Gateway rule created by CGPS. Useful for subsequent runs.
+
+## Features
+
+- Support for hosts files
+- Full support for domain lists
+- Automatically cleans up filter lists: removes duplicates, invalid domains, comments and more
+- Works fully unattended
+
+## Usage
+
+### Prerequisites
+
+1. Node.js installed on your machine
+2. Cloudflare Zero Trust account - the Free plan is enough. Use the Cloudflare documentation for details.
+3. Cloudflare email, API key (NOT the API token), and account ID
+4. A filter list of domains you want to block - **max 300,000 domains for the free plan** - in the working directory named `input.csv`. Mullvad provides awesome [DNS blocklists](https://github.com/mullvad/dns-blocklists) that work well with this project.
+
+---
+
+1. Clone this repository.
+2. Run `npm install` to install dependencies.
+3. Copy `.env.example` to `.env` and fill in the values.
+4. If this is a subsequent run, execute `cf_gateway_rule_delete.js` and `cf_list_delete.js` (in order) to clean up.
+5. If you're on Linux and haven't downloaded any filters yourself, use the `get_recommended_filters.sh` script to download recommended filter lists (about 250 000 domains).
+6. Run `cf_list_create.js` to create the lists in Cloudflare Gateway.
+7. Run `cf_gateway_rule_create.js` to create the firewall rule in Cloudflare Gateway.
+8. Profit!
+
+### Running in GitHub Actions
+
+This project can be run in GitHub Actions so your filter lists will be automatically updated and pushed to Cloudflare Gateway.
+
+[work in progress]

cf_gateway_rule_create.js

@@ -0,0 +1,65 @@
+require("dotenv").config();
+const axios = require('axios');
+
+const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
+const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
+const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
+
+// Function to read Cloudflare Zero Trust lists
+async function getZeroTrustLists() {
+  const response = await axios.get(
+    `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`,
+    {
+      headers: {
+        'Authorization': `Bearer ${API_TOKEN}`,
+        'Content-Type': 'application/json',
+        'X-Auth-Email': ACCOUNT_EMAIL,
+        'X-Auth-Key': API_TOKEN,
+      },
+    }
+  );
+
+  return response.data.result;
+}
+
+;(async() => {
+    const lists = await getZeroTrustLists();
+    const filtered_lists = lists.filter(list => list.name.startsWith('CGPS List'));
+
+    let wirefilter_expression = '';
+
+    // Build the wirefilter expression
+    for (const list of filtered_lists) {
+        wirefilter_expression += `dns.fqdn in \$${list.id} or `;
+    }
+    // Remove the trailing ' or '
+    if (wirefilter_expression.endsWith(' or ')) {
+        wirefilter_expression = wirefilter_expression.slice(0, -4);
+    }
+    wirefilter_expression = wirefilter_expression.trim().replace('\n', '');
+    console.log(`Firewall expression contains ${wirefilter_expression.length} characters, and checks against ${filtered_lists.length} filter lists.`)
+
+    const resp = await axios.request({
+        method: 'POST',
+        url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`,
+        headers: {
+            'Authorization': `Bearer ${API_TOKEN}`,
+            'Content-Type': 'application/json',
+            'X-Auth-Email': ACCOUNT_EMAIL,
+            'X-Auth-Key': API_TOKEN,
+        },
+        data: {
+            "name": "CGPS Filter Lists",
+            "description": "Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.",
+            "enabled": true,
+            "action": "block",
+            "filters": ["dns"],
+            "traffic": wirefilter_expression,
+        }
+    });
+    console.log('Success:', resp.data.success);
+})();
+
+async function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

cf_gateway_rule_delete.js

@@ -0,0 +1,50 @@
+require("dotenv").config();
+const axios = require('axios');
+
+const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
+const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
+const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
+
+// Function to read Cloudflare Zero Trust rules
+async function getZeroTrustRules() {
+  const response = await axios.get(
+    `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`,
+    {
+      headers: {
+        'Authorization': `Bearer ${API_TOKEN}`,
+        'Content-Type': 'application/json',
+        'X-Auth-Email': ACCOUNT_EMAIL,
+        'X-Auth-Key': API_TOKEN,
+      },
+    }
+  );
+
+  return response.data.result;
+}
+
+;(async() => {
+    const rules = await getZeroTrustRules();
+    const [filtered_rule] = rules.filter(rule => rule.name === "CGPS Filter Lists");
+
+    if (!filtered_rule) return console.warn("No rule with matching name found - this is not an issue if you haven't run the create script yet. Exiting.");
+
+    console.log(`Deleting rule ${filtered_rule.name} with ID ${filtered_rule.id}`);
+
+    const resp = await axios.request({
+        method: 'DELETE',
+        url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules/${filtered_rule.id}`,
+        headers: {
+            'Authorization': `Bearer ${API_TOKEN}`,
+            'Content-Type': 'application/json',
+            'X-Auth-Email': ACCOUNT_EMAIL,
+            'X-Auth-Key': API_TOKEN,
+        },
+    });
+
+    console.log('Success: ', resp.data.success);
+    await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe
+})();
+
+async function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

cf_list_create.js

@@ -0,0 +1,132 @@
+require("dotenv").config();
+const fs = require('fs');
+const axios = require('axios');
+
+const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
+const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
+const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
+const LIST_ITEM_LIMIT = Number.isSafeInteger(Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT)) ? Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT) : 300000;
+
+console.log(`List item limit set to ${LIST_ITEM_LIMIT}`);
+
+// Read input.csv and parse domains
+fs.readFile('input.csv', 'utf8', async (err, data) => {
+  if (err) {
+    console.error('Error reading input.csv:', err);
+    return;
+  }
+
+  // Convert into array and cleanup input
+  const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
+  let domains = data.split('\n').filter(domain => {
+    // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
+    return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
+  }).map(domain => {
+    // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
+    return domain
+      .replace('\r', '')
+      .replace('0.0.0.0 ', '')
+      .replace('127.0.0.1 ', '')
+      .replace('::1 ', '')
+      .replace(':: ', '');
+  }).filter(domain => {
+    return domainValidationPattern.test(domain);
+  });
+  
+  // Trim array to 300,000 domains if it's longer than that
+  if (domains.length > LIST_ITEM_LIMIT) {
+    domains = trimArray(domains, LIST_ITEM_LIMIT);
+    console.warn(`More than ${LIST_ITEM_LIMIT} domains found in input.csv - input has to be trimmed`);
+  }
+
+  // Check for duplicates in domains array
+  let uniqueDomains = [];
+  let seen = new Set(); // Use a set to store seen values
+  for (let domain of domains) {
+    if (!seen.has(domain)) { // If the domain is not in the set
+      seen.add(domain); // Add it to the set
+      uniqueDomains.push(domain); // Push the domain to the uniqueDomains array
+    } else { // If the domain is in the set
+      console.warn(`Duplicate domain found: ${domain} - removing`); // Log the duplicate domain
+    }
+  }
+
+  // Replace domains array with uniqueDomains array
+  domains = uniqueDomains;
+
+  const listsToCreate = Math.ceil(domains.length / 1000);
+
+  console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`);
+
+  // Separate domains into chunks of 1000 (Cloudflare list cap)
+  const chunks = chunkArray(domains, 1000);
+
+  // Create Cloudflare Zero Trust lists
+  for (const [index, chunk] of chunks.entries()) {
+    const listName = `CGPS List - Chunk ${index}`;
+
+    let properList = [];
+
+    chunk.forEach(domain => {
+        properList.push({ "value": domain })
+    });
+
+    try {
+      await createZeroTrustList(listName, properList, (index+1), listsToCreate);
+      await sleep(350); // Sleep for 350ms between list additions
+    } catch (error) {
+      console.error(`Error creating list "${listName}":`, error.response.data);
+    }
+  }
+});
+
+function trimArray(arr, size) {
+  return arr.slice(0, size);
+}
+
+// Function to check if a domain is valid
+function isValidDomain(domain) {
+    const regex = /^((?!-)[A-Za-z0-9-]{1,63}(?<!-)\.)+[A-Za-z]{2,6}$/;
+    return regex.test(domain);
+}
+
+// Function to split an array into chunks
+function chunkArray(array, chunkSize) {
+  const chunks = [];
+  for (let i = 0; i < array.length; i += chunkSize) {
+    chunks.push(array.slice(i, i + chunkSize));
+  }
+  return chunks;
+}
+
+// Function to create a Cloudflare Zero Trust list
+async function createZeroTrustList(name, items, currentItem, totalItems) {
+  const response = await axios.post(
+    `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`,
+    {
+      name,
+      type: 'DOMAIN', // Set list type to DOMAIN
+      items,
+    },
+    {
+      headers: {
+        'Authorization': `Bearer ${API_TOKEN}`,
+        'Content-Type': 'application/json',
+        'X-Auth-Email': ACCOUNT_EMAIL,
+        'X-Auth-Key': API_TOKEN,
+      },
+    }
+  );
+
+  const listId = response.data.result.id;
+  console.log(`Created Zero Trust list "${name}" with ID ${listId} - ${totalItems - currentItem} left`);
+}
+
+function percentage(percent, total) {
+  return Math.round((percent / 100) * total);
+}
+
+// Function to sleep for a specified duration
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

cf_list_delete.js

@@ -0,0 +1,52 @@
+require("dotenv").config();
+const axios = require('axios');
+
+const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
+const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
+const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
+
+// Function to read Cloudflare Zero Trust lists
+async function getZeroTrustLists() {
+  const response = await axios.get(
+    `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`,
+    {
+      headers: {
+        'Authorization': `Bearer ${API_TOKEN}`,
+        'Content-Type': 'application/json',
+        'X-Auth-Email': ACCOUNT_EMAIL,
+        'X-Auth-Key': API_TOKEN,
+      },
+    }
+  );
+
+  return response.data.result;
+}
+
+;(async() => {
+    const lists = await getZeroTrustLists();
+    if (!lists) return console.warn("No file lists found - this is not an issue if it's your first time running this script. Exiting.");
+    const cgps_lists = lists.filter(list => list.name.startsWith('CGPS List'));
+    if (!cgps_lists.length) return console.warn("No lists with matching name found - this is not an issue if you haven't created any filter lists before. Exiting.");
+
+    console.log(`Got ${lists.length} lists, ${cgps_lists.length} of which are CGPS lists that will be deleted.`);
+
+    for (const list of cgps_lists) {
+        console.log(`Deleting list ${list.name} with ID ${list.id}`);
+        const resp = await axios.request({
+            method: 'DELETE',
+            url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists/${list.id}`,
+            headers: {
+                'Authorization': `Bearer ${API_TOKEN}`,
+                'Content-Type': 'application/json',
+                'X-Auth-Email': ACCOUNT_EMAIL,
+                'X-Auth-Key': API_TOKEN,
+              },
+        });
+        console.log('Success:', resp.data.success);
+        await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe
+    }
+})();
+
+async function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

get_recommended_filters.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# create an empty input.csv file
+touch input.csv
+
+# declare an array of urls
+urls=(
+  https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt
+  https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_gambling.txt
+  https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_privacy.txt
+  https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts
+  https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt
+  https://rescure.me/rescure_domain_blacklist.txt
+  https://adaway.org/hosts.txt
+  https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+)
+
+# loop through the urls and download each file with curl
+for url in "${urls[@]}"; do
+  # get the file name from the url
+  file=$(basename "$url")
+  # download the file with curl and save it as file.txt
+  curl -o "$file.txt" "$url"
+  # append the file contents to input.csv and add a newline
+  cat "$file.txt" >> input.csv
+  echo "" >> input.csv
+  # remove the file.txt
+  rm "$file.txt"
+done
+
+# print a message when done
+echo "Done. The input.csv file contains merged data from recommended filter lists."

package-lock.json

@@ -0,0 +1,111 @@
+{
+  "name": "cloudflare-gateway-pihole-scripts",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "dependencies": {
+        "axios": "^1.4.0",
+        "dotenv": "^16.0.3"
+      }
+    },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "node_modules/axios": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
+      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
+      "dependencies": {
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
+      }
+    },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/dotenv": {
+      "version": "16.0.3",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz",
+      "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/follow-redirects": {
+      "version": "1.15.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    }
+  }
+}

package.json

@@ -0,0 +1,6 @@
+{
+  "dependencies": {
+    "axios": "^1.4.0",
+    "dotenv": "^16.0.3"
+  }
+}