diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b54a472 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +CLOUDFLARE_API_KEY="" +CLOUDFLARE_ACCOUNT_ID="" +CLOUDFLARE_ACCOUNT_EMAIL="" +CLOUDFLARE_LIST_ITEM_LIMIT="300000" \ No newline at end of file diff --git a/.github/images/gateway_analytics.png b/.github/images/gateway_analytics.png new file mode 100644 index 0000000..f7ab1c8 Binary files /dev/null and b/.github/images/gateway_analytics.png differ diff --git a/.gitignore b/.gitignore index 20b0c92..7b2a616 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ yarn-error.log* lerna-debug.log* .pnpm-debug.log* +# CGPS +output +input.csv + # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json diff --git a/README.md b/README.md index 349773d..9743ecc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,45 @@ -# cloudflare-gateway-pihole-scripts - Use Cloudflare Gateway VPN to block ads and tracking domains - free alternative to NextDNS, Pi-hole and Adguard +# Cloudflare Gateway Pi-hole Scripts (CGPS) + +![Cloudflare Gateway Analytics screenshot](.github/images/gateway_analytics.png) + +Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and network traffic based on your firewall policies. This is a collection of scripts that can be used to get a similar experience as if you were using Pi-hole, but with Cloudflare Gateway - so no servers to maintain or need to buy a Raspberry Pi! + +## About the individual scripts + +- `cf_list_delete.js` - Deletes all lists created by CGPS from Cloudflare Gateway. This is useful for subsequent runs. +- `cf_list_create.js` - Takes an input.csv file containing domains and creates lists in Cloudflare Gateway +- `cf_gateway_rule_create.js` - Creates a Cloudflare Gateway rule to block all traffic if it matches the lists created by CGPS. +- `cf_gateway_rule_delete.js` - Deletes the Cloudflare Gateway rule created by CGPS. Useful for subsequent runs. + +## Features + +- Support for hosts files +- Full support for domain lists +- Automatically cleans up filter lists: removes duplicates, invalid domains, comments and more +- Works fully unattended + +## Usage + +### Prerequisites + +1. Node.js installed on your machine +2. Cloudflare Zero Trust account - the Free plan is enough. Use the Cloudflare documentation for details. +3. Cloudflare email, API key (NOT the API token), and account ID +4. A filter list of domains you want to block - **max 300,000 domains for the free plan** - in the working directory named `input.csv`. Mullvad provides awesome [DNS blocklists](https://github.com/mullvad/dns-blocklists) that work well with this project. + +--- + +1. Clone this repository. +2. Run `npm install` to install dependencies. +3. Copy `.env.example` to `.env` and fill in the values. +4. If this is a subsequent run, execute `cf_gateway_rule_delete.js` and `cf_list_delete.js` (in order) to clean up. +5. If you're on Linux and haven't downloaded any filters yourself, use the `get_recommended_filters.sh` script to download recommended filter lists (about 250 000 domains). +6. Run `cf_list_create.js` to create the lists in Cloudflare Gateway. +7. Run `cf_gateway_rule_create.js` to create the firewall rule in Cloudflare Gateway. +8. Profit! + +### Running in GitHub Actions + +This project can be run in GitHub Actions so your filter lists will be automatically updated and pushed to Cloudflare Gateway. + +[work in progress] diff --git a/cf_gateway_rule_create.js b/cf_gateway_rule_create.js new file mode 100644 index 0000000..c25ff17 --- /dev/null +++ b/cf_gateway_rule_create.js @@ -0,0 +1,65 @@ +require("dotenv").config(); +const axios = require('axios'); + +const API_TOKEN = process.env.CLOUDFLARE_API_KEY; +const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; +const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; + +// Function to read Cloudflare Zero Trust lists +async function getZeroTrustLists() { + const response = await axios.get( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`, + { + headers: { + 'Authorization': `Bearer ${API_TOKEN}`, + 'Content-Type': 'application/json', + 'X-Auth-Email': ACCOUNT_EMAIL, + 'X-Auth-Key': API_TOKEN, + }, + } + ); + + return response.data.result; +} + +;(async() => { + const lists = await getZeroTrustLists(); + const filtered_lists = lists.filter(list => list.name.startsWith('CGPS List')); + + let wirefilter_expression = ''; + + // Build the wirefilter expression + for (const list of filtered_lists) { + wirefilter_expression += `dns.fqdn in \$${list.id} or `; + } + // Remove the trailing ' or ' + if (wirefilter_expression.endsWith(' or ')) { + wirefilter_expression = wirefilter_expression.slice(0, -4); + } + wirefilter_expression = wirefilter_expression.trim().replace('\n', ''); + console.log(`Firewall expression contains ${wirefilter_expression.length} characters, and checks against ${filtered_lists.length} filter lists.`) + + const resp = await axios.request({ + method: 'POST', + url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`, + headers: { + 'Authorization': `Bearer ${API_TOKEN}`, + 'Content-Type': 'application/json', + 'X-Auth-Email': ACCOUNT_EMAIL, + 'X-Auth-Key': API_TOKEN, + }, + data: { + "name": "CGPS Filter Lists", + "description": "Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.", + "enabled": true, + "action": "block", + "filters": ["dns"], + "traffic": wirefilter_expression, + } + }); + console.log('Success:', resp.data.success); +})(); + +async function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} \ No newline at end of file diff --git a/cf_gateway_rule_delete.js b/cf_gateway_rule_delete.js new file mode 100644 index 0000000..97949e5 --- /dev/null +++ b/cf_gateway_rule_delete.js @@ -0,0 +1,50 @@ +require("dotenv").config(); +const axios = require('axios'); + +const API_TOKEN = process.env.CLOUDFLARE_API_KEY; +const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; +const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; + +// Function to read Cloudflare Zero Trust rules +async function getZeroTrustRules() { + const response = await axios.get( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`, + { + headers: { + 'Authorization': `Bearer ${API_TOKEN}`, + 'Content-Type': 'application/json', + 'X-Auth-Email': ACCOUNT_EMAIL, + 'X-Auth-Key': API_TOKEN, + }, + } + ); + + return response.data.result; +} + +;(async() => { + const rules = await getZeroTrustRules(); + const [filtered_rule] = rules.filter(rule => rule.name === "CGPS Filter Lists"); + + if (!filtered_rule) return console.warn("No rule with matching name found - this is not an issue if you haven't run the create script yet. Exiting."); + + console.log(`Deleting rule ${filtered_rule.name} with ID ${filtered_rule.id}`); + + const resp = await axios.request({ + method: 'DELETE', + url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules/${filtered_rule.id}`, + headers: { + 'Authorization': `Bearer ${API_TOKEN}`, + 'Content-Type': 'application/json', + 'X-Auth-Email': ACCOUNT_EMAIL, + 'X-Auth-Key': API_TOKEN, + }, + }); + + console.log('Success: ', resp.data.success); + await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe +})(); + +async function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} \ No newline at end of file diff --git a/cf_list_create.js b/cf_list_create.js new file mode 100644 index 0000000..2e2c179 --- /dev/null +++ b/cf_list_create.js @@ -0,0 +1,132 @@ +require("dotenv").config(); +const fs = require('fs'); +const axios = require('axios'); + +const API_TOKEN = process.env.CLOUDFLARE_API_KEY; +const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; +const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; +const LIST_ITEM_LIMIT = Number.isSafeInteger(Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT)) ? Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT) : 300000; + +console.log(`List item limit set to ${LIST_ITEM_LIMIT}`); + +// Read input.csv and parse domains +fs.readFile('input.csv', 'utf8', async (err, data) => { + if (err) { + console.error('Error reading input.csv:', err); + return; + } + + // Convert into array and cleanup input + const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/; + let domains = data.split('\n').filter(domain => { + // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments + return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r'); + }).map(domain => { + // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items + return domain + .replace('\r', '') + .replace('0.0.0.0 ', '') + .replace('127.0.0.1 ', '') + .replace('::1 ', '') + .replace(':: ', ''); + }).filter(domain => { + return domainValidationPattern.test(domain); + }); + + // Trim array to 300,000 domains if it's longer than that + if (domains.length > LIST_ITEM_LIMIT) { + domains = trimArray(domains, LIST_ITEM_LIMIT); + console.warn(`More than ${LIST_ITEM_LIMIT} domains found in input.csv - input has to be trimmed`); + } + + // Check for duplicates in domains array + let uniqueDomains = []; + let seen = new Set(); // Use a set to store seen values + for (let domain of domains) { + if (!seen.has(domain)) { // If the domain is not in the set + seen.add(domain); // Add it to the set + uniqueDomains.push(domain); // Push the domain to the uniqueDomains array + } else { // If the domain is in the set + console.warn(`Duplicate domain found: ${domain} - removing`); // Log the duplicate domain + } + } + + // Replace domains array with uniqueDomains array + domains = uniqueDomains; + + const listsToCreate = Math.ceil(domains.length / 1000); + + console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`); + + // Separate domains into chunks of 1000 (Cloudflare list cap) + const chunks = chunkArray(domains, 1000); + + // Create Cloudflare Zero Trust lists + for (const [index, chunk] of chunks.entries()) { + const listName = `CGPS List - Chunk ${index}`; + + let properList = []; + + chunk.forEach(domain => { + properList.push({ "value": domain }) + }); + + try { + await createZeroTrustList(listName, properList, (index+1), listsToCreate); + await sleep(350); // Sleep for 350ms between list additions + } catch (error) { + console.error(`Error creating list "${listName}":`, error.response.data); + } + } +}); + +function trimArray(arr, size) { + return arr.slice(0, size); +} + +// Function to check if a domain is valid +function isValidDomain(domain) { + const regex = /^((?!-)[A-Za-z0-9-]{1,63}(? setTimeout(resolve, ms)); +} diff --git a/cf_list_delete.js b/cf_list_delete.js new file mode 100644 index 0000000..111f29b --- /dev/null +++ b/cf_list_delete.js @@ -0,0 +1,52 @@ +require("dotenv").config(); +const axios = require('axios'); + +const API_TOKEN = process.env.CLOUDFLARE_API_KEY; +const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; +const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; + +// Function to read Cloudflare Zero Trust lists +async function getZeroTrustLists() { + const response = await axios.get( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`, + { + headers: { + 'Authorization': `Bearer ${API_TOKEN}`, + 'Content-Type': 'application/json', + 'X-Auth-Email': ACCOUNT_EMAIL, + 'X-Auth-Key': API_TOKEN, + }, + } + ); + + return response.data.result; +} + +;(async() => { + const lists = await getZeroTrustLists(); + if (!lists) return console.warn("No file lists found - this is not an issue if it's your first time running this script. Exiting."); + const cgps_lists = lists.filter(list => list.name.startsWith('CGPS List')); + if (!cgps_lists.length) return console.warn("No lists with matching name found - this is not an issue if you haven't created any filter lists before. Exiting."); + + console.log(`Got ${lists.length} lists, ${cgps_lists.length} of which are CGPS lists that will be deleted.`); + + for (const list of cgps_lists) { + console.log(`Deleting list ${list.name} with ID ${list.id}`); + const resp = await axios.request({ + method: 'DELETE', + url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists/${list.id}`, + headers: { + 'Authorization': `Bearer ${API_TOKEN}`, + 'Content-Type': 'application/json', + 'X-Auth-Email': ACCOUNT_EMAIL, + 'X-Auth-Key': API_TOKEN, + }, + }); + console.log('Success:', resp.data.success); + await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe + } +})(); + +async function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} \ No newline at end of file diff --git a/get_recommended_filters.sh b/get_recommended_filters.sh new file mode 100644 index 0000000..e183f3f --- /dev/null +++ b/get_recommended_filters.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# create an empty input.csv file +touch input.csv + +# declare an array of urls +urls=( + https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt + https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_gambling.txt + https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_privacy.txt + https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts + https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt + https://rescure.me/rescure_domain_blacklist.txt + https://adaway.org/hosts.txt + https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts +) + +# loop through the urls and download each file with curl +for url in "${urls[@]}"; do + # get the file name from the url + file=$(basename "$url") + # download the file with curl and save it as file.txt + curl -o "$file.txt" "$url" + # append the file contents to input.csv and add a newline + cat "$file.txt" >> input.csv + echo "" >> input.csv + # remove the file.txt + rm "$file.txt" +done + +# print a message when done +echo "Done. The input.csv file contains merged data from recommended filter lists." \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..ebd7ac6 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,111 @@ +{ + "name": "cloudflare-gateway-pihole-scripts", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "axios": "^1.4.0", + "dotenv": "^16.0.3" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", + "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dotenv": { + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz", + "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==", + "engines": { + "node": ">=12" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..5265f2e --- /dev/null +++ b/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "axios": "^1.4.0", + "dotenv": "^16.0.3" + } +}