nezumi/cloudflare-gateway-pihole-scripts
1
0
Fork 0
mirror of https://github.com/Nezumi-2711/cloudflare-gateway-pihole-scripts.git synced 2026-09-22 13:38:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into upstream

Browse Source
This commit is contained in:
Việt Huỳnh
2023-09-09 02:39:15 +07:00
committed by GitHub
parent 28d359e02f b667585cbc
commit f7f9001ec5
8 changed files with 192 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -3
View File
@@ -20,7 +20,6 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne
- Whitelist support, allowing you to prevent false positives and breakage by forcing trusted domains to always be unblocked.
- Optional health check: Sends a ping request ensuring continuous monitoring and alerting for the workflow execution.
## Usage
### Prerequisites
@@ -57,7 +56,6 @@ Please note that the GitHub Action downloads the recommended blocklists and whit
- `CLOUDFLARE_LIST_ITEM_LIMIT`: The maximum number of blocked domains allowed for your Cloudflare Zero Trust plan. Use 300000 for the free plan or if you're unsure.
- `PING_URL`: /Optional/ The HTTP(S) URL to ping (using curl) after the GitHub Action has successfully updated your filters. Useful for monitoring.
3. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property.
4. Enable GitHub Actions in your repository settings.
@@ -71,7 +69,7 @@ Alternatively, you can install the Cloudflare WARP client and log in to Zero Tru
### Dry runs
To see if e.g. your filter lists are valid without actually changing anything in your Cloudflare account, you can set the `DRY_RUN` environment variable to "true", either in `.env` or the regular way. This will only print info such as the lists that would be created or the amount of duplicate domains to the console.
To see if e.g. your filter lists are valid without actually changing anything in your Cloudflare account, you can set the `DRY_RUN` environment variable to "true" or any value other than empty, either in `.env` or the regular way. This will only print info such as the lists that would be created or the amount of duplicate domains to the console.
**Warning:** This currently only works for `cf_list_create.js`.
cf_gateway_rule_create.js
+25 -13
View File
@@ -1,5 +1,5 @@
require("dotenv").config();
const axios = require('axios');
import 'dotenv/config';
import fetch from 'node-fetch';
const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
@@ -7,19 +7,24 @@ const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
// Function to read Cloudflare Zero Trust lists
async function getZeroTrustLists() {
const response = await axios.get(
`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`,
{
const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
const response = await fetch(url, {
method: 'GET',
headers: {
'Authorization': `Bearer ${API_TOKEN}`,
'Content-Type': 'application/json',
'X-Auth-Email': ACCOUNT_EMAIL,
'X-Auth-Key': API_TOKEN,
},
}
);
});
return response.data.result;
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const data = await response.json();
return data.result;
}
;(async() => {
@@ -39,25 +44,32 @@ async function getZeroTrustLists() {
wirefilter_expression = wirefilter_expression.trim().replace('\n', '');
if (!process.env.CI) console.log(`Firewall expression contains ${wirefilter_expression.length} characters, and checks against ${filtered_lists.length} filter lists.`)
const resp = await axios.request({
const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`;
const response = await fetch(url, {
method: 'POST',
url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`,
headers: {
'Authorization': `Bearer ${API_TOKEN}`,
'Content-Type': 'application/json',
'X-Auth-Email': ACCOUNT_EMAIL,
'X-Auth-Key': API_TOKEN,
},
data: {
body: JSON.stringify({
"name": "CGPS Filter Lists",
"description": "Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.",
"enabled": true,
"action": "block",
"filters": ["dns"],
"traffic": wirefilter_expression,
}
}),
});
console.log('Success:', resp.data.success);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const data = await response.json();
console.log('Success:', data.success);
})();
async function sleep(ms) {
cf_gateway_rule_delete.js
+19 -11
View File
@@ -1,5 +1,5 @@
require("dotenv").config();
const axios = require('axios');
import 'dotenv/config';
import fetch from 'node-fetch';
const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
@@ -7,19 +7,24 @@ const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
// Function to read Cloudflare Zero Trust rules
async function getZeroTrustRules() {
const response = await axios.get(
`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`,
{
const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules`;
const response = await fetch(url, {
method: 'GET',
headers: {
'Authorization': `Bearer ${API_TOKEN}`,
'Content-Type': 'application/json',
'X-Auth-Email': ACCOUNT_EMAIL,
'X-Auth-Key': API_TOKEN,
},
}
);
});
return response.data.result;
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const data = await response.json();
return data.result;
}
;(async() => {
@@ -30,9 +35,10 @@ async function getZeroTrustRules() {
console.log(`Deleting rule`, process.env.CI ? "(redacted, running in CI)" : `${filtered_rule.name} with ID ${filtered_rule.id}`);
const resp = await axios.request({
const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules/${filtered_rule.id}`;
const resp = await fetch(url, {
method: 'DELETE',
url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/rules/${filtered_rule.id}`,
headers: {
'Authorization': `Bearer ${API_TOKEN}`,
'Content-Type': 'application/json',
@@ -41,7 +47,9 @@ async function getZeroTrustRules() {
},
});
console.log('Success: ', resp.data.success);
const data = await resp.json();
console.log('Success: ', data.success);
await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe
})();
cf_list_create.js
+19 -15
View File
@@ -1,6 +1,6 @@
require("dotenv").config();
const fs = require('fs');
const axios = require('axios');
import 'dotenv/config';
import fetch from 'node-fetch';
import fs from 'fs';
const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
@@ -91,13 +91,15 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
domains = uniqueDomains;
// Remove domains from the domains array that are present in the whitelist array
let whitelistedDomainCount = 0;
domains = domains.filter(domain => {
if (whitelist.includes(domain)) {
console.warn(`Domain found in the whitelist: ${domain} - removing`);
whitelistedDomainCount++;
return false;
}
return true;
});
if (whitelistedDomainCount > 0) console.warn(`Found ${whitelistedDomainCount} domains in input.csv that are present in the whitelist - removing them`);
// Trim array to 300,000 domains if it's longer than that
if (domains.length > LIST_ITEM_LIMIT) {
@@ -156,24 +158,26 @@ function chunkArray(array, chunkSize) {
// Function to create a Cloudflare Zero Trust list
async function createZeroTrustList(name, items, currentItem, totalItems) {
const response = await axios.post(
`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`,
{
name,
type: 'DOMAIN', // Set list type to DOMAIN
items,
},
{
const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
const response = await fetch(url, {
method: 'POST',
headers: {
'Authorization': `Bearer ${API_TOKEN}`,
'Content-Type': 'application/json',
'X-Auth-Email': ACCOUNT_EMAIL,
'X-Auth-Key': API_TOKEN,
},
}
);
body: JSON.stringify({
name,
type: 'DOMAIN', // Set list type to DOMAIN
items,
}),
});
const data = await response.json();
const listId = data.result.id;
const listId = response.data.result.id;
console.log(`Created Zero Trust list`, process.env.CI ? "(redacted on CI)" : `"${name}" with ID ${listId} - ${totalItems - currentItem} left`);
}
cf_list_delete.js
+18 -12
View File
@@ -1,5 +1,5 @@
require("dotenv").config();
const axios = require('axios');
import 'dotenv/config';
import fetch from 'node-fetch';
const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
@@ -7,19 +7,20 @@ const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
// Function to read Cloudflare Zero Trust lists
async function getZeroTrustLists() {
const response = await axios.get(
`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`,
{
const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
const response = await fetch(url, {
method: 'GET',
headers: {
'Authorization': `Bearer ${API_TOKEN}`,
'Content-Type': 'application/json',
'X-Auth-Email': ACCOUNT_EMAIL,
'X-Auth-Key': API_TOKEN,
},
}
);
});
return response.data.result;
const data = await response.json();
return data.result;
}
;(async() => {
@@ -30,11 +31,12 @@ async function getZeroTrustLists() {
if (!process.env.CI) console.log(`Got ${lists.length} lists, ${cgps_lists.length} of which are CGPS lists that will be deleted.`);
let lists_processed = 0;
for (const list of cgps_lists) {
console.log(`Deleting list`, process.env.CI ? "(redacted, running in CI)" : `${list.name} with ID ${list.id}`);
const resp = await axios.request({
console.log(`Deleting list`, process.env.CI ? "(info redacted, running in CI)" : `${list.name} with ID ${list.id}, ${cgps_lists.length - lists_processed - 1} left`);
const resp = await fetch(`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists/${list.id}`, {
method: 'DELETE',
url: `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists/${list.id}`,
headers: {
'Authorization': `Bearer ${API_TOKEN}`,
'Content-Type': 'application/json',
@@ -42,7 +44,11 @@ async function getZeroTrustLists() {
'X-Auth-Key': API_TOKEN,
},
});
console.log('Success:', resp.data.success);
const data = await resp.json();
console.log('Success:', data.success);
lists_processed++;
await sleep(350); // Cloudflare API rate limit is 1200 requests per 5 minutes, so we sleep for 350ms to be safe
}
})();
package-lock.json
Generated
+68 -74
View File
@@ -5,42 +5,16 @@
"packages": {
"": {
"dependencies": {
"axios": "^1.4.0",
"dotenv": "^16.0.3"
"dotenv": "^16.0.3",
"node-fetch": "^3.3.2"
}
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
"node_modules/axios": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
"integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
"dependencies": {
"follow-redirects": "^1.15.0",
"form-data": "^4.0.0",
"proxy-from-env": "^1.1.0"
}
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"node_modules/data-uri-to-buffer": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"engines": {
"node": ">=0.4.0"
"node": ">= 12"
}
},
"node_modules/dotenv": {
@@ -51,61 +25,81 @@
"node": ">=12"
}
},
"node_modules/follow-redirects": {
"version": "1.15.2",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
"integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
"node_modules/fetch-blob": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "paypal",
"url": "https://paypal.me/jimmywarting"
}
],
"dependencies": {
"node-domexception": "^1.0.0",
"web-streams-polyfill": "^3.0.3"
},
"engines": {
"node": "^12.20 || >= 14.13"
}
},
"node_modules/formdata-polyfill": {
"version": "4.0.10",
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
"dependencies": {
"fetch-blob": "^3.1.2"
},
"engines": {
"node": ">=12.20.0"
}
},
"node_modules/node-domexception": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "github",
"url": "https://paypal.me/jimmywarting"
}
],
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
"node": ">=10.5.0"
}
},
"node_modules/form-data": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
"node_modules/node-fetch": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
"data-uri-to-buffer": "^4.0.0",
"fetch-blob": "^3.1.4",
"formdata-polyfill": "^4.0.10"
},
"engines": {
"node": ">= 6"
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/node-fetch"
}
},
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"node_modules/web-streams-polyfill": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
"engines": {
"node": ">= 0.6"
"node": ">= 8"
}
},
"node_modules/mime-types": {
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dependencies": {
"mime-db": "1.52.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
}
}
}
package.json
+3 -2
View File
@@ -1,6 +1,7 @@
{
"type": "module",
"dependencies": {
"axios": "^1.4.0",
"dotenv": "^16.0.3"
"dotenv": "^16.0.3",
"node-fetch": "^3.3.2"
}
}