mirror of
https://github.com/Nezumi-2711/cloudflare-gateway-pihole-scripts.git
synced 2026-09-22 20:01:04 +00:00
refactored domain processing code
This commit is contained in:
Viet Huynh
+76
-97
@@ -1,115 +1,94 @@
|
||||
import fs from 'fs';
|
||||
import { DRY_RUN, FAST_MODE, LIST_ITEM_LIMIT } from './lib/constants.js';
|
||||
import { createZeroTrustListsAtOnce, createZeroTrustListsOneByOne } from './lib/api.js';
|
||||
import { truncateArray } from './lib/utils.js';
|
||||
import { resolve } from "path";
|
||||
|
||||
if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`);
|
||||
import {
|
||||
createZeroTrustListsAtOnce,
|
||||
createZeroTrustListsOneByOne,
|
||||
} from "./lib/api.js";
|
||||
import {
|
||||
DRY_RUN,
|
||||
FAST_MODE,
|
||||
LIST_ITEM_LIMIT,
|
||||
LIST_ITEM_SIZE,
|
||||
} from "./lib/constants.js";
|
||||
import { normalizeDomain } from "./lib/helpers.js";
|
||||
import { isComment, isValidDomain, readFile } from "./lib/utils.js";
|
||||
|
||||
let whitelist = []; // Define an empty array for the whitelist
|
||||
const allowlistFilename = "whitelist.csv";
|
||||
const blocklistFilename = "input.csv";
|
||||
const allowlist = new Map();
|
||||
const blocklist = new Map();
|
||||
const domains = [];
|
||||
let processedDomainCount = 0;
|
||||
let duplicateDomainCount = 0;
|
||||
|
||||
// Read whitelist.csv and parse
|
||||
fs.readFile('whitelist.csv', 'utf8', async (err, data) => {
|
||||
if (err) {
|
||||
console.warn('Error reading whitelist.csv:', err);
|
||||
console.warn('Assuming whitelist is empty.')
|
||||
} else {
|
||||
// Convert into array and cleanup whitelist
|
||||
const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
|
||||
whitelist = data.split('\n').filter(domain => {
|
||||
// Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
|
||||
return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
|
||||
}).map(domain => {
|
||||
// Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
|
||||
return domain
|
||||
.replace('\r', '')
|
||||
.replace('0.0.0.0 ', '')
|
||||
.replace('127.0.0.1 ', '')
|
||||
.replace('::1 ', '')
|
||||
.replace(':: ', '')
|
||||
.replace('||', '')
|
||||
.replace('@@||', '')
|
||||
.replace('^$important', '')
|
||||
.replace('*.', '')
|
||||
.replace('^', '');
|
||||
}).filter(domain => {
|
||||
return domainValidationPattern.test(domain);
|
||||
});
|
||||
console.log(`Found ${whitelist.length} valid domains in whitelist.`);
|
||||
}
|
||||
// Read allowlist
|
||||
console.log(`Processing ${allowlistFilename}`);
|
||||
await readFile(resolve(allowlistFilename), (line) => {
|
||||
if (isComment(line)) return;
|
||||
|
||||
const domain = normalizeDomain(line, true);
|
||||
|
||||
if (!isValidDomain(domain)) return;
|
||||
|
||||
allowlist.set(domain, 1);
|
||||
});
|
||||
|
||||
|
||||
// Read input.csv and parse domains
|
||||
fs.readFile('input.csv', 'utf8', async (err, data) => {
|
||||
if (err) {
|
||||
console.error('Error reading input.csv:', err);
|
||||
// Read blocklist
|
||||
console.log(`Processing ${blocklistFilename}`);
|
||||
await readFile(resolve(blocklistFilename), (line, rl) => {
|
||||
if (domains.length === LIST_ITEM_LIMIT) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Convert into array and cleanup input
|
||||
const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
|
||||
let domains = data.split('\n').filter(domain => {
|
||||
// Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
|
||||
return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
|
||||
}).map(domain => {
|
||||
// Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
|
||||
return domain
|
||||
.replace('\r', '')
|
||||
.replace('0.0.0.0 ', '')
|
||||
.replace('127.0.0.1 ', '')
|
||||
.replace('::1 ', '')
|
||||
.replace(':: ', '')
|
||||
.replace('^', '')
|
||||
.replace('||', '')
|
||||
.replace('@@||', '')
|
||||
.replace('^$important', '')
|
||||
.replace('*.', '')
|
||||
.replace('^', '');
|
||||
}).filter(domain => {
|
||||
return domainValidationPattern.test(domain);
|
||||
});
|
||||
if (isComment(line)) return;
|
||||
|
||||
// Check for duplicates in domains array
|
||||
let duplicateDomainCount = 0;
|
||||
let uniqueDomains = [];
|
||||
let seen = new Set(); // Use a set to store seen values
|
||||
for (let domain of domains) {
|
||||
if (!seen.has(domain)) { // If the domain is not in the set
|
||||
seen.add(domain); // Add it to the set
|
||||
uniqueDomains.push(domain); // Push the domain to the uniqueDomains array
|
||||
} else { // If the domain is in the set
|
||||
duplicateDomainCount++; // Increment the duplicateDomainCount
|
||||
}
|
||||
}
|
||||
if (duplicateDomainCount > 0) console.warn(`Found ${duplicateDomainCount} duplicate domains in input.csv - removing`);
|
||||
const domain = normalizeDomain(line);
|
||||
|
||||
// Replace domains array with uniqueDomains array
|
||||
domains = uniqueDomains;
|
||||
if (!isValidDomain(domain)) return;
|
||||
|
||||
// Remove domains from the domains array that are present in the whitelist array
|
||||
let whitelistedDomainCount = 0;
|
||||
domains = domains.filter(domain => {
|
||||
if (whitelist.includes(domain)) {
|
||||
whitelistedDomainCount++;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
if (whitelistedDomainCount > 0) console.warn(`Found ${whitelistedDomainCount} domains in input.csv that are present in the whitelist - removing them`);
|
||||
processedDomainCount++;
|
||||
|
||||
// Trim array to 300,000 domains if it's longer than that
|
||||
if (domains.length > LIST_ITEM_LIMIT) {
|
||||
console.warn(`${domains.length} domains found in input.csv - input has to be trimmed to ${LIST_ITEM_LIMIT} domains`);
|
||||
domains = truncateArray(domains, LIST_ITEM_LIMIT);
|
||||
if (blocklist.has(domain)) {
|
||||
console.log(`Found ${domain} in blocklist already - Skipping...`);
|
||||
duplicateDomainCount++;
|
||||
return;
|
||||
}
|
||||
|
||||
const listsToCreate = Math.ceil(domains.length / 1000);
|
||||
if (allowlist.has(domain)) {
|
||||
console.log(`Found ${domain} in allowlist - Skipping...`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!process.env.CI) console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`);
|
||||
blocklist.set(domain, 1);
|
||||
domains.push(domain);
|
||||
|
||||
// If we are dry-running, stop here because we don't want to create lists
|
||||
// TODO: we should probably continue, just without making any real requests to Cloudflare
|
||||
if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.');
|
||||
if (domains.length === LIST_ITEM_LIMIT) {
|
||||
console.log(
|
||||
"Maximum number of blocked domains reached - Stopping processing blocklist..."
|
||||
);
|
||||
rl.close();
|
||||
}
|
||||
});
|
||||
|
||||
console.log("\n\n");
|
||||
console.log(`Number of processed domains: ${processedDomainCount}`);
|
||||
console.log(`Number of blocked domains: ${domains.length}`);
|
||||
console.log(`Number of allowed domains: ${allowlist.size}`);
|
||||
console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
|
||||
console.log(
|
||||
`Number of lists which will be created: ${Math.ceil(
|
||||
domains.length / LIST_ITEM_SIZE
|
||||
)}`
|
||||
);
|
||||
console.log("\n\n");
|
||||
|
||||
(async () => {
|
||||
if (DRY_RUN) {
|
||||
console.log(
|
||||
"Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again."
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (FAST_MODE) {
|
||||
await createZeroTrustListsAtOnce(domains);
|
||||
@@ -117,4 +96,4 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
|
||||
}
|
||||
|
||||
await createZeroTrustListsOneByOne(domains);
|
||||
});
|
||||
})();
|
||||
|
||||
@@ -40,3 +40,22 @@ const request = async (url, options) => {
|
||||
*/
|
||||
export const requestGateway = (path, options) =>
|
||||
request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway${path}`, options);
|
||||
|
||||
/**
|
||||
* Normalizes a domain.
|
||||
* @param {string} value The value to be normalized.
|
||||
* @param {boolean} isAllowlisting Whether the value is to be whitelisted.
|
||||
* @returns {string}
|
||||
*/
|
||||
export const normalizeDomain = (value, isAllowlisting) => {
|
||||
const normalized = value
|
||||
.replace(/(0\.0\.0\.0|127\.0\.0\.1|::1|::)\s+/, "")
|
||||
.replace("||", "")
|
||||
.replace("^$important", "")
|
||||
.replace("*.", "")
|
||||
.replace("^", "");
|
||||
|
||||
if (isAllowlisting) return normalized.replace("@@||", "");
|
||||
|
||||
return normalized;
|
||||
};
|
||||
|
||||
+48
-5
@@ -1,3 +1,8 @@
|
||||
import { once } from "events";
|
||||
import { createReadStream } from "fs";
|
||||
import { basename } from "path";
|
||||
import { createInterface } from "readline";
|
||||
|
||||
/**
|
||||
* Sleeps for a specified amount of time.
|
||||
* @param {number} [ms=350] The amount of time in ms.
|
||||
@@ -6,9 +11,47 @@ export const sleep = (ms = 350) =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Truncates an array to the specified size.
|
||||
* @param {any[]} arr The array to be truncated.
|
||||
* @param {number} size The size to which the array will be truncated.
|
||||
* @returns {any[]}
|
||||
* Checks if the value is a valid domain.
|
||||
* @param {string} value The value to be checked.
|
||||
*/
|
||||
export const truncateArray = (arr, size) => arr.slice(0, size);
|
||||
export const isValidDomain = (value) =>
|
||||
/^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/.test(value);
|
||||
|
||||
/**
|
||||
* Checks if the value is a comment.
|
||||
* @param {string} value The value to be checked.
|
||||
*/
|
||||
export const isComment = (value) =>
|
||||
value.startsWith("#") ||
|
||||
value.startsWith("//") ||
|
||||
value.startsWith("!") ||
|
||||
value.startsWith("/*") ||
|
||||
value.startsWith("*/");
|
||||
|
||||
/**
|
||||
* @callback onLine
|
||||
* @param {string} line The current line.
|
||||
* @param {ReturnType<createInterface>} rl The readline interface.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Asynchronously reads a file line by line.
|
||||
* @param {string} filePath The path to the file.
|
||||
* @param {onLine} onLine The callback executed on each line read.
|
||||
*/
|
||||
export const readFile = async (filePath, onLine) => {
|
||||
try {
|
||||
const rl = createInterface({
|
||||
input: createReadStream(filePath),
|
||||
crlfDelay: Infinity,
|
||||
});
|
||||
|
||||
rl.on("line", (line) => onLine(line, rl));
|
||||
|
||||
await once(rl, "close");
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`Error occurred while reading ${basename(filePath)} - ${err.toString()}`
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user