nezumi/cloudflare-gateway-pihole-scripts
1
0
Fork 0
mirror of https://github.com/Nezumi-2711/cloudflare-gateway-pihole-scripts.git synced 2026-09-22 13:38:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactored domain processing code

Browse Source
This commit is contained in:
Viet Huynh
2023-09-12 16:35:16 +07:00
parent e96d0f719a
commit 40772b567d
3 changed files with 143 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cf_list_create.js
+76 -97
View File
@@ -1,115 +1,94 @@
import fs from 'fs';
import { DRY_RUN, FAST_MODE, LIST_ITEM_LIMIT } from './lib/constants.js';
import { createZeroTrustListsAtOnce, createZeroTrustListsOneByOne } from './lib/api.js';
import { truncateArray } from './lib/utils.js';
import { resolve } from "path";
if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`);
import {
createZeroTrustListsAtOnce,
createZeroTrustListsOneByOne,
} from "./lib/api.js";
import {
DRY_RUN,
FAST_MODE,
LIST_ITEM_LIMIT,
LIST_ITEM_SIZE,
} from "./lib/constants.js";
import { normalizeDomain } from "./lib/helpers.js";
import { isComment, isValidDomain, readFile } from "./lib/utils.js";
let whitelist = []; // Define an empty array for the whitelist
const allowlistFilename = "whitelist.csv";
const blocklistFilename = "input.csv";
const allowlist = new Map();
const blocklist = new Map();
const domains = [];
let processedDomainCount = 0;
let duplicateDomainCount = 0;
// Read whitelist.csv and parse
fs.readFile('whitelist.csv', 'utf8', async (err, data) => {
if (err) {
console.warn('Error reading whitelist.csv:', err);
console.warn('Assuming whitelist is empty.')
} else {
// Convert into array and cleanup whitelist
const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
whitelist = data.split('\n').filter(domain => {
// Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
}).map(domain => {
// Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
return domain
.replace('\r', '')
.replace('0.0.0.0 ', '')
.replace('127.0.0.1 ', '')
.replace('::1 ', '')
.replace(':: ', '')
.replace('||', '')
.replace('@@||', '')
.replace('^$important', '')
.replace('*.', '')
.replace('^', '');
}).filter(domain => {
return domainValidationPattern.test(domain);
});
console.log(`Found ${whitelist.length} valid domains in whitelist.`);
}
// Read allowlist
console.log(`Processing ${allowlistFilename}`);
await readFile(resolve(allowlistFilename), (line) => {
if (isComment(line)) return;
const domain = normalizeDomain(line, true);
if (!isValidDomain(domain)) return;
allowlist.set(domain, 1);
});
// Read input.csv and parse domains
fs.readFile('input.csv', 'utf8', async (err, data) => {
if (err) {
console.error('Error reading input.csv:', err);
// Read blocklist
console.log(`Processing ${blocklistFilename}`);
await readFile(resolve(blocklistFilename), (line, rl) => {
if (domains.length === LIST_ITEM_LIMIT) {
return;
}
// Convert into array and cleanup input
const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
let domains = data.split('\n').filter(domain => {
// Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
}).map(domain => {
// Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
return domain
.replace('\r', '')
.replace('0.0.0.0 ', '')
.replace('127.0.0.1 ', '')
.replace('::1 ', '')
.replace(':: ', '')
.replace('^', '')
.replace('||', '')
.replace('@@||', '')
.replace('^$important', '')
.replace('*.', '')
.replace('^', '');
}).filter(domain => {
return domainValidationPattern.test(domain);
});
if (isComment(line)) return;
// Check for duplicates in domains array
let duplicateDomainCount = 0;
let uniqueDomains = [];
let seen = new Set(); // Use a set to store seen values
for (let domain of domains) {
if (!seen.has(domain)) { // If the domain is not in the set
seen.add(domain); // Add it to the set
uniqueDomains.push(domain); // Push the domain to the uniqueDomains array
} else { // If the domain is in the set
duplicateDomainCount++; // Increment the duplicateDomainCount
}
}
if (duplicateDomainCount > 0) console.warn(`Found ${duplicateDomainCount} duplicate domains in input.csv - removing`);
const domain = normalizeDomain(line);
// Replace domains array with uniqueDomains array
domains = uniqueDomains;
if (!isValidDomain(domain)) return;
// Remove domains from the domains array that are present in the whitelist array
let whitelistedDomainCount = 0;
domains = domains.filter(domain => {
if (whitelist.includes(domain)) {
whitelistedDomainCount++;
return false;
}
return true;
});
if (whitelistedDomainCount > 0) console.warn(`Found ${whitelistedDomainCount} domains in input.csv that are present in the whitelist - removing them`);
processedDomainCount++;
// Trim array to 300,000 domains if it's longer than that
if (domains.length > LIST_ITEM_LIMIT) {
console.warn(`${domains.length} domains found in input.csv - input has to be trimmed to ${LIST_ITEM_LIMIT} domains`);
domains = truncateArray(domains, LIST_ITEM_LIMIT);
if (blocklist.has(domain)) {
console.log(`Found ${domain} in blocklist already - Skipping...`);
duplicateDomainCount++;
return;
}
const listsToCreate = Math.ceil(domains.length / 1000);
if (allowlist.has(domain)) {
console.log(`Found ${domain} in allowlist - Skipping...`);
return;
}
if (!process.env.CI) console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`);
blocklist.set(domain, 1);
domains.push(domain);
// If we are dry-running, stop here because we don't want to create lists
// TODO: we should probably continue, just without making any real requests to Cloudflare
if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.');
if (domains.length === LIST_ITEM_LIMIT) {
console.log(
"Maximum number of blocked domains reached - Stopping processing blocklist..."
);
rl.close();
}
});
console.log("\n\n");
console.log(`Number of processed domains: ${processedDomainCount}`);
console.log(`Number of blocked domains: ${domains.length}`);
console.log(`Number of allowed domains: ${allowlist.size}`);
console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
console.log(
`Number of lists which will be created: ${Math.ceil(
domains.length / LIST_ITEM_SIZE
)}`
);
console.log("\n\n");
(async () => {
if (DRY_RUN) {
console.log(
"Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again."
);
return;
}
if (FAST_MODE) {
await createZeroTrustListsAtOnce(domains);
@@ -117,4 +96,4 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
}
await createZeroTrustListsOneByOne(domains);
});
})();