From 40772b567d17df95084d02400aab68000fde5e40 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Tue, 12 Sep 2023 16:35:16 +0700 Subject: [PATCH] refactored domain processing code --- cf_list_create.js | 173 ++++++++++++++++++++-------------------------- lib/helpers.js | 19 +++++ lib/utils.js | 53 ++++++++++++-- 3 files changed, 143 insertions(+), 102 deletions(-) diff --git a/cf_list_create.js b/cf_list_create.js index 1ea61f6..28d451a 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -1,115 +1,94 @@ -import fs from 'fs'; -import { DRY_RUN, FAST_MODE, LIST_ITEM_LIMIT } from './lib/constants.js'; -import { createZeroTrustListsAtOnce, createZeroTrustListsOneByOne } from './lib/api.js'; -import { truncateArray } from './lib/utils.js'; +import { resolve } from "path"; -if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`); +import { + createZeroTrustListsAtOnce, + createZeroTrustListsOneByOne, +} from "./lib/api.js"; +import { + DRY_RUN, + FAST_MODE, + LIST_ITEM_LIMIT, + LIST_ITEM_SIZE, +} from "./lib/constants.js"; +import { normalizeDomain } from "./lib/helpers.js"; +import { isComment, isValidDomain, readFile } from "./lib/utils.js"; -let whitelist = []; // Define an empty array for the whitelist +const allowlistFilename = "whitelist.csv"; +const blocklistFilename = "input.csv"; +const allowlist = new Map(); +const blocklist = new Map(); +const domains = []; +let processedDomainCount = 0; +let duplicateDomainCount = 0; -// Read whitelist.csv and parse -fs.readFile('whitelist.csv', 'utf8', async (err, data) => { - if (err) { - console.warn('Error reading whitelist.csv:', err); - console.warn('Assuming whitelist is empty.') - } else { - // Convert into array and cleanup whitelist - const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/; - whitelist = data.split('\n').filter(domain => { - // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments - return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r'); - }).map(domain => { - // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items - return domain - .replace('\r', '') - .replace('0.0.0.0 ', '') - .replace('127.0.0.1 ', '') - .replace('::1 ', '') - .replace(':: ', '') - .replace('||', '') - .replace('@@||', '') - .replace('^$important', '') - .replace('*.', '') - .replace('^', ''); - }).filter(domain => { - return domainValidationPattern.test(domain); - }); - console.log(`Found ${whitelist.length} valid domains in whitelist.`); - } +// Read allowlist +console.log(`Processing ${allowlistFilename}`); +await readFile(resolve(allowlistFilename), (line) => { + if (isComment(line)) return; + + const domain = normalizeDomain(line, true); + + if (!isValidDomain(domain)) return; + + allowlist.set(domain, 1); }); - -// Read input.csv and parse domains -fs.readFile('input.csv', 'utf8', async (err, data) => { - if (err) { - console.error('Error reading input.csv:', err); +// Read blocklist +console.log(`Processing ${blocklistFilename}`); +await readFile(resolve(blocklistFilename), (line, rl) => { + if (domains.length === LIST_ITEM_LIMIT) { return; } - // Convert into array and cleanup input - const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/; - let domains = data.split('\n').filter(domain => { - // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments - return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r'); - }).map(domain => { - // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items - return domain - .replace('\r', '') - .replace('0.0.0.0 ', '') - .replace('127.0.0.1 ', '') - .replace('::1 ', '') - .replace(':: ', '') - .replace('^', '') - .replace('||', '') - .replace('@@||', '') - .replace('^$important', '') - .replace('*.', '') - .replace('^', ''); - }).filter(domain => { - return domainValidationPattern.test(domain); - }); + if (isComment(line)) return; - // Check for duplicates in domains array - let duplicateDomainCount = 0; - let uniqueDomains = []; - let seen = new Set(); // Use a set to store seen values - for (let domain of domains) { - if (!seen.has(domain)) { // If the domain is not in the set - seen.add(domain); // Add it to the set - uniqueDomains.push(domain); // Push the domain to the uniqueDomains array - } else { // If the domain is in the set - duplicateDomainCount++; // Increment the duplicateDomainCount - } - } - if (duplicateDomainCount > 0) console.warn(`Found ${duplicateDomainCount} duplicate domains in input.csv - removing`); + const domain = normalizeDomain(line); - // Replace domains array with uniqueDomains array - domains = uniqueDomains; + if (!isValidDomain(domain)) return; - // Remove domains from the domains array that are present in the whitelist array - let whitelistedDomainCount = 0; - domains = domains.filter(domain => { - if (whitelist.includes(domain)) { - whitelistedDomainCount++; - return false; - } - return true; - }); - if (whitelistedDomainCount > 0) console.warn(`Found ${whitelistedDomainCount} domains in input.csv that are present in the whitelist - removing them`); + processedDomainCount++; - // Trim array to 300,000 domains if it's longer than that - if (domains.length > LIST_ITEM_LIMIT) { - console.warn(`${domains.length} domains found in input.csv - input has to be trimmed to ${LIST_ITEM_LIMIT} domains`); - domains = truncateArray(domains, LIST_ITEM_LIMIT); + if (blocklist.has(domain)) { + console.log(`Found ${domain} in blocklist already - Skipping...`); + duplicateDomainCount++; + return; } - const listsToCreate = Math.ceil(domains.length / 1000); + if (allowlist.has(domain)) { + console.log(`Found ${domain} in allowlist - Skipping...`); + return; + } - if (!process.env.CI) console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`); + blocklist.set(domain, 1); + domains.push(domain); - // If we are dry-running, stop here because we don't want to create lists - // TODO: we should probably continue, just without making any real requests to Cloudflare - if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.'); + if (domains.length === LIST_ITEM_LIMIT) { + console.log( + "Maximum number of blocked domains reached - Stopping processing blocklist..." + ); + rl.close(); + } +}); + +console.log("\n\n"); +console.log(`Number of processed domains: ${processedDomainCount}`); +console.log(`Number of blocked domains: ${domains.length}`); +console.log(`Number of allowed domains: ${allowlist.size}`); +console.log(`Number of duplicate domains: ${duplicateDomainCount}`); +console.log( + `Number of lists which will be created: ${Math.ceil( + domains.length / LIST_ITEM_SIZE + )}` +); +console.log("\n\n"); + +(async () => { + if (DRY_RUN) { + console.log( + "Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again." + ); + return; + } if (FAST_MODE) { await createZeroTrustListsAtOnce(domains); @@ -117,4 +96,4 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { } await createZeroTrustListsOneByOne(domains); -}); +})(); diff --git a/lib/helpers.js b/lib/helpers.js index 40c7b14..707c037 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -40,3 +40,22 @@ const request = async (url, options) => { */ export const requestGateway = (path, options) => request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway${path}`, options); + +/** + * Normalizes a domain. + * @param {string} value The value to be normalized. + * @param {boolean} isAllowlisting Whether the value is to be whitelisted. + * @returns {string} + */ +export const normalizeDomain = (value, isAllowlisting) => { + const normalized = value + .replace(/(0\.0\.0\.0|127\.0\.0\.1|::1|::)\s+/, "") + .replace("||", "") + .replace("^$important", "") + .replace("*.", "") + .replace("^", ""); + + if (isAllowlisting) return normalized.replace("@@||", ""); + + return normalized; +}; diff --git a/lib/utils.js b/lib/utils.js index 1d13e0c..ade1427 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,3 +1,8 @@ +import { once } from "events"; +import { createReadStream } from "fs"; +import { basename } from "path"; +import { createInterface } from "readline"; + /** * Sleeps for a specified amount of time. * @param {number} [ms=350] The amount of time in ms. @@ -6,9 +11,47 @@ export const sleep = (ms = 350) => new Promise((resolve) => setTimeout(resolve, ms)); /** - * Truncates an array to the specified size. - * @param {any[]} arr The array to be truncated. - * @param {number} size The size to which the array will be truncated. - * @returns {any[]} + * Checks if the value is a valid domain. + * @param {string} value The value to be checked. */ -export const truncateArray = (arr, size) => arr.slice(0, size); +export const isValidDomain = (value) => + /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/.test(value); + +/** + * Checks if the value is a comment. + * @param {string} value The value to be checked. + */ +export const isComment = (value) => + value.startsWith("#") || + value.startsWith("//") || + value.startsWith("!") || + value.startsWith("/*") || + value.startsWith("*/"); + +/** + * @callback onLine + * @param {string} line The current line. + * @param {ReturnType} rl The readline interface. + */ + +/** + * Asynchronously reads a file line by line. + * @param {string} filePath The path to the file. + * @param {onLine} onLine The callback executed on each line read. + */ +export const readFile = async (filePath, onLine) => { + try { + const rl = createInterface({ + input: createReadStream(filePath), + crlfDelay: Infinity, + }); + + rl.on("line", (line) => onLine(line, rl)); + + await once(rl, "close"); + } catch (err) { + console.error( + `Error occurred while reading ${basename(filePath)} - ${err.toString()}` + ); + } +};