refactored domain processing code

cf_list_create.js

@@ -1,115 +1,94 @@
-import fs from 'fs';
+import { resolve } from "path";
-import { DRY_RUN, FAST_MODE, LIST_ITEM_LIMIT } from './lib/constants.js';
-import { createZeroTrustListsAtOnce, createZeroTrustListsOneByOne } from './lib/api.js';
-import { truncateArray } from './lib/utils.js';
 
-if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`);
+import {
+  createZeroTrustListsAtOnce,
+  createZeroTrustListsOneByOne,
+} from "./lib/api.js";
+import {
+  DRY_RUN,
+  FAST_MODE,
+  LIST_ITEM_LIMIT,
+  LIST_ITEM_SIZE,
+} from "./lib/constants.js";
+import { normalizeDomain } from "./lib/helpers.js";
+import { isComment, isValidDomain, readFile } from "./lib/utils.js";
 
-let whitelist = []; // Define an empty array for the whitelist
+const allowlistFilename = "whitelist.csv";
+const blocklistFilename = "input.csv";
+const allowlist = new Map();
+const blocklist = new Map();
+const domains = [];
+let processedDomainCount = 0;
+let duplicateDomainCount = 0;
 
-// Read whitelist.csv and parse
+// Read allowlist
-fs.readFile('whitelist.csv', 'utf8', async (err, data) => {
+console.log(`Processing ${allowlistFilename}`);
-  if (err) {
+await readFile(resolve(allowlistFilename), (line) => {
-    console.warn('Error reading whitelist.csv:', err);
+  if (isComment(line)) return;
-    console.warn('Assuming whitelist is empty.')
+
-  } else {
+  const domain = normalizeDomain(line, true);
-    // Convert into array and cleanup whitelist
+
-    const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
+  if (!isValidDomain(domain)) return;
-    whitelist = data.split('\n').filter(domain => {
+
-      // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
+  allowlist.set(domain, 1);
-      return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
-    }).map(domain => {
-      // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
-      return domain
-        .replace('\r', '')
-        .replace('0.0.0.0 ', '')
-        .replace('127.0.0.1 ', '')
-        .replace('::1 ', '')
-        .replace(':: ', '')
-        .replace('||', '')
-        .replace('@@||', '')
-        .replace('^$important', '')
-        .replace('*.', '')
-        .replace('^', '');
-    }).filter(domain => {
-      return domainValidationPattern.test(domain);
-    });
-    console.log(`Found ${whitelist.length} valid domains in whitelist.`);
-  }  
 });
 
-
+// Read blocklist
-// Read input.csv and parse domains
+console.log(`Processing ${blocklistFilename}`);
-fs.readFile('input.csv', 'utf8', async (err, data) => {
+await readFile(resolve(blocklistFilename), (line, rl) => {
-  if (err) {
+  if (domains.length === LIST_ITEM_LIMIT) {
-    console.error('Error reading input.csv:', err);
     return;
   }
 
-  // Convert into array and cleanup input
+  if (isComment(line)) return;
-  const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
-  let domains = data.split('\n').filter(domain => {
-    // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
-    return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
-  }).map(domain => {
-    // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
-    return domain
-      .replace('\r', '')
-      .replace('0.0.0.0 ', '')
-      .replace('127.0.0.1 ', '')
-      .replace('::1 ', '')
-      .replace(':: ', '')
-      .replace('^', '')
-      .replace('||', '')
-      .replace('@@||', '')
-      .replace('^$important', '')
-      .replace('*.', '')
-      .replace('^', '');
-  }).filter(domain => {
-    return domainValidationPattern.test(domain);
-  });
 
-  // Check for duplicates in domains array
+  const domain = normalizeDomain(line);
-  let duplicateDomainCount = 0;
-  let uniqueDomains = [];
-  let seen = new Set(); // Use a set to store seen values
-  for (let domain of domains) {
-    if (!seen.has(domain)) { // If the domain is not in the set
-      seen.add(domain); // Add it to the set
-      uniqueDomains.push(domain); // Push the domain to the uniqueDomains array
-    } else { // If the domain is in the set
-      duplicateDomainCount++; // Increment the duplicateDomainCount
-    }
-  }
-  if (duplicateDomainCount > 0) console.warn(`Found ${duplicateDomainCount} duplicate domains in input.csv - removing`);
 
-  // Replace domains array with uniqueDomains array
+  if (!isValidDomain(domain)) return;
-  domains = uniqueDomains;
 
-  // Remove domains from the domains array that are present in the whitelist array
+  processedDomainCount++;
-  let whitelistedDomainCount = 0;
-  domains = domains.filter(domain => {
-    if (whitelist.includes(domain)) {
-      whitelistedDomainCount++;
-      return false;
-    }
-    return true;
-  });
-  if (whitelistedDomainCount > 0) console.warn(`Found ${whitelistedDomainCount} domains in input.csv that are present in the whitelist - removing them`);
 
-  // Trim array to 300,000 domains if it's longer than that
+  if (blocklist.has(domain)) {
-  if (domains.length > LIST_ITEM_LIMIT) {
+    console.log(`Found ${domain} in blocklist already - Skipping...`);
-    console.warn(`${domains.length} domains found in input.csv - input has to be trimmed to ${LIST_ITEM_LIMIT} domains`);
+    duplicateDomainCount++;
-    domains = truncateArray(domains, LIST_ITEM_LIMIT);
+    return;
   }
 
-  const listsToCreate = Math.ceil(domains.length / 1000);
+  if (allowlist.has(domain)) {
+    console.log(`Found ${domain} in allowlist - Skipping...`);
+    return;
+  }
 
-  if (!process.env.CI) console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`);
+  blocklist.set(domain, 1);
+  domains.push(domain);
 
-  // If we are dry-running, stop here because we don't want to create lists
+  if (domains.length === LIST_ITEM_LIMIT) {
-  // TODO: we should probably continue, just without making any real requests to Cloudflare
+    console.log(
-  if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.');
+      "Maximum number of blocked domains reached - Stopping processing blocklist..."
+    );
+    rl.close();
+  }
+});
+
+console.log("\n\n");
+console.log(`Number of processed domains: ${processedDomainCount}`);
+console.log(`Number of blocked domains: ${domains.length}`);
+console.log(`Number of allowed domains: ${allowlist.size}`);
+console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
+console.log(
+  `Number of lists which will be created: ${Math.ceil(
+    domains.length / LIST_ITEM_SIZE
+  )}`
+);
+console.log("\n\n");
+
+(async () => {
+  if (DRY_RUN) {
+    console.log(
+      "Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again."
+    );
+    return;
+  }
 
   if (FAST_MODE) {
     await createZeroTrustListsAtOnce(domains);
@@ -117,4 +96,4 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
   }
 
   await createZeroTrustListsOneByOne(domains);
-});
+})();

lib/helpers.js

@@ -40,3 +40,22 @@ const request = async (url, options) => {
  */
 export const requestGateway = (path, options) =>
   request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway${path}`, options);
+
+/**
+ * Normalizes a domain.
+ * @param {string} value The value to be normalized.
+ * @param {boolean} isAllowlisting Whether the value is to be whitelisted.
+ * @returns {string}
+ */
+export const normalizeDomain = (value, isAllowlisting) => {
+  const normalized = value
+    .replace(/(0\.0\.0\.0|127\.0\.0\.1|::1|::)\s+/, "")
+    .replace("||", "")
+    .replace("^$important", "")
+    .replace("*.", "")
+    .replace("^", "");
+
+  if (isAllowlisting) return normalized.replace("@@||", "");
+
+  return normalized;
+};

lib/utils.js

@@ -1,3 +1,8 @@
+import { once } from "events";
+import { createReadStream } from "fs";
+import { basename } from "path";
+import { createInterface } from "readline";
+
 /**
  * Sleeps for a specified amount of time.
  * @param {number} [ms=350] The amount of time in ms.
@@ -6,9 +11,47 @@ export const sleep = (ms = 350) =>
   new Promise((resolve) => setTimeout(resolve, ms));
 
 /**
- * Truncates an array to the specified size.
+ * Checks if the value is a valid domain.
- * @param {any[]} arr The array to be truncated.
+ * @param {string} value The value to be checked.
- * @param {number} size The size to which the array will be truncated.
- * @returns {any[]}
  */
-export const truncateArray = (arr, size) => arr.slice(0, size);
+export const isValidDomain = (value) =>
+  /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/.test(value);
+
+/**
+ * Checks if the value is a comment.
+ * @param {string} value The value to be checked.
+ */
+export const isComment = (value) =>
+  value.startsWith("#") ||
+  value.startsWith("//") ||
+  value.startsWith("!") ||
+  value.startsWith("/*") ||
+  value.startsWith("*/");
+
+/**
+ * @callback onLine
+ * @param {string} line The current line.
+ * @param {ReturnType<createInterface>} rl The readline interface.
+ */
+
+/**
+ * Asynchronously reads a file line by line.
+ * @param {string} filePath The path to the file.
+ * @param {onLine} onLine The callback executed on each line read.
+ */
+export const readFile = async (filePath, onLine) => {
+  try {
+    const rl = createInterface({
+      input: createReadStream(filePath),
+      crlfDelay: Infinity,
+    });
+
+    rl.on("line", (line) => onLine(line, rl));
+
+    await once(rl, "close");
+  } catch (err) {
+    console.error(
+      `Error occurred while reading ${basename(filePath)} - ${err.toString()}`
+    );
+  }
+};