From f103f839e636aa49fa7666d03ed7133b581da7f3 Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Tue, 12 Sep 2023 03:39:15 +0700
Subject: [PATCH 01/10] set NODE_ENV to production

---
 auto_update_github_action.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml
index e36c4ce..5d8b92b 100644
--- a/auto_update_github_action.yml
+++ b/auto_update_github_action.yml
@@ -8,6 +8,9 @@ on:
       - main
   workflow_dispatch:
 
+env:
+  NODE_ENV: production
+
 jobs:
   cgps:
     runs-on: ubuntu-latest

From e96d0f719acd2cbd79a9982152508d8f11518739 Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Tue, 12 Sep 2023 16:28:10 +0700
Subject: [PATCH 02/10] added .editorconfig

---
 .editorconfig | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 .editorconfig

diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 0000000..c6c8b36
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,9 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true

From 40772b567d17df95084d02400aab68000fde5e40 Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Tue, 12 Sep 2023 16:35:16 +0700
Subject: [PATCH 03/10] refactored domain processing code

---
 cf_list_create.js | 173 ++++++++++++++++++++--------------------------
 lib/helpers.js    |  19 +++++
 lib/utils.js      |  53 ++++++++++++--
 3 files changed, 143 insertions(+), 102 deletions(-)

diff --git a/cf_list_create.js b/cf_list_create.js
index 1ea61f6..28d451a 100644
--- a/cf_list_create.js
+++ b/cf_list_create.js
@@ -1,115 +1,94 @@
-import fs from 'fs';
-import { DRY_RUN, FAST_MODE, LIST_ITEM_LIMIT } from './lib/constants.js';
-import { createZeroTrustListsAtOnce, createZeroTrustListsOneByOne } from './lib/api.js';
-import { truncateArray } from './lib/utils.js';
+import { resolve } from "path";
 
-if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`);
+import {
+  createZeroTrustListsAtOnce,
+  createZeroTrustListsOneByOne,
+} from "./lib/api.js";
+import {
+  DRY_RUN,
+  FAST_MODE,
+  LIST_ITEM_LIMIT,
+  LIST_ITEM_SIZE,
+} from "./lib/constants.js";
+import { normalizeDomain } from "./lib/helpers.js";
+import { isComment, isValidDomain, readFile } from "./lib/utils.js";
 
-let whitelist = []; // Define an empty array for the whitelist
+const allowlistFilename = "whitelist.csv";
+const blocklistFilename = "input.csv";
+const allowlist = new Map();
+const blocklist = new Map();
+const domains = [];
+let processedDomainCount = 0;
+let duplicateDomainCount = 0;
 
-// Read whitelist.csv and parse
-fs.readFile('whitelist.csv', 'utf8', async (err, data) => {
-  if (err) {
-    console.warn('Error reading whitelist.csv:', err);
-    console.warn('Assuming whitelist is empty.')
-  } else {
-    // Convert into array and cleanup whitelist
-    const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
-    whitelist = data.split('\n').filter(domain => {
-      // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
-      return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
-    }).map(domain => {
-      // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
-      return domain
-        .replace('\r', '')
-        .replace('0.0.0.0 ', '')
-        .replace('127.0.0.1 ', '')
-        .replace('::1 ', '')
-        .replace(':: ', '')
-        .replace('||', '')
-        .replace('@@||', '')
-        .replace('^$important', '')
-        .replace('*.', '')
-        .replace('^', '');
-    }).filter(domain => {
-      return domainValidationPattern.test(domain);
-    });
-    console.log(`Found ${whitelist.length} valid domains in whitelist.`);
-  }  
+// Read allowlist
+console.log(`Processing ${allowlistFilename}`);
+await readFile(resolve(allowlistFilename), (line) => {
+  if (isComment(line)) return;
+
+  const domain = normalizeDomain(line, true);
+
+  if (!isValidDomain(domain)) return;
+
+  allowlist.set(domain, 1);
 });
 
-
-// Read input.csv and parse domains
-fs.readFile('input.csv', 'utf8', async (err, data) => {
-  if (err) {
-    console.error('Error reading input.csv:', err);
+// Read blocklist
+console.log(`Processing ${blocklistFilename}`);
+await readFile(resolve(blocklistFilename), (line, rl) => {
+  if (domains.length === LIST_ITEM_LIMIT) {
     return;
   }
 
-  // Convert into array and cleanup input
-  const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
-  let domains = data.split('\n').filter(domain => {
-    // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
-    return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
-  }).map(domain => {
-    // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
-    return domain
-      .replace('\r', '')
-      .replace('0.0.0.0 ', '')
-      .replace('127.0.0.1 ', '')
-      .replace('::1 ', '')
-      .replace(':: ', '')
-      .replace('^', '')
-      .replace('||', '')
-      .replace('@@||', '')
-      .replace('^$important', '')
-      .replace('*.', '')
-      .replace('^', '');
-  }).filter(domain => {
-    return domainValidationPattern.test(domain);
-  });
+  if (isComment(line)) return;
 
-  // Check for duplicates in domains array
-  let duplicateDomainCount = 0;
-  let uniqueDomains = [];
-  let seen = new Set(); // Use a set to store seen values
-  for (let domain of domains) {
-    if (!seen.has(domain)) { // If the domain is not in the set
-      seen.add(domain); // Add it to the set
-      uniqueDomains.push(domain); // Push the domain to the uniqueDomains array
-    } else { // If the domain is in the set
-      duplicateDomainCount++; // Increment the duplicateDomainCount
-    }
-  }
-  if (duplicateDomainCount > 0) console.warn(`Found ${duplicateDomainCount} duplicate domains in input.csv - removing`);
+  const domain = normalizeDomain(line);
 
-  // Replace domains array with uniqueDomains array
-  domains = uniqueDomains;
+  if (!isValidDomain(domain)) return;
 
-  // Remove domains from the domains array that are present in the whitelist array
-  let whitelistedDomainCount = 0;
-  domains = domains.filter(domain => {
-    if (whitelist.includes(domain)) {
-      whitelistedDomainCount++;
-      return false;
-    }
-    return true;
-  });
-  if (whitelistedDomainCount > 0) console.warn(`Found ${whitelistedDomainCount} domains in input.csv that are present in the whitelist - removing them`);
+  processedDomainCount++;
 
-  // Trim array to 300,000 domains if it's longer than that
-  if (domains.length > LIST_ITEM_LIMIT) {
-    console.warn(`${domains.length} domains found in input.csv - input has to be trimmed to ${LIST_ITEM_LIMIT} domains`);
-    domains = truncateArray(domains, LIST_ITEM_LIMIT);
+  if (blocklist.has(domain)) {
+    console.log(`Found ${domain} in blocklist already - Skipping...`);
+    duplicateDomainCount++;
+    return;
   }
 
-  const listsToCreate = Math.ceil(domains.length / 1000);
+  if (allowlist.has(domain)) {
+    console.log(`Found ${domain} in allowlist - Skipping...`);
+    return;
+  }
 
-  if (!process.env.CI) console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`);
+  blocklist.set(domain, 1);
+  domains.push(domain);
 
-  // If we are dry-running, stop here because we don't want to create lists
-  // TODO: we should probably continue, just without making any real requests to Cloudflare
-  if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.');
+  if (domains.length === LIST_ITEM_LIMIT) {
+    console.log(
+      "Maximum number of blocked domains reached - Stopping processing blocklist..."
+    );
+    rl.close();
+  }
+});
+
+console.log("\n\n");
+console.log(`Number of processed domains: ${processedDomainCount}`);
+console.log(`Number of blocked domains: ${domains.length}`);
+console.log(`Number of allowed domains: ${allowlist.size}`);
+console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
+console.log(
+  `Number of lists which will be created: ${Math.ceil(
+    domains.length / LIST_ITEM_SIZE
+  )}`
+);
+console.log("\n\n");
+
+(async () => {
+  if (DRY_RUN) {
+    console.log(
+      "Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again."
+    );
+    return;
+  }
 
   if (FAST_MODE) {
     await createZeroTrustListsAtOnce(domains);
@@ -117,4 +96,4 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
   }
 
   await createZeroTrustListsOneByOne(domains);
-});
+})();
diff --git a/lib/helpers.js b/lib/helpers.js
index 40c7b14..707c037 100644
--- a/lib/helpers.js
+++ b/lib/helpers.js
@@ -40,3 +40,22 @@ const request = async (url, options) => {
  */
 export const requestGateway = (path, options) =>
   request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway${path}`, options);
+
+/**
+ * Normalizes a domain.
+ * @param {string} value The value to be normalized.
+ * @param {boolean} isAllowlisting Whether the value is to be whitelisted.
+ * @returns {string}
+ */
+export const normalizeDomain = (value, isAllowlisting) => {
+  const normalized = value
+    .replace(/(0\.0\.0\.0|127\.0\.0\.1|::1|::)\s+/, "")
+    .replace("||", "")
+    .replace("^$important", "")
+    .replace("*.", "")
+    .replace("^", "");
+
+  if (isAllowlisting) return normalized.replace("@@||", "");
+
+  return normalized;
+};
diff --git a/lib/utils.js b/lib/utils.js
index 1d13e0c..ade1427 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,3 +1,8 @@
+import { once } from "events";
+import { createReadStream } from "fs";
+import { basename } from "path";
+import { createInterface } from "readline";
+
 /**
  * Sleeps for a specified amount of time.
  * @param {number} [ms=350] The amount of time in ms.
@@ -6,9 +11,47 @@ export const sleep = (ms = 350) =>
   new Promise((resolve) => setTimeout(resolve, ms));
 
 /**
- * Truncates an array to the specified size.
- * @param {any[]} arr The array to be truncated.
- * @param {number} size The size to which the array will be truncated.
- * @returns {any[]}
+ * Checks if the value is a valid domain.
+ * @param {string} value The value to be checked.
  */
-export const truncateArray = (arr, size) => arr.slice(0, size);
+export const isValidDomain = (value) =>
+  /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/.test(value);
+
+/**
+ * Checks if the value is a comment.
+ * @param {string} value The value to be checked.
+ */
+export const isComment = (value) =>
+  value.startsWith("#") ||
+  value.startsWith("//") ||
+  value.startsWith("!") ||
+  value.startsWith("/*") ||
+  value.startsWith("*/");
+
+/**
+ * @callback onLine
+ * @param {string} line The current line.
+ * @param {ReturnType<createInterface>} rl The readline interface.
+ */
+
+/**
+ * Asynchronously reads a file line by line.
+ * @param {string} filePath The path to the file.
+ * @param {onLine} onLine The callback executed on each line read.
+ */
+export const readFile = async (filePath, onLine) => {
+  try {
+    const rl = createInterface({
+      input: createReadStream(filePath),
+      crlfDelay: Infinity,
+    });
+
+    rl.on("line", (line) => onLine(line, rl));
+
+    await once(rl, "close");
+  } catch (err) {
+    console.error(
+      `Error occurred while reading ${basename(filePath)} - ${err.toString()}`
+    );
+  }
+};

From 9c4ce9c759b8236f1f720eeac6912b2cbfa14390 Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Tue, 12 Sep 2023 17:04:28 +0700
Subject: [PATCH 04/10] cleaned up rules code

---
 cf_gateway_rule_create.js | 26 +++++++++-----------------
 cf_gateway_rule_delete.js | 20 ++++++++++++--------
 2 files changed, 21 insertions(+), 25 deletions(-)

diff --git a/cf_gateway_rule_create.js b/cf_gateway_rule_create.js
index d29ba2f..bf9e73d 100644
--- a/cf_gateway_rule_create.js
+++ b/cf_gateway_rule_create.js
@@ -1,21 +1,13 @@
-import { createZeroTrustRule, getZeroTrustLists } from './lib/api.js';
+import { createZeroTrustRule, getZeroTrustLists } from "./lib/api.js";
 
-;(async() => {
-    const { result: lists } = await getZeroTrustLists();
-    const filtered_lists = lists.filter(list => list.name.startsWith('CGPS List'));
+const { result: lists } = await getZeroTrustLists();
+const wirefilterExpression = lists.reduce((previous, current, index) => {
+  if (!current.name.startsWith("CGPS List")) return previous;
 
-    let wirefilter_expression = '';
+  // Remove the trailing ' or '
+  if (index === lists.length - 1) return previous.slice(0, -4);
 
-    // Build the wirefilter expression
-    for (const list of filtered_lists) {
-        wirefilter_expression += `any(dns.domains[*] in \$${list.id}) or `;
-    }
-    // Remove the trailing ' or '
-    if (wirefilter_expression.endsWith(' or ')) {
-        wirefilter_expression = wirefilter_expression.slice(0, -4);
-    }
-    wirefilter_expression = wirefilter_expression.trim().replace('\n', '');
-    if (!process.env.CI) console.log(`Firewall expression contains ${wirefilter_expression.length} characters, and checks against ${filtered_lists.length} filter lists.`)
+  return `${previous} any(dns.domains[*] in \$${current.id}) or `;
+}, "");
 
-    await createZeroTrustRule(wirefilter_expression);
-})();
+await createZeroTrustRule(wirefilterExpression);
diff --git a/cf_gateway_rule_delete.js b/cf_gateway_rule_delete.js
index 5c04aea..e8aedf5 100644
--- a/cf_gateway_rule_delete.js
+++ b/cf_gateway_rule_delete.js
@@ -1,12 +1,16 @@
-import { deleteZeroTrustRule, getZeroTrustRules } from './lib/api.js';
+import { deleteZeroTrustRule, getZeroTrustRules } from "./lib/api.js";
 
-;(async() => {
-    const { result: rules } = await getZeroTrustRules();
-    const [filtered_rule] = rules.filter(rule => rule.name === "CGPS Filter Lists");
+const { result: rules } = await getZeroTrustRules();
+const cgpsRule = rules.find(({ name }) => name === "CGPS Filter Lists");
 
-    if (!filtered_rule) return console.warn("No rule with matching name found - this is not an issue if you haven't run the create script yet. Exiting.");
+(async () => {
+  if (!cgpsRule) {
+    console.warn(
+      "No rule with matching name found - this is not an issue if you haven't run the create script yet. Exiting."
+    );
+    return;
+  }
 
-    console.log(`Deleting rule`, process.env.CI ? "(redacted, running in CI)" : `"${filtered_rule.name}" with ID ${filtered_rule.id}`);
-
-    await deleteZeroTrustRule(filtered_rule.id);
+  console.log(`Deleting rule ${cgpsRule.name}`);
+  await deleteZeroTrustRule(cgpsRule.id);
 })();

From cc3ebd147ab8300fe0043fa37c1c2f7756f749f5 Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Tue, 12 Sep 2023 17:06:11 +0700
Subject: [PATCH 05/10] cleaned up code

---
 cf_list_delete.js | 43 +++++++++++++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/cf_list_delete.js b/cf_list_delete.js
index ced83e9..e9d3cb6 100644
--- a/cf_list_delete.js
+++ b/cf_list_delete.js
@@ -1,18 +1,37 @@
-import { deleteZeroTrustListsAtOnce, deleteZeroTrustListsOneByOne, getZeroTrustLists } from "./lib/api.js";
+import {
+  deleteZeroTrustListsAtOnce,
+  deleteZeroTrustListsOneByOne,
+  getZeroTrustLists,
+} from "./lib/api.js";
 import { FAST_MODE } from "./lib/constants.js";
 
-;(async() => {
-    const { result: lists } = await getZeroTrustLists();
-    if (!lists) return console.warn("No file lists found - this is not an issue if it's your first time running this script. Exiting.");
-    const cgps_lists = lists.filter(list => list.name.startsWith('CGPS List'));
-    if (!cgps_lists.length) return console.warn("No lists with matching name found - this is not an issue if you haven't created any filter lists before. Exiting.");
+(async () => {
+  const { result: lists } = await getZeroTrustLists();
 
-    if (!process.env.CI) console.log(`Got ${lists.length} lists, ${cgps_lists.length} of which are CGPS lists that will be deleted.`);
+  if (!lists) {
+    console.warn(
+      "No file lists found - this is not an issue if it's your first time running this script. Exiting."
+    );
+    return;
+  }
 
-    if (FAST_MODE) {
-      await deleteZeroTrustListsAtOnce(cgps_lists);
-      return;
-    }
+  const cgpsLists = lists.filter(({ name }) => name.startsWith("CGPS List"));
 
-    await deleteZeroTrustListsOneByOne(cgps_lists);
+  if (!cgpsLists.length) {
+    console.warn(
+      "No lists with matching name found - this is not an issue if you haven't created any filter lists before. Exiting."
+    );
+    return;
+  }
+
+  console.log(
+    `Got ${lists.length} lists, ${cgpsLists.length} of which are CGPS lists that will be deleted.`
+  );
+
+  if (FAST_MODE) {
+    await deleteZeroTrustListsAtOnce(cgpsLists);
+    return;
+  }
+
+  await deleteZeroTrustListsOneByOne(cgpsLists);
 })();

From fccfdaba0043983d143b64c901104db1a385f28b Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Tue, 12 Sep 2023 17:19:32 +0700
Subject: [PATCH 06/10] fixed last chunk expression

---
 cf_gateway_rule_create.js | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/cf_gateway_rule_create.js b/cf_gateway_rule_create.js
index bf9e73d..d069a00 100644
--- a/cf_gateway_rule_create.js
+++ b/cf_gateway_rule_create.js
@@ -1,13 +1,11 @@
 import { createZeroTrustRule, getZeroTrustLists } from "./lib/api.js";
 
 const { result: lists } = await getZeroTrustLists();
-const wirefilterExpression = lists.reduce((previous, current, index) => {
+const wirefilterExpression = lists.reduce((previous, current) => {
   if (!current.name.startsWith("CGPS List")) return previous;
 
-  // Remove the trailing ' or '
-  if (index === lists.length - 1) return previous.slice(0, -4);
-
   return `${previous} any(dns.domains[*] in \$${current.id}) or `;
 }, "");
 
-await createZeroTrustRule(wirefilterExpression);
+// Remove the trailing ' or '
+await createZeroTrustRule(wirefilterExpression.slice(0, -4));

From 811205ccc24ddcca794f10d41b229ba81f67610c Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Tue, 12 Sep 2023 17:32:05 +0700
Subject: [PATCH 07/10] trim before processing domain

---
 cf_list_create.js | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/cf_list_create.js b/cf_list_create.js
index 28d451a..ac9d951 100644
--- a/cf_list_create.js
+++ b/cf_list_create.js
@@ -24,9 +24,13 @@ let duplicateDomainCount = 0;
 // Read allowlist
 console.log(`Processing ${allowlistFilename}`);
 await readFile(resolve(allowlistFilename), (line) => {
-  if (isComment(line)) return;
+  const _line = line.trim();
 
-  const domain = normalizeDomain(line, true);
+  if (!_line) return;
+
+  if (isComment(_line)) return;
+
+  const domain = normalizeDomain(_line, true);
 
   if (!isValidDomain(domain)) return;
 
@@ -40,9 +44,13 @@ await readFile(resolve(blocklistFilename), (line, rl) => {
     return;
   }
 
-  if (isComment(line)) return;
+  const _line = line.trim();
 
-  const domain = normalizeDomain(line);
+  if (!_line) return;
+
+  if (isComment(_line)) return;
+
+  const domain = normalizeDomain(_line);
 
   if (!isValidDomain(domain)) return;
 

From 695d7c7f5a465ae1b20110935f1d3fa0f17f3061 Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Tue, 12 Sep 2023 20:19:09 +0700
Subject: [PATCH 08/10] fixed to count allowed domains correctly

---
 cf_list_create.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cf_list_create.js b/cf_list_create.js
index ac9d951..76b1485 100644
--- a/cf_list_create.js
+++ b/cf_list_create.js
@@ -20,6 +20,7 @@ const blocklist = new Map();
 const domains = [];
 let processedDomainCount = 0;
 let duplicateDomainCount = 0;
+let allowedDomainCount = 0;
 
 // Read allowlist
 console.log(`Processing ${allowlistFilename}`);
@@ -64,6 +65,7 @@ await readFile(resolve(blocklistFilename), (line, rl) => {
 
   if (allowlist.has(domain)) {
     console.log(`Found ${domain} in allowlist - Skipping...`);
+    allowedDomainCount++;
     return;
   }
 
@@ -81,7 +83,7 @@ await readFile(resolve(blocklistFilename), (line, rl) => {
 console.log("\n\n");
 console.log(`Number of processed domains: ${processedDomainCount}`);
 console.log(`Number of blocked domains: ${domains.length}`);
-console.log(`Number of allowed domains: ${allowlist.size}`);
+console.log(`Number of allowed domains: ${allowedDomainCount}`);
 console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
 console.log(
   `Number of lists which will be created: ${Math.ceil(

From 083335278c314d4031bc2719407ce92a62e209bd Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Fri, 15 Sep 2023 05:36:19 +0700
Subject: [PATCH 09/10] updated blocklist check logic to prevent adding a
 subdomain if a higher-level domain is already blocked

---
 cf_list_create.js | 39 +++++++++++++++++++++++++++++++--------
 lib/utils.js      | 18 ++++++++++++++++++
 2 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/cf_list_create.js b/cf_list_create.js
index 76b1485..fb0e81b 100644
--- a/cf_list_create.js
+++ b/cf_list_create.js
@@ -11,7 +11,12 @@ import {
   LIST_ITEM_SIZE,
 } from "./lib/constants.js";
 import { normalizeDomain } from "./lib/helpers.js";
-import { isComment, isValidDomain, readFile } from "./lib/utils.js";
+import {
+  extractDomain,
+  isComment,
+  isValidDomain,
+  readFile,
+} from "./lib/utils.js";
 
 const allowlistFilename = "whitelist.csv";
 const blocklistFilename = "input.csv";
@@ -19,6 +24,7 @@ const allowlist = new Map();
 const blocklist = new Map();
 const domains = [];
 let processedDomainCount = 0;
+let unnecessaryDomainCount = 0;
 let duplicateDomainCount = 0;
 let allowedDomainCount = 0;
 
@@ -57,14 +63,30 @@ await readFile(resolve(blocklistFilename), (line, rl) => {
 
   processedDomainCount++;
 
-  if (blocklist.has(domain)) {
-    console.log(`Found ${domain} in blocklist already - Skipping...`);
-    duplicateDomainCount++;
-    return;
-  }
+  const anyDomainExists = extractDomain(domain)
+    .reverse()
+    .some((item) => {
+      if (blocklist.has(item)) {
+        if (item === domain) {
+          console.log(`Found ${item} in blocklist already - Skipping`);
+          duplicateDomainCount++;
+        } else {
+          console.log(
+            `Found ${item} in blocklist already - Skipping ${domain}`
+          );
+          unnecessaryDomainCount++;
+        }
+
+        return true;
+      }
+
+      return false;
+    });
+
+  if (anyDomainExists) return;
 
   if (allowlist.has(domain)) {
-    console.log(`Found ${domain} in allowlist - Skipping...`);
+    console.log(`Found ${domain} in allowlist - Skipping`);
     allowedDomainCount++;
     return;
   }
@@ -82,9 +104,10 @@ await readFile(resolve(blocklistFilename), (line, rl) => {
 
 console.log("\n\n");
 console.log(`Number of processed domains: ${processedDomainCount}`);
+console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
+console.log(`Number of unnecessary domains: ${unnecessaryDomainCount}`);
 console.log(`Number of blocked domains: ${domains.length}`);
 console.log(`Number of allowed domains: ${allowedDomainCount}`);
-console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
 console.log(
   `Number of lists which will be created: ${Math.ceil(
     domains.length / LIST_ITEM_SIZE
diff --git a/lib/utils.js b/lib/utils.js
index ade1427..b5b3b6b 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -17,6 +17,24 @@ export const sleep = (ms = 350) =>
 export const isValidDomain = (value) =>
   /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/.test(value);
 
+/**
+ * Extracts all subdomains from a domain including itself.
+ * @param {string} domain The domain to be extracted.
+ * @returns {string[]}
+ */
+export const extractDomain = (domain) =>
+  domain.split(".").reduce((previous, current, index, array) => {
+    const nextIndex = index + 1;
+
+    if (nextIndex > array.length - 1) return previous;
+
+    const domain = [current, ...array.slice(nextIndex)].join(".");
+
+    previous.push(domain);
+
+    return previous;
+  }, []);
+
 /**
  * Checks if the value is a comment.
  * @param {string} value The value to be checked.

From 27f6e9f0d3a72d381e1bc752d0a7dceba27b4671 Mon Sep 17 00:00:00 2001
From: Viet Huynh <xa.lach@yahoo.com>
Date: Sun, 17 Sep 2023 19:55:43 +0700
Subject: [PATCH 10/10] added code comments

---
 cf_list_create.js | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/cf_list_create.js b/cf_list_create.js
index fb0e81b..3671642 100644
--- a/cf_list_create.js
+++ b/cf_list_create.js
@@ -55,22 +55,32 @@ await readFile(resolve(blocklistFilename), (line, rl) => {
 
   if (!_line) return;
 
+  // Check if the current line is a comment in any format
   if (isComment(_line)) return;
 
+  // Remove prefixes and suffixes in hosts, wildcard or adblock format
   const domain = normalizeDomain(_line);
 
+  // Check if it is a valid domain which is not a URL or does not contain
+  // characters like * in the middle of the domain
   if (!isValidDomain(domain)) return;
 
   processedDomainCount++;
 
+  // Get all the levels of the domain and check from the highest
+  // because we are blocking all subdomains
+  // Example: fourth.third.example.com => ["example.com", "third.example.com", "fourth.third.example.com"]
   const anyDomainExists = extractDomain(domain)
     .reverse()
     .some((item) => {
       if (blocklist.has(item)) {
         if (item === domain) {
+          // The exact domain is already blocked
           console.log(`Found ${item} in blocklist already - Skipping`);
           duplicateDomainCount++;
         } else {
+          // The higher-level domain is already blocked
+          // so it's not necessary to block this domain
           console.log(
             `Found ${item} in blocklist already - Skipping ${domain}`
           );