updated blocklist check logic to prevent adding a subdomain if a higher-level domain is already blocked

cf_list_create.js

@@ -11,7 +11,12 @@ import {
   LIST_ITEM_SIZE,
 } from "./lib/constants.js";
 import { normalizeDomain } from "./lib/helpers.js";
-import { isComment, isValidDomain, readFile } from "./lib/utils.js";
+import {
+  extractDomain,
+  isComment,
+  isValidDomain,
+  readFile,
+} from "./lib/utils.js";
 
 const allowlistFilename = "whitelist.csv";
 const blocklistFilename = "input.csv";
@@ -19,6 +24,7 @@ const allowlist = new Map();
 const blocklist = new Map();
 const domains = [];
 let processedDomainCount = 0;
+let unnecessaryDomainCount = 0;
 let duplicateDomainCount = 0;
 let allowedDomainCount = 0;
 
@@ -57,14 +63,30 @@ await readFile(resolve(blocklistFilename), (line, rl) => {
 
   processedDomainCount++;
 
-  if (blocklist.has(domain)) {
-    console.log(`Found ${domain} in blocklist already - Skipping...`);
-    duplicateDomainCount++;
-    return;
-  }
+  const anyDomainExists = extractDomain(domain)
+    .reverse()
+    .some((item) => {
+      if (blocklist.has(item)) {
+        if (item === domain) {
+          console.log(`Found ${item} in blocklist already - Skipping`);
+          duplicateDomainCount++;
+        } else {
+          console.log(
+            `Found ${item} in blocklist already - Skipping ${domain}`
+          );
+          unnecessaryDomainCount++;
+        }
+
+        return true;
+      }
+
+      return false;
+    });
+
+  if (anyDomainExists) return;
 
   if (allowlist.has(domain)) {
-    console.log(`Found ${domain} in allowlist - Skipping...`);
+    console.log(`Found ${domain} in allowlist - Skipping`);
     allowedDomainCount++;
     return;
   }
@@ -82,9 +104,10 @@ await readFile(resolve(blocklistFilename), (line, rl) => {
 
 console.log("\n\n");
 console.log(`Number of processed domains: ${processedDomainCount}`);
+console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
+console.log(`Number of unnecessary domains: ${unnecessaryDomainCount}`);
 console.log(`Number of blocked domains: ${domains.length}`);
 console.log(`Number of allowed domains: ${allowedDomainCount}`);
-console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
 console.log(
   `Number of lists which will be created: ${Math.ceil(
     domains.length / LIST_ITEM_SIZE

lib/utils.js

@@ -17,6 +17,24 @@ export const sleep = (ms = 350) =>
 export const isValidDomain = (value) =>
   /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/.test(value);
 
+/**
+ * Extracts all subdomains from a domain including itself.
+ * @param {string} domain The domain to be extracted.
+ * @returns {string[]}
+ */
+export const extractDomain = (domain) =>
+  domain.split(".").reduce((previous, current, index, array) => {
+    const nextIndex = index + 1;
+
+    if (nextIndex > array.length - 1) return previous;
+
+    const domain = [current, ...array.slice(nextIndex)].join(".");
+
+    previous.push(domain);
+
+    return previous;
+  }, []);
+
 /**
  * Checks if the value is a comment.
  * @param {string} value The value to be checked.