Merge pull request #49 from hlqviet/refactor/domain-processing

Further enhance the domain processing code

cf_list_create.js

@@ -17,6 +17,7 @@ import {
   extractDomain,
   isComment,
   isValidDomain,
+  memoize,
   readFile,
 } from "./lib/utils.js";
 
@@ -33,6 +34,7 @@ let processedDomainCount = 0;
 let unnecessaryDomainCount = 0;
 let duplicateDomainCount = 0;
 let allowedDomainCount = 0;
+const memoizedNormalizeDomain = memoize(normalizeDomain);
 
 // Read allowlist
 console.log(`Processing ${allowlistFilename}`);
@@ -43,7 +45,7 @@ await readFile(resolve(`./${allowlistFilename}`), (line) => {
 
   if (isComment(_line)) return;
 
-  const domain = normalizeDomain(_line, true);
+  const domain = memoizedNormalizeDomain(_line, true);
 
   if (!isValidDomain(domain)) return;
 
@@ -65,7 +67,7 @@ await readFile(resolve(`./${blocklistFilename}`), (line, rl) => {
   if (isComment(_line)) return;
 
   // Remove prefixes and suffixes in hosts, wildcard or adblock format
-  const domain = normalizeDomain(_line);
+  const domain = memoizedNormalizeDomain(_line);
 
   // Check if it is a valid domain which is not a URL or does not contain
   // characters like * in the middle of the domain
@@ -73,40 +75,31 @@ await readFile(resolve(`./${blocklistFilename}`), (line, rl) => {
 
   processedDomainCount++;
 
-  // Get all the levels of the domain and check from the highest
-  // because we are blocking all subdomains
-  // Example: fourth.third.example.com => ["example.com", "third.example.com", "fourth.third.example.com"]
-  const anyDomainExists = extractDomain(domain)
-    .reverse()
-    .some((item) => {
-      if (blocklist.has(item)) {
-        if (item === domain) {
-          // The exact domain is already blocked
-          console.log(`Found ${item} in blocklist already - Skipping`);
-          duplicateDomainCount++;
-        } else {
-          // The higher-level domain is already blocked
-          // so it's not necessary to block this domain
-          console.log(
-            `Found ${item} in blocklist already - Skipping ${domain}`
-          );
-          unnecessaryDomainCount++;
-        }
-
-        return true;
-      }
-
-      return false;
-    });
-
-  if (anyDomainExists) return;
-
   if (allowlist.has(domain)) {
     console.log(`Found ${domain} in allowlist - Skipping`);
     allowedDomainCount++;
     return;
   }
 
+  if (blocklist.has(domain)) {
+    console.log(`Found ${domain} in blocklist already - Skipping`);
+    duplicateDomainCount++;
+    return;
+  }
+
+  // Get all the levels of the domain and check from the highest
+  // because we are blocking all subdomains
+  // Example: fourth.third.example.com => ["example.com", "third.example.com", "fourth.third.example.com"]
+  for (const item of extractDomain(domain).slice(1)) {
+    if (!blocklist.has(item)) continue;
+
+    // The higher-level domain is already blocked
+    // so it's not necessary to block this domain
+    console.log(`Found ${item} in blocklist already - Skipping ${domain}`);
+    unnecessaryDomainCount++;
+    return;
+  }
+
   blocklist.set(domain, 1);
   domains.push(domain);
 
@@ -124,8 +117,8 @@ console.log("\n\n");
 console.log(`Number of processed domains: ${processedDomainCount}`);
 console.log(`Number of duplicate domains: ${duplicateDomainCount}`);
 console.log(`Number of unnecessary domains: ${unnecessaryDomainCount}`);
-console.log(`Number of blocked domains: ${domains.length}`);
 console.log(`Number of allowed domains: ${allowedDomainCount}`);
+console.log(`Number of blocked domains: ${domains.length}`);
 console.log(`Number of lists to be created: ${numberOfLists}`);
 console.log("\n\n");
 
@@ -143,12 +136,11 @@ console.log("\n\n");
 
   if (FAST_MODE) {
     await createZeroTrustListsAtOnce(domains);
-    // TODO: make this less repetitive
+  } else {
-    await notifyWebhook(`CF List Create script finished running (${domains.length} domains, ${numberOfLists} lists)`);
+    await createZeroTrustListsOneByOne(domains);
-    return;
   }
 
-  await createZeroTrustListsOneByOne(domains);
+  await notifyWebhook(
-
+    `CF List Create script finished running (${domains.length} domains, ${numberOfLists} lists)`
-  await notifyWebhook(`CF List Create script finished running (${domains.length} domains, ${numberOfLists} lists)`);
+  );
 })();

download_lists.js

@@ -38,6 +38,7 @@ const downloadLists = async (filename, urls) => {
   } catch (err) {
     console.error(`An error occurred while processing ${filename}:\n`, err);
     console.error("URLs:\n", urls);
+    throw err;
   }
 };
 

lib/api.js

@@ -51,6 +51,7 @@ export const createZeroTrustListsOneByOne = async (items) => {
       console.log(`Created "${listName}" list - ${totalListNumber} left`);
     } catch (err) {
       console.error(`Could not create "${listName}" - ${err.toString()}`);
+      throw err;
     }
   }
 };
@@ -77,6 +78,7 @@ export const createZeroTrustListsAtOnce = async (items) => {
     console.log("Created lists successfully");
   } catch (err) {
     console.error(`Error occurred while creating lists - ${err.toString()}`);
+    throw err;
   }
 };
 
@@ -106,6 +108,7 @@ export const deleteZeroTrustListsOneByOne = async (lists) => {
       console.log(`Deleted ${name} list - ${totalListNumber} left`);
     } catch (err) {
       console.error(`Could not delete ${name} - ${err.toString()}`);
+      throw err;
     }
   }
 };
@@ -124,6 +127,7 @@ export const deleteZeroTrustListsAtOnce = async (lists) => {
     console.log("Deleted lists successfully");
   } catch (err) {
     console.error(`Error occurred while deleting lists - ${err.toString()}`);
+    throw err;
   }
 };
 
@@ -161,6 +165,7 @@ export const createZeroTrustRule = async (wirefilterExpression) => {
     console.log("Created rule successfully");
   } catch (err) {
     console.error(`Error occurred while creating rule - ${err.toString()}`);
+    throw err;
   }
 };
 
@@ -180,5 +185,6 @@ export const deleteZeroTrustRule = async (id) => {
     console.log("Deleted rule successfully");
   } catch (err) {
     console.error(`Error occurred while deleting rule - ${err.toString()}`);
+    throw err;
   }
 };

lib/utils.js

@@ -19,18 +19,18 @@ export const isValidDomain = (value) =>
  * @param {string} domain The domain to be extracted.
  * @returns {string[]}
  */
-export const extractDomain = (domain) =>
+export const extractDomain = (domain) => {
-  domain.split(".").reduce((previous, current, index, array) => {
+  const parts = domain.split(".");
-    const nextIndex = index + 1;
+  const extractedDomains = [];
 
-    if (nextIndex > array.length - 1) return previous;
+  for (let i = 0; i < parts.length; i++) {
+    const subdomains = parts.slice(i).join(".");
 
-    const domain = [current, ...array.slice(nextIndex)].join(".");
+    extractedDomains.unshift(subdomains);
+  }
 
-    previous.push(domain);
+  return extractedDomains;
-
+};
-    return previous;
-  }, []);
 
 /**
  * Checks if the value is a comment.
@@ -49,19 +49,16 @@ export const isComment = (value) =>
  * @param {string[]} urls The URLs to the files to be downloaded.
  */
 export const downloadFiles = async (filePath, urls) => {
-  const writeStream = createWriteStream(filePath, { flags: "a" });
   const responses = await Promise.all(urls.map((url) => fetch(url)));
 
   for (const response of responses) {
-    const readable = ReadStream.from(response.body, {
+    const writeStream = createWriteStream(filePath, { flags: "a" });
-      autoDestroy: true,
-    });
 
-    readable.on("end", () => {
+    ReadStream.from(response.body)
+      .on("end", () => {
         writeStream.write("\n");
-    });
+      })
-
+      .pipe(writeStream);
-    readable.pipe(writeStream, { end: false });
   }
 };
 
@@ -90,5 +87,27 @@ export const readFile = async (filePath, onLine) => {
     console.error(
       `Error occurred while reading ${basename(filePath)} - ${err.toString()}`
     );
+    throw err;
   }
 };
+
+/**
+ * Memoizes a function
+ * @template T The argument type of the function.
+ * @template R The return type of the function.
+ * @param {(...fnArgs: T[]) => R} fn The function to be memoized.
+ */
+export const memoize = (fn) => {
+  const cache = new Map();
+
+  return (...args) => {
+    const key = args.join("-");
+
+    if (cache.has(key)) return cache.get(key);
+
+    const result = fn(...args);
+
+    cache.set(key, result);
+    return result;
+  };
+};