diff --git a/cf_list_create.js b/cf_list_create.js index 68670cb..bce748b 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -17,6 +17,7 @@ import { extractDomain, isComment, isValidDomain, + memoize, readFile, } from "./lib/utils.js"; @@ -33,6 +34,7 @@ let processedDomainCount = 0; let unnecessaryDomainCount = 0; let duplicateDomainCount = 0; let allowedDomainCount = 0; +const memoizedNormalizeDomain = memoize(normalizeDomain); // Read allowlist console.log(`Processing ${allowlistFilename}`); @@ -43,7 +45,7 @@ await readFile(resolve(`./${allowlistFilename}`), (line) => { if (isComment(_line)) return; - const domain = normalizeDomain(_line, true); + const domain = memoizedNormalizeDomain(_line, true); if (!isValidDomain(domain)) return; @@ -65,7 +67,7 @@ await readFile(resolve(`./${blocklistFilename}`), (line, rl) => { if (isComment(_line)) return; // Remove prefixes and suffixes in hosts, wildcard or adblock format - const domain = normalizeDomain(_line); + const domain = memoizedNormalizeDomain(_line); // Check if it is a valid domain which is not a URL or does not contain // characters like * in the middle of the domain @@ -73,40 +75,31 @@ await readFile(resolve(`./${blocklistFilename}`), (line, rl) => { processedDomainCount++; - // Get all the levels of the domain and check from the highest - // because we are blocking all subdomains - // Example: fourth.third.example.com => ["example.com", "third.example.com", "fourth.third.example.com"] - const anyDomainExists = extractDomain(domain) - .reverse() - .some((item) => { - if (blocklist.has(item)) { - if (item === domain) { - // The exact domain is already blocked - console.log(`Found ${item} in blocklist already - Skipping`); - duplicateDomainCount++; - } else { - // The higher-level domain is already blocked - // so it's not necessary to block this domain - console.log( - `Found ${item} in blocklist already - Skipping ${domain}` - ); - unnecessaryDomainCount++; - } - - return true; - } - - return false; - }); - - if (anyDomainExists) return; - if (allowlist.has(domain)) { console.log(`Found ${domain} in allowlist - Skipping`); allowedDomainCount++; return; } + if (blocklist.has(domain)) { + console.log(`Found ${domain} in blocklist already - Skipping`); + duplicateDomainCount++; + return; + } + + // Get all the levels of the domain and check from the highest + // because we are blocking all subdomains + // Example: fourth.third.example.com => ["example.com", "third.example.com", "fourth.third.example.com"] + for (const item of extractDomain(domain).slice(1)) { + if (!blocklist.has(item)) continue; + + // The higher-level domain is already blocked + // so it's not necessary to block this domain + console.log(`Found ${item} in blocklist already - Skipping ${domain}`); + unnecessaryDomainCount++; + return; + } + blocklist.set(domain, 1); domains.push(domain); @@ -124,8 +117,8 @@ console.log("\n\n"); console.log(`Number of processed domains: ${processedDomainCount}`); console.log(`Number of duplicate domains: ${duplicateDomainCount}`); console.log(`Number of unnecessary domains: ${unnecessaryDomainCount}`); -console.log(`Number of blocked domains: ${domains.length}`); console.log(`Number of allowed domains: ${allowedDomainCount}`); +console.log(`Number of blocked domains: ${domains.length}`); console.log(`Number of lists to be created: ${numberOfLists}`); console.log("\n\n"); @@ -143,12 +136,11 @@ console.log("\n\n"); if (FAST_MODE) { await createZeroTrustListsAtOnce(domains); - // TODO: make this less repetitive - await notifyWebhook(`CF List Create script finished running (${domains.length} domains, ${numberOfLists} lists)`); - return; + } else { + await createZeroTrustListsOneByOne(domains); } - await createZeroTrustListsOneByOne(domains); - - await notifyWebhook(`CF List Create script finished running (${domains.length} domains, ${numberOfLists} lists)`); + await notifyWebhook( + `CF List Create script finished running (${domains.length} domains, ${numberOfLists} lists)` + ); })(); diff --git a/download_lists.js b/download_lists.js index 9087e0f..db61951 100644 --- a/download_lists.js +++ b/download_lists.js @@ -38,6 +38,7 @@ const downloadLists = async (filename, urls) => { } catch (err) { console.error(`An error occurred while processing ${filename}:\n`, err); console.error("URLs:\n", urls); + throw err; } }; diff --git a/lib/api.js b/lib/api.js index ef4c138..2a83f15 100644 --- a/lib/api.js +++ b/lib/api.js @@ -51,6 +51,7 @@ export const createZeroTrustListsOneByOne = async (items) => { console.log(`Created "${listName}" list - ${totalListNumber} left`); } catch (err) { console.error(`Could not create "${listName}" - ${err.toString()}`); + throw err; } } }; @@ -77,6 +78,7 @@ export const createZeroTrustListsAtOnce = async (items) => { console.log("Created lists successfully"); } catch (err) { console.error(`Error occurred while creating lists - ${err.toString()}`); + throw err; } }; @@ -106,6 +108,7 @@ export const deleteZeroTrustListsOneByOne = async (lists) => { console.log(`Deleted ${name} list - ${totalListNumber} left`); } catch (err) { console.error(`Could not delete ${name} - ${err.toString()}`); + throw err; } } }; @@ -124,6 +127,7 @@ export const deleteZeroTrustListsAtOnce = async (lists) => { console.log("Deleted lists successfully"); } catch (err) { console.error(`Error occurred while deleting lists - ${err.toString()}`); + throw err; } }; @@ -161,6 +165,7 @@ export const createZeroTrustRule = async (wirefilterExpression) => { console.log("Created rule successfully"); } catch (err) { console.error(`Error occurred while creating rule - ${err.toString()}`); + throw err; } }; @@ -180,5 +185,6 @@ export const deleteZeroTrustRule = async (id) => { console.log("Deleted rule successfully"); } catch (err) { console.error(`Error occurred while deleting rule - ${err.toString()}`); + throw err; } }; diff --git a/lib/utils.js b/lib/utils.js index 79ad4ea..dd91e21 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -19,18 +19,18 @@ export const isValidDomain = (value) => * @param {string} domain The domain to be extracted. * @returns {string[]} */ -export const extractDomain = (domain) => - domain.split(".").reduce((previous, current, index, array) => { - const nextIndex = index + 1; +export const extractDomain = (domain) => { + const parts = domain.split("."); + const extractedDomains = []; - if (nextIndex > array.length - 1) return previous; + for (let i = 0; i < parts.length; i++) { + const subdomains = parts.slice(i).join("."); - const domain = [current, ...array.slice(nextIndex)].join("."); + extractedDomains.unshift(subdomains); + } - previous.push(domain); - - return previous; - }, []); + return extractedDomains; +}; /** * Checks if the value is a comment. @@ -49,19 +49,16 @@ export const isComment = (value) => * @param {string[]} urls The URLs to the files to be downloaded. */ export const downloadFiles = async (filePath, urls) => { - const writeStream = createWriteStream(filePath, { flags: "a" }); const responses = await Promise.all(urls.map((url) => fetch(url))); for (const response of responses) { - const readable = ReadStream.from(response.body, { - autoDestroy: true, - }); + const writeStream = createWriteStream(filePath, { flags: "a" }); - readable.on("end", () => { - writeStream.write("\n"); - }); - - readable.pipe(writeStream, { end: false }); + ReadStream.from(response.body) + .on("end", () => { + writeStream.write("\n"); + }) + .pipe(writeStream); } }; @@ -90,5 +87,27 @@ export const readFile = async (filePath, onLine) => { console.error( `Error occurred while reading ${basename(filePath)} - ${err.toString()}` ); + throw err; } }; + +/** + * Memoizes a function + * @template T The argument type of the function. + * @template R The return type of the function. + * @param {(...fnArgs: T[]) => R} fn The function to be memoized. + */ +export const memoize = (fn) => { + const cache = new Map(); + + return (...args) => { + const key = args.join("-"); + + if (cache.has(key)) return cache.get(key); + + const result = fn(...args); + + cache.set(key, result); + return result; + }; +};