diff --git a/cf_list_create.js b/cf_list_create.js index 76b1485..fb0e81b 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -11,7 +11,12 @@ import { LIST_ITEM_SIZE, } from "./lib/constants.js"; import { normalizeDomain } from "./lib/helpers.js"; -import { isComment, isValidDomain, readFile } from "./lib/utils.js"; +import { + extractDomain, + isComment, + isValidDomain, + readFile, +} from "./lib/utils.js"; const allowlistFilename = "whitelist.csv"; const blocklistFilename = "input.csv"; @@ -19,6 +24,7 @@ const allowlist = new Map(); const blocklist = new Map(); const domains = []; let processedDomainCount = 0; +let unnecessaryDomainCount = 0; let duplicateDomainCount = 0; let allowedDomainCount = 0; @@ -57,14 +63,30 @@ await readFile(resolve(blocklistFilename), (line, rl) => { processedDomainCount++; - if (blocklist.has(domain)) { - console.log(`Found ${domain} in blocklist already - Skipping...`); - duplicateDomainCount++; - return; - } + const anyDomainExists = extractDomain(domain) + .reverse() + .some((item) => { + if (blocklist.has(item)) { + if (item === domain) { + console.log(`Found ${item} in blocklist already - Skipping`); + duplicateDomainCount++; + } else { + console.log( + `Found ${item} in blocklist already - Skipping ${domain}` + ); + unnecessaryDomainCount++; + } + + return true; + } + + return false; + }); + + if (anyDomainExists) return; if (allowlist.has(domain)) { - console.log(`Found ${domain} in allowlist - Skipping...`); + console.log(`Found ${domain} in allowlist - Skipping`); allowedDomainCount++; return; } @@ -82,9 +104,10 @@ await readFile(resolve(blocklistFilename), (line, rl) => { console.log("\n\n"); console.log(`Number of processed domains: ${processedDomainCount}`); +console.log(`Number of duplicate domains: ${duplicateDomainCount}`); +console.log(`Number of unnecessary domains: ${unnecessaryDomainCount}`); console.log(`Number of blocked domains: ${domains.length}`); console.log(`Number of allowed domains: ${allowedDomainCount}`); -console.log(`Number of duplicate domains: ${duplicateDomainCount}`); console.log( `Number of lists which will be created: ${Math.ceil( domains.length / LIST_ITEM_SIZE diff --git a/lib/utils.js b/lib/utils.js index ade1427..b5b3b6b 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -17,6 +17,24 @@ export const sleep = (ms = 350) => export const isValidDomain = (value) => /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/.test(value); +/** + * Extracts all subdomains from a domain including itself. + * @param {string} domain The domain to be extracted. + * @returns {string[]} + */ +export const extractDomain = (domain) => + domain.split(".").reduce((previous, current, index, array) => { + const nextIndex = index + 1; + + if (nextIndex > array.length - 1) return previous; + + const domain = [current, ...array.slice(nextIndex)].join("."); + + previous.push(domain); + + return previous; + }, []); + /** * Checks if the value is a comment. * @param {string} value The value to be checked.