From 1a1daadabaa05bb1fb50a82e3253c0e0356870ad Mon Sep 17 00:00:00 2001 From: Norbert Date: Thu, 29 Jun 2023 01:48:42 +0200 Subject: [PATCH] Check for duplicates before trim --- cf_list_create.js | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/cf_list_create.js b/cf_list_create.js index ab157cf..738af4f 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -72,17 +72,6 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { }).filter(domain => { return domainValidationPattern.test(domain); }); - - // Remove domains from the domains array that are present in the whitelist array - domains = domains.filter(domain => { - return !whitelist.includes(domain); - }); - - // Trim array to 300,000 domains if it's longer than that - if (domains.length > LIST_ITEM_LIMIT) { - domains = trimArray(domains, LIST_ITEM_LIMIT); - console.warn(`More than ${LIST_ITEM_LIMIT} domains found in input.csv - input has to be trimmed`); - } // Check for duplicates in domains array let uniqueDomains = []; @@ -99,6 +88,17 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { // Replace domains array with uniqueDomains array domains = uniqueDomains; + // Remove domains from the domains array that are present in the whitelist array + domains = domains.filter(domain => { + return !whitelist.includes(domain); + }); + + // Trim array to 300,000 domains if it's longer than that + if (domains.length > LIST_ITEM_LIMIT) { + domains = trimArray(domains, LIST_ITEM_LIMIT); + console.warn(`More than ${LIST_ITEM_LIMIT} domains found in input.csv - input has to be trimmed`); + } + const listsToCreate = Math.ceil(domains.length / 1000); if (!process.env.CI) console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`); @@ -175,4 +175,3 @@ function percentage(percent, total) { function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } -