nezumi/cloudflare-gateway-pihole-scripts
1
0
Fork 0
mirror of https://github.com/Nezumi-2711/cloudflare-gateway-pihole-scripts.git synced 2026-09-22 13:38:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Check for duplicates before trim

Browse Source
This commit is contained in:
Norbert
2023-06-29 01:48:42 +02:00
parent 9e80ebb8ea
commit 1a1daadaba
1 changed files with 11 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cf_list_create.js
+11 -12
View File
@@ -73,17 +73,6 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
return domainValidationPattern.test(domain); return domainValidationPattern.test(domain);
}); });
// Remove domains from the domains array that are present in the whitelist array
domains = domains.filter(domain => {
return !whitelist.includes(domain);
});
// Trim array to 300,000 domains if it's longer than that
if (domains.length > LIST_ITEM_LIMIT) {
domains = trimArray(domains, LIST_ITEM_LIMIT);
console.warn(`More than ${LIST_ITEM_LIMIT} domains found in input.csv - input has to be trimmed`);
}
// Check for duplicates in domains array // Check for duplicates in domains array
let uniqueDomains = []; let uniqueDomains = [];
let seen = new Set(); // Use a set to store seen values let seen = new Set(); // Use a set to store seen values
@@ -99,6 +88,17 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
// Replace domains array with uniqueDomains array // Replace domains array with uniqueDomains array
domains = uniqueDomains; domains = uniqueDomains;
// Remove domains from the domains array that are present in the whitelist array
domains = domains.filter(domain => {
return !whitelist.includes(domain);
});
// Trim array to 300,000 domains if it's longer than that
if (domains.length > LIST_ITEM_LIMIT) {
domains = trimArray(domains, LIST_ITEM_LIMIT);
console.warn(`More than ${LIST_ITEM_LIMIT} domains found in input.csv - input has to be trimmed`);
}
const listsToCreate = Math.ceil(domains.length / 1000); const listsToCreate = Math.ceil(domains.length / 1000);
if (!process.env.CI) console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`); if (!process.env.CI) console.log(`Found ${domains.length} valid domains in input.csv after cleanup - ${listsToCreate} list(s) will be created`);
@@ -175,4 +175,3 @@ function percentage(percent, total) {
function sleep(ms) { function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise(resolve => setTimeout(resolve, ms));
} }