whitelist

cf_list_create.js

@@ -9,6 +9,34 @@ const LIST_ITEM_LIMIT = Number.isSafeInteger(Number(process.env.CLOUDFLARE_LIST_
 
 if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`);
 
+
+// Read whitelist.csv and parse
+fs.readFile('whitelist.csv', 'utf8', async (err, data) => {
+  if (err) {
+    console.error('Error reading whitelist.csv:', err);
+    return;
+  }
+
+  // Convert into array and cleanup whitelist
+  const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
+  let whitelist = data.split('\n').filter(domain => {
+    // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
+    return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
+  }).map(domain => {
+    // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
+    return domain
+      .replace('\r', '')
+      .replace('0.0.0.0 ', '')
+      .replace('127.0.0.1 ', '')
+      .replace('::1 ', '')
+      .replace(':: ', '');
+  }).filter(domain => {
+    return domainValidationPattern.test(domain);
+  });
+  
+});
+
+
 // Read input.csv and parse domains
 fs.readFile('input.csv', 'utf8', async (err, data) => {
   if (err) {
@@ -33,6 +61,12 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
     return domainValidationPattern.test(domain);
   });
   
+  // Remove domains from the domains array that are present in the whitelist array
+  const whitelist = whitelist
+  domains = domains.filter(domain => {
+    return !whitelist.includes(domain);
+  });
+
   // Trim array to 300,000 domains if it's longer than that
   if (domains.length > LIST_ITEM_LIMIT) {
     domains = trimArray(domains, LIST_ITEM_LIMIT);
@@ -130,3 +164,4 @@ function percentage(percent, total) {
 function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
+