Merge pull request #7 from norbertjoni/v1

Whitelist support
2026-09-22 13:38:37 +00:00 · 2023-05-15 15:11:12 +02:00
parent d7bc517ec1 1b6c744cb2
commit 09d10297b1
4 changed files with 105 additions and 2 deletions
@@ -17,6 +17,7 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne
 - Full support for domain lists
 - Automatically cleans up filter lists: removes duplicates, invalid domains, comments and more
 - Works fully unattended
+- Whitelist support, allowing you to prevent false positives and breakage by forcing trusted domains to always be unblocked.

 ## Usage

@@ -26,6 +27,7 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne
 2. Cloudflare [Zero Trust](https://one.dash.cloudflare.com/) account - the Free plan is enough. Use the Cloudflare [documentation](https://developers.cloudflare.com/cloudflare-one/) for details.
 3. Cloudflare email, API key (NOT the API token), and account ID
 4. A file containing the domains you want to block - **max 300,000 domains for the free plan** - in the working directory named `input.csv`. Mullvad provides awesome [DNS blocklists](https://github.com/mullvad/dns-blocklists) that work well with this project. A bash script that downloads recommended blocklists, `get_recommended_filters.sh`, is included.
+5. Optional: You can whitelist domains by putting them in a file `whitelist.csv`. You can also use the `get_recomended_whitelist.sh` Bash script to get the recommended whitelists.

 ### Running locally

@@ -42,6 +44,8 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne

 These scripts can be run using GitHub Actions so your filters will be automatically updated and pushed to Cloudflare Gateway. This is useful if you are using a frequently updated malware blocklist.

+Please note that the GitHub Action downloads the recommended blocklists and whitelist by default. You can change this behavior by editing the file.
+
 1. Create a new empty, private repository. Forking or public repositories are discouraged, but supported - although the script never leaks your API keys and GitHub Actions secrets are automatically redacted from the logs, it's better to be safe than sorry.
 2. Create the following GitHub Actions secrets in your repository settings:

@@ -26,7 +26,11 @@ jobs:
    - name: Install npm dependencies
      run: npm ci
      working-directory: cloudflare-gateway-pihole-scripts
-
+      
+    - name: Download recommended whitelist
+      run: bash ./get_recommended_whitelist.sh
+      working-directory: cloudflare-gateway-pihole-scripts
+      
    - name: Download recommended filters
      run: bash ./get_recommended_filters.sh
      working-directory: cloudflare-gateway-pihole-scripts
@@ -9,6 +9,39 @@ const LIST_ITEM_LIMIT = Number.isSafeInteger(Number(process.env.CLOUDFLARE_LIST_

 if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`);

+let whitelist = []; // Define an empty array for the whitelist
+
+// Read whitelist.csv and parse
+fs.readFile('whitelist.csv', 'utf8', async (err, data) => {
+  if (err) {
+    console.warn('Error reading whitelist.csv:', err);
+    console.warn('Assuming whitelist is empty.')
+  } else {
+    // Convert into array and cleanup whitelist
+    const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/;
+    whitelist = data.split('\n').filter(domain => {
+      // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments
+      return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r');
+    }).map(domain => {
+      // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items
+      return domain
+        .replace('\r', '')
+        .replace('0.0.0.0 ', '')
+        .replace('127.0.0.1 ', '')
+        .replace('::1 ', '')
+        .replace(':: ', '')
+        .replace('||', '')
+        .replace('@@||', '')
+        .replace('^$important', '')
+        .replace('^', '');
+    }).filter(domain => {
+      return domainValidationPattern.test(domain);
+    });
+  }  
+});
+
+console.log(`Found ${whitelist.length} valid domains in whitelist.`);
+
 // Read input.csv and parse domains
 fs.readFile('input.csv', 'utf8', async (err, data) => {
  if (err) {
@@ -28,11 +61,21 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
      .replace('0.0.0.0 ', '')
      .replace('127.0.0.1 ', '')
      .replace('::1 ', '')
-      .replace(':: ', '');
+      .replace(':: ', '')
+      .replace('^', '')
+      .replace('||', '')
+      .replace('@@||', '')
+      .replace('^$important', '')
+      .replace('^', '');
  }).filter(domain => {
    return domainValidationPattern.test(domain);
  });
  
+  // Remove domains from the domains array that are present in the whitelist array
+  domains = domains.filter(domain => {
+    return !whitelist.includes(domain);
+  });
+
  // Trim array to 300,000 domains if it's longer than that
  if (domains.length > LIST_ITEM_LIMIT) {
    domains = trimArray(domains, LIST_ITEM_LIMIT);
@@ -130,3 +173,4 @@ function percentage(percent, total) {
 function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
 }
+
@@ -0,0 +1,51 @@
+#!/bin/bash
+#  
+# Use the provided lists or add you own
+# https://oisd.nl/includedlists/whitelists
+# by creating an empty whitelist.csv file
+touch whitelist.csv
+
+# declare an array of urls
+urls=(
+    https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt
+    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt
+    https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt
+    https://raw.githubusercontent.com/TogoFire-Home/AD-Settings/main/Filters/whitelist.txt
+    https://raw.githubusercontent.com/freekers/whitelist/master/domains/whitelist.txt
+    https://raw.githubusercontent.com/DandelionSprout/AdGuard-Home-Whitelist/master/whitelist.txt
+    # Commented out because it looks suspicious
+    # https://www.aadvantageeshopping.com/adBlockWhitelist.php
+    https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt
+    https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/optional-list.txt
+    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/issues.txt
+    https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt
+    https://raw.githubusercontent.com/mawenjian/china-cdn-domain-whitelist/master/china-cdn-domain-whitelist.txt
+    https://raw.githubusercontent.com/notracking/hosts-blocklists-scripts/master/hostnames.whitelist.txt
+    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/mac.txt
+    https://raw.githubusercontent.com/boutetnico/url-shorteners/master/list.txt
+    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/windows.txt
+    https://raw.githubusercontent.com/Dogino/Discord-Phishing-URLs/main/official-domains.txt
+    https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt
+    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/android.txt
+    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/sensitive.txt
+    https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt
+    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/firefox.txt
+    https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt
+
+)
+
+# loop through the urls and download each file with curl
+for url in "${urls[@]}"; do
+  # get the file name from the url
+  file=$(basename "$url")
+  # download the file with curl and save it as file.txt
+  curl -o "$file.txt" "$url"
+  # append the file contents to whitelist.csv and add a newline
+  cat "$file.txt" >> whitelist.csv
+  echo "" >> whitelist.csv
+  # remove the file.txt
+  rm "$file.txt"
+done
+
+# print a message when done
+echo "Done. The whitelist.csv file contains merged data from recommended whitelists."