diff --git a/README.md b/README.md index 0562bc9..43afebb 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne - Full support for domain lists - Automatically cleans up filter lists: removes duplicates, invalid domains, comments and more - Works fully unattended +- Whitelist support, allowing you to prevent false positives and breakage by forcing trusted domains to always be unblocked. ## Usage @@ -26,6 +27,7 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne 2. Cloudflare [Zero Trust](https://one.dash.cloudflare.com/) account - the Free plan is enough. Use the Cloudflare [documentation](https://developers.cloudflare.com/cloudflare-one/) for details. 3. Cloudflare email, API key (NOT the API token), and account ID 4. A file containing the domains you want to block - **max 300,000 domains for the free plan** - in the working directory named `input.csv`. Mullvad provides awesome [DNS blocklists](https://github.com/mullvad/dns-blocklists) that work well with this project. A bash script that downloads recommended blocklists, `get_recommended_filters.sh`, is included. +5. Optional: You can whitelist domains by putting them in a file `whitelist.csv`. You can also use the `get_recomended_whitelist.sh` Bash script to get the recommended whitelists. ### Running locally @@ -42,6 +44,8 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne These scripts can be run using GitHub Actions so your filters will be automatically updated and pushed to Cloudflare Gateway. This is useful if you are using a frequently updated malware blocklist. +Please note that the GitHub Action downloads the recommended blocklists and whitelist by default. You can change this behavior by editing the file. + 1. Create a new empty, private repository. Forking or public repositories are discouraged, but supported - although the script never leaks your API keys and GitHub Actions secrets are automatically redacted from the logs, it's better to be safe than sorry. 2. Create the following GitHub Actions secrets in your repository settings: diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index 5ab8301..747b937 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -26,7 +26,11 @@ jobs: - name: Install npm dependencies run: npm ci working-directory: cloudflare-gateway-pihole-scripts - + + - name: Download recommended whitelist + run: bash ./get_recommended_whitelist.sh + working-directory: cloudflare-gateway-pihole-scripts + - name: Download recommended filters run: bash ./get_recommended_filters.sh working-directory: cloudflare-gateway-pihole-scripts diff --git a/cf_list_create.js b/cf_list_create.js index 64535e6..07237e0 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -9,6 +9,39 @@ const LIST_ITEM_LIMIT = Number.isSafeInteger(Number(process.env.CLOUDFLARE_LIST_ if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`); +let whitelist = []; // Define an empty array for the whitelist + +// Read whitelist.csv and parse +fs.readFile('whitelist.csv', 'utf8', async (err, data) => { + if (err) { + console.warn('Error reading whitelist.csv:', err); + console.warn('Assuming whitelist is empty.') + } else { + // Convert into array and cleanup whitelist + const domainValidationPattern = /^(?!-)[A-Za-z0-9-]+([\-\.]{1}[a-z0-9]+)*\.[A-Za-z]{2,6}$/; + whitelist = data.split('\n').filter(domain => { + // Remove entire lines starting with "127.0.0.1" or "::1", empty lines or comments + return domain && !domain.startsWith('#') && !domain.startsWith('//') && !domain.startsWith('/*') && !domain.startsWith('*/') && !(domain === '\r'); + }).map(domain => { + // Remove "\r", "0.0.0.0 ", "127.0.0.1 ", "::1 " and similar from domain items + return domain + .replace('\r', '') + .replace('0.0.0.0 ', '') + .replace('127.0.0.1 ', '') + .replace('::1 ', '') + .replace(':: ', '') + .replace('||', '') + .replace('@@||', '') + .replace('^$important', '') + .replace('^', ''); + }).filter(domain => { + return domainValidationPattern.test(domain); + }); + } +}); + +console.log(`Found ${whitelist.length} valid domains in whitelist.`); + // Read input.csv and parse domains fs.readFile('input.csv', 'utf8', async (err, data) => { if (err) { @@ -28,11 +61,21 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { .replace('0.0.0.0 ', '') .replace('127.0.0.1 ', '') .replace('::1 ', '') - .replace(':: ', ''); + .replace(':: ', '') + .replace('^', '') + .replace('||', '') + .replace('@@||', '') + .replace('^$important', '') + .replace('^', ''); }).filter(domain => { return domainValidationPattern.test(domain); }); + // Remove domains from the domains array that are present in the whitelist array + domains = domains.filter(domain => { + return !whitelist.includes(domain); + }); + // Trim array to 300,000 domains if it's longer than that if (domains.length > LIST_ITEM_LIMIT) { domains = trimArray(domains, LIST_ITEM_LIMIT); @@ -130,3 +173,4 @@ function percentage(percent, total) { function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } + diff --git a/get_recommended_whitelist.sh b/get_recommended_whitelist.sh new file mode 100644 index 0000000..9f99ec9 --- /dev/null +++ b/get_recommended_whitelist.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Use the provided lists or add you own +# https://oisd.nl/includedlists/whitelists +# by creating an empty whitelist.csv file +touch whitelist.csv + +# declare an array of urls +urls=( + https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt + https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt + https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt + https://raw.githubusercontent.com/TogoFire-Home/AD-Settings/main/Filters/whitelist.txt + https://raw.githubusercontent.com/freekers/whitelist/master/domains/whitelist.txt + https://raw.githubusercontent.com/DandelionSprout/AdGuard-Home-Whitelist/master/whitelist.txt + # Commented out because it looks suspicious + # https://www.aadvantageeshopping.com/adBlockWhitelist.php + https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt + https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/optional-list.txt + https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/issues.txt + https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt + https://raw.githubusercontent.com/mawenjian/china-cdn-domain-whitelist/master/china-cdn-domain-whitelist.txt + https://raw.githubusercontent.com/notracking/hosts-blocklists-scripts/master/hostnames.whitelist.txt + https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/mac.txt + https://raw.githubusercontent.com/boutetnico/url-shorteners/master/list.txt + https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/windows.txt + https://raw.githubusercontent.com/Dogino/Discord-Phishing-URLs/main/official-domains.txt + https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt + https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/android.txt + https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/sensitive.txt + https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt + https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/firefox.txt + https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt + +) + +# loop through the urls and download each file with curl +for url in "${urls[@]}"; do + # get the file name from the url + file=$(basename "$url") + # download the file with curl and save it as file.txt + curl -o "$file.txt" "$url" + # append the file contents to whitelist.csv and add a newline + cat "$file.txt" >> whitelist.csv + echo "" >> whitelist.csv + # remove the file.txt + rm "$file.txt" +done + +# print a message when done +echo "Done. The whitelist.csv file contains merged data from recommended whitelists."