From 652101efafcafda63c59c644986dfa764ae686cd Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Mon, 18 Sep 2023 13:42:09 +0700 Subject: [PATCH 1/5] added support for lists from environment variables --- auto_update_github_action.yml | 4 ++ download_lists.js | 74 +++++++++-------------------------- lib/constants.js | 43 ++++++++++++++++++++ 3 files changed, 66 insertions(+), 55 deletions(-) diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index b23b022..cd36700 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -35,9 +35,13 @@ jobs: - name: Download recommended whitelist run: bash ./get_recommended_whitelist.sh + env: + ALLOWLIST_URLS: ${{ vars.ALLOWLIST_URLS }} - name: Download recommended filters run: bash ./get_recommended_filters.sh + env: + BLOCKLIST_URLS: ${{ vars.BLOCKLIST_URLS }} - name: Delete old rules and lists run: | diff --git a/download_lists.js b/download_lists.js index 56e4a19..34e4413 100644 --- a/download_lists.js +++ b/download_lists.js @@ -1,74 +1,38 @@ import { resolve } from "path"; -import { LIST_TYPE, PROCESSING_FILENAME } from "./lib/constants.js"; +import { + LIST_TYPE, + PROCESSING_FILENAME, + RECOMMENDED_ALLOWLIST_URLS, + RECOMMENDED_BLOCKLIST_URLS, + USER_DEFINED_ALLOWLIST_URLS, + USER_DEFINED_BLOCKLIST_URLS, +} from "./lib/constants.js"; import { downloadFiles } from "./lib/utils.js"; -const allowlistUrls = [ - "https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt", - "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt", - "https://raw.githubusercontent.com/TogoFire-Home/AD-Settings/main/Filters/whitelist.txt", - "https://raw.githubusercontent.com/freekers/whitelist/master/domains/whitelist.txt", - "https://raw.githubusercontent.com/DandelionSprout/AdGuard-Home-Whitelist/master/whitelist.txt", - "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt", - "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/optional-list.txt", - "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/issues.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt", - "https://raw.githubusercontent.com/mawenjian/china-cdn-domain-whitelist/master/china-cdn-domain-whitelist.txt", - "https://raw.githubusercontent.com/notracking/hosts-blocklists-scripts/master/hostnames.whitelist.txt", - "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/mac.txt", - "https://raw.githubusercontent.com/boutetnico/url-shorteners/master/list.txt", - "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/windows.txt", - "https://raw.githubusercontent.com/Dogino/Discord-Phishing-URLs/main/official-domains.txt", - "https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt", - "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/android.txt", - "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/sensitive.txt", - "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt", - "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/firefox.txt", - // Commented out because it whitelists sites including doubleclick.net and ad.atdmt.com - // https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt, - // Uncomment the line below to use OISD's most commmonly whitelisted list - // https://local.oisd.nl/extract/commonly_whitelisted.php, -]; -const blocklistUrls = [ - "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt", - "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_gambling.txt", - "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_privacy.txt", - "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts", - "https://adaway.org/hosts.txt", - "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts", -]; +const allowlistUrls = USER_DEFINED_ALLOWLIST_URLS || RECOMMENDED_ALLOWLIST_URLS; +const blocklistUrls = USER_DEFINED_BLOCKLIST_URLS || RECOMMENDED_BLOCKLIST_URLS; const listType = process.argv[2]; -const downloadAllowlists = async () => { - await downloadFiles( - resolve(`./${PROCESSING_FILENAME.ALLOWLIST}`), - allowlistUrls - ); +const downloadLists = async (filename, urls) => { + await downloadFiles(resolve(`./${filename}`), urls); console.log( - `Done. The ${PROCESSING_FILENAME.ALLOWLIST} file contains merged data from recommended allowlists.` - ); -}; - -const downloadBlocklists = async () => { - await downloadFiles( - resolve(`./${PROCESSING_FILENAME.BLOCKLIST}`), - blocklistUrls - ); - console.log( - `Done. The ${PROCESSING_FILENAME.BLOCKLIST} file contains merged data from recommended blocklists.` + `Done. The ${filename} file contains merged data from ${urls.length} list(s).` ); }; switch (listType) { case LIST_TYPE.ALLOWLIST: { - await downloadAllowlists(); + await downloadLists(PROCESSING_FILENAME.ALLOWLIST, allowlistUrls); break; } case LIST_TYPE.BLOCKLIST: { - await downloadBlocklists(); + await downloadLists(PROCESSING_FILENAME.BLOCKLIST, blocklistUrls); break; } default: - await Promise.all([downloadAllowlists(), downloadBlocklists()]); + await Promise.all([ + downloadLists(PROCESSING_FILENAME.ALLOWLIST, allowlistUrls), + downloadLists(PROCESSING_FILENAME.BLOCKLIST, blocklistUrls), + ]); } diff --git a/lib/constants.js b/lib/constants.js index 9d5e124..6d1b31a 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -31,3 +31,46 @@ export const LIST_TYPE = { ALLOWLIST: "allowlist", BLOCKLIST: "blocklist", }; + +export const USER_DEFINED_ALLOWLIST_URLS = + process.env.ALLOWLIST_URLS?.split("\n"); + +export const USER_DEFINED_BLOCKLIST_URLS = + process.env.BLOCKLIST_URLS?.split("\n"); + +export const RECOMMENDED_ALLOWLIST_URLS = [ + "https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt", + "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt", + "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt", + "https://raw.githubusercontent.com/TogoFire-Home/AD-Settings/main/Filters/whitelist.txt", + "https://raw.githubusercontent.com/freekers/whitelist/master/domains/whitelist.txt", + "https://raw.githubusercontent.com/DandelionSprout/AdGuard-Home-Whitelist/master/whitelist.txt", + "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt", + "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/optional-list.txt", + "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/issues.txt", + "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt", + "https://raw.githubusercontent.com/mawenjian/china-cdn-domain-whitelist/master/china-cdn-domain-whitelist.txt", + "https://raw.githubusercontent.com/notracking/hosts-blocklists-scripts/master/hostnames.whitelist.txt", + "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/mac.txt", + "https://raw.githubusercontent.com/boutetnico/url-shorteners/master/list.txt", + "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/windows.txt", + "https://raw.githubusercontent.com/Dogino/Discord-Phishing-URLs/main/official-domains.txt", + "https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt", + "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/android.txt", + "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/sensitive.txt", + "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt", + "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/firefox.txt", + // Commented out because it whitelists sites including doubleclick.net and ad.atdmt.com + // https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt, + // Uncomment the line below to use OISD's most commmonly whitelisted list + // https://local.oisd.nl/extract/commonly_whitelisted.php, +]; + +export const RECOMMENDED_BLOCKLIST_URLS = [ + "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt", + "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_gambling.txt", + "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_privacy.txt", + "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts", + "https://adaway.org/hosts.txt", + "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts", +]; From 3cf872f2629b247ab489ee4371518ce2b686fa4e Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Mon, 18 Sep 2023 13:42:26 +0700 Subject: [PATCH 2/5] updated .env example --- .env.example | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.env.example b/.env.example index ee7845c..14dd3b2 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,8 @@ CLOUDFLARE_API_KEY= CLOUDFLARE_ACCOUNT_ID= CLOUDFLARE_ACCOUNT_EMAIL= CLOUDFLARE_LIST_ITEM_LIMIT=300000 +DRY_RUN=0 +FAST_MODE=1 +ALLOWLIST_URLS="https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt +https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt" +BLOCKLIST_URLS=https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts From b20befbe310a5691d7f40a7885ebdf9c397b7983 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Mon, 18 Sep 2023 13:51:57 +0700 Subject: [PATCH 3/5] updated instructions on ALLOWLIST_URLS and BLOCKLIST_URLS --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a12f313..db6e6cb 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ Please note that the GitHub Action downloads the recommended blocklists and whit 3. Create the following GitHub Actions variables in your repository settings if you desire: - `FAST_MODE`: Enable the scripts to send the requests simultaneously. Beware that there's a rate limit of 1200 requests per five minutes (https://developers.cloudflare.com/fundamentals/api/reference/limits/) so make sure you know what you are doing. +- `ALLOWLIST_URLS`: Uses your own allowlists. One URL per line. Recommended allowlists will be used if this variable is not provided. +- `BLOCKLIST_URLS`: Uses your own blocklists. One URL per line. Recommended blocklists will be used if this variable is not provided. 4. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property. 5. Enable GitHub Actions in your repository settings. From ae9afe3b6164a2cf5b87458b1c75eae5098777bb Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Tue, 19 Sep 2023 01:31:30 +0700 Subject: [PATCH 4/5] updated defaults --- .env.example | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 14dd3b2..83fcadf 100644 --- a/.env.example +++ b/.env.example @@ -3,7 +3,9 @@ CLOUDFLARE_ACCOUNT_ID= CLOUDFLARE_ACCOUNT_EMAIL= CLOUDFLARE_LIST_ITEM_LIMIT=300000 DRY_RUN=0 -FAST_MODE=1 -ALLOWLIST_URLS="https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt -https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt" -BLOCKLIST_URLS=https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts +FAST_MODE=0 + +# The following lists will be used instead of the recommended ones if you enter any. +# Multiline is supported: https://github.com/motdotla/dotenv#multiline-values +# ALLOWLIST_URLS= +# BLOCKLIST_URLS= From f4868b59461c3daf7964a8c593fee6c5c43353d7 Mon Sep 17 00:00:00 2001 From: Viet Huynh Date: Tue, 19 Sep 2023 01:32:01 +0700 Subject: [PATCH 5/5] log the lists to be used --- download_lists.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/download_lists.js b/download_lists.js index 34e4413..3f04054 100644 --- a/download_lists.js +++ b/download_lists.js @@ -17,7 +17,13 @@ const listType = process.argv[2]; const downloadLists = async (filename, urls) => { await downloadFiles(resolve(`./${filename}`), urls); console.log( - `Done. The ${filename} file contains merged data from ${urls.length} list(s).` + `Done. The ${filename} file contains merged data from the following list(s):` + ); + console.log( + urls.reduce( + (previous, current, index) => previous + `${index + 1}. ${current}\n`, + "" + ) ); };