Merge pull request #26 from hlqviet/feature/lists-as-environment-variables

Support allowlists and blocklists from environment variables

.env.example

@@ -2,3 +2,10 @@ CLOUDFLARE_API_KEY=
 CLOUDFLARE_ACCOUNT_ID=
 CLOUDFLARE_ACCOUNT_EMAIL=
 CLOUDFLARE_LIST_ITEM_LIMIT=300000
+DRY_RUN=0
+FAST_MODE=0
+
+# The following lists will be used instead of the recommended ones if you enter any.
+# Multiline is supported: https://github.com/motdotla/dotenv#multiline-values
+# ALLOWLIST_URLS=
+# BLOCKLIST_URLS=

README.md

@@ -60,6 +60,8 @@ Please note that the GitHub Action downloads the recommended blocklists and whit
 3. Create the following GitHub Actions variables in your repository settings if you desire:
 
 - `FAST_MODE`: Enable the scripts to send the requests simultaneously. Beware that there's a rate limit of 1200 requests per five minutes (https://developers.cloudflare.com/fundamentals/api/reference/limits/) so make sure you know what you are doing.
+- `ALLOWLIST_URLS`: Uses your own allowlists. One URL per line. Recommended allowlists will be used if this variable is not provided.
+- `BLOCKLIST_URLS`: Uses your own blocklists. One URL per line. Recommended blocklists will be used if this variable is not provided.
 
 4. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property.
 5. Enable GitHub Actions in your repository settings.

auto_update_github_action.yml

@@ -35,9 +35,13 @@ jobs:
 
       - name: Download recommended whitelist
         run: bash ./get_recommended_whitelist.sh
+        env:
+          ALLOWLIST_URLS: ${{ vars.ALLOWLIST_URLS }}
 
       - name: Download recommended filters
         run: bash ./get_recommended_filters.sh
+        env:
+          BLOCKLIST_URLS: ${{ vars.BLOCKLIST_URLS }}
 
       - name: Delete old rules and lists
         run: |

download_lists.js

@@ -1,74 +1,44 @@
 import { resolve } from "path";
 
-import { LIST_TYPE, PROCESSING_FILENAME } from "./lib/constants.js";
+import {
+  LIST_TYPE,
+  PROCESSING_FILENAME,
+  RECOMMENDED_ALLOWLIST_URLS,
+  RECOMMENDED_BLOCKLIST_URLS,
+  USER_DEFINED_ALLOWLIST_URLS,
+  USER_DEFINED_BLOCKLIST_URLS,
+} from "./lib/constants.js";
 import { downloadFiles } from "./lib/utils.js";
 
-const allowlistUrls = [
+const allowlistUrls = USER_DEFINED_ALLOWLIST_URLS || RECOMMENDED_ALLOWLIST_URLS;
-  "https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt",
+const blocklistUrls = USER_DEFINED_BLOCKLIST_URLS || RECOMMENDED_BLOCKLIST_URLS;
-  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt",
-  "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt",
-  "https://raw.githubusercontent.com/TogoFire-Home/AD-Settings/main/Filters/whitelist.txt",
-  "https://raw.githubusercontent.com/freekers/whitelist/master/domains/whitelist.txt",
-  "https://raw.githubusercontent.com/DandelionSprout/AdGuard-Home-Whitelist/master/whitelist.txt",
-  "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt",
-  "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/optional-list.txt",
-  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/issues.txt",
-  "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt",
-  "https://raw.githubusercontent.com/mawenjian/china-cdn-domain-whitelist/master/china-cdn-domain-whitelist.txt",
-  "https://raw.githubusercontent.com/notracking/hosts-blocklists-scripts/master/hostnames.whitelist.txt",
-  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/mac.txt",
-  "https://raw.githubusercontent.com/boutetnico/url-shorteners/master/list.txt",
-  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/windows.txt",
-  "https://raw.githubusercontent.com/Dogino/Discord-Phishing-URLs/main/official-domains.txt",
-  "https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt",
-  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/android.txt",
-  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/sensitive.txt",
-  "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt",
-  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/firefox.txt",
-  // Commented out because it whitelists sites including doubleclick.net and ad.atdmt.com
-  // https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt,
-  // Uncomment the line below to use OISD's most commmonly whitelisted list
-  // https://local.oisd.nl/extract/commonly_whitelisted.php,
-];
-const blocklistUrls = [
-  "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt",
-  "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_gambling.txt",
-  "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_privacy.txt",
-  "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts",
-  "https://adaway.org/hosts.txt",
-  "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
-];
 const listType = process.argv[2];
 
-const downloadAllowlists = async () => {
+const downloadLists = async (filename, urls) => {
-  await downloadFiles(
+  await downloadFiles(resolve(`./${filename}`), urls);
-    resolve(`./${PROCESSING_FILENAME.ALLOWLIST}`),
+  console.log(
-    allowlistUrls
+    `Done. The ${filename} file contains merged data from the following list(s):`
   );
   console.log(
-    `Done. The ${PROCESSING_FILENAME.ALLOWLIST} file contains merged data from recommended allowlists.`
+    urls.reduce(
-  );
+      (previous, current, index) => previous + `${index + 1}. ${current}\n`,
-};
+      ""
-
+    )
-const downloadBlocklists = async () => {
-  await downloadFiles(
-    resolve(`./${PROCESSING_FILENAME.BLOCKLIST}`),
-    blocklistUrls
-  );
-  console.log(
-    `Done. The ${PROCESSING_FILENAME.BLOCKLIST} file contains merged data from recommended blocklists.`
   );
 };
 
 switch (listType) {
   case LIST_TYPE.ALLOWLIST: {
-    await downloadAllowlists();
+    await downloadLists(PROCESSING_FILENAME.ALLOWLIST, allowlistUrls);
     break;
   }
   case LIST_TYPE.BLOCKLIST: {
-    await downloadBlocklists();
+    await downloadLists(PROCESSING_FILENAME.BLOCKLIST, blocklistUrls);
     break;
   }
   default:
-    await Promise.all([downloadAllowlists(), downloadBlocklists()]);
+    await Promise.all([
+      downloadLists(PROCESSING_FILENAME.ALLOWLIST, allowlistUrls),
+      downloadLists(PROCESSING_FILENAME.BLOCKLIST, blocklistUrls),
+    ]);
 }

lib/constants.js

@@ -31,3 +31,46 @@ export const LIST_TYPE = {
   ALLOWLIST: "allowlist",
   BLOCKLIST: "blocklist",
 };
+
+export const USER_DEFINED_ALLOWLIST_URLS =
+  process.env.ALLOWLIST_URLS?.split("\n");
+
+export const USER_DEFINED_BLOCKLIST_URLS =
+  process.env.BLOCKLIST_URLS?.split("\n");
+
+export const RECOMMENDED_ALLOWLIST_URLS = [
+  "https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt",
+  "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt",
+  "https://raw.githubusercontent.com/TogoFire-Home/AD-Settings/main/Filters/whitelist.txt",
+  "https://raw.githubusercontent.com/freekers/whitelist/master/domains/whitelist.txt",
+  "https://raw.githubusercontent.com/DandelionSprout/AdGuard-Home-Whitelist/master/whitelist.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt",
+  "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/optional-list.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/issues.txt",
+  "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt",
+  "https://raw.githubusercontent.com/mawenjian/china-cdn-domain-whitelist/master/china-cdn-domain-whitelist.txt",
+  "https://raw.githubusercontent.com/notracking/hosts-blocklists-scripts/master/hostnames.whitelist.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/mac.txt",
+  "https://raw.githubusercontent.com/boutetnico/url-shorteners/master/list.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/windows.txt",
+  "https://raw.githubusercontent.com/Dogino/Discord-Phishing-URLs/main/official-domains.txt",
+  "https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/android.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/sensitive.txt",
+  "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/firefox.txt",
+  // Commented out because it whitelists sites including doubleclick.net and ad.atdmt.com
+  // https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt,
+  // Uncomment the line below to use OISD's most commmonly whitelisted list
+  // https://local.oisd.nl/extract/commonly_whitelisted.php,
+];
+
+export const RECOMMENDED_BLOCKLIST_URLS = [
+  "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt",
+  "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_gambling.txt",
+  "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_privacy.txt",
+  "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts",
+  "https://adaway.org/hosts.txt",
+  "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
+];