Merge pull request #25 from hlqviet/feature/node-download

.gitignore

@@ -11,6 +11,8 @@ lerna-debug.log*
 output
 input.csv
 whitelist.csv
+blocklist.txt
+allowlist.txt
 
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json

README.md

@@ -7,9 +7,10 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne
 ## About the individual scripts
 
 - `cf_list_delete.js` - Deletes all lists created by CGPS from Cloudflare Gateway. This is useful for subsequent runs.
-- `cf_list_create.js` - Takes an input.csv file containing domains and creates lists in Cloudflare Gateway
+- `cf_list_create.js` - Takes a blocklist.txt file containing domains and creates lists in Cloudflare Gateway
 - `cf_gateway_rule_create.js` - Creates a Cloudflare Gateway rule to block all traffic if it matches the lists created by CGPS.
 - `cf_gateway_rule_delete.js` - Deletes the Cloudflare Gateway rule created by CGPS. Useful for subsequent runs.
+- `download_lists.js` - Initiates blocklist and whitelist download.
 
 ## Features
 
@@ -27,8 +28,8 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne
 1. Node.js installed on your machine
 2. Cloudflare [Zero Trust](https://one.dash.cloudflare.com/) account - the Free plan is enough. Use the Cloudflare [documentation](https://developers.cloudflare.com/cloudflare-one/) for details.
 3. Cloudflare email, API key (NOT the API token), and account ID
-4. A file containing the domains you want to block - **max 300,000 domains for the free plan** - in the working directory named `input.csv`. Mullvad provides awesome [DNS blocklists](https://github.com/mullvad/dns-blocklists) that work well with this project. A bash script that downloads recommended blocklists, `get_recommended_filters.sh`, is included.
-5. Optional: You can whitelist domains by putting them in a file `whitelist.csv`. You can also use the `get_recomended_whitelist.sh` Bash script to get the recommended whitelists.
+4. A file containing the domains you want to block - **max 300,000 domains for the free plan** - in the working directory named `blocklist.txt`. Mullvad provides awesome [DNS blocklists](https://github.com/mullvad/dns-blocklists) that work well with this project. A bash script that downloads recommended blocklists, `get_recommended_filters.sh`, is included.
+5. Optional: You can whitelist domains by putting them in a file `allowlist.txt`. You can also use the `get_recomended_whitelist.sh` Bash script to get the recommended whitelists.
 
 ### Running locally
 

cf_list_create.js

@@ -1,3 +1,4 @@
+import { existsSync } from "fs";
 import { resolve } from "path";
 
 import {
@@ -9,6 +10,7 @@ import {
   FAST_MODE,
   LIST_ITEM_LIMIT,
   LIST_ITEM_SIZE,
+  PROCESSING_FILENAME,
 } from "./lib/constants.js";
 import { normalizeDomain } from "./lib/helpers.js";
 import {
@@ -18,8 +20,12 @@ import {
   readFile,
 } from "./lib/utils.js";
 
-const allowlistFilename = "whitelist.csv";
-const blocklistFilename = "input.csv";
+const allowlistFilename = existsSync(PROCESSING_FILENAME.OLD_ALLOWLIST)
+  ? PROCESSING_FILENAME.OLD_ALLOWLIST
+  : PROCESSING_FILENAME.ALLOWLIST;
+const blocklistFilename = existsSync(PROCESSING_FILENAME.OLD_BLOCKLIST)
+  ? PROCESSING_FILENAME.OLD_BLOCKLIST
+  : PROCESSING_FILENAME.BLOCKLIST;
 const allowlist = new Map();
 const blocklist = new Map();
 const domains = [];
@@ -30,7 +36,7 @@ let allowedDomainCount = 0;
 
 // Read allowlist
 console.log(`Processing ${allowlistFilename}`);
-await readFile(resolve(allowlistFilename), (line) => {
+await readFile(resolve(`./${allowlistFilename}`), (line) => {
   const _line = line.trim();
 
   if (!_line) return;
@@ -46,7 +52,7 @@ await readFile(resolve(allowlistFilename), (line) => {
 
 // Read blocklist
 console.log(`Processing ${blocklistFilename}`);
-await readFile(resolve(blocklistFilename), (line, rl) => {
+await readFile(resolve(`./${blocklistFilename}`), (line, rl) => {
   if (domains.length === LIST_ITEM_LIMIT) {
     return;
   }

download_lists.js

@@ -0,0 +1,74 @@
+import { resolve } from "path";
+
+import { LIST_TYPE, PROCESSING_FILENAME } from "./lib/constants.js";
+import { downloadFiles } from "./lib/utils.js";
+
+const allowlistUrls = [
+  "https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt",
+  "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt",
+  "https://raw.githubusercontent.com/TogoFire-Home/AD-Settings/main/Filters/whitelist.txt",
+  "https://raw.githubusercontent.com/freekers/whitelist/master/domains/whitelist.txt",
+  "https://raw.githubusercontent.com/DandelionSprout/AdGuard-Home-Whitelist/master/whitelist.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt",
+  "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/optional-list.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/issues.txt",
+  "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt",
+  "https://raw.githubusercontent.com/mawenjian/china-cdn-domain-whitelist/master/china-cdn-domain-whitelist.txt",
+  "https://raw.githubusercontent.com/notracking/hosts-blocklists-scripts/master/hostnames.whitelist.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/mac.txt",
+  "https://raw.githubusercontent.com/boutetnico/url-shorteners/master/list.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/windows.txt",
+  "https://raw.githubusercontent.com/Dogino/Discord-Phishing-URLs/main/official-domains.txt",
+  "https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/android.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/sensitive.txt",
+  "https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt",
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/firefox.txt",
+  // Commented out because it whitelists sites including doubleclick.net and ad.atdmt.com
+  // https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt,
+  // Uncomment the line below to use OISD's most commmonly whitelisted list
+  // https://local.oisd.nl/extract/commonly_whitelisted.php,
+];
+const blocklistUrls = [
+  "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt",
+  "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_gambling.txt",
+  "https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_privacy.txt",
+  "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts",
+  "https://adaway.org/hosts.txt",
+  "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
+];
+const listType = process.argv[2];
+
+const downloadAllowlists = async () => {
+  await downloadFiles(
+    resolve(`./${PROCESSING_FILENAME.ALLOWLIST}`),
+    allowlistUrls
+  );
+  console.log(
+    `Done. The ${PROCESSING_FILENAME.ALLOWLIST} file contains merged data from recommended allowlists.`
+  );
+};
+
+const downloadBlocklists = async () => {
+  await downloadFiles(
+    resolve(`./${PROCESSING_FILENAME.BLOCKLIST}`),
+    blocklistUrls
+  );
+  console.log(
+    `Done. The ${PROCESSING_FILENAME.BLOCKLIST} file contains merged data from recommended blocklists.`
+  );
+};
+
+switch (listType) {
+  case LIST_TYPE.ALLOWLIST: {
+    await downloadAllowlists();
+    break;
+  }
+  case LIST_TYPE.BLOCKLIST: {
+    await downloadBlocklists();
+    break;
+  }
+  default:
+    await Promise.all([downloadAllowlists(), downloadBlocklists()]);
+}

get_recommended_filters.sh

@@ -1,19 +1,3 @@
 #!/bin/bash
 
-source $(dirname "$0")/lib/helpers.sh
-
-# declare an array of urls
-urls=(
-  https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt
-  https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_gambling.txt
-  https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_privacy.txt
-  https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts
-  https://adaway.org/hosts.txt
-  https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
-)
-
-# download all files in parallel and append them to input.csv
-download_lists $urls 'input.csv'
-
-# print a message when done
-echo "Done. The input.csv file contains merged data from recommended filter lists."
+node download_lists.js blocklist

get_recommended_whitelist.sh

@@ -1,42 +1,3 @@
 #!/bin/bash
-#  
-# Use the provided lists or add your own.
-# There is no limit on the amount of whitelisted domains you can have.
 
-source $(dirname "$0")/lib/helpers.sh
-
-# declare an array of urls
-urls=(
-    https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt
-    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt
-    https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt
-    https://raw.githubusercontent.com/TogoFire-Home/AD-Settings/main/Filters/whitelist.txt
-    https://raw.githubusercontent.com/freekers/whitelist/master/domains/whitelist.txt
-    https://raw.githubusercontent.com/DandelionSprout/AdGuard-Home-Whitelist/master/whitelist.txt
-    https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt
-    https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/optional-list.txt
-    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/issues.txt
-    https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt
-    https://raw.githubusercontent.com/mawenjian/china-cdn-domain-whitelist/master/china-cdn-domain-whitelist.txt
-    https://raw.githubusercontent.com/notracking/hosts-blocklists-scripts/master/hostnames.whitelist.txt
-    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/mac.txt
-    https://raw.githubusercontent.com/boutetnico/url-shorteners/master/list.txt
-    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/windows.txt
-    https://raw.githubusercontent.com/Dogino/Discord-Phishing-URLs/main/official-domains.txt
-    https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt
-    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/android.txt
-    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/sensitive.txt
-    https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt
-    https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/firefox.txt
-    # Commented out because it whitelists sites including doubleclick.net and ad.atdmt.com
-    # https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt
-    # Uncomment the line below to use OISD's most commmonly whitelisted list
-    # https://local.oisd.nl/extract/commonly_whitelisted.php
-
-)
-
-# download all files in parallel and append them to whitelist.csv
-download_lists $urls 'whitelist.csv'
-
-# print a message when done
-echo "Done. The whitelist.csv file contains merged data from recommended whitelists."
+node download_lists.js allowlist

lib/constants.js

@@ -19,3 +19,15 @@ export const API_HOST = "https://api.cloudflare.com/client/v4";
 export const DRY_RUN = !!parseInt(process.env.DRY_RUN, 10);
 
 export const FAST_MODE = !!parseInt(process.env.FAST_MODE, 10);
+
+export const PROCESSING_FILENAME = {
+  ALLOWLIST: "allowlist.txt",
+  BLOCKLIST: "blocklist.txt",
+  OLD_ALLOWLIST: "whitelist.csv",
+  OLD_BLOCKLIST: "input.csv",
+};
+
+export const LIST_TYPE = {
+  ALLOWLIST: "allowlist",
+  BLOCKLIST: "blocklist",
+};

lib/helpers.js

@@ -1,9 +1,14 @@
 import { ACCOUNT_EMAIL, ACCOUNT_ID, API_HOST, API_TOKEN } from "./constants.js";
 
 if (!globalThis.fetch) {
-  console.warn("\nIMPORTANT: Your Node.js version doesn't have native fetch support and may not be supported in the future. Please update to v18 or later.\n")
+  console.warn(
+    "\nIMPORTANT: Your Node.js version doesn't have native fetch support and may not be supported in the future. Please update to v18 or later.\n"
+  );
   // Advise what to do if running in GitHub Actions
-  if (process.env.GITHUB_WORKSPACE) console.warn("Since you're running in GitHub Actions, you should update your Actions workflow configuration to use Node v18 or higher.") 
+  if (process.env.GITHUB_WORKSPACE)
+    console.warn(
+      "Since you're running in GitHub Actions, you should update your Actions workflow configuration to use Node v18 or higher."
+    );
   // Import node-fetch since there's no native fetch in this environment
   globalThis.fetch = (await import("node-fetch")).default;
 }
@@ -48,7 +53,7 @@ export const requestGateway = (path, options) =>
 /**
  * Normalizes a domain.
  * @param {string} value The value to be normalized.
- * @param {boolean} isAllowlisting Whether the value is to be whitelisted.
+ * @param {boolean} isAllowlisting Whether the value is to be allowlisted.
  * @returns {string}
  */
 export const normalizeDomain = (value, isAllowlisting) => {

lib/helpers.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-download_lists() {
-  urls=$1
-  output=$2
-  
-  curl -sSfL --parallel --parallel-max 10 --retry 3 ${urls[@]} > $output
-}

lib/utils.js

@@ -1,7 +1,12 @@
 import { once } from "events";
-import { createReadStream } from "fs";
+import { createReadStream, createWriteStream } from "fs";
 import { basename } from "path";
 import { createInterface } from "readline";
+import { WritableStream } from "stream/web";
+
+if (!globalThis.fetch) {
+  globalThis.fetch = (await import("node-fetch")).default;
+}
 
 /**
  * Sleeps for a specified amount of time.
@@ -46,6 +51,28 @@ export const isComment = (value) =>
   value.startsWith("/*") ||
   value.startsWith("*/");
 
+/**
+ * Downloads files and concatenates them into one file.
+ * @param {string} filePath The path to the file being written to.
+ * @param {string[]} urls The URLs to the files to be downloaded.
+ */
+export const downloadFiles = async (filePath, urls) => {
+  const writeStream = createWriteStream(filePath, { flags: "a" });
+  const writableStream = new WritableStream({
+    write(chunk) {
+      writeStream.write(chunk);
+    },
+  });
+  const responses = await Promise.all(urls.map((url) => fetch(url)));
+
+  for (const response of responses) {
+    await response.body?.pipeTo(writableStream, { preventClose: true });
+    writeStream.write("\n");
+  }
+
+  await writableStream.close();
+};
+
 /**
  * @callback onLine
  * @param {string} line The current line.