ported list download code to JS

.gitignore

@@ -9,8 +9,8 @@ lerna-debug.log*
 
 # CGPS
 output
-input.csv
-whitelist.csv
+blocklist.txt
+allowlist.txt
 
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json

cf_list_create.js

@@ -18,8 +18,8 @@ import {
   readFile,
 } from "./lib/utils.js";
 
-const allowlistFilename = "whitelist.csv";
-const blocklistFilename = "input.csv";
+const allowlistFilename = "allowlist.txt";
+const blocklistFilename = "blocklist.txt";
 const allowlist = new Map();
 const blocklist = new Map();
 const domains = [];

download_files.js

@@ -0,0 +1,5 @@
+import { resolve } from "path";
+
+import { downloadFiles } from "./lib/utils.js";
+
+downloadFiles(resolve(process.argv[2]), process.argv.slice(3));

get_recommended_filters.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-source $(dirname "$0")/lib/helpers.sh
-
 # declare an array of urls
 urls=(
   https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt
@@ -13,7 +11,7 @@ urls=(
 )
 
 # download all files in parallel and append them to input.csv
-download_lists $urls 'input.csv'
+node download_files.js blocklist.txt ${urls[@]}
 
 # print a message when done
-echo "Done. The input.csv file contains merged data from recommended filter lists."
+echo "Done. The blocklist.txt file contains merged data from recommended filter lists."

get_recommended_whitelist.sh

@@ -3,8 +3,6 @@
 # Use the provided lists or add your own.
 # There is no limit on the amount of whitelisted domains you can have.
 
-source $(dirname "$0")/lib/helpers.sh
-
 # declare an array of urls
 urls=(
     https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt
@@ -36,7 +34,7 @@ urls=(
 )
 
 # download all files in parallel and append them to whitelist.csv
-download_lists $urls 'whitelist.csv'
+node download_files.js allowlist.txt ${urls[@]}
 
 # print a message when done
-echo "Done. The whitelist.csv file contains merged data from recommended whitelists."
+echo "Done. The allowlist.txt file contains merged data from recommended whitelists."

lib/helpers.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-download_lists() {
-  urls=$1
-  output=$2
-  
-  curl -sSfL --parallel --parallel-max 10 --retry 3 ${urls[@]} > $output
-}

lib/utils.js

@@ -1,7 +1,12 @@
 import { once } from "events";
-import { createReadStream } from "fs";
+import { createReadStream, createWriteStream } from "fs";
 import { basename } from "path";
 import { createInterface } from "readline";
+import { WritableStream } from "stream/web";
+
+if (!globalThis.fetch) {
+  globalThis.fetch = (await import("node-fetch")).default;
+}
 
 /**
  * Sleeps for a specified amount of time.
@@ -46,6 +51,28 @@ export const isComment = (value) =>
   value.startsWith("/*") ||
   value.startsWith("*/");
 
+/**
+ * Downloads files and concatenates them into one file.
+ * @param {string} filePath The path to the file being written to.
+ * @param {string[]} urls The URLs to the files to be downloaded.
+ */
+export const downloadFiles = async (filePath, urls) => {
+  const writeStream = createWriteStream(filePath, { flags: "a" });
+  const writableStream = new WritableStream({
+    write(chunk) {
+      writeStream.write(chunk);
+    },
+  });
+  const responses = await Promise.all(urls.map((url) => fetch(url)));
+
+  for (const response of responses) {
+    await response.body?.pipeTo(writableStream, { preventClose: true });
+    writeStream.write("\n");
+  }
+
+  await writableStream.close();
+};
+
 /**
  * @callback onLine
  * @param {string} line The current line.