ported list download code to JS

lib/helpers.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-download_lists() {
-  urls=$1
-  output=$2
-  
-  curl -sSfL --parallel --parallel-max 10 --retry 3 ${urls[@]} > $output
-}

lib/utils.js

@@ -1,7 +1,12 @@
 import { once } from "events";
-import { createReadStream } from "fs";
+import { createReadStream, createWriteStream } from "fs";
 import { basename } from "path";
 import { createInterface } from "readline";
+import { WritableStream } from "stream/web";
+
+if (!globalThis.fetch) {
+  globalThis.fetch = (await import("node-fetch")).default;
+}
 
 /**
  * Sleeps for a specified amount of time.
@@ -46,6 +51,28 @@ export const isComment = (value) =>
   value.startsWith("/*") ||
   value.startsWith("*/");
 
+/**
+ * Downloads files and concatenates them into one file.
+ * @param {string} filePath The path to the file being written to.
+ * @param {string[]} urls The URLs to the files to be downloaded.
+ */
+export const downloadFiles = async (filePath, urls) => {
+  const writeStream = createWriteStream(filePath, { flags: "a" });
+  const writableStream = new WritableStream({
+    write(chunk) {
+      writeStream.write(chunk);
+    },
+  });
+  const responses = await Promise.all(urls.map((url) => fetch(url)));
+
+  for (const response of responses) {
+    await response.body?.pipeTo(writableStream, { preventClose: true });
+    writeStream.write("\n");
+  }
+
+  await writableStream.close();
+};
+
 /**
  * @callback onLine
  * @param {string} line The current line.