diff --git a/.gitignore b/.gitignore index a6995ee..c8446c6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,8 @@ lerna-debug.log* # CGPS output -input.csv -whitelist.csv +blocklist.txt +allowlist.txt # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json diff --git a/cf_list_create.js b/cf_list_create.js index fb0e81b..b560c54 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -18,8 +18,8 @@ import { readFile, } from "./lib/utils.js"; -const allowlistFilename = "whitelist.csv"; -const blocklistFilename = "input.csv"; +const allowlistFilename = "allowlist.txt"; +const blocklistFilename = "blocklist.txt"; const allowlist = new Map(); const blocklist = new Map(); const domains = []; diff --git a/download_files.js b/download_files.js new file mode 100644 index 0000000..f9ef927 --- /dev/null +++ b/download_files.js @@ -0,0 +1,5 @@ +import { resolve } from "path"; + +import { downloadFiles } from "./lib/utils.js"; + +downloadFiles(resolve(process.argv[2]), process.argv.slice(3)); diff --git a/get_recommended_filters.sh b/get_recommended_filters.sh index 2c8547c..e030f48 100644 --- a/get_recommended_filters.sh +++ b/get_recommended_filters.sh @@ -1,7 +1,5 @@ #!/bin/bash -source $(dirname "$0")/lib/helpers.sh - # declare an array of urls urls=( https://raw.githubusercontent.com/mullvad/dns-blocklists/main/output/doh/doh_adblock.txt @@ -13,7 +11,7 @@ urls=( ) # download all files in parallel and append them to input.csv -download_lists $urls 'input.csv' +node download_files.js blocklist.txt ${urls[@]} # print a message when done -echo "Done. The input.csv file contains merged data from recommended filter lists." +echo "Done. The blocklist.txt file contains merged data from recommended filter lists." diff --git a/get_recommended_whitelist.sh b/get_recommended_whitelist.sh index 4035957..e1e2cc2 100644 --- a/get_recommended_whitelist.sh +++ b/get_recommended_whitelist.sh @@ -1,10 +1,8 @@ #!/bin/bash -# +# # Use the provided lists or add your own. # There is no limit on the amount of whitelisted domains you can have. -source $(dirname "$0")/lib/helpers.sh - # declare an array of urls urls=( https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt @@ -36,7 +34,7 @@ urls=( ) # download all files in parallel and append them to whitelist.csv -download_lists $urls 'whitelist.csv' +node download_files.js allowlist.txt ${urls[@]} # print a message when done -echo "Done. The whitelist.csv file contains merged data from recommended whitelists." +echo "Done. The allowlist.txt file contains merged data from recommended whitelists." diff --git a/lib/helpers.sh b/lib/helpers.sh deleted file mode 100644 index 80cfcd2..0000000 --- a/lib/helpers.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -download_lists() { - urls=$1 - output=$2 - - curl -sSfL --parallel --parallel-max 10 --retry 3 ${urls[@]} > $output -} diff --git a/lib/utils.js b/lib/utils.js index b5b3b6b..0eb065c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,7 +1,12 @@ import { once } from "events"; -import { createReadStream } from "fs"; +import { createReadStream, createWriteStream } from "fs"; import { basename } from "path"; import { createInterface } from "readline"; +import { WritableStream } from "stream/web"; + +if (!globalThis.fetch) { + globalThis.fetch = (await import("node-fetch")).default; +} /** * Sleeps for a specified amount of time. @@ -46,6 +51,28 @@ export const isComment = (value) => value.startsWith("/*") || value.startsWith("*/"); +/** + * Downloads files and concatenates them into one file. + * @param {string} filePath The path to the file being written to. + * @param {string[]} urls The URLs to the files to be downloaded. + */ +export const downloadFiles = async (filePath, urls) => { + const writeStream = createWriteStream(filePath, { flags: "a" }); + const writableStream = new WritableStream({ + write(chunk) { + writeStream.write(chunk); + }, + }); + const responses = await Promise.all(urls.map((url) => fetch(url))); + + for (const response of responses) { + await response.body?.pipeTo(writableStream, { preventClose: true }); + writeStream.write("\n"); + } + + await writableStream.close(); +}; + /** * @callback onLine * @param {string} line The current line.