optimized list downloading process

get_recommended_filters.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-# create an empty input.csv file
+source $(dirname "$0")/helpers.sh
-touch input.csv
 
 # declare an array of urls
 urls=(
@@ -14,18 +13,8 @@ urls=(
   https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
 )
 
-# loop through the urls and download each file with curl
+# download all files in parallel and append them to input.csv
-for url in "${urls[@]}"; do
+download_lists $urls 'input.csv'
-  # get the file name from the url
-  file=$(basename "$url")
-  # download the file with curl and save it as file.txt
-  curl -o "$file.txt" "$url"
-  # append the file contents to input.csv and add a newline
-  cat "$file.txt" >> input.csv
-  echo "" >> input.csv
-  # remove the file.txt
-  rm "$file.txt"
-done
 
 # print a message when done
 echo "Done. The input.csv file contains merged data from recommended filter lists."

get_recommended_whitelist.sh

@@ -2,8 +2,8 @@
 #  
 # Use the provided lists or add you own
 # https://oisd.nl/includedlists/whitelists
-# by creating an empty whitelist.csv file
+
-touch whitelist.csv
+source $(dirname "$0")/helpers.sh
 
 # declare an array of urls
 urls=(
@@ -35,18 +35,8 @@ urls=(
 
 )
 
-# loop through the urls and download each file with curl
+# download all files in parallel and append them to whitelist.csv
-for url in "${urls[@]}"; do
+download_lists $urls 'whitelist.csv'
-  # get the file name from the url
-  file=$(basename "$url")
-  # download the file with curl and save it as file.txt
-  curl -o "$file.txt" "$url"
-  # append the file contents to whitelist.csv and add a newline
-  cat "$file.txt" >> whitelist.csv
-  echo "" >> whitelist.csv
-  # remove the file.txt
-  rm "$file.txt"
-done
 
 # print a message when done
 echo "Done. The whitelist.csv file contains merged data from recommended whitelists."

helpers.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+download_lists() {
+  urls=$1
+  output=$2
+  
+  curl -sSfL --parallel --parallel-max 10 --retry 3 ${urls[@]} > $output
+}