diff --git a/get_recommended_filters.sh b/get_recommended_filters.sh index 9e0d28c..3d83c22 100644 --- a/get_recommended_filters.sh +++ b/get_recommended_filters.sh @@ -1,7 +1,6 @@ #!/bin/bash -# create an empty input.csv file -touch input.csv +source $(dirname "$0")/helpers.sh # declare an array of urls urls=( @@ -14,18 +13,8 @@ urls=( https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts ) -# loop through the urls and download each file with curl -for url in "${urls[@]}"; do - # get the file name from the url - file=$(basename "$url") - # download the file with curl and save it as file.txt - curl -o "$file.txt" "$url" - # append the file contents to input.csv and add a newline - cat "$file.txt" >> input.csv - echo "" >> input.csv - # remove the file.txt - rm "$file.txt" -done +# download all files in parallel and append them to input.csv +download_lists $urls 'input.csv' # print a message when done echo "Done. The input.csv file contains merged data from recommended filter lists." diff --git a/get_recommended_whitelist.sh b/get_recommended_whitelist.sh index 74c4c92..c9d9ebe 100644 --- a/get_recommended_whitelist.sh +++ b/get_recommended_whitelist.sh @@ -2,8 +2,8 @@ # # Use the provided lists or add you own # https://oisd.nl/includedlists/whitelists -# by creating an empty whitelist.csv file -touch whitelist.csv + +source $(dirname "$0")/helpers.sh # declare an array of urls urls=( @@ -35,18 +35,8 @@ urls=( ) -# loop through the urls and download each file with curl -for url in "${urls[@]}"; do - # get the file name from the url - file=$(basename "$url") - # download the file with curl and save it as file.txt - curl -o "$file.txt" "$url" - # append the file contents to whitelist.csv and add a newline - cat "$file.txt" >> whitelist.csv - echo "" >> whitelist.csv - # remove the file.txt - rm "$file.txt" -done +# download all files in parallel and append them to whitelist.csv +download_lists $urls 'whitelist.csv' # print a message when done echo "Done. The whitelist.csv file contains merged data from recommended whitelists." diff --git a/helpers.sh b/helpers.sh new file mode 100644 index 0000000..80cfcd2 --- /dev/null +++ b/helpers.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +download_lists() { + urls=$1 + output=$2 + + curl -sSfL --parallel --parallel-max 10 --retry 3 ${urls[@]} > $output +}