optimized list downloading process

This commit is contained in:
Viet Huynh
2023-09-05 03:46:00 +07:00
parent 9f3c32c912
commit 33cf719edb
3 changed files with 15 additions and 28 deletions
+3 -14
View File
@@ -1,7 +1,6 @@
#!/bin/bash
# create an empty input.csv file
touch input.csv
source $(dirname "$0")/helpers.sh
# declare an array of urls
urls=(
@@ -14,18 +13,8 @@ urls=(
https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
)
# loop through the urls and download each file with curl
for url in "${urls[@]}"; do
# get the file name from the url
file=$(basename "$url")
# download the file with curl and save it as file.txt
curl -o "$file.txt" "$url"
# append the file contents to input.csv and add a newline
cat "$file.txt" >> input.csv
echo "" >> input.csv
# remove the file.txt
rm "$file.txt"
done
# download all files in parallel and append them to input.csv
download_lists $urls 'input.csv'
# print a message when done
echo "Done. The input.csv file contains merged data from recommended filter lists."