refactored downloadFiles to a more efficient approach

This commit is contained in:
Viet Huynh
2023-11-19 16:39:54 +07:00
parent c91aea1b39
commit 6eb55bf6a1
+6 -9
View File
@@ -49,19 +49,16 @@ export const isComment = (value) =>
* @param {string[]} urls The URLs to the files to be downloaded.
*/
export const downloadFiles = async (filePath, urls) => {
const writeStream = createWriteStream(filePath, { flags: "a" });
const responses = await Promise.all(urls.map((url) => fetch(url)));
for (const response of responses) {
const readable = ReadStream.from(response.body, {
autoDestroy: true,
});
const writeStream = createWriteStream(filePath, { flags: "a" });
readable.on("end", () => {
writeStream.write("\n");
});
readable.pipe(writeStream, { end: false });
ReadStream.from(response.body)
.on("end", () => {
writeStream.write("\n");
})
.pipe(writeStream);
}
};