CODEWITHSUNDEEP
javascriptnode.jsasync-awaitstreamaxios
Mattia Piano
Mattia Piano

Reputation: 23

array files downloader with nodejs axios and async await pattern in vanilla javascript

i have a problem with a script write in vanilla javascript and runs in nodejs. What i want to get is:

// Get axios request
  // response stream
  // download file1..........finish
// When file is downloaded 
// go to the next request 

// Get axios request
  // response stream
  // download file2..........finish
// When file is downloaded 
// go to the next request

I write this script in node and async await pattern

// ARRAY OF BOOKS 
ex. [{title: 'awesome book', link: 'http://www.example.com/awesome_book.pdf'},
     {title: 'new book', link: 'http://www.example.com/new.pdf'} ...etc]
const Books = require('./Books');

const fs = require('fs');
const path = require('path');
const axios = require('axios');


// 1)
// loop the array of books and create array of links
let booksLinkinkArray = Books.map( book => book.link);

// 2)
// Function get request with axios
// response is stream
function request (element) {
  try{
    return axios({
      url: element,
      method: "GET",
      responseType: "stream"
    });
  } catch(e) {
    console.log( 'errore: ' + e)
  }
}


// 3)
// Function that downloads the file
async function download(urls) {
  urls.map(async url => {
    try{
      const saveFile = await request(url);
      let file = url.split('/')[3];
        console.log(file + ' :  ' + 'init download');
      let download = fs.createWriteStream(path.join(__dirname, 'books_dir', file));
      saveFile.data.pipe(download);
      console.log(file + ' :  ' + 'downloaded')

    } catch(e) {
      console.log( 'error: ' + e)
    }

  }) }

download(booksLinkinkArray);

This script is ok , but the loop of request is too fast, and file downloads overlap as:

// get request
// response
//download file init
// get request
// response
//download file init
ect...

file 1.......
file 2........
file 1...........
file 2..........
file 2.............. finish
file 1.............. finish

Is there a way to manage the stream so that the call is only be done after all the chuncks have been drained from the stream?

thanks for any replies

Upvotes: 2

Views: 8640

Answers (3)

namen3645
namen3645

Reputation: 1

const fs = require('fs');
const path = require('path');
const axios = require('axios');

function request (element) {
    try{
        return axios({
        url: element,
        method: "GET",
        responseType: "stream"
        });
    } catch(e) {
        console.log( 'errore: ' + e)
    }
}

async function download(url){
    const saveFile = await request(url);
    const file = url.split('/')[3];
    const download = fs.createWriteStream(path.join(__dirname, 'tmp', file));
    new Promise((resolve, reject)=> {
        saveFile.data.pipe(download);
        download.on("close", resolve);
        download.on("error", console.error);
    });
}

for(const url of booksLinkinkArray) {
    download(url);
}

Upvotes: 0

欧阳斌
欧阳斌

Reputation: 2351

async download(urls){
  for(const url of urls) {
     const saveFile = await request(url);
     const file = url.split('/')[3];
     const download = fs.createWriteStream(path.join(__dirname, 'books_dir', file));
     await new Promise((resolve, reject)=> {
        saveFile.data.pipe(download);
        download.on("close", resolve);
        download.on("error", console.error);
     });
  }
}

Upvotes: 4

欧阳斌
欧阳斌

Reputation: 2351

async download(){
  for(const url of urls) {
    .....
  }
}

Upvotes: 0

Related Questions