Nodejs fucntion return before operations finished

Question

I have a question about my Nodejs program. I have a FindItemFromUrlList function which in a for loop calls another TestUrl. I would like FindItemFromUrlList to return the urls found (so the list foundUrls) once all the urls have been tested. But when the for loop is traversed, and that the urls are tested and found, when I do console.log (foundUrls) I have an empty list while in TestUrl. I can see that the Urls have been found and I presume that the algorithm perforsm the console.log function before the for loop, I tried to do a .then but it doesn't work.

function FindItemFromUrlList(KeyworList, UrlList, Color, Excludedurls){
    var completed_requests = 0 ;
    var itemfound = 0;
    foundUrls = [];
    for(urls of UrlList){
        completed_requests++;
        TestUrl(KeyworList, urls, itemfound, completed_requests, foundUrls, UrlList, Color, Excludedurls);
    }
    console.log(foundUrls);
}

function TestUrl(KeyworList, urls, itemfound, completed_requests, foundUrls, UrlList, Color, Excludedurls){
    https.get(urls, function (res) {
        var htmlcode2 = "";
        res.on('data', (d) => {
            htmlcode2 += d;
        })
            .on('end', () => {
                for (keyword of KeyworList) {
                    if (htmlcode2.toString().includes(keyword) && !foundUrls.includes(urls) && (!(Excludedurls && urls.includes(Excludedurls))) ){
                        foundUrls.push(urls);
                    }
                }
                if (completed_requests == UrlList.length) {
                    // All download done, process responses
                    console.log("Done all requests");
                    console.log("nombre d'items trouvés : ",foundUrls.length);
                    console.log("Url trouvées :", foundUrls);
                }
            })})
}

Here is the result of my code running :

[]
Done all requests
nombre d'items trouvés :  5
Url trouvées : [
  'https://www.supremenewyork.com/shop/tops-sweaters/jmqsjoxe5/iuawc8jf5',
  'https://www.supremenewyork.com/shop/tops-sweaters/jmqsjoxe5/if60rct4l',
  'https://www.supremenewyork.com/shop/tops-sweaters/jmqsjoxe5/dgkjtbima',
  'https://www.supremenewyork.com/shop/tops-sweaters/jmqsjoxe5/wwx6gdai0',
  'https://www.supremenewyork.com/shop/tops-sweaters/jmqsjoxe5/x2y3lk890'
]

Answer 1

Your https.get(...) is async and receives a callback, which means, your console.log(foundUrls) will execute before the TestUrl func is finished and foundUrls array populated.

async function FindItemFromUrlList(KeyworList, UrlList, Color, Excludedurls){
    var completed_requests = 0 ;
    var itemfound = 0;
    const promises = []
    foundUrls = [];
    for(urls of UrlList){
        completed_requests++;
        const promises = TestUrl(KeyworList, urls, itemfound, completed_requests, foundUrls, UrlList, Color, Excludedurls);
       promises.push(promise)
    }
    
    await Promise.all(promises) // this will wait for promises to resolve
    console.log(foundUrls);
}

async function TestUrl(KeyworList, urls, itemfound, completed_requests, foundUrls, 
  UrlList, Color, Excludedurls) {
  return new Promise((resolve, reject) => {
    https.get(urls, function (res) {
        var htmlcode2 = "";
        res.on('data', (d) => {
            htmlcode2 += d;
        })
            .on('end', () => {
                for (keyword of KeyworList) {
                    if (htmlcode2.toString().includes(keyword) && !foundUrls.includes(urls) && (!(Excludedurls && urls.includes(Excludedurls))) ){
                        foundUrls.push(urls);
                    }
                }

                resolve() // here you resolve the promise

                if (completed_requests == UrlList.length) {
                    // All download done, process responses
                    console.log("Done all requests");
                    console.log("nombre d'items trouvés : ",foundUrls.length);
                    console.log("Url trouvées :", foundUrls);
                }
            })})
  })
    
}

You can use async keyword and Promises to handle this. Using await Promise.all(...) will wait for all promises to be completed before moving on. If you've never used Promises before, i suggest you read next article.

Edit: Added code and explanation