CODEWITHSUNDEEP
javascriptnode.jsrequestpipefs
Isaac L
Isaac L

Reputation: 73

Node issue with request.pipe() and large files, error: cannot create a string longer than 0x1fffffe8 characters

I'm having a hard time figuring out what's going on here. I have a function that retrieves URLs to .mp4 files and then attempts to download them, one at a time. Certain larger files throw an exception:

Uncaught Exception: Error: Cannot create a string longer than 0x1fffffe8 characters at Buffer.toString (buffer.js:776:17) at Request.

Everything I've read says that you'll encounter issues downloading large files if you don't create a writeable stream - but that's exactly what I'm doing, so I'm not sure what's going wrong.

async function downloadProjectAssets(event, apikey, projectId) {
        let url = `https://api.xxx.com/v1/medias.json?access_token=${apikey}&project_id=${projectId}`
    
        // Make an initial retrieve to find out where we'll need to make a request to for the 4-5 .mp4 files
        let mediaObjectCollection = [];
        await request(url, { headers: { 'username': 'api'}, json: true, gzip: true }, function (error, response, responseBody) {
            if (error) console.error('error:', error);
            responseBody.forEach(mediaObject => { 
              mediaObjectCollection.push(mediaObject) 
            })
        });
    
        // Loop over all links to assets found (4-5 .mp4s per project) 
        let projectAssetsDownloaded = [];
        for (let i = 0; i < mediaObjectCollection.length; i++) {
            let mediaObject = mediaObjectCollection[i];
            event.reply('mediaBeingProcessed', mediaObject);
            
            await new Promise(resolve => {
                let fileExtension = mediaObject.assets[0].contentType.substring(mediaObject.assets[0].contentType.lastIndexOf('/') + 1);
                fs.mkdirSync(`./Downloaded Files/${sanitize(mediaObject.project.name)}`, { recursive: true });
                
                // Download the actual file
                request(mediaObject.assets[0].url, {gzip: true} )
                    .on('error', function (err) {
                      console.log(err);
                    })
                    .pipe(fs.createWriteStream(`./Downloaded Files/${sanitize(mediaObject.project.name)}/${sanitize(mediaObject.name)}.${fileExtension}`))
                    .on('finish', resolve);
            });
            projectAssetsDownloaded.push(mediaObject);
            event.reply('mediaFinishedProcessed', mediaObject);
        } 
        return projectAssetsDownloaded;
    }

Upvotes: 4

Views: 4388

Answers (1)

Isaac L
Isaac L

Reputation: 73

Solved! Turns out it's either a bug or limitation of the Node request library. After spending hours on the problem, I switched my code to use http and it works like a charm. For anyone else struggling, here's my code:

    await new Promise(resolve => {
        let fileExtension = mediaObject.assets[0].contentType.substring(mediaObject.assets[0].contentType.lastIndexOf('/') + 1);
        fs.mkdirSync(`./Downloaded Files/${sanitize(mediaObject.project.name)}`, { recursive: true });
        let writeStream = fs.createWriteStream(`./Downloaded Files/${sanitize(mediaObject.project.name)}/${sanitize(mediaObject.name)}.${fileExtension}`);
        http.get(mediaObject.assets[0].url, function(response) {
            response.pipe(writeStream);
            writeStream.on('finish', resolve);
        })
    })

Upvotes: 2

Related Questions