CODEWITHSUNDEEP
node.jsaxios
Phantom007
Phantom007

Reputation: 2249

Unable to upload large files using nodejs/axios

I am writing a nodejs client that would upload files (files can be both binary or text files) from my local dev machine to my server which is written in Java, configuring which is not an option. I am using the following code to upload files, it works fine for files upto 2 gb, but beyond that it throws an error mentioned below. Now you may think that the server might not be allowing files more than 2 gb but I have successfully uploaded files upto 10 gb using Rest clients like Postman and Insomnia on the same instance.

const fs = require("fs");
const path = require("path");
const axios = require("axios");
const FormData = require("form-data");

function uploadAxios({ filePath }) {
  
let formData;
try {
  formData = new FormData();
  formData.append("filedata", fs.createReadStream(filePath));
} catch (e) {
  console.error(e)
}

  axios
    .post(
      `https://myinstance.com`,
      formData,
      {
        headers: {
          ...formData.getHeaders(),
          "Content-Type": "multipart/form-data",
          Authorization:
            "Basic xyz==",
        },
        maxContentLength: Infinity,
        maxBodyLength: Infinity,
        // maxContentLength: 21474836480,
        // maxBodyLength: 21474836480, // I have tried setting these values with both numbers and the keyword Infinity but nothing works
      }
    )
    .then(console.log)
    .catch(console.error);
}

const filePath = "C:\\Users\\phantom007\\Documents\\BigFiles\\3gb.txt";
 
uploadAxios({  filePath  });

Error I get:

#
# Fatal error in , line 0
# API fatal error handler returned after process out of memory
#

<--- Last few GCs --->
es[7844:0000023DC49CE190]    47061 ms: Mark-sweep 33.8 (41.8) -> 33.8 (41.8) MB, 417.2 / 0.1 ms  (+ 947.1 ms in 34029 steps since start of marking, biggest step 431.0 ms, walltime since start of marking 15184 ms) finalize incremental marking via stack guard[7844:0000023D
C49CE190]    48358 ms: Mark-sweep 34.4 (41.8) -> 31.8 (40.5) MB, 1048.4 / 0.0 ms  (+ 0.0 ms in 1 steps since start of marking, biggest step 0.0 ms, walltime since start of marking 1049 ms) finalize incremental marking via task GC in old spac

<--- JS stacktrace --->

==== JS stack trace =========================================

Security context: 000002E294C255E9 <JSObject>
    0: builtin exit frame: new ArrayBuffer(aka ArrayBuffer)(this=0000022FFFF822D1 <undefined>,65536)

    1: _read [fs.js:~2078] [pc=0000004AD942D301](this=0000039E67337641 <ReadStream map = 000002F26D804989>,n=65536)
    2: read [_stream_readable.js:454] [bytecode=000002A16EB59689 offset=357](this=0000039E67337641 <ReadStream map = 000002F26D804989>,n=0)
    3: push [_stream_readable.js:~201]...

FATAL ERROR: Committing semi space failed. Allocation failed - process out of memory

It looks like the error is because it has exceed the memory limit, i know by passing the flag --max-old-space-size i can overcome this, but i want this to be scalable and not hardcode an upper limit.

PS: My dev machine has 12 GB free memory

Edit: I added the error trace.

Upvotes: 2

Views: 3792

Answers (1)

Jesus Velazquez
Jesus Velazquez

Reputation: 61

I'm using multer to define limit, see next code:

  app.use(multer({
  storage: storage,
  dest: path.join(pathApp),
  limits: {
    fileSize: 5000000
  },
  fileFilter: function fileFilter(req, file, cb) {
    var filetypes = /json/;
    var mimetype = filetypes.test(file.mimetype);
    var extname = filetypes.test(path.extname(file.originalname));

    if (mimetype && extname) {
      console.log("Port ".concat(app.get('port')) + " - Uploading file " + file.originalname);
      return cb(null, true, req);
    }

    cb(JSON.stringify({
      "success": false,
      "payload": {
        "app": "upload",
        "function": "upload"
      },
      "error": {
        "code": 415,
        "message": 'File type not valid'
      }
    }));
  }
}).single('file1'));

Upvotes: 1

Related Questions