Compiling Webpack in memory but resolving to node_modules on disk

Question

I'm trying to use web pack to compile an in memory string of valid javascript code. I'm using memory fs as outlined here: https://webpack.github.io/docs/node.js-api.html#compile-to-memory.

So I'm taking a string containing raw javascript, writing that to memory fs, and then web pack resolves to that entry point. But the compilation fails on the first require statement, presumably because it's not able to look in the real fs for node_modules.

Any ideas on how can I accomplish this?

import webpack from 'webpack';
import MemoryFS from 'memory-fs';
import thenify from 'thenify';

function* compile(code) {
    const fs = new MemoryFS();
    fs.writeFileSync('/file.js', code);
    const compiler = webpack({
        entry: { file: '/file.js' },
        output: {
            path: '/build',
            filename: '[name].js'
        },
        module: {
            loaders: [
                { test: /\.json$/, loader: 'json' }
            ],  
        }
    });
    compiler.run = thenify(compiler.run);

    compiler.inputFileSystem = fs;
    compiler.resolvers.normal.fileSystem = fs; //this is needed for memfs
    compiler.outputFileSystem = fs;
    const stats = yield compiler.run();
    //retrieve the output of the compilation
    const res = stats.compilation.assets['file.js'].source();
    return res;
}

Usage

var code = "var _ = require('underscore'); console.log(_);";
var bundle = yield compile(code); //should be a bundle containing the underscore source.

The error is

ModuleNotFoundError: Module not found: Error: Cannot resolve module underscore in /

This question indicates that others have tried the same thing: https://github.com/webpack/webpack/issues/1562. there's a gist referenced at https://gist.github.com/DatenMetzgerX/2a96ebf287b4311f4c18 that I believe was intended to do what I'm hoping to accomplish, but in it's current form I don't see how. It assigns an instance of MemoryFs to all of the resolvers. I've tried assigning node's fs module, but no dice.

So in short, I'm trying to set an entry point to an in memory string of raw javascript, but still have require and import statements resolved to node_modules on disk.

UPDATE

I've been able to get the result I'm looking for but it's not pretty. I'm basically overriding the implementation of #stat and #readFile in MemoryFS to check the real filesystem if it gets any request for a file that doesn't exist in memory. I could clean this up a bit by subclassing MemoryFS instead of swapping method implementations at runtime, but the idea would still be the same.

Working solution

import webpack from 'webpack';
import JsonLoader from 'json-loader';
import MemoryFS from 'memory-fs';
import UglifyJS from "uglify-js";
import thenify from 'thenify';
import path from 'path';
import fs from 'fs';
import root from 'app-root-path';
/*
* Provide webpack with an instance of MemoryFS for
* in-memory compilation. We're currently overriding
* #stat and #readFile. Webpack will ask MemoryFS for the 
* entry file, which it will find successfully. However, 
* all dependencies are on the real filesystem, so any require 
* or import statements will fail. When that happens, our wrapper 
* functions will then check fs for the requested file. 
*/
const memFs = new MemoryFS();
const statOrig = memFs.stat.bind(memFs);
const readFileOrig = memFs.readFile.bind(memFs);
memFs.stat = function (_path, cb) {
    statOrig(_path, function(err, result) {
        if (err) {
            return fs.stat(_path, cb);
        } else {
            return cb(err, result);
        }
    });
};
memFs.readFile = function (path, cb) {
    readFileOrig(path, function (err, result) {
        if (err) {
            return fs.readFile(path, cb);
        } else {
            return cb(err, result);
        }
    });
};


export default function* compile(code) {
    // Setup webpack 
    //create a directory structure in MemoryFS that matches
    //the real filesystem
    const rootDir = root.toString();
    //write code snippet to memoryfs
    const outputName = `file.js`;
    const entry = path.join(rootDir, outputName);
    const rootExists = memFs.existsSync(rootDir);
    if (!rootExists) {
        memFs.mkdirpSync(rootDir);
    }
    memFs.writeFileSync(entry, code);
    //point webpack to memoryfs for the entry file
    const compiler = webpack({
        entry: entry,
        output: {
            filename: outputName
        },
        module: {
            loaders: [
                { test: /\.json$/, loader: 'json' }
            ]
        }
    });
    compiler.run = thenify(compiler.run);

    //direct webpack to use memoryfs for file input
    compiler.inputFileSystem = memFs;
    compiler.resolvers.normal.fileSystem = memFs;

    //direct webpack to output to memoryfs rather than to disk
    compiler.outputFileSystem = memFs;
    const stats = yield compiler.run();
    //remove entry from memory. we're done with it
    memFs.unlinkSync(entry);
    const errors = stats.compilation.errors;
    if (errors && errors.length > 0) {
        //if there are errors, throw the first one
        throw errors[0];
    }
    //retrieve the output of the compilation
    const res = stats.compilation.assets[outputName].source(); 
    return res;
}

Usage

var code = "var _ = require('underscore'); console.log(_);";
var bundle = yield compile(code); //is a valid js bundle containing the underscore source and a log statement logging _.

If there's not a better way, then I'll definitely encapsulate this into a subclass of MemoryFS, but I'm hoping there's a more sane way to accomplish this with Webpack's api.

Answer 1

I know it's late but for the record here comes a code snippet.

import * as fs from 'fs';
import { resolve } from 'path';
import  { Volume } from 'memfs';
import { ufs } from 'unionfs';

const volume = Volume.fromJSON({
 [resolve(process.cwd(), 'test.js')]: 'this file is on memory not on disk'
});

ufs.use(fs).use(volume);

// Reads from memory
console.log(ufs.readFileSync(resolve(process.cwd(), 'test.js'), 'utf8'));
// Reads from disk
console.log(ufs.readFileSync(resolve(process.cwd(), 'package.json'), 'utf8'));

// Writing into memory
volume.writeFileSync(resolve(process.cwd(), 'test.memory'), 'This should be 
on memory');
console.log(ufs.readFileSync(resolve(process.cwd(), 'test.memory'), 'utf8'));

// Writing into disk
ufs.writeFileSync(resolve(process.cwd(), 'test.disk'), 'This should be on disk');
console.log(ufs.readFileSync(resolve(process.cwd(), 'test.disk'), 'utf8'));

Hers the console output:

user1@pc playground % node inMem.mjs
this file is on memory not on disk
{
  "name": "playground",
  "version": "1.0.0",
  "main": "index.js",
  "license": "MIT",
  "dependencies": {
    "memfs": "^3.3.0",
    "unionfs": "^4.4.0"
  }
}

This should be on memory
This should be on disk
user1@pc playground % ls .
inMem.mjs       node_modules    package.json            yarn.lock

Answer 2

Instead of memory-fs, the combination of unionfs/memfs/linkfs should help.

• https://npmjs.com/unionfs

• https://npmjs.com/memfs

• https://npmjs.com/linkfs

Answer 3

You're using MemoryFS, which is a JavaScript reimplementation of a feature normally handled by the Operating System. I wonder, could you mount a directory using tmpfs at the Operating System level, then use that? webpack would then not know or care that the input file is actually stored in memory.

Assuming that you have mounted a memory-based filesystem at /media/memory, the webpack configuration code could be as simple as this:

resolve: {
  root: ['/media/memory', ...other paths...],
  },
  output: {
    path: '/wherever/you/want/the/output/files'
  }
}

This approach also has a hidden benefit: If you want to debug the input code, you just mount /media/memory with a non-RAM-based filesystem and you can see what's being generated.

Answer 4

I have created this snippet untested. I think you want the inputFS to be the real one and the output fs to be the in memory one. On the other hand you want all the dependencies of file.js to be constructed separately. For that I figured the webpack.optimize.CommonsChunkPlugin plugin could help. I expect webpack to write everything to the memory. I hope it works.

import webpack from 'webpack';
import MemoryFS from 'memory-fs';
import thenify from 'thenify';
import realFS from 'fs';

function* compile(code) {
    const fs = new MemoryFS();
    const compiler = webpack({
        entry: {
            file: '/file.js',
            vendor: [
                'underscore',
                'other-package-name'
            ]

        },
        output: {
            path: '/build',
            filename: '[name].js'
        },
        module: {
            loaders: [
                { test: /\.json$/, loader: 'json' }
            ],
        },
        plugins: [
            new webpack.optimize.CommonsChunkPlugin('vendor', 'vendor.js')
        ]
    });
    compiler.run = thenify(compiler.run);

    compiler.inputFileSystem = realFS;
    compiler.resolvers.normal.fileSystem = fs; //this is needed for memfs
    compiler.outputFileSystem = fs;
    const stats = yield compiler.run();
    //retrieve the output of the compilation
    const res = stats.compilation.assets['file.js'].source();
    return res;
}