From ce2c4cefd0ba080ce8ca1a2d17cb0424b633da1f Mon Sep 17 00:00:00 2001 From: calebboyd Date: Wed, 5 Jul 2017 23:18:08 -0500 Subject: [PATCH] wip: bundling work --- .babelrc | 2 +- .gitignore | 1 + README.md | 6 +- package.json | 7 +- src/bundle.js | 54 ++------------ src/bundling/bindings-rewrite.js | 91 +++++++++++++++++++++++ src/bundling/native-module-fuse-plugin.js | 30 ++++++++ 7 files changed, 135 insertions(+), 56 deletions(-) create mode 100644 src/bundling/bindings-rewrite.js create mode 100644 src/bundling/native-module-fuse-plugin.js diff --git a/.babelrc b/.babelrc index 81699c2..f8dfde2 100644 --- a/.babelrc +++ b/.babelrc @@ -2,7 +2,7 @@ "presets": [ ["env", { "targets": { - "node": "6.5" + "node": "4.0" } }] ] diff --git a/.gitignore b/.gitignore index 580e70a..e8cf86f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ temp/ coverage node_modules +examples *.log *.exe .idea diff --git a/README.md b/README.md index 72b43b5..cd382f1 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ ## Motivation and Features -- Supports production ready ([secure](#security)) builds +- Supports production ready, ([securable](#security)) builds - Ability to run multiple applications with *different* node.js runtimes. - Distribute binaries without needing node / npm. - Idempotent builds @@ -53,7 +53,7 @@ Additional resources can be added to the binary by passing `-r glob/pattern/**/* By default `nexe` will attempt to download a pre-built executable. However, some users may want to customize the way node is built, either by changing the flags, providing a different icon, or different executable details. These options are supported out of the box, and subsequent builds with compatible options will result in instant build times. -Nexe also exposes its patching pipeline to the user. This allows limitless and simple patching of node sources prior to compilation. +Nexe also exposes its patching pipeline to the user. This allows the application of simple patching of node sources prior to compilation. ## Node.js API @@ -178,7 +178,7 @@ For examples, see the built in patches: [src/patches](src/patches) #### `SourceFile` - `contents: string` - `filename: string` - + Any modifications made to `SourceFile#contents` will be maintained in the cache _without_ the need to explicitly write them back out, e.g. using `NexeCompiler#setFileContentsAsync`. ## Security diff --git a/package.json b/package.json index 147953c..49e5a30 100644 --- a/package.json +++ b/package.json @@ -35,18 +35,13 @@ "chalk": "^1.1.3", "download": "^6.2.0", "globby": "^6.1.0", - "html-loader": "^0.4.5", - "json-loader": "^0.5.4", "memory-fs": "^0.4.1", "minimist": "^1.2.0", "mkdirp": "^0.5.1", "nigel": "^2.0.2", "ora": "^1.2.0", - "raw-loader": "^0.5.1", "rimraf": "^2.6.1", - "tar": "^3.1.3", - "webpack": "^2.3.3", - "xbin-loader": "^1.0.1" + "tar": "^3.1.3" }, "devDependencies": { "babel-cli": "^6.24.1", diff --git a/src/bundle.js b/src/bundle.js index 4d15765..7658b8f 100644 --- a/src/bundle.js +++ b/src/bundle.js @@ -1,53 +1,15 @@ -import Mfs from 'memory-fs' -import webpack from 'webpack' +import { + FuseBox, + CSSPlugin, + JSONPlugin, + HTMLPlugin } from 'fuse-box' import { fromCallback } from 'bluebird' -import { resolve, join, relative } from 'path' - -const isObject = (x) => typeof x === 'object' -const isString = (x) => x instanceof String || typeof x === 'string' +import * as path from 'path' export default async function bundle (compiler, next) { - let bundleConfig = compiler.options.bundle - if (!bundleConfig) { + let options = compiler.options.bundle + if (!options) { return next() } - const inputFilePath = compiler.options.input || require.resolve(process.cwd()) - const mfs = new Mfs() - const path = resolve('nexe') - const filename = 'virtual-bundle.js' - - if (isString(bundleConfig)) { - bundleConfig = require(relative(process.cwd(), bundleConfig)) - } else if (!isObject(bundleConfig)) { - bundleConfig = { - entry: resolve(inputFilePath), - target: 'node', - module: { - rules: [ - { test: /\.html$/, use: 'html-loader' }, - { test: /\.json$/, use: 'json-loader' }, - { test: /\.node$/, use: 'xbin-loader' }, - { test: /license$|\.md$/, use: 'raw-loader' } - ] - } - } - } - - bundleConfig.output = { path, filename } - - const bundler = webpack(bundleConfig) - bundler.outputFileSystem = mfs - const stats = await fromCallback(cb => bundler.run(cb)) - - if (stats.hasErrors()) { - // compiler.log.error(stats.toString()) - return null - } - - compiler.input = mfs.readFileSync( - join(bundleConfig.output.path, bundleConfig.output.filename) - ) - - return next() } diff --git a/src/bundling/bindings-rewrite.js b/src/bundling/bindings-rewrite.js new file mode 100644 index 0000000..1ae998f --- /dev/null +++ b/src/bundling/bindings-rewrite.js @@ -0,0 +1,91 @@ +import * as fs from 'fs' +import * as path from 'path' +import * as child from 'child_process' + +function findNativeModulePath (filePath, bindingsArg) { + const dirname = path.dirname(filePath) + const tempFile = Math.random() * 100 + '.js' + const tempFilePath = path.join(dirname, tempFile) + fs.writeFileSync(tempFilePath, ` + var bindings = require('bindings'); + var Module = require('module'); + var originalRequire = Module.prototype.require; + Module.prototype.require = function(path) { + const mod = originalRequire.apply(this, arguments); + process.stdout.write(path) + return mod + }; + bindings('${bindingsArg}') + `) + //using exec because it can be done syncronously + const nativeFileName = child.execSync('node ' + tempFile, { cwd: dirname }).toString() + fs.unlinkSync(tempFilePath) + const relativePath = './' + path.relative(dirname, nativeFileName).replace(/\\/g, '/') + return relativePath +} +/** + * Traverse all nodes in a file and evaluate usages of the bindings module + */ +export class BindingsRewrite { + + constructor () { + this.bindingsIdNodes = [] + this.nativeModulePaths = [] + this.rewrite = false + } + + isRequireBindings (node) { + return node.callee.name === 'require' && node.arguments[0].value === 'bindings' + } + + onNode (absolutePath, node, parent) { + if (node.type === 'CallExpression') { + if (this.isRequireBindings(node) && parent.type === 'VariableDeclarator') { + /** + * const loadBindings = require('bindings'); + * -> const loadBindings = String(''); + */ + this.bindingsIdNodes.push(parent.id) + node.callee.name = 'String' + node.arguments[0].value = '' + this.rewrite = true + return + } + + if (this.isRequireBindings(node) && parent.type === 'CallExpression') { + /** + *const bindings = require('bindings')('native-module').... + * -> const bindings = require('./path/to/native/module.node')..... + */ + const bindingsArgNode = parent.arguments[0] + if (bindingsArgNode.type === 'Literal') { + parent.callee = { type: 'Identifier', name: 'require' } + bindingsArgNode.value = findNativeModulePath(absolutePath, bindingsArgNode.value) + this.nativeModulePaths.push(bindingsArgNode.value) + this.rewrite = true + return + } + } + + const bindingsInvocationIdx = this.bindingsIdNodes.findIndex(x => node.callee.name === x.name) + + if (this.bindingsIdNodes[bindingsInvocationIdx]) { + /** + * const bindings = loadBindings('native-module') + * -> const bindings = require('./path/to/native/module.node') + */ + const bindingsIdNode = this.bindingsIdNodes[bindingsInvocationIdx] + const bindingsArgNode = node.arguments[0] + this.bindingsIdNodes.splice(bindingsInvocationIdx, 1) + + if (bindingsArgNode.type === 'Literal') { + node.callee.name = 'require' + bindingsArgNode.value = findNativeModulePath(absolutePath, bindingsArgNode.value) + this.nativeModulePaths.push(bindingsArgNode.value) + this.rewrite = true + return + } + } + } + } +} diff --git a/src/bundling/native-module-fuse-plugin.js b/src/bundling/native-module-fuse-plugin.js new file mode 100644 index 0000000..ad96a89 --- /dev/null +++ b/src/bundling/native-module-fuse-plugin.js @@ -0,0 +1,30 @@ +import { BindingsRewrite } from './bindings-rewrite' + +export class NativeModulePlugin { + constructor (...moduleNames) { + this.test = new RegExp(`node_modules\/(${moduleNames.join('|')}).*\.js`) + this.limit2project = false + } + + transform (file) { + const bindingsRewrite = new BindingsRewrite() + file.loadContents() + file.makeAnalysis(null, { + plugins: [{ + onNode(file, node, parent) { + bindingsRewrite.onNode(file.info.absPath, node, parent) + }, + onEnd(file) { + if (bindingsRewrite.rewrite) { + const index = file.analysis.dependencies.indexOf('bindings') + if (~index) { + file.analysis.dependencies.splice(index, 1) + } + file.analysis.dependencies.push(...bindingsRewrite.nativeModulePaths) + file.analysis.requiresRegeneration = true + } + } + }] + }) + } +}