feat: next

src/artifacts.js

@@ -0,0 +1,58 @@
+const
+  { join, dirname } = require('path'),
+  { readdir, stat, readFile, unlink, writeFile } = require('fs'),
+  { promisify, map, coroutine } = require('bluebird'),
+  mkdirpAsync = promisify(require('mkdirp')),
+  statAsync = promisify(stat),
+  unlinkAsync = promisify(unlink),
+  readdirAsync = promisify(readdir),
+  writeAnyFileAsync = promisify(writeFile),
+  readFileAsync = promisify(readFile),
+  readDirAsync = (dir) => {
+    return readdirAsync(dir).map((file) => {
+      const path = join(dir, file)
+      return statAsync(path).then(s => s.isDirectory() ? readDirAsync(path) : path)
+    }).reduce((a, b) => a.concat(b), [])
+  },
+  maybeReadFileContents = (file) => {
+    return readFileAsync(file, 'utf-8')
+      .catch(e => {
+        if (e.code === 'ENOENT') {
+          return ''
+        }
+        throw e
+      })
+  }
+
+/**
+ * The artifacts step is where source patches are committed, or written as "artifacts"
+ * Steps:
+ *  - A temporary directory is created in the downloaded source
+ *  - On start, any files in that directory are restored into the source tree
+ *  - After the patch functions have run, the temporary directory is emptied
+ *  - Original versions of sources to be patched are written to the temporary directory
+ *  - Finally, The patched files are written into source.
+ *
+ */
+module.exports.artifacts = function* artifacts ({ files, writeFileAsync, src }, next) {
+  const temp = join(src, 'nexe')
+  yield mkdirpAsync(temp)
+  const tmpFiles = yield readDirAsync(temp)
+
+  yield map(tmpFiles, coroutine(function* (path) {
+    return writeFileAsync(path.replace(temp, ''), yield readFileAsync(path))
+  }))
+
+  yield next()
+
+  yield map(tmpFiles, x => unlinkAsync(x))
+  return map(files, coroutine(function* (file) {
+    const sourceFile = join(src, file.filename),
+      tempFile = join(temp, file.filename),
+      fileContents = yield maybeReadFileContents(sourceFile)
+
+    yield mkdirpAsync(dirname(tempFile))
+    yield writeAnyFileAsync(tempFile, fileContents)
+    yield writeFileAsync(file.filename, file.contents)
+  }))
+}

src/bundle.js

@@ -0,0 +1,46 @@
+const Mfs = require('memory-fs'),
+  webpack = require('webpack'),
+  { fromCallback } = require('bluebird'),
+  { resolve, join, relative } = require('path'),
+  isObject = (x) => typeof x === 'object',
+  isString = (x) => typeof x === 'string'
+
+function* bundle (compiler, next) {
+  let bundleConfig = compiler.options.bundle
+  if (!bundleConfig) {
+    return next()
+  }
+
+  const
+    input = compiler.options.input || require.resolve(process.cwd()),
+    fs = new Mfs(),
+    path = resolve('nexe'),
+    filename = 'virtual-bundle.js'
+
+  if (isString(bundleConfig)) {
+    bundleConfig === require(relative(process.cwd(), bundleConfig))
+  } else if (!isObject(bundleConfig)) {
+    bundleConfig = {
+      entry: resolve(input),
+      target: 'node',
+      output: { path, filename }
+    }
+  }
+
+  const bundler = webpack(bundleConfig)
+  bundler.outputFileSystem = fs
+  const stats = yield fromCallback(cb => bundler.run(cb))
+
+  if (stats.hasErrors()) {
+    compiler.log.error(stats.toString())
+    return null
+  }
+  //eslint-disable-next-line no-sync
+  compiler.input = fs.readFileSync(
+    join(bundleConfig.output.path, bundleConfig.output.filename)
+  )
+
+  return next()
+}
+
+module.exports.bundle = bundle

src/cli.js

@@ -0,0 +1,88 @@
+const
+  { normalize } = require('path'),
+  { Promise, promisify } = require('bluebird'),
+  { createWriteStream, readFile } = require('fs')
+
+const readFileAsync = promisify(readFile),
+  isWindows = process.platform === 'win32'
+
+function dequoteStdIn (input) {
+  input = input.trim()
+  if (input.startsWith('\'') && input.endsWith('\'') ||
+    input.startsWith('"') && input.endsWith('"')) {
+    return input.slice(1).slice(0, -1)
+  }
+  return input
+}
+
+function getStdIn () {
+  return new Promise((resolve) => {
+    const bundle = []
+    process.stdin.setEncoding('utf-8')
+    process.stdin.on('data', x => bundle.push(x))
+    process.stdin.once('end', () =>
+      resolve(dequoteStdIn(Buffer.concat(bundle).toString()))
+    )
+    process.stdin.resume()
+  })
+}
+
+/**
+ * The "cli" step detects whether the process is in a tty. If it is then the input is read into memory.
+ * Otherwise, it is buffered from stdin. If no input options are passed in the tty, the package.json#main file is used.
+ * After all the build steps have run, the output (the executable) is written to a file or piped to stdout.
+ *
+ * Configuration:
+ *   - compiler.options.input - file path to the input bundle.
+ *     - fallbacks: stdin, package.json#main
+ *   - compiler.options.output - file path to the output executable.
+ *     - fallbacks: stdout, nexe_ + epoch + ext
+ *
+ * @param {*} compiler
+ * @param {*} next
+ */
+module.exports.cli = function* cli (compiler, next) {
+  const input = compiler.options.input,
+    bundled = Boolean(compiler.input)
+
+  if (bundled) {
+    yield next()
+  } else if (!input && !process.stdin.isTTY) {
+    compiler.log.verbose('Buffering stdin as main module...')
+    compiler.input = yield getStdIn()
+  } else if (input) {
+    compiler.log.verbose('Reading input as main module: ' + input)
+    compiler.input = yield readFileAsync(normalize(input))
+  } else if (!compiler.options.empty) {
+    compiler.log.verbose('Resolving cwd as main module...')
+    compiler.input = yield readFileAsync(require.resolve(process.cwd()))
+  }
+
+  if (!bundled) {
+    yield next()
+  }
+
+  const deliverable = yield compiler.getDeliverableAsync()
+
+  return new Promise((resolve, reject) => {
+    deliverable.once('error', reject)
+
+    if (!compiler.options.output && !process.stdout.isTTY) {
+      compiler.log.verbose('Writing result to stdout...')
+      deliverable.pipe(process.stdout).once('error', reject)
+      resolve()
+    } else {
+      compiler.log.verbose('Writing result to file...')
+      const output = compiler.options.output || `${compiler.options.name}${isWindows ? '.exe' : ''}`
+      deliverable.pipe(createWriteStream(normalize(output)))
+      .once('error', reject)
+      .once('close', e => {
+        if (e) {
+          reject(e)
+        } else {
+          resolve(compiler.log.info('Executable written: ' + output))
+        }
+      })
+    }
+  })
+}

src/compiler.js

@@ -0,0 +1,94 @@
+const
+  { dirname, normalize, join } = require('path'),
+  { promisify, Promise, coroutine } = require('bluebird'),
+  { normalizeOptions } = require('./options'),
+  { readFile, writeFile, createReadStream } = require('fs'),
+  { spawn } = require('child_process'),
+  { logger } = require('./logger')
+
+const
+  isWindows = process.platform === 'win32',
+  isBsd = Boolean(~process.platform.indexOf('bsd')),
+  make = isWindows && 'vcbuild.bat' ||
+    isBsd && 'gmake' ||
+    'make',
+  configure = isWindows ? 'configure' : './configure',
+  readFileAsync = promisify(readFile),
+  writeFileAsync = promisify(writeFile)
+
+module.exports.NexeCompiler = class NexeCompiler {
+  constructor (options) {
+    options = this.options = normalizeOptions(options)
+    logger.setLevel(options.loglevel)
+    this.log = logger
+    this.python = options.python
+    this.configure = options.configure
+    this.make = isWindows ? options.vcBuild : options.make
+    this.src = join(options.temp, options.version)
+    this.env = Object.assign({}, process.env)
+    this.files = []
+
+    this.readFileAsync = coroutine(function * (file) {
+      const cachedFile = this.files.find(x => normalize(x.filename) === normalize(file))
+      if (cachedFile) {
+        return Promise.resolve(cachedFile)
+      }
+      this.files.push({
+        filename: file,
+        contents: yield readFileAsync(join(this.src, file), 'utf-8').catch(e => {
+          if (e.code !== 'ENOENT') {
+            throw e
+          }
+          return ''
+        })
+      })
+      return this.readFileAsync(file)
+    }.bind(this))
+    this.writeFileAsync = (file, contents) => writeFileAsync(join(this.src, file), contents)
+  }
+
+  set python (pythonPath) {
+    if (!pythonPath) {
+      return
+    }
+    if (isWindows) {
+      this.env.PATH = this.env.PATH + ';' + normalize(dirname(pythonPath))
+    } else {
+      this.env.PYTHON = pythonPath
+    }
+  }
+
+  get _deliverableLocation () {
+    return isWindows
+      ? join(this.src, 'Release', 'node.exe')
+      : join(this.src, 'out', 'Release', 'node')
+  }
+
+  _runBuildCommandAsync (command, args) {
+    return new Promise((resolve, reject) => {
+      spawn(command, args, {
+        cwd: this.src,
+        env: this.env,
+        stdio: 'ignore'
+      })
+      .once('error', reject)
+      .once('close', resolve)
+    })
+  }
+
+  _configureAsync () {
+    return this._runBuildCommandAsync(
+      this.env.PYTHON || 'python',
+      [configure, ...this.configure]
+    )
+  }
+
+  buildAsync () {
+    return this._configureAsync()
+      .then(() => this._runBuildCommandAsync(make, this.make))
+  }
+
+  getDeliverableAsync () {
+    return Promise.resolve(createReadStream(this._deliverableLocation))
+  }
+}

src/download.js

@@ -0,0 +1,75 @@
+const
+  unZip = require('zlib').createGunzip,
+  unTar = require('tar').Extract,
+  request = require('request'),
+  { Promise, promisify } = require('bluebird'),
+  statAsync = promisify(require('fs').stat),
+  rimrafAsync = promisify(require('rimraf'))
+
+function progress (req, log, precision = 10) {
+  const logged = {}
+  let length = 0,
+    total = 1
+
+  return req.on('response', (res) => {
+    total = res.headers['content-length']
+  }).on('data', (chunk) => {
+    length += chunk.length
+    const percentComplete = +(length / total * 100).toFixed()
+    if (percentComplete % precision === 0 && !logged[percentComplete]) {
+      logged[percentComplete] = true
+      log(percentComplete)
+    }
+  })
+}
+
+function fetchNodeSource (path, url, log) {
+  log.info('Downloading Node: ' + url)
+  return new Promise((resolve, reject) => {
+    progress(request.get(url), (pc) => {
+      log.verbose(`Downloading Node: ${pc}%...`)
+      if (pc === 100) {
+        log.info('Extracting Node...')
+      }
+    }).on('error', reject)
+      .pipe(unZip().on('error', reject))
+      .pipe(unTar({ path, strip: 1 }))
+      .on('error', reject)
+      .on('end', () => resolve(log.info('Extracted to: ' + path)))
+  })
+}
+
+function cleanSrc (clean, src, log) {
+  if (clean === true) {
+    log.info('Removing source: ' + src)
+    return rimrafAsync(src).then(() => {
+      log.info('Source deleted.' + src)
+    })
+  }
+  return Promise.resolve()
+}
+
+/**
+ * Deletes (maybe) and downloads the node source to the configured temporary directory
+ * @param {*} compiler
+ * @param {*} next
+ */
+function download (compiler, next) {
+  const
+    { src, log } = compiler,
+    { version, sourceUrl, clean } = compiler.options,
+    url = sourceUrl ||
+      `https://nodejs.org/dist/v${version}/node-v${version}.tar.gz`
+
+  return cleanSrc(clean, src, log).then(() => statAsync(src).then(
+    x => !x.isDirectory() && fetchNodeSource(src, url, log),
+    e => {
+      if (e.code !== 'ENOENT') {
+        throw e
+      }
+      return fetchNodeSource(src, url, log)
+    }
+  )).then(next)
+}
+
+module.exports.download = download

src/logger.js

@@ -0,0 +1,35 @@
+const
+  EOL = require('os').EOL,
+  colors = require('chalk'),
+  isFn = f => typeof f === 'function',
+  logNoop = (x, cb) => {
+    isFn(cb) && process.nextTick(cb)
+  },
+  safeCb = f => isFn(f) ? f : logNoop,
+  logLevels = [
+    ['verbose', 'green'],
+    ['info', 'blue'],
+    ['error', 'red']
+  ],
+  logger = logLevels.reduce((logMethods, info) => {
+    const [level, color] = info
+    logMethods[level] = function (output, cb) {
+      process.stderr.write(colors[color](`${EOL}[${level}]: ${output}`), safeCb(cb))
+    }
+    return logMethods
+  }, {
+    setLevel (level) {
+      if (level === 'silent') {
+        return void logLevels.forEach(([x]) => logger[x] = logNoop)
+      }
+      process.on('beforeExit', () => {
+        process.stderr.write(EOL)
+      })
+      if (level === 'info') {
+        logger.verbose = logNoop
+      }
+      return logger
+    }
+  })
+
+module.exports.logger = logger

src/nexe.js

@@ -0,0 +1,51 @@
+const
+  { compose } = require('app-builder'),
+  { bundle } = require('./bundle'),
+  { NexeCompiler } = require('./compiler'),
+  { options } = require('./options'),
+  { cli } = require('./cli'),
+  { download } = require('./download'),
+  { artifacts } = require('./artifacts'),
+  { patches } = require('./patches'),
+  { EOL } = require('os'),
+  { coroutine, Promise } = require('bluebird'),
+  { logger } = require('./logger')
+
+function wrap (fns) {
+  return [].concat(...fns).map(fn => {
+    return fn.constructor.name === 'GeneratorFunction'
+      ? coroutine(fn) : fn
+  })
+}
+
+function compile (compilerOptions, callback) {
+  const compiler = new NexeCompiler(compilerOptions)
+
+  compiler.log.verbose('Compiler options:' +
+    EOL + JSON.stringify(compiler.options, null, 4)
+  )
+
+  const nexe = compose(wrap([
+    bundle,
+    cli,
+    download,
+    (nexeCompiler, next) => {
+      return next().then(() => {
+        return nexeCompiler.buildAsync()
+      })
+    },
+    artifacts,
+    patches,
+    compiler.options.patches
+  ]))
+  return Promise.resolve(nexe(compiler)).asCallback(callback)
+}
+
+module.exports.options = options
+module.exports.compile = compile
+
+if (require.main === module || process.__nexe) {
+  compile(options).catch((e) => {
+    logger.error(e.stack, () => process.exit(1))
+  })
+}

src/options.js

@@ -0,0 +1,136 @@
+const
+  parseArgv = require('minimist'),
+  { basename, extname, join } = require('path'),
+  EOL = require('os').EOL,
+  padRight = (str, l) => {
+    while (str.length < l) {
+      str += ' '
+    }
+    return str.substr(0, l)
+  },
+  defaults = {
+    temp: process.env.NEXE_TEMP || join(process.cwd(), '.nexe'),
+    version: process.version.slice(1),
+    flags: [],
+    configure: [],
+    make: [],
+    vcBuild: ['nosign', 'release'],
+    quick: false,
+    bundle: true,
+    enableNodeCli: false,
+    verbose: false,
+    silent: false,
+    padding: 2,
+    patches: []
+  },
+  alias = {
+    i: 'input',
+    o: 'output',
+    t: 'temp',
+    n: 'name',
+    v: 'version',
+    p: 'python',
+    f: 'flag',
+    c: 'configure',
+    m: 'make',
+    vc: 'vcBuild',
+    q: 'quick',
+    s: 'snapshot',
+    b: 'bundle',
+    cli: 'enableNodeCli',
+    h: 'help',
+    l: 'loglevel'
+  },
+  argv = parseArgv(process.argv, { alias, default: defaults }),
+  help = `
+nexe --help              CLI OPTIONS
+
+  -i   --input      =./index.js             -- application entry point
+  -o   --output     =./nexe.exe             -- path to output file
+  -t   --temp       =./.nexe                -- nexe temp directory (3Gb+) ~ NEXE_TEMP
+  -n   --name       =nexe.js                -- file name for error reporting at run time
+  -v   --version    =${padRight(process.version.slice(1), 23)}-- node version
+  -p   --python     =/path/to/python2       -- python executable
+  -f   --flag       ="--expose-gc"          -- *v8 flags to include during compilation
+  -c   --configure  ="--with-dtrace"        -- *pass arguments to the configure command
+  -m   --make       ="--loglevel"           -- *pass arguments to the make command
+  -vc  --vcBuild    =x64                    -- *pass arguments to vcbuild.bat
+  -s   --snapshot   =/path/to/snapshot      -- build with warmup snapshot
+  -b   --bundle                             -- attempt bundling application
+       --ico                                -- file name for alternate icon file (windows)
+       --rc-*                               -- populate rc file options (windows)
+       --clean                              -- force download of sources
+       --enableNodeCli                      -- enable node cli enforcement (blocks app cli)
+       --sourceUrl                          -- pass an alternate source (node.tar.gz) url
+       --silent                             -- disable logging
+       --verbose                            -- set logging to verbose
+
+
+       -* variable key name                 * option can be used more than once
+
+  TODO -q   --quick   =win32-x64-X.X.X        -- use prebuilt binary (url, key or path)
+  TODO     --resource-* =/path/to/resource      -- *embed file bytes within the binary
+`.trim()
+
+function flattenFilter (...args) {
+  return [].concat(...args).filter(x => x)
+}
+
+/**
+ * Extract keys such as { "rc-CompanyName": "Node.js" } to
+ * { CompanyName: "Node.js" }
+ * @param {*} match
+ * @param {*} options
+ */
+function extractCliMap (match, options) {
+  Object.keys(options).filter(x => match.test(x))
+    .reduce((map, option) => {
+      const key = option.split('-')[1]
+      map = map || {}
+      map[key] = options[option]
+      delete options[option]
+      return map
+    }, null)
+}
+
+function extractName (options) {
+  if (typeof options.input === 'string') {
+    return options.name
+      || basename(options.input).replace(extname(options.input), '')
+  }
+  const mainName = basename(require.resolve(process.cwd()))
+  return options.name || mainName.replace(extname(mainName), '')
+}
+
+function normalizeOptions (input) {
+  if (input.__normalized) {
+    return input
+  }
+
+  if (argv.help || Boolean(argv._.find(x => x === 'version'))) {
+    process.stderr.write(
+      argv.help ? help : 'next' + EOL,
+      () => process.exit(0)
+    )
+  }
+
+  const options = Object.assign({}, defaults, input)
+  delete options._
+  delete alias.rc
+  options.loglevel = options.loglevel
+    || options.silent && 'silent'
+    || options.verbose && 'verbose' || 'info'
+  options.name = extractName(options)
+  options.flags = flattenFilter(options.flag, options.flags)
+  options.make = flattenFilter(options.make)
+  options.vcBuild = flattenFilter(options.vcBuild)
+  options.rc = options.rc || extractCliMap(/^rc-.*/, options)
+  options.resources = options.resources || extractCliMap(/^resource-.*/, options)
+  Object.keys(alias).forEach(x => delete options[x])
+
+  options.__normalized = true
+  return options
+}
+
+module.exports.normalizeOptions = normalizeOptions
+module.exports.options = normalizeOptions(argv)

src/patches/content.js

@@ -0,0 +1,27 @@
+const Buffer = require('buffer').Buffer
+
+function* content (compiler, next) {
+  yield next()
+
+  const filename = 'lib/' + compiler.options.name + '.js',
+    file = yield compiler.readFileAsync(filename),
+    header = '/' + '*'.repeat(19) + 'nexe_',
+    end = '_' + '*'.repeat(19) + '/',
+    padding = Array(compiler.options.padding * 1000000).fill('*').join('')
+
+  if (!padding) {
+    file.contents = compiler.input
+    return
+  }
+
+  file.contents = [
+    compiler.input,
+    '/',
+    padding,
+    '/'
+  ].join('')
+
+  file.contents = header + Buffer.byteLength(file.contents) + end + file.contents
+}
+
+module.exports.content = content

src/patches/disable-node-cli.js

@@ -0,0 +1,17 @@
+function * disableNodeCli (compiler, next) {
+  if (compiler.options.enableNodeCli) {
+    return next()
+  }
+
+  const nodecc = yield compiler.readFileAsync('src/node.cc'),
+    nodeccMarker = "argv[index][0] == '-'"
+
+  nodecc.contents = nodecc.contents.replace(
+    nodeccMarker,
+    nodeccMarker.replace('-', ']')
+  )
+
+  return next()
+}
+
+module.exports.disableNodeCli = disableNodeCli

src/patches/flags.js

@@ -0,0 +1,14 @@
+module.exports.flags = function* flags (compiler, next) {
+  const nodeflags = compiler.options.flags
+  if (!nodeflags.length) {
+    return next()
+  }
+
+  const nodegyp = yield compiler.readFileAsync('node.gyp')
+
+  nodegyp.contents = nodegyp.contents.replace(
+    "'node_v8_options%': ''",
+    `'node_v8_options%': '${nodeflags.join(' ')}'`)
+
+  return next()
+}

src/patches/gyp.js

@@ -0,0 +1,15 @@
+module.exports.nodeGyp = function * nodeGyp ({ files, readFileAsync, download }, next) {
+  yield next()
+
+  const nodegyp = yield readFileAsync('node.gyp'),
+    nodeGypMarker = "'lib/fs.js',"
+
+  nodegyp.contents = nodegyp.contents
+    .replace(nodeGypMarker, `
+      ${nodeGypMarker}
+      ${files
+          .filter(x => x.filename.startsWith('lib'))
+          .map(x => `'${x.filename}'`)
+          .toString()},
+    `.trim())
+}

src/patches/ico.js

@@ -0,0 +1,16 @@
+const
+  { readFile } = require('fs'),
+  { normalize } = require('path'),
+  { promisify } = require('bluebird')
+
+const readFileAsync = promisify(readFile)
+
+module.exports.ico = function * ico (compiler, next) {
+  const iconFile = compiler.options.ico
+  if (!iconFile) {
+    return next()
+  }
+  const file = yield compiler.readFileAsync('src/res/node.ico')
+  file.contents = yield readFileAsync(normalize(iconFile))
+  return next()
+}

src/patches/index.js

@@ -0,0 +1,9 @@
+module.exports.patches = [
+  require('./gyp').nodeGyp, //upstream, sets manifest
+  require('./content').content, //upstream, sets main module content
+  require('./third-party-main').main, //loads main module
+  require('./disable-node-cli').disableNodeCli,
+  require('./flags').flags,
+  require('./ico').ico,
+  require('./node-rc').nodeRc
+]

src/patches/node-rc.js

@@ -0,0 +1,21 @@
+function* nodeRc (compiler, next) {
+  const options = compiler.options.rc
+  if (!options) {
+    return next()
+  }
+
+  const file = yield compiler.readFileAsync('src/res/node.rc')
+
+  Object.keys(options).forEach((key) => {
+    const value = options[key],
+      isVar = /^[A-Z_]+$/.test(value)
+
+    value = isVar ? value : `"${value}"`
+    file.contents.replace(new RegExp(`VALUE "${key}",*`), `VALUE "${key}", ${value}`)
+    // TODO support keys that are not present?
+  })
+
+  return next()
+}
+
+module.exports.nodeRc = nodeRc

src/patches/snapshot.js

@@ -0,0 +1,15 @@
+module.exports.snapshot = function* snapshot (compiler, next) {
+  const snapshotFile = compiler.options.snapshot
+
+  if (!snapshotFile) {
+    return next()
+  }
+
+  const file = yield compiler.readFileAsync('configure')
+  file.contents = file.contents
+    .replace(
+      'def configure_v8(o):',
+      `def configure_v8(o):\n  o['variables']['embed_script'] = '${snapshotFile}'\n  o['variables']['warmup_script'] = '${snapshotFile}'`
+    )
+  return next()
+}

src/patches/third-party-main.js

@@ -0,0 +1,21 @@
+function * main (compiler, next) {
+  const mainFile = yield compiler.readFileAsync('lib/_third_party_main.js')
+  mainFile.contents = `
+    Object.defineProperty(process, '__nexe', {
+      value: true,
+      enumerable: false,
+      configurable: false
+    });
+    require("${compiler.options.name}");
+    `.trim()
+
+  if (compiler.options.empty === true) {
+    compiler.options.resources.length = 0
+    compiler.input = 'console.log(`nexe-${process.platform}-${process.arch}-${process.version}`)'
+    return next()
+  }
+
+  return next()
+}
+
+module.exports.main = main