oxmc/nexe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: instant builds

Browse Source
This commit is contained in:
calebboyd
2017-05-23 00:22:23 -05:00
parent 1f35fe498e
commit e12eb8b289
16 changed files with 402 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bundle.js
+8 -15
View File
@@ -1,19 +1,10 @@
import Mfs from 'memory-fs'
import webpack from 'webpack'
import { fromCallback } from 'bluebird'
import { resolveModule } from './util'
import { resolve, join } from 'path'
import module from 'module'
import 'json-loader'
import 'html-loader'
import { resolve, join, relative } from 'path'
const isObject = (x) => typeof x === 'object'
const isString = (x) => typeof x === 'string'
function loadModule (path) {
console.log('loading module!!!!', path)
return module._load(path, module, false)
}
const isString = (x) => typeof x === 'string' || x instanceof String
export default async function bundle (compiler, next) {
let bundleConfig = compiler.options.bundle
@@ -21,21 +12,23 @@ export default async function bundle (compiler, next) {
return next()
}
const input = compiler.options.input || resolveModule(process.cwd())
const inputFilePath = compiler.options.input || require.resolve(process.cwd())
const mfs = new Mfs()
const path = resolve('nexe')
const filename = 'virtual-bundle.js'
if (isString(bundleConfig)) {
bundleConfig = loadModule(relative(process.cwd(), bundleConfig))
bundleConfig = require(relative(process.cwd(), bundleConfig))
} else if (!isObject(bundleConfig)) {
bundleConfig = {
entry: resolve(input),
entry: resolve(inputFilePath),
target: 'node',
module: {
rules: [
{ test: /\.html$/, use: 'html-loader' },
{ test: /\.json$/, use: 'json-loader' }
{ test: /\.json$/, use: 'json-loader' },
{ test: /\.node$/, use: 'xbin-loader' },
{ test: /license$|\.md$/, use: 'raw-loader' }
]
}
}
src/cli.js
+22 -31
View File
@@ -1,21 +1,7 @@
import { normalize } from 'path'
import Bluebird from 'bluebird'
import { createWriteStream } from 'fs'
import { dequote, readFileAsync, resolveModule } from './util'
const isWindows = process.platform === 'win32'
function getStdIn () {
return new Bluebird((resolve) => {
const bundle = []
process.stdin.setEncoding('utf-8')
process.stdin.on('data', x => bundle.push(x))
process.stdin.once('end', () =>
resolve(dequote(Buffer.concat(bundle).toString()))
)
process.stdin.resume()
})
}
import { readFileAsync, readStreamAsync, isWindows } from './util'
/**
* The "cli" step detects whether the process is in a tty. If it is then the input is read into memory.
@@ -38,41 +24,46 @@ export default async function cli (compiler, next) {
if (bundled) {
await next()
} else if (!input && !process.stdin.isTTY) {
compiler.log.verbose('Buffering stdin as main module...')
compiler.input = await getStdIn()
compiler.log.verbose('Buffering stdin as bundle...')
compiler.input = await readStreamAsync(process.stdin)
} else if (input) {
compiler.log.verbose('Reading input as main module: ' + input)
compiler.log.verbose('Reading input as bundle: ' + input)
compiler.input = await readFileAsync(normalize(input))
} else if (!compiler.options.empty) {
compiler.log.verbose('Resolving cwd as main module...')
compiler.input = await readFileAsync(resolveModule(process.cwd()))
const bundle = require.resolve(process.cwd())
compiler.log.verbose('Resolving cwd as main bundle: ' + bundle)
compiler.input = await readFileAsync(bundle)
}
if (!bundled) {
await next()
}
const shouldPipeOutput = Boolean(!compiler.options.output && !process.stdout.isTTY)
const outputName = compiler.options.output ||
`${compiler.options.name}${isWindows ? '.exe' : ''}`
compiler.output = shouldPipeOutput ? null : outputName
const deliverable = await compiler.compileAsync()
return new Bluebird((resolve, reject) => {
deliverable.once('error', reject)
if (!compiler.options.output && !process.stdout.isTTY) {
if (!compiler.output) {
compiler.log.verbose('Writing result to stdout...')
deliverable.pipe(process.stdout).once('error', reject)
resolve()
} else {
compiler.log.verbose('Writing result to file...')
const output = compiler.options.output || `${compiler.options.name}${isWindows ? '.exe' : ''}`
deliverable.pipe(createWriteStream(normalize(output)))
.once('error', reject)
.once('close', e => {
if (e) {
reject(e)
} else {
compiler.log.info('Executable written: ' + output, resolve)
}
})
deliverable.pipe(createWriteStream(normalize(compiler.output)))
.once('error', reject)
.once('close', e => {
if (e) {
reject(e)
} else {
compiler.log.info('Executable written: ' + compiler.output, resolve)
}
})
}
})
}
src/compiler.js
+194 -11
View File
@@ -1,14 +1,32 @@
import { normalize, join } from 'path'
import Bluebird from 'bluebird'
import { Buffer } from 'buffer'
import { createReadStream } from 'fs'
import { Readable } from 'stream'
import { spawn } from 'child_process'
import * as logger from './logger'
import { readFileAsync, writeFileAsync, dequote } from './util'
import { Stream as Needle } from 'nigel'
import logger from './logger'
import { deepEqual } from 'assert'
import {
readFileAsync,
writeFileAsync,
fileExistsAsync,
dequote,
isWindows
} from './util'
const isWindows = process.platform === 'win32'
const isBsd = Boolean(~process.platform.indexOf('bsd'))
const make = isWindows ? 'vcbuild.bat' : isBsd ? 'gmake' : 'make'
const configure = isWindows ? 'configure' : './configure'
const marker = Buffer.from('<nexe~sentinel>').toString('hex')
const needle = Buffer.from(marker)
const padLeft = (x, l, c = '0') => (c.repeat(l) + x).slice(-l)
const inflate = (value, size) => {
if (!size | value.length >= size) {
return value
}
return value + Array(size - value.length + 1).join(';')
}
export class NexeCompiler {
constructor (options) {
@@ -16,8 +34,6 @@ export class NexeCompiler {
logger.setLevel(options.loglevel)
this.log = logger
this.python = options.python
this.configure = options.configure
this.make = isWindows ? options.vcBuild : options.make
this.src = join(options.temp, options.version)
this.env = Object.assign({}, process.env)
this.files = []
@@ -48,7 +64,24 @@ export class NexeCompiler {
}
}
get _deliverableLocation () {
_findPaddingSize (size, override) {
if (override === 0) {
return 0
}
size = override > size ? override : size
const padding = [3, 6, 9, 16, 25, 40].map(x => x * 1e6)
.filter(p => size <= p)[0]
if (!padding) {
throw new Error(`No prebuilt target large enough (${(size / 1024).toFixed(2)}Mb).\nUse the --build flag and build for the current platform`)
}
return padding
}
_getArtifactLocation (target) {
if (target) {
return join(this.options.temp, target)
}
return isWindows
? join(this.src, 'Release', 'node.exe')
: join(this.src, 'out', 'Release', 'node')
@@ -69,18 +102,168 @@ export class NexeCompiler {
_configureAsync () {
return this._runBuildCommandAsync(
this.env.PYTHON || 'python',
[configure, ...this.configure]
[configure, ...this.options.configure]
)
}
async _buildAsync () {
if (this.options.clean) {
await this._runBuildCommandAsync(make, ['clean'])
}
await this._configureAsync()
return this._runBuildCommandAsync(make, this.make)
const buildOptions = isWindows ? this.options.vcBuild : this.options.make
await this._runBuildCommandAsync(make, buildOptions)
return createReadStream(this._getArtifactLocation())
}
compileAsync () {
return this._buildAsync().then(() => {
return createReadStream(this._deliverableLocation)
_fetchPrebuiltBinary () {
/**
* TODO implement hosted builds
* - CircleCI artifacts
* - AppVeyor windows build
* - Publish to github release from appveyor
* github.io site build
*/
return this._buildAsync()
}
_getPayload (header) {
return this._serializeHeader(header) + this.input + '/**' + this.resources.bundle + `**/`
}
_generateHeader (paddingOverride) {
const zeros = padLeft(0, 20)
const header = {
configure: this.options.configure.slice().sort(),
make: this.options.make.slice().sort(),
enableNodeCli: this.options.enableNodeCli,
vcBuild: this.options.vcBuild.slice().sort(),
resources: this.resources.index,
contentSize: zeros,
paddingSize: zeros,
resourceOffset: zeros,
binaryOffset: zeros
}
const serializedHeader = this._serializeHeader(header)
header.contentSize = padLeft(Buffer.byteLength(this._getPayload(header)), 20)
header.paddingSize = padLeft(this._findPaddingSize(+header.contentSize, paddingOverride), 20)
header.resourceOffset = padLeft(Buffer.byteLength(serializedHeader + this.input + '/**'), 20)
return header
}
async _getExistingBinaryHeaderAsync (target) {
const filename = this._getArtifactLocation(target)
const existingBinary = await fileExistsAsync(filename)
if (existingBinary) {
return this._extractHeaderAsync(filename)
}
return null
}
_extractHeaderAsync (path) {
const haystack = createReadStream(path)
const stream = new Needle(needle)
let needles = 0
let lastStack = null
haystack.pipe(stream)
return new Promise(resolve => {
stream.on('needle', () => needles++)
.on('haystack', x => {
if (needles === 2) {
haystack.close()
stream.end()
resolve(JSON.parse(lastStack.toString()))
}
lastStack = x
}).on('close', () => resolve(null))
})
}
_serializeHeader (header) {
return `/**${marker}${JSON.stringify(header)}${marker}**/process.__nexe=${JSON.stringify(header)};/**${marker}**/`
}
_headersAreEqual (rhs, lhs) {
const ignoreKeys = [
'resourceOffset', 'binaryOffset', 'contentSize', 'resources'
].reduce((acc, c) => {
acc[c] = true
return acc
}, {})
rhs = Object.assign({}, rhs, ignoreKeys)
lhs = Object.assign({}, lhs, ignoreKeys)
try {
deepEqual(rhs, lhs)
return true
} catch (e) { void e }
return false
}
async setMainModule (compiler, next) {
if (compiler.options.targets.length) {
return
}
await next()
const mainFile = await compiler.readFileAsync(`lib/${compiler.options.name}.js`)
const header = compiler._generateHeader(compiler.options.padding)
mainFile.contents = inflate(this._getPayload(header), +header.paddingSize) +
'\n//' + marker
}
async compileAsync () {
let target = this.options.targets.slice().shift() // TODO support multiple targets
let prebuiltSource = null
const header = this._generateHeader(this.options.padding)
target = target && `${target}-${header.paddingSize}`
const existingBinaryHeader = await this._getExistingBinaryHeaderAsync(target)
if (existingBinaryHeader && this._headersAreEqual(header, existingBinaryHeader)) {
prebuiltSource = createReadStream(this._getArtifactLocation(target))
}
if (target) {
prebuiltSource = this._fetchPrebuiltBinary(target)
}
if (!prebuiltSource) {
prebuiltSource = await this._buildAsync()
}
return this._assembleDeliverable(
header,
prebuiltSource
)
}
_assembleDeliverable (header, binary) {
const firstNeedle = Buffer.concat([Buffer.from('/**'), needle])
const stream = new Needle(firstNeedle)
const artifact = new Readable({ read () {} })
let needles = 0
let currentStackSize = 0
binary.pipe(stream)
stream.on('needle', () => {
needles++
stream.needle(needle)
})
.on('haystack', x => {
if (needles < 1) {
currentStackSize += x.length
artifact.push(x)
}
if (needles === 1 && !+header.binaryOffset) {
header.binaryOffset = padLeft(currentStackSize, 20)
artifact.push(
Buffer.from(
inflate(this._getPayload(header), +header.paddingSize) +
'\n//' + marker
)
)
}
if (needles > 3) {
artifact.push(x)
}
}).on('close', () => artifact.push(null))
return artifact
}
}
src/logger.js
+1 -8
View File
@@ -43,11 +43,4 @@ const logger = logLevels.reduce((logMethods, info) => {
}
})
const { info, error, verbose, setLevel } = logger
export {
setLevel,
info,
error,
verbose
}
export default logger
src/nexe.js
+11 -10
View File
@@ -9,7 +9,7 @@ import artifacts from './artifacts'
import patches from './patches'
import { EOL } from 'os'
import Bluebird from 'bluebird'
import { error } from './logger'
import logger from './logger'
PromiseConfig.constructor = Bluebird
Bluebird.longStackTraces()
@@ -17,25 +17,26 @@ Bluebird.longStackTraces()
async function compile (compilerOptions, callback) {
const options = await normalizeOptionsAsync(compilerOptions)
const compiler = new NexeCompiler(options)
const build = compiler.options.build
compiler.log.verbose('Compiler options:' +
EOL + JSON.stringify(compiler.options, null, 4)
)
const nexe = compose(
const nexe = compose(...[
resource,
bundle,
cli,
download,
artifacts,
patches,
options.patches
)
build && download,
build && artifacts,
build && patches,
build && options.patches
].filter(x => x))
return nexe(compiler).asCallback(callback)
}
function isNexe (callback) {
return Bluebird.resolve(Boolean(process.__nexe)).asCallback(callback)
function isNexe () {
return Boolean(process.__nexe)
}
export {
@@ -47,6 +48,6 @@ export {
if (process.__nexe) {
compile(argv)
.catch((e) => {
error(e.stack, () => process.exit(e.exitCode || 1))
logger.error(e.stack, () => process.exit(e.exitCode || 1))
})
}
src/options.js
+35 -26
View File
@@ -1,7 +1,6 @@
import parseArgv from 'minimist'
import { basename, extname, join } from 'path'
import Bluebird from 'bluebird'
import { resolveModule } from './util'
import { EOL } from 'os'
function padRight (str, l) {
@@ -13,19 +12,17 @@ const defaults = {
flags: [],
configure: [],
make: [],
targets: [],
vcBuild: ['nosign', 'release'],
quick: false,
bundle: true,
bundle: false,
build: false,
enableNodeCli: false,
verbose: false,
silent: false,
padding: 2,
patches: []
}
const alias = {
i: 'input',
o: 'output',
t: 'temp',
t: 'target',
n: 'name',
v: 'version',
p: 'python',
@@ -33,7 +30,6 @@ const alias = {
c: 'configure',
m: 'make',
vc: 'vcBuild',
q: 'quick',
s: 'snapshot',
b: 'bundle',
cli: 'enableNodeCli',
@@ -44,9 +40,9 @@ const argv = parseArgv(process.argv, { alias, default: defaults })
const help = `
nexe --help CLI OPTIONS
-i --input =./index.js -- application entry point
-o --output =./nexe.exe -- path to output file
-t --temp =./.nexe -- nexe temp directory (3Gb+) ~ NEXE_TEMP
-i --input =index.js -- application entry point
-o --output =nexe.exe -- path to output file
-t --target =win32-x64-6.10.3 -- *target a prebuilt binary
-n --name =nexe.js -- file name for error reporting at run time
-v --version =${padRight(process.version.slice(1), 23)}-- node version
-p --python =/path/to/python2 -- python executable
@@ -55,7 +51,9 @@ nexe --help CLI OPTIONS
-m --make ="--loglevel" -- *pass arguments to the make command
-vc --vcBuild =x64 -- *pass arguments to vcbuild.bat
-s --snapshot =/path/to/snapshot -- build with warmup snapshot
-b --bundle -- attempt bundling application
-r --resource =./paths/**/* -- *embed file bytes within the binary
-b --bundle =webpack-config.js -- use default configuration or provide custom webpack config
--temp =./path/to/temp -- nexe temp files (for downloads and source builds)
--ico -- file name for alternate icon file (windows)
--rc-* -- populate rc file options (windows)
--clean -- force download of sources
@@ -64,11 +62,10 @@ nexe --help CLI OPTIONS
--silent -- disable logging
--verbose -- set logging to verbose
-* variable key name * option can be used more than once
TODO -q --quick =win32-x64-X.X.X -- use prebuilt binary (url, key or path)
TODO --resource-* =/path/to/resource -- *embed file bytes within the binary
`.trim()
function flattenFilter (...args) {
@@ -95,7 +92,7 @@ function extractCliMap (match, options) {
function tryResolveMainFileName () {
let filename = 'nexe'
try {
const file = resolveModule(process.cwd())
const file = require.resolve(process.cwd())
filename = basename(file).replace(extname(file), '')
} catch (_) {}
@@ -106,23 +103,23 @@ function extractLogLevel (options) {
if (options.loglevel) return options.loglevel
if (options.silent) return 'silent'
if (options.verbose) return 'verbose'
if (options.info) return 'info'
return 'info'
}
function extractName (options) {
if (typeof options.input === 'string') {
return options.name ||
basename(options.input).replace(extname(options.input), '')
let name = options.name
if (typeof options.input === 'string' && !name) {
name = basename(options.input).replace(extname(options.input), '')
}
const mainName = tryResolveMainFileName()
return options.name || mainName
name = name || tryResolveMainFileName()
return name.replace(/\.exe$/, '')
}
function normalizeOptionsAsync (input) {
if (argv.help || Boolean(argv._.find(x => x === 'version'))) {
if (argv.help || Boolean(argv._.filter(x => x === 'version')[0])) {
return Bluebird.fromCallback(cb => process.stderr.write(
argv.help ? help : '2.0.0-beta.1' + EOL,
() => cb(null, process.exit(1))
argv.help ? help : '2.0.0-rc.1' + EOL,
() => cb(null, process.exit(0))
))
}
@@ -131,10 +128,22 @@ function normalizeOptionsAsync (input) {
options.loglevel = extractLogLevel(options)
options.name = extractName(options)
options.flags = flattenFilter(options.flag, options.flags)
options.targets = flattenFilter(options.target, options.targets)
options.make = flattenFilter(options.make)
options.vcBuild = flattenFilter(options.vcBuild)
options.resources = flattenFilter(options.resource, options.resources)
options.rc = options.rc || extractCliMap(/^rc-.*/, options)
options.resources = options.resources || extractCliMap(/^resource-.*/, options)
if (options.build || options.padding === 0) {
options.targets = []
options.build = true
} else if (!options.targets.length) {
const defaultTarget = [process.platform, process.arch, options.version].join('-')
options.targets = [defaultTarget]
}
// TODO validate target(s)
Object.keys(alias)
.filter(k => k !== 'rc')
.forEach(x => delete options[x])
src/patches/content.js
-25
View File
@@ -1,25 +0,0 @@
import { Buffer } from 'buffer'
export default async function content (compiler, next) {
await next()
const filename = 'lib/' + compiler.options.name + '.js'
const file = await compiler.readFileAsync(filename)
const header = '/' + '*'.repeat(19) + 'nexe_'
const end = '_' + '*'.repeat(19) + '/'
const padding = Array(compiler.options.padding * 1000000).fill('*').join('')
if (!padding) {
file.contents = compiler.input
return
}
file.contents = [
compiler.input,
'/',
padding,
'/'
].join('')
file.contents = header + Buffer.byteLength(file.contents) + end + file.contents
}
src/patches/index.js
+3 -4
View File
@@ -1,6 +1,5 @@
import gyp from './gyp'
import content from './content'
import main from './third-party-main'
import nexePatches from './third-party-main'
import cli from './disable-node-cli'
import flags from './flags'
import ico from './ico'
@@ -8,8 +7,8 @@ import rc from './node-rc'
const patches = [
gyp,
content,
main,
(compiler, next) => compiler.setMainModule(compiler, next),
nexePatches,
cli,
flags,
ico,
src/patches/third-party-main.js
+49 -14
View File
@@ -1,22 +1,57 @@
export default async function main (compiler, next) {
const mainFile = await compiler.readFileAsync('lib/_third_party_main.js')
mainFile.contents = `
Object.defineProperty(process, '__nexe', {
value: true,
enumerable: false,
writable: false,
configurable: false
});
Object.defineProperty(process, '__nexe', (function () {
let nexeHeader = null
return {
get: function () {
return nexeHeader
},
set: function (value) {
if (nexeHeader) {
throw new Error('__nexe cannot be reconfigured')
}
nexeHeader = value
Object.freeze(nexeHeader)
},
enumerable: false,
configurable: false
}
})());
const fs = require('fs')
const originalReadFile = fs.readFile
const Buffer = require('buffer').Buffer
const isString = x => typeof x === 'string' || x instanceof String
fs.readFile = function readFile (file, options, callback) {
const manifest = process.__nexe
const entry = manifest && manifest.resources[file]
if (!manifest || !entry || !isString(file)) {
return originalReadFile.apply(fs, arguments)
}
const [offset, length] = entry
const resourceOffset = +manifest.binaryOffset + +manifest.resourceOffset + offset
callback = typeof options === 'function' ? options : callback
fs.open(process.execPath, 'r', function (err, fd) {
if (err) return callback(err, null)
fs.read(fd, Buffer.alloc(length), 0, length, binaryOffset, function (error, bytesRead, buffer) {
if (error) {
return fs.close(fd, function () {
callback(error, null)
})
}
fs.close(fd, function (err) {
if (err) {
return callback(err, buffer)
}
callback(err, Buffer.from(buffer.toString(), 'base64'))
})
})
})
}
require("${compiler.options.name}");
`.trim()
if (compiler.options.empty === true) {
compiler.options.resources.length = 0
//eslint-disable-next-line
compiler.input = 'console.log(`nexe: ${process.platform}-${process.arch}-${process.version.slice(1)}`)'
return next()
}
return next()
}
src/resource.js
+34
View File
@@ -0,0 +1,34 @@
import { each, fromCallback } from 'bluebird'
import { readFileAsync } from './util'
import { Buffer } from 'buffer'
import globs from 'globby'
import { stat } from 'fs'
async function isDirectoryAsync (path) {
const stats = fromCallback(cb => stat(path, cb))
return stats.isDirectory()
}
export default async function resource (compiler, next) {
const resources = compiler.resources = {
index: {},
bundle: ''
}
if (!compiler.options.resources.length) {
return next()
}
await each(globs(compiler.options.resources), async (file) => {
if (await isDirectoryAsync(file)) {
return
}
const contents = await readFileAsync(file)
const encodedContents = contents.toString('base64')
resources.index[file] = [
Buffer.byteLength(resources.bundle),
Buffer.byteLength(encodedContents)
]
resources.bundle = resources.bundle + encodedContents
})
}
src/util.js
+27 -12
View File
@@ -1,16 +1,8 @@
import { readFile, writeFile } from 'fs'
import { readFile, writeFile, stat } from 'fs'
import Bluebird from 'bluebird'
import module from 'module'
function resolveModule (path) {
const filename = module._resolveFilename(path, module)
console.log('resolving module ', path)
return filename
}
function dequote (input) {
input = input.trim()
const singleQuote = input.startsWith('\'') && input.endsWith('\'')
const doubleQuote = input.startsWith('"') && input.endsWith('"')
if (singleQuote || doubleQuote) {
@@ -19,12 +11,35 @@ function dequote (input) {
return input
}
function readStreamAsync (stream) {
return new Bluebird((resolve) => {
let input = ''
stream.setEncoding('utf-8')
stream.on('data', x => { input += x })
stream.once('end', () => resolve(dequote(input)))
stream.resume && stream.resume()
})
}
const readFileAsync = Bluebird.promisify(readFile)
const writeFileAsync = Bluebird.promisify(writeFile)
const statAsync = Bluebird.promisify(stat)
const isWindows = process.platform === 'win32'
function fileExistsAsync (path) {
return statAsync(path).then(x => true, err => {
if (err.code !== 'ENOENT') {
throw err
}
return false
})
}
export {
readFileAsync,
writeFileAsync,
dequote,
resolveModule
isWindows,
readFileAsync,
fileExistsAsync,
readStreamAsync,
writeFileAsync
}