oxmc/nexe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: move startup scripts to shims, add plugins

Browse Source
This commit is contained in:
calebboyd
2017-09-26 20:16:03 -05:00
parent b822984776
commit 11fa087e9d
26 changed files with 2573 additions and 3205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/compiler.ts
+10 -15
View File
@@ -6,7 +6,7 @@ import { Readable } from 'stream'
import { spawn } from 'child_process'
import { Logger } from './logger'
import { readFileAsync, writeFileAsync, pathExistsAsync, dequote, isWindows, bound } from './util'
import { NexeOptions, nexeVersion } from './options'
import { NexeOptions, version } from './options'
import { NexeTarget } from './target'
import download = require('download')
import { getLatestGitRelease } from './releases'
@@ -22,12 +22,14 @@ export interface NexeFile {
contents: string
}
export { NexeOptions }
interface NexeHeader {
resources: { [key: string]: number[] }
version: string
}
export class NexeCompiler {
export class NexeCompiler<T extends NexeOptions = NexeOptions> {
private start = Date.now()
private env = { ...process.env }
private compileStep: { modify: Function; log: Function }
@@ -48,7 +50,7 @@ export class NexeCompiler {
: `${this.options.output || this.options.name}`
private nodeSrcBinPath: string
constructor(public options: NexeOptions) {
constructor(public options: T) {
const { python } = (this.options = options)
this.targets = options.targets as NexeTarget[]
this.target = this.targets[0]
@@ -56,7 +58,7 @@ export class NexeCompiler {
this.nodeSrcBinPath = isWindows
? join(this.src, 'Release', 'node.exe')
: join(this.src, 'out', 'Release', 'node')
this.log.step('nexe ' + nexeVersion, 'info')
this.log.step('nexe ' + version, 'info')
if (python) {
if (isWindows) {
this.env.PATH = '"' + dequote(normalize(python)) + '";' + this.env.PATH
@@ -192,7 +194,7 @@ export class NexeCompiler {
return createReadStream(filename)
}
private _generateHeader() {
getHeader() {
const version =
['configure', 'vcBuild', 'make'].reduce((a, c) => {
return (a += (this.options as any)[c]
@@ -206,11 +208,6 @@ export class NexeCompiler {
.update(version)
.digest('hex')
}
const serializedHeader = this._serializeHeader(header)
return header
}
private _serializeHeader(header: NexeHeader) {
return `process.__nexe=${JSON.stringify(header)};`
}
@@ -219,7 +216,6 @@ export class NexeCompiler {
const build = this.options.build
const location = this.getNodeExecutableLocation(build ? undefined : target)
let binary = (await pathExistsAsync(location)) ? createReadStream(location) : null
const header = this._generateHeader()
if (!build && !binary) {
step.modify('Fetching prebuilt binary')
binary = await this._fetchPrebuiltBinaryAsync(target)
@@ -228,10 +224,10 @@ export class NexeCompiler {
binary = await this._buildAsync()
step.log('Node binary compiled')
}
return this._assembleDeliverable(header, binary)
return this._assembleDeliverable(binary)
}
private _assembleDeliverable(header: NexeHeader, binary: NodeJS.ReadableStream) {
private _assembleDeliverable(binary: NodeJS.ReadableStream) {
if (this.options.empty) {
return binary
}
@@ -240,8 +236,7 @@ export class NexeCompiler {
artifact.push(chunk)
})
binary.on('close', () => {
const content = this._serializeHeader(header) + this.shims.join(';') + ';' + this.input
const content = [this.shims.join(''), this.input].join(';')
artifact.push(content)
artifact.push(this.resources.bundle)
const lengths = Buffer.from(Array(16))
src/nexe.ts
+4 -10
View File
@@ -1,13 +1,7 @@
import { compose, PromiseConfig, Middleware } from 'app-builder'
import { compose, Middleware } from 'app-builder'
import resource from './steps/resource'
import { NexeCompiler } from './compiler'
import {
argv,
nexeVersion as version,
normalizeOptionsAsync,
NexeOptions,
NexePatch
} from './options'
import { argv, version, help, normalizeOptionsAsync, NexeOptions, NexePatch } from './options'
import cli from './steps/cli'
import bundle from './steps/bundle'
import download from './steps/download'
@@ -40,7 +34,7 @@ async function compile(
const buildSteps = build
? [download, artifacts, ...patches, ...(options.patches as NexePatch[])]
: []
const nexe = compose(resource, bundle, cli, buildSteps, shim)
const nexe = compose(resource, bundle, cli, buildSteps, shim, options.plugins as NexePatch[])
return callback
? void nexe(compiler).then(
() => callback && callback(null),
@@ -52,4 +46,4 @@ async function compile(
: nexe(compiler)
}
export { argv, compile, version }
export { argv, compile, version, NexeCompiler, NexeOptions, help }
src/options.ts
+35 -35
View File
@@ -6,7 +6,7 @@ import { getTarget, NexeTarget } from './target'
import { EOL } from 'os'
import * as c from 'chalk'
export const nexeVersion = '2.0.0-rc.7'
export const version = '{{replace:0}}'
export interface NexePatch {
(compiler: NexeCompiler, next: () => Promise<void>): Promise<void>
@@ -30,6 +30,7 @@ export interface NexeOptions {
enableNodeCli: boolean
bundle: boolean | string
patches: (string | NexePatch)[]
plugins: (string | NexePatch)[]
native: any
empty: boolean
sourceUrl?: string
@@ -61,7 +62,8 @@ const defaults = {
compress: false,
build: false,
bundle: true,
patches: []
patches: [],
plugins: []
}
const alias = {
i: 'input',
@@ -87,37 +89,37 @@ ${c.bold('nexe <entry-file> [options]')}
${c.underline.bold('Options:')}
-i --input ${g('=index.js')} -- application entry point
-o --output ${g('=my-app.exe')} -- path to output file
-t --target ${g('=8.4.0-x64')} -- node version description
-n --name ${g('=my-app')} -- main app module name
-r --resource -- *embed files (glob) within the binary
-i --input -- application entry point
-o --output -- path to output file
-t --target -- node version description
-n --name -- main app module name
-r --resource -- *embed files (glob) within the binary
--plugin -- extend nexe runtime behavior
${c.underline.bold('Building from source:')}
-b --build -- build from source
-p --python -- python2 (as python) executable path
-f --flag -- *v8 flags to include during compilation
-c --configure -- *arguments to the configure step
-m --make -- *arguments to the make/build step
--snapshot -- path to a warmup snapshot
--ico -- file name for alternate icon file (windows)
--rc-* -- populate rc file options (windows)
--sourceUrl -- pass an alternate source (node.tar.gz) url
--enableNodeCli -- enable node cli enforcement (blocks app cli)
-b --build -- build from source
-p --python -- python2 (as python) executable path
-f --flag -- *v8 flags to include during compilation
-c --configure -- *arguments to the configure step
-m --make -- *arguments to the make/build step
--snapshot -- path to a warmup snapshot
--ico -- file name for alternate icon file (windows)
--rc-* -- populate rc file options (windows)
--sourceUrl -- pass an alternate source (node.tar.gz) url
--enableNodeCli -- enable node cli enforcement (blocks app cli)
${c.underline.bold('Other options:')}
--bundle -- custom bundling module with 'createBundle' export
--temp -- temp file storage default './nexe'
--cwd -- set the current working directory for the command
--fake-argv TODO -- fake argv[1] with entry file
--clean -- force download of sources
--silent -- disable logging
--verbose -- set logging to verbose
--bundle -- custom bundling module with 'createBundle' export
--temp -- temp file storage default './nexe'
--cwd -- set the current working directory for the command
--fake-argv -- fake argv[1] with entry file
--clean -- force download of sources
--silent -- disable logging
--verbose -- set logging to verbose
-* variable key name * option can be used more than once`.trim()
-* variable key name * option can be used more than once`.trim()
help = EOL + help + EOL
function flatten(...args: any[]): string[] {
@@ -203,12 +205,7 @@ function findInput(input: string, cwd: string) {
return ''
}
function normalizeOptionsAsync(input?: Partial<NexeOptions>): Promise<NexeOptions | never> {
if (argv.help || argv._.some((x: string) => x === 'version') || argv.version === true) {
return new Promise(() => {
process.stderr.write(argv.help ? help : nexeVersion + EOL, () => process.exit(0))
})
}
function normalizeOptionsAsync(input?: Partial<NexeOptions>): Promise<NexeOptions> {
const options = Object.assign({}, defaults, input) as NexeOptions
const opts = options as any
@@ -236,12 +233,15 @@ function normalizeOptionsAsync(input?: Partial<NexeOptions>): Promise<NexeOption
}
}
options.patches = options.patches.map(x => {
const requireDefault = (x: string) => {
if (typeof x === 'string') {
return require(x).default
}
return x
})
}
options.plugins = options.plugins.map(requireDefault)
options.patches = options.patches.map(requireDefault)
Object.keys(alias)
.filter(k => k !== 'rc')
@@ -250,4 +250,4 @@ function normalizeOptionsAsync(input?: Partial<NexeOptions>): Promise<NexeOption
return Promise.resolve(options)
}
export { argv, normalizeOptionsAsync }
export { argv, normalizeOptionsAsync, help }
src/patches/boot-nexe.ts
+53
View File
@@ -0,0 +1,53 @@
const fs = require('fs')
const fd = fs.openSync(process.execPath, 'r')
const stat = fs.statSync(process.execPath)
const footer = Buffer.from(Array(32))
fs.readSync(fd, footer, 0, 32, stat.size - 32)
if (!footer.slice(0, 16).equals(Buffer.from('<nexe~~sentinel>'))) {
throw 'Invalid Nexe binary'
}
const contentSize = footer.readDoubleLE(16)
const resourceSize = footer.readDoubleLE(24)
const contentStart = stat.size - 32 - resourceSize - contentSize
const resourceStart = contentStart + contentSize
Object.defineProperty(
process,
'__nexe',
(function() {
let nexeHeader: any = null
return {
get: function() {
return nexeHeader
},
set: function(value: any) {
if (nexeHeader) {
throw new Error('__nexe cannot be reconfigured')
}
nexeHeader = Object.assign({}, value, {
layout: {
stat,
contentSize,
contentStart,
resourceSize,
resourceStart
}
})
Object.freeze(nexeHeader)
},
enumerable: false,
configurable: false
}
})()
)
const contentBuffer = Buffer.from(Array(contentSize))
fs.readSync(fd, contentBuffer, 0, contentSize, contentStart)
fs.closeSync(fd)
const Module = require('module')
process.mainModule = new Module(process.execPath, null)
process.mainModule!.loaded = true
;(process.mainModule as any)._compile(contentBuffer.toString(), process.execPath)
src/patches/index.ts
+1 -1
View File
@@ -5,7 +5,7 @@ import cli from './disable-node-cli'
import flags from './flags'
import ico from './ico'
import rc from './node-rc'
import { NexeCompiler } from '../compiler'
import { NexeCompiler, NexeOptions } from '../compiler'
const patches = [gyp, nexePatches, buildFixes, cli, flags, ico, rc]
src/patches/third-party-main.ts
+1 -116
View File
@@ -5,122 +5,7 @@ import { join } from 'path'
export default async function main(compiler: NexeCompiler, next: () => Promise<void>) {
await compiler.setFileContentsAsync(
'lib/_third_party_main.js',
`
const fs = require('fs')
const path = require('path')
const Buffer = require('buffer').Buffer
const isString = x => typeof x === 'string' || x instanceof String
const fd = fs.openSync(process.execPath, 'r')
const size = fs.statSync(process.execPath).size
const footer = Buffer.from(Array(32))
fs.readSync(fd, footer, 0, 32, size - 32)
if (!footer.slice(0, 16).equals(Buffer.from('<nexe~~sentinel>'))) {
throw 'Invalid Nexe binary'
}
const contentSize = footer.readDoubleLE(16)
const resourceSize = footer.readDoubleLE(24)
const contentStart = size - 32 - resourceSize - contentSize
const resourceStart = contentStart + contentSize
Object.defineProperty(process, '__nexe', (function () {
let nexeHeader = null
return {
get: function () {
return nexeHeader
},
set: function (value) {
if (nexeHeader) {
throw new Error('__nexe cannot be reconfigured')
}
nexeHeader = value
Object.freeze(nexeHeader)
},
enumerable: false,
configurable: false
}
})());
if (resourceSize) {
const originalReadFile = fs.readFile
const originalReadFileSync = fs.readFileSync
let setupManifest = () => {
const manifest = process.__nexe && process.__nexe.resources
if (!manifest) {
return
}
Object.keys(manifest).forEach((key) => {
const absolutePath = path.resolve(key)
if (!manifest[absolutePath]) {
manifest[absolutePath] = manifest[key]
}
const normalizedPath = path.normalize(key)
if (!manifest[normalizedPath]) {
manifest[normalizedPath] = manifest[key]
}
})
normalizeManifest = () => {}
};
fs.readFile = function readFile (file, options, callback) {
setupManifest()
const manifest = process.__nexe
const entry = manifest && manifest.resources[file]
if (!manifest || !entry || !isString(file)) {
return originalReadFile.apply(fs, arguments)
}
const [offset, length] = entry
const resourceOffset = resourceStart + offset
const encoding = isString(options) ? options : null
callback = typeof options === 'function' ? options : callback
fs.open(process.execPath, 'r', function (err, fd) {
if (err) return callback(err, null)
fs.read(fd, Buffer.alloc(length), 0, length, resourceOffset, function (error, bytesRead, buffer) {
if (error) {
return fs.close(fd, function () {
callback(error, null)
})
}
fs.close(fd, function (err) {
if (err) {
return callback(err, buffer)
}
const result = Buffer.from(buffer.toString(), 'base64')
callback(err, encoding ? result.toString(encoding) : result)
})
})
})
}
fs.readFileSync = function readFileSync (file, options) {
setupManifest()
const manifest = process.__nexe
const entry = manifest && manifest.resources[file]
if (!manifest || !entry || !isString(file)) {
return originalReadFileSync.apply(fs, arguments)
}
const [offset, length] = entry
const resourceOffset = resourceStart + offset
const encoding = isString(options) ? options : null
const fd = fs.openSync(process.execPath, 'r')
const result = Buffer.alloc(length)
fs.readSync(fd, result, 0, length, resourceOffset)
fs.closeSync(fd)
const contents = Buffer.from(result.toString(), 'base64')
return encoding ? contents.toString(encoding) : contents
}
}
const contentBuffer = Buffer.from(Array(contentSize));
fs.readSync(fd, contentBuffer, 0, contentSize, contentStart);
fs.closeSync(fd);
const Module = require('module');
process.mainModule = new Module(process.execPath, null);
process.mainModule.loaded = true;
process.mainModule._compile(contentBuffer.toString(), process.execPath);
`.trim()
'{{replace:lib/patches/boot-nexe.js}}'
)
return next()
}
src/steps/bundle.ts
+3 -3
View File
@@ -1,11 +1,11 @@
import { NexeCompiler } from '../compiler'
import { FuseBox, JSONPlugin, CSSPlugin, HTMLPlugin, QuantumPlugin } from 'fuse-box'
import { readFileAsync, writeFileAsync } from '../util'
import { resolve, relative } from 'path'
import NativeModulePlugin from '../bundling/fuse-native-module-plugin'
import { NexeOptions } from '../options'
function createBundle(options: NexeOptions) {
const { FuseBox, JSONPlugin, CSSPlugin, HTMLPlugin, QuantumPlugin } = require('fuse-box')
const plugins: any = [JSONPlugin(), CSSPlugin(), HTMLPlugin(), NativeModulePlugin(options.native)]
if (options.compress) {
plugins.push(
@@ -28,9 +28,9 @@ function createBundle(options: NexeOptions) {
})
const input = relative(options.cwd, options.input).replace(/\\/g, '/')
fuse.bundle(options.name).instructions(`> ${input}`)
return fuse.run().then(x => {
return fuse.run().then((x: any) => {
let output = ''
x.bundles.forEach(y => (output = y.context.output.lastPrimaryOutput.content!.toString()))
x.bundles.forEach((y: any) => (output = y.context.output.lastPrimaryOutput.content.toString()))
return output
})
}
src/steps/cli.ts
+4 -2
View File
@@ -33,14 +33,16 @@ function readStreamAsync(stream: NodeJS.ReadableStream): PromiseLike<string> {
*/
export default async function cli(compiler: NexeCompiler, next: () => Promise<void>) {
const { log } = compiler
let stdInUsed = false
if (!process.stdin.isTTY) {
log.step('Using stdin as input')
stdInUsed = true
compiler.input = await readStreamAsync(process.stdin)
}
await next()
log.step(`Bundling: '${stdInUsed ? '[stdin]' : compiler.options.input}'`)
const target = compiler.options.targets.shift() as NexeTarget
const deliverable = await compiler.compileAsync(target)
src/steps/shim-fs.ts
+93
View File
@@ -0,0 +1,93 @@
import { Stats } from 'fs'
import { ok } from 'assert'
import { resolve, normalize } from 'path'
const binary = (process as any).__nexe as NexeBinary
ok(binary)
const manifest = binary.resources
const isString = (x: any): x is string => typeof x === 'string' || x instanceof String
if (Object.keys(manifest).length) {
const fs = require('fs')
const originalReadFile = fs.readFile
const originalReadFileSync = fs.readFileSync
const resourceStart = binary.layout.resourceStart
let setupManifest = () => {
const manifest = binary.resources
Object.keys(manifest).forEach(key => {
const absolutePath = resolve(key)
if (!manifest[absolutePath]) {
manifest[absolutePath] = manifest[key]
}
const normalizedPath = normalize(key)
if (!manifest[normalizedPath]) {
manifest[normalizedPath] = manifest[key]
}
})
setupManifest = () => {}
}
//TODO track inflight fs reqs??
var nfs = {
readFile: function readFile(file: any, options: any, callback: any) {
setupManifest()
const entry = manifest[file]
if (!entry || !isString(file)) {
return originalReadFile.apply(fs, arguments)
}
const [offset, length] = entry
const resourceOffset = resourceStart + offset
const encoding = isString(options) ? options : null
callback = typeof options === 'function' ? options : callback
fs.open(process.execPath, 'r', function(err: Error, fd: number) {
if (err) return callback(err, null)
fs.read(fd, Buffer.alloc(length), 0, length, resourceOffset, function(
error: Error,
bytesRead: number,
result: Buffer
) {
if (error) {
return fs.close(fd, function() {
callback(error, null)
})
}
fs.close(fd, function(err: Error) {
if (err) {
return callback(err, result)
}
callback(err, encoding ? result.toString(encoding) : result)
})
})
})
},
readFileSync: function readFileSync(file: any, options: any) {
setupManifest()
const entry = manifest[file]
if (!entry || !isString(file)) {
return originalReadFileSync.apply(fs, arguments)
}
const [offset, length] = entry
const resourceOffset = resourceStart + offset
const encoding = isString(options) ? options : null
const fd = fs.openSync(process.execPath, 'r')
const result = Buffer.alloc(length)
fs.readSync(fd, result, 0, length, resourceOffset)
fs.closeSync(fd)
return encoding ? result.toString(encoding) : result
}
}
Object.assign(fs, nfs)
}
interface NexeBinary {
resources: { [key: string]: number[] }
version: string
layout: {
stat: Stats
contentSize: number
contentStart: number
resourceSize: number
resourceStart: number
}
}
src/steps/shim.ts
+17 -10
View File
@@ -1,18 +1,25 @@
import { NexeCompiler } from '../compiler'
function wrap(code: string) {
return '!(function () {' + code + '})();'
}
export default function(compiler: NexeCompiler, next: () => Promise<void>) {
if (!compiler.options.fakeArgv) {
return next()
compiler.shims.push(wrap(compiler.getHeader()))
if (compiler.options.resources.length) {
compiler.shims.push(wrap('{{replace:lib/steps/shim-fs.js}}'))
}
const nty = !process.stdin.isTTY
const input = nty ? '[stdin]' : compiler.options.input
compiler.input =
`!(() => {
var r = require('path').resolve;
process.argv.splice(1,0, ${nty ? `'${input}'` : `r("${input}")`});
})();` + compiler.input
if (compiler.options.fakeArgv) {
const nty = !process.stdin.isTTY
const input = nty ? '[stdin]' : compiler.options.input
compiler.shims.push(
wrap(`
var r = require('path').resolve;
process.argv.splice(1,0, ${nty ? `'${input}'` : `r("${input}")`});`)
)
}
return next()
}