From ba78c7dc54f174a83ff9eed555d035f35323374c Mon Sep 17 00:00:00 2001 From: calebboyd Date: Wed, 23 Aug 2017 22:35:01 -0500 Subject: [PATCH] refactor: append in lieu of inserting payload --- package.json | 2 +- src/compiler.ts | 174 +++++++------------------------- src/options.ts | 5 +- src/patches/index.ts | 11 +- src/patches/third-party-main.ts | 109 ++++++++++++-------- 5 files changed, 109 insertions(+), 192 deletions(-) diff --git a/package.json b/package.json index 678d887..e6f622b 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "nexe", "description": "Create a single executable out of your Node.js application", "license": "MIT", - "version": "2.0.0-beta.6", + "version": "2.0.0-beta.7", "contributors": [ "Craig Condon (http://crcn.io)", "Jared Allard ", diff --git a/src/compiler.ts b/src/compiler.ts index 09a9e00..56a42a8 100644 --- a/src/compiler.ts +++ b/src/compiler.ts @@ -13,32 +13,15 @@ import { NexeOptions } from './options' const isBsd = Boolean(~process.platform.indexOf('bsd')) const make = isWindows ? 'vcbuild.bat' : isBsd ? 'gmake' : 'make' const configure = isWindows ? 'configure' : './configure' -const marker = Buffer.from('').toString('hex') -const tail = `\n//${marker}` -const needle = Buffer.from(marker) -const fixedIntegerLength = 10 -const padLeft = (x: string | number, l = fixedIntegerLength, c = '0') => (c.repeat(l) + x).slice(-l) -const inflate = (value: string, size: number) => { - if (!size || value.length >= size) { - return value - } - return value + ' '.repeat(size - value.length) -} export interface NexeFile { filename: string + absPath: string contents: string } interface NexeHeader { - /** - * Zero padded number indicating the extra size available in the binary - */ - paddingSize: string - /** - * - */ - binaryOffset: string + resources: { [key: string]: number[] } version: string } @@ -79,12 +62,11 @@ export class NexeCompiler { this.readFileAsync = async (file: string) => { let cachedFile = this.files.find(x => normalize(x.filename) === normalize(file)) if (!cachedFile) { + const absPath = join(this.src, file) cachedFile = { + absPath, filename: file, - contents: await readFileAsync(join(this.src, file), 'utf-8').catch( - { code: 'ENOENT' }, - () => '' - ) + contents: await readFileAsync(absPath, 'utf-8').catch({ code: 'ENOENT' }, () => '') } this.files.push(cachedFile) } @@ -107,22 +89,6 @@ export class NexeCompiler { return this.log.flush().then(x => process.exit(code)) } - private _findPaddingSize(size: number, override = this.options.padding) { - if (override === 0) { - return 0 - } - size = override > size ? override : size - const padding = [3, 6, 9, 16, 25, 40].map(x => x * 1e6).find(p => size <= p) - if (!padding) { - throw new Error( - `No prebuilt target large enough (${(size / 1024).toFixed( - 2 - )}Mb).\nUse the --build flag and build for the current platform` - ) - } - return padding - } - private _getNodeExecutableLocation(target?: string | null) { if (target) { return join(this.options.temp, target) @@ -150,10 +116,16 @@ export class NexeCompiler { } private async _buildAsync() { - this.compileStep.log(`Configuring node build: ${this.options.configure}`) + this.compileStep.log( + `Configuring node build${this.options.configure.length + ? ': ' + this.options.configure + : '...'}` + ) await this._configureAsync() const buildOptions = isWindows ? this.options.vcBuild : this.options.make - this.compileStep.log(`Compiling Node with arguments: ${buildOptions}`) + this.compileStep.log( + `Compiling Node${buildOptions.length ? ' with arguments: ' + buildOptions : '...'}` + ) await this._runBuildCommandAsync(make, buildOptions) return createReadStream(this._getNodeExecutableLocation()) } @@ -162,128 +134,58 @@ export class NexeCompiler { return this._buildAsync() } - private _getPayload(header: NexeHeader) { - return this._serializeHeader(header) + this.input + '/**' + this.resources.bundle + `**/` - } - private _generateHeader() { - const zeros = padLeft(0) const version = ['configure', 'vcBuild', 'make'].reduce((a, c) => { return (a += (this.options as any)[c].slice().sort().join()) }, '') + this.options.enableNodeCli const header = { - version: padLeft(0, 32), resources: this.resources.index, - contentSize: zeros, - paddingSize: zeros, - resourceOffset: zeros, - binaryOffset: zeros + version: createHash('md5').update(version).digest('hex') } const serializedHeader = this._serializeHeader(header) - header.contentSize = padLeft(Buffer.byteLength(this._getPayload(header))) - header.paddingSize = padLeft(this._findPaddingSize(+header.contentSize)) - header.resourceOffset = padLeft(Buffer.byteLength(serializedHeader + this.input + '/**')) - header.version = createHash('md5').update(version + header.paddingSize).digest('hex') return header } - private async _getExistingBinaryHeaderAsync(target: string | undefined | null) { - const filename = this._getNodeExecutableLocation(target) - const existingBinary = await pathExistsAsync(filename) - if (existingBinary) { - return this._extractHeaderAsync(filename) - } - return null - } - - private _extractHeaderAsync(path: string): Promise { - const binary = createReadStream(path) - const haystack = new Needle(needle) - let needles = 0 - let stackCache: Buffer[] = [] - return new Promise((resolve, reject) => { - binary - .on('error', reject) - .pipe(haystack) - .on('error', reject) - .on('close', () => reject(new Error(`Binary: ${path} is not compatible with nexe`))) - .on('haystack', (x: Buffer) => needles && stackCache.push(x)) - .on('needle', () => { - if (++needles === 2) { - resolve(JSON.parse(Buffer.concat(stackCache).toString())) - binary.close() - haystack.end() - } - }) - }) - } - private _serializeHeader(header: NexeHeader) { - return `/**${marker}${JSON.stringify(header)}${marker}**/process.__nexe=${JSON.stringify( - header - )};` - } - - async setMainModule(compiler: NexeCompiler, next: () => Promise) { - await next() - const header = compiler._generateHeader() - const contents = inflate(this._getPayload(header), +header.paddingSize) + tail - const file = await compiler.readFileAsync('lib/_third_party_main.js') - file.contents += '\n' + contents - const mod = Buffer.byteLength(file.contents) % 16 - if (mod) { - file.contents += ' '.repeat(16 - mod) - } + return `/**${JSON.stringify(header)}**/process.__nexe=${JSON.stringify(header)};` } async compileAsync() { const step = (this.compileStep = this.log.step('Compiling result')) let target = this.options.targets.slice().shift() - let prebuiltBinary = null + const location = this._getNodeExecutableLocation(target) + let binary = (await pathExistsAsync(location)) ? createReadStream(location) : null const header = this._generateHeader() - target = target && `${target}-${header.version.slice(0, 6)}` step.log(`Scanning existing binary...`) - const existingHeader = await this._getExistingBinaryHeaderAsync(target) - if (existingHeader && existingHeader.version === header.version) { - const location = this._getNodeExecutableLocation(target) - step.log(`Source already built: ${location}`) - prebuiltBinary = createReadStream(location) + + if (target && !binary) { + //throw new Error('\nNot Implemented, use --build during beta\n') + binary = await this._fetchPrebuiltBinaryAsync() } - if (target) { - throw new Error('\nNot Implemented, use --build during beta\n') - // prebuiltBinary = await this._fetchPrebuiltBinaryAsync(target) - } - if (!prebuiltBinary) { - prebuiltBinary = await this._buildAsync() + + if (!binary) { + binary = await this._buildAsync() step.log('Node binary compiled') } - return this._assembleDeliverable(header, prebuiltBinary) + return this._assembleDeliverable(header, binary) } private _assembleDeliverable(header: NexeHeader, binary: NodeJS.ReadableStream) { - const haystack = new Needle(Buffer.concat([Buffer.from('/**'), needle])) const artifact = new Readable({ read() {} }) - let needles = 0 - let currentStackSize = 0 - binary.pipe(haystack) - haystack - .on('close', () => artifact.push(null)) - .on('needle', () => ++needles && haystack.needle(needle)) - .on('haystack', (x: Buffer) => { - if (!needles) { - currentStackSize += x.length - artifact.push(x) - } - if (needles === 1 && !+header.binaryOffset) { - header.binaryOffset = padLeft(currentStackSize) - const content = Buffer.from(inflate(this._getPayload(header), +header.paddingSize) + tail) - artifact.push(content) - } - if (needles > 2) { - artifact.push(x) - } - }) + binary.on('data', (chunk: Buffer) => { + artifact.push(chunk) + }) + binary.on('close', () => { + const content = this._serializeHeader(header) + this.input + artifact.push(content) + artifact.push(this.resources.bundle) + const lengths = Buffer.from(Array(16)) + lengths.writeDoubleLE(Buffer.byteLength(content), 0) + lengths.writeDoubleLE(Buffer.byteLength(this.resources.bundle), 8) + artifact.push(Buffer.concat([Buffer.from(''), lengths])) + artifact.push(null) + }) return artifact } } diff --git a/src/options.ts b/src/options.ts index fd64f9a..b6a890d 100644 --- a/src/options.ts +++ b/src/options.ts @@ -61,7 +61,6 @@ export interface NexeOptions { silent?: boolean verbose?: boolean info?: boolean - padding: number patches: NexePatch[] empty: boolean @@ -183,7 +182,7 @@ function extractName(options: NexeOptions) { function normalizeOptionsAsync(input: Partial) { if (argv.help || argv._.some((x: string) => x === 'version')) { - process.stderr.write(argv.help ? help : '2.0.0-beta.6' + EOL, () => process.exit(0)) + process.stderr.write(argv.help ? help : '2.0.0-beta.7' + EOL, () => process.exit(0)) } const options = Object.assign({}, defaults, input) as NexeOptions @@ -199,7 +198,7 @@ function normalizeOptionsAsync(input: Partial) { options.resources = flattenFilter(opts.resource, options.resources) options.rc = options.rc || extractCliMap(/^rc-.*/, options) - if (options.build || options.padding === 0) { + if (options.build) { options.targets = [] options.build = true } else if (!options.targets.length) { diff --git a/src/patches/index.ts b/src/patches/index.ts index 998289a..c878b98 100644 --- a/src/patches/index.ts +++ b/src/patches/index.ts @@ -7,15 +7,6 @@ import ico from './ico' import rc from './node-rc' import { NexeCompiler } from '../compiler' -const patches = [ - gyp, - (compiler: NexeCompiler, next: () => Promise) => compiler.setMainModule(compiler, next), - nexePatches, - buildFixes, - cli, - flags, - ico, - rc -] +const patches = [gyp, nexePatches, buildFixes, cli, flags, ico, rc] export default patches diff --git a/src/patches/third-party-main.ts b/src/patches/third-party-main.ts index 80cac86..2f2db7a 100644 --- a/src/patches/third-party-main.ts +++ b/src/patches/third-party-main.ts @@ -8,6 +8,20 @@ const fs = require('fs') const Buffer = require('buffer').Buffer const isString = x => typeof x === 'string' || x instanceof String +const fd = fs.openSync(process.execPath, 'r') +const size = fs.statSync(process.execPath).size +const footer = Buffer.from(Array(46)) +fs.readSync(fd, footer, 0, 32, size - 32) + +if (!footer.slice(0, 16).equals(Buffer.from(''))) { + throw new Error('Invalid Nexe binary') +} + +const contentSize = footer.readDoubleLE(16) +const resourceSize = footer.readDoubleLE(24) +const contentStart = size - 32 - resourceSize - contentSize +const resourceStart = contentStart + contentSize + Object.defineProperty(process, '__nexe', (function () { let nexeHeader = null return { @@ -26,55 +40,66 @@ Object.defineProperty(process, '__nexe', (function () { } })()); -const originalReadFile = fs.readFile -const originalReadFileSync = fs.readFileSync +if (resourceSize) { + const originalReadFile = fs.readFile + const originalReadFileSync = fs.readFileSync -fs.readFile = function readFile (file, options, callback) { - const manifest = process.__nexe - const entry = manifest && manifest.resources[file] - if (!manifest || !entry || !isString(file)) { - return originalReadFile.apply(fs, arguments) - } - const [offset, length] = entry - const resourceOffset = +manifest.binaryOffset + +manifest.resourceOffset + offset - const encoding = isString(options) ? options : null - callback = typeof options === 'function' ? options : callback + fs.readFile = function readFile (file, options, callback) { + const manifest = process.__nexe + const entry = manifest && manifest.resources[file] + if (!manifest || !entry || !isString(file)) { + return originalReadFile.apply(fs, arguments) + } + const [offset, length] = entry + const resourceOffset = resourceStart + offset + const encoding = isString(options) ? options : null + callback = typeof options === 'function' ? options : callback - fs.open(process.execPath, 'r', function (err, fd) { - if (err) return callback(err, null) - fs.read(fd, Buffer.alloc(length), 0, length, resourceOffset, function (error, bytesRead, buffer) { - if (error) { - return fs.close(fd, function () { - callback(error, null) - }) - } - fs.close(fd, function (err) { - if (err) { - return callback(err, buffer) + fs.open(process.execPath, 'r', function (err, fd) { + if (err) return callback(err, null) + fs.read(fd, Buffer.alloc(length), 0, length, resourceOffset, function (error, bytesRead, buffer) { + if (error) { + return fs.close(fd, function () { + callback(error, null) + }) } - const result = Buffer.from(buffer.toString(), 'base64') - callback(err, encoding ? result.toString(encoding) : result) + fs.close(fd, function (err) { + if (err) { + return callback(err, buffer) + } + const result = Buffer.from(buffer.toString(), 'base64') + callback(err, encoding ? result.toString(encoding) : result) + }) }) }) - }) + } + + fs.readFileSync = function readFileSync (file, options) { + const manifest = process.__nexe + const entry = manifest && manifest.resources[file] + if (!manifest || !entry || !isString(file)) { + return originalReadFileSync.apply(fs, arguments) + } + const [offset, length] = entry + const resourceOffset = resourceStart + offset + const encoding = isString(options) ? options : null + const fd = fs.openSync(process.execPath, 'r') + const result = Buffer.alloc(length) + fs.readSync(fd, result, 0, length, resourceOffset) + fs.closeSync(fd) + const contents = Buffer.from(result.toString(), 'base64') + return encoding ? contents.toString(encoding) : contents + } } -fs.readFileSync = function readFileSync (file, options) { - const manifest = process.__nexe - const entry = manifest && manifest.resources[file] - if (!manifest || !entry || !isString(file)) { - return originalReadFileSync.apply(fs, arguments) - } - const [offset, length] = entry - const resourceOffset = +manifest.binaryOffset + +manifest.resourceOffset + offset - const encoding = isString(options) ? options : null - const fd = fs.openSync(process.execPath, 'r') - const result = Buffer.alloc(length) - fs.readSync(fd, result, 0, length, resourceOffset) - fs.closeSync(fd) - const contents = Buffer.from(result.toString(), 'base64') - return encoding ? contents.toString(encoding) : contents -}`.trim() +const contentBuffer = Buffer.from(Array(contentSize)) +fs.readSync(fd, contentBuffer, 0, contentSize, contentStart) +fs.closeSync(fd) +const Module = require('module') +process.mainModule = new Module(process.execPath, null) +process.mainModule.loaded = true +process.mainModule._compile(contentBuffer.toString(), process.execPath) +`.trim() ) return next() }