refactor: append in lieu of inserting payload

package.json

@@ -2,7 +2,7 @@
   "name": "nexe",
   "description": "Create a single executable out of your Node.js application",
   "license": "MIT",
-  "version": "2.0.0-beta.6",
+  "version": "2.0.0-beta.7",
   "contributors": [
     "Craig Condon <craig.j.condon@gmail.com> (http://crcn.io)",
     "Jared Allard <jaredallard@outlook.com>",

src/compiler.ts

@@ -13,32 +13,15 @@ import { NexeOptions } from './options'
 const isBsd = Boolean(~process.platform.indexOf('bsd'))
 const make = isWindows ? 'vcbuild.bat' : isBsd ? 'gmake' : 'make'
 const configure = isWindows ? 'configure' : './configure'
-const marker = Buffer.from('<nexe~sentinel>').toString('hex')
-const tail = `\n//${marker}`
-const needle = Buffer.from(marker)
-const fixedIntegerLength = 10
-const padLeft = (x: string | number, l = fixedIntegerLength, c = '0') => (c.repeat(l) + x).slice(-l)
-const inflate = (value: string, size: number) => {
-  if (!size || value.length >= size) {
-    return value
-  }
-  return value + ' '.repeat(size - value.length)
-}
 
 export interface NexeFile {
   filename: string
+  absPath: string
   contents: string
 }
 
 interface NexeHeader {
-  /**
-   * Zero padded number indicating the extra size available in the binary
-   */
-  paddingSize: string
-  /**
-   *
-   */
-  binaryOffset: string
+  resources: { [key: string]: number[] }
   version: string
 }
 
@@ -79,12 +62,11 @@ export class NexeCompiler {
     this.readFileAsync = async (file: string) => {
       let cachedFile = this.files.find(x => normalize(x.filename) === normalize(file))
       if (!cachedFile) {
+        const absPath = join(this.src, file)
         cachedFile = {
+          absPath,
           filename: file,
-          contents: await readFileAsync(join(this.src, file), 'utf-8').catch(
-            { code: 'ENOENT' },
-            () => ''
-          )
+          contents: await readFileAsync(absPath, 'utf-8').catch({ code: 'ENOENT' }, () => '')
         }
         this.files.push(cachedFile)
       }
@@ -107,22 +89,6 @@ export class NexeCompiler {
     return this.log.flush().then(x => process.exit(code))
   }
 
-  private _findPaddingSize(size: number, override = this.options.padding) {
-    if (override === 0) {
-      return 0
-    }
-    size = override > size ? override : size
-    const padding = [3, 6, 9, 16, 25, 40].map(x => x * 1e6).find(p => size <= p)
-    if (!padding) {
-      throw new Error(
-        `No prebuilt target large enough (${(size / 1024).toFixed(
-          2
-        )}Mb).\nUse the --build flag and build for the current platform`
-      )
-    }
-    return padding
-  }
-
   private _getNodeExecutableLocation(target?: string | null) {
     if (target) {
       return join(this.options.temp, target)
@@ -150,10 +116,16 @@ export class NexeCompiler {
   }
 
   private async _buildAsync() {
-    this.compileStep.log(`Configuring node build: ${this.options.configure}`)
+    this.compileStep.log(
+      `Configuring node build${this.options.configure.length
+        ? ': ' + this.options.configure
+        : '...'}`
+    )
     await this._configureAsync()
     const buildOptions = isWindows ? this.options.vcBuild : this.options.make
-    this.compileStep.log(`Compiling Node with arguments: ${buildOptions}`)
+    this.compileStep.log(
+      `Compiling Node${buildOptions.length ? ' with arguments: ' + buildOptions : '...'}`
+    )
     await this._runBuildCommandAsync(make, buildOptions)
     return createReadStream(this._getNodeExecutableLocation())
   }
@@ -162,128 +134,58 @@ export class NexeCompiler {
     return this._buildAsync()
   }
 
-  private _getPayload(header: NexeHeader) {
-    return this._serializeHeader(header) + this.input + '/**' + this.resources.bundle + `**/`
-  }
-
   private _generateHeader() {
-    const zeros = padLeft(0)
     const version =
       ['configure', 'vcBuild', 'make'].reduce((a, c) => {
         return (a += (this.options as any)[c].slice().sort().join())
       }, '') + this.options.enableNodeCli
     const header = {
-      version: padLeft(0, 32),
       resources: this.resources.index,
-      contentSize: zeros,
-      paddingSize: zeros,
-      resourceOffset: zeros,
-      binaryOffset: zeros
+      version: createHash('md5').update(version).digest('hex')
     }
     const serializedHeader = this._serializeHeader(header)
-    header.contentSize = padLeft(Buffer.byteLength(this._getPayload(header)))
-    header.paddingSize = padLeft(this._findPaddingSize(+header.contentSize))
-    header.resourceOffset = padLeft(Buffer.byteLength(serializedHeader + this.input + '/**'))
-    header.version = createHash('md5').update(version + header.paddingSize).digest('hex')
     return header
   }
 
-  private async _getExistingBinaryHeaderAsync(target: string | undefined | null) {
-    const filename = this._getNodeExecutableLocation(target)
-    const existingBinary = await pathExistsAsync(filename)
-    if (existingBinary) {
-      return this._extractHeaderAsync(filename)
-    }
-    return null
-  }
-
-  private _extractHeaderAsync(path: string): Promise<NexeHeader> {
-    const binary = createReadStream(path)
-    const haystack = new Needle(needle)
-    let needles = 0
-    let stackCache: Buffer[] = []
-    return new Promise((resolve, reject) => {
-      binary
-        .on('error', reject)
-        .pipe(haystack)
-        .on('error', reject)
-        .on('close', () => reject(new Error(`Binary: ${path} is not compatible with nexe`)))
-        .on('haystack', (x: Buffer) => needles && stackCache.push(x))
-        .on('needle', () => {
-          if (++needles === 2) {
-            resolve(JSON.parse(Buffer.concat(stackCache).toString()))
-            binary.close()
-            haystack.end()
-          }
-        })
-    })
-  }
-
   private _serializeHeader(header: NexeHeader) {
-    return `/**${marker}${JSON.stringify(header)}${marker}**/process.__nexe=${JSON.stringify(
-      header
-    )};`
-  }
-
-  async setMainModule(compiler: NexeCompiler, next: () => Promise<void>) {
-    await next()
-    const header = compiler._generateHeader()
-    const contents = inflate(this._getPayload(header), +header.paddingSize) + tail
-    const file = await compiler.readFileAsync('lib/_third_party_main.js')
-    file.contents += '\n' + contents
-    const mod = Buffer.byteLength(file.contents) % 16
-    if (mod) {
-      file.contents += ' '.repeat(16 - mod)
-    }
+    return `/**${JSON.stringify(header)}**/process.__nexe=${JSON.stringify(header)};`
   }
 
   async compileAsync() {
     const step = (this.compileStep = this.log.step('Compiling result'))
     let target = this.options.targets.slice().shift()
-    let prebuiltBinary = null
+    const location = this._getNodeExecutableLocation(target)
+    let binary = (await pathExistsAsync(location)) ? createReadStream(location) : null
     const header = this._generateHeader()
-    target = target && `${target}-${header.version.slice(0, 6)}`
     step.log(`Scanning existing binary...`)
-    const existingHeader = await this._getExistingBinaryHeaderAsync(target)
-    if (existingHeader && existingHeader.version === header.version) {
-      const location = this._getNodeExecutableLocation(target)
-      step.log(`Source already built: ${location}`)
-      prebuiltBinary = createReadStream(location)
+
+    if (target && !binary) {
+      //throw new Error('\nNot Implemented, use --build during beta\n')
+      binary = await this._fetchPrebuiltBinaryAsync()
     }
-    if (target) {
-      throw new Error('\nNot Implemented, use --build during beta\n')
-      // prebuiltBinary = await this._fetchPrebuiltBinaryAsync(target)
-    }
-    if (!prebuiltBinary) {
-      prebuiltBinary = await this._buildAsync()
+
+    if (!binary) {
+      binary = await this._buildAsync()
       step.log('Node binary compiled')
     }
-    return this._assembleDeliverable(header, prebuiltBinary)
+    return this._assembleDeliverable(header, binary)
   }
 
   private _assembleDeliverable(header: NexeHeader, binary: NodeJS.ReadableStream) {
-    const haystack = new Needle(Buffer.concat([Buffer.from('/**'), needle]))
     const artifact = new Readable({ read() {} })
-    let needles = 0
-    let currentStackSize = 0
-    binary.pipe(haystack)
-    haystack
-      .on('close', () => artifact.push(null))
-      .on('needle', () => ++needles && haystack.needle(needle))
-      .on('haystack', (x: Buffer) => {
-        if (!needles) {
-          currentStackSize += x.length
-          artifact.push(x)
-        }
-        if (needles === 1 && !+header.binaryOffset) {
-          header.binaryOffset = padLeft(currentStackSize)
-          const content = Buffer.from(inflate(this._getPayload(header), +header.paddingSize) + tail)
-          artifact.push(content)
-        }
-        if (needles > 2) {
-          artifact.push(x)
-        }
-      })
+    binary.on('data', (chunk: Buffer) => {
+      artifact.push(chunk)
+    })
+    binary.on('close', () => {
+      const content = this._serializeHeader(header) + this.input
+      artifact.push(content)
+      artifact.push(this.resources.bundle)
+      const lengths = Buffer.from(Array(16))
+      lengths.writeDoubleLE(Buffer.byteLength(content), 0)
+      lengths.writeDoubleLE(Buffer.byteLength(this.resources.bundle), 8)
+      artifact.push(Buffer.concat([Buffer.from('<nexe~~sentinel>'), lengths]))
+      artifact.push(null)
+    })
     return artifact
   }
 }

src/options.ts

@@ -61,7 +61,6 @@ export interface NexeOptions {
   silent?: boolean
   verbose?: boolean
   info?: boolean
-  padding: number
   patches: NexePatch[]
 
   empty: boolean
@@ -183,7 +182,7 @@ function extractName(options: NexeOptions) {
 
 function normalizeOptionsAsync(input: Partial<NexeOptions>) {
   if (argv.help || argv._.some((x: string) => x === 'version')) {
-    process.stderr.write(argv.help ? help : '2.0.0-beta.6' + EOL, () => process.exit(0))
+    process.stderr.write(argv.help ? help : '2.0.0-beta.7' + EOL, () => process.exit(0))
   }
 
   const options = Object.assign({}, defaults, input) as NexeOptions
@@ -199,7 +198,7 @@ function normalizeOptionsAsync(input: Partial<NexeOptions>) {
   options.resources = flattenFilter(opts.resource, options.resources)
   options.rc = options.rc || extractCliMap(/^rc-.*/, options)
 
-  if (options.build || options.padding === 0) {
+  if (options.build) {
     options.targets = []
     options.build = true
   } else if (!options.targets.length) {

src/patches/index.ts

@@ -7,15 +7,6 @@ import ico from './ico'
 import rc from './node-rc'
 import { NexeCompiler } from '../compiler'
 
-const patches = [
-  gyp,
-  (compiler: NexeCompiler, next: () => Promise<void>) => compiler.setMainModule(compiler, next),
-  nexePatches,
-  buildFixes,
-  cli,
-  flags,
-  ico,
-  rc
-]
+const patches = [gyp, nexePatches, buildFixes, cli, flags, ico, rc]
 
 export default patches

src/patches/third-party-main.ts

@@ -8,6 +8,20 @@ const fs = require('fs')
 const Buffer = require('buffer').Buffer
 const isString = x => typeof x === 'string' || x instanceof String
 
+const fd = fs.openSync(process.execPath, 'r')
+const size = fs.statSync(process.execPath).size
+const footer = Buffer.from(Array(46))
+fs.readSync(fd, footer, 0, 32, size - 32)
+
+if (!footer.slice(0, 16).equals(Buffer.from('<nexe~~sentinel>'))) {
+  throw new Error('Invalid Nexe binary')
+}
+
+const contentSize = footer.readDoubleLE(16)
+const resourceSize = footer.readDoubleLE(24)
+const contentStart = size - 32 - resourceSize - contentSize
+const resourceStart = contentStart + contentSize
+
 Object.defineProperty(process, '__nexe', (function () {
   let nexeHeader = null
   return {
@@ -26,55 +40,66 @@ Object.defineProperty(process, '__nexe', (function () {
   }
 })());
 
-const originalReadFile = fs.readFile
-const originalReadFileSync = fs.readFileSync
+if (resourceSize) {
+  const originalReadFile = fs.readFile
+  const originalReadFileSync = fs.readFileSync
 
-fs.readFile = function readFile (file, options, callback) {
-  const manifest = process.__nexe
-  const entry = manifest && manifest.resources[file]
-  if (!manifest || !entry || !isString(file)) {
-    return originalReadFile.apply(fs, arguments)
-  }
-  const [offset, length] = entry
-  const resourceOffset = +manifest.binaryOffset + +manifest.resourceOffset + offset
-  const encoding = isString(options) ? options : null
-  callback = typeof options === 'function' ? options : callback
+  fs.readFile = function readFile (file, options, callback) {
+    const manifest = process.__nexe
+    const entry = manifest && manifest.resources[file]
+    if (!manifest || !entry || !isString(file)) {
+      return originalReadFile.apply(fs, arguments)
+    }
+    const [offset, length] = entry
+    const resourceOffset = resourceStart + offset
+    const encoding = isString(options) ? options : null
+    callback = typeof options === 'function' ? options : callback
 
-  fs.open(process.execPath, 'r', function (err, fd) {
-    if (err) return callback(err, null)
-    fs.read(fd, Buffer.alloc(length), 0, length, resourceOffset, function (error, bytesRead, buffer) {
-      if (error) {
-        return fs.close(fd, function () {
-          callback(error, null)
-        })
-      }
-      fs.close(fd, function (err) {
-        if (err) {
-          return callback(err, buffer)
+    fs.open(process.execPath, 'r', function (err, fd) {
+      if (err) return callback(err, null)
+      fs.read(fd, Buffer.alloc(length), 0, length, resourceOffset, function (error, bytesRead, buffer) {
+        if (error) {
+          return fs.close(fd, function () {
+            callback(error, null)
+          })
         }
-        const result = Buffer.from(buffer.toString(), 'base64')
-        callback(err, encoding ? result.toString(encoding) : result)
+        fs.close(fd, function (err) {
+          if (err) {
+            return callback(err, buffer)
+          }
+          const result = Buffer.from(buffer.toString(), 'base64')
+          callback(err, encoding ? result.toString(encoding) : result)
+        })
       })
     })
-  })
+  }
+
+  fs.readFileSync = function readFileSync (file, options) {
+    const manifest = process.__nexe
+    const entry = manifest && manifest.resources[file]
+    if (!manifest || !entry || !isString(file)) {
+      return originalReadFileSync.apply(fs, arguments)
+    }
+    const [offset, length] = entry
+    const resourceOffset = resourceStart + offset
+    const encoding = isString(options) ? options : null
+    const fd = fs.openSync(process.execPath, 'r')
+    const result = Buffer.alloc(length)
+    fs.readSync(fd, result, 0, length, resourceOffset)
+    fs.closeSync(fd)
+    const contents = Buffer.from(result.toString(), 'base64')
+    return encoding ? contents.toString(encoding) : contents
+  }
 }
 
-fs.readFileSync = function readFileSync (file, options) {
-  const manifest = process.__nexe
-  const entry = manifest && manifest.resources[file]
-  if (!manifest || !entry || !isString(file)) {
-    return originalReadFileSync.apply(fs, arguments)
-  }
-  const [offset, length] = entry
-  const resourceOffset = +manifest.binaryOffset + +manifest.resourceOffset + offset
-  const encoding = isString(options) ? options : null
-  const fd = fs.openSync(process.execPath, 'r')
-  const result = Buffer.alloc(length)
-  fs.readSync(fd, result, 0, length, resourceOffset)
-  fs.closeSync(fd)
-  const contents = Buffer.from(result.toString(), 'base64')
-  return encoding ? contents.toString(encoding) : contents
-}`.trim()
+const contentBuffer = Buffer.from(Array(contentSize))
+fs.readSync(fd, contentBuffer, 0, contentSize, contentStart)
+fs.closeSync(fd)
+const Module = require('module')
+process.mainModule = new Module(process.execPath, null)
+process.mainModule.loaded = true
+process.mainModule._compile(contentBuffer.toString(), process.execPath)
+`.trim()
   )
   return next()
 }