feat: standalone bundler

2018-08-16 20:52:39 -05:00
parent c70d93b72f
commit 53ef02f87a
28 changed files with 638 additions and 2538 deletions
@@ -1,11 +0,0 @@
-{
-  "name": "nexe-bundle-fs",
-  "version": "1.0.0",
-  "description": "Set of fs patches for nexe bundles",
-  "main": "patch.ts",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Caleb Boyd",
-  "license": "MIT"
-}
@@ -1,7 +1,7 @@
 import { delimiter, dirname, normalize, join } from 'path'
 import { Buffer } from 'buffer'
 import { createReadStream } from 'fs'
-import { Readable } from 'stream'
+import { Readable, Stream } from 'stream'
 import { spawn } from 'child_process'
 import { Logger, LogStep } from './logger'
 import { readFileAsync, writeFileAsync, pathExistsAsync, dequote, isWindows, bound } from './util'
@@ -10,6 +10,8 @@ import { NexeTarget } from './target'
 import download = require('download')
 import { getLatestGitRelease } from './releases'
 import { IncomingMessage } from 'http'
+import combineStreams = require('multistream')
+import { Bundle, toStream } from './fs/bundle'

 const isBsd = Boolean(~process.platform.indexOf('bsd'))
 const make = isWindows ? 'vcbuild.bat' : isBsd ? 'gmake' : 'make'
@@ -23,29 +25,23 @@ export interface NexeFile {

 export { NexeOptions }

-interface NexeHeader {
-  resources: { [key: string]: number[] }
-  version: string
-}
-
 export class NexeCompiler {
  private start = Date.now()
  private env = { ...process.env }
+  private bundle: Bundle
  private compileStep: LogStep | undefined
  public log = new Logger(this.options.loglevel)
  public src: string
  public files: NexeFile[] = []
  public shims: string[] = []
-  public input: string | undefined
+  public startup: string = ''
+  public entrypoint: string | undefined
  public bundledInput?: string
  public targets: NexeTarget[]
  public target: NexeTarget
-  public resources: { bundle: Buffer; index: { [key: string]: number[] } } = {
-    index: {},
-    bundle: Buffer.from('')
-  }
  public output = this.options.output
  private nodeSrcBinPath: string
+
  constructor(public options: NexeOptions) {
    const { python } = (this.options = options)
    this.targets = options.targets as NexeTarget[]
@@ -55,7 +51,7 @@ export class NexeCompiler {
      ? join(this.src, 'Release', 'node.exe')
      : join(this.src, 'out', 'Release', 'node')
    this.log.step('nexe ' + version, 'info')
-
+    this.bundle = new Bundle(options)
    if (isWindows) {
      const originalPath = process.env.PATH
      delete process.env.PATH
@@ -71,10 +67,16 @@ export class NexeCompiler {
  }

  @bound
-  addResource(file: string, contents: Buffer) {
-    const { resources } = this
-    resources.index[file] = [resources.bundle.byteLength, contents.byteLength]
-    resources.bundle = Buffer.concat([resources.bundle, contents])
+  addResource(file: string, content?: Buffer | string) {
+    return this.bundle.addResource(file, content)
+  }
+
+  get binaryConfiguration() {
+    return { resources: this.bundle.index }
+  }
+
+  get resourceSize() {
+    return this.bundle.blobSize
  }

  @bound
@@ -234,27 +236,25 @@ export class NexeCompiler {
  }

  code() {
-    return [this.shims.join(''), this.input].join(';')
+    return [this.shims.join(''), this.startup].join(';')
  }

  private _assembleDeliverable(binary: NodeJS.ReadableStream) {
    if (!this.options.mangle) {
      return binary
    }
-    const artifact = new Readable({ read() {} })
-    binary.on('data', (chunk: Buffer) => {
-      artifact.push(chunk)
-    })
-    binary.on('close', () => {
-      const content = this.code()
-      artifact.push(content)
-      artifact.push(this.resources.bundle)
-      const lengths = Buffer.from(Array(16))
-      lengths.writeDoubleLE(Buffer.byteLength(content), 0)
-      lengths.writeDoubleLE(this.resources.bundle.byteLength, 8)
-      artifact.push(Buffer.concat([Buffer.from('<nexe~~sentinel>'), lengths]))
-      artifact.push(null)
-    })
-    return artifact
+
+    const startup = this.code(),
+      codeSize = Buffer.byteLength(startup)
+
+    const lengths = Buffer.from(Array(16))
+    lengths.writeDoubleLE(codeSize, 0)
+    lengths.writeDoubleLE(this.bundle.blobSize, 8)
+    return combineStreams([
+      binary,
+      toStream(startup),
+      this.bundle.toStream(),
+      toStream(Buffer.concat([Buffer.from('<nexe~~sentinel>'), lengths]))
+    ])
  }
 }
@@ -0,0 +1,53 @@
+# nexe-fs
+
+This module contains a set of patches used to create (and use) the nexe virtual filesystem
+
+---
+
+### Getting Started:
+
+To patch the module loader, monkey patches must be installed on a few methods at the time that node boots up. This requires a special build of node. Nexe provides these builds or the patch can be applied from this module manually in your own build setup.
+
+```javascript
+const bootstrapPatch = fs.readFileSync(require.resolve('nexe-fs/bootstrap'), 'utf-8')
+// insert boostrapPatch wherever node starts up. For reference, Nexe does this
+// in third-party-main.ts
+```
+
+To create a virtual file system use the `Bundle` Object
+
+```javascript
+const { Bundle } = require('nexe-fs')
+
+const bundle = new Bundle({ cwd })
+await bundle.addResource(absoluteFileName)
+
+bundle.toStream().pipe(fs.createWriteStream('./myFsBlob'))
+
+// You'll also want to save the bundle index to use at runtime
+// It can be saved by accessing bundle.index (object/hash) or serializing the bundle with JSON.stringify(bundle)
+```
+
+In the entrypoint of the custom node build, the fs patch needs to be applied
+```javascript
+const fakeFs = fs.readFileSync(require.resolve('nexe-fs/patch'), 'utf-8')
+//apply this code to your entrypoint
+```
+
+The patch exposes two methods, `shimFs` and `restoreFs` to be used as follows:
+
+```javascript
+shimFs({
+  blobPath: 'location/of/myFsBlob',
+  resources: bundle.index,
+  layout: {
+    stat: blobStats //fs.Stats object about the blob at blobPath
+    resourceStart: 0 // the offset within the blob that is referenced by the bundle index
+  }
+})
+```
+
+At this point executing the original entrypoint shoudl result in usage of the virtual filesystem.
+
+
+
@@ -0,0 +1,70 @@
+import { stat as getStat, Stats, createReadStream } from 'fs'
+import { relative } from 'path'
+import { Readable } from 'stream'
+import combineStreams = require('multistream')
+
+const stat = (file: string): Promise<Stats> => {
+  return new Promise((resolve, reject) => {
+    getStat(file, (err, stats) => (err ? reject(err) : resolve(stats)))
+  })
+}
+
+function makeRelative(cwd: string, path: string) {
+  return './' + relative(cwd, path)
+}
+
+export function toStream(content: Buffer | string) {
+  const readable = new Readable({ read() {} })
+  readable.push(content)
+  readable.push(null)
+  return readable
+}
+
+export type File = { absPath: string; contents: string; deps: FileMap }
+export type FileMap = { [absPath: string]: File | null }
+
+export interface BundleOptions {
+  entries: string[]
+  cwd: string
+  expand: boolean
+  loadContent: boolean
+  files: FileMap
+}
+
+export class Bundle {
+  constructor({ cwd }: { cwd: string } = { cwd: process.cwd() }) {
+    this.cwd = cwd
+  }
+  cwd: string
+  blobSize: number = 0
+  index: { [relativeFilePath: string]: [number, number] } = {}
+  streams: (Readable | (() => Readable))[] = []
+
+  async addResource(absoluteFileName: string, content?: Buffer | string) {
+    let length = 0
+    if (content) {
+      length = Buffer.byteLength(content)
+    } else {
+      const stats = await stat(absoluteFileName)
+      length = stats.size
+    }
+
+    const start = this.blobSize
+
+    this.blobSize += length
+    this.index[makeRelative(this.cwd, absoluteFileName)] = [start, length]
+    this.streams.push(() => (content ? toStream(content) : createReadStream(absoluteFileName)))
+  }
+
+  concat() {
+    throw new Error('Not Implemented')
+  }
+
+  toStream() {
+    return combineStreams(this.streams)
+  }
+
+  toJSON() {
+    return this.index
+  }
+}
@@ -0,0 +1,15 @@
+{
+  "name": "nexe-fs",
+  "version": "1.0.1",
+  "description": "Virtual File System used by nexe",
+  "main": "bundle.js",
+  "files": [
+    "*.d.ts",
+    "*.js"
+  ],
+  "author": "Caleb Boyd",
+  "dependencies": {
+    "multistream": "^2.1.1"
+  },
+  "license": "MIT"
+}
@@ -13,16 +13,24 @@ export interface NexeBinary {
 }

 let originalFsMethods: any = null
+let nexeBinary: NexeBinary | null = null

-function restoreFs(fs: any) {
-  if (!originalFsMethods) {
-    return
+function restoreFs(fs: any = require('fs')): NexeBinary | false {
+  if (!nexeBinary) {
+    return false
  }
+  const source = nexeBinary
  Object.assign(fs, originalFsMethods)
+  nexeBinary = originalFsMethods = null
+  return source
 }

 function shimFs(binary: NexeBinary, fs: any = require('fs')) {
+  if (originalFsMethods !== null) {
+    return
+  }
  originalFsMethods = Object.assign({}, fs)
+  nexeBinary = binary
  const { blobPath, resources: manifest } = binary
  const { resourceStart, stat } = binary.layout
  const directories: { [key: string]: { [key: string]: boolean } } = {},
@@ -325,6 +333,8 @@ function shimFs(binary: NexeBinary, fs: any = require('fs')) {
      process.nextTick(() => cb(exists))
    }
  }
+  Object.assign(fs, nfs)
+  return true
 }

-export { shimFs as applyFsPatch, restoreFs }
+export { shimFs, restoreFs }
@@ -21,9 +21,9 @@ async function compile(
  const nexe = compose(
    clean,
    resource,
+    cli,
    bundle,
    shim,
-    cli,
    options.build ? [download, artifacts, ...patches, ...(options.patches as NexePatch[])] : [],
    options.plugins as NexePatch[]
  )
@@ -63,7 +63,7 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
  })

  const fileLines = file.contents.split('\n')
-  fileLines.splice(location.start.line, 0, '{{replace:lib/patches/bootstrap.js}}' + '\n')
+  fileLines.splice(location.start.line, 0, '{{replace:lib/fs/bootstrap.js}}' + '\n')
  file.contents = fileLines.join('\n')

  await compiler.setFileContentsAsync(
@@ -1,49 +1,22 @@
 import { NexeCompiler } from '../compiler'
-import { readFileAsync, writeFileAsync } from '../util'
-import { resolve, relative } from 'path'
+import { resolve } from 'path'
+import { each } from '@calebboyd/semaphore'
 import resolveFiles from 'resolve-dependencies'

-function makeRelative(cwd: string, path: string) {
-  return './' + relative(cwd, path)
-}
-
-let producer = async function(compiler: NexeCompiler): Promise<string> {
-  const { cwd, input } = compiler.options
-  const { files, entries } = await resolveFiles(input, { cwd, expand: true })
-  const mainFileContents = (entries[input].contents as string) || ''
-
-  Object.keys(files).forEach(filename => {
-    const file = files[filename]!
-    if (file && file.contents) {
-      compiler.addResource(makeRelative(cwd, filename), Buffer.from(file.contents))
-    }
-  })
-  return Promise.resolve(mainFileContents)
-}
-
 export default async function bundle(compiler: NexeCompiler, next: any) {
-  const { bundle, cwd, empty, input } = compiler.options
+  const { bundle, cwd, input } = compiler.options
  if (!bundle) {
-    compiler.input = await readFileAsync(resolve(cwd, input), 'utf-8')
+    await compiler.addResource(resolve(cwd, input))
    return next()
  }

  if (!input) {
-    compiler.input = ''
    return next()
  }

-  if (typeof bundle === 'string') {
-    producer = require(resolve(cwd, bundle)).createBundle
-  }
-
-  compiler.input = await producer(compiler)
-  const debugBundle = compiler.options.debugBundle
-
-  if (debugBundle) {
-    let bundleDebugFile = typeof debugBundle === 'string' ? debugBundle : 'nexe-debug.bundle.js'
-    await writeFileAsync(resolve(cwd, bundleDebugFile), compiler.input)
-  }
-
+  const { files } = await resolveFiles(input, { cwd, expand: true, loadContent: false })
+  await each(Object.keys(files), (filename: string) => compiler.addResource(filename), {
+    concurrency: 10
+  })
  return next()
 }
@@ -1,10 +1,11 @@
-import { normalize, relative } from 'path'
+import { normalize, relative, resolve } from 'path'
 import { createWriteStream, chmodSync, statSync } from 'fs'
 import { dequote } from '../util'
+import { Readable } from 'stream'
 import { NexeCompiler } from '../compiler'
 import { NexeTarget } from '../target'

-function getStdIn(stdin: NodeJS.ReadStream): Promise<string> {
+function getStdIn(stdin: Readable): Promise<string> {
  return new Promise(resolve => {
    let out = ''
    stdin
@@ -38,9 +39,13 @@ export default async function cli(compiler: NexeCompiler, next: () => Promise<vo
  let stdInUsed = false
  if (!process.stdin.isTTY && compiler.options.enableStdIn) {
    stdInUsed = true
-    compiler.input = dequote(await getStdIn(process.stdin))
+    compiler.entrypoint = './__nexe_stdin.js'
+    const code = dequote(await getStdIn(process.stdin))
+    await compiler.addResource(resolve(compiler.options.cwd, compiler.entrypoint), code)
+  } else {
+    compiler.entrypoint = './' + relative(compiler.options.cwd, compiler.options.input)
  }
-
+  compiler.startup = ';require("module").runMain();'
  await next()

  const target = compiler.options.targets.shift() as NexeTarget
@@ -1,11 +1,9 @@
-import { readFileAsync, isDirectoryAsync, each } from '../util'
+import { isDirectoryAsync, each } from '../util'
 import * as globs from 'globby'
 import { NexeCompiler } from '../compiler'

 export default async function resource(compiler: NexeCompiler, next: () => Promise<any>) {
-  const resources = compiler.resources
  const { cwd } = compiler.options
-
  if (!compiler.options.resources.length) {
    return next()
  }
@@ -17,9 +15,8 @@ export default async function resource(compiler: NexeCompiler, next: () => Promi
    }
    count++
    step.log(`Including file: ${file}`)
-    const contents = await readFileAsync(file)
-    compiler.addResource(file, contents)
+    await compiler.addResource(file)
  })
-  step.log(`Included ${count} file(s). ${(resources.bundle.byteLength / 1e6).toFixed(3)} MB`)
+  step.log(`Included ${count} file(s). ${(compiler.resourceSize / 1e6).toFixed(3)} MB`)
  return next()
 }
@@ -1,13 +1,12 @@
 import { NexeCompiler } from '../compiler'
-import { relative } from 'path'
 import { wrap } from '../util'

 export default function(compiler: NexeCompiler, next: () => Promise<void>) {
  compiler.shims.push(
    wrap(
-      `process.__nexe = ${JSON.stringify({ resources: compiler.resources.index })};\n` +
-        '{{replace:lib/bundle/fs/patch.js}}' +
-        '\nshimFs(process.__nexe, require("fs"))'
+      `process.__nexe = ${JSON.stringify(compiler.binaryConfiguration)};\n` +
+        '{{replace:lib/fs/patch.js}}' +
+        '\nshimFs(process.__nexe)'
    )
  )
  compiler.shims.push(
@@ -20,18 +19,13 @@ export default function(compiler: NexeCompiler, next: () => Promise<void>) {
  `)
  )

-  if (compiler.options.fakeArgv !== false) {
-    const nty = !process.stdin.isTTY
-    const input = nty
-      ? '[stdin]'
-      : JSON.stringify(relative(compiler.options.cwd, compiler.options.input))
-
-    compiler.shims.push(
-      wrap(`
-        var r = require('path').resolve; 
-        process.argv.splice(1,0, ${nty ? `'${input}'` : `r(${input})`});`)
-    )
-  }
+  compiler.shims.push(
+    wrap(`
+      if (!process.send) {
+        process.argv.splice(1,0, require.resolve("${compiler.entrypoint}"))  
+      }
+    `)
+  )

  return next()
 }