Update src/steps/download.ts

Co-authored-by: Jared Allard <jaredallard@users.noreply.github.com>

README.md

@@ -51,7 +51,7 @@ Additional files or resources can be added to the binary by passing `-r "glob/pa
 
 ## Compiling the nexe Executable
 
-By default `nexe` will attempt to download a pre-built executable. These are listed on the [releases page](https://github.com/nexe/nexe/releases/tag/v3.3.3). The exact version you want may be unavailable or you may want to customize what is built. See `nexe --help` for a list of options available when passing the [`--build`](#build-boolean) option. You will also need to ensure your environment is setup to [build node](https://github.com/nodejs/node/blob/master/BUILDING.md). Note: the `python` binary in your path should be an acceptable version of python 2. eg. Systems that have python 2 will need to create a [symlink](https://github.com/nexe/nexe/issues/354#issuecomment-319874486).
+By default `nexe` will attempt to download a pre-built executable. These are listed on the [releases page](https://github.com/nexe/nexe/releases/tag/v3.3.3). The exact version you want may be unavailable or you may want to customize what is built. See `nexe --help` for a list of options available when passing the [`--build`](#build-boolean) option. You will also need to ensure your environment is setup to [build node](https://github.com/nodejs/node/blob/master/BUILDING.md). Note: the `python` binary in your path should be an acceptable version of python 3; you can create a [symlink](https://github.com/nexe/nexe/issues/354#issuecomment-319874486) or use the `--python` parameter (e.g. `nexe --build --python=$(which python3)`).
 
 ### Linux and macOS
 [Prerequisites & details](https://github.com/nodejs/node/blob/master/BUILDING.md#unix-and-macos)
@@ -74,7 +74,7 @@ Install-BoxstarterPackage https://raw.githubusercontent.com/nodejs/node/master/t
 **Set config:**
 ```
 npm config set msvs_version 2019
-npm config set python python2.7
+npm config set python python3.8
 ```
 Where `2019` is the version of Visual Studio you have (if you have it).
 
@@ -154,7 +154,7 @@ compile({
  - #### `asset: string`
     - Provide a pre-built nexe binary asset, this is a file path is resolved relative to cwd.
  - #### `python: string`
-    - On Linux this is the path pointing to your python2 executable
+    - On Linux this is the path pointing to your python3 executable
     - On Windows this is the directory where `python` can be accessed
     - default: `null`
  - #### `flags: string[]`

azure-pipelines.yml

@@ -47,7 +47,7 @@ jobs:
   - script: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 gcc-multilib g++-multilib
     condition: contains(variables['Agent.JobName'], 'linux')
     displayName: Install Linux build dependencies
-  - script: choco install visualcpp-build-tools --version 14.0.25420.1 -fy && npm config set msvs_version 2015
+  - script: npm config set msvs_version 2015
     condition: contains(variables['Agent.JobName'], 'windows_2015')
     displayName: Install Windows build dependencies
   - script: npm install && npm run ci:build
@@ -58,8 +58,7 @@ jobs:
       msbuild_args: /3G
     condition: not(contains(variables['Agent.JobName'], 'windows'))
   - powershell: |
-      iex (new-object net.webclient).downloadstring('https://get.scoop.sh')
+      choco install nasm
-      scoop install nasm
       npm install
       npm run ci:build
     displayName: Build Nexe (Windows)
@@ -67,3 +66,8 @@ jobs:
     env:
       GITHUB_TOKEN: $(PersonalGithubToken)
       NEXE_VERBOSE: $(NEXE_VERBOSE)
+  - script: npm run test:integration
+    displayName: Test built artefact
+    env:
+      NEXE_VERBOSE: $(NEXE_VERBOSE)
+      BUILD: ''

package.json

@@ -2,7 +2,7 @@
   "name": "nexe",
   "description": "Create a single executable out of your Node.js application",
   "license": "MIT",
-  "version": "4.0.0-beta.19",
+  "version": "5.0.0-beta.1",
   "contributors": [
     "Craig Condon <craig.j.condon@gmail.com> (http://crcn.io)",
     "Jared Allard <jaredallard@outlook.com>",
@@ -11,10 +11,14 @@
   "scripts": {
     "ci:build": "ts-node tasks/build",
     "test": "mocha",
+    "test:integration": "node index.js --target 14.15.3 -i test/integration/index.js -o integration-tests --verbose -r test/integration -r node_modules/ && npm run test:integration:run",
+    "test:integration:run": "run-script-os",
+    "test:integration:run:win32": "integration-tests.exe",
+    "test:integration:run:default": "./integration-tests",
     "lint": "tslint \"{src,plugins,tasks}/**/*.ts\" --fix",
     "prepare": "npm run lint && npm run build && npm test",
     "prebuild": "rimraf lib",
-    "build": "tsc --declaration",
+    "build": "tsc --declaration && tsc -p tasks && webpack",
     "postbuild": "ts-node tasks/post-build"
   },
   "repository": {
@@ -41,39 +45,48 @@
   },
   "dependencies": {
     "@calebboyd/semaphore": "^1.3.1",
+    "@yarnpkg/fslib": "^3.0.0-rc.43",
+    "@yarnpkg/libzip": "^3.0.0-rc.43",
     "app-builder": "^7.0.4",
+    "archiver": "^5.3.1",
     "caw": "^2.0.1",
     "chalk": "^2.4.2",
-    "cherow": "1.6.9",
     "download": "^8.0.0",
     "globby": "^11.0.2",
-    "got": "^11.8.2",
+    "got": "^12.6.0",
-    "minimist": "^1.2.5",
+    "meriyah": "^4.3.5",
+    "minimist": "^1.2.8",
     "mkdirp": "^1.0.4",
     "multistream": "^4.1.0",
     "ora": "^3.4.0",
-    "resolve-dependencies": "^6.0.7",
+    "resolve-dependencies": "^6.0.9",
-    "rimraf": "^3.0.2"
+    "rimraf": "^3.0.2",
+    "run-script-os": "^1.1.6",
+    "webpack-config-prefabs": "0.0.5"
   },
   "devDependencies": {
+    "@types/archiver": "^5.3.2",
     "@types/chai": "^4",
-    "@types/download": "^6",
+    "@types/download": "^8",
     "@types/globby": "^9",
+    "@types/lodash": "^4.14.192",
     "@types/minimist": "^1.2.2",
     "@types/mkdirp": "^1.0.2",
-    "@types/mocha": "^9.0.0",
+    "@types/mocha": "^10.0.1",
-    "@types/multistream": "^2.1.1",
+    "@types/multistream": "^4.1.0",
     "@types/ora": "^3.2.0",
     "@types/rimraf": "3.0.2",
-    "@types/semver": "^7.3.8",
+    "@types/semver": "^7.3.13",
-    "chai": "^4.3.4",
+    "chai": "^4.3.7",
     "execa": "^5.1.1",
-    "mocha": "^9.1.2",
+    "lodash": "^4.17.21",
-    "prettier": "^2.4.1",
+    "mocha": "^10.2.0",
-    "ts-node": "^10.2.1",
+    "prettier": "^2.8.7",
+    "ts-node": "^10.9.1",
     "tslint": "^6.1.1",
     "tslint-config-prettier": "^1.18.0",
     "tslint-plugin-prettier": "^2.3.0",
-    "typescript": "^4.4.3"
+    "typescript": "^5.0.3",
+    "webpack-cli": "^5.0.1"
   }
 }

src/compiler.ts

@@ -16,9 +16,11 @@ import {
 } from './util'
 import { NexeOptions, version } from './options'
 import { NexeTarget } from './target'
+import { PassThrough, Readable, Stream, Transform } from 'stream'
 import MultiStream = require('multistream')
 import { Bundle, toStream } from './fs/bundle'
 import { File } from 'resolve-dependencies'
+import { FactoryStream, LazyStream } from 'multistream'
 
 const isBsd = Boolean(~process.platform.indexOf('bsd'))
 const make = isWindows ? 'vcbuild.bat' : isBsd ? 'gmake' : 'make'
@@ -287,24 +289,40 @@ export class NexeCompiler {
     return [this.shims.join(''), this.startup].join(';')
   }
 
-  private async _assembleDeliverable(binary: NodeJS.ReadableStream) {
+  private async _assembleDeliverable(binary: Readable) {
     if (!this.options.mangle) {
       return binary
     }
-    const resources = this.bundle.renderIndex()
+    const launchCode = this.code()
-    this.shims.unshift(wrap(`process.__nexe = ${JSON.stringify({ resources })};\n`))
+    const codeSize = Buffer.byteLength(launchCode)
+    const sentinel = Buffer.from('<nexe~~sentinel>')
 
-    const code = this.code(),
+    let vfsSize = 0
-      codeSize = Buffer.byteLength(code),
+    const streams = [
-      lengths = Buffer.from(Array(16))
-
-    lengths.writeDoubleLE(codeSize, 0)
-    lengths.writeDoubleLE(this.bundle.size, 8)
-    return new (MultiStream as any)([
       binary,
-      toStream(code),
+      toStream(launchCode),
-      this.bundle.toStream(),
+      this.bundle.toStream().pipe(
-      toStream(Buffer.concat([Buffer.from('<nexe~~sentinel>'), lengths])),
+        new Transform({
-    ])
+          transform: (chunk, _, cb) => {
+            vfsSize || this.bundle.finalize()
+            chunk && (vfsSize += chunk.length)
+            cb(null, chunk)
+          },
+        })
+      ),
+    ]
+
+    let done = false
+    return new MultiStream((cb) => {
+      if (done) cb(null, null)
+      else if (streams.length) cb(null, streams.shift() as Readable)
+      else {
+        done = true
+        const trailers = Buffer.alloc(16)
+        trailers.writeDoubleLE(codeSize, 0)
+        trailers.writeDoubleLE(vfsSize, 8)
+        cb(null, toStream(Buffer.concat([sentinel, trailers])))
+      }
+    })
   }
 }

src/fs/SnapshotZipFS.ts

@@ -0,0 +1,395 @@
+import { Libzip } from '@yarnpkg/libzip'
+import {
+  FakeFS,
+  PortablePath,
+  RmdirOptions,
+  OpendirOptions,
+  BasePortableFakeFS,
+  Dirent,
+  Dir,
+  MkdirOptions,
+  FSPath,
+  ppath,
+  npath,
+  Filename,
+  CustomDir,
+} from '@yarnpkg/fslib'
+import { ZipFS, ZipOpenFS } from '@yarnpkg/libzip'
+import { resolve, toNamespacedPath } from 'path'
+import { constants } from 'fs'
+
+export type SnapshotZipFSOptions = {
+  baseFs: FakeFS<PortablePath>
+  libzip: Libzip | (() => Libzip)
+  zipFs: ZipFS
+  root: string
+}
+
+const uniqBy = (arr: Array<string | Dirent>, pick: (...arg: any[]) => any) => {
+  const seen = new Set()
+  return arr.filter((x) => {
+    const key = pick(x)
+    if (seen.has(key)) return false
+    seen.add(key)
+    return true
+  })
+}
+function uniqReaddir(arr: Array<string | Dirent>) {
+  return uniqBy(arr, (s: string | Dirent) => (typeof s === 'string' ? s : s.name))
+}
+
+export class SnapshotZipFS extends BasePortableFakeFS {
+  zipFs: ZipFS
+  baseFs: FakeFS<PortablePath>
+  root: string
+  magic: number
+  constructor(opts: SnapshotZipFSOptions) {
+    super()
+    this.zipFs = opts.zipFs
+    this.baseFs = opts.baseFs
+    this.root = opts.root
+    this.magic = 0x2a << 24
+  }
+  private readonly fdMap: Map<number, [ZipFS, number]> = new Map()
+  private nextFd = 3
+
+  async makeCallPromise<T>(
+    p: FSPath<PortablePath>,
+    discard: () => Promise<T>,
+    accept: (zipFS: ZipFS, zipInfo: { subPath: PortablePath }) => Promise<T>,
+    { requireSubpath = true }: { requireSubpath?: boolean } = {}
+  ): Promise<T> {
+    if (typeof p !== 'string') return await discard()
+
+    const normalizedP = this.resolve(p)
+
+    const zipInfo = this.findZip(normalizedP)
+    if (!zipInfo) return await discard()
+
+    if (requireSubpath && zipInfo.subPath === '/') return await discard()
+
+    return await accept(this.zipFs, zipInfo)
+  }
+
+  makeCallSync<T>(
+    p: FSPath<PortablePath>,
+    discard: () => T,
+    accept: (zipFS: ZipFS, zipInfo: { subPath: PortablePath; archivePath: string }) => T,
+    { requireSubpath = true }: { requireSubpath?: boolean } = {}
+  ): T {
+    if (typeof p !== 'string') return discard()
+
+    const normalizedP = this.resolve(p)
+
+    const zipInfo = this.findZip(normalizedP)
+    if (!zipInfo) return discard()
+
+    if (requireSubpath && zipInfo.subPath === '/') return discard()
+
+    return accept(this.zipFs, { archivePath: '', ...zipInfo })
+  }
+
+  async realpathPromise(p: PortablePath) {
+    return await this.realpathSync(p)
+  }
+
+  realpathSync(p: PortablePath) {
+    return this.makeCallSync(
+      p,
+      () => this.baseFs.realpathSync(p),
+      (zipFs, { subPath }) => {
+        if (zipFs.lstatSync(subPath).isSymbolicLink()) {
+          return zipFs.realpathSync(subPath)
+        } else {
+          // return the original path in case it wasn't under /snapshot, e.g. if it was for a node module - otherwise the node module parent path is the wrong one (and other things resolve relative to that)
+          return p
+        }
+      }
+    )
+  }
+
+  findZip(p: PortablePath) {
+    p = this.resolve(p)
+    const snapshotPP = npath.toPortablePath('/snapshot')
+    const pathsToTry = Array.from(
+      new Set([
+        p,
+        resolve('/snapshot', p),
+        ppath.resolve(
+          snapshotPP,
+          npath.toPortablePath(
+            npath.relative(npath.fromPortablePath(this.root), npath.fromPortablePath(p))
+          )
+        ),
+        ppath.resolve(
+          snapshotPP,
+          npath.toPortablePath(
+            npath.relative(
+              toNamespacedPath(npath.fromPortablePath(this.root)),
+              toNamespacedPath(npath.fromPortablePath(p))
+            )
+          )
+        ),
+      ])
+    )
+    for (const path of pathsToTry) {
+      const portablePath = npath.toPortablePath(path)
+      if (this.zipFs.existsSync(portablePath)) {
+        return {
+          subPath: portablePath,
+        }
+      }
+    }
+  }
+
+  async copyFilePromise(sourceP: PortablePath, destP: PortablePath, flags: number = 0) {
+    const fallback = async (
+      sourceFs: FakeFS<PortablePath>,
+      sourceP: PortablePath,
+      destFs: FakeFS<PortablePath>,
+      destP: PortablePath
+    ) => {
+      if ((flags & constants.COPYFILE_FICLONE_FORCE) !== 0)
+        throw Object.assign(
+          new Error(`EXDEV: cross-device clone not permitted, copyfile '${sourceP}' -> ${destP}'`),
+          { code: `EXDEV` }
+        )
+      if (flags & constants.COPYFILE_EXCL && (await this.existsPromise(sourceP)))
+        throw Object.assign(
+          new Error(`EEXIST: file already exists, copyfile '${sourceP}' -> '${destP}'`),
+          { code: `EEXIST` }
+        )
+
+      let content
+      try {
+        content = await sourceFs.readFilePromise(sourceP)
+      } catch (error) {
+        throw Object.assign(
+          new Error(`EINVAL: invalid argument, copyfile '${sourceP}' -> '${destP}'`),
+          { code: `EINVAL` }
+        )
+      }
+
+      await destFs.writeFilePromise(destP, content)
+    }
+
+    return await this.makeCallPromise(
+      sourceP,
+      async () => {
+        return await this.baseFs.copyFilePromise(sourceP, destP, flags)
+      },
+      async (zipFsS, { subPath: subPathS }) => {
+        return await fallback(zipFsS, subPathS, this.baseFs, destP)
+      }
+    )
+  }
+
+  copyFileSync(sourceP: PortablePath, destP: PortablePath, flags: number = 0) {
+    const fallback = (
+      sourceFs: FakeFS<PortablePath>,
+      sourceP: PortablePath,
+      destFs: FakeFS<PortablePath>,
+      destP: PortablePath
+    ) => {
+      if ((flags & constants.COPYFILE_FICLONE_FORCE) !== 0)
+        throw Object.assign(
+          new Error(`EXDEV: cross-device clone not permitted, copyfile '${sourceP}' -> ${destP}'`),
+          { code: `EXDEV` }
+        )
+      if (flags & constants.COPYFILE_EXCL && this.existsSync(sourceP))
+        throw Object.assign(
+          new Error(`EEXIST: file already exists, copyfile '${sourceP}' -> '${destP}'`),
+          { code: `EEXIST` }
+        )
+
+      let content
+      try {
+        content = sourceFs.readFileSync(sourceP)
+      } catch (error) {
+        throw Object.assign(
+          new Error(`EINVAL: invalid argument, copyfile '${sourceP}' -> '${destP}'`),
+          { code: `EINVAL` }
+        )
+      }
+
+      destFs.writeFileSync(destP, content)
+    }
+
+    return this.makeCallSync(
+      sourceP,
+      () => {
+        return this.baseFs.copyFileSync(sourceP, destP, flags)
+      },
+      (zipFsS, { subPath: subPathS }) => {
+        return fallback(zipFsS, subPathS, this.baseFs, destP)
+      }
+    )
+  }
+  async readdirPromise(p: PortablePath): Promise<Array<Filename>>
+  async readdirPromise(
+    p: PortablePath,
+    opts: { withFileTypes: false } | null
+  ): Promise<Array<Filename>>
+  async readdirPromise(p: PortablePath, opts: { withFileTypes: true }): Promise<Array<Dirent>>
+  async readdirPromise(
+    p: PortablePath,
+    opts: { withFileTypes: boolean }
+  ): Promise<Array<Filename> | Array<Dirent>>
+  async readdirPromise(
+    p: PortablePath,
+    opts?: { withFileTypes?: boolean } | null
+  ): Promise<Array<string | Dirent>> {
+    const fallback = async () => {
+      return await this.baseFs.readdirPromise(p, opts as any)
+    }
+    return await this.makeCallPromise(
+      p,
+      fallback,
+      async (zipFs, { subPath }) => {
+        const fallbackPaths: Array<string | Dirent> = await fallback().catch(() => [])
+        return Promise.resolve(
+          uniqReaddir(fallbackPaths.concat(await zipFs.readdirPromise(subPath, opts as any)))
+        )
+      },
+      {
+        requireSubpath: false,
+      }
+    )
+  }
+
+  readdirSync(p: PortablePath): Array<Filename>
+  readdirSync(p: PortablePath, opts: { withFileTypes: false } | null): Array<Filename>
+  readdirSync(p: PortablePath, opts: { withFileTypes: true }): Array<Dirent>
+  readdirSync(p: PortablePath, opts: { withFileTypes: boolean }): Array<Filename> | Array<Dirent>
+  readdirSync(p: PortablePath, opts?: { withFileTypes?: boolean } | null): Array<string | Dirent> {
+    const fallback = () => {
+      return this.baseFs.readdirSync(p, opts as any)
+    }
+    return this.makeCallSync(
+      p,
+      fallback,
+      (zipFs, { subPath }) => {
+        let fallbackPaths: Array<string | Dirent> = []
+        try {
+          fallbackPaths = fallback()
+        } catch (e) {}
+        return fallbackPaths.concat(uniqReaddir(zipFs.readdirSync(subPath, opts as any)))
+      },
+      {
+        requireSubpath: false,
+      }
+    )
+  }
+
+  async mkdirPromise(p: PortablePath, opts?: MkdirOptions) {
+    return await this.baseFs.mkdirPromise(p, opts)
+  }
+
+  mkdirSync(p: PortablePath, opts?: MkdirOptions) {
+    return this.baseFs.mkdirSync(p, opts)
+  }
+
+  async rmdirPromise(p: PortablePath, opts?: RmdirOptions) {
+    return await this.baseFs.rmdirPromise(p, opts)
+  }
+
+  rmdirSync(p: PortablePath, opts?: RmdirOptions) {
+    return this.baseFs.rmdirSync(p, opts)
+  }
+
+  async opendirPromise(p: PortablePath, opts?: OpendirOptions) {
+    return this.opendirSync(p, opts)
+  }
+
+  opendirSync(p: PortablePath, opts?: OpendirOptions) {
+    const zipInfo = this.findZip(p)
+    let zipFsDir: Dir<PortablePath> | null = null
+    if (zipInfo) {
+      zipFsDir = this.zipFs.opendirSync(zipInfo.subPath)
+    }
+    let realFsDir: Dir<PortablePath> | null = null
+    try {
+      realFsDir = this.baseFs.opendirSync(p)
+    } catch (e) {
+      if (!zipFsDir) throw e
+    }
+    const seen = new Set()
+    const nextDirent = () => {
+      const entry = realFsDir?.readSync() || zipFsDir?.readSync()
+      if (entry && !seen.has(entry.name)) {
+        seen.add(entry.name)
+        return entry
+      }
+      return null
+    }
+
+    const onClose = () => {
+      zipFsDir?.closeSync()
+      realFsDir?.closeSync()
+    }
+
+    return new CustomDir(p, nextDirent, { onClose })
+  }
+
+  accessPromise = ZipOpenFS.prototype.accessPromise
+  accessSync = ZipOpenFS.prototype.accessSync
+  appendFilePromise = ZipOpenFS.prototype.appendFilePromise
+  appendFileSync = ZipOpenFS.prototype.appendFileSync
+  chmodPromise = ZipOpenFS.prototype.chmodPromise
+  chmodSync = ZipOpenFS.prototype.chmodSync
+  fchmodPromise = ZipOpenFS.prototype.fchmodPromise
+  fchmodSync = ZipOpenFS.prototype.fchmodSync
+  chownPromise = ZipOpenFS.prototype.chownPromise
+  chownSync = ZipOpenFS.prototype.chownSync
+  fchownPromise = ZipOpenFS.prototype.fchownPromise
+  fchownSync = ZipOpenFS.prototype.fchownSync
+  closePromise = ZipOpenFS.prototype.closePromise
+  closeSync = ZipOpenFS.prototype.closeSync
+  createReadStream = ZipOpenFS.prototype.createReadStream
+  createWriteStream = ZipOpenFS.prototype.createWriteStream
+  existsPromise = ZipOpenFS.prototype.existsPromise
+  existsSync = ZipOpenFS.prototype.existsSync
+  fstatPromise = ZipOpenFS.prototype.fstatPromise
+  fstatSync = ZipOpenFS.prototype.fstatSync
+  getExtractHint = ZipOpenFS.prototype.getExtractHint
+  getRealPath = ZipOpenFS.prototype.getRealPath
+  linkPromise = ZipOpenFS.prototype.linkPromise
+  linkSync = ZipOpenFS.prototype.linkSync
+  lstatPromise = ZipOpenFS.prototype.lstatPromise
+  lstatSync = ZipOpenFS.prototype.lstatSync
+  openPromise = ZipOpenFS.prototype.openPromise
+  openSync = ZipOpenFS.prototype.openSync
+  readFilePromise = ZipOpenFS.prototype.readFilePromise
+  readFileSync = ZipOpenFS.prototype.readFileSync
+  readlinkPromise = ZipOpenFS.prototype.readlinkPromise
+  readlinkSync = ZipOpenFS.prototype.readlinkSync
+  readPromise = ZipOpenFS.prototype.readPromise
+  readSync = ZipOpenFS.prototype.readSync
+  renamePromise = ZipOpenFS.prototype.renamePromise
+  renameSync = ZipOpenFS.prototype.renameSync
+  resolve = ZipOpenFS.prototype.resolve
+  statPromise = ZipOpenFS.prototype.statPromise
+  statSync = ZipOpenFS.prototype.statSync
+  symlinkPromise = ZipOpenFS.prototype.symlinkPromise
+  symlinkSync = ZipOpenFS.prototype.symlinkSync
+  truncatePromise = ZipOpenFS.prototype.truncatePromise
+  truncateSync = ZipOpenFS.prototype.truncateSync
+  ftruncatePromise = ZipOpenFS.prototype.ftruncatePromise
+  ftruncateSync = ZipOpenFS.prototype.ftruncateSync
+  unlinkPromise = ZipOpenFS.prototype.unlinkPromise
+  unlinkSync = ZipOpenFS.prototype.unlinkSync
+  unwatchFile = ZipOpenFS.prototype.unwatchFile
+  utimesPromise = ZipOpenFS.prototype.utimesPromise
+  utimesSync = ZipOpenFS.prototype.utimesSync
+  lutimesPromise = ZipOpenFS.prototype.utimesPromise
+  lutimesSync = ZipOpenFS.prototype.utimesSync
+  watch = ZipOpenFS.prototype.watch
+  watchFile = ZipOpenFS.prototype.watchFile
+  writeFilePromise = ZipOpenFS.prototype.writeFilePromise
+  writeFileSync = ZipOpenFS.prototype.writeFileSync
+  writePromise = ZipOpenFS.prototype.writePromise
+  writeSync = ZipOpenFS.prototype.writeSync
+
+  // @ts-ignore
+  remapFd = ZipOpenFS.prototype.remapFd
+}

src/fs/bundle.ts

@@ -1,24 +1,10 @@
-import * as fs from 'fs'
-import { promisify } from 'util'
 import { relative } from 'path'
 import { Readable } from 'stream'
-import { argv } from '../options'
 import { File } from 'resolve-dependencies'
-import MultiStream = require('multistream')
+import archiver from 'archiver'
 
-const lstat = promisify(fs.lstat),
+function makeRelativeToZip(cwd: string, path: string) {
-  realpath = promisify(fs.realpath),
+  return '/snapshot/' + relative(cwd, path)
-  createReadStream = fs.createReadStream,
-  stat = promisify(fs.stat)
-
-export type MultiStreams = (Readable | (() => Readable))[]
-
-function makeRelative(cwd: string, path: string) {
-  return './' + relative(cwd, path)
-}
-
-function sortEntry(a: [string, File], b: [string, File]) {
-  return a[0] > b[0] ? 1 : -1
 }
 
 export function toStream(content: Buffer | string) {
@@ -30,130 +16,36 @@ export function toStream(content: Buffer | string) {
   })
   return readable
 }
-
-async function createFile(absoluteFileName: string) {
-  const stats = await lstat(absoluteFileName),
-    file: File = {
-      size: stats.size,
-      moduleType: 'commonjs',
-      contents: '',
-      absPath: absoluteFileName,
-      deps: {},
-    }
-  if (stats.isSymbolicLink()) {
-    file.size = stats.size
-    const [realPath, realStat] = await Promise.all([
-      realpath(absoluteFileName),
-      stat(absoluteFileName),
-    ])
-    file.realPath = realPath
-    file.realSize = realStat.size
-  }
-  return file
-}
-
 export class Bundle {
-  size = 0
   cwd: string
-  rendered = false
+  files = new Set<string>()
-  private offset = 0
+  zip: any
-  private index: { [key: string]: [number, number] } = {}
-  private files: { [key: string]: File } = {}
-  streams: MultiStreams = []
   constructor({ cwd }: { cwd: string } = { cwd: process.cwd() }) {
     this.cwd = cwd
+    this.zip = archiver('zip')
   }
 
   get list() {
-    return Object.keys(this.files)
+    return Array.from(this.files)
   }
 
-  public async addResource(
+  public addResource(absoluteFileName: string, content?: File | Buffer | string) {
-    absoluteFileName: string,
+    const destPath = makeRelativeToZip(this.cwd, absoluteFileName)
-    content?: File | Buffer | string
+    if (!this.files.has(destPath)) {
-  ): Promise<number> {
+      if (content == null) {
-    if (this.files[absoluteFileName]) {
+        this.zip.file(absoluteFileName, { name: destPath })
-      return this.size
-    }
-    if (typeof content === 'string' || Buffer.isBuffer(content)) {
-      this.files[absoluteFileName] = {
-        size: Buffer.byteLength(content),
-        moduleType: 'commonjs',
-        contents: content as any, //todo type is wrong here... should allow buffer
-        deps: {},
-        absPath: absoluteFileName,
-      }
-    } else if (content) {
-      this.files[content.absPath] = content
-    } else {
-      this.files[absoluteFileName] = {
-        absPath: absoluteFileName,
-        moduleType: 'commonjs',
-        contents: '',
-        deps: {},
-        size: 0,
-      }
-      this.files[absoluteFileName] = await createFile(absoluteFileName)
-    }
-    return (this.size += this.files[absoluteFileName].size)
-  }
-
-  /**
-   * De-dupe files by absolute path, partition by symlink/real
-   * Iterate over real, add entries
-   * Iterate over symlinks, add symlinks
-   */
-  renderIndex() {
-    if (this.rendered) {
-      throw new Error('Bundle index already rendered')
-    }
-    const files = Object.entries(this.files),
-      realFiles: [string, File][] = [],
-      symLinks: [string, File][] = []
-    for (const entry of files) {
-      if (entry[1].realPath) {
-        symLinks.push(entry)
       } else {
-        realFiles.push(entry)
+        this.zip.append(content, { name: destPath })
       }
-    }
+      this.files.add(destPath)
-    realFiles.sort(sortEntry)
-    symLinks.sort(sortEntry)
-
-    for (const [absPath, file] of realFiles) {
-      this.addEntry(absPath, file)
-    }
-    for (const [absPath, file] of symLinks) {
-      this.addEntry(file.realPath as string, file)
-      this.addEntry(absPath, file, file.realPath)
-    }
-    this.rendered = true
-    return this.index
-  }
-
-  /**
-   * Add a stream if needed and an entry with the required offset and size
-   * Ensure the calling order of this method is idempotent (eg, while iterating a sorted set)
-   * @param entryPath
-   * @param file
-   * @param useEntry
-   */
-  addEntry(entryPath: string, file: File, useEntry?: string) {
-    const existingName = useEntry && makeRelative(this.cwd, useEntry),
-      name = makeRelative(this.cwd, entryPath),
-      size = file.realSize ?? file.size,
-      existingEntry = this.index[existingName ?? name]
-
-    this.index[name] = existingEntry || [this.offset, size]
-    if (!existingEntry) {
-      this.streams.push(() =>
-        file.contents ? toStream(file.contents) : createReadStream(file.absPath)
-      )
-      this.offset += size
     }
   }
 
-  toStream() {
+  public finalize() {
-    return new (MultiStream as any)(this.streams)
+    return this.zip.finalize()
+  }
+
+  public toStream(): Readable {
+    return this.zip
   }
 }

src/fs/patch.ts

@@ -1,432 +1,108 @@
-import { Stats } from 'fs'
+import { ZipFS, getLibzipSync } from '@yarnpkg/libzip'
-import { getLatestGitRelease } from '../releases'
+import { patchFs, npath, PosixFS, NodeFS } from '@yarnpkg/fslib'
+import { SnapshotZipFS } from './SnapshotZipFS'
+import * as assert from 'assert'
+import * as constants from 'constants'
+import { dirname } from 'path'
 
-export interface NexeBinary {
+export interface NexeHeader {
   blobPath: string
-  resources: { [key: string]: number[] }
   layout: {
-    stat: Stats
     resourceStart: number
-    contentSize?: number
+    resourceSize: number
-    contentStart?: number
+    contentSize: number
-    resourceSize?: number
+    contentStart: number
   }
 }
 
 let originalFsMethods: any = null
 let lazyRestoreFs = () => {}
+const patches = (process as any).nexe.patches || {}
+const originalPatches = { ...patches }
+delete (process as any).nexe
 
-// optional Win32 file namespace prefix followed by drive letter and colon
+function shimFs(binary: NexeHeader, fs: typeof import('fs') = require('fs')) {
-const windowsFullPathRegex = /^(\\{2}\?\\)?([a-zA-Z]):/
-
-const upcaseDriveLetter = (s: string): string =>
-  s.replace(windowsFullPathRegex, (_match, ns, drive) => `${ns || ''}${drive.toUpperCase()}:`)
-
-function shimFs(binary: NexeBinary, fs: any = require('fs')) {
   if (originalFsMethods !== null) {
     return
   }
+
   originalFsMethods = Object.assign({}, fs)
-  const { blobPath, resources: manifest } = binary,
+
-    { resourceStart, stat } = binary.layout,
+  const realFs: typeof fs = { ...fs }
-    directories: { [key: string]: { [key: string]: boolean } } = {},
+  const nodeFs = new NodeFS(realFs)
-    notAFile = '!@#$%^&*',
+
-    isWin = process.platform.startsWith('win'),
+  const blob = Buffer.allocUnsafe(binary.layout.resourceSize)
-    isString = (x: any): x is string => typeof x === 'string' || x instanceof String,
+  const blobFd = realFs.openSync(binary.blobPath, 'r')
-    noop = () => {},
+  const bytesRead = realFs.readSync(
-    path = require('path'),
+    blobFd,
-    winPath: (key: string) => string = isWin ? upcaseDriveLetter : (s) => s,
+    blob,
-    baseDir = winPath(path.dirname(process.execPath))
+    0,
+    binary.layout.resourceSize,
+    binary.layout.resourceStart
+  )
+  assert.equal(bytesRead, binary.layout.resourceSize)
+
+  const zipFs = new ZipFS(blob, { readOnly: true })
+  const snapshotZipFS = new SnapshotZipFS({
+    libzip: getLibzipSync(),
+    zipFs,
+    baseFs: nodeFs,
+    root: dirname(process.argv[0]),
+  })
+  const posixSnapshotZipFs = new PosixFS(snapshotZipFS)
+  patchFs(fs, posixSnapshotZipFs)
 
   let log = (_: string) => true
-  let loggedManifest = false
   if ((process.env.DEBUG || '').toLowerCase().includes('nexe:require')) {
+    process.stderr.write(
+      // @ts-ignore
+      `[nexe] - FILES ${JSON.stringify(Array.from(zipFs.entries.keys()), null, 4)}\n`
+    )
+    process.stderr.write(
+      // @ts-ignore
+      `[nexe] - DIRECTORIES ${JSON.stringify(Array.from(zipFs.listings.keys()), null, 4)}\n`
+    )
     log = (text: string) => {
-      setupManifest()
+      return process.stderr.write(`[nexe] - ${text}\n`)
-      if (!loggedManifest) {
-        process.stderr.write('[nexe] - MANIFEST' + JSON.stringify(manifest, null, 4) + '\n')
-        process.stderr.write('[nexe] - DIRECTORIES' + JSON.stringify(directories, null, 4) + '\n')
-        loggedManifest = true
-      }
-      return process.stderr.write('[nexe] - ' + text + '\n')
     }
   }
-
+  function internalModuleReadFile(this: any, original: any, ...args: any[]) {
-  const getKey = function getKey(filepath: string | Buffer | null): string {
+    log(`internalModuleReadFile ${args[0]}`)
-    if (Buffer.isBuffer(filepath)) {
+    try {
-      filepath = filepath.toString()
+      return posixSnapshotZipFs.readFileSync(args[0], 'utf-8')
-    }
+    } catch (e) {
-    if (!isString(filepath)) {
+      return ''
-      return notAFile
-    }
-    let key = path.resolve(baseDir, filepath)
-
-    return winPath(key)
-  }
-
-  const statTime = function () {
-    return {
-      atime: new Date(stat.atime),
-      mtime: new Date(stat.mtime),
-      ctime: new Date(stat.ctime),
-      birthtime: new Date(stat.birthtime),
     }
   }
-
+  if (patches.internalModuleReadFile) {
-  let BigInt: Function
+    patches.internalModuleReadFile = internalModuleReadFile
-  try {
-    BigInt = eval('BigInt')
-  } catch (ignored) {}
-
-  const minBlocks = Math.max(Math.ceil(stat.blksize / 512), 1)
-  const createStat = function (extensions: any, options: any) {
-    const stat = Object.assign(new fs.Stats(), binary.layout.stat, statTime(), extensions)
-    if ('size' in extensions) {
-      //Assume non adjustable allocation size for file system
-      stat.blocks = Math.ceil(stat.size / stat.blksize) * minBlocks
-    }
-    if (options && options.bigint && typeof BigInt !== 'undefined') {
-      for (const k in stat) {
-        if (Object.prototype.hasOwnProperty.call(stat, k) && typeof stat[k] === 'number') {
-          stat[k] = BigInt(stat[k])
-        }
-      }
-    }
-    return stat
-  }
-
-  const ownStat = function (filepath: any, options: any) {
-    setupManifest()
-    const key = getKey(filepath)
-    if (directories[key]) {
-      let mode = binary.layout.stat.mode
-      mode |= fs.constants.S_IFDIR
-      mode &= ~fs.constants.S_IFREG
-      return createStat({ mode, size: 0 }, options)
-    }
-    if (manifest[key]) {
-      return createStat({ size: manifest[key][1] }, options)
-    }
-  }
-
-  const getStat = function (fn: string) {
-    return function stat(filepath: string | Buffer, options: any, callback: any) {
-      let stat: any
-      if (typeof options === 'function') {
-        callback = options
-        stat = ownStat(filepath, null)
-      } else {
-        stat = ownStat(filepath, options)
-      }
-      if (stat) {
-        process.nextTick(() => {
-          callback(null, stat)
-        })
-      } else {
-        return originalFsMethods[fn].apply(fs, arguments)
-      }
-    }
-  }
-
-  function makeLong(filepath: string) {
-    return (path as any)._makeLong && (path as any)._makeLong(filepath)
-  }
-
-  function fileOpts(options: any) {
-    return !options ? {} : isString(options) ? { encoding: options } : options
-  }
-
-  let setupManifest = () => {
-    Object.keys(manifest).forEach((filepath) => {
-      const entry = manifest[filepath]
-      const absolutePath = getKey(filepath)
-      const longPath = makeLong(absolutePath)
-      const normalizedPath = winPath(path.normalize(filepath))
-
-      if (!manifest[absolutePath]) {
-        manifest[absolutePath] = entry
-      }
-      if (longPath && !manifest[longPath]) {
-        manifest[longPath] = entry
-      }
-      if (!manifest[normalizedPath]) {
-        manifest[normalizedPath] = manifest[filepath]
-      }
-
-      let currentDir = path.dirname(absolutePath)
-      let prevDir = absolutePath
-
-      while (currentDir !== prevDir) {
-        directories[currentDir] = directories[currentDir] || {}
-        directories[currentDir][path.basename(prevDir)] = true
-        const longDir = makeLong(currentDir)
-        if (longDir && !directories[longDir]) {
-          directories[longDir] = directories[currentDir]
-        }
-        prevDir = currentDir
-        currentDir = path.dirname(currentDir)
-      }
-    })
-    ;(manifest[notAFile] as any) = false
-    ;(directories[notAFile] as any) = false
-    setupManifest = noop
-  }
-
-  //naive patches intended to work for most use cases
-  const nfs: any = {
-    existsSync: function existsSync(filepath: string) {
-      setupManifest()
-      const key = getKey(filepath)
-      if (manifest[key] || directories[key]) {
-        return true
-      }
-      return originalFsMethods.existsSync.apply(fs, arguments)
-    },
-    realpath: function realpath(filepath: any, options: any, cb: any): void {
-      setupManifest()
-      const key = getKey(filepath)
-      if (isString(filepath) && (manifest[filepath] || manifest[key])) {
-        return process.nextTick(() => cb(null, filepath))
-      }
-      return originalFsMethods.realpath.call(fs, filepath, options, cb)
-    },
-    realpathSync: function realpathSync(filepath: any, options: any) {
-      setupManifest()
-      const key = getKey(filepath)
-      if (manifest[key]) {
-        return filepath
-      }
-      return originalFsMethods.realpathSync.call(fs, filepath, options)
-    },
-    readdir: function readdir(filepath: string | Buffer, options: any, callback: any) {
-      setupManifest()
-      const dir = directories[getKey(filepath)]
-      if (dir) {
-        if ('function' === typeof options) {
-          callback = options
-          options = { encoding: 'utf8' }
-        }
-        process.nextTick(() => callback(null, Object.keys(dir)))
-      } else {
-        return originalFsMethods.readdir.apply(fs, arguments)
-      }
-    },
-    readdirSync: function readdirSync(filepath: string | Buffer, options: any) {
-      setupManifest()
-      const dir = directories[getKey(filepath)]
-      if (dir) {
-        return Object.keys(dir)
-      }
-      return originalFsMethods.readdirSync.apply(fs, arguments)
-    },
-
-    readFile: function readFile(filepath: any, options: any, callback: any) {
-      setupManifest()
-      const entry = manifest[getKey(filepath)]
-      if (!entry) {
-        return originalFsMethods.readFile.apply(fs, arguments)
-      }
-      const [offset, length] = entry
-      const resourceOffset = resourceStart + offset
-      const encoding = fileOpts(options).encoding
-      callback = typeof options === 'function' ? options : callback
-
-      originalFsMethods.open(blobPath, 'r', function (err: Error, fd: number) {
-        if (err) return callback(err, null)
-        originalFsMethods.read(
-          fd,
-          Buffer.alloc(length),
-          0,
-          length,
-          resourceOffset,
-          function (error: Error, bytesRead: number, result: Buffer) {
-            if (error) {
-              return originalFsMethods.close(fd, function () {
-                callback(error, null)
-              })
-            }
-            originalFsMethods.close(fd, function (err: Error) {
-              if (err) {
-                return callback(err, result)
-              }
-              callback(err, encoding ? result.toString(encoding) : result)
-            })
-          }
-        )
-      })
-    },
-    createReadStream: function createReadStream(filepath: any, options: any) {
-      setupManifest()
-      const entry = manifest[getKey(filepath)]
-      if (!entry) {
-        return originalFsMethods.createReadStream.apply(fs, arguments)
-      }
-      const [offset, length] = entry
-      const resourceOffset = resourceStart + offset
-      const opts = fileOpts(options)
-
-      return originalFsMethods.createReadStream(
-        blobPath,
-        Object.assign({}, opts, {
-          start: resourceOffset,
-          end: resourceOffset + length - 1,
-        })
-      )
-    },
-    readFileSync: function readFileSync(filepath: any, options: any) {
-      setupManifest()
-      const entry = manifest[getKey(filepath)]
-      if (!entry) {
-        return originalFsMethods.readFileSync.apply(fs, arguments)
-      }
-      const [offset, length] = entry
-      const resourceOffset = resourceStart + offset
-      const encoding = fileOpts(options).encoding
-      const fd = originalFsMethods.openSync(process.execPath, 'r')
-      const result = Buffer.alloc(length)
-      originalFsMethods.readSync(fd, result, 0, length, resourceOffset)
-      originalFsMethods.closeSync(fd)
-      return encoding ? result.toString(encoding) : result
-    },
-    statSync: function statSync(filepath: string | Buffer, options: any) {
-      const stat = ownStat(filepath, options)
-      if (stat) {
-        return stat
-      }
-      return originalFsMethods.statSync.apply(fs, arguments)
-    },
-    stat: getStat('stat'),
-    lstat: getStat('lstat'),
-    lstatSync: function statSync(filepath: string | Buffer, options: any) {
-      const stat = ownStat(filepath, options)
-      if (stat) {
-        return stat
-      }
-      return originalFsMethods.lstatSync.apply(fs, arguments)
-    },
-  }
-  if (typeof fs.exists === 'function') {
-    nfs.exists = function (filepath: string, cb: Function) {
-      cb = cb || noop
-      const exists = nfs.existsSync(filepath)
-      process.nextTick(() => cb(exists))
-    }
-  }
-
-  const patches = (process as any).nexe.patches || {}
-  delete (process as any).nexe
-  patches.internalModuleReadFile = function (this: any, original: any, ...args: any[]) {
-    setupManifest()
-    const filepath = getKey(args[0])
-    if (manifest[filepath]) {
-      log('read     (hit)              ' + filepath)
-      return nfs.readFileSync(filepath, 'utf-8')
-    }
-    log('read          (miss)       ' + filepath)
-    return original.call(this, ...args)
   }
   let returningArray: boolean
   patches.internalModuleReadJSON = function (this: any, original: any, ...args: any[]) {
     if (returningArray == null) returningArray = Array.isArray(original.call(this, ''))
-    const res = patches.internalModuleReadFile.call(this, original, ...args)
+    const res = internalModuleReadFile.call(this, original, ...args)
     return returningArray && !Array.isArray(res)
-      ? [res, /"(main|name|type|exports|imports)"/.test(res)]
+      ? res === ''
+        ? []
+        : [res, /"(main|name|type|exports|imports)"/.test(res)]
       : res
   }
   patches.internalModuleStat = function (this: any, original: any, ...args: any[]) {
-    setupManifest()
+    log(`internalModuleStat ${args[0]}`)
-    const filepath = getKey(args[0])
+    try {
-    if (manifest[filepath]) {
+      const stat = posixSnapshotZipFs.statSync(args[0])
-      log('stat     (hit)              ' + filepath + '   ' + 0)
+      if (stat.isDirectory()) return 1
       return 0
-    }
+    } catch (e) {
-    if (directories[filepath]) {
+      return -constants.ENOENT
-      log('stat dir (hit)              ' + filepath + '   ' + 1)
-      return 1
-    }
-    const res = original.call(this, ...args)
-    if (res === 0) {
-      log('stat          (miss)        ' + filepath + '   ' + res)
-    } else if (res === 1) {
-      log('stat dir      (miss)        ' + filepath + '   ' + res)
-    } else {
-      log('stat                 (fail) ' + filepath + '   ' + res)
-    }
-    return res
-  }
-
-  if (typeof fs.exists === 'function') {
-    nfs.exists = function (filepath: string, cb: Function) {
-      cb = cb || noop
-      const exists = nfs.existsSync(filepath)
-      if (!exists) {
-        return originalFsMethods.exists(filepath, cb)
-      }
-      process.nextTick(() => cb(exists))
     }
   }
 
-  if (typeof fs.copyFile === 'function') {
-    nfs.copyFile = function (filepath: string, dest: string, flags: number, callback: Function) {
-      setupManifest()
-      const entry = manifest[getKey(filepath)]
-      if (!entry) {
-        return originalFsMethods.copyFile.apply(fs, arguments)
-      }
-      if (typeof flags === 'function') {
-        callback = flags
-        flags = 0
-      }
-      nfs.readFile(filepath, (err: any, buffer: any) => {
-        if (err) {
-          return callback(err)
-        }
-        originalFsMethods.writeFile(dest, buffer, (err: any) => {
-          if (err) {
-            return callback(err)
-          }
-          callback(null)
-        })
-      })
-    }
-    nfs.copyFileSync = function (filepath: string, dest: string) {
-      setupManifest()
-      const entry = manifest[getKey(filepath)]
-      if (!entry) {
-        return originalFsMethods.copyFileSync.apply(fs, arguments)
-      }
-      return originalFsMethods.writeFileSync(dest, nfs.readFileSync(filepath))
-    }
-  }
-
-  if (typeof fs.realpath.native === 'function') {
-    nfs.realpath.native = function realpathNative(filepath: any, options: any, cb: any): void {
-      setupManifest()
-      const key = getKey(filepath)
-      if (isString(filepath) && (manifest[filepath] || manifest[key])) {
-        return process.nextTick(() => cb(null, filepath))
-      }
-      return originalFsMethods.realpath.native.call(fs, filepath, options, cb)
-    }
-    nfs.realpathSync.native = function realpathSyncNative(filepath: any, options: any) {
-      setupManifest()
-      const key = getKey(filepath)
-      if (manifest[key]) {
-        return filepath
-      }
-      return originalFsMethods.realpathSync.native.call(fs, filepath, options)
-    }
-  }
-
-  Object.assign(fs, nfs)
-
   lazyRestoreFs = () => {
-    Object.keys(nfs).forEach((key) => {
+    Object.assign(fs, originalFsMethods)
-      fs[key] = originalFsMethods[key]
+    Object.assign(patches, originalPatches)
-    })
     lazyRestoreFs = () => {}
   }
-  return true
 }
 
 function restoreFs() {

src/options.ts

@@ -1,4 +1,4 @@
-import * as parseArgv from 'minimist'
+import parseArgv from 'minimist'
 import { NexeCompiler, NexeError } from './compiler'
 import { isWindows, STDIN_FLAG } from './util'
 import { basename, extname, join, isAbsolute, resolve } from 'path'
@@ -9,7 +9,7 @@ import { resolveSync } from 'resolve-dependencies'
 const caw = require('caw')
 const c = process.platform === 'win32' ? chalk.constructor({ enabled: false }) : chalk
 
-export const version = '{{replace:0}}'
+export const version = '{{ version }}'
 
 export interface NexePatch {
   (compiler: NexeCompiler, next: () => Promise<void>): Promise<void>
@@ -109,7 +109,7 @@ ${c.bold('nexe <entry-file> [options]')}
    ${c.underline.bold('Building from source:')}
 
   -b   --build                      -- build from source
-  -p   --python                     -- python2 (as python) executable path
+  -p   --python                     -- python3 (as python) executable path
   -f   --flag                       -- *v8 flags to include during compilation
   -c   --configure                  -- *arguments to the configure step
   -m   --make                       -- *arguments to the make/build step
@@ -261,7 +261,9 @@ function normalizeOptions(input?: Partial<NexeOptions>): NexeOptions {
   options.downloadOptions.agent = process.env.HTTPS_PROXY
     ? caw(process.env.HTTPS_PROXY, { protocol: 'https' })
     : options.downloadOptions.agent || require('https').globalAgent
-  options.downloadOptions.rejectUnauthorized = process.env.HTTPS_PROXY ? false : true
+  options.downloadOptions.rejectUnauthorized = process.env.NODE_TLS_REJECT_UNAUTHORIZED
+    ? false
+    : true
 
   options.rc = options.rc || extractCliMap(/^rc-.*/, options)
   options.output =

src/patches/third-party-main.ts

@@ -1,5 +1,5 @@
 import { NexeCompiler } from '../compiler'
-import { parse } from 'cherow'
+import { parse } from 'meriyah'
 import { wrap, semverGt } from '../util'
 
 function walkSome(node: any, visit: Function) {
@@ -29,6 +29,8 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
 
   if (version.startsWith('4.')) {
     bootFile = 'src/node.js'
+  } else if (semverGt(version, '18.11.99')) {
+    bootFile = 'lib/internal/process/pre_execution.js'
   } else if (semverGt(version, '11.99')) {
     bootFile = 'lib/internal/bootstrap/pre_execution.js'
   } else if (semverGt(version, '9.10.1')) {
@@ -37,12 +39,10 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
 
   const file = await compiler.readFileAsync(bootFile),
     ast = parse(file.contents.toString(), {
-      loc: true,
-      tolerant: true,
       next: true,
       globalReturn: true,
-      node: true,
+      loc: true,
-      skipShebang: true,
+      specDeviation: true,
     }),
     location = { start: { line: 0 } }
 
@@ -55,13 +55,22 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
   })
 
   const fileLines = file.contents.toString().split('\n')
-  fileLines.splice(
+  if (semverGt(version, '19.99')) {
-    location.start.line,
+    await compiler.replaceInFileAsync(
-    0,
+      'lib/internal/modules/cjs/loader.js',
-    '{{replace:lib/fs/bootstrap.js}}' +
+      "'use strict';",
-      '\n' +
+      "'use strict';\n" + '{{ file("lib/fs/bootstrap.js") }}' + '\n'
-      (semverGt(version, '11.99') ? 'expandArgv1 = false;\n' : '')
+    )
-  )
+    fileLines.splice(location.start.line, 0, 'expandArgv1 = false;')
+  } else {
+    fileLines.splice(
+      location.start.line,
+      0,
+      '{{ file("lib/fs/bootstrap.js") }}' +
+        '\n' +
+        (semverGt(version, '11.99') ? 'expandArgv1 = false;\n' : '')
+    )
+  }
   file.contents = fileLines.join('\n')
 
   if (semverGt(version, '11.99')) {
@@ -69,13 +78,13 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
       await compiler.replaceInFileAsync(
         bootFile,
         'initializeFrozenIntrinsics();',
-        'initializeFrozenIntrinsics();\n' + wrap('{{replace:lib/patches/boot-nexe.js}}')
+        'initializeFrozenIntrinsics();\n' + wrap('{{ file("lib/patches/boot-nexe.js") }}')
       )
     } else {
       await compiler.replaceInFileAsync(
         bootFile,
         'initializePolicy();',
-        'initializePolicy();\n' + wrap('{{replace:lib/patches/boot-nexe.js}}')
+        'initializePolicy();\n' + wrap('{{ file("lib/patches/boot-nexe.js") }}')
       )
     }
     await compiler.replaceInFileAsync(
@@ -102,7 +111,7 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
   } else {
     await compiler.setFileContentsAsync(
       'lib/_third_party_main.js',
-      '{{replace:lib/patches/boot-nexe.js}}'
+      '{{ file("lib/patches/boot-nexe.js") }}'
     )
   }
   return next()

src/steps/bundle.ts

@@ -61,9 +61,11 @@ export default async function bundle(compiler: NexeCompiler, next: any) {
     compiler.entrypoint = './' + relative(cwd, input)
   }
 
-  const { files, warnings } = resolveFiles(
+  const step = compiler.log.step('Resolving dependencies...')
+
+  const { files, warnings } = await resolveFiles(
     input,
-    ...Object.keys(compiler.bundle.list).filter((x) => x.endsWith('.js')),
+    ...Object.keys(compiler.bundle.list).filter((x) => x.endsWith('.js') || x.endsWith('.mjs')),
     { cwd, expand: 'variable', loadContent: false }
   )
 
@@ -76,8 +78,9 @@ export default async function bundle(compiler: NexeCompiler, next: any) {
   //TODO: warnings.forEach((x) => console.log(x))
 
   await Promise.all(
-    Object.entries(files).map(([key, file]) => {
+    Object.entries(files).map(([key]) => {
-      return compiler.addResource(key, file)
+      step.log(`Including dependency: ${key}`)
+      return compiler.addResource(key)
     })
   )
 

src/steps/download.ts

@@ -58,7 +58,7 @@ export default async function downloadNode(compiler: NexeCompiler, next: () => P
     { sourceUrl, downloadOptions, build } = compiler.options,
     url = sourceUrl || `https://nodejs.org/dist/v${version}/node-v${version}.tar.gz`,
     step = log.step(
-      `Downloading ${build ? '' : 'pre-built '}Node.js${build ? `source from: ${url}` : ''}`
+      `Downloading ${build ? '' : 'pre-built '}Node.js${build ? ` source from: ${url}` : ''}`
     ),
     exeLocation = compiler.getNodeExecutableLocation(build ? undefined : target),
     downloadExists = await pathExistsAsync(build ? src : exeLocation)

src/steps/resource.ts

@@ -1,5 +1,5 @@
 import { each } from '../util'
-import * as globs from 'globby'
+import globs from 'globby'
 import { resolve } from 'path'
 import { NexeCompiler } from '../compiler'
 

src/steps/shim.ts

@@ -5,10 +5,17 @@ export default async function (compiler: NexeCompiler, next: () => Promise<void>
   await next()
   compiler.shims.push(
     wrap(
-      '' +
+      [
-        '{{replace:lib/fs/patch.js}}' +
+        'process.__nexe = {};',
-        '\nshimFs(process.__nexe)' +
+        'const fsPatcher = (function() {',
-        `\n${compiler.options.fs ? '' : 'restoreFs()'}`
+        'const module = {exports: {}};',
+        'const exports = module.exports;',
+        '{{file("lib/fs/patch.bundle.js")}}',
+        'return module.exports;',
+        '})()',
+        'fsPatcher.shimFs(process.__nexe);',
+        compiler.options.fs ? '' : 'restoreFs();',
+      ].join('\n')
       //TODO support only restoring specific methods
     )
   )

src/util.ts

@@ -26,7 +26,7 @@ function falseOnEnoent(e: any) {
 }
 
 function padRight(str: string, l: number) {
-  return (str + ' '.repeat(l)).substr(0, l)
+  return (str + ' '.repeat(l)).slice(0, l)
 }
 
 const bound: MethodDecorator = function bound<T>(

tasks/docker.ts

@@ -50,7 +50,7 @@ export async function runDockerBuild(target: NexeTarget) {
     output.push(await execa(`docker run -d --name nexe nexe-docker sh`, { shell: true }))
     output.push(await execa(`docker cp nexe:/out out`, { shell: true }))
     output.push(await execa(`docker rm nexe`, { shell: true }))
-  } catch (e) {
+  } catch (e: any) {
     console.log('Error running docker')
     appendFileSync(outFilename, e.message)
   } finally {

tasks/post-build.ts

@@ -1,11 +1,10 @@
 import { writeFileSync, readFileSync } from 'fs'
+import { template } from 'lodash'
 
 /**
- * post build step to insert code files into code files (naively).
+ * post build step to insert code files into code files.
- * '{{replace:path/to/file}}' => "file contents"
  * And the package.json version.
  */
-//inject('plugins/nexe-daemon/lib/index.js')
 cp('src/fs/package.json', 'lib/fs/package.json')
 cp('src/fs/bootstrap.js', 'lib/fs/bootstrap.js')
 cp('src/fs/README.md', 'lib/fs/README.md')
@@ -14,18 +13,12 @@ inject('lib/patches/third-party-main.js')
 inject('lib/steps/shim.js')
 inject('lib/options.js', JSON.stringify(require('../package.json').version))
 
-function inject(filename: string, ...replacements: string[]) {
+function inject(filename: string, version?: string) {
-  let contents = readFileSync(filename, 'utf8')
+  const contents = template(readFileSync(filename, 'utf8'), {
-  contents = contents.replace(/('{{(.*)}}')/g, (substring: string, ...matches: string[]) => {
+    interpolate: /'\{\{([\s\S]*?)\}\}'/,
-    if (!matches || !matches[1]) {
+  })({
-      return substring
+    file: (path: string) => JSON.stringify(readFileSync(path, 'utf8')),
-    }
+    version,
-    const [replace, file] = matches[1].split(':')
-    if (replace !== 'replace') {
-      return substring
-    }
-    console.log('Replacing: ', substring)
-    return replacements[+file] ? replacements[+file] : JSON.stringify(readFileSync(file, 'utf8'))
   })
   writeFileSync(filename, contents)
   console.log(`Wrote: ${filename}`)

tasks/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true
+  },
+  "include":[
+    "*.ts"
+  ]
+}
+

test/integration/index.js

@@ -0,0 +1,23 @@
+const { mkdtemp, copyFile } = require('fs/promises')
+const os = require('os')
+const path = require('path')
+const rimraf = require('rimraf')
+const cp = require('child_process')
+
+async function runTests() {
+  const tempdir = await mkdtemp(path.join(os.tmpdir(), 'nexe-integration-tests-'))
+  const executable = path.join(tempdir, path.basename(process.argv[0]))
+  await copyFile(process.argv[0], executable)
+  process.on('beforeExit', () => rimraf.sync(tempdir))
+  cp.spawn(executable,
+    [
+    path.join(tempdir, 'node_modules/mocha/bin/mocha.js'),
+      path.join(tempdir, 'test/integration/tests.integration-spec.js')
+    ],
+    { stdio: ['inherit', 'inherit', 'inherit', 'ipc'], cwd: tempdir }
+  ).on('exit', (code) => {
+    process.exitCode = code
+  })
+}
+
+runTests()

test/integration/tests.integration-spec.js

@@ -0,0 +1,120 @@
+const { expect } = require('chai')
+const fs = require('fs')
+const { dirname, basename } = require('path')
+
+const files = { zipFs: 'test/integration/tests.integration-spec.js', real: 'real', real2: 'real2' }
+describe('SnapshotZipFS', () => {
+  console.error('Note: these tests are order dependant, so subsequent failures are likely a flow on effect.')
+  it('checks existance from the zipFs', () => expect(fs.existsSync(files.zipFs)).to.equal(true))
+  it('checks non-existance from the zipFs', () => expect(fs.existsSync('not found')).to.equal(false))
+  it('reads from the zipFs', () => expect(fs.readFileSync(files.zipFs, 'utf8')).to.include('SnapshotZipFS'))
+  it('copies from the zipFs', () => expect(() => fs.copyFileSync(files.zipFs, files.real)).not.to.throw())
+  it('reads from the real fs', () => expect(fs.readFileSync(files.real, 'utf8')).to.include('SnapshotZipFS'))
+  it('checks non-existance from the real fs', () => expect(fs.existsSync(files.real2)).to.equal(false))
+  it('copies from the real fs', () => expect(() => fs.copyFileSync(files.real, files.real2)).not.to.throw())
+  it('checks existance from the real fs', () => expect(fs.existsSync(files.real2)).to.equal(true))
+  it('unlinks from the real fs', () => expect(() => fs.unlinkSync(files.real2)).not.to.throw())
+  it('symlinks the real fs', () => expect(() => fs.symlinkSync('tmp', files.real2)).not.to.throw())
+  it('readlinks the real fs', () => expect(fs.readlinkSync(files.real2)).to.equal('tmp'))
+  it('truncates the real fs', () => expect(() => fs.truncateSync(files.real)).not.to.throw())
+  it('writeFiles the real fs', () => expect(() => fs.writeFileSync(files.real2, 'testing')).not.to.throw())
+  it('accesses the real fs', () => expect(() => fs.accessSync(files.real)).not.to.throw())
+  it('accesses the zipFs', () => expect(() => fs.accessSync(files.zipFs)).not.to.throw())
+  it('stats the real fs', () => expect(fs.statSync(files.real2).size).to.be.greaterThan(0))
+  it('stats the zipFs', () => expect(fs.statSync(files.zipFs).size).to.be.greaterThan(0))
+  it('readdirs from the zipFs', () => expect(fs.readdirSync(dirname(files.zipFs))).to.include(basename(files.zipFs)))
+  it('readdirs from the real fs', () => expect(fs.readdirSync('.')).to.include(files.real))
+  it('readdirs from both', () => {
+    expect(fs.readdirSync('.')).to.include(files.real),
+    expect(fs.readdirSync('.')).to.include('test')
+  })
+  let dir
+  it('opendirs the zipFs fs', async () => {
+    expect(() => dir = fs.opendirSync(dirname(files.zipFs))).not.to.throw()
+    const names = []
+    for await (const dirent of dir) {
+      names.push(dirent.name)
+    }
+    expect(names).to.include(basename(files.zipFs))
+  })
+  it('opendirs the real fs and zipFs', async () => {
+    expect(() => dir = fs.opendirSync('.')).not.to.throw()
+    const names = []
+    for await (const dirent of dir) {
+      names.push(dirent.name)
+    }
+    expect(names).to.include(files.real)
+    expect(names).to.include('test')
+  })
+  it('createWriteStreams to the real fs', () => expect(() => fs.createWriteStream('createWriteStream')).not.to.throw())
+  it('createReadStreams to the real fs', () => expect(() => fs.createReadStream(files.real)).not.to.throw())
+  it('createReadStreams to the zipFs', () => expect(() => fs.createReadStream(files.zipFs)).not.to.throw())
+  it('renames to the realFs', () => {
+    expect(() => fs.renameSync(files.real2, 'rename')).not.to.throw()
+    expect(fs.existsSync('rename')).to.equal(true)
+  })
+
+  describe('write actions to the zipFs', () => {
+    it(`won't truncate to the zipFs`, () => expect(() => fs.truncateSync(files.zipFs)).to.throw())
+    it(`won't unlink from the zipFs`, () => expect(() => fs.unlinkSync(files.zipFs)).to.throw())
+    it(`won't appendFile to the zipFs`, () => expect(() => fs.appendFileSync(files.zipFs, 'xxx')).to.throw())
+    it(`won't symlink to the zipFs`, () => expect(() => fs.symlinkSync('tmp', files.zipFs)).to.throw())
+    it(`won't chmod to the zipFs`, () => expect(() => fs.chmodSync(files.zipFs, 0o666)).to.throw())
+    it(`won't chown to the zipFs`, () => expect(() => fs.chownSync(files.zipFs, 0, 0)).to.throw())
+    it(`won't writeFile the zipFs`, () => expect(() => fs.writeFileSync(files.zipFs, 'testing')).to.throw())
+    it(`won't createWriteStreams to the zipFs`, () => expect(() => fs.createWriteStream(files.zipFs)).to.throw())
+    it(`won't rename to the zipFs`, () => expect(() => fs.renameSync(files.real, files.zipFs)).to.throw())
+  })
+
+  ;[ 'zipFs', 'real' ].forEach((k) => {
+    it(`watches the ${k} fs`, () => expect(() => fs.watch(files[k], () => null).close()).not.to.throw())
+    it(`watchFiles the ${k} fs`, () => expect(() => fs.watchFile(files[k], () => null)).not.to.throw())
+    it(`unwatchFiles the ${k} fs`, () => expect(() => fs.unwatchFile(files[k])).not.to.throw())
+  })
+  let buffer = Buffer.alloc(8092), fd
+  describe('zipfs file descriptors', () => {
+    it('opens files in the zipFs', () => {
+      expect(() => (fd = fs.openSync(files.zipFs, 'a+'))).not.to.throw()
+      expect(fd).to.not.be.null
+    })
+    it('reads file descriptors from the zipFs', () => {
+      expect(() => fs.readSync(fd, buffer)).not.to.throw()
+      expect(buffer.toString()).to.include('SnapshotZipFS')
+    })
+    it('fstats the zipFs', () => expect(fs.fstatSync(fd).size).to.be.greaterThan(0))
+    it(`won't write to the zipFs`, () => expect(() => fs.writeSync(fd, 'xxx')).to.throw())
+    it(`won't fchmod the zipFs`, () => expect(() => fs.fchmodSync(fd)).to.throw())
+    it(`won't fchown the zipFs`, () => expect(() => fs.fchownSync(fd)).to.throw())
+    it(`won't ftruncate the zipFs`, () => expect(() => fs.ftruncateSync(fd)).to.throw())
+    it('closes file descriptors from the zipFs', () => expect(() => fs.closeSync(fd)).not.to.throw())
+  })
+  fd = null
+  buffer = Buffer.alloc(8092)
+  describe('real file descriptors', () => {
+    it('opens files in the real fs', () => {
+      expect(() => (fd = fs.openSync(files.real2, 'a+'))).not.to.throw()
+      expect(fd).to.not.be.null
+    })
+    it('reads file descriptors from the real fs', () => {
+      expect(() => fs.readSync(fd, buffer)).not.to.throw()
+      expect(buffer.toString()).to.include('SnapshotZipFS')
+    })
+    it('will write to the real fs', () => expect(() => fs.writeSync(fd, 'xxx')).not.to.throw())
+    it('fstats the real fs', () => expect(fs.fstatSync(fd).size).to.be.greaterThan(0))
+    it('closes file descriptors from the real fs', () => expect(() => fs.closeSync(fd)).not.to.throw())
+  })
+  describe('path clashes', () => {
+    it('delegates directory creation to the real fs', () => expect(() => fs.mkdirSync('test')).not.to.throw())
+    it('delegates directory removal to the real fs', () => expect(() => fs.rmdirSync('test')).not.to.throw())
+  })
+  describe('requiring modules', () => {
+    it('resolves modules from the zipFs', () => expect(() => require('resolve-dependencies')).not.to.throw())
+    it('resolves modules from the real fs', () => {
+      fs.mkdirSync('node_modules')
+      fs.mkdirSync('node_modules/fake-package')
+      fs.writeFileSync('node_modules/fake-package/package.json', '{ "name": "fake-package", "main": "some-file.js"}')
+      fs.writeFileSync('node_modules/fake-package/some-file.js', `module.exports = { fake: 'package' }`)
+      expect(require('fake-package')).to.eql({ fake: 'package' })
+    })
+  })
+})

tsconfig.json

@@ -3,6 +3,7 @@
     "target":"es2015",
     "skipLibCheck": true,
     "experimentalDecorators": true,
+    "esModuleInterop": true,
     "lib": ["es2018"],
     "module": "commonjs",
     "outDir": "./lib",

webpack.config.ts

@@ -0,0 +1,7 @@
+import * as wcp from 'webpack-config-prefabs'
+export = wcp.nodeLibrary(module, {
+  enableTypescript: true,
+  entry: './src/fs/patch.ts',
+  minimize: false,
+  outputFilepath: './lib/fs/patch.bundle.js',
+});