fix: cleanup zip finalization

There's probably a cleaner way to do this...

package.json

@@ -44,22 +44,21 @@
     ]
   },
   "dependencies": {
-    "@calebboyd/semaphore": "^1.3.1",
+    "@yarnpkg/fslib": "^2.6.0-rc.8",
-    "@yarnpkg/fslib": "^3.0.0-rc.43",
+    "@yarnpkg/libzip": "^2.2.2",
-    "@yarnpkg/libzip": "^3.0.0-rc.43",
     "app-builder": "^7.0.4",
     "archiver": "^5.3.1",
     "caw": "^2.0.1",
     "chalk": "^2.4.2",
     "download": "^8.0.0",
     "globby": "^11.0.2",
-    "got": "^12.6.0",
+    "got": "^11.8.2",
-    "meriyah": "^4.3.5",
+    "meriyah": "^4.3.3",
-    "minimist": "^1.2.8",
+    "minimist": "^1.2.6",
     "mkdirp": "^1.0.4",
     "multistream": "^4.1.0",
     "ora": "^3.4.0",
-    "resolve-dependencies": "^6.0.9",
+    "resolve-dependencies": "^6.0.8",
     "rimraf": "^3.0.2",
     "run-script-os": "^1.1.6",
     "webpack-config-prefabs": "0.0.5"
@@ -67,26 +66,29 @@
   "devDependencies": {
     "@types/archiver": "^5.3.2",
     "@types/chai": "^4",
-    "@types/download": "^8",
+    "@types/download": "^6",
     "@types/globby": "^9",
-    "@types/lodash": "^4.14.192",
+    "@types/got": "^9.6.12",
+    "@types/highland": "^2.12.14",
+    "@types/lodash": "^4.14.173",
     "@types/minimist": "^1.2.2",
     "@types/mkdirp": "^1.0.2",
-    "@types/mocha": "^10.0.1",
+    "@types/mocha": "^9.0.0",
     "@types/multistream": "^4.1.0",
+    "@types/node": "^16.10.2",
     "@types/ora": "^3.2.0",
     "@types/rimraf": "3.0.2",
-    "@types/semver": "^7.3.13",
+    "@types/semver": "^7.3.8",
-    "chai": "^4.3.7",
+    "chai": "^4.3.6",
     "execa": "^5.1.1",
     "lodash": "^4.17.21",
-    "mocha": "^10.2.0",
+    "mocha": "^10.0.0",
-    "prettier": "^2.8.7",
+    "prettier": "^2.6.2",
-    "ts-node": "^10.9.1",
+    "ts-node": "^10.7.0",
     "tslint": "^6.1.1",
     "tslint-config-prettier": "^1.18.0",
     "tslint-plugin-prettier": "^2.3.0",
-    "typescript": "^5.0.3",
+    "typescript": "^4.6.4",
-    "webpack-cli": "^5.0.1"
+    "webpack-cli": "^4.8.0"
   }
 }

src/compiler.ts

@@ -298,19 +298,13 @@ export class NexeCompiler {
     const sentinel = Buffer.from('<nexe~~sentinel>')
 
     let vfsSize = 0
-    const streams = [
+    const streams = [binary, toStream(launchCode), this.bundle.toStream().pipe(new Transform({
-      binary,
+      transform: (chunk, _, cb) => {
-      toStream(launchCode),
+        vfsSize || this.bundle.finalize()
-      this.bundle.toStream().pipe(
+        chunk && (vfsSize += chunk.length)
-        new Transform({
+        cb(null, chunk)
-          transform: (chunk, _, cb) => {
+      },
-            vfsSize || this.bundle.finalize()
+    }))]
-            chunk && (vfsSize += chunk.length)
-            cb(null, chunk)
-          },
-        })
-      ),
-    ]
 
     let done = false
     return new MultiStream((cb) => {

src/fs/SnapshotZipFS.ts

@@ -2,19 +2,14 @@ import { Libzip } from '@yarnpkg/libzip'
 import {
   FakeFS,
   PortablePath,
+  ZipFS,
+  ZipOpenFS,
   RmdirOptions,
   OpendirOptions,
-  BasePortableFakeFS,
-  Dirent,
-  Dir,
-  MkdirOptions,
-  FSPath,
-  ppath,
-  npath,
-  Filename,
-  CustomDir,
 } from '@yarnpkg/fslib'
-import { ZipFS, ZipOpenFS } from '@yarnpkg/libzip'
+import { BasePortableFakeFS, Dirent, MkdirOptions, Dir } from '@yarnpkg/fslib/lib/FakeFS'
+import { CustomDir } from '@yarnpkg/fslib/lib/algorithms/opendir'
+import { FSPath, ppath, npath, Filename } from '@yarnpkg/fslib/lib/path'
 import { resolve, toNamespacedPath } from 'path'
 import { constants } from 'fs'
 
@@ -22,17 +17,10 @@ export type SnapshotZipFSOptions = {
   baseFs: FakeFS<PortablePath>
   libzip: Libzip | (() => Libzip)
   zipFs: ZipFS
-  root: string
 }
 
 const uniqBy = (arr: Array<string | Dirent>, pick: (...arg: any[]) => any) => {
-  const seen = new Set()
+  return arr.filter((x, i, self) => i === self.findIndex((y) => pick(x) === pick(y)))
-  return arr.filter((x) => {
-    const key = pick(x)
-    if (seen.has(key)) return false
-    seen.add(key)
-    return true
-  })
 }
 function uniqReaddir(arr: Array<string | Dirent>) {
   return uniqBy(arr, (s: string | Dirent) => (typeof s === 'string' ? s : s.name))
@@ -41,14 +29,10 @@ function uniqReaddir(arr: Array<string | Dirent>) {
 export class SnapshotZipFS extends BasePortableFakeFS {
   zipFs: ZipFS
   baseFs: FakeFS<PortablePath>
-  root: string
-  magic: number
   constructor(opts: SnapshotZipFSOptions) {
     super()
     this.zipFs = opts.zipFs
     this.baseFs = opts.baseFs
-    this.root = opts.root
-    this.magic = 0x2a << 24
   }
   private readonly fdMap: Map<number, [ZipFS, number]> = new Map()
   private nextFd = 3
@@ -90,42 +74,46 @@ export class SnapshotZipFS extends BasePortableFakeFS {
   }
 
   async realpathPromise(p: PortablePath) {
-    return await this.realpathSync(p)
+    return await this.makeCallPromise(
+      p,
+      async () => {
+        return await this.baseFs.realpathPromise(p)
+      },
+      async (zipFs, { subPath }) => {
+        return await zipFs.realpathPromise(subPath)
+      }
+    )
   }
 
   realpathSync(p: PortablePath) {
     return this.makeCallSync(
       p,
-      () => this.baseFs.realpathSync(p),
+      () => {
+        return this.baseFs.realpathSync(p)
+      },
       (zipFs, { subPath }) => {
-        if (zipFs.lstatSync(subPath).isSymbolicLink()) {
+        return zipFs.realpathSync(subPath)
-          return zipFs.realpathSync(subPath)
-        } else {
-          // return the original path in case it wasn't under /snapshot, e.g. if it was for a node module - otherwise the node module parent path is the wrong one (and other things resolve relative to that)
-          return p
-        }
       }
     )
   }
 
   findZip(p: PortablePath) {
     p = this.resolve(p)
-    const snapshotPP = npath.toPortablePath('/snapshot')
     const pathsToTry = Array.from(
       new Set([
         p,
         resolve('/snapshot', p),
         ppath.resolve(
-          snapshotPP,
+          npath.toPortablePath('/snapshot'),
           npath.toPortablePath(
-            npath.relative(npath.fromPortablePath(this.root), npath.fromPortablePath(p))
+            npath.relative(npath.fromPortablePath(process.cwd()), npath.fromPortablePath(p))
           )
         ),
         ppath.resolve(
-          snapshotPP,
+          npath.toPortablePath('/snapshot'),
           npath.toPortablePath(
             npath.relative(
-              toNamespacedPath(npath.fromPortablePath(this.root)),
+              toNamespacedPath(npath.fromPortablePath(process.cwd())),
               toNamespacedPath(npath.fromPortablePath(p))
             )
           )
@@ -248,7 +236,9 @@ export class SnapshotZipFS extends BasePortableFakeFS {
       async (zipFs, { subPath }) => {
         const fallbackPaths: Array<string | Dirent> = await fallback().catch(() => [])
         return Promise.resolve(
-          uniqReaddir(fallbackPaths.concat(await zipFs.readdirPromise(subPath, opts as any)))
+          uniqReaddir(
+            (fallbackPaths as any[]).concat(await zipFs.readdirPromise(subPath, opts as any))
+          )
         )
       },
       {
@@ -273,7 +263,7 @@ export class SnapshotZipFS extends BasePortableFakeFS {
         try {
           fallbackPaths = fallback()
         } catch (e) {}
-        return fallbackPaths.concat(uniqReaddir(zipFs.readdirSync(subPath, opts as any)))
+        return (fallbackPaths as any[]).concat(uniqReaddir(zipFs.readdirSync(subPath, opts as any)))
       },
       {
         requireSubpath: false,
@@ -298,7 +288,7 @@ export class SnapshotZipFS extends BasePortableFakeFS {
   }
 
   async opendirPromise(p: PortablePath, opts?: OpendirOptions) {
-    return this.opendirSync(p, opts)
+    return Promise.resolve(this.zipFs.opendirSync(p, opts))
   }
 
   opendirSync(p: PortablePath, opts?: OpendirOptions) {
@@ -381,8 +371,6 @@ export class SnapshotZipFS extends BasePortableFakeFS {
   unwatchFile = ZipOpenFS.prototype.unwatchFile
   utimesPromise = ZipOpenFS.prototype.utimesPromise
   utimesSync = ZipOpenFS.prototype.utimesSync
-  lutimesPromise = ZipOpenFS.prototype.utimesPromise
-  lutimesSync = ZipOpenFS.prototype.utimesSync
   watch = ZipOpenFS.prototype.watch
   watchFile = ZipOpenFS.prototype.watchFile
   writeFilePromise = ZipOpenFS.prototype.writeFilePromise

src/fs/patch.ts

@@ -1,9 +1,8 @@
-import { ZipFS, getLibzipSync } from '@yarnpkg/libzip'
+import { getLibzipSync } from '@yarnpkg/libzip'
-import { patchFs, npath, PosixFS, NodeFS } from '@yarnpkg/fslib'
+import { patchFs, npath, PosixFS, NodeFS, ZipFS } from '@yarnpkg/fslib'
 import { SnapshotZipFS } from './SnapshotZipFS'
 import * as assert from 'assert'
 import * as constants from 'constants'
-import { dirname } from 'path'
 
 export interface NexeHeader {
   blobPath: string
@@ -18,7 +17,6 @@ export interface NexeHeader {
 let originalFsMethods: any = null
 let lazyRestoreFs = () => {}
 const patches = (process as any).nexe.patches || {}
-const originalPatches = { ...patches }
 delete (process as any).nexe
 
 function shimFs(binary: NexeHeader, fs: typeof import('fs') = require('fs')) {
@@ -42,65 +40,60 @@ function shimFs(binary: NexeHeader, fs: typeof import('fs') = require('fs')) {
   )
   assert.equal(bytesRead, binary.layout.resourceSize)
 
-  const zipFs = new ZipFS(blob, { readOnly: true })
+  const zipFs = new ZipFS(blob, {
+    libzip: getLibzipSync(),
+    readOnly: true,
+  })
   const snapshotZipFS = new SnapshotZipFS({
     libzip: getLibzipSync(),
     zipFs,
     baseFs: nodeFs,
-    root: dirname(process.argv[0]),
   })
   const posixSnapshotZipFs = new PosixFS(snapshotZipFS)
   patchFs(fs, posixSnapshotZipFs)
+  const { readFileSync } = fs
 
   let log = (_: string) => true
+  let loggedManifest = false
   if ((process.env.DEBUG || '').toLowerCase().includes('nexe:require')) {
-    process.stderr.write(
-      // @ts-ignore
-      `[nexe] - FILES ${JSON.stringify(Array.from(zipFs.entries.keys()), null, 4)}\n`
-    )
-    process.stderr.write(
-      // @ts-ignore
-      `[nexe] - DIRECTORIES ${JSON.stringify(Array.from(zipFs.listings.keys()), null, 4)}\n`
-    )
     log = (text: string) => {
-      return process.stderr.write(`[nexe] - ${text}\n`)
+      if (!loggedManifest) {
+        // TODO anything meaningful to log here?
+        loggedManifest = true
+      }
+      return process.stderr.write('[nexe] - ' + text + '\n')
     }
   }
-  function internalModuleReadFile(this: any, original: any, ...args: any[]) {
+  patches.internalModuleReadFile = function (this: any, original: any, ...args: any[]) {
     log(`internalModuleReadFile ${args[0]}`)
     try {
-      return posixSnapshotZipFs.readFileSync(args[0], 'utf-8')
+      return fs.readFileSync(args[0], 'utf-8')
     } catch (e) {
-      return ''
+      return null
     }
   }
-  if (patches.internalModuleReadFile) {
-    patches.internalModuleReadFile = internalModuleReadFile
-  }
   let returningArray: boolean
   patches.internalModuleReadJSON = function (this: any, original: any, ...args: any[]) {
     if (returningArray == null) returningArray = Array.isArray(original.call(this, ''))
-    const res = internalModuleReadFile.call(this, original, ...args)
+    const res = patches.internalModuleReadFile.call(this, original, ...args)
     return returningArray && !Array.isArray(res)
-      ? res === ''
+      ? [res, /"(main|name|type|exports|imports)"/.test(res)]
-        ? []
-        : [res, /"(main|name|type|exports|imports)"/.test(res)]
       : res
   }
   patches.internalModuleStat = function (this: any, original: any, ...args: any[]) {
     log(`internalModuleStat ${args[0]}`)
     try {
-      const stat = posixSnapshotZipFs.statSync(args[0])
+      const stat = fs.statSync(args[0])
       if (stat.isDirectory()) return 1
       return 0
     } catch (e) {
       return -constants.ENOENT
     }
   }
+  // TODO restore patches in restoreFs
 
   lazyRestoreFs = () => {
     Object.assign(fs, originalFsMethods)
-    Object.assign(patches, originalPatches)
     lazyRestoreFs = () => {}
   }
 }

src/patches/boot-nexe.ts

@@ -1,14 +1,15 @@
 const fs = require('fs'),
   fd = fs.openSync(process.execPath, 'r'),
   stat = fs.statSync(process.execPath),
-  tailSize = Math.min(stat.size, 16000),
+  tailSize = Math.min(stat.size, 4096),
   tailWindow = Buffer.from(Array(tailSize))
 
 fs.readSync(fd, tailWindow, 0, tailSize, stat.size - tailSize)
 
 const footerPosition = tailWindow.indexOf('<nexe~~sentinel>')
-if (footerPosition == -1) {
+if (footerPosition === -1) {
-  throw 'Invalid Nexe binary'
+  process.stderr.write('Invalid Nexe binary')
+  process.exit(1)
 }
 
 const footer = tailWindow.slice(footerPosition, footerPosition + 32),

src/patches/third-party-main.ts

@@ -55,22 +55,13 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
   })
 
   const fileLines = file.contents.toString().split('\n')
-  if (semverGt(version, '19.99')) {
+  fileLines.splice(
-    await compiler.replaceInFileAsync(
+    location.start.line,
-      'lib/internal/modules/cjs/loader.js',
+    0,
-      "'use strict';",
+    '{{ file("lib/fs/bootstrap.js") }}' +
-      "'use strict';\n" + '{{ file("lib/fs/bootstrap.js") }}' + '\n'
+      '\n' +
-    )
+      (semverGt(version, '11.99') ? 'expandArgv1 = false;\n' : '')
-    fileLines.splice(location.start.line, 0, 'expandArgv1 = false;')
+  )
-  } else {
-    fileLines.splice(
-      location.start.line,
-      0,
-      '{{ file("lib/fs/bootstrap.js") }}' +
-        '\n' +
-        (semverGt(version, '11.99') ? 'expandArgv1 = false;\n' : '')
-    )
-  }
   file.contents = fileLines.join('\n')
 
   if (semverGt(version, '11.99')) {

src/steps/download.ts

@@ -58,7 +58,7 @@ export default async function downloadNode(compiler: NexeCompiler, next: () => P
     { sourceUrl, downloadOptions, build } = compiler.options,
     url = sourceUrl || `https://nodejs.org/dist/v${version}/node-v${version}.tar.gz`,
     step = log.step(
-      `Downloading ${build ? '' : 'pre-built '}Node.js${build ? ` source from: ${url}` : ''}`
+      `Downloading ${build ? '' : 'pre-built '}Node.js${build ? `source from: ${url}` : ''}`
     ),
     exeLocation = compiler.getNodeExecutableLocation(build ? undefined : target),
     downloadExists = await pathExistsAsync(build ? src : exeLocation)

tasks/docker.ts

@@ -58,11 +58,17 @@ export async function runDockerBuild(target: NexeTarget) {
       appendFileSync(outFilename, x.stderr)
       appendFileSync(outFilename, x.stdout)
     })
-    await got(`https://transfer.sh/${Math.random().toString(36).substring(2)}.txt`, {
+    try {
-      body: await readFileAsync(outFilename),
+      const { body } = await got(
-      method: 'PUT',
+        `https://transfer.sh/${Math.random().toString(36).substring(2)}.txt`,
-    })
+        {
-      .then((x) => console.log('Posted docker log: ', x.body))
+          body: await readFileAsync(outFilename),
-      .catch((e) => console.log('Error posting log', e))
+          method: 'PUT',
+        }
+      )
+      console.log('Posted docker log: ', body)
+    } catch (e) {
+      console.log('Error posting log', e)
+    }
   }
 }

test/integration/index.js

@@ -1,22 +1,17 @@
-const { mkdtemp, copyFile } = require('fs/promises')
+const { mkdtemp } = require('fs/promises')
 const os = require('os')
 const path = require('path')
 const rimraf = require('rimraf')
-const cp = require('child_process')
 
 async function runTests() {
-  const tempdir = await mkdtemp(path.join(os.tmpdir(), 'nexe-integration-tests-'))
+  const tempdir = await mkdtemp(path.join(os.tmpdir(), 'nexe-'))
-  const executable = path.join(tempdir, path.basename(process.argv[0]))
-  await copyFile(process.argv[0], executable)
   process.on('beforeExit', () => rimraf.sync(tempdir))
-  cp.spawn(executable,
+  process.chdir(tempdir)
-    [
+  const Mocha = require('mocha')
-    path.join(tempdir, 'node_modules/mocha/bin/mocha.js'),
+  const mocha = new Mocha()
-      path.join(tempdir, 'test/integration/tests.integration-spec.js')
+  mocha.addFile('test/integration/tests.integration-spec.js')
-    ],
+  mocha.run((failures) => {
-    { stdio: ['inherit', 'inherit', 'inherit', 'ipc'], cwd: tempdir }
+    process.exitCode = failures ? 1 : 0
-  ).on('exit', (code) => {
-    process.exitCode = code
   })
 }
 

test/integration/tests.integration-spec.js

@@ -1,6 +1,6 @@
 const { expect } = require('chai')
 const fs = require('fs')
-const { dirname, basename } = require('path')
+const { dirname, basename }= require('path')
 
 const files = { zipFs: 'test/integration/tests.integration-spec.js', real: 'real', real2: 'real2' }
 describe('SnapshotZipFS', () => {
@@ -88,6 +88,7 @@ describe('SnapshotZipFS', () => {
     it(`won't ftruncate the zipFs`, () => expect(() => fs.ftruncateSync(fd)).to.throw())
     it('closes file descriptors from the zipFs', () => expect(() => fs.closeSync(fd)).not.to.throw())
   })
+
   fd = null
   buffer = Buffer.alloc(8092)
   describe('real file descriptors', () => {
@@ -95,7 +96,7 @@ describe('SnapshotZipFS', () => {
       expect(() => (fd = fs.openSync(files.real2, 'a+'))).not.to.throw()
       expect(fd).to.not.be.null
     })
-    it('reads file descriptors from the real fs', () => {
+    it('reads file descriptors from the real', () => {
       expect(() => fs.readSync(fd, buffer)).not.to.throw()
       expect(buffer.toString()).to.include('SnapshotZipFS')
     })
@@ -103,18 +104,9 @@ describe('SnapshotZipFS', () => {
     it('fstats the real fs', () => expect(fs.fstatSync(fd).size).to.be.greaterThan(0))
     it('closes file descriptors from the real fs', () => expect(() => fs.closeSync(fd)).not.to.throw())
   })
+
   describe('path clashes', () => {
     it('delegates directory creation to the real fs', () => expect(() => fs.mkdirSync('test')).not.to.throw())
     it('delegates directory removal to the real fs', () => expect(() => fs.rmdirSync('test')).not.to.throw())
   })
-  describe('requiring modules', () => {
-    it('resolves modules from the zipFs', () => expect(() => require('resolve-dependencies')).not.to.throw())
-    it('resolves modules from the real fs', () => {
-      fs.mkdirSync('node_modules')
-      fs.mkdirSync('node_modules/fake-package')
-      fs.writeFileSync('node_modules/fake-package/package.json', '{ "name": "fake-package", "main": "some-file.js"}')
-      fs.writeFileSync('node_modules/fake-package/some-file.js', `module.exports = { fake: 'package' }`)
-      expect(require('fake-package')).to.eql({ fake: 'package' })
-    })
-  })
 })