Update src/steps/download.ts

Co-authored-by: Jared Allard <jaredallard@users.noreply.github.com>

package.json

@@ -44,21 +44,22 @@
     ]
   },
   "dependencies": {
-    "@yarnpkg/fslib": "^2.6.0-rc.8",
-    "@yarnpkg/libzip": "^2.2.2",
+    "@calebboyd/semaphore": "^1.3.1",
+    "@yarnpkg/fslib": "^3.0.0-rc.43",
+    "@yarnpkg/libzip": "^3.0.0-rc.43",
     "app-builder": "^7.0.4",
     "archiver": "^5.3.1",
     "caw": "^2.0.1",
     "chalk": "^2.4.2",
     "download": "^8.0.0",
     "globby": "^11.0.2",
-    "got": "^11.8.2",
-    "meriyah": "^4.3.3",
-    "minimist": "^1.2.6",
+    "got": "^12.6.0",
+    "meriyah": "^4.3.5",
+    "minimist": "^1.2.8",
     "mkdirp": "^1.0.4",
     "multistream": "^4.1.0",
     "ora": "^3.4.0",
-    "resolve-dependencies": "^6.0.8",
+    "resolve-dependencies": "^6.0.9",
     "rimraf": "^3.0.2",
     "run-script-os": "^1.1.6",
     "webpack-config-prefabs": "0.0.5"
@@ -66,29 +67,26 @@
   "devDependencies": {
     "@types/archiver": "^5.3.2",
     "@types/chai": "^4",
-    "@types/download": "^6",
+    "@types/download": "^8",
     "@types/globby": "^9",
-    "@types/got": "^9.6.12",
-    "@types/highland": "^2.12.14",
-    "@types/lodash": "^4.14.173",
+    "@types/lodash": "^4.14.192",
     "@types/minimist": "^1.2.2",
     "@types/mkdirp": "^1.0.2",
-    "@types/mocha": "^9.0.0",
+    "@types/mocha": "^10.0.1",
     "@types/multistream": "^4.1.0",
-    "@types/node": "^16.10.2",
     "@types/ora": "^3.2.0",
     "@types/rimraf": "3.0.2",
-    "@types/semver": "^7.3.8",
-    "chai": "^4.3.6",
+    "@types/semver": "^7.3.13",
+    "chai": "^4.3.7",
     "execa": "^5.1.1",
     "lodash": "^4.17.21",
-    "mocha": "^10.0.0",
-    "prettier": "^2.6.2",
-    "ts-node": "^10.7.0",
+    "mocha": "^10.2.0",
+    "prettier": "^2.8.7",
+    "ts-node": "^10.9.1",
     "tslint": "^6.1.1",
     "tslint-config-prettier": "^1.18.0",
     "tslint-plugin-prettier": "^2.3.0",
-    "typescript": "^4.6.4",
-    "webpack-cli": "^4.8.0"
+    "typescript": "^5.0.3",
+    "webpack-cli": "^5.0.1"
   }
 }

src/compiler.ts

@@ -298,13 +298,19 @@ export class NexeCompiler {
     const sentinel = Buffer.from('<nexe~~sentinel>')
 
     let vfsSize = 0
-    const streams = [binary, toStream(launchCode), this.bundle.toStream().pipe(new Transform({
-      transform: (chunk, _, cb) => {
-        vfsSize || this.bundle.finalize()
-        chunk && (vfsSize += chunk.length)
-        cb(null, chunk)
-      },
-    }))]
+    const streams = [
+      binary,
+      toStream(launchCode),
+      this.bundle.toStream().pipe(
+        new Transform({
+          transform: (chunk, _, cb) => {
+            vfsSize || this.bundle.finalize()
+            chunk && (vfsSize += chunk.length)
+            cb(null, chunk)
+          },
+        })
+      ),
+    ]
 
     let done = false
     return new MultiStream((cb) => {

src/fs/SnapshotZipFS.ts

@@ -2,14 +2,19 @@ import { Libzip } from '@yarnpkg/libzip'
 import {
   FakeFS,
   PortablePath,
-  ZipFS,
-  ZipOpenFS,
   RmdirOptions,
   OpendirOptions,
+  BasePortableFakeFS,
+  Dirent,
+  Dir,
+  MkdirOptions,
+  FSPath,
+  ppath,
+  npath,
+  Filename,
+  CustomDir,
 } from '@yarnpkg/fslib'
-import { BasePortableFakeFS, Dirent, MkdirOptions, Dir } from '@yarnpkg/fslib/lib/FakeFS'
-import { CustomDir } from '@yarnpkg/fslib/lib/algorithms/opendir'
-import { FSPath, ppath, npath, Filename } from '@yarnpkg/fslib/lib/path'
+import { ZipFS, ZipOpenFS } from '@yarnpkg/libzip'
 import { resolve, toNamespacedPath } from 'path'
 import { constants } from 'fs'
 
@@ -17,10 +22,17 @@ export type SnapshotZipFSOptions = {
   baseFs: FakeFS<PortablePath>
   libzip: Libzip | (() => Libzip)
   zipFs: ZipFS
+  root: string
 }
 
 const uniqBy = (arr: Array<string | Dirent>, pick: (...arg: any[]) => any) => {
-  return arr.filter((x, i, self) => i === self.findIndex((y) => pick(x) === pick(y)))
+  const seen = new Set()
+  return arr.filter((x) => {
+    const key = pick(x)
+    if (seen.has(key)) return false
+    seen.add(key)
+    return true
+  })
 }
 function uniqReaddir(arr: Array<string | Dirent>) {
   return uniqBy(arr, (s: string | Dirent) => (typeof s === 'string' ? s : s.name))
@@ -29,10 +41,14 @@ function uniqReaddir(arr: Array<string | Dirent>) {
 export class SnapshotZipFS extends BasePortableFakeFS {
   zipFs: ZipFS
   baseFs: FakeFS<PortablePath>
+  root: string
+  magic: number
   constructor(opts: SnapshotZipFSOptions) {
     super()
     this.zipFs = opts.zipFs
     this.baseFs = opts.baseFs
+    this.root = opts.root
+    this.magic = 0x2a << 24
   }
   private readonly fdMap: Map<number, [ZipFS, number]> = new Map()
   private nextFd = 3
@@ -74,46 +90,42 @@ export class SnapshotZipFS extends BasePortableFakeFS {
   }
 
   async realpathPromise(p: PortablePath) {
-    return await this.makeCallPromise(
-      p,
-      async () => {
-        return await this.baseFs.realpathPromise(p)
-      },
-      async (zipFs, { subPath }) => {
-        return await zipFs.realpathPromise(subPath)
-      }
-    )
+    return await this.realpathSync(p)
   }
 
   realpathSync(p: PortablePath) {
     return this.makeCallSync(
       p,
-      () => {
-        return this.baseFs.realpathSync(p)
-      },
+      () => this.baseFs.realpathSync(p),
       (zipFs, { subPath }) => {
-        return zipFs.realpathSync(subPath)
+        if (zipFs.lstatSync(subPath).isSymbolicLink()) {
+          return zipFs.realpathSync(subPath)
+        } else {
+          // return the original path in case it wasn't under /snapshot, e.g. if it was for a node module - otherwise the node module parent path is the wrong one (and other things resolve relative to that)
+          return p
+        }
       }
     )
   }
 
   findZip(p: PortablePath) {
     p = this.resolve(p)
+    const snapshotPP = npath.toPortablePath('/snapshot')
     const pathsToTry = Array.from(
       new Set([
         p,
         resolve('/snapshot', p),
         ppath.resolve(
-          npath.toPortablePath('/snapshot'),
+          snapshotPP,
           npath.toPortablePath(
-            npath.relative(npath.fromPortablePath(process.cwd()), npath.fromPortablePath(p))
+            npath.relative(npath.fromPortablePath(this.root), npath.fromPortablePath(p))
           )
         ),
         ppath.resolve(
-          npath.toPortablePath('/snapshot'),
+          snapshotPP,
           npath.toPortablePath(
             npath.relative(
-              toNamespacedPath(npath.fromPortablePath(process.cwd())),
+              toNamespacedPath(npath.fromPortablePath(this.root)),
               toNamespacedPath(npath.fromPortablePath(p))
             )
           )
@@ -236,9 +248,7 @@ export class SnapshotZipFS extends BasePortableFakeFS {
       async (zipFs, { subPath }) => {
         const fallbackPaths: Array<string | Dirent> = await fallback().catch(() => [])
         return Promise.resolve(
-          uniqReaddir(
-            (fallbackPaths as any[]).concat(await zipFs.readdirPromise(subPath, opts as any))
-          )
+          uniqReaddir(fallbackPaths.concat(await zipFs.readdirPromise(subPath, opts as any)))
         )
       },
       {
@@ -263,7 +273,7 @@ export class SnapshotZipFS extends BasePortableFakeFS {
         try {
           fallbackPaths = fallback()
         } catch (e) {}
-        return (fallbackPaths as any[]).concat(uniqReaddir(zipFs.readdirSync(subPath, opts as any)))
+        return fallbackPaths.concat(uniqReaddir(zipFs.readdirSync(subPath, opts as any)))
       },
       {
         requireSubpath: false,
@@ -288,7 +298,7 @@ export class SnapshotZipFS extends BasePortableFakeFS {
   }
 
   async opendirPromise(p: PortablePath, opts?: OpendirOptions) {
-    return Promise.resolve(this.zipFs.opendirSync(p, opts))
+    return this.opendirSync(p, opts)
   }
 
   opendirSync(p: PortablePath, opts?: OpendirOptions) {
@@ -371,6 +381,8 @@ export class SnapshotZipFS extends BasePortableFakeFS {
   unwatchFile = ZipOpenFS.prototype.unwatchFile
   utimesPromise = ZipOpenFS.prototype.utimesPromise
   utimesSync = ZipOpenFS.prototype.utimesSync
+  lutimesPromise = ZipOpenFS.prototype.utimesPromise
+  lutimesSync = ZipOpenFS.prototype.utimesSync
   watch = ZipOpenFS.prototype.watch
   watchFile = ZipOpenFS.prototype.watchFile
   writeFilePromise = ZipOpenFS.prototype.writeFilePromise

src/fs/patch.ts

@@ -1,8 +1,9 @@
-import { getLibzipSync } from '@yarnpkg/libzip'
-import { patchFs, npath, PosixFS, NodeFS, ZipFS } from '@yarnpkg/fslib'
+import { ZipFS, getLibzipSync } from '@yarnpkg/libzip'
+import { patchFs, npath, PosixFS, NodeFS } from '@yarnpkg/fslib'
 import { SnapshotZipFS } from './SnapshotZipFS'
 import * as assert from 'assert'
 import * as constants from 'constants'
+import { dirname } from 'path'
 
 export interface NexeHeader {
   blobPath: string
@@ -17,6 +18,7 @@ export interface NexeHeader {
 let originalFsMethods: any = null
 let lazyRestoreFs = () => {}
 const patches = (process as any).nexe.patches || {}
+const originalPatches = { ...patches }
 delete (process as any).nexe
 
 function shimFs(binary: NexeHeader, fs: typeof import('fs') = require('fs')) {
@@ -40,60 +42,65 @@ function shimFs(binary: NexeHeader, fs: typeof import('fs') = require('fs')) {
   )
   assert.equal(bytesRead, binary.layout.resourceSize)
 
-  const zipFs = new ZipFS(blob, {
-    libzip: getLibzipSync(),
-    readOnly: true,
-  })
+  const zipFs = new ZipFS(blob, { readOnly: true })
   const snapshotZipFS = new SnapshotZipFS({
     libzip: getLibzipSync(),
     zipFs,
     baseFs: nodeFs,
+    root: dirname(process.argv[0]),
   })
   const posixSnapshotZipFs = new PosixFS(snapshotZipFS)
   patchFs(fs, posixSnapshotZipFs)
-  const { readFileSync } = fs
 
   let log = (_: string) => true
-  let loggedManifest = false
   if ((process.env.DEBUG || '').toLowerCase().includes('nexe:require')) {
+    process.stderr.write(
+      // @ts-ignore
+      `[nexe] - FILES ${JSON.stringify(Array.from(zipFs.entries.keys()), null, 4)}\n`
+    )
+    process.stderr.write(
+      // @ts-ignore
+      `[nexe] - DIRECTORIES ${JSON.stringify(Array.from(zipFs.listings.keys()), null, 4)}\n`
+    )
     log = (text: string) => {
-      if (!loggedManifest) {
-        // TODO anything meaningful to log here?
-        loggedManifest = true
-      }
-      return process.stderr.write('[nexe] - ' + text + '\n')
+      return process.stderr.write(`[nexe] - ${text}\n`)
     }
   }
-  patches.internalModuleReadFile = function (this: any, original: any, ...args: any[]) {
+  function internalModuleReadFile(this: any, original: any, ...args: any[]) {
     log(`internalModuleReadFile ${args[0]}`)
     try {
-      return fs.readFileSync(args[0], 'utf-8')
+      return posixSnapshotZipFs.readFileSync(args[0], 'utf-8')
     } catch (e) {
-      return null
+      return ''
     }
   }
+  if (patches.internalModuleReadFile) {
+    patches.internalModuleReadFile = internalModuleReadFile
+  }
   let returningArray: boolean
   patches.internalModuleReadJSON = function (this: any, original: any, ...args: any[]) {
     if (returningArray == null) returningArray = Array.isArray(original.call(this, ''))
-    const res = patches.internalModuleReadFile.call(this, original, ...args)
+    const res = internalModuleReadFile.call(this, original, ...args)
     return returningArray && !Array.isArray(res)
-      ? [res, /"(main|name|type|exports|imports)"/.test(res)]
+      ? res === ''
+        ? []
+        : [res, /"(main|name|type|exports|imports)"/.test(res)]
       : res
   }
   patches.internalModuleStat = function (this: any, original: any, ...args: any[]) {
     log(`internalModuleStat ${args[0]}`)
     try {
-      const stat = fs.statSync(args[0])
+      const stat = posixSnapshotZipFs.statSync(args[0])
       if (stat.isDirectory()) return 1
       return 0
     } catch (e) {
       return -constants.ENOENT
     }
   }
-  // TODO restore patches in restoreFs
 
   lazyRestoreFs = () => {
     Object.assign(fs, originalFsMethods)
+    Object.assign(patches, originalPatches)
     lazyRestoreFs = () => {}
   }
 }

src/patches/boot-nexe.ts

@@ -1,15 +1,14 @@
 const fs = require('fs'),
   fd = fs.openSync(process.execPath, 'r'),
   stat = fs.statSync(process.execPath),
-  tailSize = Math.min(stat.size, 4096),
+  tailSize = Math.min(stat.size, 16000),
   tailWindow = Buffer.from(Array(tailSize))
 
 fs.readSync(fd, tailWindow, 0, tailSize, stat.size - tailSize)
 
 const footerPosition = tailWindow.indexOf('<nexe~~sentinel>')
-if (footerPosition === -1) {
-  process.stderr.write('Invalid Nexe binary')
-  process.exit(1)
+if (footerPosition == -1) {
+  throw 'Invalid Nexe binary'
 }
 
 const footer = tailWindow.slice(footerPosition, footerPosition + 32),

src/patches/third-party-main.ts

@@ -55,13 +55,22 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
   })
 
   const fileLines = file.contents.toString().split('\n')
-  fileLines.splice(
-    location.start.line,
-    0,
-    '{{ file("lib/fs/bootstrap.js") }}' +
-      '\n' +
-      (semverGt(version, '11.99') ? 'expandArgv1 = false;\n' : '')
-  )
+  if (semverGt(version, '19.99')) {
+    await compiler.replaceInFileAsync(
+      'lib/internal/modules/cjs/loader.js',
+      "'use strict';",
+      "'use strict';\n" + '{{ file("lib/fs/bootstrap.js") }}' + '\n'
+    )
+    fileLines.splice(location.start.line, 0, 'expandArgv1 = false;')
+  } else {
+    fileLines.splice(
+      location.start.line,
+      0,
+      '{{ file("lib/fs/bootstrap.js") }}' +
+        '\n' +
+        (semverGt(version, '11.99') ? 'expandArgv1 = false;\n' : '')
+    )
+  }
   file.contents = fileLines.join('\n')
 
   if (semverGt(version, '11.99')) {

src/steps/download.ts

@@ -58,7 +58,7 @@ export default async function downloadNode(compiler: NexeCompiler, next: () => P
     { sourceUrl, downloadOptions, build } = compiler.options,
     url = sourceUrl || `https://nodejs.org/dist/v${version}/node-v${version}.tar.gz`,
     step = log.step(
-      `Downloading ${build ? '' : 'pre-built '}Node.js${build ? `source from: ${url}` : ''}`
+      `Downloading ${build ? '' : 'pre-built '}Node.js${build ? ` source from: ${url}` : ''}`
     ),
     exeLocation = compiler.getNodeExecutableLocation(build ? undefined : target),
     downloadExists = await pathExistsAsync(build ? src : exeLocation)

tasks/docker.ts

@@ -58,17 +58,11 @@ export async function runDockerBuild(target: NexeTarget) {
       appendFileSync(outFilename, x.stderr)
       appendFileSync(outFilename, x.stdout)
     })
-    try {
-      const { body } = await got(
-        `https://transfer.sh/${Math.random().toString(36).substring(2)}.txt`,
-        {
-          body: await readFileAsync(outFilename),
-          method: 'PUT',
-        }
-      )
-      console.log('Posted docker log: ', body)
-    } catch (e) {
-      console.log('Error posting log', e)
-    }
+    await got(`https://transfer.sh/${Math.random().toString(36).substring(2)}.txt`, {
+      body: await readFileAsync(outFilename),
+      method: 'PUT',
+    })
+      .then((x) => console.log('Posted docker log: ', x.body))
+      .catch((e) => console.log('Error posting log', e))
   }
 }

test/integration/index.js

@@ -1,17 +1,22 @@
-const { mkdtemp } = require('fs/promises')
+const { mkdtemp, copyFile } = require('fs/promises')
 const os = require('os')
 const path = require('path')
 const rimraf = require('rimraf')
+const cp = require('child_process')
 
 async function runTests() {
-  const tempdir = await mkdtemp(path.join(os.tmpdir(), 'nexe-'))
+  const tempdir = await mkdtemp(path.join(os.tmpdir(), 'nexe-integration-tests-'))
+  const executable = path.join(tempdir, path.basename(process.argv[0]))
+  await copyFile(process.argv[0], executable)
   process.on('beforeExit', () => rimraf.sync(tempdir))
-  process.chdir(tempdir)
-  const Mocha = require('mocha')
-  const mocha = new Mocha()
-  mocha.addFile('test/integration/tests.integration-spec.js')
-  mocha.run((failures) => {
-    process.exitCode = failures ? 1 : 0
+  cp.spawn(executable,
+    [
+    path.join(tempdir, 'node_modules/mocha/bin/mocha.js'),
+      path.join(tempdir, 'test/integration/tests.integration-spec.js')
+    ],
+    { stdio: ['inherit', 'inherit', 'inherit', 'ipc'], cwd: tempdir }
+  ).on('exit', (code) => {
+    process.exitCode = code
   })
 }
 

test/integration/tests.integration-spec.js

@@ -1,6 +1,6 @@
 const { expect } = require('chai')
 const fs = require('fs')
-const { dirname, basename }= require('path')
+const { dirname, basename } = require('path')
 
 const files = { zipFs: 'test/integration/tests.integration-spec.js', real: 'real', real2: 'real2' }
 describe('SnapshotZipFS', () => {
@@ -88,7 +88,6 @@ describe('SnapshotZipFS', () => {
     it(`won't ftruncate the zipFs`, () => expect(() => fs.ftruncateSync(fd)).to.throw())
     it('closes file descriptors from the zipFs', () => expect(() => fs.closeSync(fd)).not.to.throw())
   })
-
   fd = null
   buffer = Buffer.alloc(8092)
   describe('real file descriptors', () => {
@@ -96,7 +95,7 @@ describe('SnapshotZipFS', () => {
       expect(() => (fd = fs.openSync(files.real2, 'a+'))).not.to.throw()
       expect(fd).to.not.be.null
     })
-    it('reads file descriptors from the real', () => {
+    it('reads file descriptors from the real fs', () => {
       expect(() => fs.readSync(fd, buffer)).not.to.throw()
       expect(buffer.toString()).to.include('SnapshotZipFS')
     })
@@ -104,9 +103,18 @@ describe('SnapshotZipFS', () => {
     it('fstats the real fs', () => expect(fs.fstatSync(fd).size).to.be.greaterThan(0))
     it('closes file descriptors from the real fs', () => expect(() => fs.closeSync(fd)).not.to.throw())
   })
-
   describe('path clashes', () => {
     it('delegates directory creation to the real fs', () => expect(() => fs.mkdirSync('test')).not.to.throw())
     it('delegates directory removal to the real fs', () => expect(() => fs.rmdirSync('test')).not.to.throw())
   })
+  describe('requiring modules', () => {
+    it('resolves modules from the zipFs', () => expect(() => require('resolve-dependencies')).not.to.throw())
+    it('resolves modules from the real fs', () => {
+      fs.mkdirSync('node_modules')
+      fs.mkdirSync('node_modules/fake-package')
+      fs.writeFileSync('node_modules/fake-package/package.json', '{ "name": "fake-package", "main": "some-file.js"}')
+      fs.writeFileSync('node_modules/fake-package/some-file.js', `module.exports = { fake: 'package' }`)
+      expect(require('fake-package')).to.eql({ fake: 'package' })
+    })
+  })
 })