replace got with axios, and add macOS sign support with pr #1049

package.json

@@ -2,7 +2,7 @@
   "name": "nexe",
   "description": "Create a single executable out of your Node.js application",
   "license": "MIT",
-  "version": "5.0.0-beta.4",
+  "version": "5.0.0-beta.5",
   "contributors": [
     "Craig Condon <craig.j.condon@gmail.com> (http://crcn.io)",
     "Jared Allard <jaredallard@outlook.com>",
@@ -49,11 +49,10 @@
     "@yarnpkg/libzip": "^3.0.0-rc.43",
     "app-builder": "^7.0.4",
     "archiver": "^5.3.1",
+    "axios": "^1.13.2",
     "caw": "^2.0.1",
     "chalk": "^2.4.2",
-    "download": "^8.0.0",
     "globby": "^11.0.2",
-    "got": "^12.6.0",
     "meriyah": "^4.3.5",
     "minimist": "^1.2.8",
     "mkdirp": "^1.0.4",
@@ -62,12 +61,12 @@
     "resolve-dependencies": "^6.0.9",
     "rimraf": "^3.0.2",
     "run-script-os": "^1.1.6",
+    "tar": "^7.5.2",
     "webpack-config-prefabs": "0.0.5"
   },
   "devDependencies": {
     "@types/archiver": "^5.3.2",
     "@types/chai": "^4",
-    "@types/download": "^8",
     "@types/globby": "^9",
     "@types/lodash": "^4.14.192",
     "@types/minimist": "^1.2.2",

src/patches/boot-nexe.ts

@@ -1,20 +1,42 @@
 const fs = require('fs'),
   fd = fs.openSync(process.execPath, 'r'),
-  stat = fs.statSync(process.execPath),
+  stat = fs.statSync(fd),
   tailSize = Math.min(stat.size, 16000),
-  tailWindow = Buffer.from(Array(tailSize))
+  tailWindow = Buffer.alloc(tailSize),
+  match = '<nexe' + '~~sentinel>',
+  matchLength = match.length,
+  lastBuffer = Buffer.alloc(matchLength + 32)
 
-fs.readSync(fd, tailWindow, 0, tailSize, stat.size - tailSize)
+let offset = stat.size,
+  footerPosition = -1,
+  footerPositionOffset = 0,
+  footer: Buffer
+
+while (true) {
+  const bytesRead = fs.readSync(fd, tailWindow, 0, tailSize, offset - tailSize)
+  if (bytesRead === 0) break
+
+  const combinedBuffers = Buffer.concat([tailWindow, lastBuffer])
+  footerPosition = combinedBuffers.indexOf(match)
+
+  if (footerPosition > -1) {
+    footer = combinedBuffers.slice(footerPosition, footerPosition + 32)
+    break
+  }
+
+  if (offset < 0) break
+
+  tailWindow.copy(lastBuffer)
+  offset = offset - bytesRead
+}
 
-const footerPosition = tailWindow.indexOf('<nexe~~sentinel>')
 if (footerPosition == -1) {
   throw 'Invalid Nexe binary'
 }
 
-const footer = tailWindow.slice(footerPosition, footerPosition + 32),
-  contentSize = footer.readDoubleLE(16),
-  resourceSize = footer.readDoubleLE(24),
-  contentStart = stat.size - tailSize + footerPosition - resourceSize - contentSize,
+const contentSize = footer!.readDoubleLE(16),
+  resourceSize = footer!.readDoubleLE(24),
+  contentStart = offset - tailSize + footerPosition - resourceSize - contentSize,
   resourceStart = contentStart + contentSize
 
 Object.defineProperty(
@@ -48,10 +70,10 @@ Object.defineProperty(
   })()
 )
 
-const contentBuffer = Buffer.from(Array(contentSize)),
+const contentBuffer = Buffer.alloc(contentSize),
   Module = require('module')
 
 fs.readSync(fd, contentBuffer, 0, contentSize, contentStart)
 fs.closeSync(fd)
 
-new Module(process.execPath, null)._compile(contentBuffer.toString(), process.execPath)
+new Module(process.execPath, null)._compile(contentBuffer.slice(1).toString(), process.execPath)

src/patches/third-party-main.ts

@@ -55,7 +55,7 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
   })
 
   const fileLines = file.contents.toString().split('\n')
-  if (semverGt(version, '19.99')) {
+  if (semverGt(version, '18.16.99')) {
     await compiler.replaceInFileAsync(
       'lib/internal/modules/cjs/loader.js',
       "'use strict';",

src/releases.ts

@@ -1,4 +1,4 @@
-import got = require('got')
+import axios from 'axios';
 import { platforms, architectures, NexeTarget, getTarget, targetsEqual } from './target'
 export { NexeTarget }
 
@@ -20,8 +20,9 @@ interface NodeRelease {
   version: string
 }
 
-async function getJson<T>(url: string, options?: any) {
-  return JSON.parse((await got(url, options)).body) as T
+async function getJson<T>(url: string, options?: any): Promise<T> {
+  const response = await axios.get(url, options);
+  return response.data as T;
 }
 
 function isBuildableVersion(version: string) {
@@ -31,7 +32,7 @@ function isBuildableVersion(version: string) {
   return !versionsToSkip.includes(Number(version.split('.')[0]))
 }
 
-export function getLatestGitRelease(options?: any) {
+export function getLatestGitRelease(options?: any): Promise<GitRelease> {
   return getJson<GitRelease>('https://api.github.com/repos/nexe/nexe/releases/latest', options)
 }
 

src/steps/cli.ts

@@ -1,8 +1,10 @@
+import { platform } from 'os'
 import { dirname, normalize, relative, resolve } from 'path'
-import { createWriteStream, chmodSync, statSync } from 'fs'
+import { createWriteStream, chmodSync, statSync, readFileSync, writeFileSync } from 'fs'
 import { NexeCompiler } from '../compiler'
 import { NexeTarget } from '../target'
 import { STDIN_FLAG } from '../util'
+import { patchMachOExecutable } from './mach-o'
 import mkdirp = require('mkdirp')
 
 /**
@@ -41,6 +43,11 @@ export default async function cli(compiler: NexeCompiler, next: () => Promise<vo
             ),
             outputFileLogOutput = relative(process.cwd(), output)
 
+          if (platform() === 'darwin') {
+            step.log('Preparing binary for macOS signing.')
+            writeFileSync(output, patchMachOExecutable(readFileSync(output)))
+          }
+
           chmodSync(output, mode.toString(8).slice(-3))
           step.log(
             `Entry: '${

src/steps/download.ts

@@ -1,25 +1,90 @@
-import download = require('download')
+import axios, { AxiosResponse } from 'axios'
 import { pathExistsAsync } from '../util'
 import { LogStep } from '../logger'
 import { IncomingMessage } from 'http'
 import { NexeCompiler, NexeError } from '../compiler'
 import { dirname } from 'path'
+import { createWriteStream } from 'fs'
+import { pipeline } from 'stream/promises'
+import { createBrotliDecompress, createGunzip, createInflate } from 'zlib'
+import tar from 'tar'
+import fs from 'fs/promises'
 
-function fetchNodeSourceAsync(dest: string, url: string, step: LogStep, options = {}) {
+async function downloadWithProgress(url: string, dest: string, options: any = {}, step?: LogStep): Promise<void> {
+  const response = await axios({
+    url,
+    method: 'GET',
+    responseType: 'stream',
+    ...options
+  })
+
+  const total = parseInt(response.headers['content-length'] || '0', 10)
+  let current = 0
+
+  // Create write stream
+  const writer = createWriteStream(dest)
+
+  // Handle progress
+  response.data.on('data', (chunk: Buffer) => {
+    current += chunk.length
+    if (step && total > 0) {
+      step.modify(`Downloading...${((current / total) * 100).toFixed()}%`)
+    }
+  })
+
+  // Pipe the response to file
+  await pipeline(response.data, writer)
+
+  if (step) {
+    step.log(`Download completed: ${dest}`)
+  }
+}
+
+async function fetchNodeSourceAsync(dest: string, url: string, step: LogStep, options = {}) {
   const setText = (p: number) => step.modify(`Downloading Node: ${p.toFixed()}%...`)
-  return download(url, dest, Object.assign(options, { extract: true, strip: 1 }))
-    .on('response', (res: IncomingMessage) => {
-      const total = +res.headers['content-length']!
-      let current = 0
-      res.on('data', (data) => {
-        current += data.length
-        setText((current / total) * 100)
-        if (current === total) {
-          step.log('Extracting Node...')
-        }
-      })
+
+  // Download the file first
+  const tempFile = dest + '.tar.gz'
+  const response = await axios({
+    url,
+    method: 'GET',
+    responseType: 'stream',
+    ...options
+  })
+
+  const total = parseInt(response.headers['content-length'] || '0', 10)
+  let current = 0
+
+  // Create write stream for the temp file
+  const writer = createWriteStream(tempFile)
+
+  // Track progress
+  response.data.on('data', (chunk: Buffer) => {
+    current += chunk.length
+    if (total > 0) {
+      setText((current / total) * 100)
+    }
+  })
+
+  // Wait for download to complete
+  await pipeline(response.data, writer)
+
+  step.log('Extracting Node...')
+
+  // Extract the tar.gz file
+  await pipeline(
+    createReadStream(tempFile),
+    createGunzip(),
+    tar.extract({
+      cwd: dest,
+      strip: 1
     })
-    .then(() => step.log(`Node source extracted to: ${dest}`))
+  )
+
+  // Clean up temp file
+  await fs.unlink(tempFile)
+
+  step.log(`Node source extracted to: ${dest}`)
 }
 
 async function fetchPrebuiltBinary(compiler: NexeCompiler, step: any) {
@@ -27,22 +92,12 @@ async function fetchPrebuiltBinary(compiler: NexeCompiler, step: any) {
     filename = compiler.getNodeExecutableLocation(target)
 
   try {
-    await download(remoteAsset, dirname(filename), compiler.options.downloadOptions).on(
-      'response',
-      (res: IncomingMessage) => {
-        const total = +res.headers['content-length']!
-        let current = 0
-        res.on('data', (data) => {
-          current += data.length
-          step!.modify(`Downloading...${((current / total) * 100).toFixed()}%`)
-        })
-      }
-    )
+    await downloadWithProgress(remoteAsset, filename, compiler.options.downloadOptions, step)
   } catch (e: any) {
-    if (e.statusCode === 404) {
+    if (e.response?.status === 404) {
       throw new NexeError(`${remoteAsset} is not available, create it using the --build flag`)
     } else {
-      throw new NexeError('Error downloading prebuilt binary: ' + e)
+      throw new NexeError('Error downloading prebuilt binary: ' + e.message)
     }
   }
 }
@@ -76,3 +131,8 @@ export default async function downloadNode(compiler: NexeCompiler, next: () => P
 
   return next()
 }
+
+// Helper function to create a readable stream
+function createReadStream(path: string) {
+  return require('fs').createReadStream(path)
+}

src/steps/mach-o.ts

@@ -0,0 +1,84 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2021 Vercel, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * This code is originally from the [pkg](https://github.com/vercel/pkg).
+ */
+
+function parseCStr(buf: Buffer) {
+  for (let i = 0; i < buf.length; i += 1) {
+    if (buf[i] === 0) {
+      return buf.slice(0, i).toString()
+    }
+  }
+}
+
+function patchCommand(type: number, buf: Buffer, file: Buffer) {
+  // segment_64
+  if (type === 0x19) {
+    const name = parseCStr(buf.slice(0, 16))
+
+    if (name === '__LINKEDIT') {
+      const fileoff = buf.readBigUInt64LE(32)
+      const vmsizePatched = BigInt(file.length) - fileoff
+      const filesizePatched = vmsizePatched
+
+      buf.writeBigUInt64LE(vmsizePatched, 24)
+      buf.writeBigUInt64LE(filesizePatched, 40)
+    }
+  }
+
+  // symtab
+  if (type === 0x2) {
+    const stroff = buf.readUInt32LE(8)
+    const strsizePatched = file.length - stroff
+
+    buf.writeUInt32LE(strsizePatched, 12)
+  }
+}
+
+function patchMachOExecutable(file: Buffer) {
+  const align = 8
+  const hsize = 32
+
+  const ncmds = file.readUInt32LE(16)
+  const buf = file.slice(hsize)
+
+  for (let offset = 0, i = 0; i < ncmds; i += 1) {
+    const type = buf.readUInt32LE(offset)
+
+    offset += 4
+    const size = buf.readUInt32LE(offset) - 8
+
+    offset += 4
+    patchCommand(type, buf.slice(offset, offset + size), file)
+
+    offset += size
+    if (offset & align) {
+      offset += align - (offset & align)
+    }
+  }
+
+  return file
+}
+
+export { patchMachOExecutable }

tasks/asset-build.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const { resolve } = require('path')
+const resolvePath = (path) => resolve(__dirname, '..', path)
+
+async function compile() {
+  const executableSuffix = require('os').platform().startsWith('win') ? '.exe' : ''
+  const builtExecutablePath = require('path').join(process.env.NEXE_TMP, process.versions.node, `out/Release/node${executableSuffix}`)
+  try {
+    require('fs').unlinkSync(builtExecutablePath)
+  } catch(e) {}
+  const nexe = require('..')
+  console.error('Building asset')
+  return nexe.compile({
+    loglevel: 'verbose',
+    python: process.env.PYTHON || 'python',
+    mangle: false,
+    build: true,
+    output: process.env.NEXE_ASSET || `nexe-asset${executableSuffix}`,
+    input: resolvePath('test/asset-build-input.js'),
+    configure: process.env.MUSL_BUILD ? ['--fully-static'] : [],
+    temp: process.env.NEXE_TMP,
+  })
+}
+
+if (!('MAKEFLAGS' in process.env)) {
+  process.env.MAKEFLAGS = `-j${require('os').cpus().length + 1}`
+}
+
+compile()

tasks/asset-test-build.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const { resolve } = require('path')
+const resolvePath = (path) => resolve(__dirname, '..', path)
+
+async function build() {
+  const nexe = require('..')
+  console.error('Doing test build using asset')
+  if (!require('fs').existsSync(process.env.NEXE_ASSET)) {
+    console.error(`Pre-built asset not found at ${process.env.NEXE_ASSET}`)
+    process.exitCode = 1
+    return
+  }
+  return nexe.compile({
+    loglevel: 'verbose',
+    python: process.env.PYTHON || 'python',
+    build: true,
+    output: resolvePath('integration-tests'),
+    input: resolvePath('test/integration/index.js'),
+    resources: [resolvePath('test/integration'), resolvePath('node_modules')],
+    asset: process.env.NEXE_ASSET,
+  })
+}
+
+build()

tasks/build.ts

@@ -3,7 +3,8 @@ import { getUnBuiltReleases, getLatestGitRelease } from '../lib/releases'
 import { runDockerBuild } from './docker'
 import { getTarget, targetsEqual, NexeTarget } from '../lib/target'
 import { pathExistsAsync, readFileAsync, execFileAsync, semverGt } from '../lib/util'
-import got from 'got'
+import axios from 'axios'
+import FormData from 'form-data'
 import { cpus } from 'os'
 
 const env = process.env,
@@ -85,15 +86,23 @@ async function build() {
       process.exit(0)
       return
     }
-    await got(gitRelease.upload_url.split('{')[0], {
-      searchParams: { name: target.toString() },
-      body: await readFileAsync(output),
+
+    // Create form data for file upload
+    const formData = new FormData();
+    formData.append('file', await readFileAsync(output), {
+      filename: target.toString(),
+      contentType: 'application/octet-stream'
+    });
+
+    const uploadUrl = gitRelease.upload_url.split('{')[0];
+    await axios.post(uploadUrl, formData, {
+      params: { name: target.toString() },
       headers: {
         ...headers,
-        'Content-Type': 'application/octet-stream',
+        ...formData.getHeaders(),
       },
     }).catch((reason) => {
-      console.log(reason && reason.response && reason.response.body)
+      console.log(reason && reason.response && reason.response.data)
       throw reason
     })
     console.log(target + ' uploaded.')

tasks/docker.ts

@@ -1,6 +1,6 @@
 import { NexeTarget, architectures } from '../lib/target'
 import { writeFileAsync, readFileAsync } from '../lib/util'
-import got from 'got'
+import axios from 'axios'
 import execa = require('execa')
 import { appendFileSync } from 'fs'
 
@@ -58,11 +58,20 @@ export async function runDockerBuild(target: NexeTarget) {
       appendFileSync(outFilename, x.stderr)
       appendFileSync(outFilename, x.stdout)
     })
-    await got(`https://transfer.sh/${Math.random().toString(36).substring(2)}.txt`, {
-      body: await readFileAsync(outFilename),
-      method: 'PUT',
-    })
-      .then((x) => console.log('Posted docker log: ', x.body))
-      .catch((e) => console.log('Error posting log', e))
+
+    try {
+      const response = await axios.put(
+        `https://transfer.sh/${Math.random().toString(36).substring(2)}.txt`,
+        await readFileAsync(outFilename),
+        {
+          headers: {
+            'Content-Type': 'application/octet-stream',
+          }
+        }
+      )
+      console.log('Posted docker log: ', response.data)
+    } catch (e: any) {
+      console.log('Error posting log', e)
+    }
   }
 }

test/integration/index.js

@@ -1,17 +1,22 @@
-const { mkdtemp, copyFile } = require('fs/promises')
+const { mkdtemp, copyFile, realpath } = require('fs/promises')
 const os = require('os')
 const path = require('path')
 const rimraf = require('rimraf')
 const cp = require('child_process')
 
 async function runTests() {
-  const tempdir = await mkdtemp(path.join(os.tmpdir(), 'nexe-integration-tests-'))
+  const tempdir = await realpath(await mkdtemp(path.join(os.tmpdir(), 'nexe-integration-tests-')))
   const secondTempdir = await mkdtemp(path.join(os.tmpdir(), 'nexe-integration-tests-without-executable-'))
   const executable = path.join(tempdir, path.basename(process.argv[0]))
   await copyFile(process.argv[0], executable)
   process.on('beforeExit', () => {
-    rimraf.sync(tempdir)
-    rimraf.sync(secondTempdir)
+    try {
+      rimraf.sync(tempdir)
+      rimraf.sync(secondTempdir)
+    } catch(e) {
+      console.error(`Ignoring error during integration test cleanup of "${tempdir}".`)
+      console.error(`Error: ${e}`)
+    }
   })
   console.error('Running integration tests with the binary in the current working directory.')
   spawnExecutable(tempdir, () => {