fix: serialize shims/resources in upstream step

closes #677

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,23 @@
+---
+name: Bug Report
+about: Report a bug encountered while running Nexe
+labels: bug
+
+---
+<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!
+-->
+
+**What happened**:
+
+**What you expected to happen**:
+
+**How to reproduce it (as minimally and precisely as possible)**:
+
+**Anything else we need to know?**:
+
+**Environment**:
+- Platform(OS/Version):
+- Host Node Version:
+- Target Node Version:
+- Nexe version:
+- Python Version:

.github/ISSUE_TEMPLATE/feature.md

@@ -0,0 +1,11 @@
+---
+name: Feature Request
+about: Suggest an feature request
+labels: feature
+
+---
+<!-- Please only use this template for submitting feature requests -->
+
+**What would you like to be added**:
+
+**Why is this needed**:

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+**What this PR does / why we need it**:
+
+**Which issue(s) this PR fixes**:
+<!--
+*Automatically closes linked issue when PR is merged.
+Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
+_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
+-->
+Fixes #
+
+**Special notes for your reviewer**:

azure-pipelines.yml

@@ -1,6 +1,25 @@
+schedules:
+- cron: "0 8 * * *"
+  displayName: "Daily Build 1"
+  always: true
+  branches:
+    include:
+      - master
+- cron: "0 13 * * *"
+  displayName: "Daily Build 2"
+  always: true
+  branches:
+    include:
+      - master
+- cron: "0 19 * * *"
+  displayName: "Daily Build 3"
+  always: true
+  branches:
+    include:
+      - master
 jobs:
 - job: BaseBuild
-  timeoutInMinutes: 0
+  timeoutInMinutes: 180
   strategy:
     matrix:
       linux:
@@ -27,15 +46,23 @@ jobs:
       versionSpec: '10.x'
   - script: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 gcc-multilib g++-multilib
     condition: contains(variables['Agent.JobName'], 'linux')
-    displayName: Install Linux Build Dependencies
+    displayName: Install Linux build dependencies
   - script: choco install visualcpp-build-tools --version 14.0.25420.1 -fy && npm config set msvs_version 2015
     condition: contains(variables['Agent.JobName'], 'windows_2015')
-    displayName: Install Windows Build Dependencies
-  - script: choco install nasm -fy
-    displayName: Install Windows Assembler (NASM)
-    condition: contains(variables['Agent.JobName'], 'windows')
+    displayName: Install Windows build dependencies
   - script: npm install && npm run ci:build
     displayName: Build Nexe
     env:
       GITHUB_TOKEN: $(PersonalGithubToken)
       NEXE_VERBOSE: $(NEXE_VERBOSE)
+    condition: not(contains(variables['Agent.JobName'], 'windows'))
+  - powershell: |
+      iex (new-object net.webclient).downloadstring('https://get.scoop.sh')
+      scoop install nasm
+      npm install
+      npm run ci:build
+    displayName: Build Nexe (Windows)
+    condition: contains(variables['Agent.JobName'], 'windows')
+    env:
+      GITHUB_TOKEN: $(PersonalGithubToken)
+      NEXE_VERBOSE: $(NEXE_VERBOSE)

issue_template.md

@@ -1,19 +0,0 @@
-<!-- If you need help or think you have found a bug, please help us with your issue by entering the following information (otherwise you can delete this text): -->
-
-**Is this a BUG or a FEATURE REQUEST?:** 
-
-**What happened:**
-
-**What you expected to happen:**
-
-**How to reproduce it (as minimally and precisely as possible):**
-
-**Anything else we need to know?:**
-
-**Environment**
-
-- Platform(OS/Version):
-- Host Node Version:
-- Target Node Version:
-- Nexe version:
-- Python Version:

package.json

@@ -2,7 +2,7 @@
   "name": "nexe",
   "description": "Create a single executable out of your Node.js application",
   "license": "MIT",
-  "version": "3.3.0",
+  "version": "4.0.0-beta.3",
   "contributors": [
     "Craig Condon <craig.j.condon@gmail.com> (http://crcn.io)",
     "Jared Allard <jaredallard@outlook.com>",
@@ -39,22 +39,22 @@
     "chalk": "^2.4.2",
     "cherow": "1.6.9",
     "download": "^7.1.0",
-    "globby": "^9.2.0",
+    "globby": "^10.0.1",
     "got": "^9.6.0",
     "minimist": "^1.2.0",
     "mkdirp": "^0.5.1",
-    "multistream": "^2.1.1",
+    "multistream": "^4.0.0",
     "ora": "^3.4.0",
     "pify": "^4.0.1",
-    "resolve-dependencies": "^2.2.3",
-    "rimraf": "^2.6.3"
+    "resolve-dependencies": "^3.0.2",
+    "rimraf": "^3.0.0"
   },
   "devDependencies": {
-    "@types/chai": "^4.1.7",
+    "@types/chai": "^4.2.3",
     "@types/download": "^6.2.4",
-    "@types/execa": "^0.9.0",
+    "@types/execa": "^2.0.0",
     "@types/globby": "^9.1.0",
-    "@types/got": "^9.4.4",
+    "@types/got": "^9.6.7",
     "@types/minimist": "^1.2.0",
     "@types/mkdirp": "^0.5.2",
     "@types/mocha": "^5.2.7",
@@ -62,15 +62,15 @@
     "@types/ora": "^3.2.0",
     "@types/pify": "3.0.2",
     "@types/rimraf": "2.0.2",
-    "@types/semver": "^6.0.0",
+    "@types/semver": "^6.0.2",
     "chai": "^4.2.0",
-    "execa": "^1.0.0",
-    "mocha": "^6.1.4",
+    "execa": "^2.1.0",
+    "mocha": "^6.2.1",
     "prettier": "^1.18.2",
-    "ts-node": "^8.2.0",
-    "tslint": "^5.17.0",
+    "ts-node": "^8.4.1",
+    "tslint": "^5.20.0",
     "tslint-config-prettier": "^1.18.0",
     "tslint-plugin-prettier": "^2.0.1",
-    "typescript": "^3.5.1"
+    "typescript": "^3.6.3"
   }
 }

src/compiler.ts

@@ -15,7 +15,7 @@ import {
 } from './util'
 import { NexeOptions, version } from './options'
 import { NexeTarget } from './target'
-import combineStreams = require('multistream')
+import MultiStream = require('multistream')
 import { Bundle, toStream } from './fs/bundle'
 
 const isBsd = Boolean(~process.platform.indexOf('bsd'))
@@ -136,8 +136,8 @@ export class NexeCompiler {
   }
 
   @bound
-  addResource(file: string, content?: Buffer | string) {
-    return this.bundle.addResource(file, content)
+  addResource(absoluteFileName: string, content?: Buffer | string) {
+    return this.bundle.addResource(absoluteFileName, content)
   }
 
   get binaryConfiguration() {
@@ -237,7 +237,7 @@ export class NexeCompiler {
   }
 
   private _configureAsync() {
-    if (isWindows && semverGt(this.target.version, '11.999')) {
+    if (isWindows && semverGt(this.target.version, '10.15.99')) {
       return Promise.resolve()
     }
     return this._runBuildCommandAsync(this.env.PYTHON || 'python', [
@@ -309,7 +309,7 @@ export class NexeCompiler {
     const lengths = Buffer.from(Array(16))
     lengths.writeDoubleLE(codeSize, 0)
     lengths.writeDoubleLE(this.bundle.blobSize, 8)
-    return combineStreams([
+    return new (MultiStream as any)([
       binary,
       toStream(startup),
       this.bundle.toStream(),

src/fs/bundle.ts

@@ -1,7 +1,7 @@
 import { stat as getStat, Stats, createReadStream } from 'fs'
 import { relative } from 'path'
 import { Readable } from 'stream'
-import combineStreams = require('multistream')
+import MultiStream = require('multistream')
 
 const stat = (file: string): Promise<Stats> => {
   return new Promise((resolve, reject) => {
@@ -14,9 +14,12 @@ function makeRelative(cwd: string, path: string) {
 }
 
 export function toStream(content: Buffer | string) {
-  const readable = new Readable({ read() {} })
-  readable.push(content)
-  readable.push(null)
+  const readable = new Readable({
+    read() {
+      this.push(content)
+      this.push(null)
+    }
+  })
   return readable
 }
 
@@ -61,7 +64,7 @@ export class Bundle {
   }
 
   toStream() {
-    return combineStreams(this.streams)
+    return new (MultiStream as any)(this.streams)
   }
 
   toJSON() {

src/fs/patch.ts

@@ -75,21 +75,34 @@ function shimFs(binary: NexeBinary, fs: any = require('fs')) {
     }
   }
 
-  const createStat = function(extensions: any) {
-    return Object.assign(new fs.Stats(), binary.layout.stat, statTime(), extensions)
+  let BigInt: Function
+  try {
+    BigInt = eval('BigInt')
+  } catch (ignored) {}
+
+  const createStat = function(extensions: any, options: any) {
+    const stat = Object.assign(new fs.Stats(), binary.layout.stat, statTime(), extensions)
+    if (options && options.bigint && BigInt) {
+      for (const k in stat) {
+        if (Object.prototype.hasOwnProperty.call(stat, k) && typeof stat[k] === 'number') {
+          stat[k] = BigInt(stat[k])
+        }
+      }
+    }
+    return stat
   }
 
-  const ownStat = function(filepath: any) {
+  const ownStat = function(filepath: any, options: any) {
     setupManifest()
     const key = getKey(filepath)
     if (directories[key]) {
       let mode = binary.layout.stat.mode
       mode |= fs.constants.S_IFDIR
       mode &= ~fs.constants.S_IFREG
-      return createStat({ mode, size: 0 })
+      return createStat({ mode, size: 0 }, options)
     }
     if (manifest[key]) {
-      return createStat({ size: manifest[key][1] })
+      return createStat({ size: manifest[key][1] }, options)
     }
   }
 
@@ -250,15 +263,21 @@ function shimFs(binary: NexeBinary, fs: any = require('fs')) {
       originalFsMethods.closeSync(fd)
       return encoding ? result.toString(encoding) : result
     },
-    statSync: function statSync(filepath: string | Buffer) {
-      const stat = ownStat(filepath)
+    statSync: function statSync(filepath: string | Buffer, options: any) {
+      const stat = ownStat(filepath, options)
       if (stat) {
         return stat
       }
       return originalFsMethods.statSync.apply(fs, arguments)
     },
-    stat: function stat(filepath: string | Buffer, callback: any) {
-      const stat = ownStat(filepath)
+    stat: function stat(filepath: string | Buffer, options: any, callback: any) {
+      let stat: any
+      if (typeof options === 'function') {
+        callback = options
+        stat = ownStat(filepath, null)
+      } else {
+        stat = ownStat(filepath, options)
+      }
       if (stat) {
         process.nextTick(() => {
           callback(null, stat)

src/options.ts

@@ -5,7 +5,7 @@ import { basename, extname, join, isAbsolute, resolve } from 'path'
 import { getTarget, NexeTarget } from './target'
 import { EOL, homedir } from 'os'
 import chalk from 'chalk'
-import { resolveFileNameSync } from 'resolve-dependencies'
+import { resolveSync } from 'resolve-dependencies'
 const caw = require('caw')
 const c = process.platform === 'win32' ? chalk.constructor({ enabled: false }) : chalk
 
@@ -208,17 +208,17 @@ export function resolveEntry(
   }
   if (input) {
     const inputPath = padRelative(input)
-    result = resolveFileNameSync(cwd, inputPath)
+    result = resolveSync(cwd, inputPath)
   }
   if (isEntryFile(maybeEntry) && (!result || !result.absPath)) {
     const inputPath = padRelative(maybeEntry)
-    result = resolveFileNameSync(cwd, inputPath)
+    result = resolveSync(cwd, inputPath)
   }
   if (!process.stdin.isTTY && (!result || !result.absPath) && bundle === defaults.bundle) {
     return STDIN_FLAG
   }
   if (!result || !result.absPath) {
-    result = resolveFileNameSync(cwd, '.')
+    result = resolveSync(cwd, '.')
   }
   if (!result.absPath) {
     throw new NexeError(`Entry file "${input || ''}" not found!`)

src/patches/disable-node-cli.ts

@@ -7,13 +7,9 @@ export default async function disableNodeCli(compiler: NexeCompiler, next: () =>
   }
 
   if (semverGt(compiler.target.version, '11.6.0')) {
-    await compiler.replaceInFileAsync(
-      'src/node.cc',
-      /(?<!int )ProcessGlobalArgs\(/g,
-      '0;//ProcessGlobalArgs('
-    )
+    await compiler.replaceInFileAsync('src/node.cc', /(?<!int )ProcessGlobalArgs\(argv/g, '0;//$&')
   } else if (semverGt(compiler.target.version, '10.9')) {
-    await compiler.replaceInFileAsync('src/node.cc', /(?<!void )ProcessArgv\(/g, '//ProcessArgv(')
+    await compiler.replaceInFileAsync('src/node.cc', /(?<!void )ProcessArgv\(argv/g, '//$&')
   } else if (semverGt(compiler.target.version, '9.999')) {
     await compiler.replaceInFileAsync(
       'src/node.cc',

src/patches/third-party-main.ts

@@ -55,7 +55,13 @@ export default async function main(compiler: NexeCompiler, next: () => Promise<v
   })
 
   const fileLines = file.contents.split('\n')
-  fileLines.splice(location.start.line, 0, '{{replace:lib/fs/bootstrap.js}}' + '\n')
+  fileLines.splice(
+    location.start.line,
+    0,
+    '{{replace:lib/fs/bootstrap.js}}' +
+      '\n' +
+      (semverGt(version, '11.99') ? 'expandArgv1 = false;\n' : '')
+  )
   file.contents = fileLines.join('\n')
 
   if (semverGt(version, '11.99')) {

src/releases.ts

@@ -2,7 +2,7 @@ import got = require('got')
 import { platforms, architectures, NexeTarget, getTarget, targetsEqual } from './target'
 export { NexeTarget }
 
-const versionsToSkip = [0, 1, 2, 3, 4, 5, 7, 9, 11]
+const versionsToSkip = [0, 1, 2, 3, 4, 5, 7, 9, 11, 13, 15, 17, 19, 21, 22]
 export interface GitAsset {
   name: string
   url: string
@@ -25,8 +25,13 @@ async function getJson<T>(url: string, options?: any) {
 }
 
 function isBuildableVersion(version: string) {
-  const major = Number(version.split('.')[0])
-  return !versionsToSkip.includes(major) || version === '4.8.4'
+  if (version === '12.11.0') {
+    return false
+  }
+  if (version === '4.9.1') {
+    return true
+  }
+  return !versionsToSkip.includes(Number(version.split('.')[0]))
 }
 
 export function getLatestGitRelease(options?: any) {

src/steps/bundle.ts

@@ -1,7 +1,7 @@
 import { NexeCompiler, NexeError } from '../compiler'
 import { resolve, relative } from 'path'
 import { each } from '@calebboyd/semaphore'
-import resolveFiles, { resolveFileNameSync } from 'resolve-dependencies'
+import resolveFiles, { resolveSync } from 'resolve-dependencies'
 import { dequote, STDIN_FLAG, semverGt } from '../util'
 import { Readable } from 'stream'
 
@@ -54,7 +54,7 @@ export default async function bundle(compiler: NexeCompiler, next: any) {
   }
 
   if (input === STDIN_FLAG) {
-    const maybeInput = resolveFileNameSync(cwd, '.')
+    const maybeInput = resolveSync(cwd, '.')
     if (!maybeInput || !maybeInput.absPath) {
       throw new NexeError('No valid input detected')
     }
@@ -65,7 +65,7 @@ export default async function bundle(compiler: NexeCompiler, next: any) {
   const { files, warnings } = await resolveFiles(
     input,
     ...Object.keys(compiler.bundle.index).filter(x => x.endsWith('.js')),
-    { cwd, expand: true, loadContent: false }
+    { cwd, expand: 'variable', loadContent: false }
   )
 
   if (

src/steps/resource.ts

@@ -1,5 +1,5 @@
 import { isDirectoryAsync, each } from '../util'
-import globs from 'globby'
+import * as globs from 'globby'
 import { NexeCompiler } from '../compiler'
 
 export default async function resource(compiler: NexeCompiler, next: () => Promise<any>) {

src/steps/shim.ts

@@ -1,7 +1,9 @@
 import { NexeCompiler } from '../compiler'
 import { wrap } from '../util'
 
-export default function(compiler: NexeCompiler, next: () => Promise<void>) {
+export default async function(compiler: NexeCompiler, next: () => Promise<void>) {
+  await next()
+
   compiler.shims.push(
     wrap(
       `process.__nexe = ${JSON.stringify(compiler.binaryConfiguration)};\n` +
@@ -32,6 +34,4 @@ export default function(compiler: NexeCompiler, next: () => Promise<void>) {
       }
     `)
   )
-
-  return next()
 }

src/target.ts

@@ -1,8 +1,8 @@
 export type NodePlatform = 'windows' | 'mac' | 'alpine' | 'linux'
-export type NodeArch = 'x86' | 'x64' | 'arm7l' | 'arm6l' | 'arm64'
+export type NodeArch = 'x86' | 'x64' | 'arm' | 'arm64'
 
 const platforms: NodePlatform[] = ['windows', 'mac', 'alpine', 'linux'],
-  architectures: NodeArch[] = ['x86', 'x64', 'arm7l', 'arm6l', 'arm64']
+  architectures: NodeArch[] = ['x86', 'x64', 'arm', 'arm64']
 
 export { platforms, architectures }
 
@@ -26,12 +26,12 @@ const prettyPlatform: { [key: string]: NodePlatform } = {
 
 const prettyArch: { [key: string]: NodeArch } = {
   x86: 'x86',
-  arm6: 'arm6l',
+  arm6: 'arm',
   arm64: 'arm64',
-  arm6l: 'arm6l',
-  arm: 'arm7l',
-  arm7: 'arm7l',
-  arm7l: 'arm7l',
+  arm6l: 'arm',
+  arm: 'arm',
+  arm7: 'arm',
+  arm7l: 'arm',
   amd64: 'x64',
   ia32: 'x86',
   x32: 'x86',

tasks/build.ts

@@ -4,6 +4,7 @@ import { runDockerBuild } from './docker'
 import { getTarget, targetsEqual, NexeTarget } from '../lib/target'
 import { pathExistsAsync, readFileAsync, execFileAsync, semverGt } from '../lib/util'
 import got = require('got')
+import { cpus } from 'os'
 
 const env = process.env,
   isPullRequest = env.BUILD_REASON === 'PullRequest',
@@ -60,6 +61,7 @@ async function build() {
       build: true,
       verbose: Boolean(env.NEXE_VERBOSE!),
       target,
+      make: ['-j' + cpus().length],
       output
     }
   console.log('Building: ' + target + ' on ' + buildHost)
@@ -80,6 +82,7 @@ async function build() {
       unbuiltReleases = await getUnBuiltReleases({ headers })
     if (!unbuiltReleases.some(x => targetsEqual(x, target!))) {
       console.log(`${target} already uploaded.`)
+      process.exit(0)
       return
     }
     await got(gitRelease.upload_url.split('{')[0], {

tasks/docker.ts

@@ -46,10 +46,10 @@ export async function runDockerBuild(target: NexeTarget) {
   let output: any = []
 
   try {
-    output.push(await execa.shell(`docker build -t nexe-docker .`))
-    output.push(await execa.shell(`docker run -d --name nexe nexe-docker sh`))
-    output.push(await execa.shell(`docker cp nexe:/out out`))
-    output.push(await execa.shell(`docker rm nexe`))
+    output.push(await execa(`docker build -t nexe-docker .`, { shell: true }))
+    output.push(await execa(`docker run -d --name nexe nexe-docker sh`, { shell: true }))
+    output.push(await execa(`docker cp nexe:/out out`, { shell: true }))
+    output.push(await execa(`docker rm nexe`, { shell: true }))
   } catch (e) {
     console.log('Error running docker')
     appendFileSync(outFilename, e.message)