diff --git a/README.md b/README.md index dc5d0b9..74ca880 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ For examples, see the built in patches: [src/patches](src/patches). - Quickly perform a replace in a file within the downloaded Node.js source. The rest arguments are passed along to `String.prototype.replace` - `readFileAsync(filename: string): Promise` - Access (or create) a file within the downloaded Node.js source. - - `addResource(filename: string, contents: Buffer): void` + - `addResource(filename: string, contents: Buffer): Promise` - Add a resource to the nexe bundle - `files: NexeFile[]` - The cache of the currently read, modified, or created files within the downloaded Node.js source. @@ -216,9 +216,7 @@ Any modifications made to `NexeFile#contents` will be maintained in the cache _w ## Native Modules -TODO - -- [ ] Implement support for `node-pre-gyp#find`. +In order to use native modules, the native binaries must be shipped alongside the binary generated by nexe. ## Contributing @@ -226,20 +224,10 @@ Building ``` $ git clone git@github.com:nexe/nexe.git $ cd nexe -$ yarn +$ npm i && npm run build ``` Testing ``` $ npm test ``` - -## Maintainers - -[![Jared Allard](https://avatars.githubusercontent.com/u/2391349?s=130)](https://jaredallard.me/) | [![Caleb Boyd](https://avatars.githubusercontent.com/u/5818726?s=130)](https://github.com/calebboyd) | [![Dustin Greif](https://avatars.githubusercontent.com/u/3026298?s=130)](https://github.com/dgreif) | ----|---|---| -[Jared Allard](https://github.com/jaredallard) | [Caleb Boyd](http://github.com/calebboyd) | [Dustin Greif](https://github.com/dgreif) | - -### Former - -- [Craig Condon](http://crcn.codes/) diff --git a/package-lock.json b/package-lock.json index e6c2c46..b0196c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "nexe", - "version": "4.0.0-beta.11", + "version": "4.0.0-beta.13", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -30,11 +30,6 @@ "js-tokens": "^4.0.0" } }, - "@calebboyd/async": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@calebboyd/async/-/async-1.4.0.tgz", - "integrity": "sha512-Q5tSWP28OF1nGd9KD6qspJselIfrHqk5+1gLGNNuQiUy80LE4g4ZAb+r+P8MYe1/FS5ICbm+OXiIalC7xhde3A==" - }, "@calebboyd/semaphore": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@calebboyd/semaphore/-/semaphore-1.3.1.tgz", @@ -119,15 +114,6 @@ "integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==", "dev": true }, - "@types/execa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@types/execa/-/execa-2.0.0.tgz", - "integrity": "sha512-aBnkJ0r3khaZkHzu9pDZeWXrDg1N/ZtDGRQkK+KIqNVvvTvW+URXMUHQQCQMYdb2GPrcwu9Fq6l9iiT+pirIbg==", - "dev": true, - "requires": { - "execa": "*" - } - }, "@types/glob": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz", @@ -618,9 +604,9 @@ "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "cross-spawn": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.2.tgz", - "integrity": "sha512-PD6G8QG3S4FK/XCGFbEQrDqO2AnMMsy0meR7lerlIOHAAbkuavGU/pOqprrlvfTNjvowivTeBsjebAL0NSoMxw==", + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, "requires": { "path-key": "^3.1.0", @@ -953,9 +939,9 @@ "dev": true }, "execa": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-4.0.0.tgz", - "integrity": "sha512-JbDUxwV3BoT5ZVXQrSVbAiaXhXUkIwvbhPIwZ0N13kX+5yCzOhUNdocxB/UQRuYOHRYYwAxKYwJYc0T4D12pDA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/execa/-/execa-4.0.2.tgz", + "integrity": "sha512-QI2zLa6CjGWdiQsmSkZoGtDx2N+cQIGb3yNolGTdjSQzydzLgYYf8LRuagp7S7fPimjcrzUDSUFd/MgzELMi4Q==", "dev": true, "requires": { "cross-spawn": "^7.0.0", @@ -2095,88 +2081,13 @@ } }, "resolve-dependencies": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/resolve-dependencies/-/resolve-dependencies-4.2.2.tgz", - "integrity": "sha512-jA5TjcYwz0MvPCdJz5NVHk68KYzVZ7+hSHEkaKaI31OLOpUGao5/u6Dvd9O3h9YF8QEMUY9EYJ7C1mdXDcQFgQ==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/resolve-dependencies/-/resolve-dependencies-6.0.2.tgz", + "integrity": "sha512-amDlyIebiLquqRpX4odZ/LEC5QimkYlAAz4SyPxNI3nUbUa+lN74K7bYvuqrYcpuZB306EidpcpwfpI6F5q9Lw==", "requires": { - "@calebboyd/async": "^1.4.0", "enhanced-resolve": "^4.2.0", "globby": "^11.0.1", "meriyah": "^1.9.15" - }, - "dependencies": { - "@nodelib/fs.scandir": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", - "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==", - "requires": { - "@nodelib/fs.stat": "2.0.3", - "run-parallel": "^1.1.9" - } - }, - "@nodelib/fs.stat": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz", - "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==" - }, - "@nodelib/fs.walk": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz", - "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==", - "requires": { - "@nodelib/fs.scandir": "2.1.3", - "fastq": "^1.6.0" - } - }, - "fast-glob": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz", - "integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==", - "requires": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.0", - "merge2": "^1.3.0", - "micromatch": "^4.0.2", - "picomatch": "^2.2.1" - } - }, - "glob-parent": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", - "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", - "requires": { - "is-glob": "^4.0.1" - } - }, - "globby": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz", - "integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==", - "requires": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.1.1", - "ignore": "^5.1.4", - "merge2": "^1.3.0", - "slash": "^3.0.0" - } - }, - "ignore": { - "version": "5.1.8", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", - "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==" - }, - "merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==" - }, - "picomatch": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", - "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==" - } } }, "responselike": { @@ -2202,9 +2113,9 @@ "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==" }, "rimraf": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.0.tgz", - "integrity": "sha512-NDGVxTsjqfunkds7CqsOiEnxln4Bo7Nddl3XhS4pXg5OzwkLqJ971ZVAAnB+DDLnF76N+VnDEiBHaVV8I06SUg==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "requires": { "glob": "^7.1.3" } diff --git a/package.json b/package.json index 100737c..053c465 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "nexe", "description": "Create a single executable out of your Node.js application", "license": "MIT", - "version": "4.0.0-beta.11", + "version": "4.0.0-beta.13", "contributors": [ "Craig Condon (http://crcn.io)", "Jared Allard ", @@ -46,13 +46,12 @@ "multistream": "^4.0.0", "ora": "^3.4.0", "pify": "^4.0.1", - "resolve-dependencies": "^4.2.2", - "rimraf": "^3.0.0" + "resolve-dependencies": "^6.0.2", + "rimraf": "^3.0.2" }, "devDependencies": { "@types/chai": "^4.2.11", "@types/download": "^6.2.4", - "@types/execa": "^2.0.0", "@types/globby": "^9.1.0", "@types/got": "^9.6.10", "@types/minimist": "^1.2.0", @@ -64,7 +63,7 @@ "@types/rimraf": "3.0.0", "@types/semver": "^7.1.0", "chai": "^4.2.0", - "execa": "^4.0.0", + "execa": "^4.0.2", "mocha": "^7.1.1", "prettier": "^2.0.5", "ts-node": "^8.9.0", diff --git a/src/compiler.ts b/src/compiler.ts index 87f4cc7..0a446b8 100644 --- a/src/compiler.ts +++ b/src/compiler.ts @@ -18,6 +18,7 @@ import { NexeOptions, version } from './options' import { NexeTarget } from './target' import MultiStream = require('multistream') import { Bundle, toStream } from './fs/bundle' +import { File } from 'resolve-dependencies' const isBsd = Boolean(~process.platform.indexOf('bsd')) const make = isWindows ? 'vcbuild.bat' : isBsd ? 'gmake' : 'make' @@ -143,7 +144,7 @@ export class NexeCompiler { } @bound - addResource(absoluteFileName: string, content?: Buffer | string) { + addResource(absoluteFileName: string, content?: Buffer | string | File) { return this.bundle.addResource(absoluteFileName, content) } @@ -298,22 +299,19 @@ export class NexeCompiler { if (!this.options.mangle) { return binary } - const bundle = await this.bundle.toStream() + const index = this.bundle.renderIndex() + this.shims.unshift(wrap(`process.__nexe = ${JSON.stringify({ resources: index }, null, 4)};\n`)) - this.shims.unshift( - wrap(`process.__nexe = ${JSON.stringify({ resources: this.bundle.fileIndex() })};\n`) - ) - - const startup = this.code(), - codeSize = Buffer.byteLength(startup), + const code = this.code(), + codeSize = Buffer.byteLength(code), lengths = Buffer.from(Array(16)) lengths.writeDoubleLE(codeSize, 0) lengths.writeDoubleLE(this.bundle.size, 8) return new (MultiStream as any)([ binary, - toStream(startup), - bundle, + toStream(code), + this.bundle.toStream(), toStream(Buffer.concat([Buffer.from(''), lengths])), ]) } diff --git a/src/fs/bundle.ts b/src/fs/bundle.ts index 34ec72a..cf0cca6 100644 --- a/src/fs/bundle.ts +++ b/src/fs/bundle.ts @@ -2,28 +2,25 @@ import * as fs from 'fs' import { promisify } from 'util' import { relative } from 'path' import { Readable } from 'stream' -import { each } from '@calebboyd/semaphore' +import { argv } from '../options' +import { File } from 'resolve-dependencies' import MultiStream = require('multistream') -const { createReadStream } = fs, - lstat = promisify(fs.lstat), +const lstat = promisify(fs.lstat), realpath = promisify(fs.realpath), + createReadStream = fs.createReadStream, stat = promisify(fs.stat) export type MultiStreams = (Readable | (() => Readable))[] -function renderSortedObject(entries: [string, [number, number]][], streams: MultiStreams) { - return entries.sort().reduce((obj, [key, value]) => { - streams.push((value as any).stream) - ;(value as any).stream = void 0 - return Object.assign(obj, { [key]: value }) - }, {}) -} - function makeRelative(cwd: string, path: string) { return './' + relative(cwd, path) } +function sortEntry(a: [string, File], b: [string, File]) { + return a[0] > b[0] ? 1 : -1 +} + export function toStream(content: Buffer | string) { const readable = new Readable({ read() { @@ -34,81 +31,126 @@ export function toStream(content: Buffer | string) { return readable } -export type File = { absPath: string; contents: string; deps: FileMap } -export type FileMap = { [absPath: string]: File | null } - -export interface BundleOptions { - entries: string[] - cwd: string - expand: boolean - loadContent: boolean - files: FileMap +async function createFile(absoluteFileName: string) { + const stats = await lstat(absoluteFileName), + file: File = { + size: 0, + contents: '', + absPath: absoluteFileName, + deps: {}, + } + if (stats.isSymbolicLink()) { + file.size = stats.size + const [realPath, realStat] = await Promise.all([ + realpath(absoluteFileName), + stat(absoluteFileName), + ]) + file.realPath = realPath + file.realSize = realStat.size + } + return file } export class Bundle { size = 0 cwd: string rendered = false - files: { [key: string]: string | Buffer | undefined } = {} + private offset = 0 + private index: { [key: string]: [number, number] } = {} + private files: { [key: string]: File } = {} streams: MultiStreams = [] - private index: { [relativeFilePath: string]: [number, number] } = {} constructor({ cwd }: { cwd: string } = { cwd: process.cwd() }) { this.cwd = cwd } - addResource(absoluteFileName: string, content?: Buffer | string) { - this.files[absoluteFileName] = content + get list() { + return Object.keys(this.files) } - async addEntry(absoluteFileName: string, content?: Buffer | string) { - let length = 0 - let linkpath = '' - if (content !== undefined) { - length = Buffer.byteLength(content) - } else { - let stats = await lstat(absoluteFileName) - if (stats.isSymbolicLink()) { - linkpath = absoluteFileName - absoluteFileName = await realpath(linkpath) - stats = await stat(linkpath) + public async addResource( + absoluteFileName: string, + content?: File | Buffer | string + ): Promise { + if (this.files[absoluteFileName]) { + return this.size + } + if (typeof content === 'string' || Buffer.isBuffer(content)) { + this.files[absoluteFileName] = { + size: Buffer.byteLength(content), + contents: content as any, //todo type is wrong here... should allow buffer + deps: {}, + absPath: absoluteFileName, } - length = stats.size - } - const name = makeRelative(this.cwd, absoluteFileName), - existing = this.index[name] - if (!existing) { - const start = this.size - this.size += length - this.index[name] = [start, length] - } - if (linkpath) { - const linkName = makeRelative(this.cwd, absoluteFileName) - this.index[linkName] = this.index[name] - } - if (!existing) { - ;(this.index[name] as any).stream = () => - content ? toStream(content) : createReadStream(absoluteFileName) + } else if (content) { + this.files[content.absPath] = content + } else { + this.files[absoluteFileName] = { + absPath: absoluteFileName, + contents: '', + deps: {}, + size: 0, + } + this.files[absoluteFileName] = await createFile(absoluteFileName) } + return (this.size += this.files[absoluteFileName].size) } - fileIndex() { - if (!this.rendered) { - throw new Error('Index not rendered must call toStream() first') + /** + * De-dupe files by absolute path, partition by symlink/real + * Iterate over real, add entries + * Iterate over symlinks, add symlinks + */ + renderIndex() { + if (this.rendered) { + throw new Error('Bundle index already rendered') } + const files = Object.entries(this.files), + realFiles: [string, File][] = [], + symLinks: [string, File][] = [] + for (const entry of files) { + if (entry[1].realPath) { + symLinks.push(entry) + } else { + realFiles.push(entry) + } + } + realFiles.sort(sortEntry) + symLinks.sort(sortEntry) + + for (const [absPath, file] of realFiles) { + this.addEntry(absPath, file) + } + for (const [absPath, file] of symLinks) { + this.addEntry(file.realPath as string, file) + this.addEntry(absPath, file, file.realPath) + } + this.rendered = true return this.index } - async toStream() { - this.files - await each( - Object.keys(this.files), - (key: string) => { - return this.addEntry(key, this.files[key]) - }, - { concurrency: 10 } - ) - this.index = renderSortedObject(Object.entries(this.index), this.streams) - this.rendered = true + /** + * Add a stream if needed and an entry with the required offset and size + * Ensure the calling order of this method is idempotent (eg, while iterating a sorted set) + * @param entryPath + * @param file + * @param useEntry + */ + addEntry(entryPath: string, file: File, useEntry?: string) { + const existingName = useEntry && makeRelative(this.cwd, useEntry), + name = makeRelative(this.cwd, entryPath), + size = file.realSize ?? file.size, + existingEntry = this.index[existingName ?? name] + + this.index[name] = existingEntry || [this.offset, size] + if (!existingEntry) { + this.streams.push(() => + file.contents ? toStream(file.contents) : createReadStream(file.absPath) + ) + this.offset += size + } + } + + toStream() { return new (MultiStream as any)(this.streams) } } diff --git a/src/steps/bundle.ts b/src/steps/bundle.ts index 98774a9..657724f 100644 --- a/src/steps/bundle.ts +++ b/src/steps/bundle.ts @@ -1,8 +1,7 @@ import { NexeCompiler, NexeError } from '../compiler' import { resolve, relative } from 'path' -import { each } from '@calebboyd/semaphore' import resolveFiles, { resolveSync } from 'resolve-dependencies' -import { dequote, STDIN_FLAG, semverGt } from '../util' +import { dequote, STDIN_FLAG, semverGt, each } from '../util' import { Readable } from 'stream' function getStdIn(stdin: Readable): Promise { @@ -26,7 +25,7 @@ function getStdIn(stdin: Readable): Promise { } export default async function bundle(compiler: NexeCompiler, next: any) { - const { bundle, cwd, input: inputPath } = compiler.options + const { bundle: doBundle, cwd, input: inputPath } = compiler.options let input = inputPath compiler.entrypoint = './' + relative(cwd, input) @@ -36,14 +35,14 @@ export default async function bundle(compiler: NexeCompiler, next: any) { compiler.startup = ';require("module").runMain();' } - if (!bundle) { + if (!doBundle) { await compiler.addResource(resolve(cwd, input)) return next() } let code = '' - if (typeof bundle === 'string') { - code = await require(bundle).createBundle(compiler.options) + if (typeof doBundle === 'string') { + code = await require(doBundle).createBundle(compiler.options) } if (input === STDIN_FLAG && (code = code || dequote(await getStdIn(process.stdin)))) { @@ -62,9 +61,9 @@ export default async function bundle(compiler: NexeCompiler, next: any) { compiler.entrypoint = './' + relative(cwd, input) } - const { files, warnings } = await resolveFiles( + const { files, warnings } = resolveFiles( input, - ...Object.keys(compiler.bundle.files).filter((x) => x.endsWith('.js')), + ...Object.keys(compiler.bundle.list).filter((x) => x.endsWith('.js')), { cwd, expand: 'variable', loadContent: false } ) @@ -74,6 +73,13 @@ export default async function bundle(compiler: NexeCompiler, next: any) { throw new NexeError('Parsing Error:\n' + warnings.join('\n')) } - Object.keys(files).forEach((x) => compiler.addResource(x)) + //TODO: warnings.forEach((x) => console.log(x)) + + await Promise.all( + Object.entries(files).map(([key, file]) => { + return compiler.addResource(key, file) + }) + ) + return next() } diff --git a/tsconfig.json b/tsconfig.json index 47b6ce5..5011d28 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target":"es5", + "target":"es2015", "skipLibCheck": true, "experimentalDecorators": true, "lib": ["es2018"],