feat: standalone bundler
This commit is contained in:
calebboyd
@@ -0,0 +1,53 @@
|
||||
# nexe-fs
|
||||
|
||||
This module contains a set of patches used to create (and use) the nexe virtual filesystem
|
||||
|
||||
---
|
||||
|
||||
### Getting Started:
|
||||
|
||||
To patch the module loader, monkey patches must be installed on a few methods at the time that node boots up. This requires a special build of node. Nexe provides these builds or the patch can be applied from this module manually in your own build setup.
|
||||
|
||||
```javascript
|
||||
const bootstrapPatch = fs.readFileSync(require.resolve('nexe-fs/bootstrap'), 'utf-8')
|
||||
// insert boostrapPatch wherever node starts up. For reference, Nexe does this
|
||||
// in third-party-main.ts
|
||||
```
|
||||
|
||||
To create a virtual file system use the `Bundle` Object
|
||||
|
||||
```javascript
|
||||
const { Bundle } = require('nexe-fs')
|
||||
|
||||
const bundle = new Bundle({ cwd })
|
||||
await bundle.addResource(absoluteFileName)
|
||||
|
||||
bundle.toStream().pipe(fs.createWriteStream('./myFsBlob'))
|
||||
|
||||
// You'll also want to save the bundle index to use at runtime
|
||||
// It can be saved by accessing bundle.index (object/hash) or serializing the bundle with JSON.stringify(bundle)
|
||||
```
|
||||
|
||||
In the entrypoint of the custom node build, the fs patch needs to be applied
|
||||
```javascript
|
||||
const fakeFs = fs.readFileSync(require.resolve('nexe-fs/patch'), 'utf-8')
|
||||
//apply this code to your entrypoint
|
||||
```
|
||||
|
||||
The patch exposes two methods, `shimFs` and `restoreFs` to be used as follows:
|
||||
|
||||
```javascript
|
||||
shimFs({
|
||||
blobPath: 'location/of/myFsBlob',
|
||||
resources: bundle.index,
|
||||
layout: {
|
||||
stat: blobStats //fs.Stats object about the blob at blobPath
|
||||
resourceStart: 0 // the offset within the blob that is referenced by the bundle index
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
At this point executing the original entrypoint shoudl result in usage of the virtual filesystem.
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
const __nexe_patches = ((process as any).nexe = { patches: {} }).patches as any
|
||||
function __nexe_noop_patch(this: any, original: any, ...args: any[]) {
|
||||
return original.call(this, ...args)
|
||||
}
|
||||
function __nexe_patch(obj: any, method: string, patch: any) {
|
||||
const original = obj[method]
|
||||
if (!original) return
|
||||
__nexe_patches[method] = patch
|
||||
obj[method] = function(this: any, ...args: any[]) {
|
||||
return __nexe_patches[method].call(this, original, ...args)
|
||||
}
|
||||
}
|
||||
__nexe_patch((process as any).binding('fs'), 'internalModuleReadFile', __nexe_noop_patch)
|
||||
__nexe_patch((process as any).binding('fs'), 'internalModuleReadJSON', __nexe_noop_patch)
|
||||
__nexe_patch((process as any).binding('fs'), 'internalModuleStat', __nexe_noop_patch)
|
||||
@@ -0,0 +1,70 @@
|
||||
import { stat as getStat, Stats, createReadStream } from 'fs'
|
||||
import { relative } from 'path'
|
||||
import { Readable } from 'stream'
|
||||
import combineStreams = require('multistream')
|
||||
|
||||
const stat = (file: string): Promise<Stats> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
getStat(file, (err, stats) => (err ? reject(err) : resolve(stats)))
|
||||
})
|
||||
}
|
||||
|
||||
function makeRelative(cwd: string, path: string) {
|
||||
return './' + relative(cwd, path)
|
||||
}
|
||||
|
||||
export function toStream(content: Buffer | string) {
|
||||
const readable = new Readable({ read() {} })
|
||||
readable.push(content)
|
||||
readable.push(null)
|
||||
return readable
|
||||
}
|
||||
|
||||
export type File = { absPath: string; contents: string; deps: FileMap }
|
||||
export type FileMap = { [absPath: string]: File | null }
|
||||
|
||||
export interface BundleOptions {
|
||||
entries: string[]
|
||||
cwd: string
|
||||
expand: boolean
|
||||
loadContent: boolean
|
||||
files: FileMap
|
||||
}
|
||||
|
||||
export class Bundle {
|
||||
constructor({ cwd }: { cwd: string } = { cwd: process.cwd() }) {
|
||||
this.cwd = cwd
|
||||
}
|
||||
cwd: string
|
||||
blobSize: number = 0
|
||||
index: { [relativeFilePath: string]: [number, number] } = {}
|
||||
streams: (Readable | (() => Readable))[] = []
|
||||
|
||||
async addResource(absoluteFileName: string, content?: Buffer | string) {
|
||||
let length = 0
|
||||
if (content) {
|
||||
length = Buffer.byteLength(content)
|
||||
} else {
|
||||
const stats = await stat(absoluteFileName)
|
||||
length = stats.size
|
||||
}
|
||||
|
||||
const start = this.blobSize
|
||||
|
||||
this.blobSize += length
|
||||
this.index[makeRelative(this.cwd, absoluteFileName)] = [start, length]
|
||||
this.streams.push(() => (content ? toStream(content) : createReadStream(absoluteFileName)))
|
||||
}
|
||||
|
||||
concat() {
|
||||
throw new Error('Not Implemented')
|
||||
}
|
||||
|
||||
toStream() {
|
||||
return combineStreams(this.streams)
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return this.index
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "nexe-fs",
|
||||
"version": "1.0.1",
|
||||
"description": "Virtual File System used by nexe",
|
||||
"main": "bundle.js",
|
||||
"files": [
|
||||
"*.d.ts",
|
||||
"*.js"
|
||||
],
|
||||
"author": "Caleb Boyd",
|
||||
"dependencies": {
|
||||
"multistream": "^2.1.1"
|
||||
},
|
||||
"license": "MIT"
|
||||
}
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
import { Stats } from 'fs'
|
||||
|
||||
export interface NexeBinary {
|
||||
blobPath: string
|
||||
resources: { [key: string]: number[] }
|
||||
layout: {
|
||||
stat: Stats
|
||||
resourceStart: number
|
||||
contentSize?: number
|
||||
contentStart?: number
|
||||
resourceSize?: number
|
||||
}
|
||||
}
|
||||
|
||||
let originalFsMethods: any = null
|
||||
let nexeBinary: NexeBinary | null = null
|
||||
|
||||
function restoreFs(fs: any = require('fs')): NexeBinary | false {
|
||||
if (!nexeBinary) {
|
||||
return false
|
||||
}
|
||||
const source = nexeBinary
|
||||
Object.assign(fs, originalFsMethods)
|
||||
nexeBinary = originalFsMethods = null
|
||||
return source
|
||||
}
|
||||
|
||||
function shimFs(binary: NexeBinary, fs: any = require('fs')) {
|
||||
if (originalFsMethods !== null) {
|
||||
return
|
||||
}
|
||||
originalFsMethods = Object.assign({}, fs)
|
||||
nexeBinary = binary
|
||||
const { blobPath, resources: manifest } = binary
|
||||
const { resourceStart, stat } = binary.layout
|
||||
const directories: { [key: string]: { [key: string]: boolean } } = {},
|
||||
isString = (x: any): x is string => typeof x === 'string' || x instanceof String,
|
||||
isNotFile = () => false,
|
||||
isNotDirectory = isNotFile,
|
||||
isFile = () => true,
|
||||
noop = () => {},
|
||||
isDirectory = isFile,
|
||||
path = require('path')
|
||||
|
||||
let log = (text: string) => {
|
||||
if ((process.env.DEBUG || '').toLowerCase().includes('nexe:require')) {
|
||||
process.stderr.write('[nexe] - ' + text + '\n')
|
||||
} else {
|
||||
log = noop
|
||||
}
|
||||
}
|
||||
|
||||
const getKey = process.platform.startsWith('win')
|
||||
? function getKey(filepath: string): string {
|
||||
let key = path.resolve(filepath)
|
||||
if (key.substr(1, 2) === ':\\') {
|
||||
key = key[0].toUpperCase() + key.substr(1)
|
||||
}
|
||||
return key
|
||||
}
|
||||
: path.resolve
|
||||
|
||||
const statTime = function() {
|
||||
return {
|
||||
dev: 0,
|
||||
ino: 0,
|
||||
nlink: 0,
|
||||
rdev: 0,
|
||||
uid: 123,
|
||||
gid: 500,
|
||||
blksize: 4096,
|
||||
blocks: 0,
|
||||
atime: new Date(stat.atime),
|
||||
atimeMs: stat.atime.getTime(),
|
||||
mtime: new Date(stat.mtime),
|
||||
mtimeMs: stat.mtime.getTime(),
|
||||
ctime: new Date(stat.ctime),
|
||||
ctimMs: stat.ctime.getTime(),
|
||||
birthtime: new Date(stat.birthtime),
|
||||
birthtimeMs: stat.birthtime.getTime()
|
||||
}
|
||||
}
|
||||
|
||||
const createStat = function(directoryExtensions: any, fileExtensions?: any) {
|
||||
if (!fileExtensions) {
|
||||
return Object.assign({}, binary.layout.stat, directoryExtensions, { size: 0 }, statTime())
|
||||
}
|
||||
const size = directoryExtensions[1]
|
||||
return Object.assign({}, binary.layout.stat, fileExtensions, { size }, statTime())
|
||||
}
|
||||
|
||||
const ownStat = function(filepath: string) {
|
||||
setupManifest()
|
||||
const key = getKey(filepath)
|
||||
if (directories[key]) {
|
||||
return createStat({ isDirectory, isFile: isNotFile })
|
||||
}
|
||||
if (manifest[key]) {
|
||||
return createStat(manifest[key], { isFile, isDirectory: isNotDirectory })
|
||||
}
|
||||
}
|
||||
|
||||
function makeLong(filepath: string) {
|
||||
return (path as any)._makeLong && (path as any)._makeLong(filepath)
|
||||
}
|
||||
|
||||
function fileOpts(options: any) {
|
||||
return !options ? {} : isString(options) ? { encoding: options } : options
|
||||
}
|
||||
|
||||
let setupManifest = () => {
|
||||
Object.keys(manifest).forEach(filepath => {
|
||||
const entry = manifest[filepath]
|
||||
const absolutePath = getKey(filepath)
|
||||
const longPath = makeLong(absolutePath)
|
||||
const normalizedPath = path.normalize(filepath)
|
||||
|
||||
if (!manifest[absolutePath]) {
|
||||
manifest[absolutePath] = entry
|
||||
}
|
||||
if (longPath && !manifest[longPath]) {
|
||||
manifest[longPath] = entry
|
||||
}
|
||||
if (!manifest[normalizedPath]) {
|
||||
manifest[normalizedPath] = manifest[filepath]
|
||||
}
|
||||
|
||||
let currentDir = path.dirname(absolutePath)
|
||||
let prevDir = absolutePath
|
||||
|
||||
while (currentDir !== prevDir) {
|
||||
directories[currentDir] = directories[currentDir] || {}
|
||||
directories[currentDir][path.basename(prevDir)] = true
|
||||
const longDir = makeLong(currentDir)
|
||||
if (longDir && !directories[longDir]) {
|
||||
directories[longDir] = directories[currentDir]
|
||||
}
|
||||
prevDir = currentDir
|
||||
currentDir = path.dirname(currentDir)
|
||||
}
|
||||
})
|
||||
setupManifest = noop
|
||||
}
|
||||
|
||||
//naive patches intended to work for most use cases
|
||||
const nfs: any = {
|
||||
existsSync: function existsSync(filepath: string) {
|
||||
setupManifest()
|
||||
const key = getKey(filepath)
|
||||
if (manifest[key] || directories[key]) {
|
||||
return true
|
||||
}
|
||||
return originalFsMethods.existsSync.apply(fs, arguments)
|
||||
},
|
||||
realpath: function realpath(filepath: any, options: any, cb: any): void {
|
||||
setupManifest()
|
||||
const key = getKey(filepath)
|
||||
if (isString(filepath) && (manifest[filepath] || manifest[key])) {
|
||||
return process.nextTick(() => cb(null, filepath))
|
||||
}
|
||||
return originalFsMethods.realpath.call(fs, filepath, options, cb)
|
||||
},
|
||||
realpathSync: function realpathSync(filepath: any, options: any) {
|
||||
setupManifest()
|
||||
const key = getKey(filepath)
|
||||
if (isString(filepath) && (manifest[filepath] || manifest[key])) {
|
||||
return filepath
|
||||
}
|
||||
return originalFsMethods.realpathSync.call(fs, filepath, options)
|
||||
},
|
||||
readdir: function readdir(filepath: string | Buffer, options: any, callback: any) {
|
||||
setupManifest()
|
||||
filepath = filepath.toString()
|
||||
if ('function' === typeof options) {
|
||||
callback = options
|
||||
options = { encoding: 'utf8' }
|
||||
}
|
||||
const dir = directories[getKey(filepath)]
|
||||
if (dir) {
|
||||
process.nextTick(() => {
|
||||
//todo merge with original?
|
||||
callback(null, Object.keys(dir))
|
||||
})
|
||||
} else {
|
||||
return originalFsMethods.readdir.apply(fs, arguments)
|
||||
}
|
||||
},
|
||||
|
||||
readdirSync: function readdirSync(filepath: string | Buffer, options: any) {
|
||||
setupManifest()
|
||||
filepath = filepath.toString()
|
||||
const dir = directories[getKey(filepath)]
|
||||
if (dir) {
|
||||
return Object.keys(dir)
|
||||
}
|
||||
return originalFsMethods.readdirSync.apply(fs, arguments)
|
||||
},
|
||||
|
||||
readFile: function readFile(filepath: any, options: any, callback: any) {
|
||||
setupManifest()
|
||||
const entry = manifest[filepath] || manifest[getKey(filepath)]
|
||||
if (!entry || !isString(filepath)) {
|
||||
return originalFsMethods.readFile.apply(fs, arguments)
|
||||
}
|
||||
const [offset, length] = entry
|
||||
const resourceOffset = resourceStart + offset
|
||||
const encoding = fileOpts(options).encoding
|
||||
callback = typeof options === 'function' ? options : callback
|
||||
|
||||
fs.open(blobPath, 'r', function(err: Error, fd: number) {
|
||||
if (err) return callback(err, null)
|
||||
fs.read(fd, Buffer.alloc(length), 0, length, resourceOffset, function(
|
||||
error: Error,
|
||||
bytesRead: number,
|
||||
result: Buffer
|
||||
) {
|
||||
if (error) {
|
||||
return fs.close(fd, function() {
|
||||
callback(error, null)
|
||||
})
|
||||
}
|
||||
fs.close(fd, function(err: Error) {
|
||||
if (err) {
|
||||
return callback(err, result)
|
||||
}
|
||||
callback(err, encoding ? result.toString(encoding) : result)
|
||||
})
|
||||
})
|
||||
})
|
||||
},
|
||||
createReadStream: function createReadStream(filepath: any, options: any) {
|
||||
setupManifest()
|
||||
const entry = manifest[filepath] || manifest[getKey(filepath)]
|
||||
if (!entry || !isString(filepath)) {
|
||||
return originalFsMethods.createReadStream.apply(fs, arguments)
|
||||
}
|
||||
const [offset, length] = entry
|
||||
const resourceOffset = resourceStart + offset
|
||||
const opts = fileOpts(options)
|
||||
|
||||
return fs.createReadStream(
|
||||
blobPath,
|
||||
Object.assign({}, opts, {
|
||||
start: resourceOffset,
|
||||
end: resourceOffset + length - 1
|
||||
})
|
||||
)
|
||||
},
|
||||
readFileSync: function readFileSync(filepath: any, options: any) {
|
||||
setupManifest()
|
||||
|
||||
const entry = manifest[filepath] || manifest[getKey(filepath)]
|
||||
if (!entry || !isString(filepath)) {
|
||||
return originalFsMethods.readFileSync.apply(fs, arguments)
|
||||
}
|
||||
const [offset, length] = entry
|
||||
const resourceOffset = resourceStart + offset
|
||||
const encoding = fileOpts(options).encoding
|
||||
const fd = fs.openSync(process.execPath, 'r')
|
||||
const result = Buffer.alloc(length)
|
||||
fs.readSync(fd, result, 0, length, resourceOffset)
|
||||
fs.closeSync(fd)
|
||||
return encoding ? result.toString(encoding) : result
|
||||
},
|
||||
statSync: function statSync(filepath: string | Buffer) {
|
||||
const stat = isString(filepath) && ownStat(filepath)
|
||||
if (stat) {
|
||||
return stat
|
||||
}
|
||||
return originalFsMethods.statSync.apply(fs, arguments)
|
||||
},
|
||||
stat: function stat(filepath: string | Buffer, callback: any) {
|
||||
const stat = isString(filepath) && ownStat(filepath)
|
||||
if (stat) {
|
||||
process.nextTick(() => {
|
||||
callback(null, stat)
|
||||
})
|
||||
} else {
|
||||
return originalFsMethods.stat.apply(fs, arguments)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof fs.exists === 'function') {
|
||||
nfs.exists = function(filepath: string, cb: Function) {
|
||||
cb = cb || noop
|
||||
const exists = nfs.existsSync(filepath)
|
||||
process.nextTick(() => cb(exists))
|
||||
}
|
||||
}
|
||||
|
||||
const patches = (process as any).nexe.patches || {}
|
||||
delete (process as any).nexe
|
||||
patches.internalModuleReadFile = function(this: any, original: any, ...args: any[]) {
|
||||
const [filepath] = args
|
||||
setupManifest()
|
||||
if (manifest[filepath]) {
|
||||
log('read (hit) ' + filepath)
|
||||
return nfs.readFileSync(filepath, 'utf-8')
|
||||
}
|
||||
log('read (miss) ' + filepath)
|
||||
return original.call(this, ...args)
|
||||
}
|
||||
patches.internalModuleStat = function(this: any, original: any, ...args: any[]) {
|
||||
setupManifest()
|
||||
const [filepath] = args
|
||||
if (manifest[filepath]) {
|
||||
log('stat (hit) ' + filepath + ' ' + 0)
|
||||
return 0
|
||||
}
|
||||
if (directories[filepath]) {
|
||||
log('stat dir (hit) ' + filepath + ' ' + 1)
|
||||
return 1
|
||||
}
|
||||
const res = original.call(this, ...args)
|
||||
if (res === 0) {
|
||||
log('stat (miss) ' + filepath + ' ' + res)
|
||||
} else if (res === 1) {
|
||||
log('stat dir (miss) ' + filepath + ' ' + res)
|
||||
} else {
|
||||
log('stat (fail) ' + filepath + ' ' + res)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
if (typeof fs.exists === 'function') {
|
||||
nfs.exists = function(filepath: string, cb: Function) {
|
||||
cb = cb || noop
|
||||
const exists = nfs.existsSync(filepath)
|
||||
if (!exists) {
|
||||
return originalFsMethods.exists(filepath, cb)
|
||||
}
|
||||
process.nextTick(() => cb(exists))
|
||||
}
|
||||
}
|
||||
Object.assign(fs, nfs)
|
||||
return true
|
||||
}
|
||||
|
||||
export { shimFs, restoreFs }
|
||||
Reference in New Issue
Block a user