This commit is contained in:
Tutur33
2023-11-24 22:35:41 +01:00
parent 3c0b507a93
commit 7644b2a0f7
45165 changed files with 4803356 additions and 3 deletions
+2
View File
@@ -0,0 +1,2 @@
types/index.d.ts
types/index.test-d.ts
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
. "$(dirname "$0")/_/husky.sh"
npm test
+5
View File
@@ -0,0 +1,5 @@
flow: false
ts: false
jsx: false
timeout: 120
check-coverage: false
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2017 Matteo Collina
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+149
View File
@@ -0,0 +1,149 @@
# sonic-boom
[![NPM Package Version](https://img.shields.io/npm/v/sonic-boom)](https://www.npmjs.com/package/sonic-boom)
[![Build Status](https://github.com/pinojs/sonic-boom/workflows/Node.js%20CI/badge.svg)](https://github.com/pinojs/sonic-boom/actions?query=workflow%3ANode.js%20CI)
[![Known Vulnerabilities](https://snyk.io/test/github/pinojs/sonic-boom/badge.svg)](https://snyk.io/test/github/pinojs/sonic-boom)
[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](https://standardjs.com/)
Extremely fast utf8-only stream implementation to write to files and
file descriptors.
This implementation is partial, but support backpressure and `.pipe()` in is here.
However, it is 2-3x faster than Node Core `fs.createWriteStream()`:
```
benchSonic*1000: 1916.904ms
benchSonicSync*1000: 8605.265ms
benchSonic4k*1000: 1965.231ms
benchSonicSync4k*1000: 1588.224ms
benchCore*1000: 5851.959ms
benchConsole*1000: 7605.713ms
```
Note that sync mode without buffering is _slower_ than a Node Core WritableStream, however
this mode matches the expected behavior of `console.log()`.
Note that if this is used to log to a windows terminal (`cmd.exe` or
powershell), it is needed to run `chcp 65001` in the terminal to
correctly display utf-8 characters, see
[chcp](https://ss64.com/nt/chcp.html) for more details.
## Install
```
npm i sonic-boom
```
## Example
```js
'use strict'
const SonicBoom = require('sonic-boom')
const sonic = new SonicBoom({ fd: process.stdout.fd }) // or { dest: '/path/to/destination' }
for (let i = 0; i < 10; i++) {
sonic.write('hello sonic\n')
}
```
## API
### SonicBoom(opts)
Creates a new instance of SonicBoom.
The options are:
* `fd`: a file descriptor, something that is returned by `fs.open` or
`fs.openSync`.
* `dest`: a string that is a path to a file to be written to (mode controlled by the `append` option).
* `minLength`: the minimum length of the internal buffer that is
required to be full before flushing.
* `maxLength`: the maximum length of the internal buffer. If a write operation would cause the buffer
to exceed `maxLength`, the data written is dropped and a `drop` event is emitted with the dropped data
* `maxWrite`: the maximum number of bytes that can be written; default: 16384
* `sync`: perform writes synchronously (similar to `console.log`).
* `append`: appends writes to dest file instead of truncating it (default `true`).
* `mode`: specify the creating file `mode` (see [fs.open()](https://nodejs.org/api/fs.html#fsopenpath-flags-mode-callback) from Node.js core).
* `mkdir`: ensure directory for dest file exists when `true` (default `false`).
* `retryEAGAIN(err, writeBufferLen, remainingBufferLen)`: a function that will be called when sonic-boom
write/writeSync/flushSync encounters a EAGAIN error. If the return value is
true sonic-boom will retry the operation, otherwise it will bubble the
error. `err` is the error that caused this function to be called,
`writeBufferLen` is the length of the buffer sonic-boom tried to write, and
`remainingBufferLen` is the length of the remaining buffer sonic-boom didn't try to write.
For `sync:false` a `SonicBoom` instance will emit the `'ready'` event when a file descriptor is available.
For `sync:true` this is not relevant because the `'ready'` event will be fired when the `SonicBoom` instance is created, before it can be subscribed to.
### SonicBoom#write(string)
Writes the string to the file.
It will return false to signal the producer to slow down.
### SonicBoom#flush()
Writes the current buffer to the file if a write was not in progress.
Do nothing if `minLength` is zero or if it is already writing.
### SonicBoom#reopen([file])
Reopen the file in place, useful for log rotation.
Example:
```js
const stream = new SonicBoom('./my.log')
process.on('SIGUSR2', function () {
stream.reopen()
})
```
### SonicBoom#flushSync()
Flushes the buffered data synchronously. This is a costly operation.
### SonicBoom#end()
Closes the stream, the data will be flushed down asynchronously
### SonicBoom#destroy()
Closes the stream immediately, the data is not flushed.
### Events
#### SonicBoom#close
See [Stream#close](https://nodejs.org/api/stream.html#event-close). The `'close'` event when the instance has been closed.
#### SonicBoom#drain
See [Stream#drain](https://nodejs.org/api/stream.html#event-drain). The `'drain'` event is emitted when source can resume sending data.
#### SonicBoom#drop <any>
When destination file maximal length is reached, the `'drop'` event is emitted with data that could not be written.
#### SonicBoom#error <Error>
The `'error'` event is emitted when the destination file can not be opened, or written.
#### SonicBoom#finish
See [Stream#finish](https://nodejs.org/api/stream.html#event-finish). The `'finish'` event after calling `end()` method and when all data was written.
#### SonicBoom#ready
The `'ready'` event occurs when the created instance is ready to process input.
#### SonicBoom#write <number>
The `'write'` event occurs every time data is written to the underlying file. It emits the number of written bytes.
## License
MIT
+67
View File
@@ -0,0 +1,67 @@
'use strict'
const bench = require('fastbench')
const SonicBoom = require('./')
const Console = require('console').Console
const fs = require('fs')
const core = fs.createWriteStream('/dev/null')
const fd = fs.openSync('/dev/null', 'w')
const sonic = new SonicBoom({ fd })
const sonic4k = new SonicBoom({ fd, minLength: 4096 })
const sonicSync = new SonicBoom({ fd, sync: true })
const sonicSync4k = new SonicBoom({ fd, minLength: 4096, sync: true })
const dummyConsole = new Console(fs.createWriteStream('/dev/null'))
const MAX = 10000
let str = ''
for (let i = 0; i < 10; i++) {
str += 'hello'
}
setTimeout(doBench, 100)
const run = bench([
function benchSonic (cb) {
sonic.once('drain', cb)
for (let i = 0; i < MAX; i++) {
sonic.write(str)
}
},
function benchSonicSync (cb) {
sonicSync.once('drain', cb)
for (let i = 0; i < MAX; i++) {
sonicSync.write(str)
}
},
function benchSonic4k (cb) {
sonic4k.once('drain', cb)
for (let i = 0; i < MAX; i++) {
sonic4k.write(str)
}
},
function benchSonicSync4k (cb) {
sonicSync4k.once('drain', cb)
for (let i = 0; i < MAX; i++) {
sonicSync4k.write(str)
}
},
function benchCore (cb) {
core.once('drain', cb)
for (let i = 0; i < MAX; i++) {
core.write(str)
}
},
function benchConsole (cb) {
for (let i = 0; i < MAX; i++) {
dummyConsole.log(str)
}
setImmediate(cb)
}
], 1000)
function doBench () {
run(run)
}
+18
View File
@@ -0,0 +1,18 @@
'use strict'
const SonicBoom = require('.')
const sonic = new SonicBoom({ fd: process.stdout.fd })
let count = 0
function scheduleWrites () {
for (let i = 0; i < 1000; i++) {
sonic.write('hello sonic\n')
console.log('hello console')
}
if (++count < 10) {
setTimeout(scheduleWrites, 100)
}
}
scheduleWrites()
+8
View File
@@ -0,0 +1,8 @@
'use strict'
const SonicBoom = require('.')
const sonic = new SonicBoom({ fd: process.stdout.fd }) // or 'destination'
for (let i = 0; i < 10; i++) {
sonic.write('hello sonic\n')
}
+22
View File
@@ -0,0 +1,22 @@
'use strict'
const SonicBoom = require('..')
const out = new SonicBoom({ fd: process.stdout.fd })
const str = Buffer.alloc(1000).fill('a').toString()
let i = 0
function write () {
if (i++ === 10) {
return
}
if (out.write(str)) {
write()
} else {
out.once('drain', write)
}
}
write()
+428
View File
@@ -0,0 +1,428 @@
'use strict'
const fs = require('fs')
const EventEmitter = require('events')
const inherits = require('util').inherits
const path = require('path')
const sleep = require('atomic-sleep')
const BUSY_WRITE_TIMEOUT = 100
// 16 KB. Don't write more than docker buffer size.
// https://github.com/moby/moby/blob/513ec73831269947d38a644c278ce3cac36783b2/daemon/logger/copier.go#L13
const MAX_WRITE = 16 * 1024
function openFile (file, sonic) {
sonic._opening = true
sonic._writing = true
sonic._asyncDrainScheduled = false
// NOTE: 'error' and 'ready' events emitted below only relevant when sonic.sync===false
// for sync mode, there is no way to add a listener that will receive these
function fileOpened (err, fd) {
if (err) {
sonic._reopening = false
sonic._writing = false
sonic._opening = false
if (sonic.sync) {
process.nextTick(() => {
if (sonic.listenerCount('error') > 0) {
sonic.emit('error', err)
}
})
} else {
sonic.emit('error', err)
}
return
}
sonic.fd = fd
sonic.file = file
sonic._reopening = false
sonic._opening = false
sonic._writing = false
if (sonic.sync) {
process.nextTick(() => sonic.emit('ready'))
} else {
sonic.emit('ready')
}
if (sonic._reopening) {
return
}
// start
if (!sonic._writing && sonic._len > sonic.minLength && !sonic.destroyed) {
actualWrite(sonic)
}
}
const flags = sonic.append ? 'a' : 'w'
const mode = sonic.mode
if (sonic.sync) {
try {
if (sonic.mkdir) fs.mkdirSync(path.dirname(file), { recursive: true })
const fd = fs.openSync(file, flags, mode)
fileOpened(null, fd)
} catch (err) {
fileOpened(err)
throw err
}
} else if (sonic.mkdir) {
fs.mkdir(path.dirname(file), { recursive: true }, (err) => {
if (err) return fileOpened(err)
fs.open(file, flags, mode, fileOpened)
})
} else {
fs.open(file, flags, mode, fileOpened)
}
}
function SonicBoom (opts) {
if (!(this instanceof SonicBoom)) {
return new SonicBoom(opts)
}
let { fd, dest, minLength, maxLength, maxWrite, sync, append = true, mode, mkdir, retryEAGAIN } = opts || {}
fd = fd || dest
this._bufs = []
this._len = 0
this.fd = -1
this._writing = false
this._writingBuf = ''
this._ending = false
this._reopening = false
this._asyncDrainScheduled = false
this._hwm = Math.max(minLength || 0, 16387)
this.file = null
this.destroyed = false
this.minLength = minLength || 0
this.maxLength = maxLength || 0
this.maxWrite = maxWrite || MAX_WRITE
this.sync = sync || false
this.append = append || false
this.mode = mode
this.retryEAGAIN = retryEAGAIN || (() => true)
this.mkdir = mkdir || false
if (typeof fd === 'number') {
this.fd = fd
process.nextTick(() => this.emit('ready'))
} else if (typeof fd === 'string') {
openFile(fd, this)
} else {
throw new Error('SonicBoom supports only file descriptors and files')
}
if (this.minLength >= this.maxWrite) {
throw new Error(`minLength should be smaller than maxWrite (${this.maxWrite})`)
}
this.release = (err, n) => {
if (err) {
if (err.code === 'EAGAIN' && this.retryEAGAIN(err, this._writingBuf.length, this._len - this._writingBuf.length)) {
if (this.sync) {
// This error code should not happen in sync mode, because it is
// not using the underlining operating system asynchronous functions.
// However it happens, and so we handle it.
// Ref: https://github.com/pinojs/pino/issues/783
try {
sleep(BUSY_WRITE_TIMEOUT)
this.release(undefined, 0)
} catch (err) {
this.release(err)
}
} else {
// Let's give the destination some time to process the chunk.
setTimeout(() => {
fs.write(this.fd, this._writingBuf, 'utf8', this.release)
}, BUSY_WRITE_TIMEOUT)
}
} else {
this._writing = false
this.emit('error', err)
}
return
}
this.emit('write', n)
this._len -= n
this._writingBuf = this._writingBuf.slice(n)
if (this._writingBuf.length) {
if (!this.sync) {
fs.write(this.fd, this._writingBuf, 'utf8', this.release)
return
}
try {
do {
const n = fs.writeSync(this.fd, this._writingBuf, 'utf8')
this._len -= n
this._writingBuf = this._writingBuf.slice(n)
} while (this._writingBuf)
} catch (err) {
this.release(err)
return
}
}
const len = this._len
if (this._reopening) {
this._writing = false
this._reopening = false
this.reopen()
} else if (len > this.minLength) {
actualWrite(this)
} else if (this._ending) {
if (len > 0) {
actualWrite(this)
} else {
this._writing = false
actualClose(this)
}
} else {
this._writing = false
if (this.sync) {
if (!this._asyncDrainScheduled) {
this._asyncDrainScheduled = true
process.nextTick(emitDrain, this)
}
} else {
this.emit('drain')
}
}
}
this.on('newListener', function (name) {
if (name === 'drain') {
this._asyncDrainScheduled = false
}
})
}
function emitDrain (sonic) {
const hasListeners = sonic.listenerCount('drain') > 0
if (!hasListeners) return
sonic._asyncDrainScheduled = false
sonic.emit('drain')
}
inherits(SonicBoom, EventEmitter)
SonicBoom.prototype.write = function (data) {
if (this.destroyed) {
throw new Error('SonicBoom destroyed')
}
const len = this._len + data.length
const bufs = this._bufs
if (this.maxLength && len > this.maxLength) {
this.emit('drop', data)
return this._len < this._hwm
}
if (
bufs.length === 0 ||
bufs[bufs.length - 1].length + data.length > this.maxWrite
) {
bufs.push('' + data)
} else {
bufs[bufs.length - 1] += data
}
this._len = len
if (!this._writing && this._len >= this.minLength) {
actualWrite(this)
}
return this._len < this._hwm
}
SonicBoom.prototype.flush = function () {
if (this.destroyed) {
throw new Error('SonicBoom destroyed')
}
if (this._writing || this.minLength <= 0) {
return
}
if (this._bufs.length === 0) {
this._bufs.push('')
}
actualWrite(this)
}
SonicBoom.prototype.reopen = function (file) {
if (this.destroyed) {
throw new Error('SonicBoom destroyed')
}
if (this._opening) {
this.once('ready', () => {
this.reopen(file)
})
return
}
if (this._ending) {
return
}
if (!this.file) {
throw new Error('Unable to reopen a file descriptor, you must pass a file to SonicBoom')
}
this._reopening = true
if (this._writing) {
return
}
const fd = this.fd
this.once('ready', () => {
if (fd !== this.fd) {
fs.close(fd, (err) => {
if (err) {
return this.emit('error', err)
}
})
}
})
openFile(file || this.file, this)
}
SonicBoom.prototype.end = function () {
if (this.destroyed) {
throw new Error('SonicBoom destroyed')
}
if (this._opening) {
this.once('ready', () => {
this.end()
})
return
}
if (this._ending) {
return
}
this._ending = true
if (this._writing) {
return
}
if (this._len > 0 && this.fd >= 0) {
actualWrite(this)
} else {
actualClose(this)
}
}
SonicBoom.prototype.flushSync = function () {
if (this.destroyed) {
throw new Error('SonicBoom destroyed')
}
if (this.fd < 0) {
throw new Error('sonic boom is not ready yet')
}
if (!this._writing && this._writingBuf.length > 0) {
this._bufs.unshift(this._writingBuf)
this._writingBuf = ''
}
while (this._bufs.length) {
const buf = this._bufs[0]
try {
this._len -= fs.writeSync(this.fd, buf, 'utf8')
this._bufs.shift()
} catch (err) {
if (err.code !== 'EAGAIN' || !this.retryEAGAIN(err, buf.length, this._len - buf.length)) {
throw err
}
sleep(BUSY_WRITE_TIMEOUT)
}
}
}
SonicBoom.prototype.destroy = function () {
if (this.destroyed) {
return
}
actualClose(this)
}
function actualWrite (sonic) {
const release = sonic.release
sonic._writing = true
sonic._writingBuf = sonic._writingBuf || sonic._bufs.shift() || ''
if (sonic.sync) {
try {
const written = fs.writeSync(sonic.fd, sonic._writingBuf, 'utf8')
release(null, written)
} catch (err) {
release(err)
}
} else {
fs.write(sonic.fd, sonic._writingBuf, 'utf8', release)
}
}
function actualClose (sonic) {
if (sonic.fd === -1) {
sonic.once('ready', actualClose.bind(null, sonic))
return
}
sonic.destroyed = true
sonic._bufs = []
if (sonic.fd !== 1 && sonic.fd !== 2) {
fs.close(sonic.fd, done)
} else {
setImmediate(done)
}
function done (err) {
if (err) {
sonic.emit('error', err)
return
}
if (sonic._ending && !sonic._writing) {
sonic.emit('finish')
}
sonic.emit('close')
}
}
/**
* These export configurations enable JS and TS developers
* to consumer SonicBoom in whatever way best suits their needs.
* Some examples of supported import syntax includes:
* - `const SonicBoom = require('SonicBoom')`
* - `const { SonicBoom } = require('SonicBoom')`
* - `import * as SonicBoom from 'SonicBoom'`
* - `import { SonicBoom } from 'SonicBoom'`
* - `import SonicBoom from 'SonicBoom'`
*/
SonicBoom.SonicBoom = SonicBoom
SonicBoom.default = SonicBoom
module.exports = SonicBoom
+55
View File
@@ -0,0 +1,55 @@
{
"name": "sonic-boom",
"version": "2.8.0",
"description": "Extremely fast utf8 only stream implementation",
"main": "index.js",
"type": "commonjs",
"types": "types/index.d.ts",
"scripts": {
"test": "npm run test:types && standard && tap test.js",
"test:unit": "tap test.js",
"test:types": "tsc && tsd && ts-node types/tests/test.ts",
"prepare": "husky install"
},
"repository": {
"type": "git",
"url": "git+https://github.com/pinojs/sonic-boom.git"
},
"keywords": [
"stream",
"fs",
"net",
"fd",
"file",
"descriptor",
"fast"
],
"author": "Matteo Collina <hello@matteocollina.com>",
"license": "MIT",
"bugs": {
"url": "https://github.com/pinojs/sonic-boom/issues"
},
"homepage": "https://github.com/pinojs/sonic-boom#readme",
"devDependencies": {
"@types/node": "^17.0.0",
"fastbench": "^1.0.1",
"husky": "^7.0.1",
"proxyquire": "^2.1.0",
"standard": "^17.0.0",
"tap": "^16.0.0",
"tsd": "^0.20.0",
"typescript": "^4.5.2",
"ts-node": "^10.4.0"
},
"dependencies": {
"atomic-sleep": "^1.0.0"
},
"husky": {
"hooks": {
"pre-commit": "npm test"
}
},
"tsd": {
"directory": "./types"
}
}
+1590
View File
File diff suppressed because it is too large Load Diff
+60
View File
@@ -0,0 +1,60 @@
// Type definitions for sonic-boom 0.7
// Definitions by: Alex Ferrando <https://github.com/alferpal>
// Igor Savin <https://github.com/kibertoad>
/// <reference types="node"/>
import { EventEmitter } from 'events';
export default SonicBoom;
export type SonicBoomOpts = {
fd?: number | string | symbol
dest?: string | number
maxLength?: number
minLength?: number
maxWrite?: number
sync?: boolean
append?: boolean
mode?: string | number
mkdir?: boolean
retryEAGAIN?: (err: Error, writeBufferLen: number, remainingBufferLen: number) => boolean
}
export class SonicBoom extends EventEmitter {
/**
* @param [fileDescriptor] File path or numerical file descriptor
* relative protocol is enabled. Default: process.stdout
* @returns a new sonic-boom instance
*/
constructor(opts: SonicBoomOpts)
/**
* Writes the string to the file. It will return false to signal the producer to slow down.
*/
write(string: string): boolean;
/**
* Writes the current buffer to the file if a write was not in progress.
* Do nothing if minLength is zero or if it is already writing.
*/
flush(): void;
/**
* Reopen the file in place, useful for log rotation.
*/
reopen(fileDescriptor?: string | number): void;
/**
* Flushes the buffered data synchronously. This is a costly operation.
*/
flushSync(): void;
/**
* Closes the stream, the data will be flushed down asynchronously
*/
end(): void;
/**
* Closes the stream immediately, the data is not flushed.
*/
destroy(): void;
}
+4
View File
@@ -0,0 +1,4 @@
import { SonicBoom } from '../../'
const sonic = new SonicBoom({ fd: process.stdout.fd })
sonic.write('hello sonic\n')