Read writer memory optimize

This commit is contained in:
Eli-Class
2026-02-05 05:36:36 +00:00
parent 132189da48
commit 92a2681292
8 changed files with 118 additions and 87 deletions

3
.gitignore vendored
View File

@@ -3,4 +3,5 @@ build/
dist/ dist/
*.log *.log
*.node *.node
.DS_Store .DS_Store
package-lock.json

View File

@@ -12,6 +12,10 @@ export interface DataHeader {
reserved: Buffer; reserved: Buffer;
} }
export interface BufferRef {
buf: Buffer;
}
export class DataProtocol { export class DataProtocol {
static createHeader(): Buffer { static createHeader(): Buffer {
const buf = Buffer.alloc(DATA_HEADER_SIZE); const buf = Buffer.alloc(DATA_HEADER_SIZE);
@@ -54,6 +58,29 @@ export class DataProtocol {
return buf; return buf;
} }
/**
* 기존 버퍼에 레코드를 직렬화하여 쓰기
* 버퍼 크기 부족시 내부에서 재할당 (BufferRef로 참조 전달)
* @returns 실제 쓰여진 바이트 수 (totalLen)
*/
static serializeRecordTo<T>(bufRef: BufferRef, data: T, serializer: Serializer<T>): number {
const dataBytes = serializer.serialize(data);
const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
// 버퍼 크기 부족시 재할당
if (bufRef.buf.length < totalLen) {
bufRef.buf = Buffer.alloc(totalLen);
}
dataBytes.copy(bufRef.buf, RECORD_HEADER_SIZE);
bufRef.buf.writeUInt32LE(dataBytes.length, 0);
const checksum = crc32(bufRef.buf, RECORD_HEADER_SIZE, totalLen);
bufRef.buf.writeUInt32LE(checksum, 4);
return totalLen;
}
static deserializeRecord<T>( static deserializeRecord<T>(
buf: Buffer, buf: Buffer,
offset: number, offset: number,

View File

@@ -1,5 +1,6 @@
// src/data-file/reader.ts // src/data-file/reader.ts
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as fsext from 'fs-ext';
import { DATA_HEADER_SIZE } from './constants.js'; import { DATA_HEADER_SIZE } from './constants.js';
import { DataProtocol, DataHeader } from './protocol.js'; import { DataProtocol, DataHeader } from './protocol.js';
import { IndexReader } from '../idx/index.js'; import { IndexReader } from '../idx/index.js';
@@ -28,7 +29,7 @@ export class DataReader<T> {
// Read header only // Read header only
const headerBuf = Buffer.alloc(DATA_HEADER_SIZE); const headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
fs.readSync(this.fd, headerBuf, 0, DATA_HEADER_SIZE, 0); fs.readSync(this.fd, headerBuf, 0, DATA_HEADER_SIZE, null);
this.header = DataProtocol.readHeader(headerBuf); this.header = DataProtocol.readHeader(headerBuf);
this.indexReader.open(this.indexPath); this.indexReader.open(this.indexPath);
@@ -36,7 +37,8 @@ export class DataReader<T> {
private readRecordAt(fd: number, offset: bigint, length: number): { data: T; length: number } | null { private readRecordAt(fd: number, offset: bigint, length: number): { data: T; length: number } | null {
const buf = Buffer.alloc(length); const buf = Buffer.alloc(length);
fs.readSync(fd, buf, 0, length, Number(offset)); fsext.seekSync(fd, Number(offset), 0);
fs.readSync(fd, buf, 0, length, null);
const result = DataProtocol.deserializeRecord( const result = DataProtocol.deserializeRecord(
buf, buf,
0, 0,

View File

@@ -1,13 +1,17 @@
// src/data-file/writer.ts // src/data-file/writer.ts
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as fsext from 'fs-ext';
import { DATA_HEADER_SIZE } from './constants.js'; import { DATA_HEADER_SIZE } from './constants.js';
import { DataProtocol } from './protocol.js'; import { DataProtocol, BufferRef } from './protocol.js';
import { IndexWriter, IndexFileOptionsRequired } from '../idx/index.js'; import { IndexWriter, IndexFileOptionsRequired } from '../idx/index.js';
import type { Serializer, DataFileOptions } from './types.js'; import type { Serializer, DataFileOptions } from './types.js';
const DEFAULT_DATA_BUF_SIZE = 4096;
export class DataWriter<T> { export class DataWriter<T> {
private fd: number | null = null; private fd: number | null = null;
private headerBuf: Buffer | null = null; private headerBuf: Buffer = Buffer.alloc(DATA_HEADER_SIZE);
private dataBufRef: BufferRef = { buf: Buffer.alloc(DEFAULT_DATA_BUF_SIZE) };
private currentOffset: bigint = BigInt(DATA_HEADER_SIZE); private currentOffset: bigint = BigInt(DATA_HEADER_SIZE);
private recordCount = 0; private recordCount = 0;
@@ -65,17 +69,19 @@ export class DataWriter<T> {
this.fd = fs.openSync(this.dataPath, this.fd = fs.openSync(this.dataPath,
isNew || this.forceTruncate ? 'w+' : 'r+'); isNew || this.forceTruncate ? 'w+' : 'r+');
try { // 버퍼 초기화
this.headerBuf = Buffer.alloc(DATA_HEADER_SIZE); this.headerBuf.fill(0);
this.dataBufRef.buf.fill(0);
try {
if (isNew || this.forceTruncate) { if (isNew || this.forceTruncate) {
const header = DataProtocol.createHeader(); const header = DataProtocol.createHeader();
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, 0); fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, null);
this.currentOffset = BigInt(DATA_HEADER_SIZE); this.currentOffset = BigInt(DATA_HEADER_SIZE);
this.recordCount = 0; this.recordCount = 0;
this.latestSequence = 0; this.latestSequence = 0;
} else { } else {
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0); fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, null);
const header = DataProtocol.readHeader(this.headerBuf); const header = DataProtocol.readHeader(this.headerBuf);
// Validate: Data file recordCount must match Index file writtenCnt // Validate: Data file recordCount must match Index file writtenCnt
@@ -88,6 +94,7 @@ export class DataWriter<T> {
this.currentOffset = header.fileSize; this.currentOffset = header.fileSize;
this.recordCount = header.recordCount; this.recordCount = header.recordCount;
this.latestSequence = this.indexWriter.getLatestSequence(); this.latestSequence = this.indexWriter.getLatestSequence();
fsext.seekSync(this.fd, 0, 2); // 가장 마지막으로
} }
} catch (error) { } catch (error) {
// Clean up resources on error // Clean up resources on error
@@ -95,7 +102,8 @@ export class DataWriter<T> {
fs.closeSync(this.fd); fs.closeSync(this.fd);
this.fd = null; this.fd = null;
} }
this.headerBuf = null; this.headerBuf.fill(0);
this.dataBufRef.buf.fill(0);
throw error; throw error;
} }
} }
@@ -103,18 +111,19 @@ export class DataWriter<T> {
append(data: T, sequence?: number, timestamp?: bigint): number { append(data: T, sequence?: number, timestamp?: bigint): number {
if (this.fd === null) throw new Error('Data file not opened'); if (this.fd === null) throw new Error('Data file not opened');
const buf = DataProtocol.serializeRecord(data, this.serializer); // dataBufRef에 직렬화 (내부에서 필요시 재할당)
const writtenLen = DataProtocol.serializeRecordTo(this.dataBufRef, data, this.serializer);
const offset = this.currentOffset; const offset = this.currentOffset;
fs.writeSync(this.fd, buf, 0, buf.length, Number(offset)); fs.writeSync(this.fd, this.dataBufRef.buf, 0, writtenLen, null);
// Write to index file // Write to index file
this.indexWriter.write(offset, buf.length, sequence, timestamp); this.indexWriter.write(offset, writtenLen, sequence, timestamp);
// Update latestSequence to the most recent sequence // Update latestSequence to the most recent sequence
this.latestSequence = this.indexWriter.getLatestSequence(); this.latestSequence = this.indexWriter.getLatestSequence();
this.currentOffset += BigInt(buf.length); this.currentOffset += BigInt(writtenLen);
++this.recordCount; ++this.recordCount;
this.writeHeader(); this.writeHeader();
@@ -151,14 +160,14 @@ export class DataWriter<T> {
} }
writeHeader(): void { writeHeader(): void {
if (this.fd === null || !this.headerBuf) return; if (this.fd === null) return;
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount); DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0); fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
} }
sync(): void { sync(): void {
if (this.fd === null || !this.headerBuf) return; if (this.fd === null) return;
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount); DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0); fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
@@ -174,7 +183,8 @@ export class DataWriter<T> {
} }
this.indexWriter.close(); this.indexWriter.close();
this.headerBuf = null; this.headerBuf.fill(0);
this.dataBufRef.buf.fill(0);
} }
writeFlags(flags: number): void { writeFlags(flags: number): void {

View File

@@ -73,39 +73,15 @@ export class IndexProtocol {
buf.writeUInt32LE(latestSequence, 36); buf.writeUInt32LE(latestSequence, 36);
} }
static writeEntry(buf: Buffer, index: number, entry: Omit<IndexEntry, 'checksum'>): void { static writeEntry(buf: Buffer, entry: Omit<IndexEntry, 'checksum'>): void {
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE; buf.writeUInt32LE(entry.sequence, 0);
buf.writeBigUInt64LE(entry.timestamp, 4);
buf.writeBigUInt64LE(entry.offset, 12);
buf.writeUInt32LE(entry.length, 20);
buf.writeUInt32LE(entry.flags | FLAG_VALID, 24);
buf.writeUInt32LE(entry.sequence, off); const checksum = crc32(buf, 0, 28);
buf.writeBigUInt64LE(entry.timestamp, off + 4); buf.writeUInt32LE(checksum, 28);
buf.writeBigUInt64LE(entry.offset, off + 12);
buf.writeUInt32LE(entry.length, off + 20);
buf.writeUInt32LE(entry.flags | FLAG_VALID, off + 24);
const checksum = crc32(buf, off, off + 28);
buf.writeUInt32LE(checksum, off + 28);
}
static readEntry(buf: Buffer, index: number): IndexEntry | null {
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
const flags = buf.readUInt32LE(off + 24);
if (!(flags & FLAG_VALID)) return null;
return {
sequence: buf.readUInt32LE(off),
timestamp: buf.readBigUInt64LE(off + 4),
offset: buf.readBigUInt64LE(off + 12),
length: buf.readUInt32LE(off + 20),
flags,
checksum: buf.readUInt32LE(off + 28),
};
}
static isValidEntry(buf: Buffer, index: number): boolean {
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
const flags = buf.readUInt32LE(off + 24);
return (flags & FLAG_VALID) !== 0;
} }
static calcFileSize(entryCount: number): number { static calcFileSize(entryCount: number): number {

View File

@@ -1,5 +1,6 @@
// src/index-file/reader.ts // src/index-file/reader.ts
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as fsext from 'fs-ext';
import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js'; import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js';
import { IndexProtocol } from './protocol.js'; import { IndexProtocol } from './protocol.js';
import type { IndexHeader, IndexEntry } from './types.js'; import type { IndexHeader, IndexEntry } from './types.js';
@@ -14,7 +15,8 @@ export class IndexReader {
private currentIndex: number = -1; private currentIndex: number = -1;
private currentSequence: number = -1; private currentSequence: number = -1;
// Reusable buffer for reading entries // Reusable buffers
private headerBuf: Buffer = Buffer.alloc(INDEX_HEADER_SIZE);
private entryBuffer: Buffer = Buffer.alloc(INDEX_ENTRY_SIZE); private entryBuffer: Buffer = Buffer.alloc(INDEX_ENTRY_SIZE);
constructor() { constructor() {
@@ -22,12 +24,16 @@ export class IndexReader {
open(idxFilePath: string): void { open(idxFilePath: string): void {
this.path = idxFilePath; this.path = idxFilePath;
// 버퍼 초기화
this.headerBuf.fill(0);
this.entryBuffer.fill(0);
this.fd = fs.openSync(this.path, 'r'); this.fd = fs.openSync(this.path, 'r');
// Read header only (64 bytes) // Read header only (64 bytes)
const headerBuf = Buffer.alloc(INDEX_HEADER_SIZE); fs.readSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, null);
fs.readSync(this.fd, headerBuf, 0, INDEX_HEADER_SIZE, 0); this.header = IndexProtocol.readHeader(this.headerBuf);
this.header = IndexProtocol.readHeader(headerBuf);
// 아예 최초에는 -1 로 두어서 readNextEntry 할때 자동 ++ 할때 오류를 검증 // 아예 최초에는 -1 로 두어서 readNextEntry 할때 자동 ++ 할때 오류를 검증
this.currentIndex = -1; this.currentIndex = -1;
this.currentSequence = -1; this.currentSequence = -1;
@@ -136,6 +142,10 @@ export class IndexReader {
this.header = null; this.header = null;
this.currentIndex = -1; this.currentIndex = -1;
this.currentSequence = -1; this.currentSequence = -1;
// 버퍼 초기화
this.headerBuf.fill(0);
this.entryBuffer.fill(0);
} }
// ###################################################################### // ######################################################################
@@ -143,7 +153,8 @@ export class IndexReader {
// ###################################################################### // ######################################################################
private readEntryAt(fd: number, index: number): IndexEntry | null { private readEntryAt(fd: number, index: number): IndexEntry | null {
const offset = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE; const offset = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
fs.readSync(fd, this.entryBuffer, 0, INDEX_ENTRY_SIZE, offset); fsext.seekSync(fd, offset, 0);
fs.readSync(fd, this.entryBuffer, 0, INDEX_ENTRY_SIZE, null);
const flags = this.entryBuffer.readUInt32LE(24); const flags = this.entryBuffer.readUInt32LE(24);
const indexEntry = this.buildIndexEntry(flags); const indexEntry = this.buildIndexEntry(flags);

View File

@@ -1,13 +1,14 @@
// src/index-file/writer.ts // src/index-file/writer.ts
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as fsext from 'fs-ext';
import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js'; import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js';
import { IndexProtocol } from './protocol.js'; import { IndexProtocol } from './protocol.js';
import { IndexFileOptionsRequired } from './types.js'; import { IndexFileOptionsRequired } from './types.js';
export class IndexWriter { export class IndexWriter {
private fd: number | null = null; private fd: number | null = null;
private headerBuf: Buffer | null = null; private headerBuf: Buffer = Buffer.alloc(INDEX_HEADER_SIZE);
private entryBuf: Buffer | null = null; private entryBuf: Buffer = Buffer.alloc(INDEX_ENTRY_SIZE);
private writtenCnt = 0; private writtenCnt = 0;
private dataFileSize = 0n; private dataFileSize = 0n;
@@ -16,6 +17,7 @@ export class IndexWriter {
private path: string | null = null; private path: string | null = null;
private fileSize: number = 0; private fileSize: number = 0;
// see IndexFileOptions // see IndexFileOptions
private maxEntries: number = 0; private maxEntries: number = 0;
private autoIncrementSequence: boolean = false; private autoIncrementSequence: boolean = false;
@@ -31,6 +33,10 @@ export class IndexWriter {
const isNew = !fs.existsSync(this.path); const isNew = !fs.existsSync(this.path);
// 버퍼 초기화
this.headerBuf.fill(0);
this.entryBuf.fill(0);
if (isNew || forceTruncate) { if (isNew || forceTruncate) {
// New file: use provided values // New file: use provided values
this.fileSize = IndexProtocol.calcFileSize(this.maxEntries); this.fileSize = IndexProtocol.calcFileSize(this.maxEntries);
@@ -41,25 +47,18 @@ export class IndexWriter {
this.fd = fs.openSync(this.path, 'w+'); this.fd = fs.openSync(this.path, 'w+');
fs.ftruncateSync(this.fd, this.fileSize); fs.ftruncateSync(this.fd, this.fileSize);
// Allocate buffers for header and entry
this.headerBuf = Buffer.alloc(INDEX_HEADER_SIZE);
this.entryBuf = Buffer.alloc(INDEX_ENTRY_SIZE);
const header = IndexProtocol.createHeader(this.maxEntries, this.autoIncrementSequence); const header = IndexProtocol.createHeader(this.maxEntries, this.autoIncrementSequence);
header.copy(this.headerBuf, 0); header.copy(this.headerBuf, 0);
// Write header to file // Write header to file
fs.writeSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0); fs.writeSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, null);
fs.fsyncSync(this.fd); fs.fsyncSync(this.fd);
} else { } else {
// Existing file: read header first // Existing file: read header first
this.fd = fs.openSync(this.path, 'r+'); this.fd = fs.openSync(this.path, 'r+');
try { try {
this.headerBuf = Buffer.alloc(INDEX_HEADER_SIZE); fs.readSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, null);
this.entryBuf = Buffer.alloc(INDEX_ENTRY_SIZE);
fs.readSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0);
const header = IndexProtocol.readHeader(this.headerBuf); const header = IndexProtocol.readHeader(this.headerBuf);
if (this.maxEntries !== header.entryCount) { if (this.maxEntries !== header.entryCount) {
@@ -86,14 +85,17 @@ export class IndexWriter {
this.writtenCnt = header.writtenCnt; this.writtenCnt = header.writtenCnt;
this.dataFileSize = header.dataFileSize; this.dataFileSize = header.dataFileSize;
this.latestSequence = header.latestSequence; this.latestSequence = header.latestSequence;
// Writer 중 이미 있었으면 가장 마지막으로 이동
const fileOffset = INDEX_HEADER_SIZE + this.writtenCnt * INDEX_ENTRY_SIZE;
fsext.seekSync(this.fd, fileOffset, 0);
} catch (error) { } catch (error) {
// Clean up resources on error // Clean up resources on error
if (this.fd !== null) { if (this.fd !== null) {
fs.closeSync(this.fd); fs.closeSync(this.fd);
this.fd = null; this.fd = null;
} }
this.headerBuf = null; this.headerBuf.fill(0);
this.entryBuf = null; this.entryBuf.fill(0);
throw error; throw error;
} }
} }
@@ -107,7 +109,7 @@ export class IndexWriter {
sequence?: number, sequence?: number,
timestamp?: bigint timestamp?: bigint
): boolean { ): boolean {
if (!this.entryBuf || this.fd === null) throw new Error('Index file not opened'); if (this.fd === null) throw new Error('Index file not opened');
if (this.writtenCnt >= this.maxEntries) { if (this.writtenCnt >= this.maxEntries) {
throw new Error(`Data count exceed provide : ${this.writtenCnt + 1} - max : ${this.maxEntries}`); throw new Error(`Data count exceed provide : ${this.writtenCnt + 1} - max : ${this.maxEntries}`);
} }
@@ -125,11 +127,8 @@ export class IndexWriter {
const ts = timestamp ?? BigInt(Date.now()) * 1000000n; const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
// Create a temporary buffer for this entry // Write entry to entryBuf (offset 0부터)
const tempBuf = Buffer.alloc(INDEX_HEADER_SIZE + (this.writtenCnt + 1) * INDEX_ENTRY_SIZE); IndexProtocol.writeEntry(this.entryBuf, {
// Write entry to temp buffer
IndexProtocol.writeEntry(tempBuf, this.writtenCnt, {
sequence: seq, sequence: seq,
timestamp: ts, timestamp: ts,
offset, offset,
@@ -137,11 +136,8 @@ export class IndexWriter {
flags: FLAG_VALID, flags: FLAG_VALID,
}); });
// Calculate file offset for this entry // Write entry to file (현재 fd 위치에 쓰기)
const fileOffset = INDEX_HEADER_SIZE + this.writtenCnt * INDEX_ENTRY_SIZE; fs.writeSync(this.fd, this.entryBuf, 0, INDEX_ENTRY_SIZE, null);
// Write entry to file
fs.writeSync(this.fd, tempBuf, fileOffset, INDEX_ENTRY_SIZE, fileOffset);
this.writtenCnt++; this.writtenCnt++;
this.latestSequence = seq; this.latestSequence = seq;
@@ -162,12 +158,12 @@ export class IndexWriter {
} }
getFlags(): number { getFlags(): number {
if (!this.headerBuf) throw new Error('Index file not opened'); if (this.fd === null) throw new Error('Index file not opened');
return this.headerBuf.readUInt32LE(41); return this.headerBuf.readUInt32LE(41);
} }
writeHeader(): void { writeHeader(): void {
if (!this.headerBuf || this.fd === null) return; if (this.fd === null) return;
// Update header counts // Update header counts
IndexProtocol.updateHeaderCounts( IndexProtocol.updateHeaderCounts(
@@ -202,12 +198,12 @@ export class IndexWriter {
this.fd = null; this.fd = null;
// 3. Clean up buffers // 3. Clean up buffers
this.headerBuf = null; this.headerBuf.fill(0);
this.entryBuf = null; this.entryBuf.fill(0);
} }
writeFlags(flags: number): void { writeFlags(flags: number): void {
if (!this.headerBuf || this.fd === null) { if (this.fd === null) {
throw new Error('Index file not opened'); throw new Error('Index file not opened');
} }

View File

@@ -13,8 +13,10 @@
"tsconfig.json" "tsconfig.json"
], ],
"dependencies": { "dependencies": {
"typescript": "^5.7.0", "@types/fs-ext": "^2.0.3",
"@types/node": "^22.0.0" "@types/node": "^22.0.0",
"fs-ext": "^2.1.1",
"typescript": "^5.7.0"
}, },
"scripts": { "scripts": {
"prepare": "tsc -p tsconfig.json", "prepare": "tsc -p tsconfig.json",
@@ -23,6 +25,12 @@
"engines": { "engines": {
"node": ">=18.0.0" "node": ">=18.0.0"
}, },
"os": ["linux", "darwin"], "os": [
"cpu": ["x64", "arm64"] "linux",
"darwin"
],
"cpu": [
"x64",
"arm64"
]
} }