sati-open/sati.n-api.index-file
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Read writer memory optimize

Browse Source
This commit is contained in:
Eli-Class
2026-02-05 05:36:36 +00:00
parent 132189da48
commit 92a2681292
8 changed files with 118 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
lib/dat/protocol.ts
View File

@@ -12,6 +12,10 @@ export interface DataHeader {
reserved: Buffer;
}
export interface BufferRef {
buf: Buffer;
}
export class DataProtocol {
static createHeader(): Buffer {
const buf = Buffer.alloc(DATA_HEADER_SIZE);
@@ -54,6 +58,29 @@ export class DataProtocol {
return buf;
}
/**
* 기존 버퍼에 레코드를 직렬화하여 쓰기
* 버퍼 크기 부족시 내부에서 재할당 (BufferRef로 참조 전달)
* @returns 실제 쓰여진 바이트 수 (totalLen)
*/
static serializeRecordTo<T>(bufRef: BufferRef, data: T, serializer: Serializer<T>): number {
const dataBytes = serializer.serialize(data);
const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
// 버퍼 크기 부족시 재할당
if (bufRef.buf.length < totalLen) {
bufRef.buf = Buffer.alloc(totalLen);
}
dataBytes.copy(bufRef.buf, RECORD_HEADER_SIZE);
bufRef.buf.writeUInt32LE(dataBytes.length, 0);
const checksum = crc32(bufRef.buf, RECORD_HEADER_SIZE, totalLen);
bufRef.buf.writeUInt32LE(checksum, 4);
return totalLen;
}
static deserializeRecord<T>(
buf: Buffer,
offset: number,

6
lib/dat/reader.ts
View File

@@ -1,5 +1,6 @@
// src/data-file/reader.ts
import * as fs from 'node:fs';
import * as fsext from 'fs-ext';
import { DATA_HEADER_SIZE } from './constants.js';
import { DataProtocol, DataHeader } from './protocol.js';
import { IndexReader } from '../idx/index.js';
@@ -28,7 +29,7 @@ export class DataReader<T> {
// Read header only
const headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
fs.readSync(this.fd, headerBuf, 0, DATA_HEADER_SIZE, 0);
fs.readSync(this.fd, headerBuf, 0, DATA_HEADER_SIZE, null);
this.header = DataProtocol.readHeader(headerBuf);
this.indexReader.open(this.indexPath);
@@ -36,7 +37,8 @@ export class DataReader<T> {
private readRecordAt(fd: number, offset: bigint, length: number): { data: T; length: number } | null {
const buf = Buffer.alloc(length);
fs.readSync(fd, buf, 0, length, Number(offset));
fsext.seekSync(fd, Number(offset), 0);
fs.readSync(fd, buf, 0, length, null);
const result = DataProtocol.deserializeRecord(
buf,
0,

38
lib/dat/writer.ts
View File

@@ -1,13 +1,17 @@
// src/data-file/writer.ts
import * as fs from 'node:fs';
import * as fsext from 'fs-ext';
import { DATA_HEADER_SIZE } from './constants.js';
import { DataProtocol } from './protocol.js';
import { DataProtocol, BufferRef } from './protocol.js';
import { IndexWriter, IndexFileOptionsRequired } from '../idx/index.js';
import type { Serializer, DataFileOptions } from './types.js';
const DEFAULT_DATA_BUF_SIZE = 4096;
export class DataWriter<T> {
private fd: number | null = null;
private headerBuf: Buffer | null = null;
private headerBuf: Buffer = Buffer.alloc(DATA_HEADER_SIZE);
private dataBufRef: BufferRef = { buf: Buffer.alloc(DEFAULT_DATA_BUF_SIZE) };
private currentOffset: bigint = BigInt(DATA_HEADER_SIZE);
private recordCount = 0;
@@ -65,17 +69,19 @@ export class DataWriter<T> {
this.fd = fs.openSync(this.dataPath,
isNew || this.forceTruncate ? 'w+' : 'r+');
try {
this.headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
// 버퍼 초기화
this.headerBuf.fill(0);
this.dataBufRef.buf.fill(0);
try {
if (isNew || this.forceTruncate) {
const header = DataProtocol.createHeader();
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, 0);
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, null);
this.currentOffset = BigInt(DATA_HEADER_SIZE);
this.recordCount = 0;
this.latestSequence = 0;
} else {
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, null);
const header = DataProtocol.readHeader(this.headerBuf);
// Validate: Data file recordCount must match Index file writtenCnt
@@ -88,6 +94,7 @@ export class DataWriter<T> {
this.currentOffset = header.fileSize;
this.recordCount = header.recordCount;
this.latestSequence = this.indexWriter.getLatestSequence();
fsext.seekSync(this.fd, 0, 2); // 가장 마지막으로
}
} catch (error) {
// Clean up resources on error
@@ -95,7 +102,8 @@ export class DataWriter<T> {
fs.closeSync(this.fd);
this.fd = null;
}
this.headerBuf = null;
this.headerBuf.fill(0);
this.dataBufRef.buf.fill(0);
throw error;
}
}
@@ -103,18 +111,19 @@ export class DataWriter<T> {
append(data: T, sequence?: number, timestamp?: bigint): number {
if (this.fd === null) throw new Error('Data file not opened');
const buf = DataProtocol.serializeRecord(data, this.serializer);
// dataBufRef에 직렬화 (내부에서 필요시 재할당)
const writtenLen = DataProtocol.serializeRecordTo(this.dataBufRef, data, this.serializer);
const offset = this.currentOffset;
fs.writeSync(this.fd, buf, 0, buf.length, Number(offset));
fs.writeSync(this.fd, this.dataBufRef.buf, 0, writtenLen, null);
// Write to index file
this.indexWriter.write(offset, buf.length, sequence, timestamp);
this.indexWriter.write(offset, writtenLen, sequence, timestamp);
// Update latestSequence to the most recent sequence
this.latestSequence = this.indexWriter.getLatestSequence();
this.currentOffset += BigInt(buf.length);
this.currentOffset += BigInt(writtenLen);
++this.recordCount;
this.writeHeader();
@@ -151,14 +160,14 @@ export class DataWriter<T> {
}
writeHeader(): void {
if (this.fd === null || !this.headerBuf) return;
if (this.fd === null) return;
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
}
sync(): void {
if (this.fd === null || !this.headerBuf) return;
if (this.fd === null) return;
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
@@ -174,7 +183,8 @@ export class DataWriter<T> {
}
this.indexWriter.close();
this.headerBuf = null;
this.headerBuf.fill(0);
this.dataBufRef.buf.fill(0);
}
writeFlags(flags: number): void {