Read writer memory optimize
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -3,4 +3,5 @@ build/
|
|||||||
dist/
|
dist/
|
||||||
*.log
|
*.log
|
||||||
*.node
|
*.node
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
package-lock.json
|
||||||
|
|||||||
@@ -12,6 +12,10 @@ export interface DataHeader {
|
|||||||
reserved: Buffer;
|
reserved: Buffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface BufferRef {
|
||||||
|
buf: Buffer;
|
||||||
|
}
|
||||||
|
|
||||||
export class DataProtocol {
|
export class DataProtocol {
|
||||||
static createHeader(): Buffer {
|
static createHeader(): Buffer {
|
||||||
const buf = Buffer.alloc(DATA_HEADER_SIZE);
|
const buf = Buffer.alloc(DATA_HEADER_SIZE);
|
||||||
@@ -54,6 +58,29 @@ export class DataProtocol {
|
|||||||
return buf;
|
return buf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 기존 버퍼에 레코드를 직렬화하여 쓰기
|
||||||
|
* 버퍼 크기 부족시 내부에서 재할당 (BufferRef로 참조 전달)
|
||||||
|
* @returns 실제 쓰여진 바이트 수 (totalLen)
|
||||||
|
*/
|
||||||
|
static serializeRecordTo<T>(bufRef: BufferRef, data: T, serializer: Serializer<T>): number {
|
||||||
|
const dataBytes = serializer.serialize(data);
|
||||||
|
const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
|
||||||
|
|
||||||
|
// 버퍼 크기 부족시 재할당
|
||||||
|
if (bufRef.buf.length < totalLen) {
|
||||||
|
bufRef.buf = Buffer.alloc(totalLen);
|
||||||
|
}
|
||||||
|
|
||||||
|
dataBytes.copy(bufRef.buf, RECORD_HEADER_SIZE);
|
||||||
|
|
||||||
|
bufRef.buf.writeUInt32LE(dataBytes.length, 0);
|
||||||
|
const checksum = crc32(bufRef.buf, RECORD_HEADER_SIZE, totalLen);
|
||||||
|
bufRef.buf.writeUInt32LE(checksum, 4);
|
||||||
|
|
||||||
|
return totalLen;
|
||||||
|
}
|
||||||
|
|
||||||
static deserializeRecord<T>(
|
static deserializeRecord<T>(
|
||||||
buf: Buffer,
|
buf: Buffer,
|
||||||
offset: number,
|
offset: number,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
// src/data-file/reader.ts
|
// src/data-file/reader.ts
|
||||||
import * as fs from 'node:fs';
|
import * as fs from 'node:fs';
|
||||||
|
import * as fsext from 'fs-ext';
|
||||||
import { DATA_HEADER_SIZE } from './constants.js';
|
import { DATA_HEADER_SIZE } from './constants.js';
|
||||||
import { DataProtocol, DataHeader } from './protocol.js';
|
import { DataProtocol, DataHeader } from './protocol.js';
|
||||||
import { IndexReader } from '../idx/index.js';
|
import { IndexReader } from '../idx/index.js';
|
||||||
@@ -28,7 +29,7 @@ export class DataReader<T> {
|
|||||||
|
|
||||||
// Read header only
|
// Read header only
|
||||||
const headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
|
const headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
|
||||||
fs.readSync(this.fd, headerBuf, 0, DATA_HEADER_SIZE, 0);
|
fs.readSync(this.fd, headerBuf, 0, DATA_HEADER_SIZE, null);
|
||||||
this.header = DataProtocol.readHeader(headerBuf);
|
this.header = DataProtocol.readHeader(headerBuf);
|
||||||
|
|
||||||
this.indexReader.open(this.indexPath);
|
this.indexReader.open(this.indexPath);
|
||||||
@@ -36,7 +37,8 @@ export class DataReader<T> {
|
|||||||
|
|
||||||
private readRecordAt(fd: number, offset: bigint, length: number): { data: T; length: number } | null {
|
private readRecordAt(fd: number, offset: bigint, length: number): { data: T; length: number } | null {
|
||||||
const buf = Buffer.alloc(length);
|
const buf = Buffer.alloc(length);
|
||||||
fs.readSync(fd, buf, 0, length, Number(offset));
|
fsext.seekSync(fd, Number(offset), 0);
|
||||||
|
fs.readSync(fd, buf, 0, length, null);
|
||||||
const result = DataProtocol.deserializeRecord(
|
const result = DataProtocol.deserializeRecord(
|
||||||
buf,
|
buf,
|
||||||
0,
|
0,
|
||||||
|
|||||||
@@ -1,13 +1,17 @@
|
|||||||
// src/data-file/writer.ts
|
// src/data-file/writer.ts
|
||||||
import * as fs from 'node:fs';
|
import * as fs from 'node:fs';
|
||||||
|
import * as fsext from 'fs-ext';
|
||||||
import { DATA_HEADER_SIZE } from './constants.js';
|
import { DATA_HEADER_SIZE } from './constants.js';
|
||||||
import { DataProtocol } from './protocol.js';
|
import { DataProtocol, BufferRef } from './protocol.js';
|
||||||
import { IndexWriter, IndexFileOptionsRequired } from '../idx/index.js';
|
import { IndexWriter, IndexFileOptionsRequired } from '../idx/index.js';
|
||||||
import type { Serializer, DataFileOptions } from './types.js';
|
import type { Serializer, DataFileOptions } from './types.js';
|
||||||
|
|
||||||
|
const DEFAULT_DATA_BUF_SIZE = 4096;
|
||||||
|
|
||||||
export class DataWriter<T> {
|
export class DataWriter<T> {
|
||||||
private fd: number | null = null;
|
private fd: number | null = null;
|
||||||
private headerBuf: Buffer | null = null;
|
private headerBuf: Buffer = Buffer.alloc(DATA_HEADER_SIZE);
|
||||||
|
private dataBufRef: BufferRef = { buf: Buffer.alloc(DEFAULT_DATA_BUF_SIZE) };
|
||||||
private currentOffset: bigint = BigInt(DATA_HEADER_SIZE);
|
private currentOffset: bigint = BigInt(DATA_HEADER_SIZE);
|
||||||
private recordCount = 0;
|
private recordCount = 0;
|
||||||
|
|
||||||
@@ -65,17 +69,19 @@ export class DataWriter<T> {
|
|||||||
this.fd = fs.openSync(this.dataPath,
|
this.fd = fs.openSync(this.dataPath,
|
||||||
isNew || this.forceTruncate ? 'w+' : 'r+');
|
isNew || this.forceTruncate ? 'w+' : 'r+');
|
||||||
|
|
||||||
try {
|
// 버퍼 초기화
|
||||||
this.headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
|
this.headerBuf.fill(0);
|
||||||
|
this.dataBufRef.buf.fill(0);
|
||||||
|
|
||||||
|
try {
|
||||||
if (isNew || this.forceTruncate) {
|
if (isNew || this.forceTruncate) {
|
||||||
const header = DataProtocol.createHeader();
|
const header = DataProtocol.createHeader();
|
||||||
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, 0);
|
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, null);
|
||||||
this.currentOffset = BigInt(DATA_HEADER_SIZE);
|
this.currentOffset = BigInt(DATA_HEADER_SIZE);
|
||||||
this.recordCount = 0;
|
this.recordCount = 0;
|
||||||
this.latestSequence = 0;
|
this.latestSequence = 0;
|
||||||
} else {
|
} else {
|
||||||
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
|
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, null);
|
||||||
const header = DataProtocol.readHeader(this.headerBuf);
|
const header = DataProtocol.readHeader(this.headerBuf);
|
||||||
|
|
||||||
// Validate: Data file recordCount must match Index file writtenCnt
|
// Validate: Data file recordCount must match Index file writtenCnt
|
||||||
@@ -88,6 +94,7 @@ export class DataWriter<T> {
|
|||||||
this.currentOffset = header.fileSize;
|
this.currentOffset = header.fileSize;
|
||||||
this.recordCount = header.recordCount;
|
this.recordCount = header.recordCount;
|
||||||
this.latestSequence = this.indexWriter.getLatestSequence();
|
this.latestSequence = this.indexWriter.getLatestSequence();
|
||||||
|
fsext.seekSync(this.fd, 0, 2); // 가장 마지막으로
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Clean up resources on error
|
// Clean up resources on error
|
||||||
@@ -95,7 +102,8 @@ export class DataWriter<T> {
|
|||||||
fs.closeSync(this.fd);
|
fs.closeSync(this.fd);
|
||||||
this.fd = null;
|
this.fd = null;
|
||||||
}
|
}
|
||||||
this.headerBuf = null;
|
this.headerBuf.fill(0);
|
||||||
|
this.dataBufRef.buf.fill(0);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -103,18 +111,19 @@ export class DataWriter<T> {
|
|||||||
append(data: T, sequence?: number, timestamp?: bigint): number {
|
append(data: T, sequence?: number, timestamp?: bigint): number {
|
||||||
if (this.fd === null) throw new Error('Data file not opened');
|
if (this.fd === null) throw new Error('Data file not opened');
|
||||||
|
|
||||||
const buf = DataProtocol.serializeRecord(data, this.serializer);
|
// dataBufRef에 직렬화 (내부에서 필요시 재할당)
|
||||||
|
const writtenLen = DataProtocol.serializeRecordTo(this.dataBufRef, data, this.serializer);
|
||||||
const offset = this.currentOffset;
|
const offset = this.currentOffset;
|
||||||
|
|
||||||
fs.writeSync(this.fd, buf, 0, buf.length, Number(offset));
|
fs.writeSync(this.fd, this.dataBufRef.buf, 0, writtenLen, null);
|
||||||
|
|
||||||
// Write to index file
|
// Write to index file
|
||||||
this.indexWriter.write(offset, buf.length, sequence, timestamp);
|
this.indexWriter.write(offset, writtenLen, sequence, timestamp);
|
||||||
|
|
||||||
// Update latestSequence to the most recent sequence
|
// Update latestSequence to the most recent sequence
|
||||||
this.latestSequence = this.indexWriter.getLatestSequence();
|
this.latestSequence = this.indexWriter.getLatestSequence();
|
||||||
|
|
||||||
this.currentOffset += BigInt(buf.length);
|
this.currentOffset += BigInt(writtenLen);
|
||||||
++this.recordCount;
|
++this.recordCount;
|
||||||
|
|
||||||
this.writeHeader();
|
this.writeHeader();
|
||||||
@@ -151,14 +160,14 @@ export class DataWriter<T> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
writeHeader(): void {
|
writeHeader(): void {
|
||||||
if (this.fd === null || !this.headerBuf) return;
|
if (this.fd === null) return;
|
||||||
|
|
||||||
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
|
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
|
||||||
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
|
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
sync(): void {
|
sync(): void {
|
||||||
if (this.fd === null || !this.headerBuf) return;
|
if (this.fd === null) return;
|
||||||
|
|
||||||
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
|
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
|
||||||
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
|
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
|
||||||
@@ -174,7 +183,8 @@ export class DataWriter<T> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
this.indexWriter.close();
|
this.indexWriter.close();
|
||||||
this.headerBuf = null;
|
this.headerBuf.fill(0);
|
||||||
|
this.dataBufRef.buf.fill(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
writeFlags(flags: number): void {
|
writeFlags(flags: number): void {
|
||||||
|
|||||||
@@ -73,39 +73,15 @@ export class IndexProtocol {
|
|||||||
buf.writeUInt32LE(latestSequence, 36);
|
buf.writeUInt32LE(latestSequence, 36);
|
||||||
}
|
}
|
||||||
|
|
||||||
static writeEntry(buf: Buffer, index: number, entry: Omit<IndexEntry, 'checksum'>): void {
|
static writeEntry(buf: Buffer, entry: Omit<IndexEntry, 'checksum'>): void {
|
||||||
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
buf.writeUInt32LE(entry.sequence, 0);
|
||||||
|
buf.writeBigUInt64LE(entry.timestamp, 4);
|
||||||
|
buf.writeBigUInt64LE(entry.offset, 12);
|
||||||
|
buf.writeUInt32LE(entry.length, 20);
|
||||||
|
buf.writeUInt32LE(entry.flags | FLAG_VALID, 24);
|
||||||
|
|
||||||
buf.writeUInt32LE(entry.sequence, off);
|
const checksum = crc32(buf, 0, 28);
|
||||||
buf.writeBigUInt64LE(entry.timestamp, off + 4);
|
buf.writeUInt32LE(checksum, 28);
|
||||||
buf.writeBigUInt64LE(entry.offset, off + 12);
|
|
||||||
buf.writeUInt32LE(entry.length, off + 20);
|
|
||||||
buf.writeUInt32LE(entry.flags | FLAG_VALID, off + 24);
|
|
||||||
|
|
||||||
const checksum = crc32(buf, off, off + 28);
|
|
||||||
buf.writeUInt32LE(checksum, off + 28);
|
|
||||||
}
|
|
||||||
|
|
||||||
static readEntry(buf: Buffer, index: number): IndexEntry | null {
|
|
||||||
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
|
||||||
const flags = buf.readUInt32LE(off + 24);
|
|
||||||
|
|
||||||
if (!(flags & FLAG_VALID)) return null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
sequence: buf.readUInt32LE(off),
|
|
||||||
timestamp: buf.readBigUInt64LE(off + 4),
|
|
||||||
offset: buf.readBigUInt64LE(off + 12),
|
|
||||||
length: buf.readUInt32LE(off + 20),
|
|
||||||
flags,
|
|
||||||
checksum: buf.readUInt32LE(off + 28),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
static isValidEntry(buf: Buffer, index: number): boolean {
|
|
||||||
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
|
||||||
const flags = buf.readUInt32LE(off + 24);
|
|
||||||
return (flags & FLAG_VALID) !== 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static calcFileSize(entryCount: number): number {
|
static calcFileSize(entryCount: number): number {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
// src/index-file/reader.ts
|
// src/index-file/reader.ts
|
||||||
import * as fs from 'node:fs';
|
import * as fs from 'node:fs';
|
||||||
|
import * as fsext from 'fs-ext';
|
||||||
import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js';
|
import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js';
|
||||||
import { IndexProtocol } from './protocol.js';
|
import { IndexProtocol } from './protocol.js';
|
||||||
import type { IndexHeader, IndexEntry } from './types.js';
|
import type { IndexHeader, IndexEntry } from './types.js';
|
||||||
@@ -14,7 +15,8 @@ export class IndexReader {
|
|||||||
private currentIndex: number = -1;
|
private currentIndex: number = -1;
|
||||||
private currentSequence: number = -1;
|
private currentSequence: number = -1;
|
||||||
|
|
||||||
// Reusable buffer for reading entries
|
// Reusable buffers
|
||||||
|
private headerBuf: Buffer = Buffer.alloc(INDEX_HEADER_SIZE);
|
||||||
private entryBuffer: Buffer = Buffer.alloc(INDEX_ENTRY_SIZE);
|
private entryBuffer: Buffer = Buffer.alloc(INDEX_ENTRY_SIZE);
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
@@ -22,12 +24,16 @@ export class IndexReader {
|
|||||||
|
|
||||||
open(idxFilePath: string): void {
|
open(idxFilePath: string): void {
|
||||||
this.path = idxFilePath;
|
this.path = idxFilePath;
|
||||||
|
|
||||||
|
// 버퍼 초기화
|
||||||
|
this.headerBuf.fill(0);
|
||||||
|
this.entryBuffer.fill(0);
|
||||||
|
|
||||||
this.fd = fs.openSync(this.path, 'r');
|
this.fd = fs.openSync(this.path, 'r');
|
||||||
|
|
||||||
// Read header only (64 bytes)
|
// Read header only (64 bytes)
|
||||||
const headerBuf = Buffer.alloc(INDEX_HEADER_SIZE);
|
fs.readSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, null);
|
||||||
fs.readSync(this.fd, headerBuf, 0, INDEX_HEADER_SIZE, 0);
|
this.header = IndexProtocol.readHeader(this.headerBuf);
|
||||||
this.header = IndexProtocol.readHeader(headerBuf);
|
|
||||||
// 아예 최초에는 -1 로 두어서 readNextEntry 할때 자동 ++ 할때 오류를 검증
|
// 아예 최초에는 -1 로 두어서 readNextEntry 할때 자동 ++ 할때 오류를 검증
|
||||||
this.currentIndex = -1;
|
this.currentIndex = -1;
|
||||||
this.currentSequence = -1;
|
this.currentSequence = -1;
|
||||||
@@ -136,6 +142,10 @@ export class IndexReader {
|
|||||||
this.header = null;
|
this.header = null;
|
||||||
this.currentIndex = -1;
|
this.currentIndex = -1;
|
||||||
this.currentSequence = -1;
|
this.currentSequence = -1;
|
||||||
|
|
||||||
|
// 버퍼 초기화
|
||||||
|
this.headerBuf.fill(0);
|
||||||
|
this.entryBuffer.fill(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ######################################################################
|
// ######################################################################
|
||||||
@@ -143,7 +153,8 @@ export class IndexReader {
|
|||||||
// ######################################################################
|
// ######################################################################
|
||||||
private readEntryAt(fd: number, index: number): IndexEntry | null {
|
private readEntryAt(fd: number, index: number): IndexEntry | null {
|
||||||
const offset = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
const offset = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
||||||
fs.readSync(fd, this.entryBuffer, 0, INDEX_ENTRY_SIZE, offset);
|
fsext.seekSync(fd, offset, 0);
|
||||||
|
fs.readSync(fd, this.entryBuffer, 0, INDEX_ENTRY_SIZE, null);
|
||||||
|
|
||||||
const flags = this.entryBuffer.readUInt32LE(24);
|
const flags = this.entryBuffer.readUInt32LE(24);
|
||||||
const indexEntry = this.buildIndexEntry(flags);
|
const indexEntry = this.buildIndexEntry(flags);
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
// src/index-file/writer.ts
|
// src/index-file/writer.ts
|
||||||
import * as fs from 'node:fs';
|
import * as fs from 'node:fs';
|
||||||
|
import * as fsext from 'fs-ext';
|
||||||
import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js';
|
import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js';
|
||||||
import { IndexProtocol } from './protocol.js';
|
import { IndexProtocol } from './protocol.js';
|
||||||
import { IndexFileOptionsRequired } from './types.js';
|
import { IndexFileOptionsRequired } from './types.js';
|
||||||
|
|
||||||
export class IndexWriter {
|
export class IndexWriter {
|
||||||
private fd: number | null = null;
|
private fd: number | null = null;
|
||||||
private headerBuf: Buffer | null = null;
|
private headerBuf: Buffer = Buffer.alloc(INDEX_HEADER_SIZE);
|
||||||
private entryBuf: Buffer | null = null;
|
private entryBuf: Buffer = Buffer.alloc(INDEX_ENTRY_SIZE);
|
||||||
|
|
||||||
private writtenCnt = 0;
|
private writtenCnt = 0;
|
||||||
private dataFileSize = 0n;
|
private dataFileSize = 0n;
|
||||||
@@ -16,6 +17,7 @@ export class IndexWriter {
|
|||||||
private path: string | null = null;
|
private path: string | null = null;
|
||||||
private fileSize: number = 0;
|
private fileSize: number = 0;
|
||||||
|
|
||||||
|
|
||||||
// see IndexFileOptions
|
// see IndexFileOptions
|
||||||
private maxEntries: number = 0;
|
private maxEntries: number = 0;
|
||||||
private autoIncrementSequence: boolean = false;
|
private autoIncrementSequence: boolean = false;
|
||||||
@@ -31,6 +33,10 @@ export class IndexWriter {
|
|||||||
|
|
||||||
const isNew = !fs.existsSync(this.path);
|
const isNew = !fs.existsSync(this.path);
|
||||||
|
|
||||||
|
// 버퍼 초기화
|
||||||
|
this.headerBuf.fill(0);
|
||||||
|
this.entryBuf.fill(0);
|
||||||
|
|
||||||
if (isNew || forceTruncate) {
|
if (isNew || forceTruncate) {
|
||||||
// New file: use provided values
|
// New file: use provided values
|
||||||
this.fileSize = IndexProtocol.calcFileSize(this.maxEntries);
|
this.fileSize = IndexProtocol.calcFileSize(this.maxEntries);
|
||||||
@@ -41,25 +47,18 @@ export class IndexWriter {
|
|||||||
this.fd = fs.openSync(this.path, 'w+');
|
this.fd = fs.openSync(this.path, 'w+');
|
||||||
fs.ftruncateSync(this.fd, this.fileSize);
|
fs.ftruncateSync(this.fd, this.fileSize);
|
||||||
|
|
||||||
// Allocate buffers for header and entry
|
|
||||||
this.headerBuf = Buffer.alloc(INDEX_HEADER_SIZE);
|
|
||||||
this.entryBuf = Buffer.alloc(INDEX_ENTRY_SIZE);
|
|
||||||
|
|
||||||
const header = IndexProtocol.createHeader(this.maxEntries, this.autoIncrementSequence);
|
const header = IndexProtocol.createHeader(this.maxEntries, this.autoIncrementSequence);
|
||||||
header.copy(this.headerBuf, 0);
|
header.copy(this.headerBuf, 0);
|
||||||
|
|
||||||
// Write header to file
|
// Write header to file
|
||||||
fs.writeSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0);
|
fs.writeSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, null);
|
||||||
fs.fsyncSync(this.fd);
|
fs.fsyncSync(this.fd);
|
||||||
} else {
|
} else {
|
||||||
// Existing file: read header first
|
// Existing file: read header first
|
||||||
this.fd = fs.openSync(this.path, 'r+');
|
this.fd = fs.openSync(this.path, 'r+');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
this.headerBuf = Buffer.alloc(INDEX_HEADER_SIZE);
|
fs.readSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, null);
|
||||||
this.entryBuf = Buffer.alloc(INDEX_ENTRY_SIZE);
|
|
||||||
|
|
||||||
fs.readSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0);
|
|
||||||
const header = IndexProtocol.readHeader(this.headerBuf);
|
const header = IndexProtocol.readHeader(this.headerBuf);
|
||||||
|
|
||||||
if (this.maxEntries !== header.entryCount) {
|
if (this.maxEntries !== header.entryCount) {
|
||||||
@@ -86,14 +85,17 @@ export class IndexWriter {
|
|||||||
this.writtenCnt = header.writtenCnt;
|
this.writtenCnt = header.writtenCnt;
|
||||||
this.dataFileSize = header.dataFileSize;
|
this.dataFileSize = header.dataFileSize;
|
||||||
this.latestSequence = header.latestSequence;
|
this.latestSequence = header.latestSequence;
|
||||||
|
// Writer 중 이미 있었으면 가장 마지막으로 이동
|
||||||
|
const fileOffset = INDEX_HEADER_SIZE + this.writtenCnt * INDEX_ENTRY_SIZE;
|
||||||
|
fsext.seekSync(this.fd, fileOffset, 0);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Clean up resources on error
|
// Clean up resources on error
|
||||||
if (this.fd !== null) {
|
if (this.fd !== null) {
|
||||||
fs.closeSync(this.fd);
|
fs.closeSync(this.fd);
|
||||||
this.fd = null;
|
this.fd = null;
|
||||||
}
|
}
|
||||||
this.headerBuf = null;
|
this.headerBuf.fill(0);
|
||||||
this.entryBuf = null;
|
this.entryBuf.fill(0);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -107,7 +109,7 @@ export class IndexWriter {
|
|||||||
sequence?: number,
|
sequence?: number,
|
||||||
timestamp?: bigint
|
timestamp?: bigint
|
||||||
): boolean {
|
): boolean {
|
||||||
if (!this.entryBuf || this.fd === null) throw new Error('Index file not opened');
|
if (this.fd === null) throw new Error('Index file not opened');
|
||||||
if (this.writtenCnt >= this.maxEntries) {
|
if (this.writtenCnt >= this.maxEntries) {
|
||||||
throw new Error(`Data count exceed provide : ${this.writtenCnt + 1} - max : ${this.maxEntries}`);
|
throw new Error(`Data count exceed provide : ${this.writtenCnt + 1} - max : ${this.maxEntries}`);
|
||||||
}
|
}
|
||||||
@@ -125,11 +127,8 @@ export class IndexWriter {
|
|||||||
|
|
||||||
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
|
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
|
||||||
|
|
||||||
// Create a temporary buffer for this entry
|
// Write entry to entryBuf (offset 0부터)
|
||||||
const tempBuf = Buffer.alloc(INDEX_HEADER_SIZE + (this.writtenCnt + 1) * INDEX_ENTRY_SIZE);
|
IndexProtocol.writeEntry(this.entryBuf, {
|
||||||
|
|
||||||
// Write entry to temp buffer
|
|
||||||
IndexProtocol.writeEntry(tempBuf, this.writtenCnt, {
|
|
||||||
sequence: seq,
|
sequence: seq,
|
||||||
timestamp: ts,
|
timestamp: ts,
|
||||||
offset,
|
offset,
|
||||||
@@ -137,11 +136,8 @@ export class IndexWriter {
|
|||||||
flags: FLAG_VALID,
|
flags: FLAG_VALID,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Calculate file offset for this entry
|
// Write entry to file (현재 fd 위치에 쓰기)
|
||||||
const fileOffset = INDEX_HEADER_SIZE + this.writtenCnt * INDEX_ENTRY_SIZE;
|
fs.writeSync(this.fd, this.entryBuf, 0, INDEX_ENTRY_SIZE, null);
|
||||||
|
|
||||||
// Write entry to file
|
|
||||||
fs.writeSync(this.fd, tempBuf, fileOffset, INDEX_ENTRY_SIZE, fileOffset);
|
|
||||||
|
|
||||||
this.writtenCnt++;
|
this.writtenCnt++;
|
||||||
this.latestSequence = seq;
|
this.latestSequence = seq;
|
||||||
@@ -162,12 +158,12 @@ export class IndexWriter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
getFlags(): number {
|
getFlags(): number {
|
||||||
if (!this.headerBuf) throw new Error('Index file not opened');
|
if (this.fd === null) throw new Error('Index file not opened');
|
||||||
return this.headerBuf.readUInt32LE(41);
|
return this.headerBuf.readUInt32LE(41);
|
||||||
}
|
}
|
||||||
|
|
||||||
writeHeader(): void {
|
writeHeader(): void {
|
||||||
if (!this.headerBuf || this.fd === null) return;
|
if (this.fd === null) return;
|
||||||
|
|
||||||
// Update header counts
|
// Update header counts
|
||||||
IndexProtocol.updateHeaderCounts(
|
IndexProtocol.updateHeaderCounts(
|
||||||
@@ -202,12 +198,12 @@ export class IndexWriter {
|
|||||||
this.fd = null;
|
this.fd = null;
|
||||||
|
|
||||||
// 3. Clean up buffers
|
// 3. Clean up buffers
|
||||||
this.headerBuf = null;
|
this.headerBuf.fill(0);
|
||||||
this.entryBuf = null;
|
this.entryBuf.fill(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
writeFlags(flags: number): void {
|
writeFlags(flags: number): void {
|
||||||
if (!this.headerBuf || this.fd === null) {
|
if (this.fd === null) {
|
||||||
throw new Error('Index file not opened');
|
throw new Error('Index file not opened');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
16
package.json
16
package.json
@@ -13,8 +13,10 @@
|
|||||||
"tsconfig.json"
|
"tsconfig.json"
|
||||||
],
|
],
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"typescript": "^5.7.0",
|
"@types/fs-ext": "^2.0.3",
|
||||||
"@types/node": "^22.0.0"
|
"@types/node": "^22.0.0",
|
||||||
|
"fs-ext": "^2.1.1",
|
||||||
|
"typescript": "^5.7.0"
|
||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"prepare": "tsc -p tsconfig.json",
|
"prepare": "tsc -p tsconfig.json",
|
||||||
@@ -23,6 +25,12 @@
|
|||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=18.0.0"
|
"node": ">=18.0.0"
|
||||||
},
|
},
|
||||||
"os": ["linux", "darwin"],
|
"os": [
|
||||||
"cpu": ["x64", "arm64"]
|
"linux",
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"cpu": [
|
||||||
|
"x64",
|
||||||
|
"arm64"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user