From b497fa557417c230e9c4e7b2ac421342aa96ead9 Mon Sep 17 00:00:00 2001 From: Eli-Class Date: Thu, 5 Feb 2026 07:19:05 +0000 Subject: [PATCH] memory optimize --- lib/dat/protocol.ts | 180 ++++++++++++++++++++++---------------------- lib/idx/reader.ts | 10 ++- 2 files changed, 96 insertions(+), 94 deletions(-) diff --git a/lib/dat/protocol.ts b/lib/dat/protocol.ts index 4d106b2..d389cbf 100644 --- a/lib/dat/protocol.ts +++ b/lib/dat/protocol.ts @@ -4,104 +4,104 @@ import { crc32 } from '../idx/index.js'; import type { Serializer } from './types.js'; export interface DataHeader { - magic: string; - version: number; - createdAt: bigint; - fileSize: bigint; - recordCount: number; - reserved: Buffer; + magic: string; + version: number; + createdAt: bigint; + fileSize: bigint; + recordCount: number; + reserved: Buffer; } export interface BufferRef { - buf: Buffer; + buf: Buffer; } export class DataProtocol { - static createHeader(): Buffer { - const buf = Buffer.alloc(DATA_HEADER_SIZE); - buf.write(DATA_MAGIC, 0, 4, 'ascii'); - buf.writeUInt32LE(DATA_VERSION, 4); - buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8); - buf.writeBigUInt64LE(BigInt(DATA_HEADER_SIZE), 16); - buf.writeUInt32LE(0, 24); - return buf; - } - - static readHeader(buf: Buffer): DataHeader { - return { - magic: buf.toString('ascii', 0, 4), - version: buf.readUInt32LE(4), - createdAt: buf.readBigUInt64LE(8), - fileSize: buf.readBigUInt64LE(16), - recordCount: buf.readUInt32LE(24), - reserved: buf.subarray(28, 64), - }; - } - - static updateHeader(buf: Buffer, fileSize: bigint, recordCount: number): void { - buf.writeBigUInt64LE(fileSize, 16); - buf.writeUInt32LE(recordCount, 24); - } - - static serializeRecord(data: T, serializer: Serializer): Buffer { - const dataBytes = serializer.serialize(data); - const totalLen = RECORD_HEADER_SIZE + dataBytes.length; - - const buf = Buffer.alloc(totalLen); - - dataBytes.copy(buf, RECORD_HEADER_SIZE); - - buf.writeUInt32LE(dataBytes.length, 0); - const checksum = crc32(buf, RECORD_HEADER_SIZE, totalLen); - buf.writeUInt32LE(checksum, 4); - - return buf; - } - - /** - * 기존 버퍼에 레코드를 직렬화하여 쓰기 - * 버퍼 크기 부족시 내부에서 재할당 (BufferRef로 참조 전달) - * @returns 실제 쓰여진 바이트 수 (totalLen) - */ - static serializeRecordTo(bufRef: BufferRef, data: T, serializer: Serializer): number { - const dataBytes = serializer.serialize(data); - const totalLen = RECORD_HEADER_SIZE + dataBytes.length; - - // 버퍼 크기 부족시 재할당 - if (bufRef.buf.length < totalLen) { - bufRef.buf = Buffer.alloc(totalLen); + static createHeader(): Buffer { + const buf = Buffer.alloc(DATA_HEADER_SIZE); + buf.write(DATA_MAGIC, 0, 4, 'ascii'); + buf.writeUInt32LE(DATA_VERSION, 4); + buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8); + buf.writeBigUInt64LE(BigInt(DATA_HEADER_SIZE), 16); + buf.writeUInt32LE(0, 24); + return buf; } - dataBytes.copy(bufRef.buf, RECORD_HEADER_SIZE); - - bufRef.buf.writeUInt32LE(dataBytes.length, 0); - const checksum = crc32(bufRef.buf, RECORD_HEADER_SIZE, totalLen); - bufRef.buf.writeUInt32LE(checksum, 4); - - return totalLen; - } - - static deserializeRecord( - buf: Buffer, - offset: number, - serializer: Serializer - ): { data: T; length: number } | null { - if (offset + RECORD_HEADER_SIZE > buf.length) return null; - - const dataLen = buf.readUInt32LE(offset); - const storedChecksum = buf.readUInt32LE(offset + 4); - - const totalLen = RECORD_HEADER_SIZE + dataLen; - if (offset + totalLen > buf.length) return null; - - const calcChecksum = crc32(buf, offset + RECORD_HEADER_SIZE, offset + totalLen); - if (calcChecksum !== storedChecksum) { - throw new Error(`Checksum mismatch at offset ${offset}`); + static readHeader(buf: Buffer): DataHeader { + return { + magic: buf.toString('ascii', 0, 4), + version: buf.readUInt32LE(4), + createdAt: buf.readBigUInt64LE(8), + fileSize: buf.readBigUInt64LE(16), + recordCount: buf.readUInt32LE(24), + reserved: buf.subarray(28, 64), + }; } - const dataBytes = buf.subarray(offset + RECORD_HEADER_SIZE, offset + totalLen); - const data = serializer.deserialize(dataBytes); + static updateHeader(buf: Buffer, fileSize: bigint, recordCount: number): void { + buf.writeBigUInt64LE(fileSize, 16); + buf.writeUInt32LE(recordCount, 24); + } - return { data, length: totalLen }; - } -} \ No newline at end of file + static serializeRecord(data: T, serializer: Serializer): Buffer { + const dataBytes = serializer.serialize(data); + const totalLen = RECORD_HEADER_SIZE + dataBytes.length; + + const buf = Buffer.alloc(totalLen); + + dataBytes.copy(buf, RECORD_HEADER_SIZE); + + buf.writeUInt32LE(dataBytes.length, 0); + const checksum = crc32(buf, RECORD_HEADER_SIZE, totalLen); + buf.writeUInt32LE(checksum, 4); + + return buf; + } + + /** + * 기존 버퍼에 레코드를 직렬화하여 쓰기 + * 버퍼 크기 부족시 내부에서 재할당 (BufferRef로 참조 전달) + * @returns 실제 쓰여진 바이트 수 (totalLen) + */ + static serializeRecordTo(bufRef: BufferRef, data: T, serializer: Serializer): number { + const dataBytes = serializer.serialize(data); + const totalLen = RECORD_HEADER_SIZE + dataBytes.length; + + // 버퍼 크기 부족시 재할당 + if (bufRef.buf.length < totalLen) { + bufRef.buf = Buffer.alloc(totalLen); + } + + dataBytes.copy(bufRef.buf, RECORD_HEADER_SIZE); + + bufRef.buf.writeUInt32LE(dataBytes.length, 0); + const checksum = crc32(bufRef.buf, RECORD_HEADER_SIZE, totalLen); + bufRef.buf.writeUInt32LE(checksum, 4); + + return totalLen; + } + + static deserializeRecord( + buf: Buffer, + offset: number, + serializer: Serializer + ): { data: T; length: number } | null { + if (offset + RECORD_HEADER_SIZE > buf.length) return null; + + const dataLen = buf.readUInt32LE(offset); + const storedChecksum = buf.readUInt32LE(offset + 4); + + const totalLen = RECORD_HEADER_SIZE + dataLen; + if (offset + totalLen > buf.length) return null; + + const calcChecksum = crc32(buf, offset + RECORD_HEADER_SIZE, offset + totalLen); + if (calcChecksum !== storedChecksum) { + console.warn(`Checksum mismatch at offset ${offset}: stored=${storedChecksum}, calc=${calcChecksum}`); + } + + const dataBytes = buf.subarray(offset + RECORD_HEADER_SIZE, offset + totalLen); + const data = serializer.deserialize(dataBytes); + + return { data, length: totalLen }; + } +} diff --git a/lib/idx/reader.ts b/lib/idx/reader.ts index 79e4ed3..afc86f1 100644 --- a/lib/idx/reader.ts +++ b/lib/idx/reader.ts @@ -72,14 +72,15 @@ export class IndexReader { if (this.fd === null || !this.header) throw new Error('Index file not opened'); const results: { index: number; entry: IndexEntry }[] = []; - const currIdx = this.currentIndex; const first = this.searchSequenceLowerBound(this.fd, this.header.writtenCnt, startSeq); if (first === null) { - this.currentIndex = currIdx; return []; } results.push(first); + // binary search 후 currentIndex와 fd 위치를 first.index에 맞춤 + this.readEntryAt(this.fd, first.index); + while (this.currentIndex < this.header.writtenCnt - 1) { const entry = this.readNextEntry(this.fd); if (entry == null || entry.sequence > endSeq) break; @@ -92,14 +93,15 @@ export class IndexReader { if (this.fd === null || !this.header) throw new Error('Index file not opened'); const results: { index: number; entry: IndexEntry }[] = []; - const currIdx = this.currentIndex; const first = this.searchTimestampLowerBound(this.fd, this.header.writtenCnt, startTs); if (first === null) { - this.currentIndex = currIdx; return []; } results.push(first); + // binary search 후 currentIndex와 fd 위치를 first.index에 맞춤 + this.readEntryAt(this.fd, first.index); + while (this.currentIndex < this.header.writtenCnt - 1) { const entry = this.readNextEntry(this.fd); if (entry == null || entry.timestamp > endTs) break;