memory optimize

lib/dat/protocol.ts

@@ -4,104 +4,104 @@ import { crc32 } from '../idx/index.js';
 import type { Serializer } from './types.js';
 
 export interface DataHeader {
-  magic: string;
+    magic: string;
-  version: number;
+    version: number;
-  createdAt: bigint;
+    createdAt: bigint;
-  fileSize: bigint;
+    fileSize: bigint;
-  recordCount: number;
+    recordCount: number;
-  reserved: Buffer;
+    reserved: Buffer;
 }
 
 export interface BufferRef {
-  buf: Buffer;
+    buf: Buffer;
 }
 
 export class DataProtocol {
-  static createHeader(): Buffer {
+    static createHeader(): Buffer {
-    const buf = Buffer.alloc(DATA_HEADER_SIZE);
+        const buf = Buffer.alloc(DATA_HEADER_SIZE);
-    buf.write(DATA_MAGIC, 0, 4, 'ascii');
+        buf.write(DATA_MAGIC, 0, 4, 'ascii');
-    buf.writeUInt32LE(DATA_VERSION, 4);
+        buf.writeUInt32LE(DATA_VERSION, 4);
-    buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
+        buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
-    buf.writeBigUInt64LE(BigInt(DATA_HEADER_SIZE), 16);
+        buf.writeBigUInt64LE(BigInt(DATA_HEADER_SIZE), 16);
-    buf.writeUInt32LE(0, 24);
+        buf.writeUInt32LE(0, 24);
-    return buf;
+        return buf;
-  }
-
-  static readHeader(buf: Buffer): DataHeader {
-    return {
-      magic: buf.toString('ascii', 0, 4),
-      version: buf.readUInt32LE(4),
-      createdAt: buf.readBigUInt64LE(8),
-      fileSize: buf.readBigUInt64LE(16),
-      recordCount: buf.readUInt32LE(24),
-      reserved: buf.subarray(28, 64),
-    };
-  }
-
-  static updateHeader(buf: Buffer, fileSize: bigint, recordCount: number): void {
-    buf.writeBigUInt64LE(fileSize, 16);
-    buf.writeUInt32LE(recordCount, 24);
-  }
-
-  static serializeRecord<T>(data: T, serializer: Serializer<T>): Buffer {
-    const dataBytes = serializer.serialize(data);
-    const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
-
-    const buf = Buffer.alloc(totalLen);
-
-    dataBytes.copy(buf, RECORD_HEADER_SIZE);
-
-    buf.writeUInt32LE(dataBytes.length, 0);
-    const checksum = crc32(buf, RECORD_HEADER_SIZE, totalLen);
-    buf.writeUInt32LE(checksum, 4);
-
-    return buf;
-  }
-
-  /**
-   * 기존 버퍼에 레코드를 직렬화하여 쓰기
-   * 버퍼 크기 부족시 내부에서 재할당 (BufferRef로 참조 전달)
-   * @returns 실제 쓰여진 바이트 수 (totalLen)
-   */
-  static serializeRecordTo<T>(bufRef: BufferRef, data: T, serializer: Serializer<T>): number {
-    const dataBytes = serializer.serialize(data);
-    const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
-
-    // 버퍼 크기 부족시 재할당
-    if (bufRef.buf.length < totalLen) {
-      bufRef.buf = Buffer.alloc(totalLen);
     }
 
-    dataBytes.copy(bufRef.buf, RECORD_HEADER_SIZE);
+    static readHeader(buf: Buffer): DataHeader {
-
+        return {
-    bufRef.buf.writeUInt32LE(dataBytes.length, 0);
+            magic: buf.toString('ascii', 0, 4),
-    const checksum = crc32(bufRef.buf, RECORD_HEADER_SIZE, totalLen);
+            version: buf.readUInt32LE(4),
-    bufRef.buf.writeUInt32LE(checksum, 4);
+            createdAt: buf.readBigUInt64LE(8),
-
+            fileSize: buf.readBigUInt64LE(16),
-    return totalLen;
+            recordCount: buf.readUInt32LE(24),
-  }
+            reserved: buf.subarray(28, 64),
-
+        };
-  static deserializeRecord<T>(
-    buf: Buffer,
-    offset: number,
-    serializer: Serializer<T>
-  ): { data: T; length: number } | null {
-    if (offset + RECORD_HEADER_SIZE > buf.length) return null;
-
-    const dataLen = buf.readUInt32LE(offset);
-    const storedChecksum = buf.readUInt32LE(offset + 4);
-
-    const totalLen = RECORD_HEADER_SIZE + dataLen;
-    if (offset + totalLen > buf.length) return null;
-
-    const calcChecksum = crc32(buf, offset + RECORD_HEADER_SIZE, offset + totalLen);
-    if (calcChecksum !== storedChecksum) {
-      throw new Error(`Checksum mismatch at offset ${offset}`);
     }
 
-    const dataBytes = buf.subarray(offset + RECORD_HEADER_SIZE, offset + totalLen);
+    static updateHeader(buf: Buffer, fileSize: bigint, recordCount: number): void {
-    const data = serializer.deserialize(dataBytes);
+        buf.writeBigUInt64LE(fileSize, 16);
+        buf.writeUInt32LE(recordCount, 24);
+    }
 
-    return { data, length: totalLen };
+    static serializeRecord<T>(data: T, serializer: Serializer<T>): Buffer {
-  }
+        const dataBytes = serializer.serialize(data);
-}
+        const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
+
+        const buf = Buffer.alloc(totalLen);
+
+        dataBytes.copy(buf, RECORD_HEADER_SIZE);
+
+        buf.writeUInt32LE(dataBytes.length, 0);
+        const checksum = crc32(buf, RECORD_HEADER_SIZE, totalLen);
+        buf.writeUInt32LE(checksum, 4);
+
+        return buf;
+    }
+
+    /**
+     * 기존 버퍼에 레코드를 직렬화하여 쓰기
+     * 버퍼 크기 부족시 내부에서 재할당 (BufferRef로 참조 전달)
+     * @returns 실제 쓰여진 바이트 수 (totalLen)
+     */
+    static serializeRecordTo<T>(bufRef: BufferRef, data: T, serializer: Serializer<T>): number {
+        const dataBytes = serializer.serialize(data);
+        const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
+
+        // 버퍼 크기 부족시 재할당
+        if (bufRef.buf.length < totalLen) {
+            bufRef.buf = Buffer.alloc(totalLen);
+        }
+
+        dataBytes.copy(bufRef.buf, RECORD_HEADER_SIZE);
+
+        bufRef.buf.writeUInt32LE(dataBytes.length, 0);
+        const checksum = crc32(bufRef.buf, RECORD_HEADER_SIZE, totalLen);
+        bufRef.buf.writeUInt32LE(checksum, 4);
+
+        return totalLen;
+    }
+
+    static deserializeRecord<T>(
+        buf: Buffer,
+        offset: number,
+        serializer: Serializer<T>
+    ): { data: T; length: number } | null {
+        if (offset + RECORD_HEADER_SIZE > buf.length) return null;
+
+        const dataLen = buf.readUInt32LE(offset);
+        const storedChecksum = buf.readUInt32LE(offset + 4);
+
+        const totalLen = RECORD_HEADER_SIZE + dataLen;
+        if (offset + totalLen > buf.length) return null;
+
+        const calcChecksum = crc32(buf, offset + RECORD_HEADER_SIZE, offset + totalLen);
+        if (calcChecksum !== storedChecksum) {
+            console.warn(`Checksum mismatch at offset ${offset}: stored=${storedChecksum}, calc=${calcChecksum}`);
+        }
+
+        const dataBytes = buf.subarray(offset + RECORD_HEADER_SIZE, offset + totalLen);
+        const data = serializer.deserialize(dataBytes);
+
+        return { data, length: totalLen };
+    }
+}

lib/idx/reader.ts

@@ -72,14 +72,15 @@ export class IndexReader {
         if (this.fd === null || !this.header) throw new Error('Index file not opened');
 
         const results: { index: number; entry: IndexEntry }[] = [];
-        const currIdx = this.currentIndex;
         const first = this.searchSequenceLowerBound(this.fd, this.header.writtenCnt, startSeq);
         if (first === null) {
-            this.currentIndex = currIdx;
             return [];
         }
 
         results.push(first);
+        // binary search 후 currentIndex와 fd 위치를 first.index에 맞춤
+        this.readEntryAt(this.fd, first.index);
+
         while (this.currentIndex < this.header.writtenCnt - 1) {
             const entry = this.readNextEntry(this.fd);
             if (entry == null || entry.sequence > endSeq) break;
@@ -92,14 +93,15 @@ export class IndexReader {
         if (this.fd === null || !this.header) throw new Error('Index file not opened');
 
         const results: { index: number; entry: IndexEntry }[] = [];
-        const currIdx = this.currentIndex;
         const first = this.searchTimestampLowerBound(this.fd, this.header.writtenCnt, startTs);
         if (first === null) {
-            this.currentIndex = currIdx;
             return [];
         }
 
         results.push(first);
+        // binary search 후 currentIndex와 fd 위치를 first.index에 맞춤
+        this.readEntryAt(this.fd, first.index);
+
         while (this.currentIndex < this.header.writtenCnt - 1) {
             const entry = this.readNextEntry(this.fd);
             if (entry == null || entry.timestamp > endTs) break;