sati-open/sati.n-api.index-file
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

memory optimize

Browse Source
This commit is contained in:
Eli-Class
2026-02-05 07:19:05 +00:00
parent 92a2681292
commit b497fa5574
2 changed files with 96 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
lib/dat/protocol.ts
View File

@@ -4,104 +4,104 @@ import { crc32 } from '../idx/index.js';
import type { Serializer } from './types.js';
export interface DataHeader {
magic: string;
version: number;
createdAt: bigint;
fileSize: bigint;
recordCount: number;
reserved: Buffer;
magic: string;
version: number;
createdAt: bigint;
fileSize: bigint;
recordCount: number;
reserved: Buffer;
}
export interface BufferRef {
buf: Buffer;
buf: Buffer;
}
export class DataProtocol {
static createHeader(): Buffer {
const buf = Buffer.alloc(DATA_HEADER_SIZE);
buf.write(DATA_MAGIC, 0, 4, 'ascii');
buf.writeUInt32LE(DATA_VERSION, 4);
buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
buf.writeBigUInt64LE(BigInt(DATA_HEADER_SIZE), 16);
buf.writeUInt32LE(0, 24);
return buf;
}
static readHeader(buf: Buffer): DataHeader {
return {
magic: buf.toString('ascii', 0, 4),
version: buf.readUInt32LE(4),
createdAt: buf.readBigUInt64LE(8),
fileSize: buf.readBigUInt64LE(16),
recordCount: buf.readUInt32LE(24),
reserved: buf.subarray(28, 64),
};
}
static updateHeader(buf: Buffer, fileSize: bigint, recordCount: number): void {
buf.writeBigUInt64LE(fileSize, 16);
buf.writeUInt32LE(recordCount, 24);
}
static serializeRecord<T>(data: T, serializer: Serializer<T>): Buffer {
const dataBytes = serializer.serialize(data);
const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
const buf = Buffer.alloc(totalLen);
dataBytes.copy(buf, RECORD_HEADER_SIZE);
buf.writeUInt32LE(dataBytes.length, 0);
const checksum = crc32(buf, RECORD_HEADER_SIZE, totalLen);
buf.writeUInt32LE(checksum, 4);
return buf;
}
/**
* 기존 버퍼에 레코드를 직렬화하여 쓰기
* 버퍼 크기 부족시 내부에서 재할당 (BufferRef로 참조 전달)
* @returns 실제 쓰여진 바이트 수 (totalLen)
*/
static serializeRecordTo<T>(bufRef: BufferRef, data: T, serializer: Serializer<T>): number {
const dataBytes = serializer.serialize(data);
const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
// 버퍼 크기 부족시 재할당
if (bufRef.buf.length < totalLen) {
bufRef.buf = Buffer.alloc(totalLen);
static createHeader(): Buffer {
const buf = Buffer.alloc(DATA_HEADER_SIZE);
buf.write(DATA_MAGIC, 0, 4, 'ascii');
buf.writeUInt32LE(DATA_VERSION, 4);
buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
buf.writeBigUInt64LE(BigInt(DATA_HEADER_SIZE), 16);
buf.writeUInt32LE(0, 24);
return buf;
}
dataBytes.copy(bufRef.buf, RECORD_HEADER_SIZE);
bufRef.buf.writeUInt32LE(dataBytes.length, 0);
const checksum = crc32(bufRef.buf, RECORD_HEADER_SIZE, totalLen);
bufRef.buf.writeUInt32LE(checksum, 4);
return totalLen;
}
static deserializeRecord<T>(
buf: Buffer,
offset: number,
serializer: Serializer<T>
): { data: T; length: number } | null {
if (offset + RECORD_HEADER_SIZE > buf.length) return null;
const dataLen = buf.readUInt32LE(offset);
const storedChecksum = buf.readUInt32LE(offset + 4);
const totalLen = RECORD_HEADER_SIZE + dataLen;
if (offset + totalLen > buf.length) return null;
const calcChecksum = crc32(buf, offset + RECORD_HEADER_SIZE, offset + totalLen);
if (calcChecksum !== storedChecksum) {
throw new Error(`Checksum mismatch at offset ${offset}`);
static readHeader(buf: Buffer): DataHeader {
return {
magic: buf.toString('ascii', 0, 4),
version: buf.readUInt32LE(4),
createdAt: buf.readBigUInt64LE(8),
fileSize: buf.readBigUInt64LE(16),
recordCount: buf.readUInt32LE(24),
reserved: buf.subarray(28, 64),
};
}
const dataBytes = buf.subarray(offset + RECORD_HEADER_SIZE, offset + totalLen);
const data = serializer.deserialize(dataBytes);
static updateHeader(buf: Buffer, fileSize: bigint, recordCount: number): void {
buf.writeBigUInt64LE(fileSize, 16);
buf.writeUInt32LE(recordCount, 24);
}
return { data, length: totalLen };
}
static serializeRecord<T>(data: T, serializer: Serializer<T>): Buffer {
const dataBytes = serializer.serialize(data);
const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
const buf = Buffer.alloc(totalLen);
dataBytes.copy(buf, RECORD_HEADER_SIZE);
buf.writeUInt32LE(dataBytes.length, 0);
const checksum = crc32(buf, RECORD_HEADER_SIZE, totalLen);
buf.writeUInt32LE(checksum, 4);
return buf;
}
/**
* 기존 버퍼에 레코드를 직렬화하여 쓰기
* 버퍼 크기 부족시 내부에서 재할당 (BufferRef로 참조 전달)
* @returns 실제 쓰여진 바이트 수 (totalLen)
*/
static serializeRecordTo<T>(bufRef: BufferRef, data: T, serializer: Serializer<T>): number {
const dataBytes = serializer.serialize(data);
const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
// 버퍼 크기 부족시 재할당
if (bufRef.buf.length < totalLen) {
bufRef.buf = Buffer.alloc(totalLen);
}
dataBytes.copy(bufRef.buf, RECORD_HEADER_SIZE);
bufRef.buf.writeUInt32LE(dataBytes.length, 0);
const checksum = crc32(bufRef.buf, RECORD_HEADER_SIZE, totalLen);
bufRef.buf.writeUInt32LE(checksum, 4);
return totalLen;
}
static deserializeRecord<T>(
buf: Buffer,
offset: number,
serializer: Serializer<T>
): { data: T; length: number } | null {
if (offset + RECORD_HEADER_SIZE > buf.length) return null;
const dataLen = buf.readUInt32LE(offset);
const storedChecksum = buf.readUInt32LE(offset + 4);
const totalLen = RECORD_HEADER_SIZE + dataLen;
if (offset + totalLen > buf.length) return null;
const calcChecksum = crc32(buf, offset + RECORD_HEADER_SIZE, offset + totalLen);
if (calcChecksum !== storedChecksum) {
console.warn(`Checksum mismatch at offset ${offset}: stored=${storedChecksum}, calc=${calcChecksum}`);
}
const dataBytes = buf.subarray(offset + RECORD_HEADER_SIZE, offset + totalLen);
const data = serializer.deserialize(dataBytes);
return { data, length: totalLen };
}
}

10
lib/idx/reader.ts
View File

@@ -72,14 +72,15 @@ export class IndexReader {
if (this.fd === null || !this.header) throw new Error('Index file not opened');
const results: { index: number; entry: IndexEntry }[] = [];
const currIdx = this.currentIndex;
const first = this.searchSequenceLowerBound(this.fd, this.header.writtenCnt, startSeq);
if (first === null) {
this.currentIndex = currIdx;
return [];
}
results.push(first);
// binary search 후 currentIndex와 fd 위치를 first.index에 맞춤
this.readEntryAt(this.fd, first.index);
while (this.currentIndex < this.header.writtenCnt - 1) {
const entry = this.readNextEntry(this.fd);
if (entry == null || entry.sequence > endSeq) break;
@@ -92,14 +93,15 @@ export class IndexReader {
if (this.fd === null || !this.header) throw new Error('Index file not opened');
const results: { index: number; entry: IndexEntry }[] = [];
const currIdx = this.currentIndex;
const first = this.searchTimestampLowerBound(this.fd, this.header.writtenCnt, startTs);
if (first === null) {
this.currentIndex = currIdx;
return [];
}
results.push(first);
// binary search 후 currentIndex와 fd 위치를 first.index에 맞춤
this.readEntryAt(this.fd, first.index);
while (this.currentIndex < this.header.writtenCnt - 1) {
const entry = this.readNextEntry(this.fd);
if (entry == null || entry.timestamp > endTs) break;