sati-open/sati.n-api.index-file
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Interface modified mmap to fd

Browse Source
This commit is contained in:
Eli-Class
2026-01-29 09:24:48 +00:00
parent abc47d4909
commit c6e3eae22e
11 changed files with 727 additions and 1733 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CUSTOM_SERIALIZER.md
View File

@@ -1,3 +1,5 @@
가변 길이 배열이 포함된 데이터의 커스텀 바이너리 직렬화 방법을 보여드릴게요.
```typescript ```typescript
// 가변 배열 직렬화 예시 // 가변 배열 직렬화 예시
import { createSerializer, DataWriter, DataReader } from './src/data-file/index.js'; import { createSerializer, DataWriter, DataReader } from './src/data-file/index.js';
@@ -391,4 +393,4 @@ const orderSerializer = createSerializer<Order>(
return { orderId, items, total }; return { orderId, items, total };
} }
); );
``` ```

4
README.md
View File

@@ -2,7 +2,7 @@
```typescript ```typescript
// example.ts // example.ts
import { DataWriter, DataReader, jsonSerializer, createSerializer } from '@elilee/index-file'; import { DataWriter, DataReader, jsonSerializer, createSerializer } from './index.js';
// ============================================ // ============================================
// 1. JSON 직렬화 (간단한 경우) // 1. JSON 직렬화 (간단한 경우)
@@ -178,4 +178,4 @@ msgpackSerializer<T>()
// 커스텀 바이너리 // 커스텀 바이너리
createSerializer<T>(serialize, deserialize) createSerializer<T>(serialize, deserialize)
``` ```

336
lib/dat/reader.ts
View File

@@ -1,213 +1,215 @@
// src/data-file/reader.ts // src/data-file/reader.ts
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import mmap from '@elilee/mmap-native';
import { DATA_HEADER_SIZE } from './constants.js'; import { DATA_HEADER_SIZE } from './constants.js';
import { DataProtocol, DataHeader } from './protocol.js'; import { DataProtocol, DataHeader } from './protocol.js';
import { IndexReader } from '../idx/index.js'; import { IndexReader } from '../idx/index.js';
import type { Serializer, DataEntry } from './types.js'; import type { Serializer, DataEntry } from './types.js';
export class DataReader<T> { export class DataReader<T> {
private fd: number | null = null; private fd: number | null = null;
private buffer: Buffer | null = null; private header: DataHeader | null = null;
private header: DataHeader | null = null;
private indexReader: IndexReader; private indexReader: IndexReader;
private serializer: Serializer<T>; private serializer: Serializer<T>;
readonly dataPath: string; readonly dataPath: string;
readonly indexPath: string; readonly indexPath: string;
constructor(basePath: string, serializer: Serializer<T>) { constructor(basePath: string, serializer: Serializer<T>) {
this.dataPath = `${basePath}.dat`; this.dataPath = `${basePath}.dat`;
this.indexPath = `${basePath}.idx`; this.indexPath = `${basePath}.idx`;
this.serializer = serializer; this.serializer = serializer;
this.indexReader = new IndexReader(this.indexPath); this.indexReader = new IndexReader(this.indexPath);
} }
open(): void { open(): void {
const stats = fs.statSync(this.dataPath); this.fd = fs.openSync(this.dataPath, 'r');
this.fd = fs.openSync(this.dataPath, 'r');
this.buffer = mmap.map( // Read header only
stats.size, const headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
mmap.PROT_READ, fs.readSync(this.fd, headerBuf, 0, DATA_HEADER_SIZE, 0);
mmap.MAP_SHARED, this.header = DataProtocol.readHeader(headerBuf);
this.fd,
0
);
this.header = DataProtocol.readHeader(this.buffer); this.indexReader.open();
this.indexReader.open(); }
}
getHeader(): DataHeader { private readRecord(offset: bigint, length: number): Buffer {
if (!this.header) throw new Error('Data file not opened'); if (this.fd === null) throw new Error('Data file not opened');
return this.header;
}
getBySequence(sequence: number): DataEntry<T> | null { const buf = Buffer.alloc(length);
if (!this.buffer) throw new Error('Data file not opened'); fs.readSync(this.fd, buf, 0, length, Number(offset));
return buf;
}
const found = this.indexReader.binarySearchBySequence(sequence); getHeader(): DataHeader {
if (!found) return null; if (!this.header) throw new Error('Data file not opened');
return this.header;
}
const result = DataProtocol.deserializeRecord( getBySequence(sequence: number): DataEntry<T> | null {
this.buffer, if (this.fd === null) throw new Error('Data file not opened');
Number(found.entry.offset),
this.serializer
);
if (!result) return null;
return { const found = this.indexReader.binarySearchBySequence(sequence);
sequence: found.entry.sequence, if (!found) return null;
timestamp: found.entry.timestamp,
data: result.data,
};
}
getByIndex(index: number): DataEntry<T> | null { const buf = this.readRecord(found.entry.offset, found.entry.length);
if (!this.buffer) throw new Error('Data file not opened');
const entry = this.indexReader.getEntry(index);
if (!entry) return null;
const result = DataProtocol.deserializeRecord(
this.buffer,
Number(entry.offset),
this.serializer
);
if (!result) return null;
return {
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
};
}
getBulkData(startSeq: number, endSeq: number): DataEntry<T>[] {
if (!this.buffer) throw new Error('Data file not opened');
const results: DataEntry<T>[] = [];
const indexHeader = this.indexReader.getHeader();
let startIdx = this.findStartIndex(startSeq, indexHeader.validCount);
for (let i = startIdx; i < indexHeader.validCount; i++) {
const entry = this.indexReader.getEntry(i);
if (!entry) continue;
if (entry.sequence > endSeq) break;
if (entry.sequence >= startSeq) {
const result = DataProtocol.deserializeRecord( const result = DataProtocol.deserializeRecord(
this.buffer, buf,
Number(entry.offset), 0,
this.serializer this.serializer
); );
if (result) { if (!result) return null;
results.push({
return {
sequence: found.entry.sequence,
timestamp: found.entry.timestamp,
data: result.data,
};
}
getByIndex(index: number): DataEntry<T> | null {
if (this.fd === null) throw new Error('Data file not opened');
const entry = this.indexReader.getEntry(index);
if (!entry) return null;
const buf = this.readRecord(entry.offset, entry.length);
const result = DataProtocol.deserializeRecord(
buf,
0,
this.serializer
);
if (!result) return null;
return {
sequence: entry.sequence, sequence: entry.sequence,
timestamp: entry.timestamp, timestamp: entry.timestamp,
data: result.data, data: result.data,
}); };
}
getBulkData(startSeq: number, endSeq: number): DataEntry<T>[] {
if (this.fd === null) throw new Error('Data file not opened');
const results: DataEntry<T>[] = [];
const indexHeader = this.indexReader.getHeader();
let startIdx = this.findStartIndex(startSeq, indexHeader.writtenCnt);
for (let i = startIdx; i < indexHeader.writtenCnt; i++) {
const entry = this.indexReader.getEntry(i);
if (!entry) continue;
if (entry.sequence > endSeq) break;
if (entry.sequence >= startSeq) {
const buf = this.readRecord(entry.offset, entry.length);
const result = DataProtocol.deserializeRecord(
buf,
0,
this.serializer
);
if (result) {
results.push({
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
});
}
}
} }
}
return results;
} }
return results; private findStartIndex(targetSeq: number, writtenCnt: number): number {
} let left = 0;
let right = writtenCnt - 1;
let result = 0;
private findStartIndex(targetSeq: number, validCount: number): number { while (left <= right) {
let left = 0; const mid = Math.floor((left + right) / 2);
let right = validCount - 1; const entry = this.indexReader.getEntry(mid);
let result = 0;
while (left <= right) { if (!entry) {
const mid = Math.floor((left + right) / 2); right = mid - 1;
const entry = this.indexReader.getEntry(mid); continue;
}
if (!entry) { if (entry.sequence >= targetSeq) {
right = mid - 1; result = mid;
continue; right = mid - 1;
} } else {
left = mid + 1;
}
}
if (entry.sequence >= targetSeq) { return result;
result = mid;
right = mid - 1;
} else {
left = mid + 1;
}
} }
return result; getBulkDataByTime(startTs: bigint, endTs: bigint): DataEntry<T>[] {
} if (this.fd === null) throw new Error('Data file not opened');
getBulkDataByTime(startTs: bigint, endTs: bigint): DataEntry<T>[] { const indexResults = this.indexReader.findByTimeRange(startTs, endTs);
if (!this.buffer) throw new Error('Data file not opened'); const results: DataEntry<T>[] = [];
const indexResults = this.indexReader.findByTimeRange(startTs, endTs); for (const { entry } of indexResults) {
const results: DataEntry<T>[] = []; const buf = this.readRecord(entry.offset, entry.length);
const result = DataProtocol.deserializeRecord(
buf,
0,
this.serializer
);
if (result) {
results.push({
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
});
}
}
for (const { entry } of indexResults) { return results;
const result = DataProtocol.deserializeRecord(
this.buffer,
Number(entry.offset),
this.serializer
);
if (result) {
results.push({
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
});
}
} }
return results; getAllData(): DataEntry<T>[] {
} if (this.fd === null) throw new Error('Data file not opened');
getAllData(): DataEntry<T>[] { const entries = this.indexReader.getAllEntries();
if (!this.buffer) throw new Error('Data file not opened'); const results: DataEntry<T>[] = [];
const entries = this.indexReader.getAllEntries(); for (const entry of entries) {
const results: DataEntry<T>[] = []; const buf = this.readRecord(entry.offset, entry.length);
const result = DataProtocol.deserializeRecord(
buf,
0,
this.serializer
);
if (result) {
results.push({
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
});
}
}
for (const entry of entries) { return results;
const result = DataProtocol.deserializeRecord(
this.buffer,
Number(entry.offset),
this.serializer
);
if (result) {
results.push({
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
});
}
} }
return results; getRecordCount(): number {
} return this.indexReader.getHeader().writtenCnt;
getRecordCount(): number {
return this.indexReader.getHeader().validCount;
}
getLastSequence(): number {
return this.indexReader.getHeader().lastSequence;
}
close(): void {
if (this.buffer) {
mmap.unmap(this.buffer);
this.buffer = null;
} }
if (this.fd !== null) {
fs.closeSync(this.fd); getLastSequence(): number {
this.fd = null; return this.indexReader.getHeader().latestSequence;
} }
this.header = null;
this.indexReader.close(); close(): void {
} if (this.fd !== null) {
} fs.closeSync(this.fd);
this.fd = null;
}
this.header = null;
this.indexReader.close();
}
}

19
lib/dat/types.ts
View File

@@ -1,16 +1,19 @@
import { IndexFileOptions } from "../idx/types.js";
// src/data-file/types.ts // src/data-file/types.ts
export interface Serializer<T> { export interface Serializer<T> {
serialize(data: T): Buffer; serialize(data: T): Buffer;
deserialize(buf: Buffer): T; deserialize(buf: Buffer): T;
} }
export interface DataEntry<T> { export interface DataEntry<T> {
sequence: number; sequence: number;
timestamp: bigint; timestamp: bigint;
data: T; data: T;
} }
export interface DataFileOptions<T> { export interface DataFileOptions<T> {
serializer: Serializer<T>; serializer: Serializer<T>;
maxEntries?: number; forceTruncate?: boolean;
} indexFileOpt: IndexFileOptions;
}

240
lib/dat/writer.ts
View File

@@ -2,119 +2,189 @@
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { DATA_HEADER_SIZE } from './constants.js'; import { DATA_HEADER_SIZE } from './constants.js';
import { DataProtocol } from './protocol.js'; import { DataProtocol } from './protocol.js';
import { IndexWriter } from '../idx/index.js'; import { IndexWriter, IndexFileOptionsRequired } from '../idx/index.js';
import type { Serializer, DataFileOptions } from './types.js'; import type { Serializer, DataFileOptions } from './types.js';
export class DataWriter<T> { export class DataWriter<T> {
private fd: number | null = null; private fd: number | null = null;
private headerBuf: Buffer | null = null; private headerBuf: Buffer | null = null;
private currentOffset: bigint = BigInt(DATA_HEADER_SIZE); private currentOffset: bigint = BigInt(DATA_HEADER_SIZE);
private recordCount = 0; private recordCount = 0;
private indexWriter: IndexWriter; private indexWriter: IndexWriter;
private serializer: Serializer<T>;
readonly dataPath: string; // See DataFileOptions
readonly indexPath: string; private readonly serializer: Serializer<T>;
private readonly forceTruncate: boolean;
constructor(basePath: string, options: DataFileOptions<T>) { private latestSequence: number = 0;
this.dataPath = `${basePath}.dat`;
this.indexPath = `${basePath}.idx`;
this.serializer = options.serializer;
const maxEntries = options.maxEntries ?? 10_000_000; private readonly indexFileOpt: IndexFileOptionsRequired;
this.indexWriter = new IndexWriter(this.indexPath, { maxEntries });
}
open(): void { private dataPath: string | null = null;
const isNew = !fs.existsSync(this.dataPath); private indexPath: string | null = null;
this.fd = fs.openSync(this.dataPath, isNew ? 'w+' : 'r+');
this.headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
if (isNew) { constructor(options: DataFileOptions<T>) {
const header = DataProtocol.createHeader(); this.serializer = options.serializer;
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, 0); this.forceTruncate = options.forceTruncate ?? false;
this.currentOffset = BigInt(DATA_HEADER_SIZE);
this.recordCount = 0; this.indexFileOpt = {
} else { maxEntries: options.indexFileOpt.maxEntries ?? 10_000_000,
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0); autoIncrementSequence: options.indexFileOpt.autoIncrementSequence ?? false
const header = DataProtocol.readHeader(this.headerBuf); }
this.currentOffset = header.fileSize;
this.recordCount = header.recordCount; this.indexWriter = new IndexWriter(this.indexFileOpt);
} }
this.indexWriter.open(); open(basePath: string): void {
} this.dataPath = `${basePath}.dat`;
this.indexPath = `${basePath}.idx`;
append(data: T, timestamp?: bigint): number { // Index file 을 Open 함으로써 파일을 시작 할 수 있는지 검증 (Throw 로써)
if (this.fd === null) throw new Error('Data file not opened'); // Open index file with maxEntries and autoIncrementSequence
const writtenCount = this.indexWriter.open(this.indexPath, this.forceTruncate);
const isNew = !fs.existsSync(this.dataPath);
const buf = DataProtocol.serializeRecord(data, this.serializer); // Index file 은 초기화인데, 신규파일 혹은 강제 클리어가 아니라면
const offset = this.currentOffset; if (writtenCount === 0 && !(isNew || this.forceTruncate)) {
throw new Error(`Index file & Data File is invalid ${this.indexPath} is initial but ${this.dataPath} is exists`);
}
if (writtenCount > 0 && isNew) {
throw new Error(`Index file & Data File is invalid data of ${this.indexPath} | ${writtenCount} is exists but ${this.dataPath} is not exists`);
}
fs.writeSync(this.fd, buf, 0, buf.length, Number(offset)); // Warn if forceTruncate will delete existing data
if (this.forceTruncate && !isNew) {
const stats = fs.statSync(this.dataPath);
const sizeMB = (stats.size / 1024 / 1024).toFixed(2);
console.warn(
`[DataWriter] forceTruncate enabled: Deleting ${sizeMB} MB of existing data\n` +
` Index: ${this.indexPath} (${writtenCount} records)\n` +
` Data: ${this.dataPath}`
);
}
const sequence = this.indexWriter.getNextSequence(); this.fd = fs.openSync(this.dataPath,
const ts = timestamp ?? BigInt(Date.now()) * 1000000n; isNew || this.forceTruncate ? 'w+' : 'r+');
this.indexWriter.append(offset, buf.length, ts); try {
this.headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
this.currentOffset += BigInt(buf.length); if (isNew || this.forceTruncate) {
this.recordCount++; const header = DataProtocol.createHeader();
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, 0);
this.currentOffset = BigInt(DATA_HEADER_SIZE);
this.recordCount = 0;
this.latestSequence = 0;
} else {
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
const header = DataProtocol.readHeader(this.headerBuf);
return sequence; // Validate: Data file recordCount must match Index file writtenCnt
} if (header.recordCount !== writtenCount) {
throw new Error(
`Data file record count mismatch: Data has ${header.recordCount} but Index has ${writtenCount}`
);
}
appendBulk(records: T[], timestamp?: bigint): number[] { this.currentOffset = header.fileSize;
const sequences: number[] = []; this.recordCount = header.recordCount;
const ts = timestamp ?? BigInt(Date.now()) * 1000000n; this.latestSequence = this.indexWriter.getLatestSequence();
}
for (const record of records) { } catch (error) {
const seq = this.append(record, ts); // Clean up resources on error
sequences.push(seq); if (this.fd !== null) {
fs.closeSync(this.fd);
this.fd = null;
}
this.headerBuf = null;
throw error;
}
} }
return sequences; append(data: T, sequence?: number, timestamp?: bigint): number {
} if (this.fd === null) throw new Error('Data file not opened');
getLastSequence(): number { const buf = DataProtocol.serializeRecord(data, this.serializer);
return this.indexWriter.getLastSequence(); const offset = this.currentOffset;
}
getNextSequence(): number { fs.writeSync(this.fd, buf, 0, buf.length, Number(offset));
return this.indexWriter.getNextSequence();
}
sync(): void { // Write to index file
if (this.fd === null || !this.headerBuf) return; this.indexWriter.write(offset, buf.length, sequence, timestamp);
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount); // Update latestSequence to the most recent sequence
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0); this.latestSequence = this.indexWriter.getLatestSequence();
fs.fsyncSync(this.fd);
this.indexWriter.syncAll(); this.currentOffset += BigInt(buf.length);
} ++this.recordCount;
close(): void { return this.latestSequence;
this.sync();
if (this.fd !== null) {
fs.closeSync(this.fd);
this.fd = null;
} }
this.indexWriter.close(); /*
this.headerBuf = null; appendBulk(records: T[], sequences?: number[], timestamp?: bigint): number[] {
} // Runtime check: sequences required when autoIncrementSequence is false
if (!this.autoIncrementSequence) {
if (!sequences) {
throw new Error('sequences is required when autoIncrementSequence is false');
}
if (sequences.length !== records.length) {
throw new Error(`sequences length (${sequences.length}) must match records length (${records.length})`);
}
}
getStats() { const resultSequences: number[] = [];
return { const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
dataPath: this.dataPath,
indexPath: this.indexPath, for (let i = 0; i < records.length; i++) {
currentOffset: this.currentOffset, const seq = sequences?.[i];
recordCount: this.recordCount, const resultSeq = this.append(records[i], seq, ts);
lastSequence: this.indexWriter.getLastSequence(), resultSequences.push(resultSeq);
}; }
}
} return resultSequences;
}
*/
getLatestSequence(): number {
return this.latestSequence;
}
getNextSequence(): number {
return this.latestSequence + 1;
}
sync(): void {
if (this.fd === null || !this.headerBuf) return;
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
fs.fsyncSync(this.fd);
this.indexWriter.syncAll();
}
close(): void {
this.sync();
if (this.fd !== null) {
fs.closeSync(this.fd);
this.fd = null;
}
this.indexWriter.close();
this.headerBuf = null;
}
getStats() {
return {
dataPath: this.dataPath,
indexPath: this.indexPath,
currentOffset: this.currentOffset,
recordCount: this.recordCount,
latestSequence: this.indexWriter.getLatestSequence(),
};
}
}

168
lib/idx/protocol.ts
View File

@@ -1,106 +1,108 @@
// src/index-file/protocol.ts // src/index-file/protocol.ts
import { import {
INDEX_MAGIC, INDEX_MAGIC,
INDEX_VERSION, INDEX_VERSION,
INDEX_HEADER_SIZE, INDEX_HEADER_SIZE,
INDEX_ENTRY_SIZE, INDEX_ENTRY_SIZE,
FLAG_VALID, FLAG_VALID,
} from './constants.js'; } from './constants.js';
import type { IndexHeader, IndexEntry } from './types.js'; import type { IndexHeader, IndexEntry } from './types.js';
const CRC_TABLE = new Uint32Array(256); const CRC_TABLE = new Uint32Array(256);
for (let i = 0; i < 256; i++) { for (let i = 0; i < 256; i++) {
let c = i; let c = i;
for (let j = 0; j < 8; j++) { for (let j = 0; j < 8; j++) {
c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1); c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
} }
CRC_TABLE[i] = c >>> 0; CRC_TABLE[i] = c >>> 0;
} }
export function crc32(buf: Buffer, start = 0, end?: number): number { export function crc32(buf: Buffer, start = 0, end?: number): number {
let crc = 0xFFFFFFFF; let crc = 0xFFFFFFFF;
const len = end ?? buf.length; const len = end ?? buf.length;
for (let i = start; i < len; i++) { for (let i = start; i < len; i++) {
crc = CRC_TABLE[(crc ^ buf[i]) & 0xFF] ^ (crc >>> 8); crc = CRC_TABLE[(crc ^ buf[i]) & 0xFF] ^ (crc >>> 8);
} }
return (~crc) >>> 0; return (~crc) >>> 0;
} }
export class IndexProtocol { export class IndexProtocol {
static createHeader(entryCount: number, magic = INDEX_MAGIC): Buffer { static createHeader(entryCount: number, autoIncrementSequence: boolean, magic = INDEX_MAGIC): Buffer {
const buf = Buffer.alloc(INDEX_HEADER_SIZE); const buf = Buffer.alloc(INDEX_HEADER_SIZE);
buf.write(magic, 0, 4, 'ascii'); buf.write(magic, 0, 4, 'ascii');
buf.writeUInt32LE(INDEX_VERSION, 4); buf.writeUInt32LE(INDEX_VERSION, 4);
buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8); buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
buf.writeUInt32LE(INDEX_ENTRY_SIZE, 16); buf.writeUInt32LE(INDEX_ENTRY_SIZE, 16);
buf.writeUInt32LE(entryCount, 20); buf.writeUInt32LE(entryCount, 20);
buf.writeUInt32LE(0, 24); buf.writeUInt32LE(0, 24); // writtenCnt
buf.writeBigUInt64LE(0n, 28); buf.writeBigUInt64LE(0n, 28); // dataFileSize
buf.writeUInt32LE(0, 36); buf.writeUInt32LE(0, 36); // latestSequence
return buf; buf.writeUInt8(autoIncrementSequence ? 1 : 0, 40); // autoIncrementSequence
} return buf;
}
static readHeader(buf: Buffer): IndexHeader { static readHeader(buf: Buffer): IndexHeader {
return { return {
magic: buf.toString('ascii', 0, 4), magic: buf.toString('ascii', 0, 4),
version: buf.readUInt32LE(4), version: buf.readUInt32LE(4),
createdAt: buf.readBigUInt64LE(8), createdAt: buf.readBigUInt64LE(8),
entrySize: buf.readUInt32LE(16), entrySize: buf.readUInt32LE(16),
entryCount: buf.readUInt32LE(20), entryCount: buf.readUInt32LE(20),
validCount: buf.readUInt32LE(24), writtenCnt: buf.readUInt32LE(24),
dataFileSize: buf.readBigUInt64LE(28), dataFileSize: buf.readBigUInt64LE(28),
lastSequence: buf.readUInt32LE(36), latestSequence: buf.readUInt32LE(36),
reserved: buf.subarray(40, 64), autoIncrementSequence: buf.readUInt8(40) === 1,
}; reserved: buf.subarray(41, 64),
} };
}
static updateHeaderCounts( static updateHeaderCounts(
buf: Buffer, buf: Buffer,
validCount: number, writtenCnt: number,
dataFileSize: bigint, dataFileSize: bigint,
lastSequence: number latestSequence: number
): void { ): void {
buf.writeUInt32LE(validCount, 24); buf.writeUInt32LE(writtenCnt, 24);
buf.writeBigUInt64LE(dataFileSize, 28); buf.writeBigUInt64LE(dataFileSize, 28);
buf.writeUInt32LE(lastSequence, 36); buf.writeUInt32LE(latestSequence, 36);
} }
static writeEntry(buf: Buffer, index: number, entry: Omit<IndexEntry, 'checksum'>): void { static writeEntry(buf: Buffer, index: number, entry: Omit<IndexEntry, 'checksum'>): void {
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE; const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
buf.writeUInt32LE(entry.sequence, off); buf.writeUInt32LE(entry.sequence, off);
buf.writeBigUInt64LE(entry.timestamp, off + 4); buf.writeBigUInt64LE(entry.timestamp, off + 4);
buf.writeBigUInt64LE(entry.offset, off + 12); buf.writeBigUInt64LE(entry.offset, off + 12);
buf.writeUInt32LE(entry.length, off + 20); buf.writeUInt32LE(entry.length, off + 20);
buf.writeUInt32LE(entry.flags | FLAG_VALID, off + 24); buf.writeUInt32LE(entry.flags | FLAG_VALID, off + 24);
const checksum = crc32(buf, off, off + 28); const checksum = crc32(buf, off, off + 28);
buf.writeUInt32LE(checksum, off + 28); buf.writeUInt32LE(checksum, off + 28);
} }
static readEntry(buf: Buffer, index: number): IndexEntry | null { static readEntry(buf: Buffer, index: number): IndexEntry | null {
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE; const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
const flags = buf.readUInt32LE(off + 24); const flags = buf.readUInt32LE(off + 24);
if (!(flags & FLAG_VALID)) return null; if (!(flags & FLAG_VALID)) return null;
return { return {
sequence: buf.readUInt32LE(off), sequence: buf.readUInt32LE(off),
timestamp: buf.readBigUInt64LE(off + 4), timestamp: buf.readBigUInt64LE(off + 4),
offset: buf.readBigUInt64LE(off + 12), offset: buf.readBigUInt64LE(off + 12),
length: buf.readUInt32LE(off + 20), length: buf.readUInt32LE(off + 20),
flags, flags,
checksum: buf.readUInt32LE(off + 28), checksum: buf.readUInt32LE(off + 28),
}; };
} }
static isValidEntry(buf: Buffer, index: number): boolean { static isValidEntry(buf: Buffer, index: number): boolean {
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE; const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
const flags = buf.readUInt32LE(off + 24); const flags = buf.readUInt32LE(off + 24);
return (flags & FLAG_VALID) !== 0; return (flags & FLAG_VALID) !== 0;
} }
static calcFileSize(entryCount: number): number { static calcFileSize(entryCount: number): number {
return INDEX_HEADER_SIZE + INDEX_ENTRY_SIZE * entryCount; return INDEX_HEADER_SIZE + INDEX_ENTRY_SIZE * entryCount;
} }
} }

219
lib/idx/reader.ts
View File

@@ -1,131 +1,114 @@
// src/index-file/reader.ts // src/index-file/reader.ts
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import mmap from '@elilee/mmap-native';
import { IndexProtocol } from './protocol.js'; import { IndexProtocol } from './protocol.js';
import type { IndexHeader, IndexEntry } from './types.js'; import type { IndexHeader, IndexEntry } from './types.js';
export class IndexReader { export class IndexReader {
private fd: number | null = null; private buffer: Buffer | null = null;
private buffer: Buffer | null = null; private header: IndexHeader | null = null;
private header: IndexHeader | null = null;
readonly path: string; readonly path: string;
constructor(path: string) { constructor(path: string) {
this.path = path; this.path = path;
}
open(): void {
const stats = fs.statSync(this.path);
this.fd = fs.openSync(this.path, 'r');
this.buffer = mmap.map(
stats.size,
mmap.PROT_READ,
mmap.MAP_SHARED,
this.fd,
0
);
this.header = IndexProtocol.readHeader(this.buffer);
}
getHeader(): IndexHeader {
if (!this.header) throw new Error('Index file not opened');
return this.header;
}
getEntry(index: number): IndexEntry | null {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
if (index < 0 || index >= this.header.entryCount) return null;
return IndexProtocol.readEntry(this.buffer, index);
}
findBySequence(sequence: number): { index: number; entry: IndexEntry } | null {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
for (let i = 0; i < this.header.validCount; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry && entry.sequence === sequence) {
return { index: i, entry };
}
}
return null;
}
findBySequenceRange(startSeq: number, endSeq: number): { index: number; entry: IndexEntry }[] {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
const results: { index: number; entry: IndexEntry }[] = [];
for (let i = 0; i < this.header.validCount; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry && entry.sequence >= startSeq && entry.sequence <= endSeq) {
results.push({ index: i, entry });
}
}
return results;
}
getAllEntries(): IndexEntry[] {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
const entries: IndexEntry[] = [];
for (let i = 0; i < this.header.validCount; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry) entries.push(entry);
}
return entries;
}
findByTimeRange(startTs: bigint, endTs: bigint): { index: number; entry: IndexEntry }[] {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
const results: { index: number; entry: IndexEntry }[] = [];
for (let i = 0; i < this.header.validCount; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry && entry.timestamp >= startTs && entry.timestamp <= endTs) {
results.push({ index: i, entry });
}
}
return results;
}
binarySearchBySequence(targetSeq: number): { index: number; entry: IndexEntry } | null {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
let left = 0;
let right = this.header.validCount - 1;
while (left <= right) {
const mid = Math.floor((left + right) / 2);
const entry = IndexProtocol.readEntry(this.buffer, mid);
if (!entry) {
right = mid - 1;
continue;
}
if (entry.sequence === targetSeq) {
return { index: mid, entry };
} else if (entry.sequence < targetSeq) {
left = mid + 1;
} else {
right = mid - 1;
}
} }
return null; open(): void {
} // Read entire file into buffer (simpler than mmap for read-only access)
this.buffer = fs.readFileSync(this.path);
this.header = IndexProtocol.readHeader(this.buffer);
}
close(): void { getHeader(): IndexHeader {
if (this.buffer) { if (!this.header) throw new Error('Index file not opened');
mmap.unmap(this.buffer); return this.header;
this.buffer = null;
} }
if (this.fd !== null) {
fs.closeSync(this.fd); getEntry(index: number): IndexEntry | null {
this.fd = null; if (!this.buffer || !this.header) throw new Error('Index file not opened');
if (index < 0 || index >= this.header.entryCount) return null;
return IndexProtocol.readEntry(this.buffer, index);
} }
this.header = null;
} findBySequence(sequence: number): { index: number; entry: IndexEntry } | null {
} if (!this.buffer || !this.header) throw new Error('Index file not opened');
for (let i = 0; i < this.header.writtenCnt; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry && entry.sequence === sequence) {
return { index: i, entry };
}
}
return null;
}
findBySequenceRange(startSeq: number, endSeq: number): { index: number; entry: IndexEntry }[] {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
const results: { index: number; entry: IndexEntry }[] = [];
for (let i = 0; i < this.header.writtenCnt; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry && entry.sequence >= startSeq && entry.sequence <= endSeq) {
results.push({ index: i, entry });
}
}
return results;
}
getAllEntries(): IndexEntry[] {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
const entries: IndexEntry[] = [];
for (let i = 0; i < this.header.writtenCnt; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry) entries.push(entry);
}
return entries;
}
findByTimeRange(startTs: bigint, endTs: bigint): { index: number; entry: IndexEntry }[] {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
const results: { index: number; entry: IndexEntry }[] = [];
for (let i = 0; i < this.header.writtenCnt; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry && entry.timestamp >= startTs && entry.timestamp <= endTs) {
results.push({ index: i, entry });
}
}
return results;
}
binarySearchBySequence(targetSeq: number): { index: number; entry: IndexEntry } | null {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
let left = 0;
let right = this.header.writtenCnt - 1;
while (left <= right) {
const mid = Math.floor((left + right) / 2);
const entry = IndexProtocol.readEntry(this.buffer, mid);
if (!entry) {
right = mid - 1;
continue;
}
if (entry.sequence === targetSeq) {
return { index: mid, entry };
} else if (entry.sequence < targetSeq) {
left = mid + 1;
} else {
right = mid - 1;
}
}
return null;
}
close(): void {
// Simply release buffer reference (GC will handle cleanup)
this.buffer = null;
this.header = null;
}
}

39
lib/idx/types.ts
View File

@@ -1,26 +1,29 @@
// src/index-file/types.ts // src/index-file/types.ts
export interface IndexHeader { export interface IndexHeader {
magic: string; magic: string;
version: number; version: number;
createdAt: bigint; createdAt: bigint;
entrySize: number; entrySize: number;
entryCount: number; entryCount: number;
validCount: number; writtenCnt: number;
dataFileSize: bigint; dataFileSize: bigint;
lastSequence: number; latestSequence: number;
reserved: Buffer; autoIncrementSequence: boolean;
reserved: Buffer;
} }
export interface IndexEntry { export interface IndexEntry {
sequence: number; sequence: number;
timestamp: bigint; timestamp: bigint;
offset: bigint; offset: bigint;
length: number; length: number;
flags: number; flags: number;
checksum: number; checksum: number;
} }
export interface IndexFileOptions { export interface IndexFileOptions {
maxEntries: number; maxEntries?: number;
magic?: string; autoIncrementSequence?: boolean;
} }
export type IndexFileOptionsRequired = Required<IndexFileOptions>;

290
lib/idx/writer.ts
View File

@@ -1,141 +1,209 @@
// src/index-file/writer.ts // src/index-file/writer.ts
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import mmap from '@elilee/mmap-native'; import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js';
import { INDEX_HEADER_SIZE, FLAG_VALID } from './constants.js';
import { IndexProtocol } from './protocol.js'; import { IndexProtocol } from './protocol.js';
import type { IndexFileOptions } from './types.js'; import { IndexFileOptionsRequired } from './types.js';
export class IndexWriter { export class IndexWriter {
private fd: number | null = null; private fd: number | null = null;
private buffer: Buffer | null = null; private headerBuf: Buffer | null = null;
private validCount = 0; private entryBuf: Buffer | null = null;
private dataFileSize = 0n;
private lastSequence = 0;
readonly path: string; private writtenCnt = 0;
readonly maxEntries: number; private dataFileSize = 0n;
readonly fileSize: number; private latestSequence = 0;
constructor(path: string, options: IndexFileOptions) { private path: string | null = null;
this.path = path; private fileSize: number = 0;
this.maxEntries = options.maxEntries;
this.fileSize = IndexProtocol.calcFileSize(options.maxEntries);
}
open(): void { // see IndexFileOptions
const isNew = !fs.existsSync(this.path); private maxEntries: number = 0;
private autoIncrementSequence: boolean = false;
this.fd = fs.openSync(this.path, isNew ? 'w+' : 'r+'); constructor(opt: IndexFileOptionsRequired) {
// Empty constructor - maxEntries provided in open()
if (isNew) { this.maxEntries = opt.maxEntries;
fs.ftruncateSync(this.fd, this.fileSize); this.autoIncrementSequence = opt.autoIncrementSequence;
} }
this.buffer = mmap.map( open(path: string, forceTruncate: boolean = false): number {
this.fileSize, this.path = path;
mmap.PROT_READ | mmap.PROT_WRITE,
mmap.MAP_SHARED,
this.fd,
0
);
if (isNew) { const isNew = !fs.existsSync(this.path);
const header = IndexProtocol.createHeader(this.maxEntries);
header.copy(this.buffer, 0);
this.syncHeader();
} else {
const header = IndexProtocol.readHeader(this.buffer);
this.validCount = header.validCount;
this.dataFileSize = header.dataFileSize;
this.lastSequence = header.lastSequence;
}
}
write( if (isNew || forceTruncate) {
index: number, // New file: use provided values
sequence: number, this.fileSize = IndexProtocol.calcFileSize(this.maxEntries);
offset: bigint, this.writtenCnt = 0;
length: number, this.dataFileSize = 0n;
timestamp?: bigint this.latestSequence = 0;
): boolean {
if (!this.buffer) throw new Error('Index file not opened');
if (index < 0 || index >= this.maxEntries) return false;
const ts = timestamp ?? BigInt(Date.now()) * 1000000n; this.fd = fs.openSync(this.path, 'w+');
fs.ftruncateSync(this.fd, this.fileSize);
IndexProtocol.writeEntry(this.buffer, index, { // Allocate buffers for header and entry
sequence, this.headerBuf = Buffer.alloc(INDEX_HEADER_SIZE);
timestamp: ts, this.entryBuf = Buffer.alloc(INDEX_ENTRY_SIZE);
offset,
length,
flags: FLAG_VALID,
});
this.validCount++; const header = IndexProtocol.createHeader(this.maxEntries, this.autoIncrementSequence);
if (sequence > this.lastSequence) { header.copy(this.headerBuf, 0);
this.lastSequence = sequence;
// Write header to file
fs.writeSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0);
fs.fsyncSync(this.fd);
} else {
// Existing file: read header first
this.fd = fs.openSync(this.path, 'r+');
try {
this.headerBuf = Buffer.alloc(INDEX_HEADER_SIZE);
this.entryBuf = Buffer.alloc(INDEX_ENTRY_SIZE);
fs.readSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0);
const header = IndexProtocol.readHeader(this.headerBuf);
if (this.maxEntries !== header.entryCount) {
throw new Error(
`maxEntries mismatch: provided ${this.maxEntries} but file has ${header.entryCount}`
);
}
if (this.autoIncrementSequence !== header.autoIncrementSequence) {
throw new Error(
`autoIncrementSequence mismatch: provided ${this.autoIncrementSequence} but file has ${header.autoIncrementSequence}`
);
}
const expectFileSize = IndexProtocol.calcFileSize(this.maxEntries);
const calcedFileSize = IndexProtocol.calcFileSize(header.entryCount);
if (expectFileSize !== calcedFileSize) {
// if (opt.version !== header.version) { 버전이 다른거니까 어떻게 처리 할지는 추후 고민 TODO }
throw new Error(
`Indexfile size calc is invalid : provided ${expectFileSize} but file has ${calcedFileSize}`
);
}
this.fileSize = calcedFileSize;
this.writtenCnt = header.writtenCnt;
this.dataFileSize = header.dataFileSize;
this.latestSequence = header.latestSequence;
} catch (error) {
// Clean up resources on error
if (this.fd !== null) {
fs.closeSync(this.fd);
this.fd = null;
}
this.headerBuf = null;
this.entryBuf = null;
throw error;
}
}
return this.writtenCnt;
} }
const newDataEnd = offset + BigInt(length); write(
if (newDataEnd > this.dataFileSize) { offset: bigint,
this.dataFileSize = newDataEnd; length: number,
sequence?: number,
timestamp?: bigint
): boolean {
if (!this.entryBuf || this.fd === null) throw new Error('Index file not opened');
if (this.writtenCnt >= this.maxEntries) {
throw new Error(`Data count exceed provide : ${this.writtenCnt + 1} - max : ${this.maxEntries}`);
}
// Calculate sequence
let seq: number;
if (!this.autoIncrementSequence) {
if (sequence === undefined) {
throw new Error('sequence is required when autoIncrementSequence is false');
}
seq = sequence;
} else {
seq = this.writtenCnt + 1;
}
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
// Create a temporary buffer for this entry
const tempBuf = Buffer.alloc(INDEX_HEADER_SIZE + (this.writtenCnt + 1) * INDEX_ENTRY_SIZE);
// Write entry to temp buffer
IndexProtocol.writeEntry(tempBuf, this.writtenCnt, {
sequence: seq,
timestamp: ts,
offset,
length,
flags: FLAG_VALID,
});
// Calculate file offset for this entry
const fileOffset = INDEX_HEADER_SIZE + this.writtenCnt * INDEX_ENTRY_SIZE;
// Write entry to file
fs.writeSync(this.fd, tempBuf, fileOffset, INDEX_ENTRY_SIZE, fileOffset);
this.writtenCnt++;
this.latestSequence = seq;
const newDataEnd = offset + BigInt(length);
if (newDataEnd > this.dataFileSize) {
this.dataFileSize = newDataEnd;
}
return true;
} }
return true; getLatestSequence(): number {
} return this.latestSequence;
}
append(offset: bigint, length: number, timestamp?: bigint): number { syncHeader(): void {
const index = this.validCount; if (!this.headerBuf || this.fd === null) return;
if (index >= this.maxEntries) return -1;
const sequence = this.lastSequence + 1; // Update header counts
this.write(index, sequence, offset, length, timestamp); IndexProtocol.updateHeaderCounts(
return index; this.headerBuf,
} this.writtenCnt,
this.dataFileSize,
this.latestSequence
);
getLastSequence(): number { // Write header to file
return this.lastSequence; fs.writeSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0);
} }
getNextSequence(): number { syncAll(): void {
return this.lastSequence + 1; if (this.fd === null) return;
}
syncHeader(): void { // Sync header first
if (!this.buffer) return; this.syncHeader();
IndexProtocol.updateHeaderCounts(
this.buffer,
this.validCount,
this.dataFileSize,
this.lastSequence
);
mmap.sync(this.buffer, 0, INDEX_HEADER_SIZE, mmap.MS_ASYNC);
}
syncAll(): void { // Sync all file changes to disk
if (!this.buffer) return; fs.fsyncSync(this.fd);
this.syncHeader(); }
mmap.sync(this.buffer, 0, this.fileSize, mmap.MS_SYNC);
}
close(): void { close(): void {
if (!this.buffer || this.fd === null) return; if (this.fd === null) return;
this.syncAll(); // 1. Sync all changes
mmap.unmap(this.buffer); this.syncAll();
fs.closeSync(this.fd);
this.buffer = null; // 2. Close file descriptor
this.fd = null; fs.closeSync(this.fd);
} this.fd = null;
getStats() { // 3. Clean up buffers
return { this.headerBuf = null;
path: this.path, this.entryBuf = null;
maxEntries: this.maxEntries, }
validCount: this.validCount,
dataFileSize: this.dataFileSize, getStats() {
lastSequence: this.lastSequence, return {
}; path: this.path,
} writtenCnt: this.writtenCnt,
} dataFileSize: this.dataFileSize,
latestSequence: this.latestSequence,
};
}
}

1138
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

3
package.json
View File

@@ -14,8 +14,7 @@
], ],
"dependencies": { "dependencies": {
"typescript": "^5.7.0", "typescript": "^5.7.0",
"@types/node": "^22.0.0", "@types/node": "^22.0.0"
"@elilee/mmap-native": "git+https://git.satitech.co.kr/sati-open/sati.n-api.mmap.git"
}, },
"scripts": { "scripts": {
"prepare": "tsc -p tsconfig.json", "prepare": "tsc -p tsconfig.json",