Interface modified mmap to fd
This commit is contained in:
Eli-Class
@@ -1,3 +1,5 @@
|
||||
가변 길이 배열이 포함된 데이터의 커스텀 바이너리 직렬화 방법을 보여드릴게요.
|
||||
|
||||
```typescript
|
||||
// 가변 배열 직렬화 예시
|
||||
import { createSerializer, DataWriter, DataReader } from './src/data-file/index.js';
|
||||
@@ -391,4 +393,4 @@ const orderSerializer = createSerializer<Order>(
|
||||
return { orderId, items, total };
|
||||
}
|
||||
);
|
||||
```
|
||||
```
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
```typescript
|
||||
// example.ts
|
||||
import { DataWriter, DataReader, jsonSerializer, createSerializer } from '@elilee/index-file';
|
||||
import { DataWriter, DataReader, jsonSerializer, createSerializer } from './index.js';
|
||||
|
||||
// ============================================
|
||||
// 1. JSON 직렬화 (간단한 경우)
|
||||
@@ -178,4 +178,4 @@ msgpackSerializer<T>()
|
||||
|
||||
// 커스텀 바이너리
|
||||
createSerializer<T>(serialize, deserialize)
|
||||
```
|
||||
```
|
||||
@@ -1,213 +1,215 @@
|
||||
// src/data-file/reader.ts
|
||||
import * as fs from 'node:fs';
|
||||
import mmap from '@elilee/mmap-native';
|
||||
import { DATA_HEADER_SIZE } from './constants.js';
|
||||
import { DataProtocol, DataHeader } from './protocol.js';
|
||||
import { IndexReader } from '../idx/index.js';
|
||||
import type { Serializer, DataEntry } from './types.js';
|
||||
|
||||
export class DataReader<T> {
|
||||
private fd: number | null = null;
|
||||
private buffer: Buffer | null = null;
|
||||
private header: DataHeader | null = null;
|
||||
private fd: number | null = null;
|
||||
private header: DataHeader | null = null;
|
||||
|
||||
private indexReader: IndexReader;
|
||||
private serializer: Serializer<T>;
|
||||
private indexReader: IndexReader;
|
||||
private serializer: Serializer<T>;
|
||||
|
||||
readonly dataPath: string;
|
||||
readonly indexPath: string;
|
||||
readonly dataPath: string;
|
||||
readonly indexPath: string;
|
||||
|
||||
constructor(basePath: string, serializer: Serializer<T>) {
|
||||
this.dataPath = `${basePath}.dat`;
|
||||
this.indexPath = `${basePath}.idx`;
|
||||
this.serializer = serializer;
|
||||
this.indexReader = new IndexReader(this.indexPath);
|
||||
}
|
||||
constructor(basePath: string, serializer: Serializer<T>) {
|
||||
this.dataPath = `${basePath}.dat`;
|
||||
this.indexPath = `${basePath}.idx`;
|
||||
this.serializer = serializer;
|
||||
this.indexReader = new IndexReader(this.indexPath);
|
||||
}
|
||||
|
||||
open(): void {
|
||||
const stats = fs.statSync(this.dataPath);
|
||||
this.fd = fs.openSync(this.dataPath, 'r');
|
||||
open(): void {
|
||||
this.fd = fs.openSync(this.dataPath, 'r');
|
||||
|
||||
this.buffer = mmap.map(
|
||||
stats.size,
|
||||
mmap.PROT_READ,
|
||||
mmap.MAP_SHARED,
|
||||
this.fd,
|
||||
0
|
||||
);
|
||||
// Read header only
|
||||
const headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
|
||||
fs.readSync(this.fd, headerBuf, 0, DATA_HEADER_SIZE, 0);
|
||||
this.header = DataProtocol.readHeader(headerBuf);
|
||||
|
||||
this.header = DataProtocol.readHeader(this.buffer);
|
||||
this.indexReader.open();
|
||||
}
|
||||
this.indexReader.open();
|
||||
}
|
||||
|
||||
getHeader(): DataHeader {
|
||||
if (!this.header) throw new Error('Data file not opened');
|
||||
return this.header;
|
||||
}
|
||||
private readRecord(offset: bigint, length: number): Buffer {
|
||||
if (this.fd === null) throw new Error('Data file not opened');
|
||||
|
||||
getBySequence(sequence: number): DataEntry<T> | null {
|
||||
if (!this.buffer) throw new Error('Data file not opened');
|
||||
const buf = Buffer.alloc(length);
|
||||
fs.readSync(this.fd, buf, 0, length, Number(offset));
|
||||
return buf;
|
||||
}
|
||||
|
||||
const found = this.indexReader.binarySearchBySequence(sequence);
|
||||
if (!found) return null;
|
||||
getHeader(): DataHeader {
|
||||
if (!this.header) throw new Error('Data file not opened');
|
||||
return this.header;
|
||||
}
|
||||
|
||||
const result = DataProtocol.deserializeRecord(
|
||||
this.buffer,
|
||||
Number(found.entry.offset),
|
||||
this.serializer
|
||||
);
|
||||
if (!result) return null;
|
||||
getBySequence(sequence: number): DataEntry<T> | null {
|
||||
if (this.fd === null) throw new Error('Data file not opened');
|
||||
|
||||
return {
|
||||
sequence: found.entry.sequence,
|
||||
timestamp: found.entry.timestamp,
|
||||
data: result.data,
|
||||
};
|
||||
}
|
||||
const found = this.indexReader.binarySearchBySequence(sequence);
|
||||
if (!found) return null;
|
||||
|
||||
getByIndex(index: number): DataEntry<T> | null {
|
||||
if (!this.buffer) throw new Error('Data file not opened');
|
||||
|
||||
const entry = this.indexReader.getEntry(index);
|
||||
if (!entry) return null;
|
||||
|
||||
const result = DataProtocol.deserializeRecord(
|
||||
this.buffer,
|
||||
Number(entry.offset),
|
||||
this.serializer
|
||||
);
|
||||
if (!result) return null;
|
||||
|
||||
return {
|
||||
sequence: entry.sequence,
|
||||
timestamp: entry.timestamp,
|
||||
data: result.data,
|
||||
};
|
||||
}
|
||||
|
||||
getBulkData(startSeq: number, endSeq: number): DataEntry<T>[] {
|
||||
if (!this.buffer) throw new Error('Data file not opened');
|
||||
|
||||
const results: DataEntry<T>[] = [];
|
||||
const indexHeader = this.indexReader.getHeader();
|
||||
let startIdx = this.findStartIndex(startSeq, indexHeader.validCount);
|
||||
|
||||
for (let i = startIdx; i < indexHeader.validCount; i++) {
|
||||
const entry = this.indexReader.getEntry(i);
|
||||
if (!entry) continue;
|
||||
|
||||
if (entry.sequence > endSeq) break;
|
||||
|
||||
if (entry.sequence >= startSeq) {
|
||||
const buf = this.readRecord(found.entry.offset, found.entry.length);
|
||||
const result = DataProtocol.deserializeRecord(
|
||||
this.buffer,
|
||||
Number(entry.offset),
|
||||
this.serializer
|
||||
buf,
|
||||
0,
|
||||
this.serializer
|
||||
);
|
||||
if (result) {
|
||||
results.push({
|
||||
if (!result) return null;
|
||||
|
||||
return {
|
||||
sequence: found.entry.sequence,
|
||||
timestamp: found.entry.timestamp,
|
||||
data: result.data,
|
||||
};
|
||||
}
|
||||
|
||||
getByIndex(index: number): DataEntry<T> | null {
|
||||
if (this.fd === null) throw new Error('Data file not opened');
|
||||
|
||||
const entry = this.indexReader.getEntry(index);
|
||||
if (!entry) return null;
|
||||
|
||||
const buf = this.readRecord(entry.offset, entry.length);
|
||||
const result = DataProtocol.deserializeRecord(
|
||||
buf,
|
||||
0,
|
||||
this.serializer
|
||||
);
|
||||
if (!result) return null;
|
||||
|
||||
return {
|
||||
sequence: entry.sequence,
|
||||
timestamp: entry.timestamp,
|
||||
data: result.data,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
getBulkData(startSeq: number, endSeq: number): DataEntry<T>[] {
|
||||
if (this.fd === null) throw new Error('Data file not opened');
|
||||
|
||||
const results: DataEntry<T>[] = [];
|
||||
const indexHeader = this.indexReader.getHeader();
|
||||
let startIdx = this.findStartIndex(startSeq, indexHeader.writtenCnt);
|
||||
|
||||
for (let i = startIdx; i < indexHeader.writtenCnt; i++) {
|
||||
const entry = this.indexReader.getEntry(i);
|
||||
if (!entry) continue;
|
||||
|
||||
if (entry.sequence > endSeq) break;
|
||||
|
||||
if (entry.sequence >= startSeq) {
|
||||
const buf = this.readRecord(entry.offset, entry.length);
|
||||
const result = DataProtocol.deserializeRecord(
|
||||
buf,
|
||||
0,
|
||||
this.serializer
|
||||
);
|
||||
if (result) {
|
||||
results.push({
|
||||
sequence: entry.sequence,
|
||||
timestamp: entry.timestamp,
|
||||
data: result.data,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
private findStartIndex(targetSeq: number, writtenCnt: number): number {
|
||||
let left = 0;
|
||||
let right = writtenCnt - 1;
|
||||
let result = 0;
|
||||
|
||||
private findStartIndex(targetSeq: number, validCount: number): number {
|
||||
let left = 0;
|
||||
let right = validCount - 1;
|
||||
let result = 0;
|
||||
while (left <= right) {
|
||||
const mid = Math.floor((left + right) / 2);
|
||||
const entry = this.indexReader.getEntry(mid);
|
||||
|
||||
while (left <= right) {
|
||||
const mid = Math.floor((left + right) / 2);
|
||||
const entry = this.indexReader.getEntry(mid);
|
||||
if (!entry) {
|
||||
right = mid - 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry) {
|
||||
right = mid - 1;
|
||||
continue;
|
||||
}
|
||||
if (entry.sequence >= targetSeq) {
|
||||
result = mid;
|
||||
right = mid - 1;
|
||||
} else {
|
||||
left = mid + 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (entry.sequence >= targetSeq) {
|
||||
result = mid;
|
||||
right = mid - 1;
|
||||
} else {
|
||||
left = mid + 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
getBulkDataByTime(startTs: bigint, endTs: bigint): DataEntry<T>[] {
|
||||
if (this.fd === null) throw new Error('Data file not opened');
|
||||
|
||||
getBulkDataByTime(startTs: bigint, endTs: bigint): DataEntry<T>[] {
|
||||
if (!this.buffer) throw new Error('Data file not opened');
|
||||
const indexResults = this.indexReader.findByTimeRange(startTs, endTs);
|
||||
const results: DataEntry<T>[] = [];
|
||||
|
||||
const indexResults = this.indexReader.findByTimeRange(startTs, endTs);
|
||||
const results: DataEntry<T>[] = [];
|
||||
for (const { entry } of indexResults) {
|
||||
const buf = this.readRecord(entry.offset, entry.length);
|
||||
const result = DataProtocol.deserializeRecord(
|
||||
buf,
|
||||
0,
|
||||
this.serializer
|
||||
);
|
||||
if (result) {
|
||||
results.push({
|
||||
sequence: entry.sequence,
|
||||
timestamp: entry.timestamp,
|
||||
data: result.data,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const { entry } of indexResults) {
|
||||
const result = DataProtocol.deserializeRecord(
|
||||
this.buffer,
|
||||
Number(entry.offset),
|
||||
this.serializer
|
||||
);
|
||||
if (result) {
|
||||
results.push({
|
||||
sequence: entry.sequence,
|
||||
timestamp: entry.timestamp,
|
||||
data: result.data,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
getAllData(): DataEntry<T>[] {
|
||||
if (this.fd === null) throw new Error('Data file not opened');
|
||||
|
||||
getAllData(): DataEntry<T>[] {
|
||||
if (!this.buffer) throw new Error('Data file not opened');
|
||||
const entries = this.indexReader.getAllEntries();
|
||||
const results: DataEntry<T>[] = [];
|
||||
|
||||
const entries = this.indexReader.getAllEntries();
|
||||
const results: DataEntry<T>[] = [];
|
||||
for (const entry of entries) {
|
||||
const buf = this.readRecord(entry.offset, entry.length);
|
||||
const result = DataProtocol.deserializeRecord(
|
||||
buf,
|
||||
0,
|
||||
this.serializer
|
||||
);
|
||||
if (result) {
|
||||
results.push({
|
||||
sequence: entry.sequence,
|
||||
timestamp: entry.timestamp,
|
||||
data: result.data,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
const result = DataProtocol.deserializeRecord(
|
||||
this.buffer,
|
||||
Number(entry.offset),
|
||||
this.serializer
|
||||
);
|
||||
if (result) {
|
||||
results.push({
|
||||
sequence: entry.sequence,
|
||||
timestamp: entry.timestamp,
|
||||
data: result.data,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
getRecordCount(): number {
|
||||
return this.indexReader.getHeader().validCount;
|
||||
}
|
||||
|
||||
getLastSequence(): number {
|
||||
return this.indexReader.getHeader().lastSequence;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.buffer) {
|
||||
mmap.unmap(this.buffer);
|
||||
this.buffer = null;
|
||||
getRecordCount(): number {
|
||||
return this.indexReader.getHeader().writtenCnt;
|
||||
}
|
||||
if (this.fd !== null) {
|
||||
fs.closeSync(this.fd);
|
||||
this.fd = null;
|
||||
|
||||
getLastSequence(): number {
|
||||
return this.indexReader.getHeader().latestSequence;
|
||||
}
|
||||
this.header = null;
|
||||
this.indexReader.close();
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.fd !== null) {
|
||||
fs.closeSync(this.fd);
|
||||
this.fd = null;
|
||||
}
|
||||
this.header = null;
|
||||
this.indexReader.close();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
import { IndexFileOptions } from "../idx/types.js";
|
||||
|
||||
// src/data-file/types.ts
|
||||
export interface Serializer<T> {
|
||||
serialize(data: T): Buffer;
|
||||
deserialize(buf: Buffer): T;
|
||||
serialize(data: T): Buffer;
|
||||
deserialize(buf: Buffer): T;
|
||||
}
|
||||
|
||||
export interface DataEntry<T> {
|
||||
sequence: number;
|
||||
timestamp: bigint;
|
||||
data: T;
|
||||
sequence: number;
|
||||
timestamp: bigint;
|
||||
data: T;
|
||||
}
|
||||
|
||||
export interface DataFileOptions<T> {
|
||||
serializer: Serializer<T>;
|
||||
maxEntries?: number;
|
||||
}
|
||||
serializer: Serializer<T>;
|
||||
forceTruncate?: boolean;
|
||||
indexFileOpt: IndexFileOptions;
|
||||
}
|
||||
|
||||
@@ -2,119 +2,189 @@
|
||||
import * as fs from 'node:fs';
|
||||
import { DATA_HEADER_SIZE } from './constants.js';
|
||||
import { DataProtocol } from './protocol.js';
|
||||
import { IndexWriter } from '../idx/index.js';
|
||||
import { IndexWriter, IndexFileOptionsRequired } from '../idx/index.js';
|
||||
import type { Serializer, DataFileOptions } from './types.js';
|
||||
|
||||
export class DataWriter<T> {
|
||||
private fd: number | null = null;
|
||||
private headerBuf: Buffer | null = null;
|
||||
private currentOffset: bigint = BigInt(DATA_HEADER_SIZE);
|
||||
private recordCount = 0;
|
||||
private fd: number | null = null;
|
||||
private headerBuf: Buffer | null = null;
|
||||
private currentOffset: bigint = BigInt(DATA_HEADER_SIZE);
|
||||
private recordCount = 0;
|
||||
|
||||
private indexWriter: IndexWriter;
|
||||
private serializer: Serializer<T>;
|
||||
private indexWriter: IndexWriter;
|
||||
|
||||
readonly dataPath: string;
|
||||
readonly indexPath: string;
|
||||
// See DataFileOptions
|
||||
private readonly serializer: Serializer<T>;
|
||||
private readonly forceTruncate: boolean;
|
||||
|
||||
constructor(basePath: string, options: DataFileOptions<T>) {
|
||||
this.dataPath = `${basePath}.dat`;
|
||||
this.indexPath = `${basePath}.idx`;
|
||||
this.serializer = options.serializer;
|
||||
private latestSequence: number = 0;
|
||||
|
||||
const maxEntries = options.maxEntries ?? 10_000_000;
|
||||
this.indexWriter = new IndexWriter(this.indexPath, { maxEntries });
|
||||
}
|
||||
private readonly indexFileOpt: IndexFileOptionsRequired;
|
||||
|
||||
open(): void {
|
||||
const isNew = !fs.existsSync(this.dataPath);
|
||||
private dataPath: string | null = null;
|
||||
private indexPath: string | null = null;
|
||||
|
||||
this.fd = fs.openSync(this.dataPath, isNew ? 'w+' : 'r+');
|
||||
this.headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
|
||||
|
||||
if (isNew) {
|
||||
const header = DataProtocol.createHeader();
|
||||
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, 0);
|
||||
this.currentOffset = BigInt(DATA_HEADER_SIZE);
|
||||
this.recordCount = 0;
|
||||
} else {
|
||||
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
|
||||
const header = DataProtocol.readHeader(this.headerBuf);
|
||||
this.currentOffset = header.fileSize;
|
||||
this.recordCount = header.recordCount;
|
||||
constructor(options: DataFileOptions<T>) {
|
||||
this.serializer = options.serializer;
|
||||
this.forceTruncate = options.forceTruncate ?? false;
|
||||
|
||||
this.indexFileOpt = {
|
||||
maxEntries: options.indexFileOpt.maxEntries ?? 10_000_000,
|
||||
autoIncrementSequence: options.indexFileOpt.autoIncrementSequence ?? false
|
||||
}
|
||||
|
||||
this.indexWriter = new IndexWriter(this.indexFileOpt);
|
||||
}
|
||||
|
||||
this.indexWriter.open();
|
||||
}
|
||||
open(basePath: string): void {
|
||||
this.dataPath = `${basePath}.dat`;
|
||||
this.indexPath = `${basePath}.idx`;
|
||||
|
||||
append(data: T, timestamp?: bigint): number {
|
||||
if (this.fd === null) throw new Error('Data file not opened');
|
||||
// Index file 을 Open 함으로써 파일을 시작 할 수 있는지 검증 (Throw 로써)
|
||||
// Open index file with maxEntries and autoIncrementSequence
|
||||
const writtenCount = this.indexWriter.open(this.indexPath, this.forceTruncate);
|
||||
const isNew = !fs.existsSync(this.dataPath);
|
||||
|
||||
const buf = DataProtocol.serializeRecord(data, this.serializer);
|
||||
const offset = this.currentOffset;
|
||||
// Index file 은 초기화인데, 신규파일 혹은 강제 클리어가 아니라면
|
||||
if (writtenCount === 0 && !(isNew || this.forceTruncate)) {
|
||||
throw new Error(`Index file & Data File is invalid ${this.indexPath} is initial but ${this.dataPath} is exists`);
|
||||
}
|
||||
if (writtenCount > 0 && isNew) {
|
||||
throw new Error(`Index file & Data File is invalid data of ${this.indexPath} | ${writtenCount} is exists but ${this.dataPath} is not exists`);
|
||||
}
|
||||
|
||||
fs.writeSync(this.fd, buf, 0, buf.length, Number(offset));
|
||||
// Warn if forceTruncate will delete existing data
|
||||
if (this.forceTruncate && !isNew) {
|
||||
const stats = fs.statSync(this.dataPath);
|
||||
const sizeMB = (stats.size / 1024 / 1024).toFixed(2);
|
||||
console.warn(
|
||||
`[DataWriter] forceTruncate enabled: Deleting ${sizeMB} MB of existing data\n` +
|
||||
` Index: ${this.indexPath} (${writtenCount} records)\n` +
|
||||
` Data: ${this.dataPath}`
|
||||
);
|
||||
}
|
||||
|
||||
const sequence = this.indexWriter.getNextSequence();
|
||||
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
|
||||
this.fd = fs.openSync(this.dataPath,
|
||||
isNew || this.forceTruncate ? 'w+' : 'r+');
|
||||
|
||||
this.indexWriter.append(offset, buf.length, ts);
|
||||
try {
|
||||
this.headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
|
||||
|
||||
this.currentOffset += BigInt(buf.length);
|
||||
this.recordCount++;
|
||||
if (isNew || this.forceTruncate) {
|
||||
const header = DataProtocol.createHeader();
|
||||
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, 0);
|
||||
this.currentOffset = BigInt(DATA_HEADER_SIZE);
|
||||
this.recordCount = 0;
|
||||
this.latestSequence = 0;
|
||||
} else {
|
||||
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
|
||||
const header = DataProtocol.readHeader(this.headerBuf);
|
||||
|
||||
return sequence;
|
||||
}
|
||||
// Validate: Data file recordCount must match Index file writtenCnt
|
||||
if (header.recordCount !== writtenCount) {
|
||||
throw new Error(
|
||||
`Data file record count mismatch: Data has ${header.recordCount} but Index has ${writtenCount}`
|
||||
);
|
||||
}
|
||||
|
||||
appendBulk(records: T[], timestamp?: bigint): number[] {
|
||||
const sequences: number[] = [];
|
||||
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
|
||||
|
||||
for (const record of records) {
|
||||
const seq = this.append(record, ts);
|
||||
sequences.push(seq);
|
||||
this.currentOffset = header.fileSize;
|
||||
this.recordCount = header.recordCount;
|
||||
this.latestSequence = this.indexWriter.getLatestSequence();
|
||||
}
|
||||
} catch (error) {
|
||||
// Clean up resources on error
|
||||
if (this.fd !== null) {
|
||||
fs.closeSync(this.fd);
|
||||
this.fd = null;
|
||||
}
|
||||
this.headerBuf = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return sequences;
|
||||
}
|
||||
append(data: T, sequence?: number, timestamp?: bigint): number {
|
||||
if (this.fd === null) throw new Error('Data file not opened');
|
||||
|
||||
getLastSequence(): number {
|
||||
return this.indexWriter.getLastSequence();
|
||||
}
|
||||
const buf = DataProtocol.serializeRecord(data, this.serializer);
|
||||
const offset = this.currentOffset;
|
||||
|
||||
getNextSequence(): number {
|
||||
return this.indexWriter.getNextSequence();
|
||||
}
|
||||
fs.writeSync(this.fd, buf, 0, buf.length, Number(offset));
|
||||
|
||||
sync(): void {
|
||||
if (this.fd === null || !this.headerBuf) return;
|
||||
// Write to index file
|
||||
this.indexWriter.write(offset, buf.length, sequence, timestamp);
|
||||
|
||||
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
|
||||
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
|
||||
fs.fsyncSync(this.fd);
|
||||
// Update latestSequence to the most recent sequence
|
||||
this.latestSequence = this.indexWriter.getLatestSequence();
|
||||
|
||||
this.indexWriter.syncAll();
|
||||
}
|
||||
this.currentOffset += BigInt(buf.length);
|
||||
++this.recordCount;
|
||||
|
||||
close(): void {
|
||||
this.sync();
|
||||
|
||||
if (this.fd !== null) {
|
||||
fs.closeSync(this.fd);
|
||||
this.fd = null;
|
||||
return this.latestSequence;
|
||||
}
|
||||
|
||||
this.indexWriter.close();
|
||||
this.headerBuf = null;
|
||||
}
|
||||
/*
|
||||
appendBulk(records: T[], sequences?: number[], timestamp?: bigint): number[] {
|
||||
// Runtime check: sequences required when autoIncrementSequence is false
|
||||
if (!this.autoIncrementSequence) {
|
||||
if (!sequences) {
|
||||
throw new Error('sequences is required when autoIncrementSequence is false');
|
||||
}
|
||||
if (sequences.length !== records.length) {
|
||||
throw new Error(`sequences length (${sequences.length}) must match records length (${records.length})`);
|
||||
}
|
||||
}
|
||||
|
||||
getStats() {
|
||||
return {
|
||||
dataPath: this.dataPath,
|
||||
indexPath: this.indexPath,
|
||||
currentOffset: this.currentOffset,
|
||||
recordCount: this.recordCount,
|
||||
lastSequence: this.indexWriter.getLastSequence(),
|
||||
};
|
||||
}
|
||||
}
|
||||
const resultSequences: number[] = [];
|
||||
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
|
||||
|
||||
for (let i = 0; i < records.length; i++) {
|
||||
const seq = sequences?.[i];
|
||||
const resultSeq = this.append(records[i], seq, ts);
|
||||
resultSequences.push(resultSeq);
|
||||
}
|
||||
|
||||
return resultSequences;
|
||||
}
|
||||
*/
|
||||
|
||||
getLatestSequence(): number {
|
||||
return this.latestSequence;
|
||||
}
|
||||
|
||||
getNextSequence(): number {
|
||||
return this.latestSequence + 1;
|
||||
}
|
||||
|
||||
sync(): void {
|
||||
if (this.fd === null || !this.headerBuf) return;
|
||||
|
||||
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
|
||||
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
|
||||
fs.fsyncSync(this.fd);
|
||||
|
||||
this.indexWriter.syncAll();
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.sync();
|
||||
|
||||
if (this.fd !== null) {
|
||||
fs.closeSync(this.fd);
|
||||
this.fd = null;
|
||||
}
|
||||
|
||||
this.indexWriter.close();
|
||||
this.headerBuf = null;
|
||||
}
|
||||
|
||||
getStats() {
|
||||
return {
|
||||
dataPath: this.dataPath,
|
||||
indexPath: this.indexPath,
|
||||
currentOffset: this.currentOffset,
|
||||
recordCount: this.recordCount,
|
||||
latestSequence: this.indexWriter.getLatestSequence(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,106 +1,108 @@
|
||||
// src/index-file/protocol.ts
|
||||
import {
|
||||
INDEX_MAGIC,
|
||||
INDEX_VERSION,
|
||||
INDEX_HEADER_SIZE,
|
||||
INDEX_ENTRY_SIZE,
|
||||
FLAG_VALID,
|
||||
INDEX_MAGIC,
|
||||
INDEX_VERSION,
|
||||
INDEX_HEADER_SIZE,
|
||||
INDEX_ENTRY_SIZE,
|
||||
FLAG_VALID,
|
||||
} from './constants.js';
|
||||
import type { IndexHeader, IndexEntry } from './types.js';
|
||||
|
||||
const CRC_TABLE = new Uint32Array(256);
|
||||
for (let i = 0; i < 256; i++) {
|
||||
let c = i;
|
||||
for (let j = 0; j < 8; j++) {
|
||||
c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
|
||||
}
|
||||
CRC_TABLE[i] = c >>> 0;
|
||||
let c = i;
|
||||
for (let j = 0; j < 8; j++) {
|
||||
c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
|
||||
}
|
||||
CRC_TABLE[i] = c >>> 0;
|
||||
}
|
||||
|
||||
export function crc32(buf: Buffer, start = 0, end?: number): number {
|
||||
let crc = 0xFFFFFFFF;
|
||||
const len = end ?? buf.length;
|
||||
for (let i = start; i < len; i++) {
|
||||
crc = CRC_TABLE[(crc ^ buf[i]) & 0xFF] ^ (crc >>> 8);
|
||||
}
|
||||
return (~crc) >>> 0;
|
||||
let crc = 0xFFFFFFFF;
|
||||
const len = end ?? buf.length;
|
||||
for (let i = start; i < len; i++) {
|
||||
crc = CRC_TABLE[(crc ^ buf[i]) & 0xFF] ^ (crc >>> 8);
|
||||
}
|
||||
return (~crc) >>> 0;
|
||||
}
|
||||
|
||||
export class IndexProtocol {
|
||||
static createHeader(entryCount: number, magic = INDEX_MAGIC): Buffer {
|
||||
const buf = Buffer.alloc(INDEX_HEADER_SIZE);
|
||||
buf.write(magic, 0, 4, 'ascii');
|
||||
buf.writeUInt32LE(INDEX_VERSION, 4);
|
||||
buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
|
||||
buf.writeUInt32LE(INDEX_ENTRY_SIZE, 16);
|
||||
buf.writeUInt32LE(entryCount, 20);
|
||||
buf.writeUInt32LE(0, 24);
|
||||
buf.writeBigUInt64LE(0n, 28);
|
||||
buf.writeUInt32LE(0, 36);
|
||||
return buf;
|
||||
}
|
||||
static createHeader(entryCount: number, autoIncrementSequence: boolean, magic = INDEX_MAGIC): Buffer {
|
||||
const buf = Buffer.alloc(INDEX_HEADER_SIZE);
|
||||
buf.write(magic, 0, 4, 'ascii');
|
||||
buf.writeUInt32LE(INDEX_VERSION, 4);
|
||||
buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
|
||||
buf.writeUInt32LE(INDEX_ENTRY_SIZE, 16);
|
||||
buf.writeUInt32LE(entryCount, 20);
|
||||
buf.writeUInt32LE(0, 24); // writtenCnt
|
||||
buf.writeBigUInt64LE(0n, 28); // dataFileSize
|
||||
buf.writeUInt32LE(0, 36); // latestSequence
|
||||
buf.writeUInt8(autoIncrementSequence ? 1 : 0, 40); // autoIncrementSequence
|
||||
return buf;
|
||||
}
|
||||
|
||||
static readHeader(buf: Buffer): IndexHeader {
|
||||
return {
|
||||
magic: buf.toString('ascii', 0, 4),
|
||||
version: buf.readUInt32LE(4),
|
||||
createdAt: buf.readBigUInt64LE(8),
|
||||
entrySize: buf.readUInt32LE(16),
|
||||
entryCount: buf.readUInt32LE(20),
|
||||
validCount: buf.readUInt32LE(24),
|
||||
dataFileSize: buf.readBigUInt64LE(28),
|
||||
lastSequence: buf.readUInt32LE(36),
|
||||
reserved: buf.subarray(40, 64),
|
||||
};
|
||||
}
|
||||
static readHeader(buf: Buffer): IndexHeader {
|
||||
return {
|
||||
magic: buf.toString('ascii', 0, 4),
|
||||
version: buf.readUInt32LE(4),
|
||||
createdAt: buf.readBigUInt64LE(8),
|
||||
entrySize: buf.readUInt32LE(16),
|
||||
entryCount: buf.readUInt32LE(20),
|
||||
writtenCnt: buf.readUInt32LE(24),
|
||||
dataFileSize: buf.readBigUInt64LE(28),
|
||||
latestSequence: buf.readUInt32LE(36),
|
||||
autoIncrementSequence: buf.readUInt8(40) === 1,
|
||||
reserved: buf.subarray(41, 64),
|
||||
};
|
||||
}
|
||||
|
||||
static updateHeaderCounts(
|
||||
buf: Buffer,
|
||||
validCount: number,
|
||||
dataFileSize: bigint,
|
||||
lastSequence: number
|
||||
): void {
|
||||
buf.writeUInt32LE(validCount, 24);
|
||||
buf.writeBigUInt64LE(dataFileSize, 28);
|
||||
buf.writeUInt32LE(lastSequence, 36);
|
||||
}
|
||||
static updateHeaderCounts(
|
||||
buf: Buffer,
|
||||
writtenCnt: number,
|
||||
dataFileSize: bigint,
|
||||
latestSequence: number
|
||||
): void {
|
||||
buf.writeUInt32LE(writtenCnt, 24);
|
||||
buf.writeBigUInt64LE(dataFileSize, 28);
|
||||
buf.writeUInt32LE(latestSequence, 36);
|
||||
}
|
||||
|
||||
static writeEntry(buf: Buffer, index: number, entry: Omit<IndexEntry, 'checksum'>): void {
|
||||
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
||||
static writeEntry(buf: Buffer, index: number, entry: Omit<IndexEntry, 'checksum'>): void {
|
||||
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
||||
|
||||
buf.writeUInt32LE(entry.sequence, off);
|
||||
buf.writeBigUInt64LE(entry.timestamp, off + 4);
|
||||
buf.writeBigUInt64LE(entry.offset, off + 12);
|
||||
buf.writeUInt32LE(entry.length, off + 20);
|
||||
buf.writeUInt32LE(entry.flags | FLAG_VALID, off + 24);
|
||||
buf.writeUInt32LE(entry.sequence, off);
|
||||
buf.writeBigUInt64LE(entry.timestamp, off + 4);
|
||||
buf.writeBigUInt64LE(entry.offset, off + 12);
|
||||
buf.writeUInt32LE(entry.length, off + 20);
|
||||
buf.writeUInt32LE(entry.flags | FLAG_VALID, off + 24);
|
||||
|
||||
const checksum = crc32(buf, off, off + 28);
|
||||
buf.writeUInt32LE(checksum, off + 28);
|
||||
}
|
||||
const checksum = crc32(buf, off, off + 28);
|
||||
buf.writeUInt32LE(checksum, off + 28);
|
||||
}
|
||||
|
||||
static readEntry(buf: Buffer, index: number): IndexEntry | null {
|
||||
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
||||
const flags = buf.readUInt32LE(off + 24);
|
||||
static readEntry(buf: Buffer, index: number): IndexEntry | null {
|
||||
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
||||
const flags = buf.readUInt32LE(off + 24);
|
||||
|
||||
if (!(flags & FLAG_VALID)) return null;
|
||||
if (!(flags & FLAG_VALID)) return null;
|
||||
|
||||
return {
|
||||
sequence: buf.readUInt32LE(off),
|
||||
timestamp: buf.readBigUInt64LE(off + 4),
|
||||
offset: buf.readBigUInt64LE(off + 12),
|
||||
length: buf.readUInt32LE(off + 20),
|
||||
flags,
|
||||
checksum: buf.readUInt32LE(off + 28),
|
||||
};
|
||||
}
|
||||
return {
|
||||
sequence: buf.readUInt32LE(off),
|
||||
timestamp: buf.readBigUInt64LE(off + 4),
|
||||
offset: buf.readBigUInt64LE(off + 12),
|
||||
length: buf.readUInt32LE(off + 20),
|
||||
flags,
|
||||
checksum: buf.readUInt32LE(off + 28),
|
||||
};
|
||||
}
|
||||
|
||||
static isValidEntry(buf: Buffer, index: number): boolean {
|
||||
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
||||
const flags = buf.readUInt32LE(off + 24);
|
||||
return (flags & FLAG_VALID) !== 0;
|
||||
}
|
||||
static isValidEntry(buf: Buffer, index: number): boolean {
|
||||
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
|
||||
const flags = buf.readUInt32LE(off + 24);
|
||||
return (flags & FLAG_VALID) !== 0;
|
||||
}
|
||||
|
||||
static calcFileSize(entryCount: number): number {
|
||||
return INDEX_HEADER_SIZE + INDEX_ENTRY_SIZE * entryCount;
|
||||
}
|
||||
}
|
||||
static calcFileSize(entryCount: number): number {
|
||||
return INDEX_HEADER_SIZE + INDEX_ENTRY_SIZE * entryCount;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,131 +1,114 @@
|
||||
// src/index-file/reader.ts
|
||||
import * as fs from 'node:fs';
|
||||
import mmap from '@elilee/mmap-native';
|
||||
import { IndexProtocol } from './protocol.js';
|
||||
import type { IndexHeader, IndexEntry } from './types.js';
|
||||
|
||||
export class IndexReader {
|
||||
private fd: number | null = null;
|
||||
private buffer: Buffer | null = null;
|
||||
private header: IndexHeader | null = null;
|
||||
private buffer: Buffer | null = null;
|
||||
private header: IndexHeader | null = null;
|
||||
|
||||
readonly path: string;
|
||||
readonly path: string;
|
||||
|
||||
constructor(path: string) {
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
open(): void {
|
||||
const stats = fs.statSync(this.path);
|
||||
this.fd = fs.openSync(this.path, 'r');
|
||||
|
||||
this.buffer = mmap.map(
|
||||
stats.size,
|
||||
mmap.PROT_READ,
|
||||
mmap.MAP_SHARED,
|
||||
this.fd,
|
||||
0
|
||||
);
|
||||
|
||||
this.header = IndexProtocol.readHeader(this.buffer);
|
||||
}
|
||||
|
||||
getHeader(): IndexHeader {
|
||||
if (!this.header) throw new Error('Index file not opened');
|
||||
return this.header;
|
||||
}
|
||||
|
||||
getEntry(index: number): IndexEntry | null {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
if (index < 0 || index >= this.header.entryCount) return null;
|
||||
return IndexProtocol.readEntry(this.buffer, index);
|
||||
}
|
||||
|
||||
findBySequence(sequence: number): { index: number; entry: IndexEntry } | null {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
for (let i = 0; i < this.header.validCount; i++) {
|
||||
const entry = IndexProtocol.readEntry(this.buffer, i);
|
||||
if (entry && entry.sequence === sequence) {
|
||||
return { index: i, entry };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
findBySequenceRange(startSeq: number, endSeq: number): { index: number; entry: IndexEntry }[] {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
const results: { index: number; entry: IndexEntry }[] = [];
|
||||
for (let i = 0; i < this.header.validCount; i++) {
|
||||
const entry = IndexProtocol.readEntry(this.buffer, i);
|
||||
if (entry && entry.sequence >= startSeq && entry.sequence <= endSeq) {
|
||||
results.push({ index: i, entry });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
getAllEntries(): IndexEntry[] {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
const entries: IndexEntry[] = [];
|
||||
for (let i = 0; i < this.header.validCount; i++) {
|
||||
const entry = IndexProtocol.readEntry(this.buffer, i);
|
||||
if (entry) entries.push(entry);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
findByTimeRange(startTs: bigint, endTs: bigint): { index: number; entry: IndexEntry }[] {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
const results: { index: number; entry: IndexEntry }[] = [];
|
||||
for (let i = 0; i < this.header.validCount; i++) {
|
||||
const entry = IndexProtocol.readEntry(this.buffer, i);
|
||||
if (entry && entry.timestamp >= startTs && entry.timestamp <= endTs) {
|
||||
results.push({ index: i, entry });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
binarySearchBySequence(targetSeq: number): { index: number; entry: IndexEntry } | null {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
let left = 0;
|
||||
let right = this.header.validCount - 1;
|
||||
|
||||
while (left <= right) {
|
||||
const mid = Math.floor((left + right) / 2);
|
||||
const entry = IndexProtocol.readEntry(this.buffer, mid);
|
||||
|
||||
if (!entry) {
|
||||
right = mid - 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.sequence === targetSeq) {
|
||||
return { index: mid, entry };
|
||||
} else if (entry.sequence < targetSeq) {
|
||||
left = mid + 1;
|
||||
} else {
|
||||
right = mid - 1;
|
||||
}
|
||||
constructor(path: string) {
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
open(): void {
|
||||
// Read entire file into buffer (simpler than mmap for read-only access)
|
||||
this.buffer = fs.readFileSync(this.path);
|
||||
this.header = IndexProtocol.readHeader(this.buffer);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.buffer) {
|
||||
mmap.unmap(this.buffer);
|
||||
this.buffer = null;
|
||||
getHeader(): IndexHeader {
|
||||
if (!this.header) throw new Error('Index file not opened');
|
||||
return this.header;
|
||||
}
|
||||
if (this.fd !== null) {
|
||||
fs.closeSync(this.fd);
|
||||
this.fd = null;
|
||||
|
||||
getEntry(index: number): IndexEntry | null {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
if (index < 0 || index >= this.header.entryCount) return null;
|
||||
return IndexProtocol.readEntry(this.buffer, index);
|
||||
}
|
||||
this.header = null;
|
||||
}
|
||||
}
|
||||
|
||||
findBySequence(sequence: number): { index: number; entry: IndexEntry } | null {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
for (let i = 0; i < this.header.writtenCnt; i++) {
|
||||
const entry = IndexProtocol.readEntry(this.buffer, i);
|
||||
if (entry && entry.sequence === sequence) {
|
||||
return { index: i, entry };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
findBySequenceRange(startSeq: number, endSeq: number): { index: number; entry: IndexEntry }[] {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
const results: { index: number; entry: IndexEntry }[] = [];
|
||||
for (let i = 0; i < this.header.writtenCnt; i++) {
|
||||
const entry = IndexProtocol.readEntry(this.buffer, i);
|
||||
if (entry && entry.sequence >= startSeq && entry.sequence <= endSeq) {
|
||||
results.push({ index: i, entry });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
getAllEntries(): IndexEntry[] {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
const entries: IndexEntry[] = [];
|
||||
for (let i = 0; i < this.header.writtenCnt; i++) {
|
||||
const entry = IndexProtocol.readEntry(this.buffer, i);
|
||||
if (entry) entries.push(entry);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
findByTimeRange(startTs: bigint, endTs: bigint): { index: number; entry: IndexEntry }[] {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
const results: { index: number; entry: IndexEntry }[] = [];
|
||||
for (let i = 0; i < this.header.writtenCnt; i++) {
|
||||
const entry = IndexProtocol.readEntry(this.buffer, i);
|
||||
if (entry && entry.timestamp >= startTs && entry.timestamp <= endTs) {
|
||||
results.push({ index: i, entry });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
binarySearchBySequence(targetSeq: number): { index: number; entry: IndexEntry } | null {
|
||||
if (!this.buffer || !this.header) throw new Error('Index file not opened');
|
||||
|
||||
let left = 0;
|
||||
let right = this.header.writtenCnt - 1;
|
||||
|
||||
while (left <= right) {
|
||||
const mid = Math.floor((left + right) / 2);
|
||||
const entry = IndexProtocol.readEntry(this.buffer, mid);
|
||||
|
||||
if (!entry) {
|
||||
right = mid - 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.sequence === targetSeq) {
|
||||
return { index: mid, entry };
|
||||
} else if (entry.sequence < targetSeq) {
|
||||
left = mid + 1;
|
||||
} else {
|
||||
right = mid - 1;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
// Simply release buffer reference (GC will handle cleanup)
|
||||
this.buffer = null;
|
||||
this.header = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,26 +1,29 @@
|
||||
// src/index-file/types.ts
|
||||
export interface IndexHeader {
|
||||
magic: string;
|
||||
version: number;
|
||||
createdAt: bigint;
|
||||
entrySize: number;
|
||||
entryCount: number;
|
||||
validCount: number;
|
||||
dataFileSize: bigint;
|
||||
lastSequence: number;
|
||||
reserved: Buffer;
|
||||
magic: string;
|
||||
version: number;
|
||||
createdAt: bigint;
|
||||
entrySize: number;
|
||||
entryCount: number;
|
||||
writtenCnt: number;
|
||||
dataFileSize: bigint;
|
||||
latestSequence: number;
|
||||
autoIncrementSequence: boolean;
|
||||
reserved: Buffer;
|
||||
}
|
||||
|
||||
export interface IndexEntry {
|
||||
sequence: number;
|
||||
timestamp: bigint;
|
||||
offset: bigint;
|
||||
length: number;
|
||||
flags: number;
|
||||
checksum: number;
|
||||
sequence: number;
|
||||
timestamp: bigint;
|
||||
offset: bigint;
|
||||
length: number;
|
||||
flags: number;
|
||||
checksum: number;
|
||||
}
|
||||
|
||||
export interface IndexFileOptions {
|
||||
maxEntries: number;
|
||||
magic?: string;
|
||||
}
|
||||
maxEntries?: number;
|
||||
autoIncrementSequence?: boolean;
|
||||
}
|
||||
|
||||
export type IndexFileOptionsRequired = Required<IndexFileOptions>;
|
||||
|
||||
@@ -1,141 +1,209 @@
|
||||
// src/index-file/writer.ts
|
||||
import * as fs from 'node:fs';
|
||||
import mmap from '@elilee/mmap-native';
|
||||
import { INDEX_HEADER_SIZE, FLAG_VALID } from './constants.js';
|
||||
import { INDEX_HEADER_SIZE, INDEX_ENTRY_SIZE, FLAG_VALID } from './constants.js';
|
||||
import { IndexProtocol } from './protocol.js';
|
||||
import type { IndexFileOptions } from './types.js';
|
||||
import { IndexFileOptionsRequired } from './types.js';
|
||||
|
||||
export class IndexWriter {
|
||||
private fd: number | null = null;
|
||||
private buffer: Buffer | null = null;
|
||||
private validCount = 0;
|
||||
private dataFileSize = 0n;
|
||||
private lastSequence = 0;
|
||||
private fd: number | null = null;
|
||||
private headerBuf: Buffer | null = null;
|
||||
private entryBuf: Buffer | null = null;
|
||||
|
||||
readonly path: string;
|
||||
readonly maxEntries: number;
|
||||
readonly fileSize: number;
|
||||
private writtenCnt = 0;
|
||||
private dataFileSize = 0n;
|
||||
private latestSequence = 0;
|
||||
|
||||
constructor(path: string, options: IndexFileOptions) {
|
||||
this.path = path;
|
||||
this.maxEntries = options.maxEntries;
|
||||
this.fileSize = IndexProtocol.calcFileSize(options.maxEntries);
|
||||
}
|
||||
private path: string | null = null;
|
||||
private fileSize: number = 0;
|
||||
|
||||
open(): void {
|
||||
const isNew = !fs.existsSync(this.path);
|
||||
// see IndexFileOptions
|
||||
private maxEntries: number = 0;
|
||||
private autoIncrementSequence: boolean = false;
|
||||
|
||||
this.fd = fs.openSync(this.path, isNew ? 'w+' : 'r+');
|
||||
|
||||
if (isNew) {
|
||||
fs.ftruncateSync(this.fd, this.fileSize);
|
||||
constructor(opt: IndexFileOptionsRequired) {
|
||||
// Empty constructor - maxEntries provided in open()
|
||||
this.maxEntries = opt.maxEntries;
|
||||
this.autoIncrementSequence = opt.autoIncrementSequence;
|
||||
}
|
||||
|
||||
this.buffer = mmap.map(
|
||||
this.fileSize,
|
||||
mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
mmap.MAP_SHARED,
|
||||
this.fd,
|
||||
0
|
||||
);
|
||||
open(path: string, forceTruncate: boolean = false): number {
|
||||
this.path = path;
|
||||
|
||||
if (isNew) {
|
||||
const header = IndexProtocol.createHeader(this.maxEntries);
|
||||
header.copy(this.buffer, 0);
|
||||
this.syncHeader();
|
||||
} else {
|
||||
const header = IndexProtocol.readHeader(this.buffer);
|
||||
this.validCount = header.validCount;
|
||||
this.dataFileSize = header.dataFileSize;
|
||||
this.lastSequence = header.lastSequence;
|
||||
}
|
||||
}
|
||||
const isNew = !fs.existsSync(this.path);
|
||||
|
||||
write(
|
||||
index: number,
|
||||
sequence: number,
|
||||
offset: bigint,
|
||||
length: number,
|
||||
timestamp?: bigint
|
||||
): boolean {
|
||||
if (!this.buffer) throw new Error('Index file not opened');
|
||||
if (index < 0 || index >= this.maxEntries) return false;
|
||||
if (isNew || forceTruncate) {
|
||||
// New file: use provided values
|
||||
this.fileSize = IndexProtocol.calcFileSize(this.maxEntries);
|
||||
this.writtenCnt = 0;
|
||||
this.dataFileSize = 0n;
|
||||
this.latestSequence = 0;
|
||||
|
||||
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
|
||||
this.fd = fs.openSync(this.path, 'w+');
|
||||
fs.ftruncateSync(this.fd, this.fileSize);
|
||||
|
||||
IndexProtocol.writeEntry(this.buffer, index, {
|
||||
sequence,
|
||||
timestamp: ts,
|
||||
offset,
|
||||
length,
|
||||
flags: FLAG_VALID,
|
||||
});
|
||||
// Allocate buffers for header and entry
|
||||
this.headerBuf = Buffer.alloc(INDEX_HEADER_SIZE);
|
||||
this.entryBuf = Buffer.alloc(INDEX_ENTRY_SIZE);
|
||||
|
||||
this.validCount++;
|
||||
if (sequence > this.lastSequence) {
|
||||
this.lastSequence = sequence;
|
||||
const header = IndexProtocol.createHeader(this.maxEntries, this.autoIncrementSequence);
|
||||
header.copy(this.headerBuf, 0);
|
||||
|
||||
// Write header to file
|
||||
fs.writeSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0);
|
||||
fs.fsyncSync(this.fd);
|
||||
} else {
|
||||
// Existing file: read header first
|
||||
this.fd = fs.openSync(this.path, 'r+');
|
||||
|
||||
try {
|
||||
this.headerBuf = Buffer.alloc(INDEX_HEADER_SIZE);
|
||||
this.entryBuf = Buffer.alloc(INDEX_ENTRY_SIZE);
|
||||
|
||||
fs.readSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0);
|
||||
const header = IndexProtocol.readHeader(this.headerBuf);
|
||||
|
||||
if (this.maxEntries !== header.entryCount) {
|
||||
throw new Error(
|
||||
`maxEntries mismatch: provided ${this.maxEntries} but file has ${header.entryCount}`
|
||||
);
|
||||
}
|
||||
if (this.autoIncrementSequence !== header.autoIncrementSequence) {
|
||||
throw new Error(
|
||||
`autoIncrementSequence mismatch: provided ${this.autoIncrementSequence} but file has ${header.autoIncrementSequence}`
|
||||
);
|
||||
}
|
||||
|
||||
const expectFileSize = IndexProtocol.calcFileSize(this.maxEntries);
|
||||
const calcedFileSize = IndexProtocol.calcFileSize(header.entryCount);
|
||||
if (expectFileSize !== calcedFileSize) {
|
||||
// if (opt.version !== header.version) { 버전이 다른거니까 어떻게 처리 할지는 추후 고민 TODO }
|
||||
throw new Error(
|
||||
`Indexfile size calc is invalid : provided ${expectFileSize} but file has ${calcedFileSize}`
|
||||
);
|
||||
}
|
||||
|
||||
this.fileSize = calcedFileSize;
|
||||
this.writtenCnt = header.writtenCnt;
|
||||
this.dataFileSize = header.dataFileSize;
|
||||
this.latestSequence = header.latestSequence;
|
||||
} catch (error) {
|
||||
// Clean up resources on error
|
||||
if (this.fd !== null) {
|
||||
fs.closeSync(this.fd);
|
||||
this.fd = null;
|
||||
}
|
||||
this.headerBuf = null;
|
||||
this.entryBuf = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return this.writtenCnt;
|
||||
}
|
||||
|
||||
const newDataEnd = offset + BigInt(length);
|
||||
if (newDataEnd > this.dataFileSize) {
|
||||
this.dataFileSize = newDataEnd;
|
||||
write(
|
||||
offset: bigint,
|
||||
length: number,
|
||||
sequence?: number,
|
||||
timestamp?: bigint
|
||||
): boolean {
|
||||
if (!this.entryBuf || this.fd === null) throw new Error('Index file not opened');
|
||||
if (this.writtenCnt >= this.maxEntries) {
|
||||
throw new Error(`Data count exceed provide : ${this.writtenCnt + 1} - max : ${this.maxEntries}`);
|
||||
}
|
||||
|
||||
// Calculate sequence
|
||||
let seq: number;
|
||||
if (!this.autoIncrementSequence) {
|
||||
if (sequence === undefined) {
|
||||
throw new Error('sequence is required when autoIncrementSequence is false');
|
||||
}
|
||||
seq = sequence;
|
||||
} else {
|
||||
seq = this.writtenCnt + 1;
|
||||
}
|
||||
|
||||
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
|
||||
|
||||
// Create a temporary buffer for this entry
|
||||
const tempBuf = Buffer.alloc(INDEX_HEADER_SIZE + (this.writtenCnt + 1) * INDEX_ENTRY_SIZE);
|
||||
|
||||
// Write entry to temp buffer
|
||||
IndexProtocol.writeEntry(tempBuf, this.writtenCnt, {
|
||||
sequence: seq,
|
||||
timestamp: ts,
|
||||
offset,
|
||||
length,
|
||||
flags: FLAG_VALID,
|
||||
});
|
||||
|
||||
// Calculate file offset for this entry
|
||||
const fileOffset = INDEX_HEADER_SIZE + this.writtenCnt * INDEX_ENTRY_SIZE;
|
||||
|
||||
// Write entry to file
|
||||
fs.writeSync(this.fd, tempBuf, fileOffset, INDEX_ENTRY_SIZE, fileOffset);
|
||||
|
||||
this.writtenCnt++;
|
||||
this.latestSequence = seq;
|
||||
|
||||
const newDataEnd = offset + BigInt(length);
|
||||
if (newDataEnd > this.dataFileSize) {
|
||||
this.dataFileSize = newDataEnd;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
getLatestSequence(): number {
|
||||
return this.latestSequence;
|
||||
}
|
||||
|
||||
append(offset: bigint, length: number, timestamp?: bigint): number {
|
||||
const index = this.validCount;
|
||||
if (index >= this.maxEntries) return -1;
|
||||
syncHeader(): void {
|
||||
if (!this.headerBuf || this.fd === null) return;
|
||||
|
||||
const sequence = this.lastSequence + 1;
|
||||
this.write(index, sequence, offset, length, timestamp);
|
||||
return index;
|
||||
}
|
||||
// Update header counts
|
||||
IndexProtocol.updateHeaderCounts(
|
||||
this.headerBuf,
|
||||
this.writtenCnt,
|
||||
this.dataFileSize,
|
||||
this.latestSequence
|
||||
);
|
||||
|
||||
getLastSequence(): number {
|
||||
return this.lastSequence;
|
||||
}
|
||||
// Write header to file
|
||||
fs.writeSync(this.fd, this.headerBuf, 0, INDEX_HEADER_SIZE, 0);
|
||||
}
|
||||
|
||||
getNextSequence(): number {
|
||||
return this.lastSequence + 1;
|
||||
}
|
||||
syncAll(): void {
|
||||
if (this.fd === null) return;
|
||||
|
||||
syncHeader(): void {
|
||||
if (!this.buffer) return;
|
||||
IndexProtocol.updateHeaderCounts(
|
||||
this.buffer,
|
||||
this.validCount,
|
||||
this.dataFileSize,
|
||||
this.lastSequence
|
||||
);
|
||||
mmap.sync(this.buffer, 0, INDEX_HEADER_SIZE, mmap.MS_ASYNC);
|
||||
}
|
||||
// Sync header first
|
||||
this.syncHeader();
|
||||
|
||||
syncAll(): void {
|
||||
if (!this.buffer) return;
|
||||
this.syncHeader();
|
||||
mmap.sync(this.buffer, 0, this.fileSize, mmap.MS_SYNC);
|
||||
}
|
||||
// Sync all file changes to disk
|
||||
fs.fsyncSync(this.fd);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (!this.buffer || this.fd === null) return;
|
||||
close(): void {
|
||||
if (this.fd === null) return;
|
||||
|
||||
this.syncAll();
|
||||
mmap.unmap(this.buffer);
|
||||
fs.closeSync(this.fd);
|
||||
// 1. Sync all changes
|
||||
this.syncAll();
|
||||
|
||||
this.buffer = null;
|
||||
this.fd = null;
|
||||
}
|
||||
// 2. Close file descriptor
|
||||
fs.closeSync(this.fd);
|
||||
this.fd = null;
|
||||
|
||||
getStats() {
|
||||
return {
|
||||
path: this.path,
|
||||
maxEntries: this.maxEntries,
|
||||
validCount: this.validCount,
|
||||
dataFileSize: this.dataFileSize,
|
||||
lastSequence: this.lastSequence,
|
||||
};
|
||||
}
|
||||
}
|
||||
// 3. Clean up buffers
|
||||
this.headerBuf = null;
|
||||
this.entryBuf = null;
|
||||
}
|
||||
|
||||
getStats() {
|
||||
return {
|
||||
path: this.path,
|
||||
writtenCnt: this.writtenCnt,
|
||||
dataFileSize: this.dataFileSize,
|
||||
latestSequence: this.latestSequence,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
1138
package-lock.json
generated
1138
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -14,8 +14,7 @@
|
||||
],
|
||||
"dependencies": {
|
||||
"typescript": "^5.7.0",
|
||||
"@types/node": "^22.0.0",
|
||||
"@elilee/mmap-native": "git+https://git.satitech.co.kr/sati-open/sati.n-api.mmap.git"
|
||||
"@types/node": "^22.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"prepare": "tsc -p tsconfig.json",
|
||||
|
||||
Reference in New Issue
Block a user