Init indexed file

lib/dat/binary-helpers.ts

@@ -0,0 +1,92 @@
+export class BinaryWriter {
+  private chunks: Buffer[] = [];
+
+  writeUInt32(value: number): this {
+    const buf = Buffer.alloc(4);
+    buf.writeUInt32LE(value, 0);
+    this.chunks.push(buf);
+    return this;
+  }
+
+  writeDouble(value: number): this {
+    const buf = Buffer.alloc(8);
+    buf.writeDoubleLE(value, 0);
+    this.chunks.push(buf);
+    return this;
+  }
+
+  writeBigUInt64(value: bigint): this {
+    const buf = Buffer.alloc(8);
+    buf.writeBigUInt64LE(value, 0);
+    this.chunks.push(buf);
+    return this;
+  }
+
+  writeString(value: string): this {
+    const strBuf = Buffer.from(value, 'utf8');
+    this.writeUInt32(strBuf.length);
+    this.chunks.push(strBuf);
+    return this;
+  }
+
+  writeNumberArray(values: number[]): this {
+    this.writeUInt32(values.length);
+    for (const v of values) this.writeDouble(v);
+    return this;
+  }
+
+  writeStringArray(values: string[]): this {
+    this.writeUInt32(values.length);
+    for (const v of values) this.writeString(v);
+    return this;
+  }
+
+  toBuffer(): Buffer {
+    return Buffer.concat(this.chunks);
+  }
+}
+
+export class BinaryReader {
+  private offset = 0;
+
+  constructor(private buf: Buffer) {}
+
+  readUInt32(): number {
+    const v = this.buf.readUInt32LE(this.offset);
+    this.offset += 4;
+    return v;
+  }
+
+  readDouble(): number {
+    const v = this.buf.readDoubleLE(this.offset);
+    this.offset += 8;
+    return v;
+  }
+
+  readBigUInt64(): bigint {
+    const v = this.buf.readBigUInt64LE(this.offset);
+    this.offset += 8;
+    return v;
+  }
+
+  readString(): string {
+    const len = this.readUInt32();
+    const v = this.buf.toString('utf8', this.offset, this.offset + len);
+    this.offset += len;
+    return v;
+  }
+
+  readNumberArray(): number[] {
+    const len = this.readUInt32();
+    const arr: number[] = [];
+    for (let i = 0; i < len; i++) arr.push(this.readDouble());
+    return arr;
+  }
+
+  readStringArray(): string[] {
+    const len = this.readUInt32();
+    const arr: string[] = [];
+    for (let i = 0; i < len; i++) arr.push(this.readString());
+    return arr;
+  }
+}

lib/dat/constants.ts

@@ -0,0 +1,5 @@
+// src/data-file/constants.ts
+export const DATA_MAGIC = 'DATA';
+export const DATA_VERSION = 1;
+export const DATA_HEADER_SIZE = 64;
+export const RECORD_HEADER_SIZE = 8;

lib/dat/index.ts

@@ -0,0 +1,7 @@
+// src/data-file/index.ts
+export { DataWriter } from './writer.js';
+export { DataReader } from './reader.js';
+export { DataProtocol } from './protocol.js';
+export * from './types.js';
+export * from './constants.js';
+export * from './serializers.js';

lib/dat/protocol.ts

@@ -0,0 +1,80 @@
+// src/data-file/protocol.ts
+import { DATA_MAGIC, DATA_VERSION, DATA_HEADER_SIZE, RECORD_HEADER_SIZE } from './constants.js';
+import { crc32 } from '../idx/index.js';
+import type { Serializer } from './types.js';
+
+export interface DataHeader {
+  magic: string;
+  version: number;
+  createdAt: bigint;
+  fileSize: bigint;
+  recordCount: number;
+  reserved: Buffer;
+}
+
+export class DataProtocol {
+  static createHeader(): Buffer {
+    const buf = Buffer.alloc(DATA_HEADER_SIZE);
+    buf.write(DATA_MAGIC, 0, 4, 'ascii');
+    buf.writeUInt32LE(DATA_VERSION, 4);
+    buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
+    buf.writeBigUInt64LE(BigInt(DATA_HEADER_SIZE), 16);
+    buf.writeUInt32LE(0, 24);
+    return buf;
+  }
+
+  static readHeader(buf: Buffer): DataHeader {
+    return {
+      magic: buf.toString('ascii', 0, 4),
+      version: buf.readUInt32LE(4),
+      createdAt: buf.readBigUInt64LE(8),
+      fileSize: buf.readBigUInt64LE(16),
+      recordCount: buf.readUInt32LE(24),
+      reserved: buf.subarray(28, 64),
+    };
+  }
+
+  static updateHeader(buf: Buffer, fileSize: bigint, recordCount: number): void {
+    buf.writeBigUInt64LE(fileSize, 16);
+    buf.writeUInt32LE(recordCount, 24);
+  }
+
+  static serializeRecord<T>(data: T, serializer: Serializer<T>): Buffer {
+    const dataBytes = serializer.serialize(data);
+    const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
+
+    const buf = Buffer.alloc(totalLen);
+
+    dataBytes.copy(buf, RECORD_HEADER_SIZE);
+
+    buf.writeUInt32LE(dataBytes.length, 0);
+    const checksum = crc32(buf, RECORD_HEADER_SIZE, totalLen);
+    buf.writeUInt32LE(checksum, 4);
+
+    return buf;
+  }
+
+  static deserializeRecord<T>(
+    buf: Buffer,
+    offset: number,
+    serializer: Serializer<T>
+  ): { data: T; length: number } | null {
+    if (offset + RECORD_HEADER_SIZE > buf.length) return null;
+
+    const dataLen = buf.readUInt32LE(offset);
+    const storedChecksum = buf.readUInt32LE(offset + 4);
+
+    const totalLen = RECORD_HEADER_SIZE + dataLen;
+    if (offset + totalLen > buf.length) return null;
+
+    const calcChecksum = crc32(buf, offset + RECORD_HEADER_SIZE, offset + totalLen);
+    if (calcChecksum !== storedChecksum) {
+      throw new Error(`Checksum mismatch at offset ${offset}`);
+    }
+
+    const dataBytes = buf.subarray(offset + RECORD_HEADER_SIZE, offset + totalLen);
+    const data = serializer.deserialize(dataBytes);
+
+    return { data, length: totalLen };
+  }
+}

lib/dat/reader.ts

@@ -0,0 +1,213 @@
+// src/data-file/reader.ts
+import * as fs from 'node:fs';
+import mmap from '@elilee/mmap-native';
+import { DATA_HEADER_SIZE } from './constants.js';
+import { DataProtocol, DataHeader } from './protocol.js';
+import { IndexReader } from '../idx/index.js';
+import type { Serializer, DataEntry } from './types.js';
+
+export class DataReader<T> {
+  private fd: number | null = null;
+  private buffer: Buffer | null = null;
+  private header: DataHeader | null = null;
+
+  private indexReader: IndexReader;
+  private serializer: Serializer<T>;
+
+  readonly dataPath: string;
+  readonly indexPath: string;
+
+  constructor(basePath: string, serializer: Serializer<T>) {
+    this.dataPath = `${basePath}.dat`;
+    this.indexPath = `${basePath}.idx`;
+    this.serializer = serializer;
+    this.indexReader = new IndexReader(this.indexPath);
+  }
+
+  open(): void {
+    const stats = fs.statSync(this.dataPath);
+    this.fd = fs.openSync(this.dataPath, 'r');
+
+    this.buffer = mmap.map(
+      stats.size,
+      mmap.PROT_READ,
+      mmap.MAP_SHARED,
+      this.fd,
+      0
+    );
+
+    this.header = DataProtocol.readHeader(this.buffer);
+    this.indexReader.open();
+  }
+
+  getHeader(): DataHeader {
+    if (!this.header) throw new Error('Data file not opened');
+    return this.header;
+  }
+
+  getBySequence(sequence: number): DataEntry<T> | null {
+    if (!this.buffer) throw new Error('Data file not opened');
+
+    const found = this.indexReader.binarySearchBySequence(sequence);
+    if (!found) return null;
+
+    const result = DataProtocol.deserializeRecord(
+      this.buffer,
+      Number(found.entry.offset),
+      this.serializer
+    );
+    if (!result) return null;
+
+    return {
+      sequence: found.entry.sequence,
+      timestamp: found.entry.timestamp,
+      data: result.data,
+    };
+  }
+
+  getByIndex(index: number): DataEntry<T> | null {
+    if (!this.buffer) throw new Error('Data file not opened');
+
+    const entry = this.indexReader.getEntry(index);
+    if (!entry) return null;
+
+    const result = DataProtocol.deserializeRecord(
+      this.buffer,
+      Number(entry.offset),
+      this.serializer
+    );
+    if (!result) return null;
+
+    return {
+      sequence: entry.sequence,
+      timestamp: entry.timestamp,
+      data: result.data,
+    };
+  }
+
+  getBulkData(startSeq: number, endSeq: number): DataEntry<T>[] {
+    if (!this.buffer) throw new Error('Data file not opened');
+
+    const results: DataEntry<T>[] = [];
+    const indexHeader = this.indexReader.getHeader();
+    let startIdx = this.findStartIndex(startSeq, indexHeader.validCount);
+
+    for (let i = startIdx; i < indexHeader.validCount; i++) {
+      const entry = this.indexReader.getEntry(i);
+      if (!entry) continue;
+
+      if (entry.sequence > endSeq) break;
+
+      if (entry.sequence >= startSeq) {
+        const result = DataProtocol.deserializeRecord(
+          this.buffer,
+          Number(entry.offset),
+          this.serializer
+        );
+        if (result) {
+          results.push({
+            sequence: entry.sequence,
+            timestamp: entry.timestamp,
+            data: result.data,
+          });
+        }
+      }
+    }
+
+    return results;
+  }
+
+  private findStartIndex(targetSeq: number, validCount: number): number {
+    let left = 0;
+    let right = validCount - 1;
+    let result = 0;
+
+    while (left <= right) {
+      const mid = Math.floor((left + right) / 2);
+      const entry = this.indexReader.getEntry(mid);
+
+      if (!entry) {
+        right = mid - 1;
+        continue;
+      }
+
+      if (entry.sequence >= targetSeq) {
+        result = mid;
+        right = mid - 1;
+      } else {
+        left = mid + 1;
+      }
+    }
+
+    return result;
+  }
+
+  getBulkDataByTime(startTs: bigint, endTs: bigint): DataEntry<T>[] {
+    if (!this.buffer) throw new Error('Data file not opened');
+
+    const indexResults = this.indexReader.findByTimeRange(startTs, endTs);
+    const results: DataEntry<T>[] = [];
+
+    for (const { entry } of indexResults) {
+      const result = DataProtocol.deserializeRecord(
+        this.buffer,
+        Number(entry.offset),
+        this.serializer
+      );
+      if (result) {
+        results.push({
+          sequence: entry.sequence,
+          timestamp: entry.timestamp,
+          data: result.data,
+        });
+      }
+    }
+
+    return results;
+  }
+
+  getAllData(): DataEntry<T>[] {
+    if (!this.buffer) throw new Error('Data file not opened');
+
+    const entries = this.indexReader.getAllEntries();
+    const results: DataEntry<T>[] = [];
+
+    for (const entry of entries) {
+      const result = DataProtocol.deserializeRecord(
+        this.buffer,
+        Number(entry.offset),
+        this.serializer
+      );
+      if (result) {
+        results.push({
+          sequence: entry.sequence,
+          timestamp: entry.timestamp,
+          data: result.data,
+        });
+      }
+    }
+
+    return results;
+  }
+
+  getRecordCount(): number {
+    return this.indexReader.getHeader().validCount;
+  }
+
+  getLastSequence(): number {
+    return this.indexReader.getHeader().lastSequence;
+  }
+
+  close(): void {
+    if (this.buffer) {
+      mmap.unmap(this.buffer);
+      this.buffer = null;
+    }
+    if (this.fd !== null) {
+      fs.closeSync(this.fd);
+      this.fd = null;
+    }
+    this.header = null;
+    this.indexReader.close();
+  }
+}

lib/dat/serializers.ts

@@ -0,0 +1,32 @@
+// src/data-file/serializers.ts
+import type { Serializer } from './types.js';
+
+export function jsonSerializer<T>(): Serializer<T> {
+  return {
+    serialize(data: T): Buffer {
+      return Buffer.from(JSON.stringify(data), 'utf8');
+    },
+    deserialize(buf: Buffer): T {
+      return JSON.parse(buf.toString('utf8'));
+    },
+  };
+}
+
+export function msgpackSerializer<T>(): Serializer<T> {
+  const { encode, decode } = require('@msgpack/msgpack');
+  return {
+    serialize(data: T): Buffer {
+      return Buffer.from(encode(data));
+    },
+    deserialize(buf: Buffer): T {
+      return decode(buf) as T;
+    },
+  };
+}
+
+export function createSerializer<T>(
+  serialize: (data: T) => Buffer,
+  deserialize: (buf: Buffer) => T
+): Serializer<T> {
+  return { serialize, deserialize };
+}

lib/dat/types.ts

@@ -0,0 +1,16 @@
+// src/data-file/types.ts
+export interface Serializer<T> {
+  serialize(data: T): Buffer;
+  deserialize(buf: Buffer): T;
+}
+
+export interface DataEntry<T> {
+  sequence: number;
+  timestamp: bigint;
+  data: T;
+}
+
+export interface DataFileOptions<T> {
+  serializer: Serializer<T>;
+  maxEntries?: number;
+}

lib/dat/writer.ts

@@ -0,0 +1,120 @@
+// src/data-file/writer.ts
+import * as fs from 'node:fs';
+import { DATA_HEADER_SIZE } from './constants.js';
+import { DataProtocol } from './protocol.js';
+import { IndexWriter } from '../idx/index.js';
+import type { Serializer, DataFileOptions } from './types.js';
+
+export class DataWriter<T> {
+  private fd: number | null = null;
+  private headerBuf: Buffer | null = null;
+  private currentOffset: bigint = BigInt(DATA_HEADER_SIZE);
+  private recordCount = 0;
+
+  private indexWriter: IndexWriter;
+  private serializer: Serializer<T>;
+
+  readonly dataPath: string;
+  readonly indexPath: string;
+
+  constructor(basePath: string, options: DataFileOptions<T>) {
+    this.dataPath = `${basePath}.dat`;
+    this.indexPath = `${basePath}.idx`;
+    this.serializer = options.serializer;
+
+    const maxEntries = options.maxEntries ?? 10_000_000;
+    this.indexWriter = new IndexWriter(this.indexPath, { maxEntries });
+  }
+
+  open(): void {
+    const isNew = !fs.existsSync(this.dataPath);
+
+    this.fd = fs.openSync(this.dataPath, isNew ? 'w+' : 'r+');
+    this.headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
+
+    if (isNew) {
+      const header = DataProtocol.createHeader();
+      fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, 0);
+      this.currentOffset = BigInt(DATA_HEADER_SIZE);
+      this.recordCount = 0;
+    } else {
+      fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
+      const header = DataProtocol.readHeader(this.headerBuf);
+      this.currentOffset = header.fileSize;
+      this.recordCount = header.recordCount;
+    }
+
+    this.indexWriter.open();
+  }
+
+  append(data: T, timestamp?: bigint): number {
+    if (this.fd === null) throw new Error('Data file not opened');
+
+    const buf = DataProtocol.serializeRecord(data, this.serializer);
+    const offset = this.currentOffset;
+
+    fs.writeSync(this.fd, buf, 0, buf.length, Number(offset));
+
+    const sequence = this.indexWriter.getNextSequence();
+    const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
+
+    this.indexWriter.append(offset, buf.length, ts);
+
+    this.currentOffset += BigInt(buf.length);
+    this.recordCount++;
+
+    return sequence;
+  }
+
+  appendBulk(records: T[], timestamp?: bigint): number[] {
+    const sequences: number[] = [];
+    const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
+
+    for (const record of records) {
+      const seq = this.append(record, ts);
+      sequences.push(seq);
+    }
+
+    return sequences;
+  }
+
+  getLastSequence(): number {
+    return this.indexWriter.getLastSequence();
+  }
+
+  getNextSequence(): number {
+    return this.indexWriter.getNextSequence();
+  }
+
+  sync(): void {
+    if (this.fd === null || !this.headerBuf) return;
+
+    DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
+    fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
+    fs.fsyncSync(this.fd);
+
+    this.indexWriter.syncAll();
+  }
+
+  close(): void {
+    this.sync();
+
+    if (this.fd !== null) {
+      fs.closeSync(this.fd);
+      this.fd = null;
+    }
+
+    this.indexWriter.close();
+    this.headerBuf = null;
+  }
+
+  getStats() {
+    return {
+      dataPath: this.dataPath,
+      indexPath: this.indexPath,
+      currentOffset: this.currentOffset,
+      recordCount: this.recordCount,
+      lastSequence: this.indexWriter.getLastSequence(),
+    };
+  }
+}

lib/idx/constants.ts

@@ -0,0 +1,8 @@
+// src/index-file/constants.ts
+export const INDEX_MAGIC = 'INDX';
+export const INDEX_VERSION = 1;
+export const INDEX_HEADER_SIZE = 64;
+export const INDEX_ENTRY_SIZE = 32;
+
+export const FLAG_VALID = 0x0001;
+export const FLAG_DELETED = 0x0002;

lib/idx/index.ts

@@ -0,0 +1,6 @@
+// src/index-file/index.ts
+export { IndexWriter } from './writer.js';
+export { IndexReader } from './reader.js';
+export { IndexProtocol, crc32 } from './protocol.js';
+export * from './types.js';
+export * from './constants.js';

lib/idx/protocol.ts

@@ -0,0 +1,106 @@
+// src/index-file/protocol.ts
+import {
+  INDEX_MAGIC,
+  INDEX_VERSION,
+  INDEX_HEADER_SIZE,
+  INDEX_ENTRY_SIZE,
+  FLAG_VALID,
+} from './constants.js';
+import type { IndexHeader, IndexEntry } from './types.js';
+
+const CRC_TABLE = new Uint32Array(256);
+for (let i = 0; i < 256; i++) {
+  let c = i;
+  for (let j = 0; j < 8; j++) {
+    c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
+  }
+  CRC_TABLE[i] = c >>> 0;
+}
+
+export function crc32(buf: Buffer, start = 0, end?: number): number {
+  let crc = 0xFFFFFFFF;
+  const len = end ?? buf.length;
+  for (let i = start; i < len; i++) {
+    crc = CRC_TABLE[(crc ^ buf[i]) & 0xFF] ^ (crc >>> 8);
+  }
+  return (~crc) >>> 0;
+}
+
+export class IndexProtocol {
+  static createHeader(entryCount: number, magic = INDEX_MAGIC): Buffer {
+    const buf = Buffer.alloc(INDEX_HEADER_SIZE);
+    buf.write(magic, 0, 4, 'ascii');
+    buf.writeUInt32LE(INDEX_VERSION, 4);
+    buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
+    buf.writeUInt32LE(INDEX_ENTRY_SIZE, 16);
+    buf.writeUInt32LE(entryCount, 20);
+    buf.writeUInt32LE(0, 24);
+    buf.writeBigUInt64LE(0n, 28);
+    buf.writeUInt32LE(0, 36);
+    return buf;
+  }
+
+  static readHeader(buf: Buffer): IndexHeader {
+    return {
+      magic: buf.toString('ascii', 0, 4),
+      version: buf.readUInt32LE(4),
+      createdAt: buf.readBigUInt64LE(8),
+      entrySize: buf.readUInt32LE(16),
+      entryCount: buf.readUInt32LE(20),
+      validCount: buf.readUInt32LE(24),
+      dataFileSize: buf.readBigUInt64LE(28),
+      lastSequence: buf.readUInt32LE(36),
+      reserved: buf.subarray(40, 64),
+    };
+  }
+
+  static updateHeaderCounts(
+    buf: Buffer,
+    validCount: number,
+    dataFileSize: bigint,
+    lastSequence: number
+  ): void {
+    buf.writeUInt32LE(validCount, 24);
+    buf.writeBigUInt64LE(dataFileSize, 28);
+    buf.writeUInt32LE(lastSequence, 36);
+  }
+
+  static writeEntry(buf: Buffer, index: number, entry: Omit<IndexEntry, 'checksum'>): void {
+    const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
+
+    buf.writeUInt32LE(entry.sequence, off);
+    buf.writeBigUInt64LE(entry.timestamp, off + 4);
+    buf.writeBigUInt64LE(entry.offset, off + 12);
+    buf.writeUInt32LE(entry.length, off + 20);
+    buf.writeUInt32LE(entry.flags | FLAG_VALID, off + 24);
+
+    const checksum = crc32(buf, off, off + 28);
+    buf.writeUInt32LE(checksum, off + 28);
+  }
+
+  static readEntry(buf: Buffer, index: number): IndexEntry | null {
+    const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
+    const flags = buf.readUInt32LE(off + 24);
+
+    if (!(flags & FLAG_VALID)) return null;
+
+    return {
+      sequence: buf.readUInt32LE(off),
+      timestamp: buf.readBigUInt64LE(off + 4),
+      offset: buf.readBigUInt64LE(off + 12),
+      length: buf.readUInt32LE(off + 20),
+      flags,
+      checksum: buf.readUInt32LE(off + 28),
+    };
+  }
+
+  static isValidEntry(buf: Buffer, index: number): boolean {
+    const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
+    const flags = buf.readUInt32LE(off + 24);
+    return (flags & FLAG_VALID) !== 0;
+  }
+
+  static calcFileSize(entryCount: number): number {
+    return INDEX_HEADER_SIZE + INDEX_ENTRY_SIZE * entryCount;
+  }
+}

lib/idx/reader.ts

@@ -0,0 +1,131 @@
+// src/index-file/reader.ts
+import * as fs from 'node:fs';
+import mmap from '@elilee/mmap-native';
+import { IndexProtocol } from './protocol.js';
+import type { IndexHeader, IndexEntry } from './types.js';
+
+export class IndexReader {
+  private fd: number | null = null;
+  private buffer: Buffer | null = null;
+  private header: IndexHeader | null = null;
+
+  readonly path: string;
+
+  constructor(path: string) {
+    this.path = path;
+  }
+
+  open(): void {
+    const stats = fs.statSync(this.path);
+    this.fd = fs.openSync(this.path, 'r');
+
+    this.buffer = mmap.map(
+      stats.size,
+      mmap.PROT_READ,
+      mmap.MAP_SHARED,
+      this.fd,
+      0
+    );
+
+    this.header = IndexProtocol.readHeader(this.buffer);
+  }
+
+  getHeader(): IndexHeader {
+    if (!this.header) throw new Error('Index file not opened');
+    return this.header;
+  }
+
+  getEntry(index: number): IndexEntry | null {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+    if (index < 0 || index >= this.header.entryCount) return null;
+    return IndexProtocol.readEntry(this.buffer, index);
+  }
+
+  findBySequence(sequence: number): { index: number; entry: IndexEntry } | null {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    for (let i = 0; i < this.header.validCount; i++) {
+      const entry = IndexProtocol.readEntry(this.buffer, i);
+      if (entry && entry.sequence === sequence) {
+        return { index: i, entry };
+      }
+    }
+    return null;
+  }
+
+  findBySequenceRange(startSeq: number, endSeq: number): { index: number; entry: IndexEntry }[] {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    const results: { index: number; entry: IndexEntry }[] = [];
+    for (let i = 0; i < this.header.validCount; i++) {
+      const entry = IndexProtocol.readEntry(this.buffer, i);
+      if (entry && entry.sequence >= startSeq && entry.sequence <= endSeq) {
+        results.push({ index: i, entry });
+      }
+    }
+    return results;
+  }
+
+  getAllEntries(): IndexEntry[] {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    const entries: IndexEntry[] = [];
+    for (let i = 0; i < this.header.validCount; i++) {
+      const entry = IndexProtocol.readEntry(this.buffer, i);
+      if (entry) entries.push(entry);
+    }
+    return entries;
+  }
+
+  findByTimeRange(startTs: bigint, endTs: bigint): { index: number; entry: IndexEntry }[] {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    const results: { index: number; entry: IndexEntry }[] = [];
+    for (let i = 0; i < this.header.validCount; i++) {
+      const entry = IndexProtocol.readEntry(this.buffer, i);
+      if (entry && entry.timestamp >= startTs && entry.timestamp <= endTs) {
+        results.push({ index: i, entry });
+      }
+    }
+    return results;
+  }
+
+  binarySearchBySequence(targetSeq: number): { index: number; entry: IndexEntry } | null {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    let left = 0;
+    let right = this.header.validCount - 1;
+
+    while (left <= right) {
+      const mid = Math.floor((left + right) / 2);
+      const entry = IndexProtocol.readEntry(this.buffer, mid);
+
+      if (!entry) {
+        right = mid - 1;
+        continue;
+      }
+
+      if (entry.sequence === targetSeq) {
+        return { index: mid, entry };
+      } else if (entry.sequence < targetSeq) {
+        left = mid + 1;
+      } else {
+        right = mid - 1;
+      }
+    }
+
+    return null;
+  }
+
+  close(): void {
+    if (this.buffer) {
+      mmap.unmap(this.buffer);
+      this.buffer = null;
+    }
+    if (this.fd !== null) {
+      fs.closeSync(this.fd);
+      this.fd = null;
+    }
+    this.header = null;
+  }
+}

lib/idx/types.ts

@@ -0,0 +1,26 @@
+// src/index-file/types.ts
+export interface IndexHeader {
+  magic: string;
+  version: number;
+  createdAt: bigint;
+  entrySize: number;
+  entryCount: number;
+  validCount: number;
+  dataFileSize: bigint;
+  lastSequence: number;
+  reserved: Buffer;
+}
+
+export interface IndexEntry {
+  sequence: number;
+  timestamp: bigint;
+  offset: bigint;
+  length: number;
+  flags: number;
+  checksum: number;
+}
+
+export interface IndexFileOptions {
+  maxEntries: number;
+  magic?: string;
+}

lib/idx/writer.ts

@@ -0,0 +1,141 @@
+// src/index-file/writer.ts
+import * as fs from 'node:fs';
+import mmap from '@elilee/mmap-native';
+import { INDEX_HEADER_SIZE, FLAG_VALID } from './constants.js';
+import { IndexProtocol } from './protocol.js';
+import type { IndexFileOptions } from './types.js';
+
+export class IndexWriter {
+  private fd: number | null = null;
+  private buffer: Buffer | null = null;
+  private validCount = 0;
+  private dataFileSize = 0n;
+  private lastSequence = 0;
+
+  readonly path: string;
+  readonly maxEntries: number;
+  readonly fileSize: number;
+
+  constructor(path: string, options: IndexFileOptions) {
+    this.path = path;
+    this.maxEntries = options.maxEntries;
+    this.fileSize = IndexProtocol.calcFileSize(options.maxEntries);
+  }
+
+  open(): void {
+    const isNew = !fs.existsSync(this.path);
+
+    this.fd = fs.openSync(this.path, isNew ? 'w+' : 'r+');
+
+    if (isNew) {
+      fs.ftruncateSync(this.fd, this.fileSize);
+    }
+
+    this.buffer = mmap.map(
+      this.fileSize,
+      mmap.PROT_READ | mmap.PROT_WRITE,
+      mmap.MAP_SHARED,
+      this.fd,
+      0
+    );
+
+    if (isNew) {
+      const header = IndexProtocol.createHeader(this.maxEntries);
+      header.copy(this.buffer, 0);
+      this.syncHeader();
+    } else {
+      const header = IndexProtocol.readHeader(this.buffer);
+      this.validCount = header.validCount;
+      this.dataFileSize = header.dataFileSize;
+      this.lastSequence = header.lastSequence;
+    }
+  }
+
+  write(
+    index: number,
+    sequence: number,
+    offset: bigint,
+    length: number,
+    timestamp?: bigint
+  ): boolean {
+    if (!this.buffer) throw new Error('Index file not opened');
+    if (index < 0 || index >= this.maxEntries) return false;
+
+    const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
+
+    IndexProtocol.writeEntry(this.buffer, index, {
+      sequence,
+      timestamp: ts,
+      offset,
+      length,
+      flags: FLAG_VALID,
+    });
+
+    this.validCount++;
+    if (sequence > this.lastSequence) {
+      this.lastSequence = sequence;
+    }
+
+    const newDataEnd = offset + BigInt(length);
+    if (newDataEnd > this.dataFileSize) {
+      this.dataFileSize = newDataEnd;
+    }
+
+    return true;
+  }
+
+  append(offset: bigint, length: number, timestamp?: bigint): number {
+    const index = this.validCount;
+    if (index >= this.maxEntries) return -1;
+
+    const sequence = this.lastSequence + 1;
+    this.write(index, sequence, offset, length, timestamp);
+    return index;
+  }
+
+  getLastSequence(): number {
+    return this.lastSequence;
+  }
+
+  getNextSequence(): number {
+    return this.lastSequence + 1;
+  }
+
+  syncHeader(): void {
+    if (!this.buffer) return;
+    IndexProtocol.updateHeaderCounts(
+      this.buffer,
+      this.validCount,
+      this.dataFileSize,
+      this.lastSequence
+    );
+    mmap.sync(this.buffer, 0, INDEX_HEADER_SIZE, mmap.MS_ASYNC);
+  }
+
+  syncAll(): void {
+    if (!this.buffer) return;
+    this.syncHeader();
+    mmap.sync(this.buffer, 0, this.fileSize, mmap.MS_SYNC);
+  }
+
+  close(): void {
+    if (!this.buffer || this.fd === null) return;
+
+    this.syncAll();
+    mmap.unmap(this.buffer);
+    fs.closeSync(this.fd);
+
+    this.buffer = null;
+    this.fd = null;
+  }
+
+  getStats() {
+    return {
+      path: this.path,
+      maxEntries: this.maxEntries,
+      validCount: this.validCount,
+      dataFileSize: this.dataFileSize,
+      lastSequence: this.lastSequence,
+    };
+  }
+}