Init indexed file

2026-01-27 02:30:42 +00:00
commit 1d13e9fc3e
22 changed files with 2812 additions and 0 deletions
--- a/lib/idx/constants.ts
+++ b/lib/idx/constants.ts
@@ -0,0 +1,8 @@
+// src/index-file/constants.ts
+export const INDEX_MAGIC = 'INDX';
+export const INDEX_VERSION = 1;
+export const INDEX_HEADER_SIZE = 64;
+export const INDEX_ENTRY_SIZE = 32;
+
+export const FLAG_VALID = 0x0001;
+export const FLAG_DELETED = 0x0002;
--- a/lib/idx/index.ts
+++ b/lib/idx/index.ts
@@ -0,0 +1,6 @@
+// src/index-file/index.ts
+export { IndexWriter } from './writer.js';
+export { IndexReader } from './reader.js';
+export { IndexProtocol, crc32 } from './protocol.js';
+export * from './types.js';
+export * from './constants.js';
--- a/lib/idx/protocol.ts
+++ b/lib/idx/protocol.ts
@@ -0,0 +1,106 @@
+// src/index-file/protocol.ts
+import {
+  INDEX_MAGIC,
+  INDEX_VERSION,
+  INDEX_HEADER_SIZE,
+  INDEX_ENTRY_SIZE,
+  FLAG_VALID,
+} from './constants.js';
+import type { IndexHeader, IndexEntry } from './types.js';
+
+const CRC_TABLE = new Uint32Array(256);
+for (let i = 0; i < 256; i++) {
+  let c = i;
+  for (let j = 0; j < 8; j++) {
+    c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
+  }
+  CRC_TABLE[i] = c >>> 0;
+}
+
+export function crc32(buf: Buffer, start = 0, end?: number): number {
+  let crc = 0xFFFFFFFF;
+  const len = end ?? buf.length;
+  for (let i = start; i < len; i++) {
+    crc = CRC_TABLE[(crc ^ buf[i]) & 0xFF] ^ (crc >>> 8);
+  }
+  return (~crc) >>> 0;
+}
+
+export class IndexProtocol {
+  static createHeader(entryCount: number, magic = INDEX_MAGIC): Buffer {
+    const buf = Buffer.alloc(INDEX_HEADER_SIZE);
+    buf.write(magic, 0, 4, 'ascii');
+    buf.writeUInt32LE(INDEX_VERSION, 4);
+    buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
+    buf.writeUInt32LE(INDEX_ENTRY_SIZE, 16);
+    buf.writeUInt32LE(entryCount, 20);
+    buf.writeUInt32LE(0, 24);
+    buf.writeBigUInt64LE(0n, 28);
+    buf.writeUInt32LE(0, 36);
+    return buf;
+  }
+
+  static readHeader(buf: Buffer): IndexHeader {
+    return {
+      magic: buf.toString('ascii', 0, 4),
+      version: buf.readUInt32LE(4),
+      createdAt: buf.readBigUInt64LE(8),
+      entrySize: buf.readUInt32LE(16),
+      entryCount: buf.readUInt32LE(20),
+      validCount: buf.readUInt32LE(24),
+      dataFileSize: buf.readBigUInt64LE(28),
+      lastSequence: buf.readUInt32LE(36),
+      reserved: buf.subarray(40, 64),
+    };
+  }
+
+  static updateHeaderCounts(
+    buf: Buffer,
+    validCount: number,
+    dataFileSize: bigint,
+    lastSequence: number
+  ): void {
+    buf.writeUInt32LE(validCount, 24);
+    buf.writeBigUInt64LE(dataFileSize, 28);
+    buf.writeUInt32LE(lastSequence, 36);
+  }
+
+  static writeEntry(buf: Buffer, index: number, entry: Omit<IndexEntry, 'checksum'>): void {
+    const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
+
+    buf.writeUInt32LE(entry.sequence, off);
+    buf.writeBigUInt64LE(entry.timestamp, off + 4);
+    buf.writeBigUInt64LE(entry.offset, off + 12);
+    buf.writeUInt32LE(entry.length, off + 20);
+    buf.writeUInt32LE(entry.flags | FLAG_VALID, off + 24);
+
+    const checksum = crc32(buf, off, off + 28);
+    buf.writeUInt32LE(checksum, off + 28);
+  }
+
+  static readEntry(buf: Buffer, index: number): IndexEntry | null {
+    const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
+    const flags = buf.readUInt32LE(off + 24);
+
+    if (!(flags & FLAG_VALID)) return null;
+
+    return {
+      sequence: buf.readUInt32LE(off),
+      timestamp: buf.readBigUInt64LE(off + 4),
+      offset: buf.readBigUInt64LE(off + 12),
+      length: buf.readUInt32LE(off + 20),
+      flags,
+      checksum: buf.readUInt32LE(off + 28),
+    };
+  }
+
+  static isValidEntry(buf: Buffer, index: number): boolean {
+    const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
+    const flags = buf.readUInt32LE(off + 24);
+    return (flags & FLAG_VALID) !== 0;
+  }
+
+  static calcFileSize(entryCount: number): number {
+    return INDEX_HEADER_SIZE + INDEX_ENTRY_SIZE * entryCount;
+  }
+}
--- a/lib/idx/reader.ts
+++ b/lib/idx/reader.ts
@@ -0,0 +1,131 @@
+// src/index-file/reader.ts
+import * as fs from 'node:fs';
+import mmap from '@elilee/mmap-native';
+import { IndexProtocol } from './protocol.js';
+import type { IndexHeader, IndexEntry } from './types.js';
+
+export class IndexReader {
+  private fd: number | null = null;
+  private buffer: Buffer | null = null;
+  private header: IndexHeader | null = null;
+
+  readonly path: string;
+
+  constructor(path: string) {
+    this.path = path;
+  }
+
+  open(): void {
+    const stats = fs.statSync(this.path);
+    this.fd = fs.openSync(this.path, 'r');
+
+    this.buffer = mmap.map(
+      stats.size,
+      mmap.PROT_READ,
+      mmap.MAP_SHARED,
+      this.fd,
+      0
+    );
+
+    this.header = IndexProtocol.readHeader(this.buffer);
+  }
+
+  getHeader(): IndexHeader {
+    if (!this.header) throw new Error('Index file not opened');
+    return this.header;
+  }
+
+  getEntry(index: number): IndexEntry | null {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+    if (index < 0 || index >= this.header.entryCount) return null;
+    return IndexProtocol.readEntry(this.buffer, index);
+  }
+
+  findBySequence(sequence: number): { index: number; entry: IndexEntry } | null {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    for (let i = 0; i < this.header.validCount; i++) {
+      const entry = IndexProtocol.readEntry(this.buffer, i);
+      if (entry && entry.sequence === sequence) {
+        return { index: i, entry };
+      }
+    }
+    return null;
+  }
+
+  findBySequenceRange(startSeq: number, endSeq: number): { index: number; entry: IndexEntry }[] {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    const results: { index: number; entry: IndexEntry }[] = [];
+    for (let i = 0; i < this.header.validCount; i++) {
+      const entry = IndexProtocol.readEntry(this.buffer, i);
+      if (entry && entry.sequence >= startSeq && entry.sequence <= endSeq) {
+        results.push({ index: i, entry });
+      }
+    }
+    return results;
+  }
+
+  getAllEntries(): IndexEntry[] {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    const entries: IndexEntry[] = [];
+    for (let i = 0; i < this.header.validCount; i++) {
+      const entry = IndexProtocol.readEntry(this.buffer, i);
+      if (entry) entries.push(entry);
+    }
+    return entries;
+  }
+
+  findByTimeRange(startTs: bigint, endTs: bigint): { index: number; entry: IndexEntry }[] {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    const results: { index: number; entry: IndexEntry }[] = [];
+    for (let i = 0; i < this.header.validCount; i++) {
+      const entry = IndexProtocol.readEntry(this.buffer, i);
+      if (entry && entry.timestamp >= startTs && entry.timestamp <= endTs) {
+        results.push({ index: i, entry });
+      }
+    }
+    return results;
+  }
+
+  binarySearchBySequence(targetSeq: number): { index: number; entry: IndexEntry } | null {
+    if (!this.buffer || !this.header) throw new Error('Index file not opened');
+
+    let left = 0;
+    let right = this.header.validCount - 1;
+
+    while (left <= right) {
+      const mid = Math.floor((left + right) / 2);
+      const entry = IndexProtocol.readEntry(this.buffer, mid);
+
+      if (!entry) {
+        right = mid - 1;
+        continue;
+      }
+
+      if (entry.sequence === targetSeq) {
+        return { index: mid, entry };
+      } else if (entry.sequence < targetSeq) {
+        left = mid + 1;
+      } else {
+        right = mid - 1;
+      }
+    }
+
+    return null;
+  }
+
+  close(): void {
+    if (this.buffer) {
+      mmap.unmap(this.buffer);
+      this.buffer = null;
+    }
+    if (this.fd !== null) {
+      fs.closeSync(this.fd);
+      this.fd = null;
+    }
+    this.header = null;
+  }
+}
--- a/lib/idx/types.ts
+++ b/lib/idx/types.ts
@@ -0,0 +1,26 @@
+// src/index-file/types.ts
+export interface IndexHeader {
+  magic: string;
+  version: number;
+  createdAt: bigint;
+  entrySize: number;
+  entryCount: number;
+  validCount: number;
+  dataFileSize: bigint;
+  lastSequence: number;
+  reserved: Buffer;
+}
+
+export interface IndexEntry {
+  sequence: number;
+  timestamp: bigint;
+  offset: bigint;
+  length: number;
+  flags: number;
+  checksum: number;
+}
+
+export interface IndexFileOptions {
+  maxEntries: number;
+  magic?: string;
+}
--- a/lib/idx/writer.ts
+++ b/lib/idx/writer.ts
@@ -0,0 +1,141 @@
+// src/index-file/writer.ts
+import * as fs from 'node:fs';
+import mmap from '@elilee/mmap-native';
+import { INDEX_HEADER_SIZE, FLAG_VALID } from './constants.js';
+import { IndexProtocol } from './protocol.js';
+import type { IndexFileOptions } from './types.js';
+
+export class IndexWriter {
+  private fd: number | null = null;
+  private buffer: Buffer | null = null;
+  private validCount = 0;
+  private dataFileSize = 0n;
+  private lastSequence = 0;
+
+  readonly path: string;
+  readonly maxEntries: number;
+  readonly fileSize: number;
+
+  constructor(path: string, options: IndexFileOptions) {
+    this.path = path;
+    this.maxEntries = options.maxEntries;
+    this.fileSize = IndexProtocol.calcFileSize(options.maxEntries);
+  }
+
+  open(): void {
+    const isNew = !fs.existsSync(this.path);
+
+    this.fd = fs.openSync(this.path, isNew ? 'w+' : 'r+');
+
+    if (isNew) {
+      fs.ftruncateSync(this.fd, this.fileSize);
+    }
+
+    this.buffer = mmap.map(
+      this.fileSize,
+      mmap.PROT_READ | mmap.PROT_WRITE,
+      mmap.MAP_SHARED,
+      this.fd,
+      0
+    );
+
+    if (isNew) {
+      const header = IndexProtocol.createHeader(this.maxEntries);
+      header.copy(this.buffer, 0);
+      this.syncHeader();
+    } else {
+      const header = IndexProtocol.readHeader(this.buffer);
+      this.validCount = header.validCount;
+      this.dataFileSize = header.dataFileSize;
+      this.lastSequence = header.lastSequence;
+    }
+  }
+
+  write(
+    index: number,
+    sequence: number,
+    offset: bigint,
+    length: number,
+    timestamp?: bigint
+  ): boolean {
+    if (!this.buffer) throw new Error('Index file not opened');
+    if (index < 0 || index >= this.maxEntries) return false;
+
+    const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
+
+    IndexProtocol.writeEntry(this.buffer, index, {
+      sequence,
+      timestamp: ts,
+      offset,
+      length,
+      flags: FLAG_VALID,
+    });
+
+    this.validCount++;
+    if (sequence > this.lastSequence) {
+      this.lastSequence = sequence;
+    }
+
+    const newDataEnd = offset + BigInt(length);
+    if (newDataEnd > this.dataFileSize) {
+      this.dataFileSize = newDataEnd;
+    }
+
+    return true;
+  }
+
+  append(offset: bigint, length: number, timestamp?: bigint): number {
+    const index = this.validCount;
+    if (index >= this.maxEntries) return -1;
+
+    const sequence = this.lastSequence + 1;
+    this.write(index, sequence, offset, length, timestamp);
+    return index;
+  }
+
+  getLastSequence(): number {
+    return this.lastSequence;
+  }
+
+  getNextSequence(): number {
+    return this.lastSequence + 1;
+  }
+
+  syncHeader(): void {
+    if (!this.buffer) return;
+    IndexProtocol.updateHeaderCounts(
+      this.buffer,
+      this.validCount,
+      this.dataFileSize,
+      this.lastSequence
+    );
+    mmap.sync(this.buffer, 0, INDEX_HEADER_SIZE, mmap.MS_ASYNC);
+  }
+
+  syncAll(): void {
+    if (!this.buffer) return;
+    this.syncHeader();
+    mmap.sync(this.buffer, 0, this.fileSize, mmap.MS_SYNC);
+  }
+
+  close(): void {
+    if (!this.buffer || this.fd === null) return;
+
+    this.syncAll();
+    mmap.unmap(this.buffer);
+    fs.closeSync(this.fd);
+
+    this.buffer = null;
+    this.fd = null;
+  }
+
+  getStats() {
+    return {
+      path: this.path,
+      maxEntries: this.maxEntries,
+      validCount: this.validCount,
+      dataFileSize: this.dataFileSize,
+      lastSequence: this.lastSequence,
+    };
+  }
+}