Init indexed file

This commit is contained in:
Eli-Class
2026-01-27 02:30:42 +00:00
commit 1d13e9fc3e
22 changed files with 2812 additions and 0 deletions

6
.gitignore vendored Normal file
View File

@@ -0,0 +1,6 @@
node_modules/
build/
dist/
*.log
*.node
.DS_Store

6
.npmignore Normal file
View File

@@ -0,0 +1,6 @@
node_modules/
build/
*.log
.DS_Store
test.ts
tsconfig.json

396
CUSTOM_SERIALIZER.md Normal file
View File

@@ -0,0 +1,396 @@
가변 길이 배열이 포함된 데이터의 커스텀 바이너리 직렬화 방법을 보여드릴게요.
```typescript
// 가변 배열 직렬화 예시
import { createSerializer, DataWriter, DataReader } from './src/data-file/index.js';
// ============================================
// 1. 단순 배열 (숫자 배열)
// ============================================
interface SensorReading {
sensorId: number;
values: number[]; // 가변 길이
}
const sensorSerializer = createSerializer<SensorReading>(
(data) => {
// 4(sensorId) + 4(배열길이) + 8*N(values)
const buf = Buffer.alloc(4 + 4 + data.values.length * 8);
let offset = 0;
buf.writeUInt32LE(data.sensorId, offset); offset += 4;
buf.writeUInt32LE(data.values.length, offset); offset += 4;
for (const v of data.values) {
buf.writeDoubleLE(v, offset); offset += 8;
}
return buf;
},
(buf) => {
let offset = 0;
const sensorId = buf.readUInt32LE(offset); offset += 4;
const len = buf.readUInt32LE(offset); offset += 4;
const values: number[] = [];
for (let i = 0; i < len; i++) {
values.push(buf.readDoubleLE(offset)); offset += 8;
}
return { sensorId, values };
}
);
// ============================================
// 2. 객체 배열
// ============================================
interface OrderItem {
sku: string;
qty: number;
price: number;
}
interface Order {
orderId: number;
items: OrderItem[]; // 가변 길이 객체 배열
total: number;
}
const orderSerializer = createSerializer<Order>(
(data) => {
// 각 item의 sku 길이를 먼저 계산
const skuBuffers = data.items.map(item => Buffer.from(item.sku, 'utf8'));
const itemsSize = skuBuffers.reduce(
(sum, skuBuf, i) => sum + 4 + skuBuf.length + 4 + 8, // skuLen + sku + qty + price
0
);
// 4(orderId) + 4(items길이) + itemsSize + 8(total)
const buf = Buffer.alloc(4 + 4 + itemsSize + 8);
let offset = 0;
buf.writeUInt32LE(data.orderId, offset); offset += 4;
buf.writeUInt32LE(data.items.length, offset); offset += 4;
for (let i = 0; i < data.items.length; i++) {
const item = data.items[i];
const skuBuf = skuBuffers[i];
buf.writeUInt32LE(skuBuf.length, offset); offset += 4;
skuBuf.copy(buf, offset); offset += skuBuf.length;
buf.writeUInt32LE(item.qty, offset); offset += 4;
buf.writeDoubleLE(item.price, offset); offset += 8;
}
buf.writeDoubleLE(data.total, offset);
return buf;
},
(buf) => {
let offset = 0;
const orderId = buf.readUInt32LE(offset); offset += 4;
const itemsLen = buf.readUInt32LE(offset); offset += 4;
const items: OrderItem[] = [];
for (let i = 0; i < itemsLen; i++) {
const skuLen = buf.readUInt32LE(offset); offset += 4;
const sku = buf.toString('utf8', offset, offset + skuLen); offset += skuLen;
const qty = buf.readUInt32LE(offset); offset += 4;
const price = buf.readDoubleLE(offset); offset += 8;
items.push({ sku, qty, price });
}
const total = buf.readDoubleLE(offset);
return { orderId, items, total };
}
);
// ============================================
// 3. 다중 가변 배열
// ============================================
interface TimeSeries {
id: number;
timestamps: bigint[]; // 가변
values: number[]; // 가변
tags: string[]; // 가변
}
const timeSeriesSerializer = createSerializer<TimeSeries>(
(data) => {
const tagBuffers = data.tags.map(t => Buffer.from(t, 'utf8'));
const tagsSize = tagBuffers.reduce((sum, b) => sum + 4 + b.length, 0);
// 4(id) + 4(tsLen) + 8*N + 4(valLen) + 8*M + 4(tagLen) + tagsSize
const size = 4 + 4 + data.timestamps.length * 8 + 4 + data.values.length * 8 + 4 + tagsSize;
const buf = Buffer.alloc(size);
let offset = 0;
// id
buf.writeUInt32LE(data.id, offset); offset += 4;
// timestamps
buf.writeUInt32LE(data.timestamps.length, offset); offset += 4;
for (const ts of data.timestamps) {
buf.writeBigUInt64LE(ts, offset); offset += 8;
}
// values
buf.writeUInt32LE(data.values.length, offset); offset += 4;
for (const v of data.values) {
buf.writeDoubleLE(v, offset); offset += 8;
}
// tags
buf.writeUInt32LE(data.tags.length, offset); offset += 4;
for (const tagBuf of tagBuffers) {
buf.writeUInt32LE(tagBuf.length, offset); offset += 4;
tagBuf.copy(buf, offset); offset += tagBuf.length;
}
return buf;
},
(buf) => {
let offset = 0;
const id = buf.readUInt32LE(offset); offset += 4;
// timestamps
const tsLen = buf.readUInt32LE(offset); offset += 4;
const timestamps: bigint[] = [];
for (let i = 0; i < tsLen; i++) {
timestamps.push(buf.readBigUInt64LE(offset)); offset += 8;
}
// values
const valLen = buf.readUInt32LE(offset); offset += 4;
const values: number[] = [];
for (let i = 0; i < valLen; i++) {
values.push(buf.readDoubleLE(offset)); offset += 8;
}
// tags
const tagLen = buf.readUInt32LE(offset); offset += 4;
const tags: string[] = [];
for (let i = 0; i < tagLen; i++) {
const len = buf.readUInt32LE(offset); offset += 4;
tags.push(buf.toString('utf8', offset, offset + len)); offset += len;
}
return { id, timestamps, values, tags };
}
);
// ============================================
// 사용 예시
// ============================================
// Order 쓰기
const writer = new DataWriter<Order>('./data/orders', {
serializer: orderSerializer,
});
writer.open();
writer.append({
orderId: 1001,
items: [
{ sku: 'ITEM-A', qty: 2, price: 10.5 },
{ sku: 'ITEM-B', qty: 1, price: 25.0 },
{ sku: 'ITEM-C-LONG-SKU', qty: 5, price: 5.0 },
],
total: 71.0,
});
writer.append({
orderId: 1002,
items: [{ sku: 'X', qty: 100, price: 1.0 }],
total: 100.0,
});
writer.close();
// Order 읽기
const reader = new DataReader<Order>('./data/orders', orderSerializer);
reader.open();
const orders = reader.getBulkData(1, 2);
console.log(orders);
// [
// { sequence: 1, data: { orderId: 1001, items: [...], total: 71 } },
// { sequence: 2, data: { orderId: 1002, items: [...], total: 100 } },
// ]
reader.close();
```
---
## 가변 배열 직렬화 패턴
```
┌─────────────────────────────────────────────────┐
│ [길이 4bytes] [요소1] [요소2] ... [요소N] │
└─────────────────────────────────────────────────┘
```
| 타입 | 직렬화 방식 |
|------|-------------|
| `number[]` | `[len:4] [val:8] [val:8] ...` |
| `string[]` | `[len:4] [strLen:4] [str] [strLen:4] [str] ...` |
| `Object[]` | `[len:4] [obj1 필드들] [obj2 필드들] ...` |
| `bigint[]` | `[len:4] [val:8] [val:8] ...` |
---
## 헬퍼 함수 (재사용)
```typescript
// dat/binary-helpers.ts
export class BinaryWriter {
private chunks: Buffer[] = [];
writeUInt32(value: number): this {
const buf = Buffer.alloc(4);
buf.writeUInt32LE(value, 0);
this.chunks.push(buf);
return this;
}
writeDouble(value: number): this {
const buf = Buffer.alloc(8);
buf.writeDoubleLE(value, 0);
this.chunks.push(buf);
return this;
}
writeBigUInt64(value: bigint): this {
const buf = Buffer.alloc(8);
buf.writeBigUInt64LE(value, 0);
this.chunks.push(buf);
return this;
}
writeString(value: string): this {
const strBuf = Buffer.from(value, 'utf8');
this.writeUInt32(strBuf.length);
this.chunks.push(strBuf);
return this;
}
writeNumberArray(values: number[]): this {
this.writeUInt32(values.length);
for (const v of values) this.writeDouble(v);
return this;
}
writeStringArray(values: string[]): this {
this.writeUInt32(values.length);
for (const v of values) this.writeString(v);
return this;
}
toBuffer(): Buffer {
return Buffer.concat(this.chunks);
}
}
export class BinaryReader {
private offset = 0;
constructor(private buf: Buffer) {}
readUInt32(): number {
const v = this.buf.readUInt32LE(this.offset);
this.offset += 4;
return v;
}
readDouble(): number {
const v = this.buf.readDoubleLE(this.offset);
this.offset += 8;
return v;
}
readBigUInt64(): bigint {
const v = this.buf.readBigUInt64LE(this.offset);
this.offset += 8;
return v;
}
readString(): string {
const len = this.readUInt32();
const v = this.buf.toString('utf8', this.offset, this.offset + len);
this.offset += len;
return v;
}
readNumberArray(): number[] {
const len = this.readUInt32();
const arr: number[] = [];
for (let i = 0; i < len; i++) arr.push(this.readDouble());
return arr;
}
readStringArray(): string[] {
const len = this.readUInt32();
const arr: string[] = [];
for (let i = 0; i < len; i++) arr.push(this.readString());
return arr;
}
}
```
---
## 헬퍼 사용 예시
```typescript
import { createSerializer } from './src/data-file/index.js';
import { BinaryWriter, BinaryReader } from './src/data-file/binary-helpers.js';
interface Order {
orderId: number;
items: { sku: string; qty: number; price: number }[];
total: number;
}
const orderSerializer = createSerializer<Order>(
(data) => {
const w = new BinaryWriter();
w.writeUInt32(data.orderId);
w.writeUInt32(data.items.length);
for (const item of data.items) {
w.writeString(item.sku);
w.writeUInt32(item.qty);
w.writeDouble(item.price);
}
w.writeDouble(data.total);
return w.toBuffer();
},
(buf) => {
const r = new BinaryReader(buf);
const orderId = r.readUInt32();
const itemsLen = r.readUInt32();
const items = [];
for (let i = 0; i < itemsLen; i++) {
items.push({
sku: r.readString(),
qty: r.readUInt32(),
price: r.readDouble(),
});
}
const total = r.readDouble();
return { orderId, items, total };
}
);
```

181
README.md Normal file
View File

@@ -0,0 +1,181 @@
## 사용 예시
```typescript
// example.ts
import { DataWriter, DataReader, jsonSerializer, createSerializer } from './index.js';
// ============================================
// 1. JSON 직렬화 (간단한 경우)
// ============================================
interface UserLog {
userId: string;
action: string;
metadata: Record<string, unknown>;
}
// 쓰기
const logWriter = new DataWriter<UserLog>('./data/logs', {
serializer: jsonSerializer<UserLog>(),
maxEntries: 100_000,
});
logWriter.open();
logWriter.append({ userId: 'u1', action: 'login', metadata: { ip: '1.2.3.4' } });
logWriter.append({ userId: 'u2', action: 'purchase', metadata: { amount: 100 } });
logWriter.append({ userId: 'u3', action: 'logout', metadata: {} });
console.log(logWriter.getStats());
logWriter.close();
// 읽기
const logReader = new DataReader<UserLog>('./data/logs', jsonSerializer<UserLog>());
logReader.open();
// 단일 조회
const single = logReader.getBySequence(2);
console.log('Single:', single);
// { sequence: 2, timestamp: 1234567890n, data: { userId: 'u2', action: 'purchase', ... } }
// 범위 조회
const bulk = logReader.getBulkData(1, 3);
console.log('Bulk:', bulk);
// [{ sequence: 1, ... }, { sequence: 2, ... }, { sequence: 3, ... }]
// 전체 조회
const all = logReader.getAllData();
console.log('Total:', all.length);
logReader.close();
// ============================================
// 2. 커스텀 바이너리 직렬화 (고성능)
// ============================================
interface SensorData {
sensorId: number;
temperature: number;
humidity: number;
}
const sensorSerializer = createSerializer<SensorData>(
// serialize
(data) => {
const buf = Buffer.alloc(12);
buf.writeUInt32LE(data.sensorId, 0);
buf.writeFloatLE(data.temperature, 4);
buf.writeFloatLE(data.humidity, 8);
return buf;
},
// deserialize
(buf) => ({
sensorId: buf.readUInt32LE(0),
temperature: buf.readFloatLE(4),
humidity: buf.readFloatLE(8),
})
);
const sensorWriter = new DataWriter<SensorData>('./data/sensors', {
serializer: sensorSerializer,
maxEntries: 1_000_000,
});
sensorWriter.open();
// 대량 추가
for (let i = 0; i < 10000; i++) {
sensorWriter.append({
sensorId: i % 100,
temperature: 20 + Math.random() * 10,
humidity: 40 + Math.random() * 30,
});
}
sensorWriter.close();
const sensorReader = new DataReader<SensorData>('./data/sensors', sensorSerializer);
sensorReader.open();
// 범위 조회
const range = sensorReader.getBulkData(5000, 5010);
console.log('Range:', range.length, 'records');
// 타임스탬프 범위 조회
const now = BigInt(Date.now()) * 1000000n;
const oneHourAgo = now - 3600n * 1000000000n;
const byTime = sensorReader.getBulkDataByTime(oneHourAgo, now);
console.log('By time:', byTime.length, 'records');
sensorReader.close();
// ============================================
// 3. 벌크 추가
// ============================================
interface Order {
orderId: string;
amount: number;
status: string;
}
const orderWriter = new DataWriter<Order>('./data/orders', {
serializer: jsonSerializer<Order>(),
});
orderWriter.open();
const orders: Order[] = [
{ orderId: 'O-001', amount: 150, status: 'pending' },
{ orderId: 'O-002', amount: 250, status: 'completed' },
{ orderId: 'O-003', amount: 350, status: 'shipped' },
];
const sequences = orderWriter.appendBulk(orders);
console.log('Added sequences:', sequences); // [1, 2, 3]
orderWriter.close();
```
---
## API 요약
### DataWriter<T>
| 메서드 | 설명 |
|--------|------|
| `open()` | 파일 열기 (없으면 생성) |
| `append(data, timestamp?)` | 레코드 추가, 시퀀스 반환 |
| `appendBulk(records, timestamp?)` | 여러 레코드 추가, 시퀀스 배열 반환 |
| `getLastSequence()` | 마지막 시퀀스 번호 |
| `getNextSequence()` | 다음 시퀀스 번호 |
| `sync()` | 디스크에 동기화 |
| `close()` | 파일 닫기 |
| `getStats()` | 상태 정보 |
### DataReader<T>
| 메서드 | 설명 |
|--------|------|
| `open()` | 파일 열기 |
| `getBySequence(seq)` | 시퀀스로 단일 조회 |
| `getByIndex(index)` | 인덱스로 단일 조회 |
| `getBulkData(startSeq, endSeq)` | 시퀀스 범위 조회 |
| `getBulkDataByTime(startTs, endTs)` | 타임스탬프 범위 조회 |
| `getAllData()` | 전체 조회 |
| `getRecordCount()` | 레코드 수 |
| `getLastSequence()` | 마지막 시퀀스 |
| `close()` | 파일 닫기 |
### Serializers
```typescript
// JSON (범용)
jsonSerializer<T>()
// MessagePack (빠름, npm install @msgpack/msgpack 필요)
msgpackSerializer<T>()
// 커스텀 바이너리
createSerializer<T>(serialize, deserialize)
```

1
index.ts Normal file
View File

@@ -0,0 +1 @@
export * from './lib/dat/index.js';

92
lib/dat/binary-helpers.ts Normal file
View File

@@ -0,0 +1,92 @@
export class BinaryWriter {
private chunks: Buffer[] = [];
writeUInt32(value: number): this {
const buf = Buffer.alloc(4);
buf.writeUInt32LE(value, 0);
this.chunks.push(buf);
return this;
}
writeDouble(value: number): this {
const buf = Buffer.alloc(8);
buf.writeDoubleLE(value, 0);
this.chunks.push(buf);
return this;
}
writeBigUInt64(value: bigint): this {
const buf = Buffer.alloc(8);
buf.writeBigUInt64LE(value, 0);
this.chunks.push(buf);
return this;
}
writeString(value: string): this {
const strBuf = Buffer.from(value, 'utf8');
this.writeUInt32(strBuf.length);
this.chunks.push(strBuf);
return this;
}
writeNumberArray(values: number[]): this {
this.writeUInt32(values.length);
for (const v of values) this.writeDouble(v);
return this;
}
writeStringArray(values: string[]): this {
this.writeUInt32(values.length);
for (const v of values) this.writeString(v);
return this;
}
toBuffer(): Buffer {
return Buffer.concat(this.chunks);
}
}
export class BinaryReader {
private offset = 0;
constructor(private buf: Buffer) {}
readUInt32(): number {
const v = this.buf.readUInt32LE(this.offset);
this.offset += 4;
return v;
}
readDouble(): number {
const v = this.buf.readDoubleLE(this.offset);
this.offset += 8;
return v;
}
readBigUInt64(): bigint {
const v = this.buf.readBigUInt64LE(this.offset);
this.offset += 8;
return v;
}
readString(): string {
const len = this.readUInt32();
const v = this.buf.toString('utf8', this.offset, this.offset + len);
this.offset += len;
return v;
}
readNumberArray(): number[] {
const len = this.readUInt32();
const arr: number[] = [];
for (let i = 0; i < len; i++) arr.push(this.readDouble());
return arr;
}
readStringArray(): string[] {
const len = this.readUInt32();
const arr: string[] = [];
for (let i = 0; i < len; i++) arr.push(this.readString());
return arr;
}
}

5
lib/dat/constants.ts Normal file
View File

@@ -0,0 +1,5 @@
// src/data-file/constants.ts
export const DATA_MAGIC = 'DATA';
export const DATA_VERSION = 1;
export const DATA_HEADER_SIZE = 64;
export const RECORD_HEADER_SIZE = 8;

7
lib/dat/index.ts Normal file
View File

@@ -0,0 +1,7 @@
// src/data-file/index.ts
export { DataWriter } from './writer.js';
export { DataReader } from './reader.js';
export { DataProtocol } from './protocol.js';
export * from './types.js';
export * from './constants.js';
export * from './serializers.js';

80
lib/dat/protocol.ts Normal file
View File

@@ -0,0 +1,80 @@
// src/data-file/protocol.ts
import { DATA_MAGIC, DATA_VERSION, DATA_HEADER_SIZE, RECORD_HEADER_SIZE } from './constants.js';
import { crc32 } from '../idx/index.js';
import type { Serializer } from './types.js';
export interface DataHeader {
magic: string;
version: number;
createdAt: bigint;
fileSize: bigint;
recordCount: number;
reserved: Buffer;
}
export class DataProtocol {
static createHeader(): Buffer {
const buf = Buffer.alloc(DATA_HEADER_SIZE);
buf.write(DATA_MAGIC, 0, 4, 'ascii');
buf.writeUInt32LE(DATA_VERSION, 4);
buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
buf.writeBigUInt64LE(BigInt(DATA_HEADER_SIZE), 16);
buf.writeUInt32LE(0, 24);
return buf;
}
static readHeader(buf: Buffer): DataHeader {
return {
magic: buf.toString('ascii', 0, 4),
version: buf.readUInt32LE(4),
createdAt: buf.readBigUInt64LE(8),
fileSize: buf.readBigUInt64LE(16),
recordCount: buf.readUInt32LE(24),
reserved: buf.subarray(28, 64),
};
}
static updateHeader(buf: Buffer, fileSize: bigint, recordCount: number): void {
buf.writeBigUInt64LE(fileSize, 16);
buf.writeUInt32LE(recordCount, 24);
}
static serializeRecord<T>(data: T, serializer: Serializer<T>): Buffer {
const dataBytes = serializer.serialize(data);
const totalLen = RECORD_HEADER_SIZE + dataBytes.length;
const buf = Buffer.alloc(totalLen);
dataBytes.copy(buf, RECORD_HEADER_SIZE);
buf.writeUInt32LE(dataBytes.length, 0);
const checksum = crc32(buf, RECORD_HEADER_SIZE, totalLen);
buf.writeUInt32LE(checksum, 4);
return buf;
}
static deserializeRecord<T>(
buf: Buffer,
offset: number,
serializer: Serializer<T>
): { data: T; length: number } | null {
if (offset + RECORD_HEADER_SIZE > buf.length) return null;
const dataLen = buf.readUInt32LE(offset);
const storedChecksum = buf.readUInt32LE(offset + 4);
const totalLen = RECORD_HEADER_SIZE + dataLen;
if (offset + totalLen > buf.length) return null;
const calcChecksum = crc32(buf, offset + RECORD_HEADER_SIZE, offset + totalLen);
if (calcChecksum !== storedChecksum) {
throw new Error(`Checksum mismatch at offset ${offset}`);
}
const dataBytes = buf.subarray(offset + RECORD_HEADER_SIZE, offset + totalLen);
const data = serializer.deserialize(dataBytes);
return { data, length: totalLen };
}
}

213
lib/dat/reader.ts Normal file
View File

@@ -0,0 +1,213 @@
// src/data-file/reader.ts
import * as fs from 'node:fs';
import mmap from '@elilee/mmap-native';
import { DATA_HEADER_SIZE } from './constants.js';
import { DataProtocol, DataHeader } from './protocol.js';
import { IndexReader } from '../idx/index.js';
import type { Serializer, DataEntry } from './types.js';
export class DataReader<T> {
private fd: number | null = null;
private buffer: Buffer | null = null;
private header: DataHeader | null = null;
private indexReader: IndexReader;
private serializer: Serializer<T>;
readonly dataPath: string;
readonly indexPath: string;
constructor(basePath: string, serializer: Serializer<T>) {
this.dataPath = `${basePath}.dat`;
this.indexPath = `${basePath}.idx`;
this.serializer = serializer;
this.indexReader = new IndexReader(this.indexPath);
}
open(): void {
const stats = fs.statSync(this.dataPath);
this.fd = fs.openSync(this.dataPath, 'r');
this.buffer = mmap.map(
stats.size,
mmap.PROT_READ,
mmap.MAP_SHARED,
this.fd,
0
);
this.header = DataProtocol.readHeader(this.buffer);
this.indexReader.open();
}
getHeader(): DataHeader {
if (!this.header) throw new Error('Data file not opened');
return this.header;
}
getBySequence(sequence: number): DataEntry<T> | null {
if (!this.buffer) throw new Error('Data file not opened');
const found = this.indexReader.binarySearchBySequence(sequence);
if (!found) return null;
const result = DataProtocol.deserializeRecord(
this.buffer,
Number(found.entry.offset),
this.serializer
);
if (!result) return null;
return {
sequence: found.entry.sequence,
timestamp: found.entry.timestamp,
data: result.data,
};
}
getByIndex(index: number): DataEntry<T> | null {
if (!this.buffer) throw new Error('Data file not opened');
const entry = this.indexReader.getEntry(index);
if (!entry) return null;
const result = DataProtocol.deserializeRecord(
this.buffer,
Number(entry.offset),
this.serializer
);
if (!result) return null;
return {
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
};
}
getBulkData(startSeq: number, endSeq: number): DataEntry<T>[] {
if (!this.buffer) throw new Error('Data file not opened');
const results: DataEntry<T>[] = [];
const indexHeader = this.indexReader.getHeader();
let startIdx = this.findStartIndex(startSeq, indexHeader.validCount);
for (let i = startIdx; i < indexHeader.validCount; i++) {
const entry = this.indexReader.getEntry(i);
if (!entry) continue;
if (entry.sequence > endSeq) break;
if (entry.sequence >= startSeq) {
const result = DataProtocol.deserializeRecord(
this.buffer,
Number(entry.offset),
this.serializer
);
if (result) {
results.push({
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
});
}
}
}
return results;
}
private findStartIndex(targetSeq: number, validCount: number): number {
let left = 0;
let right = validCount - 1;
let result = 0;
while (left <= right) {
const mid = Math.floor((left + right) / 2);
const entry = this.indexReader.getEntry(mid);
if (!entry) {
right = mid - 1;
continue;
}
if (entry.sequence >= targetSeq) {
result = mid;
right = mid - 1;
} else {
left = mid + 1;
}
}
return result;
}
getBulkDataByTime(startTs: bigint, endTs: bigint): DataEntry<T>[] {
if (!this.buffer) throw new Error('Data file not opened');
const indexResults = this.indexReader.findByTimeRange(startTs, endTs);
const results: DataEntry<T>[] = [];
for (const { entry } of indexResults) {
const result = DataProtocol.deserializeRecord(
this.buffer,
Number(entry.offset),
this.serializer
);
if (result) {
results.push({
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
});
}
}
return results;
}
getAllData(): DataEntry<T>[] {
if (!this.buffer) throw new Error('Data file not opened');
const entries = this.indexReader.getAllEntries();
const results: DataEntry<T>[] = [];
for (const entry of entries) {
const result = DataProtocol.deserializeRecord(
this.buffer,
Number(entry.offset),
this.serializer
);
if (result) {
results.push({
sequence: entry.sequence,
timestamp: entry.timestamp,
data: result.data,
});
}
}
return results;
}
getRecordCount(): number {
return this.indexReader.getHeader().validCount;
}
getLastSequence(): number {
return this.indexReader.getHeader().lastSequence;
}
close(): void {
if (this.buffer) {
mmap.unmap(this.buffer);
this.buffer = null;
}
if (this.fd !== null) {
fs.closeSync(this.fd);
this.fd = null;
}
this.header = null;
this.indexReader.close();
}
}

32
lib/dat/serializers.ts Normal file
View File

@@ -0,0 +1,32 @@
// src/data-file/serializers.ts
import type { Serializer } from './types.js';
export function jsonSerializer<T>(): Serializer<T> {
return {
serialize(data: T): Buffer {
return Buffer.from(JSON.stringify(data), 'utf8');
},
deserialize(buf: Buffer): T {
return JSON.parse(buf.toString('utf8'));
},
};
}
export function msgpackSerializer<T>(): Serializer<T> {
const { encode, decode } = require('@msgpack/msgpack');
return {
serialize(data: T): Buffer {
return Buffer.from(encode(data));
},
deserialize(buf: Buffer): T {
return decode(buf) as T;
},
};
}
export function createSerializer<T>(
serialize: (data: T) => Buffer,
deserialize: (buf: Buffer) => T
): Serializer<T> {
return { serialize, deserialize };
}

16
lib/dat/types.ts Normal file
View File

@@ -0,0 +1,16 @@
// src/data-file/types.ts
export interface Serializer<T> {
serialize(data: T): Buffer;
deserialize(buf: Buffer): T;
}
export interface DataEntry<T> {
sequence: number;
timestamp: bigint;
data: T;
}
export interface DataFileOptions<T> {
serializer: Serializer<T>;
maxEntries?: number;
}

120
lib/dat/writer.ts Normal file
View File

@@ -0,0 +1,120 @@
// src/data-file/writer.ts
import * as fs from 'node:fs';
import { DATA_HEADER_SIZE } from './constants.js';
import { DataProtocol } from './protocol.js';
import { IndexWriter } from '../idx/index.js';
import type { Serializer, DataFileOptions } from './types.js';
export class DataWriter<T> {
private fd: number | null = null;
private headerBuf: Buffer | null = null;
private currentOffset: bigint = BigInt(DATA_HEADER_SIZE);
private recordCount = 0;
private indexWriter: IndexWriter;
private serializer: Serializer<T>;
readonly dataPath: string;
readonly indexPath: string;
constructor(basePath: string, options: DataFileOptions<T>) {
this.dataPath = `${basePath}.dat`;
this.indexPath = `${basePath}.idx`;
this.serializer = options.serializer;
const maxEntries = options.maxEntries ?? 10_000_000;
this.indexWriter = new IndexWriter(this.indexPath, { maxEntries });
}
open(): void {
const isNew = !fs.existsSync(this.dataPath);
this.fd = fs.openSync(this.dataPath, isNew ? 'w+' : 'r+');
this.headerBuf = Buffer.alloc(DATA_HEADER_SIZE);
if (isNew) {
const header = DataProtocol.createHeader();
fs.writeSync(this.fd, header, 0, DATA_HEADER_SIZE, 0);
this.currentOffset = BigInt(DATA_HEADER_SIZE);
this.recordCount = 0;
} else {
fs.readSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
const header = DataProtocol.readHeader(this.headerBuf);
this.currentOffset = header.fileSize;
this.recordCount = header.recordCount;
}
this.indexWriter.open();
}
append(data: T, timestamp?: bigint): number {
if (this.fd === null) throw new Error('Data file not opened');
const buf = DataProtocol.serializeRecord(data, this.serializer);
const offset = this.currentOffset;
fs.writeSync(this.fd, buf, 0, buf.length, Number(offset));
const sequence = this.indexWriter.getNextSequence();
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
this.indexWriter.append(offset, buf.length, ts);
this.currentOffset += BigInt(buf.length);
this.recordCount++;
return sequence;
}
appendBulk(records: T[], timestamp?: bigint): number[] {
const sequences: number[] = [];
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
for (const record of records) {
const seq = this.append(record, ts);
sequences.push(seq);
}
return sequences;
}
getLastSequence(): number {
return this.indexWriter.getLastSequence();
}
getNextSequence(): number {
return this.indexWriter.getNextSequence();
}
sync(): void {
if (this.fd === null || !this.headerBuf) return;
DataProtocol.updateHeader(this.headerBuf, this.currentOffset, this.recordCount);
fs.writeSync(this.fd, this.headerBuf, 0, DATA_HEADER_SIZE, 0);
fs.fsyncSync(this.fd);
this.indexWriter.syncAll();
}
close(): void {
this.sync();
if (this.fd !== null) {
fs.closeSync(this.fd);
this.fd = null;
}
this.indexWriter.close();
this.headerBuf = null;
}
getStats() {
return {
dataPath: this.dataPath,
indexPath: this.indexPath,
currentOffset: this.currentOffset,
recordCount: this.recordCount,
lastSequence: this.indexWriter.getLastSequence(),
};
}
}

8
lib/idx/constants.ts Normal file
View File

@@ -0,0 +1,8 @@
// src/index-file/constants.ts
export const INDEX_MAGIC = 'INDX';
export const INDEX_VERSION = 1;
export const INDEX_HEADER_SIZE = 64;
export const INDEX_ENTRY_SIZE = 32;
export const FLAG_VALID = 0x0001;
export const FLAG_DELETED = 0x0002;

6
lib/idx/index.ts Normal file
View File

@@ -0,0 +1,6 @@
// src/index-file/index.ts
export { IndexWriter } from './writer.js';
export { IndexReader } from './reader.js';
export { IndexProtocol, crc32 } from './protocol.js';
export * from './types.js';
export * from './constants.js';

106
lib/idx/protocol.ts Normal file
View File

@@ -0,0 +1,106 @@
// src/index-file/protocol.ts
import {
INDEX_MAGIC,
INDEX_VERSION,
INDEX_HEADER_SIZE,
INDEX_ENTRY_SIZE,
FLAG_VALID,
} from './constants.js';
import type { IndexHeader, IndexEntry } from './types.js';
const CRC_TABLE = new Uint32Array(256);
for (let i = 0; i < 256; i++) {
let c = i;
for (let j = 0; j < 8; j++) {
c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
}
CRC_TABLE[i] = c >>> 0;
}
export function crc32(buf: Buffer, start = 0, end?: number): number {
let crc = 0xFFFFFFFF;
const len = end ?? buf.length;
for (let i = start; i < len; i++) {
crc = CRC_TABLE[(crc ^ buf[i]) & 0xFF] ^ (crc >>> 8);
}
return (~crc) >>> 0;
}
export class IndexProtocol {
static createHeader(entryCount: number, magic = INDEX_MAGIC): Buffer {
const buf = Buffer.alloc(INDEX_HEADER_SIZE);
buf.write(magic, 0, 4, 'ascii');
buf.writeUInt32LE(INDEX_VERSION, 4);
buf.writeBigUInt64LE(BigInt(Date.now()) * 1000000n, 8);
buf.writeUInt32LE(INDEX_ENTRY_SIZE, 16);
buf.writeUInt32LE(entryCount, 20);
buf.writeUInt32LE(0, 24);
buf.writeBigUInt64LE(0n, 28);
buf.writeUInt32LE(0, 36);
return buf;
}
static readHeader(buf: Buffer): IndexHeader {
return {
magic: buf.toString('ascii', 0, 4),
version: buf.readUInt32LE(4),
createdAt: buf.readBigUInt64LE(8),
entrySize: buf.readUInt32LE(16),
entryCount: buf.readUInt32LE(20),
validCount: buf.readUInt32LE(24),
dataFileSize: buf.readBigUInt64LE(28),
lastSequence: buf.readUInt32LE(36),
reserved: buf.subarray(40, 64),
};
}
static updateHeaderCounts(
buf: Buffer,
validCount: number,
dataFileSize: bigint,
lastSequence: number
): void {
buf.writeUInt32LE(validCount, 24);
buf.writeBigUInt64LE(dataFileSize, 28);
buf.writeUInt32LE(lastSequence, 36);
}
static writeEntry(buf: Buffer, index: number, entry: Omit<IndexEntry, 'checksum'>): void {
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
buf.writeUInt32LE(entry.sequence, off);
buf.writeBigUInt64LE(entry.timestamp, off + 4);
buf.writeBigUInt64LE(entry.offset, off + 12);
buf.writeUInt32LE(entry.length, off + 20);
buf.writeUInt32LE(entry.flags | FLAG_VALID, off + 24);
const checksum = crc32(buf, off, off + 28);
buf.writeUInt32LE(checksum, off + 28);
}
static readEntry(buf: Buffer, index: number): IndexEntry | null {
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
const flags = buf.readUInt32LE(off + 24);
if (!(flags & FLAG_VALID)) return null;
return {
sequence: buf.readUInt32LE(off),
timestamp: buf.readBigUInt64LE(off + 4),
offset: buf.readBigUInt64LE(off + 12),
length: buf.readUInt32LE(off + 20),
flags,
checksum: buf.readUInt32LE(off + 28),
};
}
static isValidEntry(buf: Buffer, index: number): boolean {
const off = INDEX_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
const flags = buf.readUInt32LE(off + 24);
return (flags & FLAG_VALID) !== 0;
}
static calcFileSize(entryCount: number): number {
return INDEX_HEADER_SIZE + INDEX_ENTRY_SIZE * entryCount;
}
}

131
lib/idx/reader.ts Normal file
View File

@@ -0,0 +1,131 @@
// src/index-file/reader.ts
import * as fs from 'node:fs';
import mmap from '@elilee/mmap-native';
import { IndexProtocol } from './protocol.js';
import type { IndexHeader, IndexEntry } from './types.js';
export class IndexReader {
private fd: number | null = null;
private buffer: Buffer | null = null;
private header: IndexHeader | null = null;
readonly path: string;
constructor(path: string) {
this.path = path;
}
open(): void {
const stats = fs.statSync(this.path);
this.fd = fs.openSync(this.path, 'r');
this.buffer = mmap.map(
stats.size,
mmap.PROT_READ,
mmap.MAP_SHARED,
this.fd,
0
);
this.header = IndexProtocol.readHeader(this.buffer);
}
getHeader(): IndexHeader {
if (!this.header) throw new Error('Index file not opened');
return this.header;
}
getEntry(index: number): IndexEntry | null {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
if (index < 0 || index >= this.header.entryCount) return null;
return IndexProtocol.readEntry(this.buffer, index);
}
findBySequence(sequence: number): { index: number; entry: IndexEntry } | null {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
for (let i = 0; i < this.header.validCount; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry && entry.sequence === sequence) {
return { index: i, entry };
}
}
return null;
}
findBySequenceRange(startSeq: number, endSeq: number): { index: number; entry: IndexEntry }[] {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
const results: { index: number; entry: IndexEntry }[] = [];
for (let i = 0; i < this.header.validCount; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry && entry.sequence >= startSeq && entry.sequence <= endSeq) {
results.push({ index: i, entry });
}
}
return results;
}
getAllEntries(): IndexEntry[] {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
const entries: IndexEntry[] = [];
for (let i = 0; i < this.header.validCount; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry) entries.push(entry);
}
return entries;
}
findByTimeRange(startTs: bigint, endTs: bigint): { index: number; entry: IndexEntry }[] {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
const results: { index: number; entry: IndexEntry }[] = [];
for (let i = 0; i < this.header.validCount; i++) {
const entry = IndexProtocol.readEntry(this.buffer, i);
if (entry && entry.timestamp >= startTs && entry.timestamp <= endTs) {
results.push({ index: i, entry });
}
}
return results;
}
binarySearchBySequence(targetSeq: number): { index: number; entry: IndexEntry } | null {
if (!this.buffer || !this.header) throw new Error('Index file not opened');
let left = 0;
let right = this.header.validCount - 1;
while (left <= right) {
const mid = Math.floor((left + right) / 2);
const entry = IndexProtocol.readEntry(this.buffer, mid);
if (!entry) {
right = mid - 1;
continue;
}
if (entry.sequence === targetSeq) {
return { index: mid, entry };
} else if (entry.sequence < targetSeq) {
left = mid + 1;
} else {
right = mid - 1;
}
}
return null;
}
close(): void {
if (this.buffer) {
mmap.unmap(this.buffer);
this.buffer = null;
}
if (this.fd !== null) {
fs.closeSync(this.fd);
this.fd = null;
}
this.header = null;
}
}

26
lib/idx/types.ts Normal file
View File

@@ -0,0 +1,26 @@
// src/index-file/types.ts
export interface IndexHeader {
magic: string;
version: number;
createdAt: bigint;
entrySize: number;
entryCount: number;
validCount: number;
dataFileSize: bigint;
lastSequence: number;
reserved: Buffer;
}
export interface IndexEntry {
sequence: number;
timestamp: bigint;
offset: bigint;
length: number;
flags: number;
checksum: number;
}
export interface IndexFileOptions {
maxEntries: number;
magic?: string;
}

141
lib/idx/writer.ts Normal file
View File

@@ -0,0 +1,141 @@
// src/index-file/writer.ts
import * as fs from 'node:fs';
import mmap from '@elilee/mmap-native';
import { INDEX_HEADER_SIZE, FLAG_VALID } from './constants.js';
import { IndexProtocol } from './protocol.js';
import type { IndexFileOptions } from './types.js';
export class IndexWriter {
private fd: number | null = null;
private buffer: Buffer | null = null;
private validCount = 0;
private dataFileSize = 0n;
private lastSequence = 0;
readonly path: string;
readonly maxEntries: number;
readonly fileSize: number;
constructor(path: string, options: IndexFileOptions) {
this.path = path;
this.maxEntries = options.maxEntries;
this.fileSize = IndexProtocol.calcFileSize(options.maxEntries);
}
open(): void {
const isNew = !fs.existsSync(this.path);
this.fd = fs.openSync(this.path, isNew ? 'w+' : 'r+');
if (isNew) {
fs.ftruncateSync(this.fd, this.fileSize);
}
this.buffer = mmap.map(
this.fileSize,
mmap.PROT_READ | mmap.PROT_WRITE,
mmap.MAP_SHARED,
this.fd,
0
);
if (isNew) {
const header = IndexProtocol.createHeader(this.maxEntries);
header.copy(this.buffer, 0);
this.syncHeader();
} else {
const header = IndexProtocol.readHeader(this.buffer);
this.validCount = header.validCount;
this.dataFileSize = header.dataFileSize;
this.lastSequence = header.lastSequence;
}
}
write(
index: number,
sequence: number,
offset: bigint,
length: number,
timestamp?: bigint
): boolean {
if (!this.buffer) throw new Error('Index file not opened');
if (index < 0 || index >= this.maxEntries) return false;
const ts = timestamp ?? BigInt(Date.now()) * 1000000n;
IndexProtocol.writeEntry(this.buffer, index, {
sequence,
timestamp: ts,
offset,
length,
flags: FLAG_VALID,
});
this.validCount++;
if (sequence > this.lastSequence) {
this.lastSequence = sequence;
}
const newDataEnd = offset + BigInt(length);
if (newDataEnd > this.dataFileSize) {
this.dataFileSize = newDataEnd;
}
return true;
}
append(offset: bigint, length: number, timestamp?: bigint): number {
const index = this.validCount;
if (index >= this.maxEntries) return -1;
const sequence = this.lastSequence + 1;
this.write(index, sequence, offset, length, timestamp);
return index;
}
getLastSequence(): number {
return this.lastSequence;
}
getNextSequence(): number {
return this.lastSequence + 1;
}
syncHeader(): void {
if (!this.buffer) return;
IndexProtocol.updateHeaderCounts(
this.buffer,
this.validCount,
this.dataFileSize,
this.lastSequence
);
mmap.sync(this.buffer, 0, INDEX_HEADER_SIZE, mmap.MS_ASYNC);
}
syncAll(): void {
if (!this.buffer) return;
this.syncHeader();
mmap.sync(this.buffer, 0, this.fileSize, mmap.MS_SYNC);
}
close(): void {
if (!this.buffer || this.fd === null) return;
this.syncAll();
mmap.unmap(this.buffer);
fs.closeSync(this.fd);
this.buffer = null;
this.fd = null;
}
getStats() {
return {
path: this.path,
maxEntries: this.maxEntries,
validCount: this.validCount,
dataFileSize: this.dataFileSize,
lastSequence: this.lastSequence,
};
}
}

1194
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

29
package.json Normal file
View File

@@ -0,0 +1,29 @@
{
"name": "@elilee/index-file",
"version": "1.0.0",
"description": "Use index file with elilee mmap-native",
"license": "MIT",
"author": "",
"type": "module",
"main": "./dist/lib/index.js",
"types": "./dist/lib/index.d.ts",
"files": [
"dist/",
"lib/",
"tsconfig.json"
],
"dependencies": {
"typescript": "^5.7.0",
"@types/node": "^22.0.0",
"@elilee/mmap-native": "git+https://git.satitech.co.kr/sati-open/sati.n-api.mmap.git"
},
"scripts": {
"prepare": "tsc -p tsconfig.json",
"build:ts": "tsc -p tsconfig.json"
},
"engines": {
"node": ">=18.0.0"
},
"os": ["linux", "darwin"],
"cpu": ["x64", "arm64"]
}

16
tsconfig.json Normal file
View File

@@ -0,0 +1,16 @@
// tsconfig.json
{
"compilerOptions": {
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"outDir": "./dist",
"rootDir": ".",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"declaration": true
},
"include": ["lib/**/*", "index.ts"],
"exclude": ["node_modules", "dist"]
}