before refactor index store to complex file-line pattern

2026-03-12 16:13:44 +06:00
parent 491ccbea89
commit 8ba956d8c5
21 changed files with 7804 additions and 57 deletions
--- a/tests/linedb_index_test.go
+++ b/tests/linedb_index_test.go
@@ -0,0 +1,419 @@
+package tests
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"linedb/pkg/linedb"
+)
+
+// mockMemcached — in-memory реализация MemcachedClient для тестов.
+type mockMemcached struct {
+	data map[string][]byte
+}
+
+func newMockMemcached() *mockMemcached {
+	return &mockMemcached{data: make(map[string][]byte)}
+}
+
+func (m *mockMemcached) Get(key string) ([]byte, error) {
+	v, ok := m.data[key]
+	if !ok {
+		return nil, nil
+	}
+	return v, nil
+}
+
+func (m *mockMemcached) Set(key string, value []byte, _ int) error {
+	m.data[key] = value
+	return nil
+}
+
+func (m *mockMemcached) Delete(key string) error {
+	delete(m.data, key)
+	return nil
+}
+
+func setupIndexedCollection(t *testing.T, indexStore linedb.IndexStore) (*linedb.LineDb, func()) {
+	t.Helper()
+	os.RemoveAll("./testdata")
+	opts := &linedb.LineDbOptions{}
+	if indexStore != nil {
+		opts.IndexStore = indexStore
+	}
+	initOptions := &linedb.LineDbInitOptions{
+		CacheSize: 100,
+		CacheTTL:  time.Minute,
+		DBFolder:  "./data/test-linedb-index",
+		Collections: []linedb.JSONLFileOptions{
+			{
+				CollectionName: "users",
+				AllocSize:      256,
+				IndexedFields:  []string{"id", "email", "name"},
+			},
+		},
+	}
+	db := linedb.NewLineDb(opts)
+	if err := db.Init(false, initOptions); err != nil {
+		t.Fatalf("Init failed: %v", err)
+	}
+	return db, func() { db.Close(); os.RemoveAll("./testdata") }
+}
+
+func TestIndexInMemoryReadByFilter(t *testing.T) {
+	db, cleanup := setupIndexedCollection(t, nil) // nil = auto in-memory
+	defer cleanup()
+
+	opts := linedb.LineDbAdapterOptions{}
+
+	if err := db.Insert(map[string]any{"name": "alice", "email": "alice@test.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert 1 failed: %v", err)
+	}
+	if err := db.Insert(map[string]any{"name": "bob", "email": "bob@test.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert 2 failed: %v", err)
+	}
+	if err := db.Insert(map[string]any{"name": "charlie", "email": "charlie@test.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert 3 failed: %v", err)
+	}
+
+	// Поиск по индексированному полю email
+	found, err := db.ReadByFilter("email:bob@test.com,name:bob", "users", opts)
+	// found, err := db.ReadByFilter(map[string]any{"email": "bob@test.com"}, "users", opts)
+	if err != nil {
+		t.Fatalf("ReadByFilter failed: %v", err)
+	}
+	if len(found) != 1 {
+		t.Fatalf("Expected 1 record for email bob@test.com, got %d", len(found))
+	}
+	if m, ok := found[0].(map[string]any); ok {
+		if m["name"] != "bob" {
+			t.Errorf("Expected name bob, got %v", m["name"])
+		}
+	}
+
+	// Поиск по name
+	found2, err := db.ReadByFilter(map[string]any{"name": "alice"}, "users", opts)
+	if err != nil {
+		t.Fatalf("ReadByFilter name failed: %v", err)
+	}
+	if len(found2) != 1 {
+		t.Fatalf("Expected 1 record for name alice, got %d", len(found2))
+	}
+}
+
+// TestIndexEncodedCollectionCache проверяет шифрованную коллекцию и двойное чтение одного фильтра (кэш).
+func TestIndexEncodedCollectionCache(t *testing.T) {
+	err := os.RemoveAll("./data/test-linedb-index-enc")
+	if err != nil {
+		t.Logf("RemoveAll failed: %v", err)
+
+	}
+	initOptions := &linedb.LineDbInitOptions{
+		CacheSize: 100,
+		CacheTTL:  time.Minute*10,
+		DBFolder:  "./data/test-linedb-index-enc",
+		Collections: []linedb.JSONLFileOptions{
+			{
+				CollectionName: "users",
+				AllocSize:      512,
+				IndexedFields:  []string{"id", "email", "name"},
+				Encode:         false,
+				EncodeKey:      "test-secret-key",
+			},
+		},
+	}
+	db := linedb.NewLineDb(nil)
+	if err := db.Init(false, initOptions); err != nil {
+		t.Fatalf("Init failed: %v", err)
+	}
+	defer func() { db.Close(); os.RemoveAll("./testdata") }()
+
+	opts := linedb.LineDbAdapterOptions{}
+	if err := db.Insert(map[string]any{"name": "alice", "email": "alice@secret.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert 1 failed: %v", err)
+	}
+	if err := db.Insert(map[string]any{"name": "bob", "email": "bob@secret.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert 2 failed: %v", err)
+	}
+
+	filter := map[string]any{"email": "bob@secret.com"}
+
+	// Первое чтение — из файла (с дешифровкой)
+	found1, err := db.ReadByFilter(filter, "users", opts)
+	if err != nil {
+		t.Fatalf("ReadByFilter (1st) failed: %v", err)
+	}
+	if len(found1) != 1 {
+		t.Fatalf("Expected 1 record, got %d", len(found1))
+	}
+	if m, ok := found1[0].(map[string]any); ok {
+		if m["name"] != "bob" || m["email"] != "bob@secret.com" {
+			t.Errorf("Expected name=bob email=bob@secret.com, got %+v", m)
+		}
+	}
+	cacheSize1 := db.GetActualCacheSize()
+	if cacheSize1 < 1 {
+		t.Errorf("Expected cache to have at least 1 entry after first read, got %d", cacheSize1)
+	}
+
+	// Второе чтение — из кэша (должно вернуть те же данные без повторного чтения файла)
+	found2, err := db.ReadByFilter(filter, "users", opts)
+	if err != nil {
+		t.Fatalf("ReadByFilter (2nd, cached) failed: %v", err)
+	}
+	if len(found2) != 1 {
+		t.Fatalf("Expected 1 record on second read, got %d", len(found2))
+	}
+	if m, ok := found2[0].(map[string]any); ok {
+		if m["name"] != "bob" || m["email"] != "bob@secret.com" {
+			t.Errorf("Expected name=bob email=bob@secret.com on cached read, got %+v", m)
+		}
+	}
+	// Кэш не должен расти при повторном запросе с тем же ключом
+	cacheSize2 := db.GetActualCacheSize()
+	if cacheSize2 != cacheSize1 {
+		t.Errorf("Cache size should stay same after cache hit: was %d, now %d", cacheSize1, cacheSize2)
+	}
+
+	// Обновляем запись (name) и проверяем, что кэш и индекс обновились
+	_, err = db.Update(
+		map[string]any{"name": "bob_updated"},
+		"users",
+		map[string]any{"email": "bob@secret.com"},
+		opts,
+	)
+	if err != nil {
+		t.Fatalf("Update failed: %v", err)
+	}
+	// После Update кэш сбрасывается — читаем снова, чтобы заполнить кэш актуальными данными
+	found3, err := db.ReadByFilter(filter, "users", opts)
+	if err != nil {
+		t.Fatalf("ReadByFilter after update failed: %v", err)
+	}
+	if len(found3) != 1 {
+		t.Fatalf("Expected 1 record after update, got %d", len(found3))
+	}
+	if m, ok := found3[0].(map[string]any); ok && m["name"] != "bob_updated" {
+		t.Errorf("Expected name=bob_updated after update, got %v", m["name"])
+	}
+	// Проверяем сырой кэш: в нём должна быть запись с name=bob_updated
+	rawCache := db.GetCacheForTest("give_me_cache")
+	if len(rawCache) == 0 {
+		t.Error("Expected cache to have entries after read")
+	}
+	var foundInCache bool
+	for _, v := range rawCache {
+		arr, ok := v.([]any)
+		if !ok || len(arr) != 1 {
+			continue
+		}
+		m, ok := arr[0].(map[string]any)
+		if !ok {
+			continue
+		}
+		if m["email"] == "bob@secret.com" {
+			foundInCache = true
+			if m["name"] != "bob_updated" {
+				t.Errorf("Expected cached name=bob_updated, got %v", m["name"])
+			}
+			break
+		}
+	}
+	if !foundInCache {
+		t.Error("Expected to find bob record in raw cache with updated name")
+	}
+	// Проверяем индекс: по email bob@secret.com должна быть одна строка, по name bob_updated — тоже
+	idxSnapshot := db.GetIndexSnapshotForTest("give_me_cache")
+	if len(idxSnapshot) == 0 {
+		t.Error("Expected index snapshot to have entries")
+	}
+	if emailIdx, ok := idxSnapshot["users:email"].(map[string][]int); ok {
+		if lines, ok := emailIdx["bob@secret.com"]; !ok || len(lines) != 1 {
+			t.Errorf("Expected index users:email bob@secret.com to have 1 line, got %v", emailIdx["bob@secret.com"])
+		}
+	}
+	if nameIdx, ok := idxSnapshot["users:name"].(map[string][]int); ok {
+		if lines, ok := nameIdx["bob_updated"]; !ok || len(lines) != 1 {
+			t.Errorf("Expected index users:name bob_updated to have 1 line, got %v", nameIdx["bob_updated"])
+		}
+	}
+
+	// Удаляем запись и проверяем, что в кэше и в индексе её больше нет
+	_, err = db.Delete(map[string]any{"email": "bob@secret.com"}, "users", opts)
+	if err != nil {
+		t.Fatalf("Delete failed: %v", err)
+	}
+	// После Delete кэш сбрасывается — читаем снова
+	found4, err := db.ReadByFilter(filter, "users", opts)
+	if err != nil {
+		t.Fatalf("ReadByFilter after delete failed: %v", err)
+	}
+	if len(found4) != 0 {
+		t.Fatalf("Expected 0 records after delete, got %d", len(found4))
+	}
+	// В сыром кэше не должно остаться записи bob
+	rawCache2 := db.GetCacheForTest("give_me_cache")
+	for _, v := range rawCache2 {
+		arr, ok := v.([]any)
+		if !ok || len(arr) != 1 {
+			continue
+		}
+		if m, ok := arr[0].(map[string]any); ok && m["email"] == "bob@secret.com" {
+			t.Error("Cached result after delete should not contain bob record")
+		}
+	}
+	// Индекс: bob@secret.com и bob_updated не должны быть в индексе (или пустые срезы)
+	idxSnapshot2 := db.GetIndexSnapshotForTest("give_me_cache")
+	if emailIdx, ok := idxSnapshot2["users:email"].(map[string][]int); ok {
+		if lines, has := emailIdx["bob@secret.com"]; has && len(lines) > 0 {
+			t.Errorf("After delete, index users:email bob@secret.com should be empty, got %v", lines)
+		}
+	}
+	if nameIdx, ok := idxSnapshot2["users:name"].(map[string][]int); ok {
+		if lines, has := nameIdx["bob_updated"]; has && len(lines) > 0 {
+			t.Errorf("After delete, index users:name bob_updated should be empty, got %v", lines)
+		}
+	}
+}
+
+func TestIndexExplicitInMemory(t *testing.T) {
+	store := linedb.NewInMemoryIndexStore()
+	db, cleanup := setupIndexedCollection(t, store)
+	defer cleanup()
+
+	opts := linedb.LineDbAdapterOptions{}
+	if err := db.Insert(map[string]any{"name": "x", "email": "x@y.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert failed: %v", err)
+	}
+
+	found, err := db.ReadByFilter(map[string]any{"email": "x@y.com"}, "users", opts)
+	if err != nil {
+		t.Fatalf("ReadByFilter failed: %v", err)
+	}
+	if len(found) != 1 {
+		t.Fatalf("Expected 1 record, got %d", len(found))
+	}
+}
+
+func TestIndexMemcachedStore(t *testing.T) {
+	mock := newMockMemcached()
+	store, err := linedb.NewMemcachedIndexStore(linedb.MemcachedIndexStoreOptions{
+		Client: mock,
+	})
+	if err != nil {
+		t.Fatalf("NewMemcachedIndexStore: %v", err)
+	}
+
+	db, cleanup := setupIndexedCollection(t, store)
+	defer cleanup()
+
+	opts := linedb.LineDbAdapterOptions{}
+	if err := db.Insert(map[string]any{"name": "mem", "email": "mem@test.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert failed: %v", err)
+	}
+
+	found, err := db.ReadByFilter(map[string]any{"email": "mem@test.com"}, "users", opts)
+	if err != nil {
+		t.Fatalf("ReadByFilter failed: %v", err)
+	}
+	if len(found) != 1 {
+		t.Fatalf("Expected 1 record, got %d", len(found))
+	}
+
+	// Проверяем, что в mockMemcached что-то записалось
+	if len(mock.data) == 0 {
+		t.Error("Expected memcached store to have data")
+	}
+}
+
+func TestIndexUpdateRebuild(t *testing.T) {
+	db, cleanup := setupIndexedCollection(t, nil)
+	defer cleanup()
+
+	opts := linedb.LineDbAdapterOptions{}
+	if err := db.Insert(map[string]any{"name": "a", "email": "a@test.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert failed: %v", err)
+	}
+
+	_, err := db.Update(
+		map[string]any{"email": "a_updated@test.com"},
+		"users",
+		map[string]any{"email": "a@test.com"},
+		opts,
+	)
+	if err != nil {
+		t.Fatalf("Update failed: %v", err)
+	}
+
+	// Старый email не должен находиться
+	old, _ := db.ReadByFilter(map[string]any{"email": "a@test.com"}, "users", opts)
+	if len(old) != 0 {
+		t.Fatalf("Expected 0 records for old email, got %d", len(old))
+	}
+	// Новый — должен
+	newFound, _ := db.ReadByFilter(map[string]any{"email": "a_updated@test.com"}, "users", opts)
+	if len(newFound) != 1 {
+		t.Fatalf("Expected 1 record for new email, got %d", len(newFound))
+	}
+}
+
+func TestIndexDeleteRebuild(t *testing.T) {
+	db, cleanup := setupIndexedCollection(t, nil)
+	defer cleanup()
+
+	opts := linedb.LineDbAdapterOptions{}
+	if err := db.Insert(map[string]any{"name": "del", "email": "del@test.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert failed: %v", err)
+	}
+	if err := db.Insert(map[string]any{"name": "keep", "email": "keep@test.com"}, "users", opts); err != nil {
+		t.Fatalf("Insert 2 failed: %v", err)
+	}
+
+	_, err := db.Delete(map[string]any{"email": "del@test.com"}, "users", opts)
+	if err != nil {
+		t.Fatalf("Delete failed: %v", err)
+	}
+
+	found, _ := db.ReadByFilter(map[string]any{"email": "del@test.com"}, "users", opts)
+	if len(found) != 0 {
+		t.Fatalf("Expected 0 after delete, got %d", len(found))
+	}
+	kept, _ := db.ReadByFilter(map[string]any{"email": "keep@test.com"}, "users", opts)
+	if len(kept) != 1 {
+		t.Fatalf("Expected 1 kept record, got %d", len(kept))
+	}
+}
+
+func TestIndexNoIndexedFields(t *testing.T) {
+	os.RemoveAll("./testdata")
+	initOptions := &linedb.LineDbInitOptions{
+		CacheSize: 100,
+		CacheTTL:  time.Minute,
+		DBFolder:  "./testdata",
+		Collections: []linedb.JSONLFileOptions{
+			{
+				CollectionName: "users",
+				AllocSize:      256,
+				IndexedFields:  nil, // без индексов
+			},
+		},
+	}
+	db := linedb.NewLineDb(nil)
+	if err := db.Init(false, initOptions); err != nil {
+		t.Fatalf("Init failed: %v", err)
+	}
+	defer func() { db.Close(); os.RemoveAll("./testdata") }()
+
+	if err := db.Insert(map[string]any{"name": "a", "email": "a@test.com"}, "users", linedb.LineDbAdapterOptions{}); err != nil {
+		t.Fatalf("Insert failed: %v", err)
+	}
+	// ReadByFilter всё равно работает (полный скан)
+	found, err := db.ReadByFilter(map[string]any{"email": "a@test.com"}, "users", linedb.LineDbAdapterOptions{})
+	if err != nil {
+		t.Fatalf("ReadByFilter failed: %v", err)
+	}
+	if len(found) != 1 {
+		t.Fatalf("Expected 1 record, got %d", len(found))
+	}
+}