GiteaAdmin/elowdb-go
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

before refactor index store to complex file-line pattern

Browse Source
This commit is contained in:
Кузнецов Антон (aka Direct-dev.ru)
2026-03-12 16:13:44 +06:00
parent 491ccbea89
commit 8ba956d8c5
21 changed files with 7804 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/data/test-linedb-index-enc/users.jsonl Normal file
View File

@@ -0,0 +1 @@
{"email":"alice@secret.com","id":1,"name":"alice"}

2
tests/data/test-point-update/users.jsonl Normal file
View File

@@ -0,0 +1,2 @@
{"email":"alice@test.com","id":1,"name":"alice"}
{"email":"bob_new@test.com","id":2,"name":"bob"}

419
tests/linedb_index_test.go Normal file
View File

@@ -0,0 +1,419 @@
package tests
import (
"os"
"testing"
"time"
"linedb/pkg/linedb"
)
// mockMemcached — in-memory реализация MemcachedClient для тестов.
type mockMemcached struct {
data map[string][]byte
}
func newMockMemcached() *mockMemcached {
return &mockMemcached{data: make(map[string][]byte)}
}
func (m *mockMemcached) Get(key string) ([]byte, error) {
v, ok := m.data[key]
if !ok {
return nil, nil
}
return v, nil
}
func (m *mockMemcached) Set(key string, value []byte, _ int) error {
m.data[key] = value
return nil
}
func (m *mockMemcached) Delete(key string) error {
delete(m.data, key)
return nil
}
func setupIndexedCollection(t *testing.T, indexStore linedb.IndexStore) (*linedb.LineDb, func()) {
t.Helper()
os.RemoveAll("./testdata")
opts := &linedb.LineDbOptions{}
if indexStore != nil {
opts.IndexStore = indexStore
}
initOptions := &linedb.LineDbInitOptions{
CacheSize: 100,
CacheTTL: time.Minute,
DBFolder: "./data/test-linedb-index",
Collections: []linedb.JSONLFileOptions{
{
CollectionName: "users",
AllocSize: 256,
IndexedFields: []string{"id", "email", "name"},
},
},
}
db := linedb.NewLineDb(opts)
if err := db.Init(false, initOptions); err != nil {
t.Fatalf("Init failed: %v", err)
}
return db, func() { db.Close(); os.RemoveAll("./testdata") }
}
func TestIndexInMemoryReadByFilter(t *testing.T) {
db, cleanup := setupIndexedCollection(t, nil) // nil = auto in-memory
defer cleanup()
opts := linedb.LineDbAdapterOptions{}
if err := db.Insert(map[string]any{"name": "alice", "email": "alice@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 1 failed: %v", err)
}
if err := db.Insert(map[string]any{"name": "bob", "email": "bob@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 2 failed: %v", err)
}
if err := db.Insert(map[string]any{"name": "charlie", "email": "charlie@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 3 failed: %v", err)
}
// Поиск по индексированному полю email
found, err := db.ReadByFilter("email:bob@test.com,name:bob", "users", opts)
// found, err := db.ReadByFilter(map[string]any{"email": "bob@test.com"}, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter failed: %v", err)
}
if len(found) != 1 {
t.Fatalf("Expected 1 record for email bob@test.com, got %d", len(found))
}
if m, ok := found[0].(map[string]any); ok {
if m["name"] != "bob" {
t.Errorf("Expected name bob, got %v", m["name"])
}
}
// Поиск по name
found2, err := db.ReadByFilter(map[string]any{"name": "alice"}, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter name failed: %v", err)
}
if len(found2) != 1 {
t.Fatalf("Expected 1 record for name alice, got %d", len(found2))
}
}
// TestIndexEncodedCollectionCache проверяет шифрованную коллекцию и двойное чтение одного фильтра (кэш).
func TestIndexEncodedCollectionCache(t *testing.T) {
err := os.RemoveAll("./data/test-linedb-index-enc")
if err != nil {
t.Logf("RemoveAll failed: %v", err)
}
initOptions := &linedb.LineDbInitOptions{
CacheSize: 100,
CacheTTL: time.Minute*10,
DBFolder: "./data/test-linedb-index-enc",
Collections: []linedb.JSONLFileOptions{
{
CollectionName: "users",
AllocSize: 512,
IndexedFields: []string{"id", "email", "name"},
Encode: false,
EncodeKey: "test-secret-key",
},
},
}
db := linedb.NewLineDb(nil)
if err := db.Init(false, initOptions); err != nil {
t.Fatalf("Init failed: %v", err)
}
defer func() { db.Close(); os.RemoveAll("./testdata") }()
opts := linedb.LineDbAdapterOptions{}
if err := db.Insert(map[string]any{"name": "alice", "email": "alice@secret.com"}, "users", opts); err != nil {
t.Fatalf("Insert 1 failed: %v", err)
}
if err := db.Insert(map[string]any{"name": "bob", "email": "bob@secret.com"}, "users", opts); err != nil {
t.Fatalf("Insert 2 failed: %v", err)
}
filter := map[string]any{"email": "bob@secret.com"}
// Первое чтение — из файла (с дешифровкой)
found1, err := db.ReadByFilter(filter, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter (1st) failed: %v", err)
}
if len(found1) != 1 {
t.Fatalf("Expected 1 record, got %d", len(found1))
}
if m, ok := found1[0].(map[string]any); ok {
if m["name"] != "bob" || m["email"] != "bob@secret.com" {
t.Errorf("Expected name=bob email=bob@secret.com, got %+v", m)
}
}
cacheSize1 := db.GetActualCacheSize()
if cacheSize1 < 1 {
t.Errorf("Expected cache to have at least 1 entry after first read, got %d", cacheSize1)
}
// Второе чтение — из кэша (должно вернуть те же данные без повторного чтения файла)
found2, err := db.ReadByFilter(filter, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter (2nd, cached) failed: %v", err)
}
if len(found2) != 1 {
t.Fatalf("Expected 1 record on second read, got %d", len(found2))
}
if m, ok := found2[0].(map[string]any); ok {
if m["name"] != "bob" || m["email"] != "bob@secret.com" {
t.Errorf("Expected name=bob email=bob@secret.com on cached read, got %+v", m)
}
}
// Кэш не должен расти при повторном запросе с тем же ключом
cacheSize2 := db.GetActualCacheSize()
if cacheSize2 != cacheSize1 {
t.Errorf("Cache size should stay same after cache hit: was %d, now %d", cacheSize1, cacheSize2)
}
// Обновляем запись (name) и проверяем, что кэш и индекс обновились
_, err = db.Update(
map[string]any{"name": "bob_updated"},
"users",
map[string]any{"email": "bob@secret.com"},
opts,
)
if err != nil {
t.Fatalf("Update failed: %v", err)
}
// После Update кэш сбрасывается — читаем снова, чтобы заполнить кэш актуальными данными
found3, err := db.ReadByFilter(filter, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter after update failed: %v", err)
}
if len(found3) != 1 {
t.Fatalf("Expected 1 record after update, got %d", len(found3))
}
if m, ok := found3[0].(map[string]any); ok && m["name"] != "bob_updated" {
t.Errorf("Expected name=bob_updated after update, got %v", m["name"])
}
// Проверяем сырой кэш: в нём должна быть запись с name=bob_updated
rawCache := db.GetCacheForTest("give_me_cache")
if len(rawCache) == 0 {
t.Error("Expected cache to have entries after read")
}
var foundInCache bool
for _, v := range rawCache {
arr, ok := v.([]any)
if !ok || len(arr) != 1 {
continue
}
m, ok := arr[0].(map[string]any)
if !ok {
continue
}
if m["email"] == "bob@secret.com" {
foundInCache = true
if m["name"] != "bob_updated" {
t.Errorf("Expected cached name=bob_updated, got %v", m["name"])
}
break
}
}
if !foundInCache {
t.Error("Expected to find bob record in raw cache with updated name")
}
// Проверяем индекс: по email bob@secret.com должна быть одна строка, по name bob_updated — тоже
idxSnapshot := db.GetIndexSnapshotForTest("give_me_cache")
if len(idxSnapshot) == 0 {
t.Error("Expected index snapshot to have entries")
}
if emailIdx, ok := idxSnapshot["users:email"].(map[string][]int); ok {
if lines, ok := emailIdx["bob@secret.com"]; !ok || len(lines) != 1 {
t.Errorf("Expected index users:email bob@secret.com to have 1 line, got %v", emailIdx["bob@secret.com"])
}
}
if nameIdx, ok := idxSnapshot["users:name"].(map[string][]int); ok {
if lines, ok := nameIdx["bob_updated"]; !ok || len(lines) != 1 {
t.Errorf("Expected index users:name bob_updated to have 1 line, got %v", nameIdx["bob_updated"])
}
}
// Удаляем запись и проверяем, что в кэше и в индексе её больше нет
_, err = db.Delete(map[string]any{"email": "bob@secret.com"}, "users", opts)
if err != nil {
t.Fatalf("Delete failed: %v", err)
}
// После Delete кэш сбрасывается — читаем снова
found4, err := db.ReadByFilter(filter, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter after delete failed: %v", err)
}
if len(found4) != 0 {
t.Fatalf("Expected 0 records after delete, got %d", len(found4))
}
// В сыром кэше не должно остаться записи bob
rawCache2 := db.GetCacheForTest("give_me_cache")
for _, v := range rawCache2 {
arr, ok := v.([]any)
if !ok || len(arr) != 1 {
continue
}
if m, ok := arr[0].(map[string]any); ok && m["email"] == "bob@secret.com" {
t.Error("Cached result after delete should not contain bob record")
}
}
// Индекс: bob@secret.com и bob_updated не должны быть в индексе (или пустые срезы)
idxSnapshot2 := db.GetIndexSnapshotForTest("give_me_cache")
if emailIdx, ok := idxSnapshot2["users:email"].(map[string][]int); ok {
if lines, has := emailIdx["bob@secret.com"]; has && len(lines) > 0 {
t.Errorf("After delete, index users:email bob@secret.com should be empty, got %v", lines)
}
}
if nameIdx, ok := idxSnapshot2["users:name"].(map[string][]int); ok {
if lines, has := nameIdx["bob_updated"]; has && len(lines) > 0 {
t.Errorf("After delete, index users:name bob_updated should be empty, got %v", lines)
}
}
}
func TestIndexExplicitInMemory(t *testing.T) {
store := linedb.NewInMemoryIndexStore()
db, cleanup := setupIndexedCollection(t, store)
defer cleanup()
opts := linedb.LineDbAdapterOptions{}
if err := db.Insert(map[string]any{"name": "x", "email": "x@y.com"}, "users", opts); err != nil {
t.Fatalf("Insert failed: %v", err)
}
found, err := db.ReadByFilter(map[string]any{"email": "x@y.com"}, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter failed: %v", err)
}
if len(found) != 1 {
t.Fatalf("Expected 1 record, got %d", len(found))
}
}
func TestIndexMemcachedStore(t *testing.T) {
mock := newMockMemcached()
store, err := linedb.NewMemcachedIndexStore(linedb.MemcachedIndexStoreOptions{
Client: mock,
})
if err != nil {
t.Fatalf("NewMemcachedIndexStore: %v", err)
}
db, cleanup := setupIndexedCollection(t, store)
defer cleanup()
opts := linedb.LineDbAdapterOptions{}
if err := db.Insert(map[string]any{"name": "mem", "email": "mem@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert failed: %v", err)
}
found, err := db.ReadByFilter(map[string]any{"email": "mem@test.com"}, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter failed: %v", err)
}
if len(found) != 1 {
t.Fatalf("Expected 1 record, got %d", len(found))
}
// Проверяем, что в mockMemcached что-то записалось
if len(mock.data) == 0 {
t.Error("Expected memcached store to have data")
}
}
func TestIndexUpdateRebuild(t *testing.T) {
db, cleanup := setupIndexedCollection(t, nil)
defer cleanup()
opts := linedb.LineDbAdapterOptions{}
if err := db.Insert(map[string]any{"name": "a", "email": "a@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert failed: %v", err)
}
_, err := db.Update(
map[string]any{"email": "a_updated@test.com"},
"users",
map[string]any{"email": "a@test.com"},
opts,
)
if err != nil {
t.Fatalf("Update failed: %v", err)
}
// Старый email не должен находиться
old, _ := db.ReadByFilter(map[string]any{"email": "a@test.com"}, "users", opts)
if len(old) != 0 {
t.Fatalf("Expected 0 records for old email, got %d", len(old))
}
// Новый — должен
newFound, _ := db.ReadByFilter(map[string]any{"email": "a_updated@test.com"}, "users", opts)
if len(newFound) != 1 {
t.Fatalf("Expected 1 record for new email, got %d", len(newFound))
}
}
func TestIndexDeleteRebuild(t *testing.T) {
db, cleanup := setupIndexedCollection(t, nil)
defer cleanup()
opts := linedb.LineDbAdapterOptions{}
if err := db.Insert(map[string]any{"name": "del", "email": "del@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert failed: %v", err)
}
if err := db.Insert(map[string]any{"name": "keep", "email": "keep@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 2 failed: %v", err)
}
_, err := db.Delete(map[string]any{"email": "del@test.com"}, "users", opts)
if err != nil {
t.Fatalf("Delete failed: %v", err)
}
found, _ := db.ReadByFilter(map[string]any{"email": "del@test.com"}, "users", opts)
if len(found) != 0 {
t.Fatalf("Expected 0 after delete, got %d", len(found))
}
kept, _ := db.ReadByFilter(map[string]any{"email": "keep@test.com"}, "users", opts)
if len(kept) != 1 {
t.Fatalf("Expected 1 kept record, got %d", len(kept))
}
}
func TestIndexNoIndexedFields(t *testing.T) {
os.RemoveAll("./testdata")
initOptions := &linedb.LineDbInitOptions{
CacheSize: 100,
CacheTTL: time.Minute,
DBFolder: "./testdata",
Collections: []linedb.JSONLFileOptions{
{
CollectionName: "users",
AllocSize: 256,
IndexedFields: nil, // без индексов
},
},
}
db := linedb.NewLineDb(nil)
if err := db.Init(false, initOptions); err != nil {
t.Fatalf("Init failed: %v", err)
}
defer func() { db.Close(); os.RemoveAll("./testdata") }()
if err := db.Insert(map[string]any{"name": "a", "email": "a@test.com"}, "users", linedb.LineDbAdapterOptions{}); err != nil {
t.Fatalf("Insert failed: %v", err)
}
// ReadByFilter всё равно работает (полный скан)
found, err := db.ReadByFilter(map[string]any{"email": "a@test.com"}, "users", linedb.LineDbAdapterOptions{})
if err != nil {
t.Fatalf("ReadByFilter failed: %v", err)
}
if len(found) != 1 {
t.Fatalf("Expected 1 record, got %d", len(found))
}
}

139
tests/linedb_point_update_test.go Normal file
View File

@@ -0,0 +1,139 @@
// Тесты для дебага точечного Update через индекс.
// Случай 1: изменение поля, не входящего в индекс (поиск по email, обновляем name).
// Случай 2: изменение поля, входящего в индекс (поиск по email, обновляем email).
package tests
import (
"os"
"testing"
"time"
"linedb/pkg/linedb"
)
func setupPointUpdateDB(t *testing.T) (*linedb.LineDb, func()) {
t.Helper()
dir := "./data/test-point-update"
os.RemoveAll(dir)
initOptions := &linedb.LineDbInitOptions{
CacheSize: 100,
CacheTTL: time.Minute,
DBFolder: dir,
Collections: []linedb.JSONLFileOptions{
{
CollectionName: "users",
AllocSize: 256,
IndexedFields: []string{"id", "email", "name"},
},
},
}
db := linedb.NewLineDb(nil)
if err := db.Init(false, initOptions); err != nil {
t.Fatalf("Init failed: %v", err)
}
return db, func() { db.Close(); os.RemoveAll(dir) }
}
// TestPointUpdate_NonIndexedField: поиск по email (индекс), обновление name (не индекс).
func TestPointUpdate_NonIndexedField(t *testing.T) {
// db, cleanup := setupPointUpdateDB(t)
db, _ := setupPointUpdateDB(t)
// defer cleanup()
opts := linedb.LineDbAdapterOptions{}
if err := db.Insert(map[string]any{"id": 1, "name": "alice", "email": "alice@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 1 failed: %v", err)
}
if err := db.Insert(map[string]any{"id": 2, "name": "bob", "email": "bob@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 2 failed: %v", err)
}
if err := db.Insert(map[string]any{"id": 3, "name": "sam", "email": "sam@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 2 failed: %v", err)
}
if err := db.Insert(map[string]any{"id": 4, "name": "chuck", "email": "chuck@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 2 failed: %v", err)
}
// Поиск по email (индекс), обновляем name (не индекс)
updated, err := db.Update(
map[string]any{"name": "Bobina"},
"users",
map[string]any{"email": "bob@test.com"},
opts,
)
if err != nil {
t.Fatalf("Update failed: %v", err)
}
if len(updated) != 1 {
t.Fatalf("Expected 1 updated record, got %d", len(updated))
}
m := updated[0].(map[string]any)
if m["name"] != "Bobina" || m["email"] != "bob@test.com" {
t.Errorf("Expected name=Bobina email=bob@test.com, got name=%v email=%v", m["name"], m["email"])
}
// Проверяем ReadByFilter по старому email — должен найти обновлённую запись
found, err := db.ReadByFilter(map[string]any{"email": "bob@test.com"}, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter failed: %v", err)
}
if len(found) != 1 {
t.Fatalf("Expected 1 record by email bob@test.com, got %d", len(found))
}
if m2 := found[0].(map[string]any); m2["name"] != "Bobina" {
t.Errorf("Expected name=bob_renamed, got %v", m2["name"])
}
}
// TestPointUpdate_IndexedField: поиск по email (индекс), обновление email.
func TestPointUpdate_IndexedField(t *testing.T) {
db, cleanup := setupPointUpdateDB(t)
defer cleanup()
opts := linedb.LineDbAdapterOptions{}
if err := db.Insert(map[string]any{"name": "alice", "email": "alice@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 1 failed: %v", err)
}
if err := db.Insert(map[string]any{"name": "bob", "email": "bob@test.com"}, "users", opts); err != nil {
t.Fatalf("Insert 2 failed: %v", err)
}
// Поиск по email, обновляем тот же email
updated, err := db.Update(
map[string]any{"email": "bob_new@test.com"},
"users",
map[string]any{"email": "bob@test.com"},
opts,
)
if err != nil {
t.Fatalf("Update failed: %v", err)
}
if len(updated) != 1 {
t.Fatalf("Expected 1 updated record, got %d", len(updated))
}
m := updated[0].(map[string]any)
if m["email"] != "bob_new@test.com" || m["name"] != "bob" {
t.Errorf("Expected email=bob_new@test.com name=bob, got email=%v name=%v", m["email"], m["name"])
}
// По старому email записей быть не должно
foundOld, err := db.ReadByFilter(map[string]any{"email": "bob@test.com"}, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter old failed: %v", err)
}
if len(foundOld) != 0 {
t.Errorf("Expected 0 records for old email bob@test.com, got %d", len(foundOld))
}
// По новому email — одна запись
foundNew, err := db.ReadByFilter(map[string]any{"email": "bob_new@test.com"}, "users", opts)
if err != nil {
t.Fatalf("ReadByFilter new failed: %v", err)
}
if len(foundNew) != 1 {
t.Fatalf("Expected 1 record for new email bob_new@test.com, got %d", len(foundNew))
}
if m2 := foundNew[0].(map[string]any); m2["name"] != "bob" {
t.Errorf("Expected name=bob, got %v", m2["name"])
}
}