From a1bbd9177a4d9588db1ff1b09af016158400782e Mon Sep 17 00:00:00 2001 From: "direct-dev.ru" Date: Thu, 9 Apr 2026 14:08:27 +0600 Subject: [PATCH] change module name 3 --- examples/custom-json/custom_json_test.go | 2 +- examples/delete/delete.go | 2 +- examples/encode/main.go | 2 +- examples/perf/main.go | 2 +- examples/test-alloc-overflow/main.go | 2 +- pkg/linedb/last_id_manager.go | 45 +++------ pkg/linedb/line_db.go | 123 ++++++++++++++++++++++- tests/linedb_point_update_test.go | 3 +- 8 files changed, 138 insertions(+), 43 deletions(-) diff --git a/examples/custom-json/custom_json_test.go b/examples/custom-json/custom_json_test.go index 84d3c98..5f978fb 100644 --- a/examples/custom-json/custom_json_test.go +++ b/examples/custom-json/custom_json_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "direct-dev.ru/gitea/GiteaAdmin/elowdb-go" + "direct-dev.ru/gitea/GiteaAdmin/elowdb-go/pkg/linedb" ) func TestCustomJSONSerialization(t *testing.T) { diff --git a/examples/delete/delete.go b/examples/delete/delete.go index c6d66aa..84ae523 100644 --- a/examples/delete/delete.go +++ b/examples/delete/delete.go @@ -5,7 +5,7 @@ import ( "log" "time" - "direct-dev.ru/gitea/GiteaAdmin/elowdb-go" + "direct-dev.ru/gitea/GiteaAdmin/elowdb-go/pkg/linedb" ) // testDeleteOneRecordByID тестирует удаление одной записи по ID diff --git a/examples/encode/main.go b/examples/encode/main.go index 8c1611b..28e1e49 100644 --- a/examples/encode/main.go +++ b/examples/encode/main.go @@ -7,7 +7,7 @@ import ( "os" "time" - "direct-dev.ru/gitea/GiteaAdmin/elowdb-go" + "direct-dev.ru/gitea/GiteaAdmin/elowdb-go/pkg/linedb" ) func main() { diff --git a/examples/perf/main.go b/examples/perf/main.go index 7ad3fa6..ddcde52 100644 --- a/examples/perf/main.go +++ b/examples/perf/main.go @@ -20,7 +20,7 @@ import ( "strings" "time" - "direct-dev.ru/gitea/GiteaAdmin/elowdb-go" + "direct-dev.ru/gitea/GiteaAdmin/elowdb-go/pkg/linedb" ) const ( diff --git a/examples/test-alloc-overflow/main.go b/examples/test-alloc-overflow/main.go index b964163..6c3be41 100644 --- a/examples/test-alloc-overflow/main.go +++ b/examples/test-alloc-overflow/main.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "direct-dev.ru/gitea/GiteaAdmin/elowdb-go" + "direct-dev.ru/gitea/GiteaAdmin/elowdb-go/pkg/linedb" ) func main() { diff --git a/pkg/linedb/last_id_manager.go b/pkg/linedb/last_id_manager.go index d93ed29..4aa0e55 100644 --- a/pkg/linedb/last_id_manager.go +++ b/pkg/linedb/last_id_manager.go @@ -4,7 +4,9 @@ import ( "sync" ) -// LastIDManager управляет последними ID для коллекций +// LastIDManager управляет последними ID для коллекций. +// Ключ — полное имя коллекции (как в NextID/Insert), без обрезки по «_»: +// иначе user_data и events_A ломались бы на первый сегмент. type LastIDManager struct { lastIDs map[string]int mutex sync.RWMutex @@ -23,53 +25,32 @@ func GetLastIDManagerInstance() *LastIDManager { return lastIDManagerInstance } -// GetLastID получает последний ID для коллекции -func (l *LastIDManager) GetLastID(filename string) int { +// GetLastID получает последний ID для коллекции (ключ — полное имя коллекции). +func (l *LastIDManager) GetLastID(collectionKey string) int { l.mutex.RLock() defer l.mutex.RUnlock() - baseFileName := l.getBaseFileName(filename) - return l.lastIDs[baseFileName] + return l.lastIDs[collectionKey] } -// SetLastID устанавливает последний ID для коллекции -func (l *LastIDManager) SetLastID(filename string, id int) { +// SetLastID устанавливает последний ID для коллекции, если id больше текущего. +func (l *LastIDManager) SetLastID(collectionKey string, id int) { l.mutex.Lock() defer l.mutex.Unlock() - baseFileName := l.getBaseFileName(filename) - currentID := l.lastIDs[baseFileName] + currentID := l.lastIDs[collectionKey] if currentID < id { - l.lastIDs[baseFileName] = id + l.lastIDs[collectionKey] = id } } // IncrementLastID увеличивает последний ID для коллекции -func (l *LastIDManager) IncrementLastID(filename string) int { +func (l *LastIDManager) IncrementLastID(collectionKey string) int { l.mutex.Lock() defer l.mutex.Unlock() - baseFileName := l.getBaseFileName(filename) - currentID := l.lastIDs[baseFileName] + currentID := l.lastIDs[collectionKey] newID := currentID + 1 - l.lastIDs[baseFileName] = newID + l.lastIDs[collectionKey] = newID return newID } - -// getBaseFileName извлекает базовое имя файла -func (l *LastIDManager) getBaseFileName(filename string) string { - if idx := l.findPartitionSeparator(filename); idx != -1 { - return filename[:idx] - } - return filename -} - -// findPartitionSeparator находит разделитель партиции -func (l *LastIDManager) findPartitionSeparator(filename string) int { - for i, char := range filename { - if char == '_' { - return i - } - } - return -1 -} diff --git a/pkg/linedb/line_db.go b/pkg/linedb/line_db.go index 17d0960..2fd00fd 100644 --- a/pkg/linedb/line_db.go +++ b/pkg/linedb/line_db.go @@ -171,6 +171,81 @@ func (db *LineDb) Init(force bool, initOptions *LineDbInitOptions) error { go db.indexRebuildTimerLoop(interval) } + if err := db.seedLastIDsFromData(dbFolder); err != nil { + return err + } + + return nil +} + +// seedLastIDsFromData выставляет LastIDManager по максимальному id в существующих данных +// (после рестарта процесса счётчик не начинается с 1). +func (db *LineDb) seedLastIDsFromData(dbFolder string) error { + for i, opt := range db.initOptions.Collections { + collectionName := opt.CollectionName + if collectionName == "" { + collectionName = fmt.Sprintf("collection_%d", i+1) + } + if db.isCollectionPartitioned(collectionName) { + if err := db.seedLastIDPartitioned(dbFolder, collectionName); err != nil { + return err + } + continue + } + adapter := db.adapters[collectionName] + if adapter == nil { + continue + } + records, err := adapter.Read(LineDbAdapterOptions{}) + if err != nil { + return fmt.Errorf("seed last id: read %s: %w", collectionName, err) + } + db.lastIDManager.SetLastID(collectionName, db.GetMaxID(records)) + } + return nil +} + +// seedLastIDPartitioned — максимальный id по базовому файлу и всем events_*.jsonl на диске. +// NextID для партиций вызывается с логическим именем (например events), ключ тот же. +func (db *LineDb) seedLastIDPartitioned(dbFolder, baseName string) error { + maxID := 0 + if a := db.adapters[baseName]; a != nil { + recs, err := a.Read(LineDbAdapterOptions{}) + if err != nil { + return fmt.Errorf("seed last id: read base %s: %w", baseName, err) + } + if m := db.GetMaxID(recs); m > maxID { + maxID = m + } + } + pattern := filepath.Join(dbFolder, baseName+"_*.jsonl") + matches, err := filepath.Glob(pattern) + if err != nil { + return fmt.Errorf("seed last id: glob %s: %w", pattern, err) + } + for _, path := range matches { + base := filepath.Base(path) + partName := strings.TrimSuffix(base, ".jsonl") + var adapter *JSONLFile + if existing, ok := db.adapters[partName]; ok { + adapter = existing + } else { + adapter = NewJSONLFile(path, "", JSONLFileOptions{CollectionName: partName}) + if err := adapter.Init(false, LineDbAdapterOptions{}); err != nil { + continue + } + db.adapters[partName] = adapter + db.collections[partName] = path + } + recs, err := adapter.Read(LineDbAdapterOptions{}) + if err != nil { + continue + } + if m := db.GetMaxID(recs); m > maxID { + maxID = m + } + } + db.lastIDManager.SetLastID(baseName, maxID) return nil } @@ -1285,8 +1360,9 @@ func (db *LineDb) GetMaxID(records []any) int { for _, record := range records { if recordMap, ok := record.(map[string]any); ok { if id, ok := recordMap["id"]; ok { - if idInt, ok := id.(int); ok && idInt > maxID { - maxID = idInt + n := idToIntForMax(id) + if n > maxID { + maxID = n } } } @@ -1294,6 +1370,21 @@ func (db *LineDb) GetMaxID(records []any) int { return maxID } +func idToIntForMax(id any) int { + switch v := id.(type) { + case int: + return v + case int64: + return int(v) + case float64: + return int(v) + case float32: + return int(v) + default: + return 0 + } +} + func (db *LineDb) matchesFilter(record any, filter any, strictCompare bool) bool { if recordMap, ok := record.(map[string]any); ok { if filterMap, ok := filter.(map[string]any); ok { @@ -1383,10 +1474,32 @@ func (db *LineDb) isInvalidID(id any) bool { if id == nil { return true } - if idNum, ok := id.(int); ok { - return idNum <= -1 + switch v := id.(type) { + case int: + return v <= -1 + case int8: + return v <= -1 + case int16: + return v <= -1 + case int32: + return v <= -1 + case int64: + return v <= -1 + case uint, uint8, uint16, uint32, uint64: + return false + case float32: + if v != v { // NaN + return true + } + return float64(v) <= -1 + case float64: + if v != v { // NaN + return true + } + return v <= -1 + default: + return false } - return false } func (db *LineDb) compareIDs(a, b any) bool { diff --git a/tests/linedb_point_update_test.go b/tests/linedb_point_update_test.go index 0d80297..651e844 100644 --- a/tests/linedb_point_update_test.go +++ b/tests/linedb_point_update_test.go @@ -4,10 +4,11 @@ package tests import ( - "linedb/pkg/linedb" "os" "testing" "time" + + "direct-dev.ru/gitea/GiteaAdmin/elowdb-go/pkg/linedb" ) func setupPointUpdateDB(t *testing.T) (*linedb.LineDb, func()) {