fixed bag with alloc size in partition

pkg/linedb/cache.go

@@ -1,6 +1,7 @@
 package linedb
 
 import (
+	"strings"
 	"sync"
 	"time"
 )
@@ -94,6 +95,37 @@ func (c *RecordCache) Clear() {
 	c.cache = make(map[string]*CacheEntry)
 }
 
+// ClearCollection удаляет записи кэша, относящиеся к коллекции collectionName:
+// ключи ReadByFilter ("collection:filter" и "collection_part:filter") и SetByRecord ("id:collection", "id:collection_part").
+func (c *RecordCache) ClearCollection(collectionName string) {
+	if collectionName == "" {
+		c.Clear()
+		return
+	}
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	for key := range c.cache {
+		if cacheKeyBelongsToCollection(key, collectionName) {
+			delete(c.cache, key)
+		}
+	}
+}
+
+func cacheKeyBelongsToCollection(key, collectionName string) bool {
+	if strings.HasPrefix(key, collectionName+":") {
+		return true
+	}
+	if strings.HasPrefix(key, collectionName+"_") {
+		return true
+	}
+	idx := strings.LastIndex(key, ":")
+	if idx < 0 {
+		return false
+	}
+	suf := key[idx+1:]
+	return suf == collectionName || strings.HasPrefix(suf, collectionName+"_")
+}
+
 // ClearEntriesContainingIDs удаляет из кэша только те записи, в данных которых
 // встречается хотя бы один из переданных id. Если ids пуст — ничего не делает.
 func (c *RecordCache) ClearEntriesContainingIDs(ids []any) {

pkg/linedb/line_db.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -87,6 +88,8 @@ func (db *LineDb) Init(force bool, initOptions *LineDbInitOptions) error {
 	if dbFolder == "" {
 		dbFolder = "linedb"
 	}
+	// Нормализуем и сохраняем фактический путь, чтобы партиции создавались в той же папке
+	db.initOptions.DBFolder = dbFolder
 
 	if err := os.MkdirAll(dbFolder, 0755); err != nil {
 		return fmt.Errorf("failed to create database folder: %w", err)
@@ -137,12 +140,10 @@ func (db *LineDb) Init(force bool, initOptions *LineDbInitOptions) error {
 				continue
 			}
 			if db.isCollectionPartitioned(opt.CollectionName) {
-				// Партиции: обходим все существующие партиции (могут быть созданы ранее)
+				// Основной файл + все base_* (пустой partition id → основной файл)
 				baseName := opt.CollectionName
-				for name, adapter := range db.adapters {
-					if !strings.HasPrefix(name, baseName+"_") {
-						continue
-					}
+				for _, adapter := range db.partitionStorageAdaptersOrdered(baseName) {
+					name := adapter.GetCollectionName()
 					records, lineIdx, err := adapter.ReadWithPhysicalLineIndexes(LineDbAdapterOptions{})
 					if err != nil {
 						continue
@@ -230,7 +231,13 @@ func (db *LineDb) seedLastIDPartitioned(dbFolder, baseName string) error {
 		if existing, ok := db.adapters[partName]; ok {
 			adapter = existing
 		} else {
-			adapter = NewJSONLFile(path, "", JSONLFileOptions{CollectionName: partName})
+			// Партиции должны наследовать опции базовой коллекции (allocSize/encode/crypto/marshal/etc.)
+			opts := JSONLFileOptions{CollectionName: partName}
+			if baseOpts := db.getCollectionOptions(baseName); baseOpts != nil {
+				opts = *baseOpts
+				opts.CollectionName = partName
+			}
+			adapter = NewJSONLFile(path, opts.EncryptKeyForLineDb, opts)
 			if err := adapter.Init(false, LineDbAdapterOptions{}); err != nil {
 				continue
 			}
@@ -249,7 +256,9 @@ func (db *LineDb) seedLastIDPartitioned(dbFolder, baseName string) error {
 	return nil
 }
 
-// Read читает все записи из коллекции
+// Read читает все записи из коллекции.
+// Для партиционированной логической коллекции (имя без суффикса _*) читает основной файл
+// и все файлы партиций base_* подряд. Запрос по имени одной партиции (base_part) читает только её файл.
 func (db *LineDb) Read(collectionName string, options LineDbAdapterOptions) ([]any, error) {
 	db.mutex.RLock()
 	defer db.mutex.RUnlock()
@@ -258,6 +267,23 @@ func (db *LineDb) Read(collectionName string, options LineDbAdapterOptions) ([]a
 		collectionName = db.getFirstCollection()
 	}
 
+	baseName := db.getBaseCollectionName(collectionName)
+	if db.isCollectionPartitioned(baseName) && collectionName == baseName {
+		adapters := db.partitionStorageAdaptersOrdered(baseName)
+		if len(adapters) == 0 {
+			return nil, fmt.Errorf("collection %s not found", collectionName)
+		}
+		var all []any
+		for _, a := range adapters {
+			recs, err := a.Read(options)
+			if err != nil {
+				return nil, err
+			}
+			all = append(all, recs...)
+		}
+		return all, nil
+	}
+
 	adapter, exists := db.adapters[collectionName]
 	if !exists {
 		return nil, fmt.Errorf("collection %s not found", collectionName)
@@ -374,12 +400,7 @@ func (db *LineDb) Insert(data any, collectionName string, options LineDbAdapterO
 		return fmt.Errorf("failed to write data: %w", err)
 	}
 
-	// Обновляем кэш
-	if db.cacheExternal != nil {
-		for _, item := range resultDataArray {
-			db.cacheExternal.UpdateCacheAfterInsert(item, collectionName)
-		}
-	}
+	db.clearCacheForCollection(collectionName)
 
 	return nil
 }
@@ -512,10 +533,7 @@ func (db *LineDb) Update(data any, collectionName string, filter any, options Li
 			return nil, err
 		}
 		if used {
-			if db.cacheExternal != nil {
-				ids := extractIDsFromRecords(result)
-				db.cacheExternal.ClearEntriesContainingIDs(ids)
-			}
+			db.clearCacheForCollection(collectionName)
 			return result, nil
 		}
 	}
@@ -534,10 +552,7 @@ func (db *LineDb) Update(data any, collectionName string, filter any, options Li
 			}
 		}
 	}
-	if db.cacheExternal != nil {
-		ids := extractIDsFromRecords(result)
-		db.cacheExternal.ClearEntriesContainingIDs(ids)
-	}
+	db.clearCacheForCollection(collectionName)
 	return result, nil
 }
 
@@ -578,10 +593,7 @@ func (db *LineDb) Delete(data any, collectionName string, options LineDbAdapterO
 			return nil, err
 		}
 		if used {
-			if db.cacheExternal != nil {
-				ids := extractIDsFromRecords(result)
-				db.cacheExternal.ClearEntriesContainingIDs(ids)
-			}
+			db.clearCacheForCollection(collectionName)
 			return result, nil
 		}
 	}
@@ -600,10 +612,7 @@ func (db *LineDb) Delete(data any, collectionName string, options LineDbAdapterO
 			}
 		}
 	}
-	if db.cacheExternal != nil {
-		ids := extractIDsFromRecords(result)
-		db.cacheExternal.ClearEntriesContainingIDs(ids)
-	}
+	db.clearCacheForCollection(collectionName)
 	return result, nil
 }
 
@@ -654,7 +663,7 @@ func (db *LineDb) ReadByFilter(filter any, collectionName string, options LineDb
 			if err != nil || (options.FailOnFailureIndexRead && !hit) {
 				return nil, fmt.Errorf("index read failed: %w", err)
 			}
-			if hit && err == nil {
+			if hit {
 				if db.cacheExternal != nil && !options.InTransaction {
 					db.cacheExternal.Set(db.generateCacheKey(filter, collectionName), result)
 				}
@@ -685,7 +694,7 @@ func (db *LineDb) ReadByFilter(filter any, collectionName string, options LineDb
 		if err != nil || (options.FailOnFailureIndexRead && !hit) {
 			return nil, fmt.Errorf("index read failed: %w", err)
 		}
-		if hit && err == nil {
+		if hit {
 			if db.cacheExternal != nil && !options.InTransaction {
 				db.cacheExternal.Set(db.generateCacheKey(filter, collectionName), result)
 			}
@@ -740,15 +749,21 @@ func (db *LineDb) ClearCache(collectionName string, options LineDbAdapterOptions
 		if collectionName == "" {
 			db.cacheExternal.Clear()
 		} else {
-			// Очищаем только записи для конкретной коллекции
-			// Это упрощенная реализация
-			db.cacheExternal.Clear()
+			db.cacheExternal.ClearCollection(collectionName)
 		}
 	}
 
 	return nil
 }
 
+// clearCacheForCollection сбрасывает кэш запросов по логической коллекции (включая ключи партиций base_*).
+func (db *LineDb) clearCacheForCollection(collectionName string) {
+	if db.cacheExternal == nil || collectionName == "" {
+		return
+	}
+	db.cacheExternal.ClearCollection(collectionName)
+}
+
 // Close закрывает базу данных
 func (db *LineDb) Close() {
 	db.mutex.Lock()
@@ -798,10 +813,8 @@ func (db *LineDb) indexRebuildTimerLoop(interval time.Duration) {
 					}
 					if db.isCollectionPartitioned(opt.CollectionName) {
 						baseName := opt.CollectionName
-						for name, adapter := range db.adapters {
-							if !strings.HasPrefix(name, baseName+"_") {
-								continue
-							}
+						for _, adapter := range db.partitionStorageAdaptersOrdered(baseName) {
+							name := adapter.GetCollectionName()
 							dirty, err := adapter.HasBlankSlots(LineDbAdapterOptions{})
 							if err != nil {
 								continue
@@ -1143,7 +1156,7 @@ func (db *LineDb) getCollectionOptions(collectionName string) *JSONLFileOptions
 }
 
 // isValueEmpty проверяет, считается ли значение "пустым" (nil или "" для string) — для обратной совместимости
-func (db *LineDb) isValueEmpty(v any) bool {
+func (db *LineDb) IsValueEmpty(v any) bool {
 	if v == nil {
 		return true
 	}
@@ -1336,17 +1349,28 @@ func (db *LineDb) isCollectionPartitioned(collectionName string) bool {
 	return exists
 }
 
-func (db *LineDb) getPartitionFiles(collectionName string) ([]string, error) {
-	baseName := db.getBaseCollectionName(collectionName)
-	var files []string
-
-	for name, filename := range db.collections {
+// partitionStorageAdaptersOrdered — для логического имени партиционированной коллекции baseName:
+// сначала адаптер основного файла (baseName.jsonl), затем все baseName_* в лексикографическом порядке.
+// Пустой ключ партиции пишет в основной файл; эти записи учитываются здесь первым адаптером.
+func (db *LineDb) partitionStorageAdaptersOrdered(baseName string) []*JSONLFile {
+	var out []*JSONLFile
+	if a := db.adapters[baseName]; a != nil {
+		out = append(out, a)
+	}
+	var partNames []string
+	for name := range db.adapters {
+		if name == baseName {
+			continue
+		}
 		if strings.HasPrefix(name, baseName+"_") {
-			files = append(files, filename)
+			partNames = append(partNames, name)
 		}
 	}
-
-	return files, nil
+	sort.Strings(partNames)
+	for _, name := range partNames {
+		out = append(out, db.adapters[name])
+	}
+	return out
 }
 
 func (db *LineDb) getPartitionAdapter(data any, collectionName string) (*JSONLFile, error) {
@@ -1356,13 +1380,27 @@ func (db *LineDb) getPartitionAdapter(data any, collectionName string) (*JSONLFi
 	}
 
 	partitionID := partitionFn(data)
+	if strings.TrimSpace(partitionID) == "" {
+		adapter, ok := db.adapters[collectionName]
+		if !ok {
+			return nil, fmt.Errorf("collection %s not found", collectionName)
+		}
+		return adapter, nil
+	}
+
 	partitionName := fmt.Sprintf("%s_%s", collectionName, partitionID)
 
 	adapter, exists := db.adapters[partitionName]
 	if !exists {
 		// Создаем новый адаптер для партиции
 		filename := filepath.Join(db.initOptions.DBFolder, partitionName+".jsonl")
-		adapter = NewJSONLFile(filename, "", JSONLFileOptions{CollectionName: partitionName})
+		// Партиции должны наследовать опции базовой коллекции (allocSize/encode/crypto/marshal/etc.)
+		opts := JSONLFileOptions{CollectionName: partitionName}
+		if baseOpts := db.getCollectionOptions(collectionName); baseOpts != nil {
+			opts = *baseOpts
+			opts.CollectionName = partitionName
+		}
+		adapter = NewJSONLFile(filename, opts.EncryptKeyForLineDb, opts)
 
 		if err := adapter.Init(false, LineDbAdapterOptions{}); err != nil {
 			return nil, fmt.Errorf("failed to init partition adapter: %w", err)
@@ -1526,24 +1564,6 @@ func (db *LineDb) compareIDs(a, b any) bool {
 	return a == b
 }
 
-// extractIDsFromRecords извлекает id из списка записей (map[string]any).
-func extractIDsFromRecords(records []any) []any {
-	if len(records) == 0 {
-		return nil
-	}
-	ids := make([]any, 0, len(records))
-	for _, rec := range records {
-		m, ok := rec.(map[string]any)
-		if !ok {
-			continue
-		}
-		if id, exists := m["id"]; exists && id != nil {
-			ids = append(ids, id)
-		}
-	}
-	return ids
-}
-
 func (db *LineDb) generateCacheKey(filter any, collectionName string) string {
 	// Упрощенная реализация генерации ключа кэша
 	return fmt.Sprintf("%s:%v", collectionName, filter)
@@ -1609,130 +1629,87 @@ func (db *LineDb) normalizeFilter(filter any) (any, error) {
 }
 
 func (db *LineDb) updatePartitioned(data any, collectionName string, filter any, options LineDbAdapterOptions) ([]any, error) {
-	partitionFiles, err := db.getPartitionFiles(collectionName)
-	if err != nil {
-		return nil, err
-	}
 	dataMap, err := db.toMap(data)
 	if err != nil {
 		return nil, fmt.Errorf("invalid update data format: %w", err)
 	}
 	opts := db.getCollectionOptions(collectionName)
+	baseName := db.getBaseCollectionName(collectionName)
 
 	var allResults []any
-	for _, filename := range partitionFiles {
-		var adapter *JSONLFile
-		var partitionName string
-		for name, adapterFile := range db.collections {
-			if adapterFile == filename {
-				adapter = db.adapters[name]
-				partitionName = name
-				break
+	for _, adapter := range db.partitionStorageAdaptersOrdered(baseName) {
+		partitionName := adapter.GetCollectionName()
+		if db.indexStore != nil && opts != nil && len(opts.IndexedFields) > 0 {
+			upd, used, terr := db.tryIndexUpdate(adapter, filter, dataMap, collectionName, partitionName, opts.IndexedFields, options)
+			if terr != nil {
+				return nil, terr
+			}
+			if used {
+				allResults = append(allResults, upd...)
+				continue
 			}
 		}
-
-		if adapter != nil {
-			if db.indexStore != nil && opts != nil && len(opts.IndexedFields) > 0 {
-				upd, used, terr := db.tryIndexUpdate(adapter, filter, dataMap, collectionName, partitionName, opts.IndexedFields, options)
-				if terr != nil {
-					return nil, terr
-				}
-				if used {
-					allResults = append(allResults, upd...)
-					continue
-				}
-			}
-			results, err := adapter.Update(data, filter, options)
-			if err != nil {
-				return nil, err
-			}
-			allResults = append(allResults, results...)
-			if db.indexStore != nil && opts != nil && len(opts.IndexedFields) > 0 {
-				records, lineIdx, readErr := adapter.ReadWithPhysicalLineIndexes(LineDbAdapterOptions{InTransaction: true})
-				if readErr == nil {
-					_ = db.indexStore.Rebuild(collectionName, partitionName, opts.IndexedFields, records, lineIdx)
-				}
+		results, err := adapter.Update(data, filter, options)
+		if err != nil {
+			return nil, err
+		}
+		allResults = append(allResults, results...)
+		if db.indexStore != nil && opts != nil && len(opts.IndexedFields) > 0 {
+			records, lineIdx, readErr := adapter.ReadWithPhysicalLineIndexes(LineDbAdapterOptions{InTransaction: true})
+			if readErr == nil {
+				_ = db.indexStore.Rebuild(collectionName, partitionName, opts.IndexedFields, records, lineIdx)
 			}
 		}
 	}
 
+	db.clearCacheForCollection(collectionName)
 	return allResults, nil
 }
 
 func (db *LineDb) deletePartitioned(data any, collectionName string, options LineDbAdapterOptions) ([]any, error) {
-	partitionFiles, err := db.getPartitionFiles(collectionName)
-	if err != nil {
-		return nil, err
-	}
 	opts := db.getCollectionOptions(collectionName)
+	baseName := db.getBaseCollectionName(collectionName)
 
 	var allResults []any
-	for _, filename := range partitionFiles {
-		var adapter *JSONLFile
-		var partitionName string
-		for name, adapterFile := range db.collections {
-			if adapterFile == filename {
-				adapter = db.adapters[name]
-				partitionName = name
-				break
+	for _, adapter := range db.partitionStorageAdaptersOrdered(baseName) {
+		partitionName := adapter.GetCollectionName()
+		if db.indexStore != nil && opts != nil && len(opts.IndexedFields) > 0 {
+			del, used, derr := db.tryIndexDelete(adapter, data, collectionName, partitionName, opts.IndexedFields, options)
+			if derr != nil {
+				return nil, derr
+			}
+			if used {
+				allResults = append(allResults, del...)
+				continue
 			}
 		}
-
-		if adapter != nil {
-			if db.indexStore != nil && opts != nil && len(opts.IndexedFields) > 0 {
-				del, used, derr := db.tryIndexDelete(adapter, data, collectionName, partitionName, opts.IndexedFields, options)
-				if derr != nil {
-					return nil, derr
-				}
-				if used {
-					allResults = append(allResults, del...)
-					continue
-				}
-			}
-			results, err := adapter.Delete(data, options)
-			if err != nil {
-				return nil, err
-			}
-			allResults = append(allResults, results...)
-			if db.indexStore != nil && opts != nil && len(opts.IndexedFields) > 0 {
-				records, lineIdx, readErr := adapter.ReadWithPhysicalLineIndexes(LineDbAdapterOptions{InTransaction: true})
-				if readErr == nil {
-					_ = db.indexStore.Rebuild(collectionName, partitionName, opts.IndexedFields, records, lineIdx)
-				}
+		results, err := adapter.Delete(data, options)
+		if err != nil {
+			return nil, err
+		}
+		allResults = append(allResults, results...)
+		if db.indexStore != nil && opts != nil && len(opts.IndexedFields) > 0 {
+			records, lineIdx, readErr := adapter.ReadWithPhysicalLineIndexes(LineDbAdapterOptions{InTransaction: true})
+			if readErr == nil {
+				_ = db.indexStore.Rebuild(collectionName, partitionName, opts.IndexedFields, records, lineIdx)
 			}
 		}
 	}
 
+	db.clearCacheForCollection(collectionName)
 	return allResults, nil
 }
 
 func (db *LineDb) readByFilterPartitioned(filter any, collectionName string, options LineDbAdapterOptions) ([]any, error) {
-	// Получаем все партиции
-	partitionFiles, err := db.getPartitionFiles(collectionName)
-	if err != nil {
-		return nil, err
-	}
-
+	baseName := db.getBaseCollectionName(collectionName)
 	var allResults []any
-	for _, filename := range partitionFiles {
-		// Находим адаптер по имени файла
-		var adapter *JSONLFile
-		for name, adapterFile := range db.collections {
-			if adapterFile == filename {
-				adapter = db.adapters[name]
-				break
-			}
-		}
-
-		if adapter != nil {
-			results, err := adapter.ReadByFilter(filter, options)
-			if err != nil {
-				return nil, err
-			}
-			allResults = append(allResults, results...)
+	for _, adapter := range db.partitionStorageAdaptersOrdered(baseName) {
+		results, err := adapter.ReadByFilter(filter, options)
+		if err != nil {
+			return nil, err
 		}
+		allResults = append(allResults, results...)
 	}
-
 	return allResults, nil
 }