329 lines
9.4 KiB
Go
329 lines
9.4 KiB
Go
// Пример комплексного performance-теста LineDb.
|
||
// Запуск:
|
||
//
|
||
// go run ./examples/perf/main.go
|
||
//
|
||
// Сценарий:
|
||
// 1. Вставка 5000 записей, каждая ~1800 символов.
|
||
// 2. N выборок ReadByFilter по индексируемому полю, замер среднего времени.
|
||
// 3. 100 случайных удалений, замер среднего времени.
|
||
// 4. Замеры памяти процесса через runtime.MemStats.
|
||
package main
|
||
|
||
import (
|
||
"fmt"
|
||
"log"
|
||
"math/rand"
|
||
"os"
|
||
"path/filepath"
|
||
"runtime"
|
||
"strings"
|
||
"time"
|
||
|
||
"direct-dev.ru/gitea/GiteaAdmin/elowdb-go"
|
||
)
|
||
|
||
const (
|
||
recordsCount = 3000
|
||
payloadSize = 1024
|
||
readOps = 2000
|
||
updateOps = 500
|
||
deleteOps = 500
|
||
collectionName = "perf_items"
|
||
baseDBDir = "./data/perf-benchmark"
|
||
allocSizeEstimate = 2048
|
||
)
|
||
|
||
func main() {
|
||
rand.Seed(time.Now().UnixNano())
|
||
|
||
if err := os.MkdirAll(baseDBDir, 0755); err != nil {
|
||
log.Fatalf("mkdir: %v", err)
|
||
}
|
||
|
||
fmt.Printf("LineDB perf сравнение: без индекса vs с индексом\n")
|
||
fmt.Printf("records=%d payload~%d readOps=%d updateOps=%d deleteOps=%d allocSize=%d\n\n",
|
||
recordsCount, payloadSize, readOps, updateOps, deleteOps, allocSizeEstimate)
|
||
|
||
noIdx := runScenario(false)
|
||
runtime.GC()
|
||
withIdx := runScenario(true)
|
||
|
||
fmt.Printf("\n=== Сводка (рядом) ===\n")
|
||
fmt.Printf("%-26s | %-18s | %-18s\n", "Метрика", "Без индекса", "С индексом")
|
||
fmt.Printf("%-26s | %-18v | %-18v\n", "Insert total", noIdx.insertTotal, withIdx.insertTotal)
|
||
fmt.Printf("%-26s | %-18v | %-18v\n", "Index build", noIdx.indexBuildTotal, withIdx.indexBuildTotal)
|
||
fmt.Printf("%-26s | %-18v | %-18v\n", "ReadByFilter avg", noIdx.readAvg, withIdx.readAvg)
|
||
fmt.Printf("%-26s | %-18v | %-18v\n", "Update avg", noIdx.updateAvg, withIdx.updateAvg)
|
||
fmt.Printf("%-26s | %-18v | %-18v\n", "Delete avg", noIdx.deleteAvg, withIdx.deleteAvg)
|
||
fmt.Printf("%-26s | %-18d | %-18d\n", "Final records", noIdx.finalCount, withIdx.finalCount)
|
||
fmt.Printf("%-26s | %-18.2f | %-18.2f\n", "Mem Alloc (MB)", noIdx.memAllocMB, withIdx.memAllocMB)
|
||
fmt.Printf("%-26s | %-18.2f | %-18.2f\n", "Mem Sys (MB)", noIdx.memSysMB, withIdx.memSysMB)
|
||
fmt.Printf("%-26s | %-18d | %-18d\n", "NumGC", noIdx.numGC, withIdx.numGC)
|
||
fmt.Printf("\nData directories:\n no-index: %s\n with-index:%s\n", noIdx.dbDir, withIdx.dbDir)
|
||
}
|
||
|
||
type scenarioResult struct {
|
||
dbDir string
|
||
insertTotal time.Duration
|
||
indexBuildTotal time.Duration
|
||
readAvg time.Duration
|
||
updateAvg time.Duration
|
||
deleteAvg time.Duration
|
||
finalCount int
|
||
memAllocMB float64
|
||
memSysMB float64
|
||
numGC uint32
|
||
}
|
||
|
||
func runScenario(useIndex bool) scenarioResult {
|
||
label := "no-index"
|
||
if useIndex {
|
||
label = "with-index"
|
||
}
|
||
dbDir := filepath.Join(baseDBDir, label)
|
||
_ = os.RemoveAll(dbDir)
|
||
|
||
var store linedb.IndexStore
|
||
lineOpts := linedb.LineDbOptions{}
|
||
collOpts := linedb.JSONLFileOptions{
|
||
CollectionName: collectionName,
|
||
AllocSize: allocSizeEstimate,
|
||
}
|
||
if useIndex {
|
||
// Индексируем поля id и index (по ним идут фильтры в тесте)
|
||
collOpts.IndexedFields = []string{"id", "index"}
|
||
store = linedb.NewInMemoryIndexStore()
|
||
lineOpts.IndexStore = store
|
||
}
|
||
|
||
initOptions := &linedb.LineDbInitOptions{
|
||
CacheSize: 1000,
|
||
CacheTTL: time.Minute,
|
||
DBFolder: dbDir,
|
||
Collections: []linedb.JSONLFileOptions{
|
||
collOpts,
|
||
},
|
||
}
|
||
|
||
db := linedb.NewLineDb(&lineOpts)
|
||
if err := db.Init(true, initOptions); err != nil {
|
||
log.Fatalf("[%s] Init failed: %v", label, err)
|
||
}
|
||
defer db.Close()
|
||
|
||
fmt.Printf("=== %s ===\n", label)
|
||
printMem("Before insert")
|
||
|
||
fmt.Printf("1) Insert %d records...\n", recordsCount)
|
||
start := time.Now()
|
||
if err := insertRecords(db); err != nil {
|
||
log.Fatalf("[%s] insertRecords failed: %v", label, err)
|
||
}
|
||
insertDur := time.Since(start)
|
||
fmt.Printf(" Total insert time: %v, per record: %v\n", insertDur, insertDur/time.Duration(recordsCount))
|
||
// Индекс строится внутри Write при DoIndexing: true (точечная индексация)
|
||
|
||
printMem("After insert")
|
||
|
||
all, err := db.Read(collectionName, linedb.LineDbAdapterOptions{})
|
||
if err != nil {
|
||
log.Fatalf("[%s] Read after insert failed: %v", label, err)
|
||
}
|
||
fmt.Printf(" Records in collection: %d\n", len(all))
|
||
ids := collectIDs(all)
|
||
indexVals := collectFieldValues(all, "index")
|
||
if len(ids) == 0 || len(indexVals) == 0 {
|
||
log.Fatalf("[%s] No IDs or index values collected, cannot continue", label)
|
||
}
|
||
|
||
fmt.Printf("\n2) Random ReadByFilter of %d ops (field=index)...\n", readOps)
|
||
readAvg := benchmarkReads(db, indexVals, readOps)
|
||
fmt.Printf(" Average ReadByFilter time: %v\n", readAvg)
|
||
|
||
fmt.Printf("\n3) Random update of %d records...\n", updateOps)
|
||
avgUpdate := benchmarkUpdates(db, ids, updateOps)
|
||
fmt.Printf(" Average update time: %v\n", avgUpdate)
|
||
printMem("After updates")
|
||
|
||
fmt.Printf("\n4) Random delete of %d records...\n", deleteOps)
|
||
avgDelete := benchmarkDeletes(db, ids, deleteOps)
|
||
fmt.Printf(" Average delete time: %v\n", avgDelete)
|
||
printMem("After deletes")
|
||
|
||
final, err := db.Read(collectionName, linedb.LineDbAdapterOptions{})
|
||
if err != nil {
|
||
log.Fatalf("[%s] Final read failed: %v", label, err)
|
||
}
|
||
fmt.Printf("\nFinal records in collection: %d\n", len(final))
|
||
|
||
mem := memSnapshot()
|
||
return scenarioResult{
|
||
dbDir: dbDir,
|
||
insertTotal: insertDur,
|
||
indexBuildTotal: 0, // индекс строится точечно при Write с DoIndexing
|
||
readAvg: readAvg,
|
||
updateAvg: avgUpdate,
|
||
deleteAvg: avgDelete,
|
||
finalCount: len(final),
|
||
memAllocMB: mem.allocMB,
|
||
memSysMB: mem.sysMB,
|
||
numGC: mem.numGC,
|
||
}
|
||
}
|
||
|
||
func insertRecords(db *linedb.LineDb) error {
|
||
// Сгенерируем базовый payload
|
||
base := strings.Repeat("X", payloadSize)
|
||
|
||
batch := make([]any, 0, 100)
|
||
for i := 0; i < recordsCount; i++ {
|
||
rec := map[string]any{
|
||
"id": i + 1,
|
||
"index": i,
|
||
"payload": base,
|
||
"created": time.Now().Format(time.RFC3339Nano),
|
||
}
|
||
batch = append(batch, rec)
|
||
if len(batch) >= cap(batch) {
|
||
if err := db.Write(batch, collectionName, linedb.LineDbAdapterOptions{DoIndexing: true}); err != nil {
|
||
return err
|
||
}
|
||
batch = batch[:0]
|
||
}
|
||
}
|
||
if len(batch) > 0 {
|
||
if err := db.Write(batch, collectionName, linedb.LineDbAdapterOptions{DoIndexing: true}); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func collectIDs(all []any) []any {
|
||
ids := make([]any, 0, len(all))
|
||
for _, r := range all {
|
||
if m, ok := r.(map[string]any); ok {
|
||
if id, ok := m["id"]; ok {
|
||
ids = append(ids, id)
|
||
}
|
||
}
|
||
}
|
||
return ids
|
||
}
|
||
|
||
func collectFieldValues(all []any, field string) []any {
|
||
vals := make([]any, 0, len(all))
|
||
for _, r := range all {
|
||
if m, ok := r.(map[string]any); ok {
|
||
if v, ok := m[field]; ok {
|
||
vals = append(vals, v)
|
||
}
|
||
}
|
||
}
|
||
return vals
|
||
}
|
||
|
||
func benchmarkReads(db *linedb.LineDb, values []any, ops int) time.Duration {
|
||
if len(values) == 0 || ops == 0 {
|
||
return 0
|
||
}
|
||
var total time.Duration
|
||
for i := 0; i < ops; i++ {
|
||
v := values[rand.Intn(len(values))]
|
||
start := time.Now()
|
||
_, err := db.ReadByFilter(map[string]any{"index": v}, collectionName, linedb.LineDbAdapterOptions{})
|
||
dur := time.Since(start)
|
||
if err != nil {
|
||
log.Printf("ReadByFilter error (index=%v): %v", v, err)
|
||
continue
|
||
}
|
||
total += dur
|
||
}
|
||
return total / time.Duration(ops)
|
||
}
|
||
|
||
func benchmarkUpdates(db *linedb.LineDb, ids []any, ops int) time.Duration {
|
||
if len(ids) == 0 {
|
||
return 0
|
||
}
|
||
var total time.Duration
|
||
for i := 0; i < ops; i++ {
|
||
id := ids[rand.Intn(len(ids))]
|
||
update := map[string]any{
|
||
"updatedAt": time.Now().Format(time.RFC3339Nano),
|
||
}
|
||
start := time.Now()
|
||
_, err := db.Update(update, collectionName, map[string]any{"id": id}, linedb.LineDbAdapterOptions{})
|
||
dur := time.Since(start)
|
||
if err != nil {
|
||
log.Printf("Update error (id=%v): %v", id, err)
|
||
continue
|
||
}
|
||
total += dur
|
||
}
|
||
if ops == 0 {
|
||
return 0
|
||
}
|
||
return total / time.Duration(ops)
|
||
}
|
||
|
||
func benchmarkDeletes(db *linedb.LineDb, ids []any, ops int) time.Duration {
|
||
if len(ids) == 0 {
|
||
return 0
|
||
}
|
||
var total time.Duration
|
||
used := make(map[int]struct{})
|
||
for i := 0; i < ops; i++ {
|
||
// чтобы не удалять по одному и тому же id постоянно, постараемся брать разные индексы
|
||
var idx int
|
||
for tries := 0; tries < 10; tries++ {
|
||
idx = rand.Intn(len(ids))
|
||
if _, ok := used[idx]; !ok {
|
||
break
|
||
}
|
||
}
|
||
used[idx] = struct{}{}
|
||
id := ids[idx]
|
||
|
||
start := time.Now()
|
||
_, err := db.Delete(map[string]any{"id": id}, collectionName, linedb.LineDbAdapterOptions{})
|
||
dur := time.Since(start)
|
||
if err != nil {
|
||
log.Printf("Delete error (id=%v): %v", id, err)
|
||
continue
|
||
}
|
||
total += dur
|
||
}
|
||
if ops == 0 {
|
||
return 0
|
||
}
|
||
return total / time.Duration(ops)
|
||
}
|
||
|
||
func printMem(label string) {
|
||
var m runtime.MemStats
|
||
runtime.ReadMemStats(&m)
|
||
fmt.Printf("\n=== %s ===\n", label)
|
||
fmt.Printf(" Alloc: %.2f MB\n", float64(m.Alloc)/1024.0/1024.0)
|
||
fmt.Printf(" TotalAlloc: %.2f MB\n", float64(m.TotalAlloc)/1024.0/1024.0)
|
||
fmt.Printf(" Sys: %.2f MB\n", float64(m.Sys)/1024.0/1024.0)
|
||
fmt.Printf(" NumGC: %d\n", m.NumGC)
|
||
}
|
||
|
||
type memSnap struct {
|
||
allocMB float64
|
||
sysMB float64
|
||
numGC uint32
|
||
}
|
||
|
||
func memSnapshot() memSnap {
|
||
var m runtime.MemStats
|
||
runtime.ReadMemStats(&m)
|
||
return memSnap{
|
||
allocMB: float64(m.Alloc) / 1024.0 / 1024.0,
|
||
sysMB: float64(m.Sys) / 1024.0 / 1024.0,
|
||
numGC: m.NumGC,
|
||
}
|
||
}
|