From 3d46c697a03039463613efdf38bd1f2cd7077c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=96=87=E5=B3=B0?= Date: Thu, 9 Apr 2026 11:05:02 +0800 Subject: [PATCH] up --- config/config.go | 4 ++-- storage/storage.go | 28 +++++++++++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/config/config.go b/config/config.go index 1f40213..d060cf5 100644 --- a/config/config.go +++ b/config/config.go @@ -101,7 +101,7 @@ func GetDefaultConfig() Config { Index: IndexConfig{ MaxURLsPerKey: 11000, MaxSameDomainPerKey: 20, - BigCleanThreshold: 10000000, + BigCleanThreshold: 2000000, MaxNewURLsPerKey: 10000, MinURLsForNewKey: 3, }, @@ -123,7 +123,7 @@ func GetDefaultConfig() Config { ConsecutiveKeyWeight: 1.3, BacklinkWeight: 1.0, ServerPort: 80, - FlushIntervalSeconds: 60, + FlushIntervalSeconds: 30, }, Backlink: BacklinkConfig{ Baseline: 200000, diff --git a/storage/storage.go b/storage/storage.go index 4b4893a..ac5213b 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -90,7 +90,7 @@ func (d *DB) Close() error { // brotli 压缩比高于 gzip,适合大量文本的存储空间优化。 func compress(data []byte) ([]byte, error) { buf := make([]byte, 0, len(data)) // 预分配,避免反复扩容 - w := brotli.NewWriterLevel((*appendWriter)(&buf), 6) // 压缩级别 6(平衡速度和压缩比) + w := brotli.NewWriterLevel((*appendWriter)(&buf), 3) // 压缩级别 3(优先速度,压缩比损失约 10-15%) if _, err := w.Write(data); err != nil { return nil, err } @@ -210,14 +210,28 @@ func (d *DB) BatchSetIndex(batch map[string][]IndexEntry) error { end = len(items) } batchNum := i/batchSize + 1 + + // 事务外预先完成所有序列化和压缩,减少事务持锁时间 + preItems := make([]struct { + keyword string + data []byte + }, 0, end-i) + for _, item := range items[i:end] { + data, err := marshalCompress(item.entries) + if err != nil { + return err + } + preItems = append(preItems, struct { + keyword string + data []byte + }{item.keyword, data}) + } + + // 事务内只做纯内存写入,持锁时间极短 if err := d.db.Update(func(tx *bolt.Tx) error { b := tx.Bucket(bucketIndex) - for _, item := range items[i:end] { - data, err := marshalCompress(item.entries) - if err != nil { - return err - } - if err := b.Put([]byte(item.keyword), data); err != nil { + for _, p := range preItems { + if err := b.Put([]byte(p.keyword), p.data); err != nil { return err } }