/context 增加 token 统计：tiktoken 精确分词与窗口使用百分比

2026-08-14 19:10:48 +08:00
parent bb5bc619c9
commit ba2ee4b0a0
6 changed files with 124 additions and 0 deletions
@@ -8,6 +8,7 @@ import (
 	"io"
 	"log"
 	"os"
+	"path/filepath"
 	"strings"

 	"github.com/peterh/liner"
@@ -19,6 +20,7 @@ import (
 )

 func main() {
+	os.Setenv("TIKTOKEN_CACHE_DIR", filepath.Join("data", "tiktoken"))
 	cfg, err := config.Load()
 	if err != nil {
 		log.Fatalf("加载配置失败: %v", err)