Signed-off-by: 吴文峰 <kevin@lmve.net>

2026-04-08 17:29:39 +08:00
commit 6c2f5ad978
15 changed files with 3651 additions and 0 deletions
@@ -0,0 +1,90 @@
+// sese-engine — Go rewrite
+//
+// All modules (harvester, search server, crawler, backlink calculator) are
+// launched as goroutines from this single binary.  The binary blocks until
+// interrupted (Ctrl-C / SIGTERM).
+//
+// Usage:
+//
+//	cd golang && go run . [--storage ./savedata] [--entry https://zh.wikipedia.org/]
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"sese-engine/analyzer"
+	"sese-engine/backlink"
+	"sese-engine/config"
+	"sese-engine/crawler"
+	"sese-engine/harvester"
+	"sese-engine/info"
+	"sese-engine/search"
+	"sese-engine/storage"
+)
+
+func main() {
+	storageDir := flag.String("storage", config.StoragePath, "path to savedata directory")
+	entryURL   := flag.String("entry", config.EntryURL, "BFS crawl entry URL")
+	stopWords  := flag.String("stopwords", "../data/标点符号.json", "path to stop-words JSON")
+	flag.Parse()
+
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+	log.Printf("sese-engine starting  storage=%s  entry=%s", *storageDir, *entryURL)
+
+	// ---- 1. Storage ----
+	db, err := storage.Open(*storageDir)
+	if err != nil {
+		log.Fatalf("failed to open storage: %v", err)
+	}
+	defer db.Close()
+
+	// ---- 2. Info service ----
+	infoSvc := info.New(*storageDir)
+
+	// ---- 3. Analyzer ----
+	// modelPath is unused (lingua-go uses built-in language models, no external file needed)
+	anal, err := analyzer.New("", *stopWords)
+	if err != nil {
+		log.Fatalf("failed to init analyzer: %v", err)
+	}
+	defer anal.Close()
+
+	// ---- 4. Harvester (index write server on :5000) ----
+	harvSrv := harvester.New(db, infoSvc)
+	go func() {
+		if err := harvSrv.ListenAndServe(":5000"); err != nil {
+			log.Fatalf("[harvester] fatal: %v", err)
+		}
+	}()
+
+	// ---- 5. Search server ----
+	searchSrv := search.New(db, infoSvc, anal)
+	go func() {
+		addr := fmt.Sprintf(":%d", config.SearchServerPort)
+		if err := searchSrv.ListenAndServe(addr); err != nil {
+			log.Fatalf("[search] fatal: %v", err)
+		}
+	}()
+
+	// ---- 6. Backlink calculator (runs every 48 h) ----
+	bl := backlink.New(db, *storageDir)
+	go bl.Run()
+
+	// ---- 7. Crawler ----
+	prosperMap := infoSvc.ProsperMap()
+	crawl := crawler.New(db, anal, prosperMap)
+	go crawl.Run(*entryURL, config.MaxEpoch)
+
+	log.Println("all modules started — press Ctrl-C to stop")
+
+	// ---- Graceful shutdown ----
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
+	<-quit
+	log.Println("shutdown signal received, exiting...")
+}