91 lines
2.4 KiB
Go
91 lines
2.4 KiB
Go
// sese-engine — Go rewrite
|
|
//
|
|
// All modules (harvester, search server, crawler, backlink calculator) are
|
|
// launched as goroutines from this single binary. The binary blocks until
|
|
// interrupted (Ctrl-C / SIGTERM).
|
|
//
|
|
// Usage:
|
|
//
|
|
// cd golang && go run . [--storage ./savedata] [--entry https://zh.wikipedia.org/]
|
|
package main
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
|
|
"sese-engine/analyzer"
|
|
"sese-engine/backlink"
|
|
"sese-engine/config"
|
|
"sese-engine/crawler"
|
|
"sese-engine/harvester"
|
|
"sese-engine/info"
|
|
"sese-engine/search"
|
|
"sese-engine/storage"
|
|
)
|
|
|
|
func main() {
|
|
storageDir := flag.String("storage", config.StoragePath, "path to savedata directory")
|
|
entryURL := flag.String("entry", config.EntryURL, "BFS crawl entry URL")
|
|
stopWords := flag.String("stopwords", "../data/标点符号.json", "path to stop-words JSON")
|
|
flag.Parse()
|
|
|
|
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
|
log.Printf("sese-engine starting storage=%s entry=%s", *storageDir, *entryURL)
|
|
|
|
// ---- 1. Storage ----
|
|
db, err := storage.Open(*storageDir)
|
|
if err != nil {
|
|
log.Fatalf("failed to open storage: %v", err)
|
|
}
|
|
defer db.Close()
|
|
|
|
// ---- 2. Info service ----
|
|
infoSvc := info.New(*storageDir)
|
|
|
|
// ---- 3. Analyzer ----
|
|
// modelPath is unused (lingua-go uses built-in language models, no external file needed)
|
|
anal, err := analyzer.New("", *stopWords)
|
|
if err != nil {
|
|
log.Fatalf("failed to init analyzer: %v", err)
|
|
}
|
|
defer anal.Close()
|
|
|
|
// ---- 4. Harvester (index write server on :5000) ----
|
|
harvSrv := harvester.New(db, infoSvc)
|
|
go func() {
|
|
if err := harvSrv.ListenAndServe(":5000"); err != nil {
|
|
log.Fatalf("[harvester] fatal: %v", err)
|
|
}
|
|
}()
|
|
|
|
// ---- 5. Search server ----
|
|
searchSrv := search.New(db, infoSvc, anal)
|
|
go func() {
|
|
addr := fmt.Sprintf(":%d", config.SearchServerPort)
|
|
if err := searchSrv.ListenAndServe(addr); err != nil {
|
|
log.Fatalf("[search] fatal: %v", err)
|
|
}
|
|
}()
|
|
|
|
// ---- 6. Backlink calculator (runs every 48 h) ----
|
|
bl := backlink.New(db, *storageDir)
|
|
go bl.Run()
|
|
|
|
// ---- 7. Crawler ----
|
|
prosperMap := infoSvc.ProsperMap()
|
|
crawl := crawler.New(db, anal, prosperMap)
|
|
go crawl.Run(*entryURL, config.MaxEpoch)
|
|
|
|
log.Println("all modules started — press Ctrl-C to stop")
|
|
|
|
// ---- Graceful shutdown ----
|
|
quit := make(chan os.Signal, 1)
|
|
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
|
<-quit
|
|
log.Println("shutdown signal received, exiting...")
|
|
}
|