// sese-engine — Go rewrite // // All modules (harvester, search server, crawler, backlink calculator) are // launched as goroutines from this single binary. The binary blocks until // interrupted (Ctrl-C / SIGTERM). // // Usage: // // cd golang && go run . [--storage ./savedata] [--entry https://zh.wikipedia.org/] package main import ( "flag" "fmt" "log" "os" "os/signal" "syscall" "sese-engine/analyzer" "sese-engine/backlink" "sese-engine/config" "sese-engine/crawler" "sese-engine/harvester" "sese-engine/info" "sese-engine/search" "sese-engine/storage" ) func main() { storageDir := flag.String("storage", config.StoragePath, "path to savedata directory") entryURL := flag.String("entry", config.EntryURL, "BFS crawl entry URL") stopWords := flag.String("stopwords", "../data/标点符号.json", "path to stop-words JSON") flag.Parse() log.SetFlags(log.LstdFlags | log.Lshortfile) log.Printf("sese-engine starting storage=%s entry=%s", *storageDir, *entryURL) // ---- 1. Storage ---- db, err := storage.Open(*storageDir) if err != nil { log.Fatalf("failed to open storage: %v", err) } defer db.Close() // ---- 2. Info service ---- infoSvc := info.New(*storageDir) // ---- 3. Analyzer ---- // modelPath is unused (lingua-go uses built-in language models, no external file needed) anal, err := analyzer.New("", *stopWords) if err != nil { log.Fatalf("failed to init analyzer: %v", err) } defer anal.Close() // ---- 4. Harvester (index write server on :5000) ---- harvSrv := harvester.New(db, infoSvc) go func() { if err := harvSrv.ListenAndServe(":5000"); err != nil { log.Fatalf("[harvester] fatal: %v", err) } }() // ---- 5. Search server ---- searchSrv := search.New(db, infoSvc, anal) go func() { addr := fmt.Sprintf(":%d", config.SearchServerPort) if err := searchSrv.ListenAndServe(addr); err != nil { log.Fatalf("[search] fatal: %v", err) } }() // ---- 6. Backlink calculator (runs every 48 h) ---- bl := backlink.New(db, *storageDir) go bl.Run() // ---- 7. Crawler ---- prosperMap := infoSvc.ProsperMap() crawl := crawler.New(db, anal, prosperMap) go crawl.Run(*entryURL, config.MaxEpoch) log.Println("all modules started — press Ctrl-C to stop") // ---- Graceful shutdown ---- quit := make(chan os.Signal, 1) signal.Notify(quit, os.Interrupt, syscall.SIGTERM) <-quit log.Println("shutdown signal received, exiting...") }