优化刷盘轮询，修复优先队列数量错误

2026-04-10 15:28:47 +08:00
parent 3256877dc4
commit e6c89c1c6a
6 changed files with 35 additions and 9 deletions
@@ -246,12 +246,14 @@ func (c *Crawler) runPriorityWorker() {
 			// 将子链接加入优先队列（保持优先级）
 			if len(hrefs) > 0 {
 				c.priorityChildrenMu.Lock()
 				added := 0
 				for _, child := range hrefs {
 					if !c.isVisited(child) {
 						c.priorityChildren = append(c.priorityChildren, child)
 						added++
 					}
 				}
-				IncrementPriorityChildren(int64(len(hrefs)))
+				IncrementPriorityChildren(int64(added))
 				c.priorityChildrenMu.Unlock()
 			}
@@ -5,7 +5,7 @@
    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>SESE 爬取管理</title>
-    <script type="module" crossorigin src="/assets/index-CiQvUT0P.js"></script>
+    <script type="module" crossorigin src="/assets/index-UuqaCCT8.js"></script>
    <link rel="stylesheet" crossorigin href="/assets/index-Dr22_wUg.css">
  </head>
  <body>
@@ -128,6 +128,7 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("/admin/priority", s.handleAdminPriority)
 	mux.HandleFunc("/admin/priority/status", s.handleAdminPriorityStatus)
 	mux.HandleFunc("/admin/flush", s.handleAdminFlush)
 	mux.HandleFunc("/admin/flush/status", s.handleAdminFlushStatus)
 	mux.HandleFunc("/admin/pending", s.handleAdminPending)
 	mux.HandleFunc("/admin/workers", s.handleAdminWorkers)
 	mux.HandleFunc("/admin/backlink", s.handleAdminBacklink)
@@ -592,6 +593,29 @@ func (s *Server) handleAdminFlush(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("flushed"))
 }
 // handleAdminFlushStatus 返回当前是否正在刷盘。
 // GET: 返回 flushing (bool)
 func (s *Server) handleAdminFlushStatus(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	if r.Method != http.MethodGet && r.Method != http.MethodOptions {
 		http.Error(w, `{"error":"method not allowed"}`, 405)
 		return
 	}
 	if r.Method == http.MethodOptions {
 		w.WriteHeader(204)
 		return
 	}
 	// flushMu.TryLock() 返回 false 表示锁被占用（正在刷盘）
 	flushing := !s.flushMu.TryLock()
 	if flushing {
 		json.NewEncoder(w).Encode(map[string]bool{"flushing": true})
 	} else {
 		s.flushMu.Unlock()
 		json.NewEncoder(w).Encode(map[string]bool{"flushing": false})
 	}
 }
 // handleAdminPending 返回内存中未刷盘的索引条目数量。
 func (s *Server) handleAdminPending(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")