优化刷盘轮询，修复优先队列数量错误

2026-04-10 15:28:47 +08:00
parent 3256877dc4
commit e6c89c1c6a
6 changed files with 35 additions and 9 deletions
@@ -246,12 +246,14 @@ func (c *Crawler) runPriorityWorker() {
 			// 将子链接加入优先队列（保持优先级）
 			if len(hrefs) > 0 {
 				c.priorityChildrenMu.Lock()
+				added := 0
 				for _, child := range hrefs {
 					if !c.isVisited(child) {
 						c.priorityChildren = append(c.priorityChildren, child)
+						added++
 					}
 				}
-				IncrementPriorityChildren(int64(len(hrefs)))
+				IncrementPriorityChildren(int64(added))
 				c.priorityChildrenMu.Unlock()
 			}

@@ -5,7 +5,7 @@
    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>SESE 爬取管理</title>
-    <script type="module" crossorigin src="/assets/index-CiQvUT0P.js"></script>
+    <script type="module" crossorigin src="/assets/index-UuqaCCT8.js"></script>
    <link rel="stylesheet" crossorigin href="/assets/index-Dr22_wUg.css">
  </head>
  <body>
@@ -128,6 +128,7 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("/admin/priority", s.handleAdminPriority)
 	mux.HandleFunc("/admin/priority/status", s.handleAdminPriorityStatus)
 	mux.HandleFunc("/admin/flush", s.handleAdminFlush)
+	mux.HandleFunc("/admin/flush/status", s.handleAdminFlushStatus)
 	mux.HandleFunc("/admin/pending", s.handleAdminPending)
 	mux.HandleFunc("/admin/workers", s.handleAdminWorkers)
 	mux.HandleFunc("/admin/backlink", s.handleAdminBacklink)
@@ -592,6 +593,29 @@ func (s *Server) handleAdminFlush(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("flushed"))
 }

+// handleAdminFlushStatus 返回当前是否正在刷盘。
+// GET: 返回 flushing (bool)
+func (s *Server) handleAdminFlushStatus(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	if r.Method != http.MethodGet && r.Method != http.MethodOptions {
+		http.Error(w, `{"error":"method not allowed"}`, 405)
+		return
+	}
+	if r.Method == http.MethodOptions {
+		w.WriteHeader(204)
+		return
+	}
+	// flushMu.TryLock() 返回 false 表示锁被占用（正在刷盘）
+	flushing := !s.flushMu.TryLock()
+	if flushing {
+		json.NewEncoder(w).Encode(map[string]bool{"flushing": true})
+	} else {
+		s.flushMu.Unlock()
+		json.NewEncoder(w).Encode(map[string]bool{"flushing": false})
+	}
+}
+
 // handleAdminPending 返回内存中未刷盘的索引条目数量。
 func (s *Server) handleAdminPending(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")