子链接不再被 isVisited 过滤
This commit is contained in:
+2
-5
@@ -258,16 +258,13 @@ func (c *Crawler) runPriorityWorker() {
|
||||
hrefs := c.visitURLUnlimited(rawURL)
|
||||
|
||||
// 将子链接加入优先队列(保持优先级)
|
||||
// 注意:Priority URL 的子链接强制加入队列,即使已访问过也要重新爬取
|
||||
if len(hrefs) > 0 {
|
||||
c.priorityChildrenMu.Lock()
|
||||
added := 0
|
||||
for _, child := range hrefs {
|
||||
if !c.isVisited(child) {
|
||||
c.priorityChildren = append(c.priorityChildren, child)
|
||||
added++
|
||||
}
|
||||
}
|
||||
IncrementPriorityChildren(int64(added))
|
||||
IncrementPriorityChildren(int64(len(hrefs)))
|
||||
c.priorityChildrenMu.Unlock()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user