up
This commit is contained in:
+1
-1
@@ -425,7 +425,7 @@ func (c *Crawler) visitURL(rawURL string) (hrefs []string) {
|
||||
oldEntry, _ := c.db.GetSnippet(res.FinalURL)
|
||||
if oldEntry != nil && oldEntry.ContentHash != "" && oldEntry.ContentHash == contentHash {
|
||||
isRecrawl = true
|
||||
log.Printf("[crawler] unchanged (recrawl skip): %s", res.FinalURL)
|
||||
//log.Printf("[crawler] unchanged (recrawl skip): %s", res.FinalURL)
|
||||
}
|
||||
|
||||
// 缓存 URL 摘要(仅对短 URL 缓存,防止超长 URL 浪费空间)
|
||||
|
||||
Reference in New Issue
Block a user