加固sizeLimit 兜底

2026-04-09 16:51:46 +08:00
parent 4137343fe7
commit 2ab89b39db
2 changed files with 7 additions and 2 deletions
@@ -204,8 +204,8 @@ func (c *Crawler) visitURL(rawURL string) (hrefs []string) {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		// 礼貌模式抓取（遵守 robots.txt + 限流），不限制大小
-		res, fetchErr = c.fetcher.fetchWithHistory(rawURL, true, fetchTimeout, 0)
+		// 礼貌模式抓取（遵守 robots.txt + 限流），限制页面大小防止内存爆炸
+		res, fetchErr = c.fetcher.fetchWithHistory(rawURL, true, fetchTimeout, config.MaxPageSize())
 	}()
 	waitCh := make(chan struct{})
 	go func() {