589 lines
14 KiB
Go
589 lines
14 KiB
Go
// crawler.go — BFS crawl loop, URL scheduling, and site-info updating.
|
|
package crawler
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"log"
|
|
"math"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"sese-engine/analyzer"
|
|
"sese-engine/config"
|
|
"sese-engine/parser"
|
|
"sese-engine/storage"
|
|
)
|
|
|
|
|
|
// Stats holds real-time crawl counters (read with atomic).
|
|
type Stats struct {
|
|
VisitedURLs int64
|
|
SuccessURLs int64
|
|
KeywordsFetched int64
|
|
}
|
|
|
|
// Crawler orchestrates the BFS crawl.
|
|
type Crawler struct {
|
|
fetcher *Fetcher
|
|
db *storage.DB
|
|
analyzer *analyzer.Analyzer
|
|
prosperMap map[string]float64 // domain → backlink score (loaded from info)
|
|
stats Stats
|
|
}
|
|
|
|
// New creates a Crawler.
|
|
func New(db *storage.DB, a *analyzer.Analyzer, prosperMap map[string]float64) *Crawler {
|
|
return &Crawler{
|
|
fetcher: NewFetcher(config.SpiderName, config.CrawlerCooldown*time.Second),
|
|
db: db,
|
|
analyzer: a,
|
|
prosperMap: prosperMap,
|
|
}
|
|
}
|
|
|
|
// URLWeight pairs a URL with its discovery weight.
|
|
type URLWeight struct {
|
|
URL string
|
|
Weight float64
|
|
}
|
|
|
|
// Run starts the BFS crawl from entryURL, running for maxEpoch rounds.
|
|
// It blocks until completion.
|
|
func (c *Crawler) Run(entryURL string, maxEpoch int) {
|
|
visited := make(map[string]bool)
|
|
queue := []string{entryURL}
|
|
|
|
for ep := 0; ep < maxEpoch; ep++ {
|
|
log.Printf("[crawler] epoch %d/%d queue=%d", ep+1, maxEpoch, len(queue))
|
|
for _, u := range queue {
|
|
visited[u] = true
|
|
}
|
|
|
|
var (
|
|
newLinks []URLWeight
|
|
mu sync.Mutex
|
|
wg sync.WaitGroup
|
|
)
|
|
|
|
sem := make(chan struct{}, config.CrawlerWorkers)
|
|
for _, u := range queue {
|
|
wg.Add(1)
|
|
sem <- struct{}{}
|
|
go func(rawURL string) {
|
|
defer wg.Done()
|
|
defer func() { <-sem }()
|
|
hrefs := c.visitURL(rawURL)
|
|
n := len(hrefs)
|
|
if n > 0 {
|
|
w := 1.0 / float64(n)
|
|
mu.Lock()
|
|
for _, h := range hrefs {
|
|
if !visited[h] {
|
|
newLinks = append(newLinks, URLWeight{URL: h, Weight: w})
|
|
}
|
|
}
|
|
mu.Unlock()
|
|
}
|
|
}(u)
|
|
}
|
|
wg.Wait()
|
|
|
|
if len(newLinks) == 0 {
|
|
log.Println("[crawler] empty queue — stopping")
|
|
return
|
|
}
|
|
|
|
queue = c.schedule(newLinks)
|
|
}
|
|
}
|
|
|
|
// visitURL fetches a URL, stores keywords, updates site info, returns discovered hrefs.
|
|
func (c *Crawler) visitURL(rawURL string) []string {
|
|
atomic.AddInt64(&c.stats.VisitedURLs, 1)
|
|
|
|
res, err := c.fetcher.fetchWithHistory(rawURL, true, 10*time.Second, 0)
|
|
if err != nil || res == nil {
|
|
c.updateSiteFailure(rawURL)
|
|
return nil
|
|
}
|
|
|
|
atomic.AddInt64(&c.stats.SuccessURLs, 1)
|
|
|
|
title, desc, text, hrefs := parser.ParseHTML(res.Body, res.FinalURL)
|
|
|
|
// Cache snippet
|
|
if len(res.FinalURL) < 250 {
|
|
_ = c.db.SetSnippet(res.FinalURL, &storage.SnippetEntry{
|
|
Title: title,
|
|
Description: truncate(desc, 256),
|
|
Text: truncate(text, 256),
|
|
Timestamp: time.Now().Unix(),
|
|
})
|
|
}
|
|
|
|
// Keyword extraction → send to harvester
|
|
kws := c.analyzer.Analyze(title, desc, text)
|
|
if len(kws) > 0 {
|
|
if len(kws) > config.MaxKeywordsPerPage {
|
|
kws = kws[:config.MaxKeywordsPerPage]
|
|
}
|
|
atomic.AddInt64(&c.stats.KeywordsFetched, int64(len(kws)))
|
|
go c.sendToHarvester(res.FinalURL, kws)
|
|
}
|
|
|
|
// Update site info
|
|
host := netloc(res.FinalURL)
|
|
c.updateSiteSuccess(host, res, title, desc, text, hrefs)
|
|
|
|
// Handle permanent redirects in site info
|
|
for from, to := range res.Redirects {
|
|
fromHost := netloc(from)
|
|
if fromHost == "" {
|
|
continue
|
|
}
|
|
info, _ := c.db.GetSiteInfo(fromHost)
|
|
if info.Redirects == nil {
|
|
info.Redirects = make(map[string]string)
|
|
}
|
|
info.Redirects[from] = to
|
|
if len(info.Redirects) > 50 {
|
|
// keep most important (just truncate randomly for now)
|
|
info.Redirects = truncateMap(info.Redirects, 40)
|
|
}
|
|
_ = c.db.SetSiteInfo(fromHost, info)
|
|
}
|
|
|
|
// Trim hrefs
|
|
if len(hrefs) > 100 {
|
|
hrefs = sampleStrings(hrefs, 100)
|
|
}
|
|
return hrefs
|
|
}
|
|
|
|
func (c *Crawler) updateSiteFailure(rawURL string) {
|
|
host := netloc(rawURL)
|
|
if host == "" {
|
|
return
|
|
}
|
|
info, _ := c.db.GetSiteInfo(host)
|
|
if info.SuccessRate == nil {
|
|
zero := 0.0
|
|
info.SuccessRate = &zero
|
|
}
|
|
*info.SuccessRate *= 0.99
|
|
_ = c.db.SetSiteInfo(host, info)
|
|
}
|
|
|
|
func (c *Crawler) updateSiteSuccess(host string, res *FetchResult, title, desc, text string, hrefs []string) {
|
|
info, _ := c.db.GetSiteInfo(host)
|
|
|
|
info.VisitCount++
|
|
info.LastVisitTime = time.Now().Unix()
|
|
|
|
one := 1.0
|
|
if info.SuccessRate == nil {
|
|
info.SuccessRate = &one
|
|
}
|
|
*info.SuccessRate = *info.SuccessRate*0.99 + 0.01
|
|
|
|
if strings.HasPrefix(res.FinalURL, "https://") {
|
|
t := true
|
|
info.HTTPSAvailable = &t
|
|
}
|
|
|
|
if res.ServerType != "" {
|
|
found := false
|
|
for _, s := range info.ServerTypes {
|
|
if s == res.ServerType {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
info.ServerTypes = append(info.ServerTypes, res.ServerType)
|
|
if len(info.ServerTypes) > 5 {
|
|
info.ServerTypes = info.ServerTypes[len(info.ServerTypes)-5:]
|
|
}
|
|
}
|
|
}
|
|
|
|
// Language detection — sample 10% or first 10 visits
|
|
if info.VisitCount < 10 || rand.Float64() < 0.1 {
|
|
lang := c.analyzer.DetectLanguage(title + " " + desc + " " + text)
|
|
if lang != "" {
|
|
if info.Languages == nil {
|
|
info.Languages = make(map[string]float64)
|
|
}
|
|
intensity := math.Min(0.2, 1/math.Sqrt(float64(info.VisitCount+1)))
|
|
for k := range info.Languages {
|
|
info.Languages[k] *= (1 - intensity)
|
|
}
|
|
info.Languages[lang] += intensity
|
|
}
|
|
// Collect external links
|
|
superHost := superNetloc(res.FinalURL)
|
|
var external []string
|
|
for _, h := range hrefs {
|
|
if superNetloc(h) != superHost {
|
|
external = append(external, h)
|
|
}
|
|
}
|
|
sampled := sampleStrings(external, 10)
|
|
info.OutLinks = append(info.OutLinks, sampled...)
|
|
if len(info.OutLinks) > 250 {
|
|
info.OutLinks = sampleStrings(info.OutLinks, 200)
|
|
}
|
|
}
|
|
|
|
_ = c.db.SetSiteInfo(host, info)
|
|
}
|
|
|
|
// sendToHarvester POSTs keyword data to the harvester service.
|
|
func (c *Crawler) sendToHarvester(finalURL string, kws []analyzer.Keyword) {
|
|
type payload struct {
|
|
URL string `json:"url"`
|
|
Keywords []analyzer.Keyword `json:"keywords"`
|
|
}
|
|
p := payload{URL: finalURL, Keywords: kws}
|
|
data, err := json.Marshal(p)
|
|
if err != nil {
|
|
return
|
|
}
|
|
resp, err := http.Post(config.HarvesterAddr+"/l", "application/json", bytes.NewReader(data))
|
|
if err != nil {
|
|
log.Printf("[crawler] harvester post failed: %v", err)
|
|
return
|
|
}
|
|
resp.Body.Close()
|
|
}
|
|
|
|
// schedule selects and prioritises the next BFS queue from raw discovered links.
|
|
func (c *Crawler) schedule(links []URLWeight) []string {
|
|
if len(links) > 100000 {
|
|
links = sampleURLWeights(links, 100000)
|
|
}
|
|
|
|
// Pre-fetch site info for all involved domains
|
|
domains := make(map[string]bool)
|
|
for _, lw := range links {
|
|
if h := netloc(lw.URL); h != "" {
|
|
domains[h] = true
|
|
}
|
|
if h := superNetloc(lw.URL); h != "" {
|
|
domains[h] = true
|
|
}
|
|
}
|
|
siteCache := make(map[string]*storage.SiteInfo, len(domains))
|
|
var mu sync.Mutex
|
|
var wg sync.WaitGroup
|
|
for d := range domains {
|
|
wg.Add(1)
|
|
go func(host string) {
|
|
defer wg.Done()
|
|
info, _ := c.db.GetSiteInfo(host)
|
|
mu.Lock()
|
|
siteCache[host] = info
|
|
mu.Unlock()
|
|
}(d)
|
|
}
|
|
wg.Wait()
|
|
|
|
// Score each URL
|
|
scored_list := make([]scoredURL, len(links))
|
|
for i, lw := range links {
|
|
scored_list[i] = scoredURL{url: lw.URL, score: c.scoreURL(lw, siteCache)}
|
|
}
|
|
|
|
// Weighted random sample (45000 or 1/3+250 whichever smaller)
|
|
k := min(45000, len(scored_list)/3+250)
|
|
selected := weightedSample(scored_list, k)
|
|
|
|
// Domain concentration filtering
|
|
selected = concentrationFilter(selected, config.CrawlFocus)
|
|
|
|
// Separate https/http, cap http at 1/4 of https count
|
|
var httpsURLs, httpURLs []string
|
|
for _, s := range selected {
|
|
if strings.HasPrefix(s, "https://") {
|
|
httpsURLs = append(httpsURLs, s)
|
|
} else {
|
|
httpURLs = append(httpURLs, s)
|
|
}
|
|
}
|
|
maxHTTP := len(httpsURLs) / 4
|
|
if len(httpURLs) > maxHTTP {
|
|
httpURLs = sampleStrings(httpURLs, maxHTTP)
|
|
}
|
|
|
|
// Separate prosperous / non-prosperous
|
|
var prosperURLs, otherURLs []string
|
|
for _, u := range append(httpsURLs, httpURLs...) {
|
|
if c.prosperMap[netloc(u)] > 0 {
|
|
prosperURLs = append(prosperURLs, u)
|
|
} else {
|
|
otherURLs = append(otherURLs, u)
|
|
}
|
|
}
|
|
n := int(float64(len(prosperURLs)) * (1-config.ExpectedProsperRatio) / config.ExpectedProsperRatio)
|
|
if len(otherURLs) > n {
|
|
keep := max(len(otherURLs)-len(selected)/10, n)
|
|
if keep < len(otherURLs) {
|
|
otherURLs = sampleStrings(otherURLs, keep)
|
|
}
|
|
}
|
|
|
|
result := append(prosperURLs, otherURLs...)
|
|
rand.Shuffle(len(result), func(i, j int) { result[i], result[j] = result[j], result[i] })
|
|
return result
|
|
}
|
|
|
|
// scoreURL computes the scheduling priority for a URL.
|
|
func (c *Crawler) scoreURL(lw URLWeight, siteCache map[string]*storage.SiteInfo) float64 {
|
|
host := netloc(lw.URL)
|
|
super := superNetloc(lw.URL)
|
|
|
|
info := siteCache[host]
|
|
if info == nil {
|
|
info = &storage.SiteInfo{}
|
|
}
|
|
|
|
// Chinese-ness
|
|
var chineseness float64 = 0.5
|
|
if len(info.Languages) > 0 {
|
|
total := 0.0
|
|
for _, v := range info.Languages {
|
|
total += v
|
|
}
|
|
if total > 0 {
|
|
chineseness = info.Languages["zh"] / total
|
|
}
|
|
}
|
|
|
|
// Interest decay based on visit count
|
|
prosper := math.Min(62, c.prosperMap[host])
|
|
limit := prosper*500 + 50
|
|
b := math.Pow(0.1, 1/limit)
|
|
interest := math.Pow(b, float64(info.VisitCount))
|
|
|
|
var interest2 float64 = 1.0
|
|
if super != host {
|
|
superInfo := siteCache[super]
|
|
if superInfo != nil {
|
|
limit2 := math.Min(62, c.prosperMap[super])*500 + 50
|
|
b2 := math.Pow(0.1, 1/limit2)
|
|
interest2 = math.Pow(b2, float64(superInfo.VisitCount))
|
|
}
|
|
}
|
|
|
|
quality := 1.0
|
|
if info.Quality != nil {
|
|
quality = *info.Quality
|
|
}
|
|
|
|
prosperity := prosper
|
|
if prosperity > 0 {
|
|
prosperity += 0.5
|
|
}
|
|
prosperity = math.Log2(2+prosperity) + 1
|
|
|
|
bad := badURL(lw.URL)
|
|
return (0.1 + chineseness) * math.Min(0.05+interest, 0.05+interest2) * quality * (1 - bad) * lw.Weight * prosperity
|
|
}
|
|
|
|
// ---- helper functions ----
|
|
|
|
func netloc(rawURL string) string {
|
|
parts := strings.SplitN(rawURL, "/", 4)
|
|
if len(parts) >= 3 && (parts[0] == "http:" || parts[0] == "https:") && parts[1] == "" {
|
|
return parts[2]
|
|
}
|
|
u, err := url.Parse(rawURL)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return u.Host
|
|
}
|
|
|
|
// superNetloc returns "domain.tld" (strips subdomains).
|
|
func superNetloc(rawURL string) string {
|
|
host := netloc(rawURL)
|
|
parts := strings.Split(host, ".")
|
|
if len(parts) >= 2 {
|
|
return strings.Join(parts[len(parts)-2:], ".")
|
|
}
|
|
return host
|
|
}
|
|
|
|
func badURL(u string) float64 {
|
|
s := math.Max(0, float64(len(u)-30)/200.0)
|
|
if strings.Contains(u, ".htm") || strings.Contains(u, ".php") {
|
|
s += (1 - s) * 0.3
|
|
}
|
|
if strings.Count(strings.TrimRight(u, "/"), "/") > 2 {
|
|
s += (1 - s) * 0.1
|
|
}
|
|
if len(u) < 5 || u[4] == ':' {
|
|
s += (1 - s) * 0.3
|
|
}
|
|
return math.Min(s, 0.9)
|
|
}
|
|
|
|
func truncate(s string, n int) string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
return s[:n]
|
|
}
|
|
|
|
func sampleStrings(s []string, n int) []string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
perm := rand.Perm(len(s))
|
|
out := make([]string, n)
|
|
for i := range out {
|
|
out[i] = s[perm[i]]
|
|
}
|
|
return out
|
|
}
|
|
|
|
func sampleURLWeights(s []URLWeight, n int) []URLWeight {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
perm := rand.Perm(len(s))
|
|
out := make([]URLWeight, n)
|
|
for i := range out {
|
|
out[i] = s[perm[i]]
|
|
}
|
|
return out
|
|
}
|
|
|
|
type scoredURL struct {
|
|
url string
|
|
score float64
|
|
}
|
|
|
|
func weightedSample(items []scoredURL, k int) []string {
|
|
if k >= len(items) {
|
|
out := make([]string, len(items))
|
|
for i, s := range items {
|
|
out[i] = s.url
|
|
}
|
|
return out
|
|
}
|
|
// Simple weighted sampling without replacement using alias method approximation
|
|
totalWeight := 0.0
|
|
for _, s := range items {
|
|
totalWeight += s.score
|
|
}
|
|
selected := make(map[int]bool)
|
|
out := make([]string, 0, k)
|
|
for len(out) < k && len(selected) < len(items) {
|
|
r := rand.Float64() * totalWeight
|
|
cum := 0.0
|
|
for i, s := range items {
|
|
if selected[i] {
|
|
continue
|
|
}
|
|
cum += s.score
|
|
if cum >= r {
|
|
selected[i] = true
|
|
out = append(out, s.url)
|
|
totalWeight -= s.score
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func concentrationFilter(urls []string, k float64) []string {
|
|
domainGroups := make(map[string][]string)
|
|
shuffled := make([]string, len(urls))
|
|
copy(shuffled, urls)
|
|
rand.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] })
|
|
|
|
for _, u := range shuffled {
|
|
d := superNetloc(u)
|
|
domainGroups[d] = append(domainGroups[d], u)
|
|
}
|
|
|
|
limit := 10
|
|
if len(domainGroups) > 1 {
|
|
sizes := make([]int, 0, len(domainGroups))
|
|
for _, g := range domainGroups {
|
|
sizes = append(sizes, int(math.Pow(float64(len(g)), k)))
|
|
}
|
|
// sort sizes ascending, drop last (largest)
|
|
for i := 0; i < len(sizes)-1; i++ {
|
|
for j := i + 1; j < len(sizes)-1; j++ {
|
|
if sizes[j] < sizes[i] {
|
|
sizes[i], sizes[j] = sizes[j], sizes[i]
|
|
}
|
|
}
|
|
}
|
|
total := 0
|
|
for _, s := range sizes[:len(sizes)-1] {
|
|
total += s
|
|
}
|
|
limit = max(10, int(float64(total)*0.6))
|
|
}
|
|
|
|
var result []string
|
|
for _, g := range domainGroups {
|
|
sn := 1 + min(limit, int(math.Pow(float64(len(g)), k)))
|
|
if sn > len(g) {
|
|
sn = len(g)
|
|
}
|
|
result = append(result, g[:sn]...)
|
|
}
|
|
rand.Shuffle(len(result), func(i, j int) { result[i], result[j] = result[j], result[i] })
|
|
return result
|
|
}
|
|
|
|
func truncateMap(m map[string]string, n int) map[string]string {
|
|
if len(m) <= n {
|
|
return m
|
|
}
|
|
out := make(map[string]string, n)
|
|
i := 0
|
|
for k, v := range m {
|
|
if i >= n {
|
|
break
|
|
}
|
|
out[k] = v
|
|
i++
|
|
}
|
|
return out
|
|
}
|
|
|
|
func min(a, b int) int {
|
|
if a < b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func max(a, b int) int {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
// Expose Stats for monitoring.
|
|
func (c *Crawler) GetStats() Stats {
|
|
return Stats{
|
|
VisitedURLs: atomic.LoadInt64(&c.stats.VisitedURLs),
|
|
SuccessURLs: atomic.LoadInt64(&c.stats.SuccessURLs),
|
|
KeywordsFetched: atomic.LoadInt64(&c.stats.KeywordsFetched),
|
|
}
|
|
}
|