85 lines
1.8 KiB
Go
85 lines
1.8 KiB
Go
package scraper
|
|
|
|
import (
|
|
"log"
|
|
"net/url"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/gocolly/colly"
|
|
)
|
|
|
|
const tapology_url string = "https://www.tapology.com"
|
|
|
|
type TapologyResult struct {
|
|
Name string `json:"name"`
|
|
Url string `json:"url"`
|
|
}
|
|
|
|
func tapology_getter() func(n string) []TapologyResult {
|
|
csrf_token := ""
|
|
name := ""
|
|
results := []TapologyResult{}
|
|
|
|
index := tapology_collector()
|
|
index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) {
|
|
csrf_token = h.Attr("content")
|
|
time.Sleep(5 * time.Second)
|
|
})
|
|
|
|
search := tapology_collector()
|
|
search.AllowURLRevisit = true
|
|
search.OnRequest(func(r *colly.Request) {
|
|
r.Headers.Add("X-CSRF-Token", csrf_token)
|
|
|
|
query := url.Values{}
|
|
query.Set("ajax", "true")
|
|
query.Set("model", "fighters")
|
|
query.Set("term", name)
|
|
r.URL.RawQuery = query.Encode()
|
|
|
|
log.Printf("Making tapology request to %s", r.URL)
|
|
})
|
|
search.OnResponse(func(r *colly.Response) {
|
|
results = parse_tapology_results(r.Body)
|
|
time.Sleep(5 * time.Second)
|
|
})
|
|
|
|
if err := index.Visit(tapology_url); err != nil {
|
|
log.Println(err)
|
|
}
|
|
|
|
return func(n string) []TapologyResult {
|
|
if csrf_token == "" {
|
|
return nil
|
|
}
|
|
|
|
name = n
|
|
if err := search.Visit(tapology_url + "/search/nav"); err != nil {
|
|
log.Println(err)
|
|
return nil
|
|
}
|
|
|
|
return results
|
|
}
|
|
}
|
|
|
|
func parse_tapology_results(b []byte) []TapologyResult {
|
|
results := []TapologyResult{}
|
|
|
|
document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
|
|
url, _ := s.Attr("href")
|
|
name := s.Text()
|
|
results = append(results, TapologyResult{Name: name, Url: url})
|
|
})
|
|
|
|
return results
|
|
}
|
|
|
|
func tapology_collector() *colly.Collector {
|
|
return colly.NewCollector(
|
|
colly.UserAgent(USER_AGENT),
|
|
colly.AllowedDomains("tapology.com", "www.tapology.com"),
|
|
)
|
|
}
|