From c5d3538e60454436d2eaca24a9a2ae1b1bc03130 Mon Sep 17 00:00:00 2001 From: long Date: Sun, 23 Feb 2025 16:35:24 -0600 Subject: [PATCH] removed old scraper code --- scraper/helpers.go | 43 ------------- scraper/one.go | 115 ----------------------------------- scraper/one_test.go | 26 -------- scraper/scraper.go | 64 -------------------- scraper/tapology.go | 107 --------------------------------- scraper/tapology_test.go | 28 --------- scraper/ufc.go | 127 --------------------------------------- scraper/ufc_test.go | 35 ----------- 8 files changed, 545 deletions(-) delete mode 100644 scraper/helpers.go delete mode 100644 scraper/one.go delete mode 100644 scraper/one_test.go delete mode 100644 scraper/scraper.go delete mode 100644 scraper/tapology.go delete mode 100644 scraper/tapology_test.go delete mode 100644 scraper/ufc.go delete mode 100644 scraper/ufc_test.go diff --git a/scraper/helpers.go b/scraper/helpers.go deleted file mode 100644 index 12b1502..0000000 --- a/scraper/helpers.go +++ /dev/null @@ -1,43 +0,0 @@ -package scraper - -import ( - "bytes" - "encoding/json" - "log" - "regexp" - "strings" - - "github.com/PuerkitoBio/goquery" - "github.com/adrg/strutil" - "github.com/adrg/strutil/metrics" -) - -func text_content(doc *goquery.Selection, selector string) string { - return cleanup_whitespace(doc.Find(selector).First().Text()) -} - -var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`) - -func cleanup_whitespace(s string) string { - return strings.TrimSpace(whitespace.ReplaceAllString(s, " ")) -} - -func document_from_bytes(b []byte) *goquery.Selection { - doc, _ := goquery.NewDocumentFromReader(bytes.NewReader(b)) - return doc.Find("html").First() -} - -func print_json(v any) { - out, err := json.MarshalIndent(v, "", " ") - if err != nil { - log.Println(err) - } else { - log.Println(string(out)) - } -} - -var hamming *metrics.Hamming = metrics.NewHamming() - -func hamming_score(a, b string) float64 { - return strutil.Similarity(a, b, hamming) -} diff --git a/scraper/one.go b/scraper/one.go deleted file mode 100644 index 51d93c3..0000000 --- a/scraper/one.go +++ /dev/null @@ -1,115 +0,0 @@ -package scraper - -import ( - "encoding/json" - "fmt" - "log" - "time" - - "github.com/PuerkitoBio/goquery" - "github.com/gocolly/colly" -) - -type event_location struct { - Name string `json:"name"` -} - -type event_data struct { - Name string `json:"name"` - StartDate string `json:"startDate"` - Image []string `json:"image"` - Location event_location `json:"location"` -} - -func scrape_one(callback EventCallback) { - list := one_collector() - detail := one_collector() - url := "" - - list.OnResponse(func(r *colly.Response) { - for _, u := range parse_one_event_list(r.Body) { - url = u - log.Printf("Visiting %s\n", u) - if err := detail.Visit(u); err != nil { - log.Println(err) - } - } - }) - - detail.OnResponse(func(r *colly.Response) { - callback(parse_one_event(url, r.Body)) - }) - - if err := list.Visit("https://www.onefc.com/events/"); err != nil { - log.Println(err) - } -} - -func parse_one_event_list(b []byte) []string { - return document_from_bytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string { - url, _ := s.Attr("href") - return url - }) -} - -func parse_one_event(url string, b []byte) *Event { - event := Event{Organization: "ONE", Url: url} - doc := document_from_bytes(b) - - data := event_data{} - _ = json.Unmarshal([]byte(doc.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data) - - event.Name = data.Name - event.Location = data.Location.Name - - if len(data.Image) > 2 { - event.Image = data.Image[2] - } - - date, err := time.Parse(time.RFC3339, data.StartDate) - if err == nil { - event.Date = int(date.Unix()) - } - - doc.Find(".event-matchup").Each(func(i int, s *goquery.Selection) { - event.Fights = append(event.Fights, *parse_one_fight(s)) - }) - - return &event -} - -func parse_one_fight(s *goquery.Selection) *Fight { - fight := Fight{} - - fight.Weight = text_content(s, ".title") - fight.FighterA = parse_one_fighter(s, true) - fight.FighterB = parse_one_fighter(s, false) - - return &fight -} - -func parse_one_fighter(s *goquery.Selection, is_a bool) *Fighter { - fighter := Fighter{} - nth_child := "1" - face := "1" - - if is_a { - nth_child = "3" - face = "2" - } - - fighter.Name = text_content(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child)) - fighter.Country = text_content(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child)) - - image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src") - fighter.Image = image - - return &fighter -} - -func one_collector() *colly.Collector { - return colly.NewCollector( - colly.UserAgent(USER_AGENT), - colly.AllowedDomains("onefc.com", "www.onefc.com"), - ) -} diff --git a/scraper/one_test.go b/scraper/one_test.go deleted file mode 100644 index aac9739..0000000 --- a/scraper/one_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package scraper - -import ( - _ "embed" - "testing" -) - -//go:embed testdata/one-fight-night-23.html -var ONEEventHTML []byte - -//go:embed testdata/one-events.html -var ONEEventListHTML []byte - -func TestONEEventList(t *testing.T) { - print_json(parse_one_event_list(ONEEventListHTML)) -} - -func TestONEEvent(t *testing.T) { - print_json(parse_one_event("some_url", ONEEventHTML)) -} - -func testScrapeONE(t *testing.T) { - scrape_one(func(e *Event) { - print_json(e) - }) -} diff --git a/scraper/scraper.go b/scraper/scraper.go deleted file mode 100644 index 8ca3fb2..0000000 --- a/scraper/scraper.go +++ /dev/null @@ -1,64 +0,0 @@ -package scraper - -import ( - "bytes" - "crypto/sha1" - "encoding/gob" -) - -type Event struct { - Url string `json:"url"` - Slug string `json:"slug"` - Name string `json:"name"` - Location string `json:"location"` - Organization string `json:"organization"` - Image string `json:"image"` - Date int `json:"date"` - Fights []Fight `json:"fights"` -} - -func (e *Event) Hash() ([]byte, error) { - var buf bytes.Buffer - hasher := sha1.New() - enc := gob.NewEncoder(&buf) - if err := enc.Encode(e); err != nil { - return nil, err - } - return hasher.Sum(buf.Bytes()), nil -} - -type Fight struct { - Weight string `json:"weight"` - FighterA *Fighter `json:"fighter_a"` - FighterB *Fighter `json:"fighter_b"` -} - -type Fighter struct { - Name string `json:"name"` - Image string `json:"image"` - Country string `json:"country"` - Link string `json:"link"` -} - -type EventCallback func(e *Event) -type EventScraper func(c EventCallback) - -const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36" - -var scrapers []EventScraper = []EventScraper{ - scrape_ufc, - scrape_one, -} - -func ScrapeEvents(callback EventCallback) { - get_tapology := tapology_getter() - - for _, scraper := range scrapers { - scraper(func(e *Event) { - for _, fight := range e.Fights { - fight.FighterA.Link = get_tapology(fight.FighterA.Name) - } - callback(e) - }) - } -} diff --git a/scraper/tapology.go b/scraper/tapology.go deleted file mode 100644 index 95cd096..0000000 --- a/scraper/tapology.go +++ /dev/null @@ -1,107 +0,0 @@ -package scraper - -import ( - "log" - "net/url" - "regexp" - "slices" - "time" - - "github.com/PuerkitoBio/goquery" - "github.com/gocolly/colly" -) - -const tapology_url string = "https://www.tapology.com" - -type TapologyResult struct { - Name string `json:"name"` - Url string `json:"url"` -} - -type TapologyGetter func(n string) string - -func tapology_getter() TapologyGetter { - csrf_token := "" - name := "" - result := "" - - index := tapology_collector() - index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) { - csrf_token = h.Attr("content") - time.Sleep(5 * time.Second) - }) - - if err := index.Visit(tapology_url); err != nil { - log.Println(err) - } - - search := tapology_collector() - search.AllowURLRevisit = true - search.OnRequest(func(r *colly.Request) { - r.Headers.Add("X-CSRF-Token", csrf_token) - - query := url.Values{} - query.Set("ajax", "true") - query.Set("model", "fighters") - query.Set("term", name) - r.URL.RawQuery = query.Encode() - - log.Printf("Making tapology request to %s", r.URL) - }) - search.OnResponse(func(r *colly.Response) { - result = parse_tapology_results(name, r.Body) - time.Sleep(5 * time.Second) - }) - - return func(n string) string { - if csrf_token == "" { - return "" - } - - name = n - if err := search.Visit(tapology_url + "/search/nav"); err != nil { - log.Println(err) - return "" - } - - return result - } -} - -var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`) - -func parse_tapology_results(name string, b []byte) string { - results := []TapologyResult{} - - document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) { - url, _ := s.Attr("href") - name := cleanup_whitespace(nickname.ReplaceAllString(s.Text(), "")) - results = append(results, TapologyResult{Name: name, Url: url}) - }) - - slices.SortStableFunc(results, func(a, b TapologyResult) int { - score_a := hamming_score(name, a.Name) - score_b := hamming_score(name, b.Name) - - if score_a > score_b { - return -1 - } else if score_a < score_b { - return 1 - } else { - return 0 - } - }) - - if len(results) < 1 { - return "" - } - - return tapology_url + results[0].Url -} - -func tapology_collector() *colly.Collector { - return colly.NewCollector( - colly.UserAgent(USER_AGENT), - colly.AllowedDomains("tapology.com", "www.tapology.com"), - ) -} diff --git a/scraper/tapology_test.go b/scraper/tapology_test.go deleted file mode 100644 index 79803ba..0000000 --- a/scraper/tapology_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package scraper - -import ( - _ "embed" - "testing" -) - -//go:embed testdata/tapology-search.html -var search_result []byte - -func testTapologyGetter(t *testing.T) { - get_tapology_for := tapology_getter() - names := []string{ - "Dustin Poirier", - "Jon Jones", - "Conor Mcgregor", - "Justin Gaethje", - "Benoit Saint Denis", - } - - for _, name := range names { - print_json(get_tapology_for(name)) - } -} - -func TestParseTapologyResults(t *testing.T) { - print_json(parse_tapology_results("Jon Jones", search_result)) -} diff --git a/scraper/ufc.go b/scraper/ufc.go deleted file mode 100644 index 29e1f14..0000000 --- a/scraper/ufc.go +++ /dev/null @@ -1,127 +0,0 @@ -package scraper - -import ( - "log" - "regexp" - "strconv" - "strings" - - "github.com/PuerkitoBio/goquery" - "github.com/gocolly/colly" -) - -func scrape_ufc(callback EventCallback) { - list := ufc_collector() - detail := ufc_collector() - url := "" - - list.OnResponse(func(r *colly.Response) { - for _, u := range parse_ufc_event_list(r.Body) { - url = u - log.Printf("Visiting %s\n", u) - if err := detail.Visit(u); err != nil { - log.Println(err) - } - } - }) - - detail.OnResponse(func(r *colly.Response) { - callback(parse_ufc_event(url, r.Body)) - }) - - if err := list.Visit("https://www.ufc.com/events"); err != nil { - log.Println(err) - } -} - -func parse_ufc_event_list(b []byte) []string { - urls := []string{} - doc := document_from_bytes(b) - - doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) { - slug, ok := s.Attr("href") - if ok { - urls = append(urls, "https://ufc.com"+slug) - } - }) - - return urls -} - -func parse_ufc_event(url string, b []byte) *Event { - event := Event{Organization: "UFC", Url: url} - doc := document_from_bytes(b) - - parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string { - return cleanup_whitespace(s.Text()) - }) - event.Name = strings.Join(parts, ": ") - event.Location = text_content(doc, ".field--name-venue") - event.Slug = parse_ufc_slug(url) - - image, ok := doc.Find(".c-hero__image img").First().Attr("src") - if ok { - event.Image = image - } - - datestr, ok := doc.Find(` - - #early-prelims .c-event-fight-card-broadcaster__time, - #prelims-card .c-event-fight-card-broadcaster__time, - .c-hero__headline-suffix - - `).Last().Attr("data-timestamp") - if ok { - date, err := strconv.Atoi(datestr) - if err == nil { - event.Date = date - } - } - - doc.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) { - event.Fights = append(event.Fights, *parse_ufc_fight(s)) - }) - - return &event -} - -func parse_ufc_fight(s *goquery.Selection) *Fight { - fight := Fight{} - - fight.Weight = strings.Replace(text_content(s, ".c-listing-fight__class-text"), " Bout", "", -1) - fight.FighterA = parse_ufc_fighter(s, "red") - fight.FighterB = parse_ufc_fighter(s, "blue") - - return &fight -} - -func parse_ufc_fighter(s *goquery.Selection, color string) *Fighter { - fighter := Fighter{} - - fighter.Name = text_content(s, ".c-listing-fight__corner-name--"+color) - fighter.Country = text_content(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text") - image, ok := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src") - if ok { - fighter.Image = image - } - - return &fighter -} - -var ufc_event_url *regexp.Regexp = regexp.MustCompile(`https://ufc.com/event/(?P(\w|-)+)`) - -func parse_ufc_slug(url string) string { - matches := ufc_event_url.FindStringSubmatch(url) - index := ufc_event_url.SubexpIndex("Slug") - if len(matches) > index { - return matches[index] - } - return "" -} - -func ufc_collector() *colly.Collector { - return colly.NewCollector( - colly.UserAgent(USER_AGENT), - colly.AllowedDomains("ufc.com", "www.ufc.com"), - ) -} diff --git a/scraper/ufc_test.go b/scraper/ufc_test.go deleted file mode 100644 index ba9f86e..0000000 --- a/scraper/ufc_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package scraper - -import ( - _ "embed" - "testing" -) - -//go:embed testdata/ufc-297-good.html -var UFCEventHTML []byte - -//go:embed testdata/ufc-events.html -var UFCEventListHTML []byte - -func TestUFCEventList(t *testing.T) { - print_json(parse_ufc_event_list(UFCEventListHTML)) -} - -func TestUFCEvent(t *testing.T) { - print_json(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML)) -} - -func TestParseUFCSlug(t *testing.T) { - result := parse_ufc_slug("https://ufc.com/event/ufc-fight-night-july-20-2024") - expected := "ufc-fight-night-july-20-2024" - if result != expected { - print_json(result) - t.FailNow() - } -} - -func testScrapeUFC(t *testing.T) { - scrape_ufc(func(e *Event) { - print_json(e) - }) -}