diff --git a/main.go b/main.go index 37ebfaf..da29a2c 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,4 @@ package main -import "mmaschedule-go/scraper" - func main() { - scraper.CleanupString("asd") } diff --git a/scraper/helpers.go b/scraper/helpers.go index 6b43262..1b35cbe 100644 --- a/scraper/helpers.go +++ b/scraper/helpers.go @@ -3,7 +3,6 @@ package scraper import ( "bytes" "encoding/json" - "hash/fnv" "log" "regexp" "strings" @@ -11,32 +10,22 @@ import ( "github.com/PuerkitoBio/goquery" ) -func TextContent(doc *goquery.Selection, selector string) string { - return CleanupString(doc.Find(selector).First().Text()) +func text_content(doc *goquery.Selection, selector string) string { + return cleanup_string(doc.Find(selector).First().Text()) } var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`) -func CleanupString(s string) string { +func cleanup_string(s string) string { return strings.TrimSpace(whitespace.ReplaceAllString(s, " ")) } -func DocumentFromBytes(b []byte) *goquery.Selection { +func document_from_bytes(b []byte) *goquery.Selection { doc, _ := goquery.NewDocumentFromReader(bytes.NewReader(b)) return doc.Find("html").First() } -func HashJSON(v any) uint32 { - out, err := json.Marshal(v) - if err != nil { - return 0 - } - h := fnv.New32a() - h.Write(out) - return h.Sum32() -} - -func PrintJSON(v any) { +func print_json(v any) { out, err := json.MarshalIndent(v, "", " ") if err != nil { log.Println(err) diff --git a/scraper/one.go b/scraper/one.go index 5251eaa..51d93c3 100644 --- a/scraper/one.go +++ b/scraper/one.go @@ -10,31 +10,24 @@ import ( "github.com/gocolly/colly" ) -type ONEEventLocation struct { +type event_location struct { Name string `json:"name"` } -type ONEEventData struct { - Name string `json:"name"` - StartDate string `json:"startDate"` - Image []string `json:"image"` - Location ONEEventLocation `json:"location"` +type event_data struct { + Name string `json:"name"` + StartDate string `json:"startDate"` + Image []string `json:"image"` + Location event_location `json:"location"` } -func ONECollector() *colly.Collector { - return colly.NewCollector( - colly.UserAgent(USER_AGENT), - colly.AllowedDomains("onefc.com", "www.onefc.com"), - ) -} - -func ScrapeONE(callback EventCallback) { - list := ONECollector() - detail := ONECollector() +func scrape_one(callback EventCallback) { + list := one_collector() + detail := one_collector() url := "" list.OnResponse(func(r *colly.Response) { - for _, u := range ParseONEEventList(r.Body) { + for _, u := range parse_one_event_list(r.Body) { url = u log.Printf("Visiting %s\n", u) if err := detail.Visit(u); err != nil { @@ -44,7 +37,7 @@ func ScrapeONE(callback EventCallback) { }) detail.OnResponse(func(r *colly.Response) { - callback(ParseONEEvent(url, r.Body)) + callback(parse_one_event(url, r.Body)) }) if err := list.Visit("https://www.onefc.com/events/"); err != nil { @@ -52,47 +45,50 @@ func ScrapeONE(callback EventCallback) { } } -func ParseONEEventList(b []byte) []string { - return DocumentFromBytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string { +func parse_one_event_list(b []byte) []string { + return document_from_bytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string { url, _ := s.Attr("href") return url }) } -func ParseONEEvent(url string, b []byte) *Event { +func parse_one_event(url string, b []byte) *Event { event := Event{Organization: "ONE", Url: url} - doc := DocumentFromBytes(b) + doc := document_from_bytes(b) - data := ONEEventData{} + data := event_data{} _ = json.Unmarshal([]byte(doc.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data) event.Name = data.Name - event.Image = data.Image[2] event.Location = data.Location.Name + if len(data.Image) > 2 { + event.Image = data.Image[2] + } + date, err := time.Parse(time.RFC3339, data.StartDate) if err == nil { event.Date = int(date.Unix()) } doc.Find(".event-matchup").Each(func(i int, s *goquery.Selection) { - event.Fights = append(event.Fights, *ParseONEFight(s)) + event.Fights = append(event.Fights, *parse_one_fight(s)) }) return &event } -func ParseONEFight(s *goquery.Selection) *Fight { +func parse_one_fight(s *goquery.Selection) *Fight { fight := Fight{} - fight.Weight = TextContent(s, ".title") - fight.FighterA = ParseONEFighter(s, true) - fight.FighterB = ParseONEFighter(s, false) + fight.Weight = text_content(s, ".title") + fight.FighterA = parse_one_fighter(s, true) + fight.FighterB = parse_one_fighter(s, false) return &fight } -func ParseONEFighter(s *goquery.Selection, is_a bool) *Fighter { +func parse_one_fighter(s *goquery.Selection, is_a bool) *Fighter { fighter := Fighter{} nth_child := "1" face := "1" @@ -102,11 +98,18 @@ func ParseONEFighter(s *goquery.Selection, is_a bool) *Fighter { face = "2" } - fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child)) - fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child)) + fighter.Name = text_content(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child)) + fighter.Country = text_content(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child)) image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src") fighter.Image = image return &fighter } + +func one_collector() *colly.Collector { + return colly.NewCollector( + colly.UserAgent(USER_AGENT), + colly.AllowedDomains("onefc.com", "www.onefc.com"), + ) +} diff --git a/scraper/one_test.go b/scraper/one_test.go index b260d2d..aac9739 100644 --- a/scraper/one_test.go +++ b/scraper/one_test.go @@ -12,15 +12,15 @@ var ONEEventHTML []byte var ONEEventListHTML []byte func TestONEEventList(t *testing.T) { - PrintJSON(ParseONEEventList(ONEEventListHTML)) + print_json(parse_one_event_list(ONEEventListHTML)) } func TestONEEvent(t *testing.T) { - PrintJSON(ParseONEEvent("some_url", ONEEventHTML)) + print_json(parse_one_event("some_url", ONEEventHTML)) } func testScrapeONE(t *testing.T) { - ScrapeONE(func(e *Event) { - PrintJSON(e) + scrape_one(func(e *Event) { + print_json(e) }) } diff --git a/scraper/scraper.go b/scraper/scraper.go index e4b6c40..3208bcb 100644 --- a/scraper/scraper.go +++ b/scraper/scraper.go @@ -29,6 +29,7 @@ type EventScraper func(c EventCallback) const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36" -var Scrapers []EventScraper = []EventScraper{ - ScrapeUFC, +var scrapers []EventScraper = []EventScraper{ + scrape_ufc, + scrape_one, } diff --git a/scraper/tapology.go b/scraper/tapology.go index f6821f5..8cef823 100644 --- a/scraper/tapology.go +++ b/scraper/tapology.go @@ -16,14 +16,7 @@ type TapologyResult struct { Url string `json:"url"` } -func tapology_collector() *colly.Collector { - return colly.NewCollector( - colly.UserAgent(USER_AGENT), - colly.AllowedDomains("tapology.com", "www.tapology.com"), - ) -} - -func TapologyGetter() func(n string) []TapologyResult { +func tapology_getter() func(n string) []TapologyResult { csrf_token := "" name := "" results := []TapologyResult{} @@ -74,7 +67,7 @@ func TapologyGetter() func(n string) []TapologyResult { func parse_tapology_results(b []byte) []TapologyResult { results := []TapologyResult{} - DocumentFromBytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) { + document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) { url, _ := s.Attr("href") name := s.Text() results = append(results, TapologyResult{Name: name, Url: url}) @@ -82,3 +75,10 @@ func parse_tapology_results(b []byte) []TapologyResult { return results } + +func tapology_collector() *colly.Collector { + return colly.NewCollector( + colly.UserAgent(USER_AGENT), + colly.AllowedDomains("tapology.com", "www.tapology.com"), + ) +} diff --git a/scraper/tapology_test.go b/scraper/tapology_test.go index a45101e..aadce12 100644 --- a/scraper/tapology_test.go +++ b/scraper/tapology_test.go @@ -9,12 +9,12 @@ import ( var search_result []byte func testTapologyGetter(t *testing.T) { - get_tapology_for := TapologyGetter() - PrintJSON(get_tapology_for("Dustin Poirier")) - PrintJSON(get_tapology_for("Jon Jones")) - PrintJSON(get_tapology_for("Conor Mcgregor")) + get_tapology_for := tapology_getter() + print_json(get_tapology_for("Dustin Poirier")) + print_json(get_tapology_for("Jon Jones")) + print_json(get_tapology_for("Conor Mcgregor")) } func TestTapologyParseFragment(t *testing.T) { - PrintJSON(parse_tapology_results(search_result)) + print_json(parse_tapology_results(search_result)) } diff --git a/scraper/ufc.go b/scraper/ufc.go index 0d39859..f6ab0c3 100644 --- a/scraper/ufc.go +++ b/scraper/ufc.go @@ -10,14 +10,7 @@ import ( "github.com/gocolly/colly" ) -func ufc_collector() *colly.Collector { - return colly.NewCollector( - colly.UserAgent(USER_AGENT), - colly.AllowedDomains("ufc.com", "www.ufc.com"), - ) -} - -func ScrapeUFC(callback EventCallback) { +func scrape_ufc(callback EventCallback) { list := ufc_collector() detail := ufc_collector() url := "" @@ -43,7 +36,7 @@ func ScrapeUFC(callback EventCallback) { func parse_ufc_event_list(b []byte) []string { urls := []string{} - doc := DocumentFromBytes(b) + doc := document_from_bytes(b) doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) { slug, ok := s.Attr("href") @@ -57,13 +50,13 @@ func parse_ufc_event_list(b []byte) []string { func parse_ufc_event(url string, b []byte) *Event { event := Event{Organization: "UFC", Url: url} - doc := DocumentFromBytes(b) + doc := document_from_bytes(b) parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string { - return CleanupString(s.Text()) + return cleanup_string(s.Text()) }) event.Name = strings.Join(parts, ": ") - event.Location = TextContent(doc, ".field--name-venue") + event.Location = text_content(doc, ".field--name-venue") event.Slug = parse_ufc_slug(url) image, ok := doc.Find(".c-hero__image img").First().Attr("src") @@ -95,7 +88,7 @@ func parse_ufc_event(url string, b []byte) *Event { func parse_ufc_fight(s *goquery.Selection) *Fight { fight := Fight{} - fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1) + fight.Weight = strings.Replace(text_content(s, ".c-listing-fight__class-text"), " Bout", "", -1) fight.FighterA = parse_ufc_fighter(s, "red") fight.FighterB = parse_ufc_fighter(s, "blue") @@ -105,8 +98,8 @@ func parse_ufc_fight(s *goquery.Selection) *Fight { func parse_ufc_fighter(s *goquery.Selection, color string) *Fighter { fighter := Fighter{} - fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color) - fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text") + fighter.Name = text_content(s, ".c-listing-fight__corner-name--"+color) + fighter.Country = text_content(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text") image, ok := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src") if ok { fighter.Image = image @@ -125,3 +118,10 @@ func parse_ufc_slug(url string) string { } return "" } + +func ufc_collector() *colly.Collector { + return colly.NewCollector( + colly.UserAgent(USER_AGENT), + colly.AllowedDomains("ufc.com", "www.ufc.com"), + ) +} diff --git a/scraper/ufc_test.go b/scraper/ufc_test.go index 9c107a7..ba9f86e 100644 --- a/scraper/ufc_test.go +++ b/scraper/ufc_test.go @@ -12,24 +12,24 @@ var UFCEventHTML []byte var UFCEventListHTML []byte func TestUFCEventList(t *testing.T) { - PrintJSON(parse_ufc_event_list(UFCEventListHTML)) + print_json(parse_ufc_event_list(UFCEventListHTML)) } func TestUFCEvent(t *testing.T) { - PrintJSON(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML)) + print_json(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML)) } func TestParseUFCSlug(t *testing.T) { result := parse_ufc_slug("https://ufc.com/event/ufc-fight-night-july-20-2024") expected := "ufc-fight-night-july-20-2024" if result != expected { - PrintJSON(result) + print_json(result) t.FailNow() } } func testScrapeUFC(t *testing.T) { - ScrapeUFC(func(e *Event) { - PrintJSON(e) + scrape_ufc(func(e *Event) { + print_json(e) }) }