diff --git a/main.go b/main.go index da29a2c..37ebfaf 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,7 @@ package main +import "mmaschedule-go/scraper" + func main() { + scraper.CleanupString("asd") } diff --git a/helpers.go b/scraper/helpers.go similarity index 80% rename from helpers.go rename to scraper/helpers.go index a97238d..6b43262 100644 --- a/helpers.go +++ b/scraper/helpers.go @@ -1,9 +1,10 @@ -package main +package scraper import ( "bytes" "encoding/json" "hash/fnv" + "log" "regexp" "strings" @@ -34,3 +35,12 @@ func HashJSON(v any) uint32 { h.Write(out) return h.Sum32() } + +func PrintJSON(v any) { + out, err := json.MarshalIndent(v, "", " ") + if err != nil { + log.Println(err) + } else { + log.Println(string(out)) + } +} diff --git a/scraper/one.go b/scraper/one.go new file mode 100644 index 0000000..5251eaa --- /dev/null +++ b/scraper/one.go @@ -0,0 +1,112 @@ +package scraper + +import ( + "encoding/json" + "fmt" + "log" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/gocolly/colly" +) + +type ONEEventLocation struct { + Name string `json:"name"` +} + +type ONEEventData struct { + Name string `json:"name"` + StartDate string `json:"startDate"` + Image []string `json:"image"` + Location ONEEventLocation `json:"location"` +} + +func ONECollector() *colly.Collector { + return colly.NewCollector( + colly.UserAgent(USER_AGENT), + colly.AllowedDomains("onefc.com", "www.onefc.com"), + ) +} + +func ScrapeONE(callback EventCallback) { + list := ONECollector() + detail := ONECollector() + url := "" + + list.OnResponse(func(r *colly.Response) { + for _, u := range ParseONEEventList(r.Body) { + url = u + log.Printf("Visiting %s\n", u) + if err := detail.Visit(u); err != nil { + log.Println(err) + } + } + }) + + detail.OnResponse(func(r *colly.Response) { + callback(ParseONEEvent(url, r.Body)) + }) + + if err := list.Visit("https://www.onefc.com/events/"); err != nil { + log.Println(err) + } +} + +func ParseONEEventList(b []byte) []string { + return DocumentFromBytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string { + url, _ := s.Attr("href") + return url + }) +} + +func ParseONEEvent(url string, b []byte) *Event { + event := Event{Organization: "ONE", Url: url} + doc := DocumentFromBytes(b) + + data := ONEEventData{} + _ = json.Unmarshal([]byte(doc.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data) + + event.Name = data.Name + event.Image = data.Image[2] + event.Location = data.Location.Name + + date, err := time.Parse(time.RFC3339, data.StartDate) + if err == nil { + event.Date = int(date.Unix()) + } + + doc.Find(".event-matchup").Each(func(i int, s *goquery.Selection) { + event.Fights = append(event.Fights, *ParseONEFight(s)) + }) + + return &event +} + +func ParseONEFight(s *goquery.Selection) *Fight { + fight := Fight{} + + fight.Weight = TextContent(s, ".title") + fight.FighterA = ParseONEFighter(s, true) + fight.FighterB = ParseONEFighter(s, false) + + return &fight +} + +func ParseONEFighter(s *goquery.Selection, is_a bool) *Fighter { + fighter := Fighter{} + nth_child := "1" + face := "1" + + if is_a { + nth_child = "3" + face = "2" + } + + fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child)) + fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child)) + + image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src") + fighter.Image = image + + return &fighter +} diff --git a/scraper/one_test.go b/scraper/one_test.go new file mode 100644 index 0000000..b260d2d --- /dev/null +++ b/scraper/one_test.go @@ -0,0 +1,26 @@ +package scraper + +import ( + _ "embed" + "testing" +) + +//go:embed testdata/one-fight-night-23.html +var ONEEventHTML []byte + +//go:embed testdata/one-events.html +var ONEEventListHTML []byte + +func TestONEEventList(t *testing.T) { + PrintJSON(ParseONEEventList(ONEEventListHTML)) +} + +func TestONEEvent(t *testing.T) { + PrintJSON(ParseONEEvent("some_url", ONEEventHTML)) +} + +func testScrapeONE(t *testing.T) { + ScrapeONE(func(e *Event) { + PrintJSON(e) + }) +} diff --git a/scraper.go b/scraper/scraper.go similarity index 86% rename from scraper.go rename to scraper/scraper.go index 848f361..e4b6c40 100644 --- a/scraper.go +++ b/scraper/scraper.go @@ -1,6 +1,8 @@ -package main +package scraper type Event struct { + Url string `json:"url"` + Slug string `json:"slug"` Name string `json:"name"` Location string `json:"location"` Organization string `json:"organization"` @@ -19,6 +21,7 @@ type Fighter struct { Name string `json:"name"` Image string `json:"image"` Country string `json:"country"` + Link string `json:"link"` } type EventCallback func(e *Event) diff --git a/scraper/tapology.go b/scraper/tapology.go new file mode 100644 index 0000000..f6821f5 --- /dev/null +++ b/scraper/tapology.go @@ -0,0 +1,84 @@ +package scraper + +import ( + "log" + "net/url" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/gocolly/colly" +) + +const tapology_url string = "https://www.tapology.com" + +type TapologyResult struct { + Name string `json:"name"` + Url string `json:"url"` +} + +func tapology_collector() *colly.Collector { + return colly.NewCollector( + colly.UserAgent(USER_AGENT), + colly.AllowedDomains("tapology.com", "www.tapology.com"), + ) +} + +func TapologyGetter() func(n string) []TapologyResult { + csrf_token := "" + name := "" + results := []TapologyResult{} + + index := tapology_collector() + index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) { + csrf_token = h.Attr("content") + time.Sleep(5 * time.Second) + }) + + search := tapology_collector() + search.AllowURLRevisit = true + search.OnRequest(func(r *colly.Request) { + r.Headers.Add("X-CSRF-Token", csrf_token) + + query := url.Values{} + query.Set("ajax", "true") + query.Set("model", "fighters") + query.Set("term", name) + r.URL.RawQuery = query.Encode() + + log.Printf("Making tapology request to %s", r.URL) + }) + search.OnResponse(func(r *colly.Response) { + results = parse_tapology_results(r.Body) + time.Sleep(5 * time.Second) + }) + + if err := index.Visit(tapology_url); err != nil { + log.Println(err) + } + + return func(n string) []TapologyResult { + if csrf_token == "" { + return nil + } + + name = n + if err := search.Visit(tapology_url + "/search/nav"); err != nil { + log.Println(err) + return nil + } + + return results + } +} + +func parse_tapology_results(b []byte) []TapologyResult { + results := []TapologyResult{} + + DocumentFromBytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) { + url, _ := s.Attr("href") + name := s.Text() + results = append(results, TapologyResult{Name: name, Url: url}) + }) + + return results +} diff --git a/scraper/tapology_test.go b/scraper/tapology_test.go new file mode 100644 index 0000000..a45101e --- /dev/null +++ b/scraper/tapology_test.go @@ -0,0 +1,20 @@ +package scraper + +import ( + _ "embed" + "testing" +) + +//go:embed testdata/tapology-search.html +var search_result []byte + +func testTapologyGetter(t *testing.T) { + get_tapology_for := TapologyGetter() + PrintJSON(get_tapology_for("Dustin Poirier")) + PrintJSON(get_tapology_for("Jon Jones")) + PrintJSON(get_tapology_for("Conor Mcgregor")) +} + +func TestTapologyParseFragment(t *testing.T) { + PrintJSON(parse_tapology_results(search_result)) +} diff --git a/scraper/ufc.go b/scraper/ufc.go new file mode 100644 index 0000000..0d39859 --- /dev/null +++ b/scraper/ufc.go @@ -0,0 +1,127 @@ +package scraper + +import ( + "log" + "regexp" + "strconv" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/gocolly/colly" +) + +func ufc_collector() *colly.Collector { + return colly.NewCollector( + colly.UserAgent(USER_AGENT), + colly.AllowedDomains("ufc.com", "www.ufc.com"), + ) +} + +func ScrapeUFC(callback EventCallback) { + list := ufc_collector() + detail := ufc_collector() + url := "" + + list.OnResponse(func(r *colly.Response) { + for _, u := range parse_ufc_event_list(r.Body) { + url = u + log.Printf("Visiting %s\n", u) + if err := detail.Visit(u); err != nil { + log.Println(err) + } + } + }) + + detail.OnResponse(func(r *colly.Response) { + callback(parse_ufc_event(url, r.Body)) + }) + + if err := list.Visit("https://www.ufc.com/events"); err != nil { + log.Println(err) + } +} + +func parse_ufc_event_list(b []byte) []string { + urls := []string{} + doc := DocumentFromBytes(b) + + doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) { + slug, ok := s.Attr("href") + if ok { + urls = append(urls, "https://ufc.com"+slug) + } + }) + + return urls +} + +func parse_ufc_event(url string, b []byte) *Event { + event := Event{Organization: "UFC", Url: url} + doc := DocumentFromBytes(b) + + parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string { + return CleanupString(s.Text()) + }) + event.Name = strings.Join(parts, ": ") + event.Location = TextContent(doc, ".field--name-venue") + event.Slug = parse_ufc_slug(url) + + image, ok := doc.Find(".c-hero__image img").First().Attr("src") + if ok { + event.Image = image + } + + datestr, ok := doc.Find(` + + #early-prelims .c-event-fight-card-broadcaster__time, + #prelims-card .c-event-fight-card-broadcaster__time, + .c-hero__headline-suffix + + `).Last().Attr("data-timestamp") + if ok { + date, err := strconv.Atoi(datestr) + if err == nil { + event.Date = date + } + } + + doc.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) { + event.Fights = append(event.Fights, *parse_ufc_fight(s)) + }) + + return &event +} + +func parse_ufc_fight(s *goquery.Selection) *Fight { + fight := Fight{} + + fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1) + fight.FighterA = parse_ufc_fighter(s, "red") + fight.FighterB = parse_ufc_fighter(s, "blue") + + return &fight +} + +func parse_ufc_fighter(s *goquery.Selection, color string) *Fighter { + fighter := Fighter{} + + fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color) + fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text") + image, ok := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src") + if ok { + fighter.Image = image + } + + return &fighter +} + +var ufc_event_url *regexp.Regexp = regexp.MustCompile(`https://ufc.com/event/(?P(\w|-)+)`) + +func parse_ufc_slug(url string) string { + matches := ufc_event_url.FindStringSubmatch(url) + index := ufc_event_url.SubexpIndex("Slug") + if len(matches) > index { + return matches[index] + } + return "" +} diff --git a/scraper/ufc_test.go b/scraper/ufc_test.go new file mode 100644 index 0000000..9c107a7 --- /dev/null +++ b/scraper/ufc_test.go @@ -0,0 +1,35 @@ +package scraper + +import ( + _ "embed" + "testing" +) + +//go:embed testdata/ufc-297-good.html +var UFCEventHTML []byte + +//go:embed testdata/ufc-events.html +var UFCEventListHTML []byte + +func TestUFCEventList(t *testing.T) { + PrintJSON(parse_ufc_event_list(UFCEventListHTML)) +} + +func TestUFCEvent(t *testing.T) { + PrintJSON(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML)) +} + +func TestParseUFCSlug(t *testing.T) { + result := parse_ufc_slug("https://ufc.com/event/ufc-fight-night-july-20-2024") + expected := "ufc-fight-night-july-20-2024" + if result != expected { + PrintJSON(result) + t.FailNow() + } +} + +func testScrapeUFC(t *testing.T) { + ScrapeUFC(func(e *Event) { + PrintJSON(e) + }) +} diff --git a/ufc.go b/ufc.go deleted file mode 100644 index 0e46a5b..0000000 --- a/ufc.go +++ /dev/null @@ -1,114 +0,0 @@ -package main - -import ( - "log" - "strconv" - "strings" - - "github.com/PuerkitoBio/goquery" - "github.com/gocolly/colly" -) - -func UFCCollector() *colly.Collector { - return colly.NewCollector( - colly.UserAgent(USER_AGENT), - colly.AllowedDomains("ufc.com", "www.ufc.com"), - ) -} - -func ScrapeUFC(callback EventCallback) { - ufclist := UFCCollector() - ufcdetail := UFCCollector() - - ufclist.OnResponse(func(r *colly.Response) { - for _, url := range ParseUFCEventList(r.Body) { - log.Printf("Visiting %s\n", url) - err := ufcdetail.Visit(url) - if err != nil { - log.Println(err) - } - } - }) - - ufcdetail.OnResponse(func(r *colly.Response) { - callback(ParseUFCEvent(r.Body)) - }) - - err := ufclist.Visit("https://www.ufc.com/events") - if err != nil { - log.Println(err) - } -} - -func ParseUFCEventList(b []byte) []string { - urls := []string{} - doc := DocumentFromBytes(b) - - doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) { - slug, ok := s.Attr("href") - if ok { - urls = append(urls, "https://ufc.com"+slug) - } - }) - - return urls -} - -func ParseUFCEvent(b []byte) *Event { - event := Event{Organization: "UFC"} - doc := DocumentFromBytes(b) - - parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string { - return CleanupString(s.Text()) - }) - event.Name = strings.Join(parts, ": ") - event.Location = TextContent(doc, ".field--name-venue") - - image, ok := doc.Find(".c-hero__image img").First().Attr("src") - if ok { - event.Image = image - } - - datestr, ok := doc.Find(` - - #early-prelims .c-event-fight-card-broadcaster__time, - #prelims-card .c-event-fight-card-broadcaster__time, - .c-hero__headline-suffix - - `).Last().Attr("data-timestamp") - if ok { - date, err := strconv.Atoi(datestr) - if err == nil { - event.Date = date - } - } - - doc.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) { - event.Fights = append(event.Fights, *ParseUFCFight(s)) - }) - - return &event -} - -func ParseUFCFight(doc *goquery.Selection) *Fight { - fight := Fight{} - - fight.Weight = strings.Replace(TextContent(doc, ".c-listing-fight__class-text"), " Bout", "", -1) - fight.FighterA = ParseUFCFighter(doc, "red") - fight.FighterB = ParseUFCFighter(doc, "blue") - - return &fight -} - -func ParseUFCFighter(doc *goquery.Selection, color string) *Fighter { - fighter := Fighter{} - - fighter.Name = TextContent(doc, ".c-listing-fight__corner-name--"+color) - fighter.Country = TextContent(doc, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text") - image, ok := doc.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src") - if ok { - fighter.Image = image - } - - return &fighter -} diff --git a/ufc_test.go b/ufc_test.go deleted file mode 100644 index be5c2b7..0000000 --- a/ufc_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - _ "embed" - "encoding/json" - "log" - "testing" -) - -//go:embed testdata/ufc-297-good.html -var UFCEventHTML []byte - -//go:embed testdata/ufc-events.html -var UFCEventListHTML []byte - -func TestUFCEventList(t *testing.T) { - urls := ParseUFCEventList(UFCEventListHTML) - PrintJSON(urls) -} - -func TestUFCEvent(t *testing.T) { - event := ParseUFCEvent(UFCEventHTML) - PrintJSON(event) -} - -func testScrapeUFC(t *testing.T) { - ScrapeUFC(func(e *Event) { - PrintJSON(e) - }) -} - -func PrintJSON(v any) { - out, err := json.MarshalIndent(v, "", " ") - if err != nil { - log.Println(err) - } else { - log.Println(string(out)) - } -}