moved stuff around

main.go

@@ -1,4 +1,7 @@
 package main
 
+import "mmaschedule-go/scraper"
+
 func main() {
+    scraper.CleanupString("asd")
 }

scraper/helpers.go

@@ -1,9 +1,10 @@
-package main
+package scraper
 
 import (
 	"bytes"
 	"encoding/json"
 	"hash/fnv"
+	"log"
 	"regexp"
 	"strings"
 
@@ -34,3 +35,12 @@ func HashJSON(v any) uint32 {
 	h.Write(out)
 	return h.Sum32()
 }
+
+func PrintJSON(v any) {
+	out, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		log.Println(err)
+	} else {
+		log.Println(string(out))
+	}
+}

scraper/one.go

@@ -0,0 +1,112 @@
+package scraper
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/gocolly/colly"
+)
+
+type ONEEventLocation struct {
+	Name string `json:"name"`
+}
+
+type ONEEventData struct {
+	Name      string           `json:"name"`
+	StartDate string           `json:"startDate"`
+	Image     []string         `json:"image"`
+	Location  ONEEventLocation `json:"location"`
+}
+
+func ONECollector() *colly.Collector {
+	return colly.NewCollector(
+		colly.UserAgent(USER_AGENT),
+		colly.AllowedDomains("onefc.com", "www.onefc.com"),
+	)
+}
+
+func ScrapeONE(callback EventCallback) {
+	list := ONECollector()
+	detail := ONECollector()
+	url := ""
+
+	list.OnResponse(func(r *colly.Response) {
+		for _, u := range ParseONEEventList(r.Body) {
+			url = u
+			log.Printf("Visiting %s\n", u)
+			if err := detail.Visit(u); err != nil {
+				log.Println(err)
+			}
+		}
+	})
+
+	detail.OnResponse(func(r *colly.Response) {
+		callback(ParseONEEvent(url, r.Body))
+	})
+
+	if err := list.Visit("https://www.onefc.com/events/"); err != nil {
+		log.Println(err)
+	}
+}
+
+func ParseONEEventList(b []byte) []string {
+	return DocumentFromBytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string {
+		url, _ := s.Attr("href")
+		return url
+	})
+}
+
+func ParseONEEvent(url string, b []byte) *Event {
+	event := Event{Organization: "ONE", Url: url}
+	doc := DocumentFromBytes(b)
+
+	data := ONEEventData{}
+	_ = json.Unmarshal([]byte(doc.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data)
+
+	event.Name = data.Name
+	event.Image = data.Image[2]
+	event.Location = data.Location.Name
+
+	date, err := time.Parse(time.RFC3339, data.StartDate)
+	if err == nil {
+		event.Date = int(date.Unix())
+	}
+
+	doc.Find(".event-matchup").Each(func(i int, s *goquery.Selection) {
+		event.Fights = append(event.Fights, *ParseONEFight(s))
+	})
+
+	return &event
+}
+
+func ParseONEFight(s *goquery.Selection) *Fight {
+	fight := Fight{}
+
+	fight.Weight = TextContent(s, ".title")
+	fight.FighterA = ParseONEFighter(s, true)
+	fight.FighterB = ParseONEFighter(s, false)
+
+	return &fight
+}
+
+func ParseONEFighter(s *goquery.Selection, is_a bool) *Fighter {
+	fighter := Fighter{}
+	nth_child := "1"
+	face := "1"
+
+	if is_a {
+		nth_child = "3"
+		face = "2"
+	}
+
+	fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
+	fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
+
+	image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src")
+	fighter.Image = image
+
+	return &fighter
+}

scraper/one_test.go

@@ -0,0 +1,26 @@
+package scraper
+
+import (
+	_ "embed"
+	"testing"
+)
+
+//go:embed testdata/one-fight-night-23.html
+var ONEEventHTML []byte
+
+//go:embed testdata/one-events.html
+var ONEEventListHTML []byte
+
+func TestONEEventList(t *testing.T) {
+	PrintJSON(ParseONEEventList(ONEEventListHTML))
+}
+
+func TestONEEvent(t *testing.T) {
+	PrintJSON(ParseONEEvent("some_url", ONEEventHTML))
+}
+
+func testScrapeONE(t *testing.T) {
+	ScrapeONE(func(e *Event) {
+		PrintJSON(e)
+	})
+}

scraper/scraper.go

@@ -1,6 +1,8 @@
-package main
+package scraper
 
 type Event struct {
+	Url          string  `json:"url"`
+	Slug         string  `json:"slug"`
 	Name         string  `json:"name"`
 	Location     string  `json:"location"`
 	Organization string  `json:"organization"`
@@ -19,6 +21,7 @@ type Fighter struct {
 	Name    string `json:"name"`
 	Image   string `json:"image"`
 	Country string `json:"country"`
+	Link    string `json:"link"`
 }
 
 type EventCallback func(e *Event)

scraper/tapology.go

@@ -0,0 +1,84 @@
+package scraper
+
+import (
+	"log"
+	"net/url"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/gocolly/colly"
+)
+
+const tapology_url string = "https://www.tapology.com"
+
+type TapologyResult struct {
+	Name string `json:"name"`
+	Url  string `json:"url"`
+}
+
+func tapology_collector() *colly.Collector {
+	return colly.NewCollector(
+		colly.UserAgent(USER_AGENT),
+		colly.AllowedDomains("tapology.com", "www.tapology.com"),
+	)
+}
+
+func TapologyGetter() func(n string) []TapologyResult {
+	csrf_token := ""
+	name := ""
+	results := []TapologyResult{}
+
+	index := tapology_collector()
+	index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) {
+		csrf_token = h.Attr("content")
+		time.Sleep(5 * time.Second)
+	})
+
+	search := tapology_collector()
+	search.AllowURLRevisit = true
+	search.OnRequest(func(r *colly.Request) {
+		r.Headers.Add("X-CSRF-Token", csrf_token)
+
+		query := url.Values{}
+		query.Set("ajax", "true")
+		query.Set("model", "fighters")
+		query.Set("term", name)
+		r.URL.RawQuery = query.Encode()
+
+		log.Printf("Making tapology request to %s", r.URL)
+	})
+	search.OnResponse(func(r *colly.Response) {
+		results = parse_tapology_results(r.Body)
+		time.Sleep(5 * time.Second)
+	})
+
+	if err := index.Visit(tapology_url); err != nil {
+		log.Println(err)
+	}
+
+	return func(n string) []TapologyResult {
+		if csrf_token == "" {
+			return nil
+		}
+
+		name = n
+		if err := search.Visit(tapology_url + "/search/nav"); err != nil {
+			log.Println(err)
+			return nil
+		}
+
+		return results
+	}
+}
+
+func parse_tapology_results(b []byte) []TapologyResult {
+	results := []TapologyResult{}
+
+	DocumentFromBytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
+		url, _ := s.Attr("href")
+		name := s.Text()
+		results = append(results, TapologyResult{Name: name, Url: url})
+	})
+
+	return results
+}

scraper/tapology_test.go

@@ -0,0 +1,20 @@
+package scraper
+
+import (
+	_ "embed"
+	"testing"
+)
+
+//go:embed testdata/tapology-search.html
+var search_result []byte
+
+func testTapologyGetter(t *testing.T) {
+	get_tapology_for := TapologyGetter()
+	PrintJSON(get_tapology_for("Dustin Poirier"))
+	PrintJSON(get_tapology_for("Jon Jones"))
+	PrintJSON(get_tapology_for("Conor Mcgregor"))
+}
+
+func TestTapologyParseFragment(t *testing.T) {
+    PrintJSON(parse_tapology_results(search_result))
+}

scraper/ufc.go

@@ -0,0 +1,127 @@
+package scraper
+
+import (
+	"log"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/gocolly/colly"
+)
+
+func ufc_collector() *colly.Collector {
+	return colly.NewCollector(
+		colly.UserAgent(USER_AGENT),
+		colly.AllowedDomains("ufc.com", "www.ufc.com"),
+	)
+}
+
+func ScrapeUFC(callback EventCallback) {
+	list := ufc_collector()
+	detail := ufc_collector()
+	url := ""
+
+	list.OnResponse(func(r *colly.Response) {
+		for _, u := range parse_ufc_event_list(r.Body) {
+			url = u
+			log.Printf("Visiting %s\n", u)
+			if err := detail.Visit(u); err != nil {
+				log.Println(err)
+			}
+		}
+	})
+
+	detail.OnResponse(func(r *colly.Response) {
+		callback(parse_ufc_event(url, r.Body))
+	})
+
+	if err := list.Visit("https://www.ufc.com/events"); err != nil {
+		log.Println(err)
+	}
+}
+
+func parse_ufc_event_list(b []byte) []string {
+	urls := []string{}
+	doc := DocumentFromBytes(b)
+
+	doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
+		slug, ok := s.Attr("href")
+		if ok {
+			urls = append(urls, "https://ufc.com"+slug)
+		}
+	})
+
+	return urls
+}
+
+func parse_ufc_event(url string, b []byte) *Event {
+	event := Event{Organization: "UFC", Url: url}
+	doc := DocumentFromBytes(b)
+
+	parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
+		return CleanupString(s.Text())
+	})
+	event.Name = strings.Join(parts, ": ")
+	event.Location = TextContent(doc, ".field--name-venue")
+	event.Slug = parse_ufc_slug(url)
+
+	image, ok := doc.Find(".c-hero__image img").First().Attr("src")
+	if ok {
+		event.Image = image
+	}
+
+	datestr, ok := doc.Find(`
+
+        #early-prelims .c-event-fight-card-broadcaster__time,
+        #prelims-card .c-event-fight-card-broadcaster__time,
+        .c-hero__headline-suffix
+
+    `).Last().Attr("data-timestamp")
+	if ok {
+		date, err := strconv.Atoi(datestr)
+		if err == nil {
+			event.Date = date
+		}
+	}
+
+	doc.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) {
+		event.Fights = append(event.Fights, *parse_ufc_fight(s))
+	})
+
+	return &event
+}
+
+func parse_ufc_fight(s *goquery.Selection) *Fight {
+	fight := Fight{}
+
+	fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1)
+	fight.FighterA = parse_ufc_fighter(s, "red")
+	fight.FighterB = parse_ufc_fighter(s, "blue")
+
+	return &fight
+}
+
+func parse_ufc_fighter(s *goquery.Selection, color string) *Fighter {
+	fighter := Fighter{}
+
+	fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color)
+	fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
+	image, ok := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
+	if ok {
+		fighter.Image = image
+	}
+
+	return &fighter
+}
+
+var ufc_event_url *regexp.Regexp = regexp.MustCompile(`https://ufc.com/event/(?P<Slug>(\w|-)+)`)
+
+func parse_ufc_slug(url string) string {
+	matches := ufc_event_url.FindStringSubmatch(url)
+	index := ufc_event_url.SubexpIndex("Slug")
+	if len(matches) > index {
+		return matches[index]
+	}
+	return ""
+}

scraper/ufc_test.go

@@ -0,0 +1,35 @@
+package scraper
+
+import (
+	_ "embed"
+	"testing"
+)
+
+//go:embed testdata/ufc-297-good.html
+var UFCEventHTML []byte
+
+//go:embed testdata/ufc-events.html
+var UFCEventListHTML []byte
+
+func TestUFCEventList(t *testing.T) {
+	PrintJSON(parse_ufc_event_list(UFCEventListHTML))
+}
+
+func TestUFCEvent(t *testing.T) {
+	PrintJSON(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML))
+}
+
+func TestParseUFCSlug(t *testing.T)  {
+    result := parse_ufc_slug("https://ufc.com/event/ufc-fight-night-july-20-2024")
+    expected := "ufc-fight-night-july-20-2024"
+    if result != expected {
+        PrintJSON(result)
+        t.FailNow()
+    }
+}
+
+func testScrapeUFC(t *testing.T) {
+	ScrapeUFC(func(e *Event) {
+		PrintJSON(e)
+	})
+}

ufc.go

@@ -1,114 +0,0 @@
-package main
-
-import (
-	"log"
-	"strconv"
-	"strings"
-
-	"github.com/PuerkitoBio/goquery"
-	"github.com/gocolly/colly"
-)
-
-func UFCCollector() *colly.Collector {
-	return colly.NewCollector(
-		colly.UserAgent(USER_AGENT),
-		colly.AllowedDomains("ufc.com", "www.ufc.com"),
-	)
-}
-
-func ScrapeUFC(callback EventCallback) {
-	ufclist := UFCCollector()
-	ufcdetail := UFCCollector()
-
-	ufclist.OnResponse(func(r *colly.Response) {
-		for _, url := range ParseUFCEventList(r.Body) {
-			log.Printf("Visiting %s\n", url)
-			err := ufcdetail.Visit(url)
-			if err != nil {
-				log.Println(err)
-			}
-		}
-	})
-
-	ufcdetail.OnResponse(func(r *colly.Response) {
-		callback(ParseUFCEvent(r.Body))
-	})
-
-	err := ufclist.Visit("https://www.ufc.com/events")
-	if err != nil {
-		log.Println(err)
-	}
-}
-
-func ParseUFCEventList(b []byte) []string {
-	urls := []string{}
-	doc := DocumentFromBytes(b)
-
-	doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
-		slug, ok := s.Attr("href")
-		if ok {
-			urls = append(urls, "https://ufc.com"+slug)
-		}
-	})
-
-	return urls
-}
-
-func ParseUFCEvent(b []byte) *Event {
-	event := Event{Organization: "UFC"}
-	doc := DocumentFromBytes(b)
-
-	parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
-		return CleanupString(s.Text())
-	})
-	event.Name = strings.Join(parts, ": ")
-	event.Location = TextContent(doc, ".field--name-venue")
-
-	image, ok := doc.Find(".c-hero__image img").First().Attr("src")
-	if ok {
-		event.Image = image
-	}
-
-	datestr, ok := doc.Find(`
-
-        #early-prelims .c-event-fight-card-broadcaster__time,
-        #prelims-card .c-event-fight-card-broadcaster__time,
-        .c-hero__headline-suffix
-
-    `).Last().Attr("data-timestamp")
-	if ok {
-		date, err := strconv.Atoi(datestr)
-		if err == nil {
-			event.Date = date
-		}
-	}
-
-	doc.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) {
-		event.Fights = append(event.Fights, *ParseUFCFight(s))
-	})
-
-	return &event
-}
-
-func ParseUFCFight(doc *goquery.Selection) *Fight {
-	fight := Fight{}
-
-	fight.Weight = strings.Replace(TextContent(doc, ".c-listing-fight__class-text"), " Bout", "", -1)
-	fight.FighterA = ParseUFCFighter(doc, "red")
-	fight.FighterB = ParseUFCFighter(doc, "blue")
-
-	return &fight
-}
-
-func ParseUFCFighter(doc *goquery.Selection, color string) *Fighter {
-	fighter := Fighter{}
-
-	fighter.Name = TextContent(doc, ".c-listing-fight__corner-name--"+color)
-	fighter.Country = TextContent(doc, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
-	image, ok := doc.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
-	if ok {
-		fighter.Image = image
-	}
-
-	return &fighter
-}

ufc_test.go

@@ -1,39 +0,0 @@
-package main
-
-import (
-	_ "embed"
-	"encoding/json"
-	"log"
-	"testing"
-)
-
-//go:embed testdata/ufc-297-good.html
-var UFCEventHTML []byte
-
-//go:embed testdata/ufc-events.html
-var UFCEventListHTML []byte
-
-func TestUFCEventList(t *testing.T) {
-	urls := ParseUFCEventList(UFCEventListHTML)
-    PrintJSON(urls)
-}
-
-func TestUFCEvent(t *testing.T) {
-	event := ParseUFCEvent(UFCEventHTML)
-    PrintJSON(event)
-}
-
-func testScrapeUFC(t *testing.T) {
-    ScrapeUFC(func(e *Event) {
-        PrintJSON(e)
-    })
-}
-
-func PrintJSON(v any) {
-	out, err := json.MarshalIndent(v, "", "  ")
-	if err != nil {
-		log.Println(err)
-	} else {
-        log.Println(string(out))
-    }
-}