made stuff private

main.go

@@ -1,7 +1,4 @@
 package main
 
-import "mmaschedule-go/scraper"
-
 func main() {
-    scraper.CleanupString("asd")
 }

scraper/helpers.go

@@ -3,7 +3,6 @@ package scraper
 import (
 	"bytes"
 	"encoding/json"
-	"hash/fnv"
 	"log"
 	"regexp"
 	"strings"
@@ -11,32 +10,22 @@ import (
 	"github.com/PuerkitoBio/goquery"
 )
 
-func TextContent(doc *goquery.Selection, selector string) string {
+func text_content(doc *goquery.Selection, selector string) string {
-	return CleanupString(doc.Find(selector).First().Text())
+	return cleanup_string(doc.Find(selector).First().Text())
 }
 
 var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
 
-func CleanupString(s string) string {
+func cleanup_string(s string) string {
 	return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
 }
 
-func DocumentFromBytes(b []byte) *goquery.Selection {
+func document_from_bytes(b []byte) *goquery.Selection {
 	doc, _ := goquery.NewDocumentFromReader(bytes.NewReader(b))
 	return doc.Find("html").First()
 }
 
-func HashJSON(v any) uint32 {
+func print_json(v any) {
-	out, err := json.Marshal(v)
-	if err != nil {
-		return 0
-	}
-	h := fnv.New32a()
-	h.Write(out)
-	return h.Sum32()
-}
-
-func PrintJSON(v any) {
 	out, err := json.MarshalIndent(v, "", "  ")
 	if err != nil {
 		log.Println(err)

scraper/one.go

@@ -10,31 +10,24 @@ import (
 	"github.com/gocolly/colly"
 )
 
-type ONEEventLocation struct {
+type event_location struct {
 	Name string `json:"name"`
 }
 
-type ONEEventData struct {
+type event_data struct {
 	Name      string         `json:"name"`
 	StartDate string         `json:"startDate"`
 	Image     []string       `json:"image"`
-	Location  ONEEventLocation `json:"location"`
+	Location  event_location `json:"location"`
 }
 
-func ONECollector() *colly.Collector {
+func scrape_one(callback EventCallback) {
-	return colly.NewCollector(
+	list := one_collector()
-		colly.UserAgent(USER_AGENT),
+	detail := one_collector()
-		colly.AllowedDomains("onefc.com", "www.onefc.com"),
-	)
-}
-
-func ScrapeONE(callback EventCallback) {
-	list := ONECollector()
-	detail := ONECollector()
 	url := ""
 
 	list.OnResponse(func(r *colly.Response) {
-		for _, u := range ParseONEEventList(r.Body) {
+		for _, u := range parse_one_event_list(r.Body) {
 			url = u
 			log.Printf("Visiting %s\n", u)
 			if err := detail.Visit(u); err != nil {
@@ -44,7 +37,7 @@ func ScrapeONE(callback EventCallback) {
 	})
 
 	detail.OnResponse(func(r *colly.Response) {
-		callback(ParseONEEvent(url, r.Body))
+		callback(parse_one_event(url, r.Body))
 	})
 
 	if err := list.Visit("https://www.onefc.com/events/"); err != nil {
@@ -52,47 +45,50 @@ func ScrapeONE(callback EventCallback) {
 	}
 }
 
-func ParseONEEventList(b []byte) []string {
+func parse_one_event_list(b []byte) []string {
-	return DocumentFromBytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string {
+	return document_from_bytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string {
 		url, _ := s.Attr("href")
 		return url
 	})
 }
 
-func ParseONEEvent(url string, b []byte) *Event {
+func parse_one_event(url string, b []byte) *Event {
 	event := Event{Organization: "ONE", Url: url}
-	doc := DocumentFromBytes(b)
+	doc := document_from_bytes(b)
 
-	data := ONEEventData{}
+	data := event_data{}
 	_ = json.Unmarshal([]byte(doc.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data)
 
 	event.Name = data.Name
-	event.Image = data.Image[2]
 	event.Location = data.Location.Name
 
+	if len(data.Image) > 2 {
+		event.Image = data.Image[2]
+	}
+
 	date, err := time.Parse(time.RFC3339, data.StartDate)
 	if err == nil {
 		event.Date = int(date.Unix())
 	}
 
 	doc.Find(".event-matchup").Each(func(i int, s *goquery.Selection) {
-		event.Fights = append(event.Fights, *ParseONEFight(s))
+		event.Fights = append(event.Fights, *parse_one_fight(s))
 	})
 
 	return &event
 }
 
-func ParseONEFight(s *goquery.Selection) *Fight {
+func parse_one_fight(s *goquery.Selection) *Fight {
 	fight := Fight{}
 
-	fight.Weight = TextContent(s, ".title")
+	fight.Weight = text_content(s, ".title")
-	fight.FighterA = ParseONEFighter(s, true)
+	fight.FighterA = parse_one_fighter(s, true)
-	fight.FighterB = ParseONEFighter(s, false)
+	fight.FighterB = parse_one_fighter(s, false)
 
 	return &fight
 }
 
-func ParseONEFighter(s *goquery.Selection, is_a bool) *Fighter {
+func parse_one_fighter(s *goquery.Selection, is_a bool) *Fighter {
 	fighter := Fighter{}
 	nth_child := "1"
 	face := "1"
@@ -102,11 +98,18 @@ func ParseONEFighter(s *goquery.Selection, is_a bool) *Fighter {
 		face = "2"
 	}
 
-	fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
+	fighter.Name = text_content(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
-	fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
+	fighter.Country = text_content(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
 
 	image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src")
 	fighter.Image = image
 
 	return &fighter
 }
+
+func one_collector() *colly.Collector {
+	return colly.NewCollector(
+		colly.UserAgent(USER_AGENT),
+		colly.AllowedDomains("onefc.com", "www.onefc.com"),
+	)
+}

scraper/one_test.go

@@ -12,15 +12,15 @@ var ONEEventHTML []byte
 var ONEEventListHTML []byte
 
 func TestONEEventList(t *testing.T) {
-	PrintJSON(ParseONEEventList(ONEEventListHTML))
+	print_json(parse_one_event_list(ONEEventListHTML))
 }
 
 func TestONEEvent(t *testing.T) {
-	PrintJSON(ParseONEEvent("some_url", ONEEventHTML))
+	print_json(parse_one_event("some_url", ONEEventHTML))
 }
 
 func testScrapeONE(t *testing.T) {
-	ScrapeONE(func(e *Event) {
+	scrape_one(func(e *Event) {
-		PrintJSON(e)
+		print_json(e)
 	})
 }

scraper/scraper.go

@@ -29,6 +29,7 @@ type EventScraper func(c EventCallback)
 
 const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
 
-var Scrapers []EventScraper = []EventScraper{
+var scrapers []EventScraper = []EventScraper{
-	ScrapeUFC,
+	scrape_ufc,
+    scrape_one,
 }

scraper/tapology.go

@@ -16,14 +16,7 @@ type TapologyResult struct {
 	Url  string `json:"url"`
 }
 
-func tapology_collector() *colly.Collector {
+func tapology_getter() func(n string) []TapologyResult {
-	return colly.NewCollector(
-		colly.UserAgent(USER_AGENT),
-		colly.AllowedDomains("tapology.com", "www.tapology.com"),
-	)
-}
-
-func TapologyGetter() func(n string) []TapologyResult {
 	csrf_token := ""
 	name := ""
 	results := []TapologyResult{}
@@ -74,7 +67,7 @@ func TapologyGetter() func(n string) []TapologyResult {
 func parse_tapology_results(b []byte) []TapologyResult {
 	results := []TapologyResult{}
 
-	DocumentFromBytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
+	document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
 		url, _ := s.Attr("href")
 		name := s.Text()
 		results = append(results, TapologyResult{Name: name, Url: url})
@@ -82,3 +75,10 @@ func parse_tapology_results(b []byte) []TapologyResult {
 
 	return results
 }
+
+func tapology_collector() *colly.Collector {
+	return colly.NewCollector(
+		colly.UserAgent(USER_AGENT),
+		colly.AllowedDomains("tapology.com", "www.tapology.com"),
+	)
+}

scraper/tapology_test.go

@@ -9,12 +9,12 @@ import (
 var search_result []byte
 
 func testTapologyGetter(t *testing.T) {
-	get_tapology_for := TapologyGetter()
+	get_tapology_for := tapology_getter()
-	PrintJSON(get_tapology_for("Dustin Poirier"))
+	print_json(get_tapology_for("Dustin Poirier"))
-	PrintJSON(get_tapology_for("Jon Jones"))
+	print_json(get_tapology_for("Jon Jones"))
-	PrintJSON(get_tapology_for("Conor Mcgregor"))
+	print_json(get_tapology_for("Conor Mcgregor"))
 }
 
 func TestTapologyParseFragment(t *testing.T) {
-    PrintJSON(parse_tapology_results(search_result))
+    print_json(parse_tapology_results(search_result))
 }

scraper/ufc.go

@@ -10,14 +10,7 @@ import (
 	"github.com/gocolly/colly"
 )
 
-func ufc_collector() *colly.Collector {
+func scrape_ufc(callback EventCallback) {
-	return colly.NewCollector(
-		colly.UserAgent(USER_AGENT),
-		colly.AllowedDomains("ufc.com", "www.ufc.com"),
-	)
-}
-
-func ScrapeUFC(callback EventCallback) {
 	list := ufc_collector()
 	detail := ufc_collector()
 	url := ""
@@ -43,7 +36,7 @@ func ScrapeUFC(callback EventCallback) {
 
 func parse_ufc_event_list(b []byte) []string {
 	urls := []string{}
-	doc := DocumentFromBytes(b)
+	doc := document_from_bytes(b)
 
 	doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
 		slug, ok := s.Attr("href")
@@ -57,13 +50,13 @@ func parse_ufc_event_list(b []byte) []string {
 
 func parse_ufc_event(url string, b []byte) *Event {
 	event := Event{Organization: "UFC", Url: url}
-	doc := DocumentFromBytes(b)
+	doc := document_from_bytes(b)
 
 	parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
-		return CleanupString(s.Text())
+		return cleanup_string(s.Text())
 	})
 	event.Name = strings.Join(parts, ": ")
-	event.Location = TextContent(doc, ".field--name-venue")
+	event.Location = text_content(doc, ".field--name-venue")
 	event.Slug = parse_ufc_slug(url)
 
 	image, ok := doc.Find(".c-hero__image img").First().Attr("src")
@@ -95,7 +88,7 @@ func parse_ufc_event(url string, b []byte) *Event {
 func parse_ufc_fight(s *goquery.Selection) *Fight {
 	fight := Fight{}
 
-	fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1)
+	fight.Weight = strings.Replace(text_content(s, ".c-listing-fight__class-text"), " Bout", "", -1)
 	fight.FighterA = parse_ufc_fighter(s, "red")
 	fight.FighterB = parse_ufc_fighter(s, "blue")
 
@@ -105,8 +98,8 @@ func parse_ufc_fight(s *goquery.Selection) *Fight {
 func parse_ufc_fighter(s *goquery.Selection, color string) *Fighter {
 	fighter := Fighter{}
 
-	fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color)
+	fighter.Name = text_content(s, ".c-listing-fight__corner-name--"+color)
-	fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
+	fighter.Country = text_content(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
 	image, ok := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
 	if ok {
 		fighter.Image = image
@@ -125,3 +118,10 @@ func parse_ufc_slug(url string) string {
 	}
 	return ""
 }
+
+func ufc_collector() *colly.Collector {
+	return colly.NewCollector(
+		colly.UserAgent(USER_AGENT),
+		colly.AllowedDomains("ufc.com", "www.ufc.com"),
+	)
+}

scraper/ufc_test.go

@@ -12,24 +12,24 @@ var UFCEventHTML []byte
 var UFCEventListHTML []byte
 
 func TestUFCEventList(t *testing.T) {
-	PrintJSON(parse_ufc_event_list(UFCEventListHTML))
+	print_json(parse_ufc_event_list(UFCEventListHTML))
 }
 
 func TestUFCEvent(t *testing.T) {
-	PrintJSON(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML))
+	print_json(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML))
 }
 
 func TestParseUFCSlug(t *testing.T)  {
     result := parse_ufc_slug("https://ufc.com/event/ufc-fight-night-july-20-2024")
     expected := "ufc-fight-night-july-20-2024"
     if result != expected {
-        PrintJSON(result)
+        print_json(result)
         t.FailNow()
     }
 }
 
 func testScrapeUFC(t *testing.T) {
-	ScrapeUFC(func(e *Event) {
+	scrape_ufc(func(e *Event) {
-		PrintJSON(e)
+		print_json(e)
 	})
 }