guy176251/mmaschedule
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

made stuff private

Browse Source
This commit is contained in:
long
2024-07-02 14:14:38 -05:00
parent eaf229abde
commit ad7b487954
9 changed files with 81 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.go
-3
View File
@@ -1,7 +1,4 @@
package main
import "mmaschedule-go/scraper"
func main() {
scraper.CleanupString("asd")
}
scraper/helpers.go
+5 -16
View File
@@ -3,7 +3,6 @@ package scraper
import (
"bytes"
"encoding/json"
"hash/fnv"
"log"
"regexp"
"strings"
@@ -11,32 +10,22 @@ import (
"github.com/PuerkitoBio/goquery"
)
func TextContent(doc *goquery.Selection, selector string) string {
return CleanupString(doc.Find(selector).First().Text())
func text_content(doc *goquery.Selection, selector string) string {
return cleanup_string(doc.Find(selector).First().Text())
}
var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
func CleanupString(s string) string {
func cleanup_string(s string) string {
return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
}
func DocumentFromBytes(b []byte) *goquery.Selection {
func document_from_bytes(b []byte) *goquery.Selection {
doc, _ := goquery.NewDocumentFromReader(bytes.NewReader(b))
return doc.Find("html").First()
}
func HashJSON(v any) uint32 {
out, err := json.Marshal(v)
if err != nil {
return 0
}
h := fnv.New32a()
h.Write(out)
return h.Sum32()
}
func PrintJSON(v any) {
func print_json(v any) {
out, err := json.MarshalIndent(v, "", " ")
if err != nil {
log.Println(err)
scraper/one.go
+35 -32
View File
@@ -10,31 +10,24 @@ import (
"github.com/gocolly/colly"
)
type ONEEventLocation struct {
type event_location struct {
Name string `json:"name"`
}
type ONEEventData struct {
Name string `json:"name"`
StartDate string `json:"startDate"`
Image []string `json:"image"`
Location ONEEventLocation `json:"location"`
type event_data struct {
Name string `json:"name"`
StartDate string `json:"startDate"`
Image []string `json:"image"`
Location event_location `json:"location"`
}
func ONECollector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("onefc.com", "www.onefc.com"),
)
}
func ScrapeONE(callback EventCallback) {
list := ONECollector()
detail := ONECollector()
func scrape_one(callback EventCallback) {
list := one_collector()
detail := one_collector()
url := ""
list.OnResponse(func(r *colly.Response) {
for _, u := range ParseONEEventList(r.Body) {
for _, u := range parse_one_event_list(r.Body) {
url = u
log.Printf("Visiting %s\n", u)
if err := detail.Visit(u); err != nil {
@@ -44,7 +37,7 @@ func ScrapeONE(callback EventCallback) {
})
detail.OnResponse(func(r *colly.Response) {
callback(ParseONEEvent(url, r.Body))
callback(parse_one_event(url, r.Body))
})
if err := list.Visit("https://www.onefc.com/events/"); err != nil {
@@ -52,47 +45,50 @@ func ScrapeONE(callback EventCallback) {
}
}
func ParseONEEventList(b []byte) []string {
return DocumentFromBytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string {
func parse_one_event_list(b []byte) []string {
return document_from_bytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string {
url, _ := s.Attr("href")
return url
})
}
func ParseONEEvent(url string, b []byte) *Event {
func parse_one_event(url string, b []byte) *Event {
event := Event{Organization: "ONE", Url: url}
doc := DocumentFromBytes(b)
doc := document_from_bytes(b)
data := ONEEventData{}
data := event_data{}
_ = json.Unmarshal([]byte(doc.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data)
event.Name = data.Name
event.Image = data.Image[2]
event.Location = data.Location.Name
if len(data.Image) > 2 {
event.Image = data.Image[2]
}
date, err := time.Parse(time.RFC3339, data.StartDate)
if err == nil {
event.Date = int(date.Unix())
}
doc.Find(".event-matchup").Each(func(i int, s *goquery.Selection) {
event.Fights = append(event.Fights, *ParseONEFight(s))
event.Fights = append(event.Fights, *parse_one_fight(s))
})
return &event
}
func ParseONEFight(s *goquery.Selection) *Fight {
func parse_one_fight(s *goquery.Selection) *Fight {
fight := Fight{}
fight.Weight = TextContent(s, ".title")
fight.FighterA = ParseONEFighter(s, true)
fight.FighterB = ParseONEFighter(s, false)
fight.Weight = text_content(s, ".title")
fight.FighterA = parse_one_fighter(s, true)
fight.FighterB = parse_one_fighter(s, false)
return &fight
}
func ParseONEFighter(s *goquery.Selection, is_a bool) *Fighter {
func parse_one_fighter(s *goquery.Selection, is_a bool) *Fighter {
fighter := Fighter{}
nth_child := "1"
face := "1"
@@ -102,11 +98,18 @@ func ParseONEFighter(s *goquery.Selection, is_a bool) *Fighter {
face = "2"
}
fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
fighter.Name = text_content(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
fighter.Country = text_content(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src")
fighter.Image = image
return &fighter
}
func one_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("onefc.com", "www.onefc.com"),
)
}
scraper/one_test.go
+4 -4
View File
@@ -12,15 +12,15 @@ var ONEEventHTML []byte
var ONEEventListHTML []byte
func TestONEEventList(t *testing.T) {
PrintJSON(ParseONEEventList(ONEEventListHTML))
print_json(parse_one_event_list(ONEEventListHTML))
}
func TestONEEvent(t *testing.T) {
PrintJSON(ParseONEEvent("some_url", ONEEventHTML))
print_json(parse_one_event("some_url", ONEEventHTML))
}
func testScrapeONE(t *testing.T) {
ScrapeONE(func(e *Event) {
PrintJSON(e)
scrape_one(func(e *Event) {
print_json(e)
})
}
scraper/scraper.go
+3 -2
View File
@@ -29,6 +29,7 @@ type EventScraper func(c EventCallback)
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
var Scrapers []EventScraper = []EventScraper{
ScrapeUFC,
var scrapers []EventScraper = []EventScraper{
scrape_ufc,
scrape_one,
}
scraper/tapology.go
+9 -9
View File
@@ -16,14 +16,7 @@ type TapologyResult struct {
Url string `json:"url"`
}
func tapology_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("tapology.com", "www.tapology.com"),
)
}
func TapologyGetter() func(n string) []TapologyResult {
func tapology_getter() func(n string) []TapologyResult {
csrf_token := ""
name := ""
results := []TapologyResult{}
@@ -74,7 +67,7 @@ func TapologyGetter() func(n string) []TapologyResult {
func parse_tapology_results(b []byte) []TapologyResult {
results := []TapologyResult{}
DocumentFromBytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
url, _ := s.Attr("href")
name := s.Text()
results = append(results, TapologyResult{Name: name, Url: url})
@@ -82,3 +75,10 @@ func parse_tapology_results(b []byte) []TapologyResult {
return results
}
func tapology_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("tapology.com", "www.tapology.com"),
)
}
scraper/tapology_test.go
+5 -5
View File
@@ -9,12 +9,12 @@ import (
var search_result []byte
func testTapologyGetter(t *testing.T) {
get_tapology_for := TapologyGetter()
PrintJSON(get_tapology_for("Dustin Poirier"))
PrintJSON(get_tapology_for("Jon Jones"))
PrintJSON(get_tapology_for("Conor Mcgregor"))
get_tapology_for := tapology_getter()
print_json(get_tapology_for("Dustin Poirier"))
print_json(get_tapology_for("Jon Jones"))
print_json(get_tapology_for("Conor Mcgregor"))
}
func TestTapologyParseFragment(t *testing.T) {
PrintJSON(parse_tapology_results(search_result))
print_json(parse_tapology_results(search_result))
}
scraper/ufc.go
+15 -15
View File
@@ -10,14 +10,7 @@ import (
"github.com/gocolly/colly"
)
func ufc_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("ufc.com", "www.ufc.com"),
)
}
func ScrapeUFC(callback EventCallback) {
func scrape_ufc(callback EventCallback) {
list := ufc_collector()
detail := ufc_collector()
url := ""
@@ -43,7 +36,7 @@ func ScrapeUFC(callback EventCallback) {
func parse_ufc_event_list(b []byte) []string {
urls := []string{}
doc := DocumentFromBytes(b)
doc := document_from_bytes(b)
doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
slug, ok := s.Attr("href")
@@ -57,13 +50,13 @@ func parse_ufc_event_list(b []byte) []string {
func parse_ufc_event(url string, b []byte) *Event {
event := Event{Organization: "UFC", Url: url}
doc := DocumentFromBytes(b)
doc := document_from_bytes(b)
parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
return CleanupString(s.Text())
return cleanup_string(s.Text())
})
event.Name = strings.Join(parts, ": ")
event.Location = TextContent(doc, ".field--name-venue")
event.Location = text_content(doc, ".field--name-venue")
event.Slug = parse_ufc_slug(url)
image, ok := doc.Find(".c-hero__image img").First().Attr("src")
@@ -95,7 +88,7 @@ func parse_ufc_event(url string, b []byte) *Event {
func parse_ufc_fight(s *goquery.Selection) *Fight {
fight := Fight{}
fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1)
fight.Weight = strings.Replace(text_content(s, ".c-listing-fight__class-text"), " Bout", "", -1)
fight.FighterA = parse_ufc_fighter(s, "red")
fight.FighterB = parse_ufc_fighter(s, "blue")
@@ -105,8 +98,8 @@ func parse_ufc_fight(s *goquery.Selection) *Fight {
func parse_ufc_fighter(s *goquery.Selection, color string) *Fighter {
fighter := Fighter{}
fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color)
fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
fighter.Name = text_content(s, ".c-listing-fight__corner-name--"+color)
fighter.Country = text_content(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
image, ok := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
if ok {
fighter.Image = image
@@ -125,3 +118,10 @@ func parse_ufc_slug(url string) string {
}
return ""
}
func ufc_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("ufc.com", "www.ufc.com"),
)
}
scraper/ufc_test.go
+5 -5
View File
@@ -12,24 +12,24 @@ var UFCEventHTML []byte
var UFCEventListHTML []byte
func TestUFCEventList(t *testing.T) {
PrintJSON(parse_ufc_event_list(UFCEventListHTML))
print_json(parse_ufc_event_list(UFCEventListHTML))
}
func TestUFCEvent(t *testing.T) {
PrintJSON(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML))
print_json(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML))
}
func TestParseUFCSlug(t *testing.T) {
result := parse_ufc_slug("https://ufc.com/event/ufc-fight-night-july-20-2024")
expected := "ufc-fight-night-july-20-2024"
if result != expected {
PrintJSON(result)
print_json(result)
t.FailNow()
}
}
func testScrapeUFC(t *testing.T) {
ScrapeUFC(func(e *Event) {
PrintJSON(e)
scrape_ufc(func(e *Event) {
print_json(e)
})
}