guy176251/mmaschedule
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved stuff around

Browse Source
This commit is contained in:
long
2024-07-02 08:04:21 -05:00
parent f38a9f9df5
commit eaf229abde
11 changed files with 422 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.go
+3
View File
@@ -1,4 +1,7 @@
package main
import "mmaschedule-go/scraper"
func main() {
scraper.CleanupString("asd")
}
helpers.go → scraper/helpers.go
+11 -1
View File
@@ -1,9 +1,10 @@
package main
package scraper
import (
"bytes"
"encoding/json"
"hash/fnv"
"log"
"regexp"
"strings"
@@ -34,3 +35,12 @@ func HashJSON(v any) uint32 {
h.Write(out)
return h.Sum32()
}
func PrintJSON(v any) {
out, err := json.MarshalIndent(v, "", " ")
if err != nil {
log.Println(err)
} else {
log.Println(string(out))
}
}
scraper/one.go
+112
View File
@@ -0,0 +1,112 @@
package scraper
import (
"encoding/json"
"fmt"
"log"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly"
)
type ONEEventLocation struct {
Name string `json:"name"`
}
type ONEEventData struct {
Name string `json:"name"`
StartDate string `json:"startDate"`
Image []string `json:"image"`
Location ONEEventLocation `json:"location"`
}
func ONECollector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("onefc.com", "www.onefc.com"),
)
}
func ScrapeONE(callback EventCallback) {
list := ONECollector()
detail := ONECollector()
url := ""
list.OnResponse(func(r *colly.Response) {
for _, u := range ParseONEEventList(r.Body) {
url = u
log.Printf("Visiting %s\n", u)
if err := detail.Visit(u); err != nil {
log.Println(err)
}
}
})
detail.OnResponse(func(r *colly.Response) {
callback(ParseONEEvent(url, r.Body))
})
if err := list.Visit("https://www.onefc.com/events/"); err != nil {
log.Println(err)
}
}
func ParseONEEventList(b []byte) []string {
return DocumentFromBytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string {
url, _ := s.Attr("href")
return url
})
}
func ParseONEEvent(url string, b []byte) *Event {
event := Event{Organization: "ONE", Url: url}
doc := DocumentFromBytes(b)
data := ONEEventData{}
_ = json.Unmarshal([]byte(doc.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data)
event.Name = data.Name
event.Image = data.Image[2]
event.Location = data.Location.Name
date, err := time.Parse(time.RFC3339, data.StartDate)
if err == nil {
event.Date = int(date.Unix())
}
doc.Find(".event-matchup").Each(func(i int, s *goquery.Selection) {
event.Fights = append(event.Fights, *ParseONEFight(s))
})
return &event
}
func ParseONEFight(s *goquery.Selection) *Fight {
fight := Fight{}
fight.Weight = TextContent(s, ".title")
fight.FighterA = ParseONEFighter(s, true)
fight.FighterB = ParseONEFighter(s, false)
return &fight
}
func ParseONEFighter(s *goquery.Selection, is_a bool) *Fighter {
fighter := Fighter{}
nth_child := "1"
face := "1"
if is_a {
nth_child = "3"
face = "2"
}
fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src")
fighter.Image = image
return &fighter
}
scraper/one_test.go
+26
View File
@@ -0,0 +1,26 @@
package scraper
import (
_ "embed"
"testing"
)
//go:embed testdata/one-fight-night-23.html
var ONEEventHTML []byte
//go:embed testdata/one-events.html
var ONEEventListHTML []byte
func TestONEEventList(t *testing.T) {
PrintJSON(ParseONEEventList(ONEEventListHTML))
}
func TestONEEvent(t *testing.T) {
PrintJSON(ParseONEEvent("some_url", ONEEventHTML))
}
func testScrapeONE(t *testing.T) {
ScrapeONE(func(e *Event) {
PrintJSON(e)
})
}
scraper.go → scraper/scraper.go
+4 -1
View File
@@ -1,6 +1,8 @@
package main
package scraper
type Event struct {
Url string `json:"url"`
Slug string `json:"slug"`
Name string `json:"name"`
Location string `json:"location"`
Organization string `json:"organization"`
@@ -19,6 +21,7 @@ type Fighter struct {
Name string `json:"name"`
Image string `json:"image"`
Country string `json:"country"`
Link string `json:"link"`
}
type EventCallback func(e *Event)
scraper/tapology.go
+84
View File
@@ -0,0 +1,84 @@
package scraper
import (
"log"
"net/url"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly"
)
const tapology_url string = "https://www.tapology.com"
type TapologyResult struct {
Name string `json:"name"`
Url string `json:"url"`
}
func tapology_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("tapology.com", "www.tapology.com"),
)
}
func TapologyGetter() func(n string) []TapologyResult {
csrf_token := ""
name := ""
results := []TapologyResult{}
index := tapology_collector()
index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) {
csrf_token = h.Attr("content")
time.Sleep(5 * time.Second)
})
search := tapology_collector()
search.AllowURLRevisit = true
search.OnRequest(func(r *colly.Request) {
r.Headers.Add("X-CSRF-Token", csrf_token)
query := url.Values{}
query.Set("ajax", "true")
query.Set("model", "fighters")
query.Set("term", name)
r.URL.RawQuery = query.Encode()
log.Printf("Making tapology request to %s", r.URL)
})
search.OnResponse(func(r *colly.Response) {
results = parse_tapology_results(r.Body)
time.Sleep(5 * time.Second)
})
if err := index.Visit(tapology_url); err != nil {
log.Println(err)
}
return func(n string) []TapologyResult {
if csrf_token == "" {
return nil
}
name = n
if err := search.Visit(tapology_url + "/search/nav"); err != nil {
log.Println(err)
return nil
}
return results
}
}
func parse_tapology_results(b []byte) []TapologyResult {
results := []TapologyResult{}
DocumentFromBytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
url, _ := s.Attr("href")
name := s.Text()
results = append(results, TapologyResult{Name: name, Url: url})
})
return results
}
scraper/tapology_test.go
+20
View File
@@ -0,0 +1,20 @@
package scraper
import (
_ "embed"
"testing"
)
//go:embed testdata/tapology-search.html
var search_result []byte
func testTapologyGetter(t *testing.T) {
get_tapology_for := TapologyGetter()
PrintJSON(get_tapology_for("Dustin Poirier"))
PrintJSON(get_tapology_for("Jon Jones"))
PrintJSON(get_tapology_for("Conor Mcgregor"))
}
func TestTapologyParseFragment(t *testing.T) {
PrintJSON(parse_tapology_results(search_result))
}
scraper/ufc.go
+127
View File
@@ -0,0 +1,127 @@
package scraper
import (
"log"
"regexp"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly"
)
func ufc_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("ufc.com", "www.ufc.com"),
)
}
func ScrapeUFC(callback EventCallback) {
list := ufc_collector()
detail := ufc_collector()
url := ""
list.OnResponse(func(r *colly.Response) {
for _, u := range parse_ufc_event_list(r.Body) {
url = u
log.Printf("Visiting %s\n", u)
if err := detail.Visit(u); err != nil {
log.Println(err)
}
}
})
detail.OnResponse(func(r *colly.Response) {
callback(parse_ufc_event(url, r.Body))
})
if err := list.Visit("https://www.ufc.com/events"); err != nil {
log.Println(err)
}
}
func parse_ufc_event_list(b []byte) []string {
urls := []string{}
doc := DocumentFromBytes(b)
doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
slug, ok := s.Attr("href")
if ok {
urls = append(urls, "https://ufc.com"+slug)
}
})
return urls
}
func parse_ufc_event(url string, b []byte) *Event {
event := Event{Organization: "UFC", Url: url}
doc := DocumentFromBytes(b)
parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
return CleanupString(s.Text())
})
event.Name = strings.Join(parts, ": ")
event.Location = TextContent(doc, ".field--name-venue")
event.Slug = parse_ufc_slug(url)
image, ok := doc.Find(".c-hero__image img").First().Attr("src")
if ok {
event.Image = image
}
datestr, ok := doc.Find(`
#early-prelims .c-event-fight-card-broadcaster__time,
#prelims-card .c-event-fight-card-broadcaster__time,
.c-hero__headline-suffix
`).Last().Attr("data-timestamp")
if ok {
date, err := strconv.Atoi(datestr)
if err == nil {
event.Date = date
}
}
doc.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) {
event.Fights = append(event.Fights, *parse_ufc_fight(s))
})
return &event
}
func parse_ufc_fight(s *goquery.Selection) *Fight {
fight := Fight{}
fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1)
fight.FighterA = parse_ufc_fighter(s, "red")
fight.FighterB = parse_ufc_fighter(s, "blue")
return &fight
}
func parse_ufc_fighter(s *goquery.Selection, color string) *Fighter {
fighter := Fighter{}
fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color)
fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
image, ok := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
if ok {
fighter.Image = image
}
return &fighter
}
var ufc_event_url *regexp.Regexp = regexp.MustCompile(`https://ufc.com/event/(?P<Slug>(\w|-)+)`)
func parse_ufc_slug(url string) string {
matches := ufc_event_url.FindStringSubmatch(url)
index := ufc_event_url.SubexpIndex("Slug")
if len(matches) > index {
return matches[index]
}
return ""
}
scraper/ufc_test.go
+35
View File
@@ -0,0 +1,35 @@
package scraper
import (
_ "embed"
"testing"
)
//go:embed testdata/ufc-297-good.html
var UFCEventHTML []byte
//go:embed testdata/ufc-events.html
var UFCEventListHTML []byte
func TestUFCEventList(t *testing.T) {
PrintJSON(parse_ufc_event_list(UFCEventListHTML))
}
func TestUFCEvent(t *testing.T) {
PrintJSON(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML))
}
func TestParseUFCSlug(t *testing.T) {
result := parse_ufc_slug("https://ufc.com/event/ufc-fight-night-july-20-2024")
expected := "ufc-fight-night-july-20-2024"
if result != expected {
PrintJSON(result)
t.FailNow()
}
}
func testScrapeUFC(t *testing.T) {
ScrapeUFC(func(e *Event) {
PrintJSON(e)
})
}
ufc.go
-114
View File
@@ -1,114 +0,0 @@
package main
import (
"log"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly"
)
func UFCCollector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("ufc.com", "www.ufc.com"),
)
}
func ScrapeUFC(callback EventCallback) {
ufclist := UFCCollector()
ufcdetail := UFCCollector()
ufclist.OnResponse(func(r *colly.Response) {
for _, url := range ParseUFCEventList(r.Body) {
log.Printf("Visiting %s\n", url)
err := ufcdetail.Visit(url)
if err != nil {
log.Println(err)
}
}
})
ufcdetail.OnResponse(func(r *colly.Response) {
callback(ParseUFCEvent(r.Body))
})
err := ufclist.Visit("https://www.ufc.com/events")
if err != nil {
log.Println(err)
}
}
func ParseUFCEventList(b []byte) []string {
urls := []string{}
doc := DocumentFromBytes(b)
doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
slug, ok := s.Attr("href")
if ok {
urls = append(urls, "https://ufc.com"+slug)
}
})
return urls
}
func ParseUFCEvent(b []byte) *Event {
event := Event{Organization: "UFC"}
doc := DocumentFromBytes(b)
parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
return CleanupString(s.Text())
})
event.Name = strings.Join(parts, ": ")
event.Location = TextContent(doc, ".field--name-venue")
image, ok := doc.Find(".c-hero__image img").First().Attr("src")
if ok {
event.Image = image
}
datestr, ok := doc.Find(`
#early-prelims .c-event-fight-card-broadcaster__time,
#prelims-card .c-event-fight-card-broadcaster__time,
.c-hero__headline-suffix
`).Last().Attr("data-timestamp")
if ok {
date, err := strconv.Atoi(datestr)
if err == nil {
event.Date = date
}
}
doc.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) {
event.Fights = append(event.Fights, *ParseUFCFight(s))
})
return &event
}
func ParseUFCFight(doc *goquery.Selection) *Fight {
fight := Fight{}
fight.Weight = strings.Replace(TextContent(doc, ".c-listing-fight__class-text"), " Bout", "", -1)
fight.FighterA = ParseUFCFighter(doc, "red")
fight.FighterB = ParseUFCFighter(doc, "blue")
return &fight
}
func ParseUFCFighter(doc *goquery.Selection, color string) *Fighter {
fighter := Fighter{}
fighter.Name = TextContent(doc, ".c-listing-fight__corner-name--"+color)
fighter.Country = TextContent(doc, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
image, ok := doc.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
if ok {
fighter.Image = image
}
return &fighter
}
ufc_test.go
-39
View File
@@ -1,39 +0,0 @@
package main
import (
_ "embed"
"encoding/json"
"log"
"testing"
)
//go:embed testdata/ufc-297-good.html
var UFCEventHTML []byte
//go:embed testdata/ufc-events.html
var UFCEventListHTML []byte
func TestUFCEventList(t *testing.T) {
urls := ParseUFCEventList(UFCEventListHTML)
PrintJSON(urls)
}
func TestUFCEvent(t *testing.T) {
event := ParseUFCEvent(UFCEventHTML)
PrintJSON(event)
}
func testScrapeUFC(t *testing.T) {
ScrapeUFC(func(e *Event) {
PrintJSON(e)
})
}
func PrintJSON(v any) {
out, err := json.MarshalIndent(v, "", " ")
if err != nil {
log.Println(err)
} else {
log.Println(string(out))
}
}