guy176251/mmaschedule
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

removed old scraper code

Browse Source
This commit is contained in:
long
2025-02-23 16:35:24 -06:00
parent e7a075988c
commit c5d3538e60
8 changed files with 0 additions and 545 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scraper/helpers.go
-43
View File
@@ -1,43 +0,0 @@
package scraper
import (
"bytes"
"encoding/json"
"log"
"regexp"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/adrg/strutil"
"github.com/adrg/strutil/metrics"
)
func text_content(doc *goquery.Selection, selector string) string {
return cleanup_whitespace(doc.Find(selector).First().Text())
}
var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
func cleanup_whitespace(s string) string {
return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
}
func document_from_bytes(b []byte) *goquery.Selection {
doc, _ := goquery.NewDocumentFromReader(bytes.NewReader(b))
return doc.Find("html").First()
}
func print_json(v any) {
out, err := json.MarshalIndent(v, "", " ")
if err != nil {
log.Println(err)
} else {
log.Println(string(out))
}
}
var hamming *metrics.Hamming = metrics.NewHamming()
func hamming_score(a, b string) float64 {
return strutil.Similarity(a, b, hamming)
}
scraper/one.go
-115
View File
@@ -1,115 +0,0 @@
package scraper
import (
"encoding/json"
"fmt"
"log"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly"
)
type event_location struct {
Name string `json:"name"`
}
type event_data struct {
Name string `json:"name"`
StartDate string `json:"startDate"`
Image []string `json:"image"`
Location event_location `json:"location"`
}
func scrape_one(callback EventCallback) {
list := one_collector()
detail := one_collector()
url := ""
list.OnResponse(func(r *colly.Response) {
for _, u := range parse_one_event_list(r.Body) {
url = u
log.Printf("Visiting %s\n", u)
if err := detail.Visit(u); err != nil {
log.Println(err)
}
}
})
detail.OnResponse(func(r *colly.Response) {
callback(parse_one_event(url, r.Body))
})
if err := list.Visit("https://www.onefc.com/events/"); err != nil {
log.Println(err)
}
}
func parse_one_event_list(b []byte) []string {
return document_from_bytes(b).Find(".is-event a.is-image-zoom[href]").Map(func(i int, s *goquery.Selection) string {
url, _ := s.Attr("href")
return url
})
}
func parse_one_event(url string, b []byte) *Event {
event := Event{Organization: "ONE", Url: url}
doc := document_from_bytes(b)
data := event_data{}
_ = json.Unmarshal([]byte(doc.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data)
event.Name = data.Name
event.Location = data.Location.Name
if len(data.Image) > 2 {
event.Image = data.Image[2]
}
date, err := time.Parse(time.RFC3339, data.StartDate)
if err == nil {
event.Date = int(date.Unix())
}
doc.Find(".event-matchup").Each(func(i int, s *goquery.Selection) {
event.Fights = append(event.Fights, *parse_one_fight(s))
})
return &event
}
func parse_one_fight(s *goquery.Selection) *Fight {
fight := Fight{}
fight.Weight = text_content(s, ".title")
fight.FighterA = parse_one_fighter(s, true)
fight.FighterB = parse_one_fighter(s, false)
return &fight
}
func parse_one_fighter(s *goquery.Selection, is_a bool) *Fighter {
fighter := Fighter{}
nth_child := "1"
face := "1"
if is_a {
nth_child = "3"
face = "2"
}
fighter.Name = text_content(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
fighter.Country = text_content(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src")
fighter.Image = image
return &fighter
}
func one_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("onefc.com", "www.onefc.com"),
)
}
scraper/one_test.go
-26
View File
@@ -1,26 +0,0 @@
package scraper
import (
_ "embed"
"testing"
)
//go:embed testdata/one-fight-night-23.html
var ONEEventHTML []byte
//go:embed testdata/one-events.html
var ONEEventListHTML []byte
func TestONEEventList(t *testing.T) {
print_json(parse_one_event_list(ONEEventListHTML))
}
func TestONEEvent(t *testing.T) {
print_json(parse_one_event("some_url", ONEEventHTML))
}
func testScrapeONE(t *testing.T) {
scrape_one(func(e *Event) {
print_json(e)
})
}
scraper/scraper.go
-64
View File
@@ -1,64 +0,0 @@
package scraper
import (
"bytes"
"crypto/sha1"
"encoding/gob"
)
type Event struct {
Url string `json:"url"`
Slug string `json:"slug"`
Name string `json:"name"`
Location string `json:"location"`
Organization string `json:"organization"`
Image string `json:"image"`
Date int `json:"date"`
Fights []Fight `json:"fights"`
}
func (e *Event) Hash() ([]byte, error) {
var buf bytes.Buffer
hasher := sha1.New()
enc := gob.NewEncoder(&buf)
if err := enc.Encode(e); err != nil {
return nil, err
}
return hasher.Sum(buf.Bytes()), nil
}
type Fight struct {
Weight string `json:"weight"`
FighterA *Fighter `json:"fighter_a"`
FighterB *Fighter `json:"fighter_b"`
}
type Fighter struct {
Name string `json:"name"`
Image string `json:"image"`
Country string `json:"country"`
Link string `json:"link"`
}
type EventCallback func(e *Event)
type EventScraper func(c EventCallback)
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
var scrapers []EventScraper = []EventScraper{
scrape_ufc,
scrape_one,
}
func ScrapeEvents(callback EventCallback) {
get_tapology := tapology_getter()
for _, scraper := range scrapers {
scraper(func(e *Event) {
for _, fight := range e.Fights {
fight.FighterA.Link = get_tapology(fight.FighterA.Name)
}
callback(e)
})
}
}
scraper/tapology.go
-107
View File
@@ -1,107 +0,0 @@
package scraper
import (
"log"
"net/url"
"regexp"
"slices"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly"
)
const tapology_url string = "https://www.tapology.com"
type TapologyResult struct {
Name string `json:"name"`
Url string `json:"url"`
}
type TapologyGetter func(n string) string
func tapology_getter() TapologyGetter {
csrf_token := ""
name := ""
result := ""
index := tapology_collector()
index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) {
csrf_token = h.Attr("content")
time.Sleep(5 * time.Second)
})
if err := index.Visit(tapology_url); err != nil {
log.Println(err)
}
search := tapology_collector()
search.AllowURLRevisit = true
search.OnRequest(func(r *colly.Request) {
r.Headers.Add("X-CSRF-Token", csrf_token)
query := url.Values{}
query.Set("ajax", "true")
query.Set("model", "fighters")
query.Set("term", name)
r.URL.RawQuery = query.Encode()
log.Printf("Making tapology request to %s", r.URL)
})
search.OnResponse(func(r *colly.Response) {
result = parse_tapology_results(name, r.Body)
time.Sleep(5 * time.Second)
})
return func(n string) string {
if csrf_token == "" {
return ""
}
name = n
if err := search.Visit(tapology_url + "/search/nav"); err != nil {
log.Println(err)
return ""
}
return result
}
}
var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`)
func parse_tapology_results(name string, b []byte) string {
results := []TapologyResult{}
document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
url, _ := s.Attr("href")
name := cleanup_whitespace(nickname.ReplaceAllString(s.Text(), ""))
results = append(results, TapologyResult{Name: name, Url: url})
})
slices.SortStableFunc(results, func(a, b TapologyResult) int {
score_a := hamming_score(name, a.Name)
score_b := hamming_score(name, b.Name)
if score_a > score_b {
return -1
} else if score_a < score_b {
return 1
} else {
return 0
}
})
if len(results) < 1 {
return ""
}
return tapology_url + results[0].Url
}
func tapology_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("tapology.com", "www.tapology.com"),
)
}
scraper/tapology_test.go
-28
View File
@@ -1,28 +0,0 @@
package scraper
import (
_ "embed"
"testing"
)
//go:embed testdata/tapology-search.html
var search_result []byte
func testTapologyGetter(t *testing.T) {
get_tapology_for := tapology_getter()
names := []string{
"Dustin Poirier",
"Jon Jones",
"Conor Mcgregor",
"Justin Gaethje",
"Benoit Saint Denis",
}
for _, name := range names {
print_json(get_tapology_for(name))
}
}
func TestParseTapologyResults(t *testing.T) {
print_json(parse_tapology_results("Jon Jones", search_result))
}
scraper/ufc.go
-127
View File
@@ -1,127 +0,0 @@
package scraper
import (
"log"
"regexp"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly"
)
func scrape_ufc(callback EventCallback) {
list := ufc_collector()
detail := ufc_collector()
url := ""
list.OnResponse(func(r *colly.Response) {
for _, u := range parse_ufc_event_list(r.Body) {
url = u
log.Printf("Visiting %s\n", u)
if err := detail.Visit(u); err != nil {
log.Println(err)
}
}
})
detail.OnResponse(func(r *colly.Response) {
callback(parse_ufc_event(url, r.Body))
})
if err := list.Visit("https://www.ufc.com/events"); err != nil {
log.Println(err)
}
}
func parse_ufc_event_list(b []byte) []string {
urls := []string{}
doc := document_from_bytes(b)
doc.Find(".c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
slug, ok := s.Attr("href")
if ok {
urls = append(urls, "https://ufc.com"+slug)
}
})
return urls
}
func parse_ufc_event(url string, b []byte) *Event {
event := Event{Organization: "UFC", Url: url}
doc := document_from_bytes(b)
parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
return cleanup_whitespace(s.Text())
})
event.Name = strings.Join(parts, ": ")
event.Location = text_content(doc, ".field--name-venue")
event.Slug = parse_ufc_slug(url)
image, ok := doc.Find(".c-hero__image img").First().Attr("src")
if ok {
event.Image = image
}
datestr, ok := doc.Find(`
#early-prelims .c-event-fight-card-broadcaster__time,
#prelims-card .c-event-fight-card-broadcaster__time,
.c-hero__headline-suffix
`).Last().Attr("data-timestamp")
if ok {
date, err := strconv.Atoi(datestr)
if err == nil {
event.Date = date
}
}
doc.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) {
event.Fights = append(event.Fights, *parse_ufc_fight(s))
})
return &event
}
func parse_ufc_fight(s *goquery.Selection) *Fight {
fight := Fight{}
fight.Weight = strings.Replace(text_content(s, ".c-listing-fight__class-text"), " Bout", "", -1)
fight.FighterA = parse_ufc_fighter(s, "red")
fight.FighterB = parse_ufc_fighter(s, "blue")
return &fight
}
func parse_ufc_fighter(s *goquery.Selection, color string) *Fighter {
fighter := Fighter{}
fighter.Name = text_content(s, ".c-listing-fight__corner-name--"+color)
fighter.Country = text_content(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
image, ok := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
if ok {
fighter.Image = image
}
return &fighter
}
var ufc_event_url *regexp.Regexp = regexp.MustCompile(`https://ufc.com/event/(?P<Slug>(\w|-)+)`)
func parse_ufc_slug(url string) string {
matches := ufc_event_url.FindStringSubmatch(url)
index := ufc_event_url.SubexpIndex("Slug")
if len(matches) > index {
return matches[index]
}
return ""
}
func ufc_collector() *colly.Collector {
return colly.NewCollector(
colly.UserAgent(USER_AGENT),
colly.AllowedDomains("ufc.com", "www.ufc.com"),
)
}
scraper/ufc_test.go
-35
View File
@@ -1,35 +0,0 @@
package scraper
import (
_ "embed"
"testing"
)
//go:embed testdata/ufc-297-good.html
var UFCEventHTML []byte
//go:embed testdata/ufc-events.html
var UFCEventListHTML []byte
func TestUFCEventList(t *testing.T) {
print_json(parse_ufc_event_list(UFCEventListHTML))
}
func TestUFCEvent(t *testing.T) {
print_json(parse_ufc_event("https://ufc.com/event/ufc-297", UFCEventHTML))
}
func TestParseUFCSlug(t *testing.T) {
result := parse_ufc_slug("https://ufc.com/event/ufc-fight-night-july-20-2024")
expected := "ufc-fight-night-july-20-2024"
if result != expected {
print_json(result)
t.FailNow()
}
}
func testScrapeUFC(t *testing.T) {
scrape_ufc(func(e *Event) {
print_json(e)
})
}