mmaschedule/scrapers.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"time"

	"mmaschedule-go/event"

	"github.com/PuerkitoBio/goquery"
)

type ClientScraper interface {
	Get(url string, options ...RequestOption) (*goquery.Selection, error)
	AddHeader(key string, value string)
}

func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
	events := []*event.Event{}
	page, err := client.Get("https://www.onefc.com/events/")

	if err != nil {
		return nil, err
	}

	page.Find("#upcoming-events-section .is-event").Each(func(i int, s *goquery.Selection) {
		_, exists := s.Find("a.smart-link").Attr("href")

		if !exists {
			return
		}

		url, exists := s.Find("a.is-image-zoom[href]").Attr("href")

		if !exists {
			return
		}

		e := event.Event{
			Url:          url,
			Organization: "ONE",
			Slug: strings.Trim(
				strings.Replace(url, "https://www.onefc.com/events/", "", 1),
				"/",
			),
		}

		err := ScrapeONEEvent(client, &e)

		if err != nil {
			log.Println(err)
			return
		}

		if e.Fights != "null" {
			return
		}

		events = append(events, &e)
	})

	return &events, nil
}

type ONEEventLocation struct {
	Name string `json:"name"`
}

type ONEEventData struct {
	Name      string           `json:"name"`
	StartDate string           `json:"startDate"`
	Image     []string         `json:"image"`
	Location  ONEEventLocation `json:"location"`
}

func ScrapeONEEvent(client ClientScraper, e *event.Event) error {
	page, err := client.Get(e.Url)

	if err != nil {
		return err
	}

	data := ONEEventData{}
	_ = json.Unmarshal([]byte(page.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data)

	e.Name = data.Name
	e.Location = data.Location.Name

	if len(data.Image) > 2 {
		e.Image = data.Image[2]
	}

	date, err := time.Parse(time.RFC3339, data.StartDate)
	if err == nil {
		e.Date = date.Unix()
	}

	var fights []*event.Fight
	page.Find(".event-matchup").Each(func(i int, s *goquery.Selection) {
		fights = append(fights, ScrapeONEFight(s))
	})
	e.MarshalFights(fights)

	return nil
}

func ScrapeONEFight(s *goquery.Selection) *event.Fight {
	fight := event.Fight{}

	fight.Weight = TextContent(s, ".title")
	fight.FighterA = ScrapeONEFighter(s, true)
	fight.FighterB = ScrapeONEFighter(s, false)

	return &fight
}

func ScrapeONEFighter(s *goquery.Selection, is_a bool) *event.Fighter {
	fighter := event.Fighter{}
	nth_child := "1"
	face := "1"

	if is_a {
		nth_child = "3"
		face = "2"
	}

	fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
	fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))

	image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src")
	fighter.Image = image

	return &fighter
}

func ScrapeUFC(client ClientScraper) (*[]*event.Event, error) {
	events := []*event.Event{}
	page, err := client.Get("https://www.ufc.com/events")

	if err != nil {
		return nil, err
	}

	page.Find("#events-list-upcoming .c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
		slug, exists := s.Attr("href")
		if exists {
			e := event.Event{
				Url:          "https://www.ufc.com" + slug,
				Organization: "UFC",
				Slug:         strings.Replace(slug, "/event/", "", 1),
			}
			err := ScrapeUFCEvent(client, &e)
			if err != nil {
				log.Println(err)
			} else if e.Fights != "null" {
				events = append(events, &e)
			}
		}
	})

	return &events, nil
}

func ScrapeUFCEvent(client ClientScraper, e *event.Event) error {
	page, err := client.Get(e.Url)

	if err != nil {
		return err
	}

	e.Name = strings.Join(
		page.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
			return CleanupWhitespace(s.Text())
		}),
		": ",
	)
	e.Location = TextContent(page, ".field--name-venue")

	image, exists := page.Find(".c-hero__image img").First().Attr("src")
	if exists {
		e.Image = image
	}

	datestr, exists := page.Find(`

        #early-prelims .c-event-fight-card-broadcaster__time,
        #prelims-card .c-event-fight-card-broadcaster__time,
        .c-hero__headline-suffix

    `).Last().Attr("data-timestamp")

	if exists {
		date, err := strconv.Atoi(datestr)
		if err == nil {
			e.Date = int64(date)
		}
	}

	var fights []*event.Fight
	page.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) {
		fights = append(fights, ScrapeUFCFight(s))
	})
	e.MarshalFights(fights)

	return nil
}

func ScrapeUFCFight(s *goquery.Selection) *event.Fight {
	fight := event.Fight{}

	fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1)
	fight.FighterA = ScrapeUFCFighter(s, "red")
	fight.FighterB = ScrapeUFCFighter(s, "blue")

	return &fight
}

func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter {
	fighter := event.Fighter{}

	fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color)
	fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")

	image, exists := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
	if exists {
		fighter.Image = image
	}

	return &fighter
}

func SetTapologyCSRF(client ClientScraper) error {
	selection, err := client.Get("https://www.tapology.com")
	if err != nil {
		return err
	}
	token, exists := selection.Find("meta[name=\"csrf-token\"]").First().Attr("content")
	if !exists {
		return fmt.Errorf("Could not parse CSRF token.")
	}
	client.AddHeader("X-CSRF-Token", token)
	return nil
}

var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`)

type TapologyResult struct {
	Name string `json:"name"`
	Url  string `json:"url"`
}

func GetTapologyLink(client ClientScraper, name string) (string, error) {
	selection, err := client.Get("https://www.tapology.com/search/nav", func(r *http.Request) {
		query := url.Values{}
		query.Set("ajax", "true")
		query.Set("model", "fighters")
		query.Set("term", name)
		r.URL.RawQuery = query.Encode()
	})

	if err != nil {
		return "", err
	}

	results := []TapologyResult{}

	selection.Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
		url, _ := s.Attr("href")
		name := CleanupWhitespace(nickname.ReplaceAllString(s.Text(), ""))
		results = append(results, TapologyResult{Name: name, Url: url})
	})

	if len(results) == 0 {
		return "", nil
	}

	slices.SortStableFunc(results, func(a, b TapologyResult) int {
		score_a := HammingScore(name, a.Name)
		score_b := HammingScore(name, b.Name)

		if score_a > score_b {
			return -1
		} else if score_a < score_b {
			return 1
		} else {
			return 0
		}
	})

	return results[0].Url, nil
}