107 lines
2.5 KiB
Go
107 lines
2.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"mmaschedule-go/event"
|
|
"time"
|
|
)
|
|
|
|
type Scraper func(ClientScraper) (*[]*event.Event, error)
|
|
|
|
func ScrapeEventsLoop(q *event.Queries, client ClientScraper, tapology bool) {
|
|
for {
|
|
time.Sleep(time.Duration(1) * time.Hour)
|
|
slog.Debug("Running hourly scraper")
|
|
ScrapeEvents(q, client, tapology)
|
|
}
|
|
}
|
|
|
|
func ScrapeEvents(q *event.Queries, client ClientScraper, tapology bool) {
|
|
events := []*event.Event{}
|
|
scrapers := []Scraper{
|
|
ScrapeONE,
|
|
ScrapeUFC,
|
|
}
|
|
|
|
for _, scraper := range scrapers {
|
|
e, err := scraper(client)
|
|
if err != nil {
|
|
slog.Error("Error scraping events", "error", err)
|
|
} else {
|
|
events = append(events, *e...)
|
|
}
|
|
}
|
|
|
|
if tapology {
|
|
UpdateTapology(q, client, &events)
|
|
}
|
|
|
|
if len(events) > 0 {
|
|
err := q.UpsertEvents(context.Background(), events)
|
|
if err != nil {
|
|
slog.Error("Failed updating events in database", "error", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Event) {
|
|
err := SetTapologyCSRF(client)
|
|
if err != nil {
|
|
slog.Error("Failed settings tapology CSRF", "error", err)
|
|
}
|
|
|
|
for _, e := range *events {
|
|
fights := e.UnmarshalFights()
|
|
for _, f := range fights {
|
|
fighters := []*event.Fighter{
|
|
f.FighterA,
|
|
f.FighterB,
|
|
}
|
|
for _, ff := range fighters {
|
|
slog.Debug("Getting tapology link from database", "name", ff.Name)
|
|
tapology, err := q.GetTapology(context.Background(), ff.Name)
|
|
if err != nil {
|
|
slog.Error("Failed getting tapology from database", "name", ff.Name, "error", err)
|
|
link, err := GetTapologyLink(client, ff.Name)
|
|
if err != nil {
|
|
slog.Error("Failed getting tapology from site", "name", ff.Name, "error", err)
|
|
} else if len(link) > 0 {
|
|
slog.Debug("Got new tapology link", "name", ff.Name, "link", link)
|
|
err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link})
|
|
if err != nil {
|
|
slog.Error("Failed creating tapology in database", "name", ff.Name, "error", err)
|
|
}
|
|
ff.Link = link
|
|
}
|
|
time.Sleep(time.Duration(5) * time.Second)
|
|
} else {
|
|
ff.Link = tapology.Url
|
|
}
|
|
}
|
|
}
|
|
e.MarshalFights(fights)
|
|
}
|
|
}
|
|
|
|
func UpdateAllTapology(q *event.Queries, client ClientScraper) error {
|
|
events_, err := q.ListEvents(context.Background())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
events := []*event.Event{}
|
|
for _, e := range events_ {
|
|
events = append(events, &e)
|
|
}
|
|
|
|
UpdateTapology(q, client, &events)
|
|
|
|
err = q.UpsertEvents(context.Background(), events)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|