- removed precompiled styles and testing html

- fixed test
- added checks for events that have empty fights
- some renaming
This commit is contained in:
2025-04-22 20:10:00 -05:00
parent 6c7d4610bd
commit 741a4cab05
21 changed files with 86 additions and 50860 deletions
+41 -27
View File
@@ -9,43 +9,49 @@ import (
type Scraper func(ClientScraper) (*[]*event.Event, error)
func ScrapeEventsLoop(q *event.Queries, client ClientScraper, tapology bool) {
func ScrapeEventsLoop(db *event.Queries, client ClientScraper, tapology bool) {
for {
time.Sleep(time.Duration(1) * time.Hour)
slog.Debug("Running hourly scraper")
ScrapeEvents(q, client, tapology)
ScrapeEvents(db, client, tapology)
}
}
func ScrapeEvents(q *event.Queries, client ClientScraper, tapology bool) {
events := []*event.Event{}
func ScrapeEvents(db *event.Queries, client ClientScraper, tapology bool) {
allEvents := []*event.Event{}
scrapers := []Scraper{
ScrapeONE,
ScrapeUFC,
}
for _, scraper := range scrapers {
e, err := scraper(client)
events, err := scraper(client)
if err != nil {
slog.Error("Error scraping events", "error", err)
} else {
events = append(events, *e...)
continue
}
for _, e := range *events {
if !e.HasEmptyFights() {
allEvents = append(allEvents, e)
}
}
}
if tapology {
UpdateTapology(q, client, &events)
UpdateTapology(db, client, &allEvents)
}
if len(events) > 0 {
err := q.UpsertEvents(context.Background(), events)
if len(allEvents) > 0 {
err := db.UpsertEvents(context.Background(), allEvents)
if err != nil {
slog.Error("Failed updating events in database", "error", err)
}
}
}
func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Event) {
func UpdateTapology(db *event.Queries, client ClientScraper, events *[]*event.Event) {
err := SetTapologyCSRF(client)
if err != nil {
slog.Error("Failed settings tapology CSRF", "error", err)
@@ -53,33 +59,41 @@ func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Eve
for _, e := range *events {
fights := e.UnmarshalFights()
for _, f := range fights {
fighters := []*event.Fighter{
f.FighterA,
f.FighterB,
}
for _, ff := range fighters {
slog.Debug("Getting tapology link from database", "name", ff.Name)
tapology, err := q.GetTapology(context.Background(), ff.Name)
if err != nil {
slog.Error("Failed getting tapology from database", "name", ff.Name, "error", err)
link, err := GetTapologyLink(client, ff.Name)
if err != nil {
slog.Error("Failed getting tapology from site", "name", ff.Name, "error", err)
} else {
slog.Debug("Got new tapology link", "name", ff.Name, "link", link)
err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link})
if err != nil {
slog.Error("Failed creating tapology in database", "name", ff.Name, "error", err)
}
ff.Link = link
}
time.Sleep(time.Duration(5) * time.Second)
} else {
tapology, err := db.GetTapology(context.Background(), ff.Name)
if err == nil {
ff.Link = tapology.Url
continue
}
slog.Error("Failed getting tapology from database", "name", ff.Name, "error", err)
link, err := GetTapologyLink(client, ff.Name)
time.Sleep(5 * time.Second)
if err != nil {
slog.Error("Failed getting tapology from site", "name", ff.Name, "error", err)
continue
}
ff.Link = link
slog.Debug("Got new tapology link", "name", ff.Name, "link", link)
err = db.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link})
if err != nil {
slog.Error("Failed creating tapology in database", "name", ff.Name, "error", err)
}
}
}
e.MarshalFights(fights)
}
}