updated a scraping test

This commit is contained in:
2025-02-23 19:48:42 -06:00
parent 2c873779ff
commit 517d6810bb
2 changed files with 31 additions and 28 deletions
+25 -23
View File
@@ -9,7 +9,7 @@ import (
type Scraper func(ClientScraper) (*[]*event.Event, error)
func ScrapeEvents(q *event.Queries, client ClientScraper) {
func ScrapeEvents(q *event.Queries, client ClientScraper, tapology bool) {
events := []*event.Event{}
scrapers := []Scraper{
ScrapeONE,
@@ -25,7 +25,9 @@ func ScrapeEvents(q *event.Queries, client ClientScraper) {
}
}
UpdateTapology(q, client, &events)
if tapology {
UpdateTapology(q, client, &events)
}
if len(events) > 0 {
err := q.UpsertEvents(context.Background(), events)
@@ -35,27 +37,6 @@ func ScrapeEvents(q *event.Queries, client ClientScraper) {
}
}
func UpdateAllTapology(q *event.Queries, client ClientScraper) error {
events_, err := q.ListEvents(context.Background())
if err != nil {
return err
}
events := []*event.Event{}
for _, e := range events_ {
events = append(events, &e)
}
UpdateTapology(q, client, &events)
err = q.UpsertEvents(context.Background(), events)
if err != nil {
return err
}
return nil
}
func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Event) {
err := SetTapologyCSRF(client)
if err != nil {
@@ -93,3 +74,24 @@ func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Eve
e.MarshalFights(fights)
}
}
func UpdateAllTapology(q *event.Queries, client ClientScraper) error {
events_, err := q.ListEvents(context.Background())
if err != nil {
return err
}
events := []*event.Event{}
for _, e := range events_ {
events = append(events, &e)
}
UpdateTapology(q, client, &events)
err = q.UpsertEvents(context.Background(), events)
if err != nil {
return err
}
return nil
}