updated a scraping test
This commit is contained in:
+25
-23
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
type Scraper func(ClientScraper) (*[]*event.Event, error)
|
||||
|
||||
func ScrapeEvents(q *event.Queries, client ClientScraper) {
|
||||
func ScrapeEvents(q *event.Queries, client ClientScraper, tapology bool) {
|
||||
events := []*event.Event{}
|
||||
scrapers := []Scraper{
|
||||
ScrapeONE,
|
||||
@@ -25,7 +25,9 @@ func ScrapeEvents(q *event.Queries, client ClientScraper) {
|
||||
}
|
||||
}
|
||||
|
||||
UpdateTapology(q, client, &events)
|
||||
if tapology {
|
||||
UpdateTapology(q, client, &events)
|
||||
}
|
||||
|
||||
if len(events) > 0 {
|
||||
err := q.UpsertEvents(context.Background(), events)
|
||||
@@ -35,27 +37,6 @@ func ScrapeEvents(q *event.Queries, client ClientScraper) {
|
||||
}
|
||||
}
|
||||
|
||||
func UpdateAllTapology(q *event.Queries, client ClientScraper) error {
|
||||
events_, err := q.ListEvents(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
events := []*event.Event{}
|
||||
for _, e := range events_ {
|
||||
events = append(events, &e)
|
||||
}
|
||||
|
||||
UpdateTapology(q, client, &events)
|
||||
|
||||
err = q.UpsertEvents(context.Background(), events)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Event) {
|
||||
err := SetTapologyCSRF(client)
|
||||
if err != nil {
|
||||
@@ -93,3 +74,24 @@ func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Eve
|
||||
e.MarshalFights(fights)
|
||||
}
|
||||
}
|
||||
|
||||
func UpdateAllTapology(q *event.Queries, client ClientScraper) error {
|
||||
events_, err := q.ListEvents(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
events := []*event.Event{}
|
||||
for _, e := range events_ {
|
||||
events = append(events, &e)
|
||||
}
|
||||
|
||||
UpdateTapology(q, client, &events)
|
||||
|
||||
err = q.UpsertEvents(context.Background(), events)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user