more debug logging, minor refactor
This commit is contained in:
+2
-2
@@ -12,7 +12,7 @@ type Scraper func(ClientScraper) (*[]*event.Event, error)
|
|||||||
func ScrapeEventsLoop(q *event.Queries, client ClientScraper, tapology bool) {
|
func ScrapeEventsLoop(q *event.Queries, client ClientScraper, tapology bool) {
|
||||||
for {
|
for {
|
||||||
time.Sleep(time.Duration(1) * time.Hour)
|
time.Sleep(time.Duration(1) * time.Hour)
|
||||||
slog.Debug("Running hourly scraper")
|
slog.Debug("Running hourly scraper")
|
||||||
ScrapeEvents(q, client, tapology)
|
ScrapeEvents(q, client, tapology)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -66,7 +66,7 @@ func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Eve
|
|||||||
link, err := GetTapologyLink(client, ff.Name)
|
link, err := GetTapologyLink(client, ff.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("Failed getting tapology from site", "name", ff.Name, "error", err)
|
slog.Error("Failed getting tapology from site", "name", ff.Name, "error", err)
|
||||||
} else if len(link) > 0 {
|
} else {
|
||||||
slog.Debug("Got new tapology link", "name", ff.Name, "link", link)
|
slog.Debug("Got new tapology link", "name", ff.Name, "link", link)
|
||||||
err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link})
|
err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+9
-3
@@ -269,7 +269,7 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) {
|
|||||||
return "", fmt.Errorf("CSRF token not set.")
|
return "", fmt.Errorf("CSRF token not set.")
|
||||||
}
|
}
|
||||||
|
|
||||||
slog.Debug("Scraping tapology link", "name", name)
|
slog.Debug("Scraping tapology link", "name", name)
|
||||||
|
|
||||||
selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) {
|
selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) {
|
||||||
query := url.Values{}
|
query := url.Values{}
|
||||||
@@ -292,7 +292,7 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
if len(results) == 0 {
|
if len(results) == 0 {
|
||||||
return "", nil
|
return "", fmt.Errorf("No tapology results for %s", name)
|
||||||
}
|
}
|
||||||
|
|
||||||
slices.SortStableFunc(results, func(a, b TapologyResult) int {
|
slices.SortStableFunc(results, func(a, b TapologyResult) int {
|
||||||
@@ -308,5 +308,11 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
return TAPOLOGY_URL + results[0].Url, nil
|
link := TAPOLOGY_URL + results[0].Url
|
||||||
|
|
||||||
|
if len(link) == 0 {
|
||||||
|
return "", fmt.Errorf("Tapology link is empty for %s", name)
|
||||||
|
} else {
|
||||||
|
return TAPOLOGY_URL + results[0].Url, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user