- added more debug logging

- fixed scraper loop not looping
This commit is contained in:
2025-02-26 22:15:26 -06:00
parent 960481c19e
commit 15287232a3
3 changed files with 22 additions and 6 deletions
+8 -1
View File
@@ -52,7 +52,6 @@ func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
"/",
),
}
err := ScrapeONEEvent(client, &e)
if err != nil {
@@ -82,6 +81,8 @@ type ONEEventData struct {
}
func ScrapeONEEvent(client ClientScraper, e *event.Event) error {
slog.Debug("Getting ONE Event", "url", e.Url)
page, err := client.Get(e.Url)
if err != nil {
@@ -170,6 +171,8 @@ func ScrapeUFC(client ClientScraper) (*[]*event.Event, error) {
}
func ScrapeUFCEvent(client ClientScraper, e *event.Event) error {
slog.Debug("Getting UFC Event", "url", e.Url)
page, err := client.Get(e.Url)
if err != nil {
@@ -240,6 +243,8 @@ func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter {
const TAPOLOGY_URL = "https://www.tapology.com"
func SetTapologyCSRF(client ClientScraper) error {
slog.Debug("Setting tapology CSRF token")
selection, err := client.Get(TAPOLOGY_URL)
if err != nil {
return err
@@ -264,6 +269,8 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) {
return "", fmt.Errorf("CSRF token not set.")
}
slog.Debug("Scraping tapology link", "name", name)
selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) {
query := url.Values{}
query.Set("ajax", "true")