- added more debug logging
- fixed scraper loop not looping
This commit is contained in:
+8
-1
@@ -52,7 +52,6 @@ func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
|
||||
"/",
|
||||
),
|
||||
}
|
||||
|
||||
err := ScrapeONEEvent(client, &e)
|
||||
|
||||
if err != nil {
|
||||
@@ -82,6 +81,8 @@ type ONEEventData struct {
|
||||
}
|
||||
|
||||
func ScrapeONEEvent(client ClientScraper, e *event.Event) error {
|
||||
slog.Debug("Getting ONE Event", "url", e.Url)
|
||||
|
||||
page, err := client.Get(e.Url)
|
||||
|
||||
if err != nil {
|
||||
@@ -170,6 +171,8 @@ func ScrapeUFC(client ClientScraper) (*[]*event.Event, error) {
|
||||
}
|
||||
|
||||
func ScrapeUFCEvent(client ClientScraper, e *event.Event) error {
|
||||
slog.Debug("Getting UFC Event", "url", e.Url)
|
||||
|
||||
page, err := client.Get(e.Url)
|
||||
|
||||
if err != nil {
|
||||
@@ -240,6 +243,8 @@ func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter {
|
||||
const TAPOLOGY_URL = "https://www.tapology.com"
|
||||
|
||||
func SetTapologyCSRF(client ClientScraper) error {
|
||||
slog.Debug("Setting tapology CSRF token")
|
||||
|
||||
selection, err := client.Get(TAPOLOGY_URL)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -264,6 +269,8 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) {
|
||||
return "", fmt.Errorf("CSRF token not set.")
|
||||
}
|
||||
|
||||
slog.Debug("Scraping tapology link", "name", name)
|
||||
|
||||
selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) {
|
||||
query := url.Values{}
|
||||
query.Set("ajax", "true")
|
||||
|
||||
Reference in New Issue
Block a user