diff --git a/src/main.go b/src/main.go index c7dc4e4..c49ecdc 100644 --- a/src/main.go +++ b/src/main.go @@ -28,6 +28,8 @@ Valid commands: func main() { var host = flag.String("host", "127.0.0.1:8000", "Set the web server host address.") var debug = flag.Bool("debug", false, "Enable debug mode.") + var notapology = flag.Bool("no-tapology", false, "Disable scraping tapology") + var noscraping = flag.Bool("no-scraping", false, "Disable scraping on runserver") flag.Parse() @@ -62,13 +64,16 @@ func main() { switch cmd { case "runserver": - go ScrapeEventsLoop(queries, client, true) + if !*noscraping { + slog.Debug("Starting scraping loop") + go ScrapeEventsLoop(queries, client, !*notapology) + } err = RunServer(*host, queries) if err != nil { slog.Error("Error starting web server:", "error", err) } case "scrape": - ScrapeEvents(queries, client, true) + ScrapeEvents(queries, client, !*notapology) default: fmt.Print(POSITIONAL_ARGS_HELP) os.Exit(1) diff --git a/src/orchestrator.go b/src/orchestrator.go index 4805dec..c43b9fc 100644 --- a/src/orchestrator.go +++ b/src/orchestrator.go @@ -10,8 +10,11 @@ import ( type Scraper func(ClientScraper) (*[]*event.Event, error) func ScrapeEventsLoop(q *event.Queries, client ClientScraper, tapology bool) { - time.Sleep(time.Duration(1) * time.Hour) - ScrapeEvents(q, client, tapology) + for { + time.Sleep(time.Duration(1) * time.Hour) + slog.Debug("Running hourly scraper") + ScrapeEvents(q, client, tapology) + } } func ScrapeEvents(q *event.Queries, client ClientScraper, tapology bool) { @@ -56,7 +59,7 @@ func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Eve f.FighterB, } for _, ff := range fighters { - slog.Debug("Getting tapology link", "name", ff.Name) + slog.Debug("Getting tapology link from database", "name", ff.Name) tapology, err := q.GetTapology(context.Background(), ff.Name) if err != nil { slog.Error("Failed getting tapology from database", "name", ff.Name, "error", err) @@ -64,6 +67,7 @@ func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Eve if err != nil { slog.Error("Failed getting tapology from site", "name", ff.Name, "error", err) } else if len(link) > 0 { + slog.Debug("Got new tapology link", "name", ff.Name, "link", link) err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link}) if err != nil { slog.Error("Failed creating tapology in database", "name", ff.Name, "error", err) diff --git a/src/scrapers.go b/src/scrapers.go index 67e459b..d507a2a 100644 --- a/src/scrapers.go +++ b/src/scrapers.go @@ -52,7 +52,6 @@ func ScrapeONE(client ClientScraper) (*[]*event.Event, error) { "/", ), } - err := ScrapeONEEvent(client, &e) if err != nil { @@ -82,6 +81,8 @@ type ONEEventData struct { } func ScrapeONEEvent(client ClientScraper, e *event.Event) error { + slog.Debug("Getting ONE Event", "url", e.Url) + page, err := client.Get(e.Url) if err != nil { @@ -170,6 +171,8 @@ func ScrapeUFC(client ClientScraper) (*[]*event.Event, error) { } func ScrapeUFCEvent(client ClientScraper, e *event.Event) error { + slog.Debug("Getting UFC Event", "url", e.Url) + page, err := client.Get(e.Url) if err != nil { @@ -240,6 +243,8 @@ func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter { const TAPOLOGY_URL = "https://www.tapology.com" func SetTapologyCSRF(client ClientScraper) error { + slog.Debug("Setting tapology CSRF token") + selection, err := client.Get(TAPOLOGY_URL) if err != nil { return err @@ -264,6 +269,8 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) { return "", fmt.Errorf("CSRF token not set.") } + slog.Debug("Scraping tapology link", "name", name) + selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) { query := url.Values{} query.Set("ajax", "true")