- added more debug logging

- fixed scraper loop not looping

src/main.go

@@ -28,6 +28,8 @@ Valid commands:
 func main() {
 	var host = flag.String("host", "127.0.0.1:8000", "Set the web server host address.")
 	var debug = flag.Bool("debug", false, "Enable debug mode.")
+	var notapology = flag.Bool("no-tapology", false, "Disable scraping tapology")
+	var noscraping = flag.Bool("no-scraping", false, "Disable scraping on runserver")
 
 	flag.Parse()
 
@@ -62,13 +64,16 @@ func main() {
 
 	switch cmd {
 	case "runserver":
-		go ScrapeEventsLoop(queries, client, true)
+		if !*noscraping {
+            slog.Debug("Starting scraping loop")
+			go ScrapeEventsLoop(queries, client, !*notapology)
+		}
 		err = RunServer(*host, queries)
 		if err != nil {
 			slog.Error("Error starting web server:", "error", err)
 		}
 	case "scrape":
-		ScrapeEvents(queries, client, true)
+		ScrapeEvents(queries, client, !*notapology)
 	default:
 		fmt.Print(POSITIONAL_ARGS_HELP)
 		os.Exit(1)

src/orchestrator.go

@@ -10,8 +10,11 @@ import (
 type Scraper func(ClientScraper) (*[]*event.Event, error)
 
 func ScrapeEventsLoop(q *event.Queries, client ClientScraper, tapology bool) {
+	for {
 		time.Sleep(time.Duration(1) * time.Hour)
+        slog.Debug("Running hourly scraper")
 		ScrapeEvents(q, client, tapology)
+	}
 }
 
 func ScrapeEvents(q *event.Queries, client ClientScraper, tapology bool) {
@@ -56,7 +59,7 @@ func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Eve
 				f.FighterB,
 			}
 			for _, ff := range fighters {
-				slog.Debug("Getting tapology link", "name", ff.Name)
+				slog.Debug("Getting tapology link from database", "name", ff.Name)
 				tapology, err := q.GetTapology(context.Background(), ff.Name)
 				if err != nil {
 					slog.Error("Failed getting tapology from database", "name", ff.Name, "error", err)
@@ -64,6 +67,7 @@ func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Eve
 					if err != nil {
 						slog.Error("Failed getting tapology from site", "name", ff.Name, "error", err)
 					} else if len(link) > 0 {
+						slog.Debug("Got new tapology link", "name", ff.Name, "link", link)
 						err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link})
 						if err != nil {
 							slog.Error("Failed creating tapology in database", "name", ff.Name, "error", err)

src/scrapers.go

@@ -52,7 +52,6 @@ func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
 				"/",
 			),
 		}
-
 		err := ScrapeONEEvent(client, &e)
 
 		if err != nil {
@@ -82,6 +81,8 @@ type ONEEventData struct {
 }
 
 func ScrapeONEEvent(client ClientScraper, e *event.Event) error {
+	slog.Debug("Getting ONE Event", "url", e.Url)
+
 	page, err := client.Get(e.Url)
 
 	if err != nil {
@@ -170,6 +171,8 @@ func ScrapeUFC(client ClientScraper) (*[]*event.Event, error) {
 }
 
 func ScrapeUFCEvent(client ClientScraper, e *event.Event) error {
+	slog.Debug("Getting UFC Event", "url", e.Url)
+
 	page, err := client.Get(e.Url)
 
 	if err != nil {
@@ -240,6 +243,8 @@ func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter {
 const TAPOLOGY_URL = "https://www.tapology.com"
 
 func SetTapologyCSRF(client ClientScraper) error {
+	slog.Debug("Setting tapology CSRF token")
+
 	selection, err := client.Get(TAPOLOGY_URL)
 	if err != nil {
 		return err
@@ -264,6 +269,8 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) {
 		return "", fmt.Errorf("CSRF token not set.")
 	}
 
+	slog.Debug("Scraping tapology link", "name", name) 
+
 	selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) {
 		query := url.Values{}
 		query.Set("ajax", "true")