events whose slug changed are now cleaned up

src/event/impl.go

@@ -103,3 +103,32 @@ func (q *Queries) UpsertEvents(ctx context.Context, e []*Event) error {
 	_, err = q.db.ExecContext(ctx, upsertEvents, string(b))
 	return err
 }
+
+const deleteDuplicateEvents = `
+WITH
+	names AS (SELECT value FROM json_each(?)),
+	urls AS (SELECT value FROM json_each(?))
+DELETE FROM event 
+WHERE 
+	name IN names
+	AND url NOT IN urls
+`
+
+func (q *Queries) DeleteDuplicateEvents(ctx context.Context, e []*Event) error {
+	var urls []string
+	var names []string
+	for _, event := range e {
+		urls = append(urls, event.Url)
+		names = append(names, event.Name)
+	}
+	urlsJson, err := json.Marshal(&urls)
+	if err != nil {
+		return err
+	}
+	namesJson, err := json.Marshal(&names)
+	if err != nil {
+		return err
+	}
+	_, err = q.db.ExecContext(ctx, deleteDuplicateEvents, string(namesJson), string(urlsJson))
+	return err
+}

src/helpers.go

@@ -11,6 +11,7 @@ import (
 	"github.com/PuerkitoBio/goquery"
 	"github.com/adrg/strutil"
 	"github.com/adrg/strutil/metrics"
+	//"github.com/enetx/surf" // better http scraper client
 )
 
 type Values = url.Values

src/orchestrator.go

@@ -11,7 +11,7 @@ type Scraper func(ClientScraper) (*[]*event.Event, error)
 
 func ScrapeEventsLoop(db *event.Queries, client ClientScraper, tapology bool) {
 	for {
-		time.Sleep(time.Duration(1) * time.Hour)
+		time.Sleep(1 * time.Hour)
 		slog.Debug("Running hourly scraper")
 		ScrapeEvents(db, client, tapology)
 	}
@@ -39,15 +39,24 @@ func ScrapeEvents(db *event.Queries, client ClientScraper, tapology bool) {
 		}
 	}
 
+	if len(allEvents) == 0 {
+		return
+	}
+
 	if tapology {
 		UpdateTapology(db, client, &allEvents)
 	}
 
-	if len(allEvents) > 0 {
+	err := db.UpsertEvents(context.Background(), allEvents)
-		err := db.UpsertEvents(context.Background(), allEvents)
+	if err != nil {
-		if err != nil {
+		slog.Error("Failed updating events in database", "error", err)
-			slog.Error("Failed updating events in database", "error", err)
+		return
-		}
+	}
+
+	err = db.DeleteDuplicateEvents(context.Background(), allEvents)
+	if err != nil {
+		slog.Error("Failed clearing out duplicate events", "error", err)
+		return
 	}
 }