diff --git a/src/event/impl.go b/src/event/impl.go index 5d5e285..36027de 100644 --- a/src/event/impl.go +++ b/src/event/impl.go @@ -103,3 +103,32 @@ func (q *Queries) UpsertEvents(ctx context.Context, e []*Event) error { _, err = q.db.ExecContext(ctx, upsertEvents, string(b)) return err } + +const deleteDuplicateEvents = ` +WITH + names AS (SELECT value FROM json_each(?)), + urls AS (SELECT value FROM json_each(?)) +DELETE FROM event +WHERE + name IN names + AND url NOT IN urls +` + +func (q *Queries) DeleteDuplicateEvents(ctx context.Context, e []*Event) error { + var urls []string + var names []string + for _, event := range e { + urls = append(urls, event.Url) + names = append(names, event.Name) + } + urlsJson, err := json.Marshal(&urls) + if err != nil { + return err + } + namesJson, err := json.Marshal(&names) + if err != nil { + return err + } + _, err = q.db.ExecContext(ctx, deleteDuplicateEvents, string(namesJson), string(urlsJson)) + return err +} diff --git a/src/helpers.go b/src/helpers.go index 17b9ab6..59de27e 100644 --- a/src/helpers.go +++ b/src/helpers.go @@ -11,6 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/adrg/strutil" "github.com/adrg/strutil/metrics" + //"github.com/enetx/surf" // better http scraper client ) type Values = url.Values diff --git a/src/orchestrator.go b/src/orchestrator.go index ebd6961..a26a5f6 100644 --- a/src/orchestrator.go +++ b/src/orchestrator.go @@ -11,7 +11,7 @@ type Scraper func(ClientScraper) (*[]*event.Event, error) func ScrapeEventsLoop(db *event.Queries, client ClientScraper, tapology bool) { for { - time.Sleep(time.Duration(1) * time.Hour) + time.Sleep(1 * time.Hour) slog.Debug("Running hourly scraper") ScrapeEvents(db, client, tapology) } @@ -39,15 +39,24 @@ func ScrapeEvents(db *event.Queries, client ClientScraper, tapology bool) { } } + if len(allEvents) == 0 { + return + } + if tapology { UpdateTapology(db, client, &allEvents) } - if len(allEvents) > 0 { - err := db.UpsertEvents(context.Background(), allEvents) - if err != nil { - slog.Error("Failed updating events in database", "error", err) - } + err := db.UpsertEvents(context.Background(), allEvents) + if err != nil { + slog.Error("Failed updating events in database", "error", err) + return + } + + err = db.DeleteDuplicateEvents(context.Background(), allEvents) + if err != nil { + slog.Error("Failed clearing out duplicate events", "error", err) + return } }