got event scraper working

.gitignore

@@ -1,2 +1,3 @@
 testdata/
 db.sqlite*
+test-db.sqlite*

event/impl.go

@@ -53,9 +53,11 @@ SELECT
   json_extract (e.value, '$.fights'),
   json_extract (e.value, '$.history')
 FROM
-  json_each (?) AS e ON CONFLICT (url, slug) DO
-UPDATE
+  json_each (?) AS e
+WHERE true
+ON CONFLICT (url) DO UPDATE
 SET
+  slug = excluded.slug,
   name = excluded.name,
   location = excluded.location,
   organization = excluded.organization,

main.go

@@ -3,9 +3,12 @@ package main
 import (
 	"database/sql"
 	"embed"
+	"fmt"
 
 	_ "github.com/mattn/go-sqlite3"
 
+	"mmaschedule-go/event"
+
 	"github.com/pressly/goose/v3"
 )
 
@@ -13,20 +16,34 @@ import (
 var migrations embed.FS
 
 func main() {
-	db, err := sql.Open("sqlite3", "db.sqlite")
+	_, err := InitDb("db.sqlite")
 	if err != nil {
-		panic(err)
+		fmt.Println("Error initializing database: ", err)
+	}
+}
+
+func InitDb(name string) (*event.Queries, error) {
+	db, err := sql.Open("sqlite3", name)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = db.Exec("PRAGMA journal_mode = WAL")
+	if err != nil {
+		return nil, err
 	}
 
 	goose.SetBaseFS(migrations)
 
 	if err := goose.SetDialect("sqlite3"); err != nil {
-		panic(err)
+		return nil, err
 	}
 
 	if err := goose.Up(db, "migrations"); err != nil {
-		panic(err)
+		return nil, err
 	}
 
-	// run app
+	queries := event.New(db)
+
+	return queries, nil
 }

orchestrator.go

@@ -1,14 +1,14 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"mmaschedule-go/event"
 )
 
 type Scraper func(ClientScraper) (*[]*event.Event, error)
 
-func ScrapeEvents() {
-	client := NewScraperClient()
+func ScrapeEvents(q *event.Queries, client ClientScraper) {
 	events := []*event.Event{}
 	scrapers := []Scraper{
 		ScrapeONE,
@@ -16,11 +16,18 @@ func ScrapeEvents() {
 	}
 
 	for _, scraper := range scrapers {
-		e, err := scraper(&client)
+		e, err := scraper(client)
 		if err != nil {
-			fmt.Println(err)
+			fmt.Println("Error scraping events: ", err)
 		} else {
 			events = append(events, *e...)
 		}
 	}
+
+	if len(events) > 0 {
+		err := q.UpsertEvents(context.Background(), events)
+		if err != nil {
+			fmt.Println("Error updating events in database: ", err)
+		}
+	}
 }

scrapers.go

@@ -59,7 +59,7 @@ func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
 			return
 		}
 
-		if e.Fights != "null" {
+		if e.Fights == "null" {
 			return
 		}
 

scrapers_test.go

@@ -105,25 +105,17 @@ func TestONE(t *testing.T) {
 	if err != nil {
 		t.Errorf("Scraping ONE failed: %s", err)
 	}
-	PrintJson(events)
+	if len(*events) == 0 {
+		t.Fatal("Scraping ONE failed: Events are empty")
+	}
+	//PrintJson(events)
 }
 
-func TestJSONString(t *testing.T) {
-	type User struct {
-		Name string
+func TestScrapeEvents(t *testing.T) {
+	client := DummyClient{}
+	q, err := InitDb("test-db.sqlite")
+	if err != nil {
+		t.Error("Error initializing database: ", err)
 	}
-
-	type Thing struct {
-		Users []User `json:"user,string"`
-		Num   int    `json:"num,string"`
-	}
-
-	u := Thing{
-		Users: []User{
-			{Name: "Bob"},
-			{Name: "John"},
-		},
-	}
-
-	PrintJson(u)
+	ScrapeEvents(q, &client)
 }