From 50c4133525f6529aec9a76d7ca93c67225b971bd Mon Sep 17 00:00:00 2001
From: long <longnguyen176251@gmail.com>
Date: Sun, 16 Feb 2025 16:21:03 -0600
Subject: [PATCH] got event scraper working

---
 .gitignore       |  1 +
 event/impl.go    |  6 ++++--
 main.go          | 27 ++++++++++++++++++++++-----
 orchestrator.go  | 15 +++++++++++----
 scrapers.go      |  2 +-
 scrapers_test.go | 28 ++++++++++------------------
 6 files changed, 49 insertions(+), 30 deletions(-)

diff --git a/.gitignore b/.gitignore
index acbfedd..e9f8ba3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 testdata/
 db.sqlite*
+test-db.sqlite*
diff --git a/event/impl.go b/event/impl.go
index 5d303b3..705f656 100644
--- a/event/impl.go
+++ b/event/impl.go
@@ -53,9 +53,11 @@ SELECT
   json_extract (e.value, '$.fights'),
   json_extract (e.value, '$.history')
 FROM
-  json_each (?) AS e ON CONFLICT (url, slug) DO
-UPDATE
+  json_each (?) AS e
+WHERE true
+ON CONFLICT (url) DO UPDATE
 SET
+  slug = excluded.slug,
   name = excluded.name,
   location = excluded.location,
   organization = excluded.organization,
diff --git a/main.go b/main.go
index 920b13f..c7c3d33 100644
--- a/main.go
+++ b/main.go
@@ -3,9 +3,12 @@ package main
 import (
 	"database/sql"
 	"embed"
+	"fmt"
 
 	_ "github.com/mattn/go-sqlite3"
 
+	"mmaschedule-go/event"
+
 	"github.com/pressly/goose/v3"
 )
 
@@ -13,20 +16,34 @@ import (
 var migrations embed.FS
 
 func main() {
-	db, err := sql.Open("sqlite3", "db.sqlite")
+	_, err := InitDb("db.sqlite")
 	if err != nil {
-		panic(err)
+		fmt.Println("Error initializing database: ", err)
+	}
+}
+
+func InitDb(name string) (*event.Queries, error) {
+	db, err := sql.Open("sqlite3", name)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = db.Exec("PRAGMA journal_mode = WAL")
+	if err != nil {
+		return nil, err
 	}
 
 	goose.SetBaseFS(migrations)
 
 	if err := goose.SetDialect("sqlite3"); err != nil {
-		panic(err)
+		return nil, err
 	}
 
 	if err := goose.Up(db, "migrations"); err != nil {
-		panic(err)
+		return nil, err
 	}
 
-	// run app
+	queries := event.New(db)
+
+	return queries, nil
 }
diff --git a/orchestrator.go b/orchestrator.go
index 27513eb..04b88a0 100644
--- a/orchestrator.go
+++ b/orchestrator.go
@@ -1,14 +1,14 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"mmaschedule-go/event"
 )
 
 type Scraper func(ClientScraper) (*[]*event.Event, error)
 
-func ScrapeEvents() {
-	client := NewScraperClient()
+func ScrapeEvents(q *event.Queries, client ClientScraper) {
 	events := []*event.Event{}
 	scrapers := []Scraper{
 		ScrapeONE,
@@ -16,11 +16,18 @@ func ScrapeEvents() {
 	}
 
 	for _, scraper := range scrapers {
-		e, err := scraper(&client)
+		e, err := scraper(client)
 		if err != nil {
-			fmt.Println(err)
+			fmt.Println("Error scraping events: ", err)
 		} else {
 			events = append(events, *e...)
 		}
 	}
+
+	if len(events) > 0 {
+		err := q.UpsertEvents(context.Background(), events)
+		if err != nil {
+			fmt.Println("Error updating events in database: ", err)
+		}
+	}
 }
diff --git a/scrapers.go b/scrapers.go
index 62f2838..7971587 100644
--- a/scrapers.go
+++ b/scrapers.go
@@ -59,7 +59,7 @@ func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
 			return
 		}
 
-		if e.Fights != "null" {
+		if e.Fights == "null" {
 			return
 		}
 
diff --git a/scrapers_test.go b/scrapers_test.go
index 43a97a5..d07ea7b 100644
--- a/scrapers_test.go
+++ b/scrapers_test.go
@@ -105,25 +105,17 @@ func TestONE(t *testing.T) {
 	if err != nil {
 		t.Errorf("Scraping ONE failed: %s", err)
 	}
-	PrintJson(events)
+	if len(*events) == 0 {
+		t.Fatal("Scraping ONE failed: Events are empty")
+	}
+	//PrintJson(events)
 }
 
-func TestJSONString(t *testing.T) {
-	type User struct {
-		Name string
+func TestScrapeEvents(t *testing.T) {
+	client := DummyClient{}
+	q, err := InitDb("test-db.sqlite")
+	if err != nil {
+		t.Error("Error initializing database: ", err)
 	}
-
-	type Thing struct {
-		Users []User `json:"user,string"`
-		Num   int    `json:"num,string"`
-	}
-
-	u := Thing{
-		Users: []User{
-			{Name: "Bob"},
-			{Name: "John"},
-		},
-	}
-
-	PrintJson(u)
+	ScrapeEvents(q, &client)
 }