added tapology link getter to scraper

event/query.sql.go

@@ -9,6 +9,23 @@ import (
 	"context"
 )
 
+const createTapology = `-- name: CreateTapology :exec
+INSERT INTO
+  tapology (name, url)
+VALUES
+  (?, ?)
+`
+
+type CreateTapologyParams struct {
+	Name string `json:"name"`
+	Url  string `json:"url"`
+}
+
+func (q *Queries) CreateTapology(ctx context.Context, arg CreateTapologyParams) error {
+	_, err := q.db.ExecContext(ctx, createTapology, arg.Name, arg.Url)
+	return err
+}
+
 const getEvent = `-- name: GetEvent :one
 SELECT
   url, slug, name, location, organization, image, date, fights, history
@@ -54,3 +71,75 @@ func (q *Queries) GetTapology(ctx context.Context, name string) (Tapology, error
 	err := row.Scan(&i.Name, &i.Url)
 	return i, err
 }
+
+const getUpcomingEvent = `-- name: GetUpcomingEvent :one
+SELECT
+  url, slug, name, location, organization, image, date, fights, history
+FROM
+  event
+WHERE
+  date >= ?
+ORDER BY
+  date ASC
+LIMIT
+  1
+`
+
+func (q *Queries) GetUpcomingEvent(ctx context.Context, date int64) (Event, error) {
+	row := q.db.QueryRowContext(ctx, getUpcomingEvent, date)
+	var i Event
+	err := row.Scan(
+		&i.Url,
+		&i.Slug,
+		&i.Name,
+		&i.Location,
+		&i.Organization,
+		&i.Image,
+		&i.Date,
+		&i.Fights,
+		&i.History,
+	)
+	return i, err
+}
+
+const listEvents = `-- name: ListEvents :many
+SELECT
+  name,
+  slug,
+  date
+FROM
+  event
+WHERE
+  date >= ?
+ORDER BY
+  date ASC
+`
+
+type ListEventsRow struct {
+	Name string `json:"name"`
+	Slug string `json:"slug"`
+	Date int64  `json:"date"`
+}
+
+func (q *Queries) ListEvents(ctx context.Context, date int64) ([]ListEventsRow, error) {
+	rows, err := q.db.QueryContext(ctx, listEvents, date)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListEventsRow
+	for rows.Next() {
+		var i ListEventsRow
+		if err := rows.Scan(&i.Name, &i.Slug, &i.Date); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}

helpers.go

@@ -78,13 +78,17 @@ func (c *ScraperClient) GetResponse(url string, options ...RequestOption) (*http
 func (c *ScraperClient) AddHeader(key string, value string) {
 	c.headers[key] = value
 }
+func (c *ScraperClient) HasHeader(key string) bool {
+	_, exists := c.headers[key]
+	return exists
+}
 
-func NewScraperClient() ScraperClient {
+func NewScraperClient() *ScraperClient {
 	client := ScraperClient{
 		headers: map[string]string{
 			"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
 		},
 		client: http.Client{},
 	}
-	return client
+	return &client
 }

main.go

@@ -16,9 +16,12 @@ import (
 var migrations embed.FS
 
 func main() {
-	_, err := InitDb("db.sqlite")
+	q, err := InitDb("db.sqlite")
 	if err != nil {
 		fmt.Println("Error initializing database: ", err)
+	} else {
+		client := NewScraperClient()
+		ScrapeEvents(q, client)
 	}
 }
 

orchestrator.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"mmaschedule-go/event"
+	"time"
 )
 
 type Scraper func(ClientScraper) (*[]*event.Event, error)
@@ -24,6 +25,41 @@ func ScrapeEvents(q *event.Queries, client ClientScraper) {
 		}
 	}
 
+	err := SetTapologyCSRF(client)
+	if err != nil {
+		fmt.Println("Error settings tapology CSRF: ", err)
+	}
+
+	for _, e := range events {
+		fights := e.UnmarshalFights()
+		for _, f := range fights {
+			fighters := []*event.Fighter{
+				f.FighterA,
+				f.FighterB,
+			}
+			for _, ff := range fighters {
+				tapology, err := q.GetTapology(context.Background(), ff.Name)
+				if err != nil {
+					fmt.Println("Error getting tapology from database: ", err)
+					link, err := GetTapologyLink(client, ff.Name)
+					if err != nil {
+						fmt.Println("Error getting tapology from site: ", err)
+					} else if len(link) > 0 {
+						err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link})
+						if err != nil {
+							fmt.Println("Error creating tapology in database: ", err)
+						}
+						ff.Link = link
+					}
+					time.Sleep(time.Duration(5_000_000_000))
+				} else {
+					ff.Link = tapology.Url
+				}
+			}
+		}
+		e.MarshalFights(fights)
+	}
+
 	if len(events) > 0 {
 		err := q.UpsertEvents(context.Background(), events)
 		if err != nil {

scrapers.go

@@ -20,6 +20,7 @@ import (
 type ClientScraper interface {
 	Get(url string, options ...RequestOption) (*goquery.Selection, error)
 	AddHeader(key string, value string)
+	HasHeader(key string) bool
 }
 
 func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
@@ -236,8 +237,10 @@ func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter {
 	return &fighter
 }
 
+const TAPOLOGY_URL = "https://www.tapology.com"
+
 func SetTapologyCSRF(client ClientScraper) error {
-	selection, err := client.Get("https://www.tapology.com")
+	selection, err := client.Get(TAPOLOGY_URL)
 	if err != nil {
 		return err
 	}
@@ -257,7 +260,11 @@ type TapologyResult struct {
 }
 
 func GetTapologyLink(client ClientScraper, name string) (string, error) {
-	selection, err := client.Get("https://www.tapology.com/search/nav", func(r *http.Request) {
+	if !client.HasHeader("X-CSRF-Token") {
+		return "", fmt.Errorf("CSRF token not set.")
+	}
+
+	selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) {
 		query := url.Values{}
 		query.Set("ajax", "true")
 		query.Set("model", "fighters")
@@ -294,5 +301,5 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) {
 		}
 	})
 
-	return results[0].Url, nil
+	return TAPOLOGY_URL + results[0].Url, nil
 }

scrapers_test.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 	"fmt"
 	"testing"
+	"net/http"
 
 	"github.com/PuerkitoBio/goquery"
 )
@@ -51,6 +52,9 @@ var ONEFN29 []byte
 //go:embed testdata/one-172.html
 var ONE172 []byte
 
+//go:embed testdata/tapology-index.html
+var TapologyIndex []byte
+
 var HTMLContent map[string][]byte = map[string][]byte{
 	"https://www.ufc.com/events":                                 UFCEventList,
 	"https://www.ufc.com/event/ufc-312":                          UFC312,
@@ -66,11 +70,21 @@ var HTMLContent map[string][]byte = map[string][]byte{
 	"https://www.onefc.com/events/one171/":                       ONE171,
 	"https://www.onefc.com/events/onefightnight29/":              ONEFN29,
 	"https://www.onefc.com/events/one172/":                       ONE172,
+	"https://www.tapology.com":                                   TapologyIndex,
 }
 
-type DummyClient struct{}
+type DummyClient struct {
+	headers map[string]string
+}
 
 func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	for _, option := range options {
+		if option != nil {
+			option(req)
+		}
+	}
+	url = req.URL.String()
 	htmlstring, exists := HTMLContent[url]
 	if !exists {
 		return nil, fmt.Errorf("Invalid URL: %s", url)
@@ -82,11 +96,23 @@ func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Select
 	return document.Find("html").First(), nil
 }
 
-func (c *DummyClient) AddHeader(key string, value string) {}
+func (c *DummyClient) AddHeader(key string, value string) {
+	c.headers[key] = value
+}
+func (c *DummyClient) HasHeader(key string) bool {
+	_, exists := c.headers[key]
+	return exists
+}
+
+func NewDummyClient() *DummyClient {
+	return &DummyClient{
+		headers: map[string]string{},
+	}
+}
 
 func TestUFC(t *testing.T) {
-	client := DummyClient{}
+	client := NewDummyClient()
-	events, err := ScrapeUFC(&client)
+	events, err := ScrapeUFC(client)
 	if err != nil {
 		t.Errorf("Scraping UFC failed: %s", err)
 	}
@@ -100,8 +126,8 @@ func TestUFC(t *testing.T) {
 }
 
 func TestONE(t *testing.T) {
-	client := DummyClient{}
+	client := NewDummyClient()
-	events, err := ScrapeONE(&client)
+	events, err := ScrapeONE(client)
 	if err != nil {
 		t.Errorf("Scraping ONE failed: %s", err)
 	}
@@ -112,10 +138,10 @@ func TestONE(t *testing.T) {
 }
 
 func TestScrapeEvents(t *testing.T) {
-	client := DummyClient{}
+	client := NewDummyClient()
 	q, err := InitDb("test-db.sqlite")
 	if err != nil {
 		t.Error("Error initializing database: ", err)
 	}
-	ScrapeEvents(q, &client)
+	ScrapeEvents(q, client)
 }

sql/query.sql

@@ -8,6 +8,30 @@ WHERE
 LIMIT
   1;
 
+-- name: GetUpcomingEvent :one
+SELECT
+  *
+FROM
+  event
+WHERE
+  date >= ?
+ORDER BY
+  date ASC
+LIMIT
+  1;
+
+-- name: ListEvents :many
+SELECT
+  name,
+  slug,
+  date
+FROM
+  event
+WHERE
+  date >= ?
+ORDER BY
+  date ASC;
+
 -- name: GetTapology :one
 SELECT
   *
@@ -17,3 +41,9 @@ WHERE
   name = ?
 LIMIT
   1;
+
+-- name: CreateTapology :exec
+INSERT INTO
+  tapology (name, url)
+VALUES
+  (?, ?);