From c8184d5a71e651ff1b5a821a4a34074e2e73d45c Mon Sep 17 00:00:00 2001 From: long Date: Sun, 16 Feb 2025 20:50:40 -0600 Subject: [PATCH] added tapology link getter to scraper --- event/query.sql.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++ helpers.go | 8 +++-- main.go | 5 ++- orchestrator.go | 36 +++++++++++++++++++ scrapers.go | 13 +++++-- scrapers_test.go | 42 +++++++++++++++++----- sql/query.sql | 30 ++++++++++++++++ 7 files changed, 209 insertions(+), 14 deletions(-) diff --git a/event/query.sql.go b/event/query.sql.go index d7f97f8..0ca4163 100644 --- a/event/query.sql.go +++ b/event/query.sql.go @@ -9,6 +9,23 @@ import ( "context" ) +const createTapology = `-- name: CreateTapology :exec +INSERT INTO + tapology (name, url) +VALUES + (?, ?) +` + +type CreateTapologyParams struct { + Name string `json:"name"` + Url string `json:"url"` +} + +func (q *Queries) CreateTapology(ctx context.Context, arg CreateTapologyParams) error { + _, err := q.db.ExecContext(ctx, createTapology, arg.Name, arg.Url) + return err +} + const getEvent = `-- name: GetEvent :one SELECT url, slug, name, location, organization, image, date, fights, history @@ -54,3 +71,75 @@ func (q *Queries) GetTapology(ctx context.Context, name string) (Tapology, error err := row.Scan(&i.Name, &i.Url) return i, err } + +const getUpcomingEvent = `-- name: GetUpcomingEvent :one +SELECT + url, slug, name, location, organization, image, date, fights, history +FROM + event +WHERE + date >= ? +ORDER BY + date ASC +LIMIT + 1 +` + +func (q *Queries) GetUpcomingEvent(ctx context.Context, date int64) (Event, error) { + row := q.db.QueryRowContext(ctx, getUpcomingEvent, date) + var i Event + err := row.Scan( + &i.Url, + &i.Slug, + &i.Name, + &i.Location, + &i.Organization, + &i.Image, + &i.Date, + &i.Fights, + &i.History, + ) + return i, err +} + +const listEvents = `-- name: ListEvents :many +SELECT + name, + slug, + date +FROM + event +WHERE + date >= ? +ORDER BY + date ASC +` + +type ListEventsRow struct { + Name string `json:"name"` + Slug string `json:"slug"` + Date int64 `json:"date"` +} + +func (q *Queries) ListEvents(ctx context.Context, date int64) ([]ListEventsRow, error) { + rows, err := q.db.QueryContext(ctx, listEvents, date) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListEventsRow + for rows.Next() { + var i ListEventsRow + if err := rows.Scan(&i.Name, &i.Slug, &i.Date); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/helpers.go b/helpers.go index eae2342..17b9ab6 100644 --- a/helpers.go +++ b/helpers.go @@ -78,13 +78,17 @@ func (c *ScraperClient) GetResponse(url string, options ...RequestOption) (*http func (c *ScraperClient) AddHeader(key string, value string) { c.headers[key] = value } +func (c *ScraperClient) HasHeader(key string) bool { + _, exists := c.headers[key] + return exists +} -func NewScraperClient() ScraperClient { +func NewScraperClient() *ScraperClient { client := ScraperClient{ headers: map[string]string{ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", }, client: http.Client{}, } - return client + return &client } diff --git a/main.go b/main.go index c7c3d33..8b1726e 100644 --- a/main.go +++ b/main.go @@ -16,9 +16,12 @@ import ( var migrations embed.FS func main() { - _, err := InitDb("db.sqlite") + q, err := InitDb("db.sqlite") if err != nil { fmt.Println("Error initializing database: ", err) + } else { + client := NewScraperClient() + ScrapeEvents(q, client) } } diff --git a/orchestrator.go b/orchestrator.go index 04b88a0..61cd1e2 100644 --- a/orchestrator.go +++ b/orchestrator.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "mmaschedule-go/event" + "time" ) type Scraper func(ClientScraper) (*[]*event.Event, error) @@ -24,6 +25,41 @@ func ScrapeEvents(q *event.Queries, client ClientScraper) { } } + err := SetTapologyCSRF(client) + if err != nil { + fmt.Println("Error settings tapology CSRF: ", err) + } + + for _, e := range events { + fights := e.UnmarshalFights() + for _, f := range fights { + fighters := []*event.Fighter{ + f.FighterA, + f.FighterB, + } + for _, ff := range fighters { + tapology, err := q.GetTapology(context.Background(), ff.Name) + if err != nil { + fmt.Println("Error getting tapology from database: ", err) + link, err := GetTapologyLink(client, ff.Name) + if err != nil { + fmt.Println("Error getting tapology from site: ", err) + } else if len(link) > 0 { + err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link}) + if err != nil { + fmt.Println("Error creating tapology in database: ", err) + } + ff.Link = link + } + time.Sleep(time.Duration(5_000_000_000)) + } else { + ff.Link = tapology.Url + } + } + } + e.MarshalFights(fights) + } + if len(events) > 0 { err := q.UpsertEvents(context.Background(), events) if err != nil { diff --git a/scrapers.go b/scrapers.go index 7971587..6082070 100644 --- a/scrapers.go +++ b/scrapers.go @@ -20,6 +20,7 @@ import ( type ClientScraper interface { Get(url string, options ...RequestOption) (*goquery.Selection, error) AddHeader(key string, value string) + HasHeader(key string) bool } func ScrapeONE(client ClientScraper) (*[]*event.Event, error) { @@ -236,8 +237,10 @@ func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter { return &fighter } +const TAPOLOGY_URL = "https://www.tapology.com" + func SetTapologyCSRF(client ClientScraper) error { - selection, err := client.Get("https://www.tapology.com") + selection, err := client.Get(TAPOLOGY_URL) if err != nil { return err } @@ -257,7 +260,11 @@ type TapologyResult struct { } func GetTapologyLink(client ClientScraper, name string) (string, error) { - selection, err := client.Get("https://www.tapology.com/search/nav", func(r *http.Request) { + if !client.HasHeader("X-CSRF-Token") { + return "", fmt.Errorf("CSRF token not set.") + } + + selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) { query := url.Values{} query.Set("ajax", "true") query.Set("model", "fighters") @@ -294,5 +301,5 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) { } }) - return results[0].Url, nil + return TAPOLOGY_URL + results[0].Url, nil } diff --git a/scrapers_test.go b/scrapers_test.go index d07ea7b..80eeded 100644 --- a/scrapers_test.go +++ b/scrapers_test.go @@ -5,6 +5,7 @@ import ( _ "embed" "fmt" "testing" + "net/http" "github.com/PuerkitoBio/goquery" ) @@ -51,6 +52,9 @@ var ONEFN29 []byte //go:embed testdata/one-172.html var ONE172 []byte +//go:embed testdata/tapology-index.html +var TapologyIndex []byte + var HTMLContent map[string][]byte = map[string][]byte{ "https://www.ufc.com/events": UFCEventList, "https://www.ufc.com/event/ufc-312": UFC312, @@ -66,11 +70,21 @@ var HTMLContent map[string][]byte = map[string][]byte{ "https://www.onefc.com/events/one171/": ONE171, "https://www.onefc.com/events/onefightnight29/": ONEFN29, "https://www.onefc.com/events/one172/": ONE172, + "https://www.tapology.com": TapologyIndex, } -type DummyClient struct{} +type DummyClient struct { + headers map[string]string +} func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) { + req, err := http.NewRequest("GET", url, nil) + for _, option := range options { + if option != nil { + option(req) + } + } + url = req.URL.String() htmlstring, exists := HTMLContent[url] if !exists { return nil, fmt.Errorf("Invalid URL: %s", url) @@ -82,11 +96,23 @@ func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Select return document.Find("html").First(), nil } -func (c *DummyClient) AddHeader(key string, value string) {} +func (c *DummyClient) AddHeader(key string, value string) { + c.headers[key] = value +} +func (c *DummyClient) HasHeader(key string) bool { + _, exists := c.headers[key] + return exists +} + +func NewDummyClient() *DummyClient { + return &DummyClient{ + headers: map[string]string{}, + } +} func TestUFC(t *testing.T) { - client := DummyClient{} - events, err := ScrapeUFC(&client) + client := NewDummyClient() + events, err := ScrapeUFC(client) if err != nil { t.Errorf("Scraping UFC failed: %s", err) } @@ -100,8 +126,8 @@ func TestUFC(t *testing.T) { } func TestONE(t *testing.T) { - client := DummyClient{} - events, err := ScrapeONE(&client) + client := NewDummyClient() + events, err := ScrapeONE(client) if err != nil { t.Errorf("Scraping ONE failed: %s", err) } @@ -112,10 +138,10 @@ func TestONE(t *testing.T) { } func TestScrapeEvents(t *testing.T) { - client := DummyClient{} + client := NewDummyClient() q, err := InitDb("test-db.sqlite") if err != nil { t.Error("Error initializing database: ", err) } - ScrapeEvents(q, &client) + ScrapeEvents(q, client) } diff --git a/sql/query.sql b/sql/query.sql index 1651cb7..e2a154c 100644 --- a/sql/query.sql +++ b/sql/query.sql @@ -8,6 +8,30 @@ WHERE LIMIT 1; +-- name: GetUpcomingEvent :one +SELECT + * +FROM + event +WHERE + date >= ? +ORDER BY + date ASC +LIMIT + 1; + +-- name: ListEvents :many +SELECT + name, + slug, + date +FROM + event +WHERE + date >= ? +ORDER BY + date ASC; + -- name: GetTapology :one SELECT * @@ -17,3 +41,9 @@ WHERE name = ? LIMIT 1; + +-- name: CreateTapology :exec +INSERT INTO + tapology (name, url) +VALUES + (?, ?);