guy176251/mmaschedule
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added tapology link getter to scraper

Browse Source
This commit is contained in:
long
2025-02-16 20:50:40 -06:00
parent 50c4133525
commit c8184d5a71
7 changed files with 209 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
event/query.sql.go
+89
View File
@@ -9,6 +9,23 @@ import (
"context"
)
const createTapology = `-- name: CreateTapology :exec
INSERT INTO
tapology (name, url)
VALUES
(?, ?)
`
type CreateTapologyParams struct {
Name string `json:"name"`
Url string `json:"url"`
}
func (q *Queries) CreateTapology(ctx context.Context, arg CreateTapologyParams) error {
_, err := q.db.ExecContext(ctx, createTapology, arg.Name, arg.Url)
return err
}
const getEvent = `-- name: GetEvent :one
SELECT
url, slug, name, location, organization, image, date, fights, history
@@ -54,3 +71,75 @@ func (q *Queries) GetTapology(ctx context.Context, name string) (Tapology, error
err := row.Scan(&i.Name, &i.Url)
return i, err
}
const getUpcomingEvent = `-- name: GetUpcomingEvent :one
SELECT
url, slug, name, location, organization, image, date, fights, history
FROM
event
WHERE
date >= ?
ORDER BY
date ASC
LIMIT
1
`
func (q *Queries) GetUpcomingEvent(ctx context.Context, date int64) (Event, error) {
row := q.db.QueryRowContext(ctx, getUpcomingEvent, date)
var i Event
err := row.Scan(
&i.Url,
&i.Slug,
&i.Name,
&i.Location,
&i.Organization,
&i.Image,
&i.Date,
&i.Fights,
&i.History,
)
return i, err
}
const listEvents = `-- name: ListEvents :many
SELECT
name,
slug,
date
FROM
event
WHERE
date >= ?
ORDER BY
date ASC
`
type ListEventsRow struct {
Name string `json:"name"`
Slug string `json:"slug"`
Date int64 `json:"date"`
}
func (q *Queries) ListEvents(ctx context.Context, date int64) ([]ListEventsRow, error) {
rows, err := q.db.QueryContext(ctx, listEvents, date)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListEventsRow
for rows.Next() {
var i ListEventsRow
if err := rows.Scan(&i.Name, &i.Slug, &i.Date); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
helpers.go
+6 -2
View File
@@ -78,13 +78,17 @@ func (c *ScraperClient) GetResponse(url string, options ...RequestOption) (*http
func (c *ScraperClient) AddHeader(key string, value string) {
c.headers[key] = value
}
func (c *ScraperClient) HasHeader(key string) bool {
_, exists := c.headers[key]
return exists
}
func NewScraperClient() ScraperClient {
func NewScraperClient() *ScraperClient {
client := ScraperClient{
headers: map[string]string{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
},
client: http.Client{},
}
return client
return &client
}
main.go
+4 -1
View File
@@ -16,9 +16,12 @@ import (
var migrations embed.FS
func main() {
_, err := InitDb("db.sqlite")
q, err := InitDb("db.sqlite")
if err != nil {
fmt.Println("Error initializing database: ", err)
} else {
client := NewScraperClient()
ScrapeEvents(q, client)
}
}
orchestrator.go
+36
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"mmaschedule-go/event"
"time"
)
type Scraper func(ClientScraper) (*[]*event.Event, error)
@@ -24,6 +25,41 @@ func ScrapeEvents(q *event.Queries, client ClientScraper) {
}
}
err := SetTapologyCSRF(client)
if err != nil {
fmt.Println("Error settings tapology CSRF: ", err)
}
for _, e := range events {
fights := e.UnmarshalFights()
for _, f := range fights {
fighters := []*event.Fighter{
f.FighterA,
f.FighterB,
}
for _, ff := range fighters {
tapology, err := q.GetTapology(context.Background(), ff.Name)
if err != nil {
fmt.Println("Error getting tapology from database: ", err)
link, err := GetTapologyLink(client, ff.Name)
if err != nil {
fmt.Println("Error getting tapology from site: ", err)
} else if len(link) > 0 {
err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link})
if err != nil {
fmt.Println("Error creating tapology in database: ", err)
}
ff.Link = link
}
time.Sleep(time.Duration(5_000_000_000))
} else {
ff.Link = tapology.Url
}
}
}
e.MarshalFights(fights)
}
if len(events) > 0 {
err := q.UpsertEvents(context.Background(), events)
if err != nil {
scrapers.go
+10 -3
View File
@@ -20,6 +20,7 @@ import (
type ClientScraper interface {
Get(url string, options ...RequestOption) (*goquery.Selection, error)
AddHeader(key string, value string)
HasHeader(key string) bool
}
func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
@@ -236,8 +237,10 @@ func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter {
return &fighter
}
const TAPOLOGY_URL = "https://www.tapology.com"
func SetTapologyCSRF(client ClientScraper) error {
selection, err := client.Get("https://www.tapology.com")
selection, err := client.Get(TAPOLOGY_URL)
if err != nil {
return err
}
@@ -257,7 +260,11 @@ type TapologyResult struct {
}
func GetTapologyLink(client ClientScraper, name string) (string, error) {
selection, err := client.Get("https://www.tapology.com/search/nav", func(r *http.Request) {
if !client.HasHeader("X-CSRF-Token") {
return "", fmt.Errorf("CSRF token not set.")
}
selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) {
query := url.Values{}
query.Set("ajax", "true")
query.Set("model", "fighters")
@@ -294,5 +301,5 @@ func GetTapologyLink(client ClientScraper, name string) (string, error) {
}
})
return results[0].Url, nil
return TAPOLOGY_URL + results[0].Url, nil
}
scrapers_test.go
+34 -8
View File
@@ -5,6 +5,7 @@ import (
_ "embed"
"fmt"
"testing"
"net/http"
"github.com/PuerkitoBio/goquery"
)
@@ -51,6 +52,9 @@ var ONEFN29 []byte
//go:embed testdata/one-172.html
var ONE172 []byte
//go:embed testdata/tapology-index.html
var TapologyIndex []byte
var HTMLContent map[string][]byte = map[string][]byte{
"https://www.ufc.com/events": UFCEventList,
"https://www.ufc.com/event/ufc-312": UFC312,
@@ -66,11 +70,21 @@ var HTMLContent map[string][]byte = map[string][]byte{
"https://www.onefc.com/events/one171/": ONE171,
"https://www.onefc.com/events/onefightnight29/": ONEFN29,
"https://www.onefc.com/events/one172/": ONE172,
"https://www.tapology.com": TapologyIndex,
}
type DummyClient struct{}
type DummyClient struct {
headers map[string]string
}
func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) {
req, err := http.NewRequest("GET", url, nil)
for _, option := range options {
if option != nil {
option(req)
}
}
url = req.URL.String()
htmlstring, exists := HTMLContent[url]
if !exists {
return nil, fmt.Errorf("Invalid URL: %s", url)
@@ -82,11 +96,23 @@ func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Select
return document.Find("html").First(), nil
}
func (c *DummyClient) AddHeader(key string, value string) {}
func (c *DummyClient) AddHeader(key string, value string) {
c.headers[key] = value
}
func (c *DummyClient) HasHeader(key string) bool {
_, exists := c.headers[key]
return exists
}
func NewDummyClient() *DummyClient {
return &DummyClient{
headers: map[string]string{},
}
}
func TestUFC(t *testing.T) {
client := DummyClient{}
events, err := ScrapeUFC(&client)
client := NewDummyClient()
events, err := ScrapeUFC(client)
if err != nil {
t.Errorf("Scraping UFC failed: %s", err)
}
@@ -100,8 +126,8 @@ func TestUFC(t *testing.T) {
}
func TestONE(t *testing.T) {
client := DummyClient{}
events, err := ScrapeONE(&client)
client := NewDummyClient()
events, err := ScrapeONE(client)
if err != nil {
t.Errorf("Scraping ONE failed: %s", err)
}
@@ -112,10 +138,10 @@ func TestONE(t *testing.T) {
}
func TestScrapeEvents(t *testing.T) {
client := DummyClient{}
client := NewDummyClient()
q, err := InitDb("test-db.sqlite")
if err != nil {
t.Error("Error initializing database: ", err)
}
ScrapeEvents(q, &client)
ScrapeEvents(q, client)
}
sql/query.sql
+30
View File
@@ -8,6 +8,30 @@ WHERE
LIMIT
1;
-- name: GetUpcomingEvent :one
SELECT
*
FROM
event
WHERE
date >= ?
ORDER BY
date ASC
LIMIT
1;
-- name: ListEvents :many
SELECT
name,
slug,
date
FROM
event
WHERE
date >= ?
ORDER BY
date ASC;
-- name: GetTapology :one
SELECT
*
@@ -17,3 +41,9 @@ WHERE
name = ?
LIMIT
1;
-- name: CreateTapology :exec
INSERT INTO
tapology (name, url)
VALUES
(?, ?);