guy176251/mmaschedule
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactored scrapers

Browse Source
This commit is contained in:
long
2025-02-16 15:07:59 -06:00
parent fc21549791
commit 283ed3eaea
13 changed files with 800 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1 -1
View File
@@ -1,2 +1,2 @@
testdata/
db.sqlite
db.sqlite*
event/db.go
+31
View File
@@ -0,0 +1,31 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.26.0
package event
import (
"context"
"database/sql"
)
type DBTX interface {
ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
PrepareContext(context.Context, string) (*sql.Stmt, error)
QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
QueryRowContext(context.Context, string, ...interface{}) *sql.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
type Queries struct {
db DBTX
}
func (q *Queries) WithTx(tx *sql.Tx) *Queries {
return &Queries{
db: tx,
}
}
event/impl.go
+75
View File
@@ -0,0 +1,75 @@
package event
import (
"context"
"encoding/json"
)
type Fight struct {
Weight string `json:"weight"`
FighterA *Fighter `json:"fighter_a"`
FighterB *Fighter `json:"fighter_b"`
}
type Fighter struct {
Name string `json:"name"`
Image string `json:"image"`
Country string `json:"country"`
Link string `json:"link"`
}
func (e *Event) UnmarshalFights() []*Fight {
var fights []*Fight
_ = json.Unmarshal([]byte(e.Fights), &fights)
return fights
}
func (e *Event) MarshalFights(fights []*Fight) {
buf, _ := json.Marshal(&fights)
e.Fights = string(buf)
}
const upsertEvents = `-- name: UpsertEvents :exec
INSERT INTO
event (
url,
slug,
name,
location,
organization,
image,
date,
fights,
history
)
SELECT
json_extract (e.value, '$.url'),
json_extract (e.value, '$.slug'),
json_extract (e.value, '$.name'),
json_extract (e.value, '$.location'),
json_extract (e.value, '$.organization'),
json_extract (e.value, '$.image'),
json_extract (e.value, '$.date'),
json_extract (e.value, '$.fights'),
json_extract (e.value, '$.history')
FROM
json_each (?) AS e ON CONFLICT (url, slug) DO
UPDATE
SET
name = excluded.name,
location = excluded.location,
organization = excluded.organization,
image = excluded.image,
date = excluded.date,
fights = excluded.fights,
history = excluded.history
`
func (q *Queries) UpsertEvents(ctx context.Context, e []*Event) error {
b, err := json.Marshal(&e)
if err != nil {
return err
}
_, err = q.db.ExecContext(ctx, upsertEvents, string(b))
return err
}
event/models.go
+22
View File
@@ -0,0 +1,22 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.26.0
package event
type Event struct {
Url string `json:"url"`
Slug string `json:"slug"`
Name string `json:"name"`
Location string `json:"location"`
Organization string `json:"organization"`
Image string `json:"image"`
Date int64 `json:"date"`
Fights string `json:"fights"`
History string `json:"history"`
}
type Tapology struct {
Name string `json:"name"`
Url string `json:"url"`
}
event/query.sql.go
+56
View File
@@ -0,0 +1,56 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.26.0
// source: query.sql
package event
import (
"context"
)
const getEvent = `-- name: GetEvent :one
SELECT
url, slug, name, location, organization, image, date, fights, history
FROM
event
WHERE
slug = ?
LIMIT
1
`
func (q *Queries) GetEvent(ctx context.Context, slug string) (Event, error) {
row := q.db.QueryRowContext(ctx, getEvent, slug)
var i Event
err := row.Scan(
&i.Url,
&i.Slug,
&i.Name,
&i.Location,
&i.Organization,
&i.Image,
&i.Date,
&i.Fights,
&i.History,
)
return i, err
}
const getTapology = `-- name: GetTapology :one
SELECT
name, url
FROM
tapology
WHERE
name = ?
LIMIT
1
`
func (q *Queries) GetTapology(ctx context.Context, name string) (Tapology, error) {
row := q.db.QueryRowContext(ctx, getTapology, name)
var i Tapology
err := row.Scan(&i.Name, &i.Url)
return i, err
}
go.mod
+9 -6
View File
@@ -1,9 +1,11 @@
module mmaschedule-go
go 1.22.4
go 1.23
toolchain go1.23.5
require (
github.com/PuerkitoBio/goquery v1.9.2
github.com/PuerkitoBio/goquery v1.10.1
github.com/adrg/strutil v0.3.1
github.com/gocolly/colly v1.2.0
github.com/mattn/go-sqlite3 v1.14.23
@@ -11,7 +13,8 @@ require (
)
require (
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/a-h/templ v0.3.833 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/antchfx/htmlquery v1.3.1 // indirect
github.com/antchfx/xmlquery v1.4.0 // indirect
github.com/antchfx/xpath v1.3.0 // indirect
@@ -24,9 +27,9 @@ require (
github.com/sethvargo/go-retry v0.3.0 // indirect
github.com/temoto/robotstxt v1.1.2 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/text v0.21.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.33.0 // indirect
)
go.sum
+44
View File
@@ -1,9 +1,15 @@
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU=
github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY=
github.com/a-h/templ v0.3.833 h1:L/KOk/0VvVTBegtE0fp2RJQiBm7/52Zxv5fqlEHiQUU=
github.com/a-h/templ v0.3.833/go.mod h1:cAu4AiZhtJfBjMY0HASlyzvkrtjnHWPeEsyGK2YYmfk=
github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4=
github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/antchfx/htmlquery v1.3.1 h1:wm0LxjLMsZhRHfQKKZscDf2COyH4vDYA3wyH+qZ+Ylc=
github.com/antchfx/htmlquery v1.3.1/go.mod h1:PTj+f1V2zksPlwNt7uVvZPsxpKNa7mlVliCRxLX6Nx8=
github.com/antchfx/xmlquery v1.4.0 h1:xg2HkfcRK2TeTbdb0m1jxCYnvsPaGY/oeZWTGqX/0hA=
@@ -27,6 +33,7 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@@ -67,21 +74,39 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -89,24 +114,43 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
helpers.go
+90
View File
@@ -0,0 +1,90 @@
package main
import (
"encoding/json"
"log"
"net/http"
"net/url"
"regexp"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/adrg/strutil"
"github.com/adrg/strutil/metrics"
)
type Values = url.Values
func TextContent(doc *goquery.Selection, selector string) string {
return CleanupWhitespace(doc.Find(selector).First().Text())
}
var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
func CleanupWhitespace(s string) string {
return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
}
func PrintJson(v any) {
out, err := json.MarshalIndent(v, "", " ")
if err != nil {
log.Println(err)
} else {
log.Println(string(out))
}
}
var hamming *metrics.Hamming = metrics.NewHamming()
func HammingScore(a, b string) float64 {
return strutil.Similarity(a, b, hamming)
}
type ScraperClient struct {
headers map[string]string
client http.Client
}
type RequestOption func(*http.Request)
func (c *ScraperClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) {
resp, err := c.GetResponse(url, options...)
if err != nil {
return nil, err
}
selection, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, err
}
return selection.Find("html").First(), nil
}
func (c *ScraperClient) GetResponse(url string, options ...RequestOption) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
for key, value := range c.headers {
req.Header.Set(key, value)
}
for _, option := range options {
if option != nil {
option(req)
}
}
return c.client.Do(req)
}
func (c *ScraperClient) AddHeader(key string, value string) {
c.headers[key] = value
}
func NewScraperClient() ScraperClient {
client := ScraperClient{
headers: map[string]string{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
},
client: http.Client{},
}
return client
}
migrations/20240915072431_initial.sql
+8
View File
@@ -1,5 +1,7 @@
-- +goose Up
-- +goose StatementBegin
--PRAGMA journal_mode = WAL;
CREATE TABLE event (
url TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL UNIQUE,
@@ -11,4 +13,10 @@ CREATE TABLE event (
fights TEXT NOT NULL,
history TEXT NOT NULL
);
CREATE TABLE tapology (
name TEXT NOT NULL UNIQUE,
url TEXT NOT NULL UNIQUE
);
-- +goose StatementEnd
orchestrator.go
+26
View File
@@ -0,0 +1,26 @@
package main
import (
"fmt"
"mmaschedule-go/event"
)
type Scraper func(ClientScraper) (*[]*event.Event, error)
func ScrapeEvents() {
client := NewScraperClient()
events := []*event.Event{}
scrapers := []Scraper{
ScrapeONE,
ScrapeUFC,
}
for _, scraper := range scrapers {
e, err := scraper(&client)
if err != nil {
fmt.Println(err)
} else {
events = append(events, *e...)
}
}
}
scrapers.go
+298
View File
@@ -0,0 +1,298 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"regexp"
"slices"
"strconv"
"strings"
"time"
"mmaschedule-go/event"
"github.com/PuerkitoBio/goquery"
)
type ClientScraper interface {
Get(url string, options ...RequestOption) (*goquery.Selection, error)
AddHeader(key string, value string)
}
func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
events := []*event.Event{}
page, err := client.Get("https://www.onefc.com/events/")
if err != nil {
return nil, err
}
page.Find("#upcoming-events-section .is-event").Each(func(i int, s *goquery.Selection) {
_, exists := s.Find("a.smart-link").Attr("href")
if !exists {
return
}
url, exists := s.Find("a.is-image-zoom[href]").Attr("href")
if !exists {
return
}
e := event.Event{
Url: url,
Organization: "ONE",
Slug: strings.Trim(
strings.Replace(url, "https://www.onefc.com/events/", "", 1),
"/",
),
}
err := ScrapeONEEvent(client, &e)
if err != nil {
log.Println(err)
return
}
if e.Fights != "null" {
return
}
events = append(events, &e)
})
return &events, nil
}
type ONEEventLocation struct {
Name string `json:"name"`
}
type ONEEventData struct {
Name string `json:"name"`
StartDate string `json:"startDate"`
Image []string `json:"image"`
Location ONEEventLocation `json:"location"`
}
func ScrapeONEEvent(client ClientScraper, e *event.Event) error {
page, err := client.Get(e.Url)
if err != nil {
return err
}
data := ONEEventData{}
_ = json.Unmarshal([]byte(page.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data)
e.Name = data.Name
e.Location = data.Location.Name
if len(data.Image) > 2 {
e.Image = data.Image[2]
}
date, err := time.Parse(time.RFC3339, data.StartDate)
if err == nil {
e.Date = date.Unix()
}
var fights []*event.Fight
page.Find(".event-matchup").Each(func(i int, s *goquery.Selection) {
fights = append(fights, ScrapeONEFight(s))
})
e.MarshalFights(fights)
return nil
}
func ScrapeONEFight(s *goquery.Selection) *event.Fight {
fight := event.Fight{}
fight.Weight = TextContent(s, ".title")
fight.FighterA = ScrapeONEFighter(s, true)
fight.FighterB = ScrapeONEFighter(s, false)
return &fight
}
func ScrapeONEFighter(s *goquery.Selection, is_a bool) *event.Fighter {
fighter := event.Fighter{}
nth_child := "1"
face := "1"
if is_a {
nth_child = "3"
face = "2"
}
fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src")
fighter.Image = image
return &fighter
}
func ScrapeUFC(client ClientScraper) (*[]*event.Event, error) {
events := []*event.Event{}
page, err := client.Get("https://www.ufc.com/events")
if err != nil {
return nil, err
}
page.Find("#events-list-upcoming .c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
slug, exists := s.Attr("href")
if exists {
e := event.Event{
Url: "https://www.ufc.com" + slug,
Organization: "UFC",
Slug: strings.Replace(slug, "/event/", "", 1),
}
err := ScrapeUFCEvent(client, &e)
if err != nil {
log.Println(err)
} else if e.Fights != "null" {
events = append(events, &e)
}
}
})
return &events, nil
}
func ScrapeUFCEvent(client ClientScraper, e *event.Event) error {
page, err := client.Get(e.Url)
if err != nil {
return err
}
e.Name = strings.Join(
page.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
return CleanupWhitespace(s.Text())
}),
": ",
)
e.Location = TextContent(page, ".field--name-venue")
image, exists := page.Find(".c-hero__image img").First().Attr("src")
if exists {
e.Image = image
}
datestr, exists := page.Find(`
#early-prelims .c-event-fight-card-broadcaster__time,
#prelims-card .c-event-fight-card-broadcaster__time,
.c-hero__headline-suffix
`).Last().Attr("data-timestamp")
if exists {
date, err := strconv.Atoi(datestr)
if err == nil {
e.Date = int64(date)
}
}
var fights []*event.Fight
page.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) {
fights = append(fights, ScrapeUFCFight(s))
})
e.MarshalFights(fights)
return nil
}
func ScrapeUFCFight(s *goquery.Selection) *event.Fight {
fight := event.Fight{}
fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1)
fight.FighterA = ScrapeUFCFighter(s, "red")
fight.FighterB = ScrapeUFCFighter(s, "blue")
return &fight
}
func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter {
fighter := event.Fighter{}
fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color)
fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
image, exists := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
if exists {
fighter.Image = image
}
return &fighter
}
func SetTapologyCSRF(client ClientScraper) error {
selection, err := client.Get("https://www.tapology.com")
if err != nil {
return err
}
token, exists := selection.Find("meta[name=\"csrf-token\"]").First().Attr("content")
if !exists {
return fmt.Errorf("Could not parse CSRF token.")
}
client.AddHeader("X-CSRF-Token", token)
return nil
}
var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`)
type TapologyResult struct {
Name string `json:"name"`
Url string `json:"url"`
}
func GetTapologyLink(client ClientScraper, name string) (string, error) {
selection, err := client.Get("https://www.tapology.com/search/nav", func(r *http.Request) {
query := url.Values{}
query.Set("ajax", "true")
query.Set("model", "fighters")
query.Set("term", name)
r.URL.RawQuery = query.Encode()
})
if err != nil {
return "", err
}
results := []TapologyResult{}
selection.Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
url, _ := s.Attr("href")
name := CleanupWhitespace(nickname.ReplaceAllString(s.Text(), ""))
results = append(results, TapologyResult{Name: name, Url: url})
})
if len(results) == 0 {
return "", nil
}
slices.SortStableFunc(results, func(a, b TapologyResult) int {
score_a := HammingScore(name, a.Name)
score_b := HammingScore(name, b.Name)
if score_a > score_b {
return -1
} else if score_a < score_b {
return 1
} else {
return 0
}
})
return results[0].Url, nil
}
scrapers_test.go
+129
View File
@@ -0,0 +1,129 @@
package main
import (
"bytes"
_ "embed"
"fmt"
"testing"
"github.com/PuerkitoBio/goquery"
)
//go:embed testdata/ufc-event-list.html
var UFCEventList []byte
//go:embed testdata/ufc-312.html
var UFC312 []byte
//go:embed testdata/ufc-fn-feb-15.html
var UFCFNFeb15 []byte
//go:embed testdata/ufc-fn-feb-22.html
var UFCFNFeb22 []byte
//go:embed testdata/ufc-fn-mar-1.html
var UFCFNMar1 []byte
//go:embed testdata/ufc-313.html
var UFC313 []byte
//go:embed testdata/ufc-fn-mar-15.html
var UFCFNMar15 []byte
//go:embed testdata/ufc-fn-mar-22.html
var UFCFNMar22 []byte
//go:embed testdata/ufc-fn-mar-29.html
var UFCFNMar29 []byte
//go:embed testdata/one-event-list.html
var ONEEventList []byte
//go:embed testdata/one-friday-fights-97.html
var ONEFridayFights97 []byte
//go:embed testdata/one-171.html
var ONE171 []byte
//go:embed testdata/one-fn-29.html
var ONEFN29 []byte
//go:embed testdata/one-172.html
var ONE172 []byte
var HTMLContent map[string][]byte = map[string][]byte{
"https://www.ufc.com/events": UFCEventList,
"https://www.ufc.com/event/ufc-312": UFC312,
"https://www.ufc.com/event/ufc-fight-night-february-15-2025": UFCFNFeb15,
"https://www.ufc.com/event/ufc-fight-night-february-22-2025": UFCFNFeb22,
"https://www.ufc.com/event/ufc-fight-night-march-01-2025": UFCFNMar1,
"https://www.ufc.com/event/ufc-313": UFC313,
"https://www.ufc.com/event/ufc-fight-night-march-15-2025": UFCFNMar15,
"https://www.ufc.com/event/ufc-fight-night-march-22-2025": UFCFNMar22,
"https://www.ufc.com/event/ufc-fight-night-march-29-2025": UFCFNMar29,
"https://www.onefc.com/events/": ONEEventList,
"https://www.onefc.com/events/one-friday-fights-97/": ONEFridayFights97,
"https://www.onefc.com/events/one171/": ONE171,
"https://www.onefc.com/events/onefightnight29/": ONEFN29,
"https://www.onefc.com/events/one172/": ONE172,
}
type DummyClient struct{}
func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) {
htmlstring, exists := HTMLContent[url]
if !exists {
return nil, fmt.Errorf("Invalid URL: %s", url)
}
document, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlstring))
if err != nil {
return nil, err
}
return document.Find("html").First(), nil
}
func (c *DummyClient) AddHeader(key string, value string) {}
func TestUFC(t *testing.T) {
client := DummyClient{}
events, err := ScrapeUFC(&client)
if err != nil {
t.Errorf("Scraping UFC failed: %s", err)
}
if len(*events) == 0 {
t.Fatal("Scraping UFC failed: Events are empty")
}
if len(*events) < 7 {
t.Fatal("Scraping UFC failed: Did not get all events")
}
//PrintJson(events)
}
func TestONE(t *testing.T) {
client := DummyClient{}
events, err := ScrapeONE(&client)
if err != nil {
t.Errorf("Scraping ONE failed: %s", err)
}
PrintJson(events)
}
func TestJSONString(t *testing.T) {
type User struct {
Name string
}
type Thing struct {
Users []User `json:"user,string"`
Num int `json:"num,string"`
}
u := Thing{
Users: []User{
{Name: "Bob"},
{Name: "John"},
},
}
PrintJson(u)
}
sql/query.sql
+11 -1
View File
@@ -4,6 +4,16 @@ SELECT
FROM
event
WHERE
url = ?
slug = ?
LIMIT
1;
-- name: GetTapology :one
SELECT
*
FROM
tapology
WHERE
name = ?
LIMIT
1;