diff --git a/.gitignore b/.gitignore index 5efed52..acbfedd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ testdata/ -db.sqlite +db.sqlite* diff --git a/event/db.go b/event/db.go new file mode 100644 index 0000000..032eef6 --- /dev/null +++ b/event/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 + +package event + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/event/impl.go b/event/impl.go new file mode 100644 index 0000000..5d303b3 --- /dev/null +++ b/event/impl.go @@ -0,0 +1,75 @@ +package event + +import ( + "context" + "encoding/json" +) + +type Fight struct { + Weight string `json:"weight"` + FighterA *Fighter `json:"fighter_a"` + FighterB *Fighter `json:"fighter_b"` +} + +type Fighter struct { + Name string `json:"name"` + Image string `json:"image"` + Country string `json:"country"` + Link string `json:"link"` +} + +func (e *Event) UnmarshalFights() []*Fight { + var fights []*Fight + _ = json.Unmarshal([]byte(e.Fights), &fights) + return fights +} + +func (e *Event) MarshalFights(fights []*Fight) { + buf, _ := json.Marshal(&fights) + e.Fights = string(buf) +} + +const upsertEvents = `-- name: UpsertEvents :exec +INSERT INTO + event ( + url, + slug, + name, + location, + organization, + image, + date, + fights, + history + ) +SELECT + json_extract (e.value, '$.url'), + json_extract (e.value, '$.slug'), + json_extract (e.value, '$.name'), + json_extract (e.value, '$.location'), + json_extract (e.value, '$.organization'), + json_extract (e.value, '$.image'), + json_extract (e.value, '$.date'), + json_extract (e.value, '$.fights'), + json_extract (e.value, '$.history') +FROM + json_each (?) AS e ON CONFLICT (url, slug) DO +UPDATE +SET + name = excluded.name, + location = excluded.location, + organization = excluded.organization, + image = excluded.image, + date = excluded.date, + fights = excluded.fights, + history = excluded.history +` + +func (q *Queries) UpsertEvents(ctx context.Context, e []*Event) error { + b, err := json.Marshal(&e) + if err != nil { + return err + } + _, err = q.db.ExecContext(ctx, upsertEvents, string(b)) + return err +} diff --git a/event/models.go b/event/models.go new file mode 100644 index 0000000..03a2173 --- /dev/null +++ b/event/models.go @@ -0,0 +1,22 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 + +package event + +type Event struct { + Url string `json:"url"` + Slug string `json:"slug"` + Name string `json:"name"` + Location string `json:"location"` + Organization string `json:"organization"` + Image string `json:"image"` + Date int64 `json:"date"` + Fights string `json:"fights"` + History string `json:"history"` +} + +type Tapology struct { + Name string `json:"name"` + Url string `json:"url"` +} diff --git a/event/query.sql.go b/event/query.sql.go new file mode 100644 index 0000000..d7f97f8 --- /dev/null +++ b/event/query.sql.go @@ -0,0 +1,56 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 +// source: query.sql + +package event + +import ( + "context" +) + +const getEvent = `-- name: GetEvent :one +SELECT + url, slug, name, location, organization, image, date, fights, history +FROM + event +WHERE + slug = ? +LIMIT + 1 +` + +func (q *Queries) GetEvent(ctx context.Context, slug string) (Event, error) { + row := q.db.QueryRowContext(ctx, getEvent, slug) + var i Event + err := row.Scan( + &i.Url, + &i.Slug, + &i.Name, + &i.Location, + &i.Organization, + &i.Image, + &i.Date, + &i.Fights, + &i.History, + ) + return i, err +} + +const getTapology = `-- name: GetTapology :one +SELECT + name, url +FROM + tapology +WHERE + name = ? +LIMIT + 1 +` + +func (q *Queries) GetTapology(ctx context.Context, name string) (Tapology, error) { + row := q.db.QueryRowContext(ctx, getTapology, name) + var i Tapology + err := row.Scan(&i.Name, &i.Url) + return i, err +} diff --git a/go.mod b/go.mod index 7861735..ffb98ba 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,11 @@ module mmaschedule-go -go 1.22.4 +go 1.23 + +toolchain go1.23.5 require ( - github.com/PuerkitoBio/goquery v1.9.2 + github.com/PuerkitoBio/goquery v1.10.1 github.com/adrg/strutil v0.3.1 github.com/gocolly/colly v1.2.0 github.com/mattn/go-sqlite3 v1.14.23 @@ -11,7 +13,8 @@ require ( ) require ( - github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/a-h/templ v0.3.833 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect github.com/antchfx/htmlquery v1.3.1 // indirect github.com/antchfx/xmlquery v1.4.0 // indirect github.com/antchfx/xpath v1.3.0 // indirect @@ -24,9 +27,9 @@ require ( github.com/sethvargo/go-retry v0.3.0 // indirect github.com/temoto/robotstxt v1.1.2 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/go.sum b/go.sum index 09a1d01..53bf59c 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,15 @@ github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= +github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU= +github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY= +github.com/a-h/templ v0.3.833 h1:L/KOk/0VvVTBegtE0fp2RJQiBm7/52Zxv5fqlEHiQUU= +github.com/a-h/templ v0.3.833/go.mod h1:cAu4AiZhtJfBjMY0HASlyzvkrtjnHWPeEsyGK2YYmfk= github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4= github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/antchfx/htmlquery v1.3.1 h1:wm0LxjLMsZhRHfQKKZscDf2COyH4vDYA3wyH+qZ+Ylc= github.com/antchfx/htmlquery v1.3.1/go.mod h1:PTj+f1V2zksPlwNt7uVvZPsxpKNa7mlVliCRxLX6Nx8= github.com/antchfx/xmlquery v1.4.0 h1:xg2HkfcRK2TeTbdb0m1jxCYnvsPaGY/oeZWTGqX/0hA= @@ -27,6 +33,7 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -67,21 +74,39 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -89,24 +114,43 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..eae2342 --- /dev/null +++ b/helpers.go @@ -0,0 +1,90 @@ +package main + +import ( + "encoding/json" + "log" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/adrg/strutil" + "github.com/adrg/strutil/metrics" +) + +type Values = url.Values + +func TextContent(doc *goquery.Selection, selector string) string { + return CleanupWhitespace(doc.Find(selector).First().Text()) +} + +var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`) + +func CleanupWhitespace(s string) string { + return strings.TrimSpace(whitespace.ReplaceAllString(s, " ")) +} + +func PrintJson(v any) { + out, err := json.MarshalIndent(v, "", " ") + if err != nil { + log.Println(err) + } else { + log.Println(string(out)) + } +} + +var hamming *metrics.Hamming = metrics.NewHamming() + +func HammingScore(a, b string) float64 { + return strutil.Similarity(a, b, hamming) +} + +type ScraperClient struct { + headers map[string]string + client http.Client +} + +type RequestOption func(*http.Request) + +func (c *ScraperClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) { + resp, err := c.GetResponse(url, options...) + if err != nil { + return nil, err + } + selection, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, err + } + return selection.Find("html").First(), nil +} + +func (c *ScraperClient) GetResponse(url string, options ...RequestOption) (*http.Response, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + for key, value := range c.headers { + req.Header.Set(key, value) + } + for _, option := range options { + if option != nil { + option(req) + } + } + return c.client.Do(req) +} + +func (c *ScraperClient) AddHeader(key string, value string) { + c.headers[key] = value +} + +func NewScraperClient() ScraperClient { + client := ScraperClient{ + headers: map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + }, + client: http.Client{}, + } + return client +} diff --git a/migrations/20240915072431_initial.sql b/migrations/20240915072431_initial.sql index a0b6621..828b087 100644 --- a/migrations/20240915072431_initial.sql +++ b/migrations/20240915072431_initial.sql @@ -1,5 +1,7 @@ -- +goose Up -- +goose StatementBegin +--PRAGMA journal_mode = WAL; + CREATE TABLE event ( url TEXT NOT NULL UNIQUE, slug TEXT NOT NULL UNIQUE, @@ -11,4 +13,10 @@ CREATE TABLE event ( fights TEXT NOT NULL, history TEXT NOT NULL ); + +CREATE TABLE tapology ( + name TEXT NOT NULL UNIQUE, + url TEXT NOT NULL UNIQUE +); + -- +goose StatementEnd diff --git a/orchestrator.go b/orchestrator.go new file mode 100644 index 0000000..27513eb --- /dev/null +++ b/orchestrator.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "mmaschedule-go/event" +) + +type Scraper func(ClientScraper) (*[]*event.Event, error) + +func ScrapeEvents() { + client := NewScraperClient() + events := []*event.Event{} + scrapers := []Scraper{ + ScrapeONE, + ScrapeUFC, + } + + for _, scraper := range scrapers { + e, err := scraper(&client) + if err != nil { + fmt.Println(err) + } else { + events = append(events, *e...) + } + } +} diff --git a/scrapers.go b/scrapers.go new file mode 100644 index 0000000..62f2838 --- /dev/null +++ b/scrapers.go @@ -0,0 +1,298 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "mmaschedule-go/event" + + "github.com/PuerkitoBio/goquery" +) + +type ClientScraper interface { + Get(url string, options ...RequestOption) (*goquery.Selection, error) + AddHeader(key string, value string) +} + +func ScrapeONE(client ClientScraper) (*[]*event.Event, error) { + events := []*event.Event{} + page, err := client.Get("https://www.onefc.com/events/") + + if err != nil { + return nil, err + } + + page.Find("#upcoming-events-section .is-event").Each(func(i int, s *goquery.Selection) { + _, exists := s.Find("a.smart-link").Attr("href") + + if !exists { + return + } + + url, exists := s.Find("a.is-image-zoom[href]").Attr("href") + + if !exists { + return + } + + e := event.Event{ + Url: url, + Organization: "ONE", + Slug: strings.Trim( + strings.Replace(url, "https://www.onefc.com/events/", "", 1), + "/", + ), + } + + err := ScrapeONEEvent(client, &e) + + if err != nil { + log.Println(err) + return + } + + if e.Fights != "null" { + return + } + + events = append(events, &e) + }) + + return &events, nil +} + +type ONEEventLocation struct { + Name string `json:"name"` +} + +type ONEEventData struct { + Name string `json:"name"` + StartDate string `json:"startDate"` + Image []string `json:"image"` + Location ONEEventLocation `json:"location"` +} + +func ScrapeONEEvent(client ClientScraper, e *event.Event) error { + page, err := client.Get(e.Url) + + if err != nil { + return err + } + + data := ONEEventData{} + _ = json.Unmarshal([]byte(page.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data) + + e.Name = data.Name + e.Location = data.Location.Name + + if len(data.Image) > 2 { + e.Image = data.Image[2] + } + + date, err := time.Parse(time.RFC3339, data.StartDate) + if err == nil { + e.Date = date.Unix() + } + + var fights []*event.Fight + page.Find(".event-matchup").Each(func(i int, s *goquery.Selection) { + fights = append(fights, ScrapeONEFight(s)) + }) + e.MarshalFights(fights) + + return nil +} + +func ScrapeONEFight(s *goquery.Selection) *event.Fight { + fight := event.Fight{} + + fight.Weight = TextContent(s, ".title") + fight.FighterA = ScrapeONEFighter(s, true) + fight.FighterB = ScrapeONEFighter(s, false) + + return &fight +} + +func ScrapeONEFighter(s *goquery.Selection, is_a bool) *event.Fighter { + fighter := event.Fighter{} + nth_child := "1" + face := "1" + + if is_a { + nth_child = "3" + face = "2" + } + + fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child)) + fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child)) + + image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src") + fighter.Image = image + + return &fighter +} + +func ScrapeUFC(client ClientScraper) (*[]*event.Event, error) { + events := []*event.Event{} + page, err := client.Get("https://www.ufc.com/events") + + if err != nil { + return nil, err + } + + page.Find("#events-list-upcoming .c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) { + slug, exists := s.Attr("href") + if exists { + e := event.Event{ + Url: "https://www.ufc.com" + slug, + Organization: "UFC", + Slug: strings.Replace(slug, "/event/", "", 1), + } + err := ScrapeUFCEvent(client, &e) + if err != nil { + log.Println(err) + } else if e.Fights != "null" { + events = append(events, &e) + } + } + }) + + return &events, nil +} + +func ScrapeUFCEvent(client ClientScraper, e *event.Event) error { + page, err := client.Get(e.Url) + + if err != nil { + return err + } + + e.Name = strings.Join( + page.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string { + return CleanupWhitespace(s.Text()) + }), + ": ", + ) + e.Location = TextContent(page, ".field--name-venue") + + image, exists := page.Find(".c-hero__image img").First().Attr("src") + if exists { + e.Image = image + } + + datestr, exists := page.Find(` + + #early-prelims .c-event-fight-card-broadcaster__time, + #prelims-card .c-event-fight-card-broadcaster__time, + .c-hero__headline-suffix + + `).Last().Attr("data-timestamp") + + if exists { + date, err := strconv.Atoi(datestr) + if err == nil { + e.Date = int64(date) + } + } + + var fights []*event.Fight + page.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) { + fights = append(fights, ScrapeUFCFight(s)) + }) + e.MarshalFights(fights) + + return nil +} + +func ScrapeUFCFight(s *goquery.Selection) *event.Fight { + fight := event.Fight{} + + fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1) + fight.FighterA = ScrapeUFCFighter(s, "red") + fight.FighterB = ScrapeUFCFighter(s, "blue") + + return &fight +} + +func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter { + fighter := event.Fighter{} + + fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color) + fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text") + + image, exists := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src") + if exists { + fighter.Image = image + } + + return &fighter +} + +func SetTapologyCSRF(client ClientScraper) error { + selection, err := client.Get("https://www.tapology.com") + if err != nil { + return err + } + token, exists := selection.Find("meta[name=\"csrf-token\"]").First().Attr("content") + if !exists { + return fmt.Errorf("Could not parse CSRF token.") + } + client.AddHeader("X-CSRF-Token", token) + return nil +} + +var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`) + +type TapologyResult struct { + Name string `json:"name"` + Url string `json:"url"` +} + +func GetTapologyLink(client ClientScraper, name string) (string, error) { + selection, err := client.Get("https://www.tapology.com/search/nav", func(r *http.Request) { + query := url.Values{} + query.Set("ajax", "true") + query.Set("model", "fighters") + query.Set("term", name) + r.URL.RawQuery = query.Encode() + }) + + if err != nil { + return "", err + } + + results := []TapologyResult{} + + selection.Find("span.star a[href]").Each(func(i int, s *goquery.Selection) { + url, _ := s.Attr("href") + name := CleanupWhitespace(nickname.ReplaceAllString(s.Text(), "")) + results = append(results, TapologyResult{Name: name, Url: url}) + }) + + if len(results) == 0 { + return "", nil + } + + slices.SortStableFunc(results, func(a, b TapologyResult) int { + score_a := HammingScore(name, a.Name) + score_b := HammingScore(name, b.Name) + + if score_a > score_b { + return -1 + } else if score_a < score_b { + return 1 + } else { + return 0 + } + }) + + return results[0].Url, nil +} diff --git a/scrapers_test.go b/scrapers_test.go new file mode 100644 index 0000000..43a97a5 --- /dev/null +++ b/scrapers_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "bytes" + _ "embed" + "fmt" + "testing" + + "github.com/PuerkitoBio/goquery" +) + +//go:embed testdata/ufc-event-list.html +var UFCEventList []byte + +//go:embed testdata/ufc-312.html +var UFC312 []byte + +//go:embed testdata/ufc-fn-feb-15.html +var UFCFNFeb15 []byte + +//go:embed testdata/ufc-fn-feb-22.html +var UFCFNFeb22 []byte + +//go:embed testdata/ufc-fn-mar-1.html +var UFCFNMar1 []byte + +//go:embed testdata/ufc-313.html +var UFC313 []byte + +//go:embed testdata/ufc-fn-mar-15.html +var UFCFNMar15 []byte + +//go:embed testdata/ufc-fn-mar-22.html +var UFCFNMar22 []byte + +//go:embed testdata/ufc-fn-mar-29.html +var UFCFNMar29 []byte + +//go:embed testdata/one-event-list.html +var ONEEventList []byte + +//go:embed testdata/one-friday-fights-97.html +var ONEFridayFights97 []byte + +//go:embed testdata/one-171.html +var ONE171 []byte + +//go:embed testdata/one-fn-29.html +var ONEFN29 []byte + +//go:embed testdata/one-172.html +var ONE172 []byte + +var HTMLContent map[string][]byte = map[string][]byte{ + "https://www.ufc.com/events": UFCEventList, + "https://www.ufc.com/event/ufc-312": UFC312, + "https://www.ufc.com/event/ufc-fight-night-february-15-2025": UFCFNFeb15, + "https://www.ufc.com/event/ufc-fight-night-february-22-2025": UFCFNFeb22, + "https://www.ufc.com/event/ufc-fight-night-march-01-2025": UFCFNMar1, + "https://www.ufc.com/event/ufc-313": UFC313, + "https://www.ufc.com/event/ufc-fight-night-march-15-2025": UFCFNMar15, + "https://www.ufc.com/event/ufc-fight-night-march-22-2025": UFCFNMar22, + "https://www.ufc.com/event/ufc-fight-night-march-29-2025": UFCFNMar29, + "https://www.onefc.com/events/": ONEEventList, + "https://www.onefc.com/events/one-friday-fights-97/": ONEFridayFights97, + "https://www.onefc.com/events/one171/": ONE171, + "https://www.onefc.com/events/onefightnight29/": ONEFN29, + "https://www.onefc.com/events/one172/": ONE172, +} + +type DummyClient struct{} + +func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) { + htmlstring, exists := HTMLContent[url] + if !exists { + return nil, fmt.Errorf("Invalid URL: %s", url) + } + document, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlstring)) + if err != nil { + return nil, err + } + return document.Find("html").First(), nil +} + +func (c *DummyClient) AddHeader(key string, value string) {} + +func TestUFC(t *testing.T) { + client := DummyClient{} + events, err := ScrapeUFC(&client) + if err != nil { + t.Errorf("Scraping UFC failed: %s", err) + } + if len(*events) == 0 { + t.Fatal("Scraping UFC failed: Events are empty") + } + if len(*events) < 7 { + t.Fatal("Scraping UFC failed: Did not get all events") + } + //PrintJson(events) +} + +func TestONE(t *testing.T) { + client := DummyClient{} + events, err := ScrapeONE(&client) + if err != nil { + t.Errorf("Scraping ONE failed: %s", err) + } + PrintJson(events) +} + +func TestJSONString(t *testing.T) { + type User struct { + Name string + } + + type Thing struct { + Users []User `json:"user,string"` + Num int `json:"num,string"` + } + + u := Thing{ + Users: []User{ + {Name: "Bob"}, + {Name: "John"}, + }, + } + + PrintJson(u) +} diff --git a/sql/query.sql b/sql/query.sql index 3fdc8fb..1651cb7 100644 --- a/sql/query.sql +++ b/sql/query.sql @@ -4,6 +4,16 @@ SELECT FROM event WHERE - url = ? + slug = ? +LIMIT + 1; + +-- name: GetTapology :one +SELECT + * +FROM + tapology +WHERE + name = ? LIMIT 1;