refactored scrapers

.gitignore

@@ -1,2 +1,2 @@
 testdata/
-db.sqlite
+db.sqlite*

event/db.go

@@ -0,0 +1,31 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.26.0
+
+package event
+
+import (
+	"context"
+	"database/sql"
+)
+
+type DBTX interface {
+	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
+	PrepareContext(context.Context, string) (*sql.Stmt, error)
+	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
+	QueryRowContext(context.Context, string, ...interface{}) *sql.Row
+}
+
+func New(db DBTX) *Queries {
+	return &Queries{db: db}
+}
+
+type Queries struct {
+	db DBTX
+}
+
+func (q *Queries) WithTx(tx *sql.Tx) *Queries {
+	return &Queries{
+		db: tx,
+	}
+}

event/impl.go

@@ -0,0 +1,75 @@
+package event
+
+import (
+	"context"
+	"encoding/json"
+)
+
+type Fight struct {
+	Weight   string   `json:"weight"`
+	FighterA *Fighter `json:"fighter_a"`
+	FighterB *Fighter `json:"fighter_b"`
+}
+
+type Fighter struct {
+	Name    string `json:"name"`
+	Image   string `json:"image"`
+	Country string `json:"country"`
+	Link    string `json:"link"`
+}
+
+func (e *Event) UnmarshalFights() []*Fight {
+	var fights []*Fight
+	_ = json.Unmarshal([]byte(e.Fights), &fights)
+	return fights
+}
+
+func (e *Event) MarshalFights(fights []*Fight) {
+	buf, _ := json.Marshal(&fights)
+	e.Fights = string(buf)
+}
+
+const upsertEvents = `-- name: UpsertEvents :exec
+INSERT INTO
+  event (
+    url,
+    slug,
+    name,
+    location,
+    organization,
+    image,
+    date,
+    fights,
+    history
+  )
+SELECT
+  json_extract (e.value, '$.url'),
+  json_extract (e.value, '$.slug'),
+  json_extract (e.value, '$.name'),
+  json_extract (e.value, '$.location'),
+  json_extract (e.value, '$.organization'),
+  json_extract (e.value, '$.image'),
+  json_extract (e.value, '$.date'),
+  json_extract (e.value, '$.fights'),
+  json_extract (e.value, '$.history')
+FROM
+  json_each (?) AS e ON CONFLICT (url, slug) DO
+UPDATE
+SET
+  name = excluded.name,
+  location = excluded.location,
+  organization = excluded.organization,
+  image = excluded.image,
+  date = excluded.date,
+  fights = excluded.fights,
+  history = excluded.history
+`
+
+func (q *Queries) UpsertEvents(ctx context.Context, e []*Event) error {
+	b, err := json.Marshal(&e)
+	if err != nil {
+		return err
+	}
+	_, err = q.db.ExecContext(ctx, upsertEvents, string(b))
+	return err
+}

event/models.go

@@ -0,0 +1,22 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.26.0
+
+package event
+
+type Event struct {
+	Url          string `json:"url"`
+	Slug         string `json:"slug"`
+	Name         string `json:"name"`
+	Location     string `json:"location"`
+	Organization string `json:"organization"`
+	Image        string `json:"image"`
+	Date         int64  `json:"date"`
+	Fights       string `json:"fights"`
+	History      string `json:"history"`
+}
+
+type Tapology struct {
+	Name string `json:"name"`
+	Url  string `json:"url"`
+}

event/query.sql.go

@@ -0,0 +1,56 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.26.0
+// source: query.sql
+
+package event
+
+import (
+	"context"
+)
+
+const getEvent = `-- name: GetEvent :one
+SELECT
+  url, slug, name, location, organization, image, date, fights, history
+FROM
+  event
+WHERE
+  slug = ?
+LIMIT
+  1
+`
+
+func (q *Queries) GetEvent(ctx context.Context, slug string) (Event, error) {
+	row := q.db.QueryRowContext(ctx, getEvent, slug)
+	var i Event
+	err := row.Scan(
+		&i.Url,
+		&i.Slug,
+		&i.Name,
+		&i.Location,
+		&i.Organization,
+		&i.Image,
+		&i.Date,
+		&i.Fights,
+		&i.History,
+	)
+	return i, err
+}
+
+const getTapology = `-- name: GetTapology :one
+SELECT
+  name, url
+FROM
+  tapology
+WHERE
+  name = ?
+LIMIT
+  1
+`
+
+func (q *Queries) GetTapology(ctx context.Context, name string) (Tapology, error) {
+	row := q.db.QueryRowContext(ctx, getTapology, name)
+	var i Tapology
+	err := row.Scan(&i.Name, &i.Url)
+	return i, err
+}

go.mod

@@ -1,9 +1,11 @@
 module mmaschedule-go
 
-go 1.22.4
+go 1.23
+
+toolchain go1.23.5
 
 require (
-	github.com/PuerkitoBio/goquery v1.9.2
+	github.com/PuerkitoBio/goquery v1.10.1
 	github.com/adrg/strutil v0.3.1
 	github.com/gocolly/colly v1.2.0
 	github.com/mattn/go-sqlite3 v1.14.23
@@ -11,7 +13,8 @@ require (
 )
 
 require (
-	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/a-h/templ v0.3.833 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/antchfx/htmlquery v1.3.1 // indirect
 	github.com/antchfx/xmlquery v1.4.0 // indirect
 	github.com/antchfx/xpath v1.3.0 // indirect
@@ -24,9 +27,9 @@ require (
 	github.com/sethvargo/go-retry v0.3.0 // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 )

go.sum

@@ -1,9 +1,15 @@
 github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
 github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
+github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU=
+github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY=
+github.com/a-h/templ v0.3.833 h1:L/KOk/0VvVTBegtE0fp2RJQiBm7/52Zxv5fqlEHiQUU=
+github.com/a-h/templ v0.3.833/go.mod h1:cAu4AiZhtJfBjMY0HASlyzvkrtjnHWPeEsyGK2YYmfk=
 github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4=
 github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA=
 github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
 github.com/antchfx/htmlquery v1.3.1 h1:wm0LxjLMsZhRHfQKKZscDf2COyH4vDYA3wyH+qZ+Ylc=
 github.com/antchfx/htmlquery v1.3.1/go.mod h1:PTj+f1V2zksPlwNt7uVvZPsxpKNa7mlVliCRxLX6Nx8=
 github.com/antchfx/xmlquery v1.4.0 h1:xg2HkfcRK2TeTbdb0m1jxCYnvsPaGY/oeZWTGqX/0hA=
@@ -27,6 +33,7 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@@ -67,21 +74,39 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
 golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -89,24 +114,43 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
 golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

helpers.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/adrg/strutil"
+	"github.com/adrg/strutil/metrics"
+)
+
+type Values = url.Values
+
+func TextContent(doc *goquery.Selection, selector string) string {
+	return CleanupWhitespace(doc.Find(selector).First().Text())
+}
+
+var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
+
+func CleanupWhitespace(s string) string {
+	return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
+}
+
+func PrintJson(v any) {
+	out, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		log.Println(err)
+	} else {
+		log.Println(string(out))
+	}
+}
+
+var hamming *metrics.Hamming = metrics.NewHamming()
+
+func HammingScore(a, b string) float64 {
+	return strutil.Similarity(a, b, hamming)
+}
+
+type ScraperClient struct {
+	headers map[string]string
+	client  http.Client
+}
+
+type RequestOption func(*http.Request)
+
+func (c *ScraperClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) {
+	resp, err := c.GetResponse(url, options...)
+	if err != nil {
+		return nil, err
+	}
+	selection, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return selection.Find("html").First(), nil
+}
+
+func (c *ScraperClient) GetResponse(url string, options ...RequestOption) (*http.Response, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	for key, value := range c.headers {
+		req.Header.Set(key, value)
+	}
+	for _, option := range options {
+		if option != nil {
+			option(req)
+		}
+	}
+	return c.client.Do(req)
+}
+
+func (c *ScraperClient) AddHeader(key string, value string) {
+	c.headers[key] = value
+}
+
+func NewScraperClient() ScraperClient {
+	client := ScraperClient{
+		headers: map[string]string{
+			"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+		},
+		client: http.Client{},
+	}
+	return client
+}

migrations/20240915072431_initial.sql

@@ -1,5 +1,7 @@
 -- +goose Up
 -- +goose StatementBegin
+--PRAGMA journal_mode = WAL;
+
 CREATE TABLE event (
   url TEXT NOT NULL UNIQUE,
   slug TEXT NOT NULL UNIQUE,
@@ -11,4 +13,10 @@ CREATE TABLE event (
   fights TEXT NOT NULL,
   history TEXT NOT NULL
 );
+
+CREATE TABLE tapology (
+  name TEXT NOT NULL UNIQUE,
+  url TEXT NOT NULL UNIQUE
+);
+
 -- +goose StatementEnd

orchestrator.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+	"mmaschedule-go/event"
+)
+
+type Scraper func(ClientScraper) (*[]*event.Event, error)
+
+func ScrapeEvents() {
+	client := NewScraperClient()
+	events := []*event.Event{}
+	scrapers := []Scraper{
+		ScrapeONE,
+		ScrapeUFC,
+	}
+
+	for _, scraper := range scrapers {
+		e, err := scraper(&client)
+		if err != nil {
+			fmt.Println(err)
+		} else {
+			events = append(events, *e...)
+		}
+	}
+}

scrapers.go

@@ -0,0 +1,298 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"mmaschedule-go/event"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type ClientScraper interface {
+	Get(url string, options ...RequestOption) (*goquery.Selection, error)
+	AddHeader(key string, value string)
+}
+
+func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
+	events := []*event.Event{}
+	page, err := client.Get("https://www.onefc.com/events/")
+
+	if err != nil {
+		return nil, err
+	}
+
+	page.Find("#upcoming-events-section .is-event").Each(func(i int, s *goquery.Selection) {
+		_, exists := s.Find("a.smart-link").Attr("href")
+
+		if !exists {
+			return
+		}
+
+		url, exists := s.Find("a.is-image-zoom[href]").Attr("href")
+
+		if !exists {
+			return
+		}
+
+		e := event.Event{
+			Url:          url,
+			Organization: "ONE",
+			Slug: strings.Trim(
+				strings.Replace(url, "https://www.onefc.com/events/", "", 1),
+				"/",
+			),
+		}
+
+		err := ScrapeONEEvent(client, &e)
+
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		if e.Fights != "null" {
+			return
+		}
+
+		events = append(events, &e)
+	})
+
+	return &events, nil
+}
+
+type ONEEventLocation struct {
+	Name string `json:"name"`
+}
+
+type ONEEventData struct {
+	Name      string           `json:"name"`
+	StartDate string           `json:"startDate"`
+	Image     []string         `json:"image"`
+	Location  ONEEventLocation `json:"location"`
+}
+
+func ScrapeONEEvent(client ClientScraper, e *event.Event) error {
+	page, err := client.Get(e.Url)
+
+	if err != nil {
+		return err
+	}
+
+	data := ONEEventData{}
+	_ = json.Unmarshal([]byte(page.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data)
+
+	e.Name = data.Name
+	e.Location = data.Location.Name
+
+	if len(data.Image) > 2 {
+		e.Image = data.Image[2]
+	}
+
+	date, err := time.Parse(time.RFC3339, data.StartDate)
+	if err == nil {
+		e.Date = date.Unix()
+	}
+
+	var fights []*event.Fight
+	page.Find(".event-matchup").Each(func(i int, s *goquery.Selection) {
+		fights = append(fights, ScrapeONEFight(s))
+	})
+	e.MarshalFights(fights)
+
+	return nil
+}
+
+func ScrapeONEFight(s *goquery.Selection) *event.Fight {
+	fight := event.Fight{}
+
+	fight.Weight = TextContent(s, ".title")
+	fight.FighterA = ScrapeONEFighter(s, true)
+	fight.FighterB = ScrapeONEFighter(s, false)
+
+	return &fight
+}
+
+func ScrapeONEFighter(s *goquery.Selection, is_a bool) *event.Fighter {
+	fighter := event.Fighter{}
+	nth_child := "1"
+	face := "1"
+
+	if is_a {
+		nth_child = "3"
+		face = "2"
+	}
+
+	fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
+	fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
+
+	image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src")
+	fighter.Image = image
+
+	return &fighter
+}
+
+func ScrapeUFC(client ClientScraper) (*[]*event.Event, error) {
+	events := []*event.Event{}
+	page, err := client.Get("https://www.ufc.com/events")
+
+	if err != nil {
+		return nil, err
+	}
+
+	page.Find("#events-list-upcoming .c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
+		slug, exists := s.Attr("href")
+		if exists {
+			e := event.Event{
+				Url:          "https://www.ufc.com" + slug,
+				Organization: "UFC",
+				Slug:         strings.Replace(slug, "/event/", "", 1),
+			}
+			err := ScrapeUFCEvent(client, &e)
+			if err != nil {
+				log.Println(err)
+			} else if e.Fights != "null" {
+				events = append(events, &e)
+			}
+		}
+	})
+
+	return &events, nil
+}
+
+func ScrapeUFCEvent(client ClientScraper, e *event.Event) error {
+	page, err := client.Get(e.Url)
+
+	if err != nil {
+		return err
+	}
+
+	e.Name = strings.Join(
+		page.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
+			return CleanupWhitespace(s.Text())
+		}),
+		": ",
+	)
+	e.Location = TextContent(page, ".field--name-venue")
+
+	image, exists := page.Find(".c-hero__image img").First().Attr("src")
+	if exists {
+		e.Image = image
+	}
+
+	datestr, exists := page.Find(`
+
+        #early-prelims .c-event-fight-card-broadcaster__time,
+        #prelims-card .c-event-fight-card-broadcaster__time,
+        .c-hero__headline-suffix
+
+    `).Last().Attr("data-timestamp")
+
+	if exists {
+		date, err := strconv.Atoi(datestr)
+		if err == nil {
+			e.Date = int64(date)
+		}
+	}
+
+	var fights []*event.Fight
+	page.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) {
+		fights = append(fights, ScrapeUFCFight(s))
+	})
+	e.MarshalFights(fights)
+
+	return nil
+}
+
+func ScrapeUFCFight(s *goquery.Selection) *event.Fight {
+	fight := event.Fight{}
+
+	fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1)
+	fight.FighterA = ScrapeUFCFighter(s, "red")
+	fight.FighterB = ScrapeUFCFighter(s, "blue")
+
+	return &fight
+}
+
+func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter {
+	fighter := event.Fighter{}
+
+	fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color)
+	fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
+
+	image, exists := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
+	if exists {
+		fighter.Image = image
+	}
+
+	return &fighter
+}
+
+func SetTapologyCSRF(client ClientScraper) error {
+	selection, err := client.Get("https://www.tapology.com")
+	if err != nil {
+		return err
+	}
+	token, exists := selection.Find("meta[name=\"csrf-token\"]").First().Attr("content")
+	if !exists {
+		return fmt.Errorf("Could not parse CSRF token.")
+	}
+	client.AddHeader("X-CSRF-Token", token)
+	return nil
+}
+
+var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`)
+
+type TapologyResult struct {
+	Name string `json:"name"`
+	Url  string `json:"url"`
+}
+
+func GetTapologyLink(client ClientScraper, name string) (string, error) {
+	selection, err := client.Get("https://www.tapology.com/search/nav", func(r *http.Request) {
+		query := url.Values{}
+		query.Set("ajax", "true")
+		query.Set("model", "fighters")
+		query.Set("term", name)
+		r.URL.RawQuery = query.Encode()
+	})
+
+	if err != nil {
+		return "", err
+	}
+
+	results := []TapologyResult{}
+
+	selection.Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
+		url, _ := s.Attr("href")
+		name := CleanupWhitespace(nickname.ReplaceAllString(s.Text(), ""))
+		results = append(results, TapologyResult{Name: name, Url: url})
+	})
+
+	if len(results) == 0 {
+		return "", nil
+	}
+
+	slices.SortStableFunc(results, func(a, b TapologyResult) int {
+		score_a := HammingScore(name, a.Name)
+		score_b := HammingScore(name, b.Name)
+
+		if score_a > score_b {
+			return -1
+		} else if score_a < score_b {
+			return 1
+		} else {
+			return 0
+		}
+	})
+
+	return results[0].Url, nil
+}

scrapers_test.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+//go:embed testdata/ufc-event-list.html
+var UFCEventList []byte
+
+//go:embed testdata/ufc-312.html
+var UFC312 []byte
+
+//go:embed testdata/ufc-fn-feb-15.html
+var UFCFNFeb15 []byte
+
+//go:embed testdata/ufc-fn-feb-22.html
+var UFCFNFeb22 []byte
+
+//go:embed testdata/ufc-fn-mar-1.html
+var UFCFNMar1 []byte
+
+//go:embed testdata/ufc-313.html
+var UFC313 []byte
+
+//go:embed testdata/ufc-fn-mar-15.html
+var UFCFNMar15 []byte
+
+//go:embed testdata/ufc-fn-mar-22.html
+var UFCFNMar22 []byte
+
+//go:embed testdata/ufc-fn-mar-29.html
+var UFCFNMar29 []byte
+
+//go:embed testdata/one-event-list.html
+var ONEEventList []byte
+
+//go:embed testdata/one-friday-fights-97.html
+var ONEFridayFights97 []byte
+
+//go:embed testdata/one-171.html
+var ONE171 []byte
+
+//go:embed testdata/one-fn-29.html
+var ONEFN29 []byte
+
+//go:embed testdata/one-172.html
+var ONE172 []byte
+
+var HTMLContent map[string][]byte = map[string][]byte{
+	"https://www.ufc.com/events":                                 UFCEventList,
+	"https://www.ufc.com/event/ufc-312":                          UFC312,
+	"https://www.ufc.com/event/ufc-fight-night-february-15-2025": UFCFNFeb15,
+	"https://www.ufc.com/event/ufc-fight-night-february-22-2025": UFCFNFeb22,
+	"https://www.ufc.com/event/ufc-fight-night-march-01-2025":    UFCFNMar1,
+	"https://www.ufc.com/event/ufc-313":                          UFC313,
+	"https://www.ufc.com/event/ufc-fight-night-march-15-2025":    UFCFNMar15,
+	"https://www.ufc.com/event/ufc-fight-night-march-22-2025":    UFCFNMar22,
+	"https://www.ufc.com/event/ufc-fight-night-march-29-2025":    UFCFNMar29,
+	"https://www.onefc.com/events/":                              ONEEventList,
+	"https://www.onefc.com/events/one-friday-fights-97/":         ONEFridayFights97,
+	"https://www.onefc.com/events/one171/":                       ONE171,
+	"https://www.onefc.com/events/onefightnight29/":              ONEFN29,
+	"https://www.onefc.com/events/one172/":                       ONE172,
+}
+
+type DummyClient struct{}
+
+func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) {
+	htmlstring, exists := HTMLContent[url]
+	if !exists {
+		return nil, fmt.Errorf("Invalid URL: %s", url)
+	}
+	document, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlstring))
+	if err != nil {
+		return nil, err
+	}
+	return document.Find("html").First(), nil
+}
+
+func (c *DummyClient) AddHeader(key string, value string) {}
+
+func TestUFC(t *testing.T) {
+	client := DummyClient{}
+	events, err := ScrapeUFC(&client)
+	if err != nil {
+		t.Errorf("Scraping UFC failed: %s", err)
+	}
+	if len(*events) == 0 {
+		t.Fatal("Scraping UFC failed: Events are empty")
+	}
+	if len(*events) < 7 {
+		t.Fatal("Scraping UFC failed: Did not get all events")
+	}
+	//PrintJson(events)
+}
+
+func TestONE(t *testing.T) {
+	client := DummyClient{}
+	events, err := ScrapeONE(&client)
+	if err != nil {
+		t.Errorf("Scraping ONE failed: %s", err)
+	}
+	PrintJson(events)
+}
+
+func TestJSONString(t *testing.T) {
+	type User struct {
+		Name string
+	}
+
+	type Thing struct {
+		Users []User `json:"user,string"`
+		Num   int    `json:"num,string"`
+	}
+
+	u := Thing{
+		Users: []User{
+			{Name: "Bob"},
+			{Name: "John"},
+		},
+	}
+
+	PrintJson(u)
+}

sql/query.sql

@@ -4,6 +4,16 @@ SELECT
 FROM
   event
 WHERE
-  url = ?
+  slug = ?
+LIMIT
+  1;
+
+-- name: GetTapology :one
+SELECT
+  *
+FROM
+  tapology
+WHERE
+  name = ?
 LIMIT
   1;