moved go files to src
This commit is contained in:
@@ -0,0 +1,277 @@
|
||||
package main
|
||||
|
||||
import "mmaschedule-go/event"
|
||||
import "strconv"
|
||||
|
||||
templ TemplEventPage(e event.Event, upcoming []event.ListUpcomingEventsRow) {
|
||||
<html data-theme="night">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width"/>
|
||||
<link rel="stylesheet" type="text/css" href="/static/styles.css"/>
|
||||
<title>MMA Schedule</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="h-screen overflow-scroll bg-neutral-focus text-neutral-content fill-neutral-content">
|
||||
<div class="drawer drawer-mobile">
|
||||
<input id="my-drawer-3" type="checkbox" class="drawer-toggle"/>
|
||||
<div class="drawer-content flex flex-col">
|
||||
<div class="w-full navbar bg-neutral sticky top-0 shadow-2xl z-50 text-neutral-content">
|
||||
<div class="flex-none lg:hidden">
|
||||
<label for="my-drawer-3" class="btn btn-square btn-ghost text-xl">
|
||||
<svg class="inline w-6 h-6" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M4 6h16v2H4zm0 5h16v2H4zm0 5h16v2H4z"></path>
|
||||
</svg>
|
||||
</label>
|
||||
</div>
|
||||
<a href="/" class="btn btn-ghost normal-case text-xl">MMA Schedule</a>
|
||||
</div>
|
||||
<div class="hidden lg:block">
|
||||
<div
|
||||
class="h-[450px] 2xl:h-[550px] bg-cover bg-[center_top_-14rem] text-white fill-white"
|
||||
style={ "background-image: url(" + e.Image + ");" }
|
||||
>
|
||||
<div class="h-full w-full bg-black/50 grid px-32">
|
||||
<div class="place-self-center flex flex-col">
|
||||
<p class="text-6xl font-light tracking-tighter border-b pb-4">
|
||||
{ e.Name }
|
||||
</p>
|
||||
<div class="pt-3 text-lg font-extrabold tracking-tighter grid justify-items-left gap-1 w-fit place-self-center">
|
||||
<p>
|
||||
@TemplIconCalendar("mr-2")
|
||||
@TemplTimestamp(e.Date)
|
||||
</p>
|
||||
if len(e.Location) > 0 {
|
||||
<p>
|
||||
@TemplIconLocation("mr-2")
|
||||
{ e.Location }
|
||||
</p>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="py-4 px-2 mx-auto container">
|
||||
<div class="lg:hidden">
|
||||
<div class="hero bg-neutral mb-4 rounded">
|
||||
<div class="hero-content flex-col justify-between">
|
||||
<div class="flex-1 grow">
|
||||
<img
|
||||
src={ e.Image }
|
||||
class="rounded-lg shadow-2xl object-cover aspect-[16/9]"
|
||||
/>
|
||||
</div>
|
||||
<div class="flex-1 grow py-2 w-full flex flex-col items-center">
|
||||
<p class="text-3xl font-bold border-b-4 w-fit pb-3 border-red-500">
|
||||
{ e.Name }
|
||||
</p>
|
||||
<div>
|
||||
<div class="pt-3 flex align-top content-center">
|
||||
@TemplIconCalendar("mr-2")
|
||||
@TemplTimestamp(e.Date)
|
||||
</div>
|
||||
if len(e.Location) > 0 {
|
||||
<p class="pt-2">
|
||||
@TemplIconLocation("mr-1")
|
||||
{ e.Location }
|
||||
</p>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col gap-4">
|
||||
for _, f := range e.UnmarshalFights() {
|
||||
@TemplFightRow(f)
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="drawer-side">
|
||||
<label for="my-drawer-3" class="drawer-overlay"></label>
|
||||
<ul class="menu p-4 w-80 bg-neutral text-neutral-content">
|
||||
<li>
|
||||
<a>
|
||||
<p class="text-2xl mb-2 pb-2 border-b-2 border-accent rounded-b-none">
|
||||
Upcoming Events
|
||||
</p>
|
||||
</a>
|
||||
</li>
|
||||
for _, u := range upcoming {
|
||||
<li class="menu-title">
|
||||
@TemplTimestamp(u.Date)
|
||||
</li>
|
||||
<li>
|
||||
<a class="pt-0" href={ templ.URL("/events/" + u.Slug + "/") }>
|
||||
{ u.Name }
|
||||
</a>
|
||||
</li>
|
||||
}
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@TemplJSTimestamp()
|
||||
</body>
|
||||
</html>
|
||||
}
|
||||
|
||||
templ TemplFightRow(f *event.Fight) {
|
||||
<div class="w-full px-4 pb-4 pt-1 rounded bg-neutral text-neutral-content">
|
||||
<div class="w-full flex justify-center">
|
||||
<p
|
||||
class="pb-1 mb-4 text-lg font-extrabold border-b border-neutral-content w-fit"
|
||||
>
|
||||
{ f.Weight }
|
||||
</p>
|
||||
</div>
|
||||
<div class="grid grid-cols-5">
|
||||
@TemplFighterCol(f.FighterA, false)
|
||||
<div class="px-2 flex">
|
||||
<div class="self-center w-full">
|
||||
<p class="text-2xl font-extrabold text-center">VS</p>
|
||||
</div>
|
||||
</div>
|
||||
@TemplFighterCol(f.FighterB, true)
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
|
||||
templ TemplFighterCol(f *event.Fighter, right bool) {
|
||||
<div class="col-span-2">
|
||||
<a
|
||||
if len(f.Link) > 0 {
|
||||
target="_blank"
|
||||
href={ templ.URL(f.Link) }
|
||||
class="no-underline hover:underline"
|
||||
}
|
||||
>
|
||||
<div
|
||||
if right {
|
||||
class="flex flex-col lg:flex-row lg:justify-end"
|
||||
} else {
|
||||
class="flex flex-col lg:flex-row"
|
||||
}
|
||||
>
|
||||
<img
|
||||
src={ f.Image }
|
||||
loading="lazy"
|
||||
alt=""
|
||||
if right {
|
||||
class="h-32 w-32 object-cover object-top rounded lg:order-2"
|
||||
} else {
|
||||
class="h-32 w-32 object-cover object-top rounded"
|
||||
}
|
||||
/>
|
||||
<div
|
||||
if right {
|
||||
class="
|
||||
lg:mx-5
|
||||
mt-3 lg:mt-0
|
||||
pt-1 lg:pt-0
|
||||
border-t-8 lg:border-t-0
|
||||
|
||||
border-blue-500
|
||||
lg:border-r-8
|
||||
lg:pr-5
|
||||
lg:order-1
|
||||
text-right
|
||||
"
|
||||
} else {
|
||||
class="
|
||||
lg:mx-5
|
||||
mt-3 lg:mt-0
|
||||
pt-1 lg:pt-0
|
||||
border-t-8 lg:border-t-0
|
||||
|
||||
border-red-500
|
||||
lg:border-l-8
|
||||
lg:pl-5
|
||||
"
|
||||
}
|
||||
>
|
||||
<p class="text-xl font-semibold mb-1 lg:mb-4">
|
||||
{ f.Name }
|
||||
</p>
|
||||
if len(f.Country) > 0 {
|
||||
<p class="text-sm lg:text-base">
|
||||
{ f.Country }
|
||||
</p>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
}
|
||||
|
||||
templ TemplTimestamp(timestamp int64) {
|
||||
<span data-timestamp={ strconv.FormatInt(timestamp, 10) }>
|
||||
//{{ date|date:'D M j, g:i A T' }}
|
||||
</span>
|
||||
}
|
||||
|
||||
templ TemplIconLocation(class string) {
|
||||
<svg class={ "w-6 h-6 inline " + class } xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path fill-rule="evenodd" d="M11.54 22.351l.07.04.028.016a.76.76 0 00.723 0l.028-.015.071-.041a16.975 16.975 0 001.144-.742 19.58 19.58 0 002.683-2.282c1.944-1.99 3.963-4.98 3.963-8.827a8.25 8.25 0 00-16.5 0c0 3.846 2.02 6.837 3.963 8.827a19.58 19.58 0 002.682 2.282 16.975 16.975 0 001.145.742zM12 13.5a3 3 0 100-6 3 3 0 000 6z" clip-rule="evenodd"></path>
|
||||
</svg>
|
||||
}
|
||||
|
||||
templ TemplIconCalendar(class string) {
|
||||
<svg class={ "inline w-6 h-6 " + class } xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path
|
||||
d="M5 22h14c1.103 0 2-.897 2-2V6c0-1.103-.897-2-2-2h-2V2h-2v2H9V2H7v2H5c-1.103 0-2 .897-2 2v14c0 1.103.897 2 2 2zM5 7h14v2H5V7z"
|
||||
></path>
|
||||
</svg>
|
||||
}
|
||||
|
||||
templ TemplJSTimestamp() {
|
||||
<script>
|
||||
(() => {
|
||||
const DATA_TIMESTAMP = "data-timestamp";
|
||||
|
||||
const MONTHS = [
|
||||
"Jan",
|
||||
"Feb",
|
||||
"Mar",
|
||||
"Apr",
|
||||
"May",
|
||||
"Jun",
|
||||
"Jul",
|
||||
"Aug",
|
||||
"Sep",
|
||||
"Oct",
|
||||
"Nov",
|
||||
"Dec",
|
||||
];
|
||||
|
||||
const DAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
|
||||
for (const e of document.querySelectorAll(`[${DATA_TIMESTAMP}]`)) {
|
||||
const timestamp = e.getAttribute(DATA_TIMESTAMP);
|
||||
if (timestamp) {
|
||||
e.innerHTML = dateStr(+timestamp);
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {number} timestamp */
|
||||
function dateStr(timestamp) {
|
||||
let d = new Date(timestamp * 1000);
|
||||
let day = d.getDate(),
|
||||
month = MONTHS[d.getMonth()],
|
||||
hour = d.getHours() % 12 || 12,
|
||||
minute = `${d.getMinutes()}`.padStart(2, "0"),
|
||||
pm = d.getHours() >= 12 ? "PM" : "AM",
|
||||
dow = DAYS[d.getDay()];
|
||||
|
||||
let timezone = d
|
||||
.toLocaleString("en", { timeZoneName: "short" })
|
||||
.split(" ")
|
||||
.pop();
|
||||
|
||||
return `${dow} ${month} ${day}, ${hour}:${minute} ${pm} ${timezone}`;
|
||||
}
|
||||
})();
|
||||
</script>
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.26.0
|
||||
|
||||
package event
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
type DBTX interface {
|
||||
ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
|
||||
PrepareContext(context.Context, string) (*sql.Stmt, error)
|
||||
QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
|
||||
QueryRowContext(context.Context, string, ...interface{}) *sql.Row
|
||||
}
|
||||
|
||||
func New(db DBTX) *Queries {
|
||||
return &Queries{db: db}
|
||||
}
|
||||
|
||||
type Queries struct {
|
||||
db DBTX
|
||||
}
|
||||
|
||||
func (q *Queries) WithTx(tx *sql.Tx) *Queries {
|
||||
return &Queries{
|
||||
db: tx,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package event
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
type Fight struct {
|
||||
Weight string `json:"weight"`
|
||||
FighterA *Fighter `json:"fighter_a"`
|
||||
FighterB *Fighter `json:"fighter_b"`
|
||||
}
|
||||
|
||||
type Fighter struct {
|
||||
Name string `json:"name"`
|
||||
Image string `json:"image"`
|
||||
Country string `json:"country"`
|
||||
Link string `json:"link"`
|
||||
}
|
||||
|
||||
func (e *Event) UnmarshalFights() []*Fight {
|
||||
var fights []*Fight
|
||||
_ = json.Unmarshal([]byte(e.Fights), &fights)
|
||||
return fights
|
||||
}
|
||||
|
||||
func (e *Event) MarshalFights(fights []*Fight) {
|
||||
buf, _ := json.Marshal(&fights)
|
||||
e.Fights = string(buf)
|
||||
}
|
||||
|
||||
const upsertEvents = `-- name: UpsertEvents :exec
|
||||
INSERT INTO
|
||||
event (
|
||||
url,
|
||||
slug,
|
||||
name,
|
||||
location,
|
||||
organization,
|
||||
image,
|
||||
date,
|
||||
fights,
|
||||
history
|
||||
)
|
||||
SELECT
|
||||
json_extract (e.value, '$.url'),
|
||||
json_extract (e.value, '$.slug'),
|
||||
json_extract (e.value, '$.name'),
|
||||
json_extract (e.value, '$.location'),
|
||||
json_extract (e.value, '$.organization'),
|
||||
json_extract (e.value, '$.image'),
|
||||
json_extract (e.value, '$.date'),
|
||||
json_extract (e.value, '$.fights'),
|
||||
json_extract (e.value, '$.history')
|
||||
FROM
|
||||
json_each (?) AS e
|
||||
WHERE true
|
||||
ON CONFLICT (url) DO UPDATE
|
||||
SET
|
||||
slug = excluded.slug,
|
||||
name = excluded.name,
|
||||
location = excluded.location,
|
||||
organization = excluded.organization,
|
||||
image = excluded.image,
|
||||
date = excluded.date,
|
||||
fights = excluded.fights,
|
||||
history = excluded.history
|
||||
`
|
||||
|
||||
func (q *Queries) UpsertEvents(ctx context.Context, e []*Event) error {
|
||||
b, err := json.Marshal(&e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = q.db.ExecContext(ctx, upsertEvents, string(b))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.26.0
|
||||
|
||||
package event
|
||||
|
||||
type Event struct {
|
||||
Url string `json:"url"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Location string `json:"location"`
|
||||
Organization string `json:"organization"`
|
||||
Image string `json:"image"`
|
||||
Date int64 `json:"date"`
|
||||
Fights string `json:"fights"`
|
||||
History string `json:"history"`
|
||||
}
|
||||
|
||||
type Tapology struct {
|
||||
Name string `json:"name"`
|
||||
Url string `json:"url"`
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.26.0
|
||||
// source: query.sql
|
||||
|
||||
package event
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
const createTapology = `-- name: CreateTapology :exec
|
||||
INSERT INTO
|
||||
tapology (name, url)
|
||||
VALUES
|
||||
(?, ?)
|
||||
`
|
||||
|
||||
type CreateTapologyParams struct {
|
||||
Name string `json:"name"`
|
||||
Url string `json:"url"`
|
||||
}
|
||||
|
||||
func (q *Queries) CreateTapology(ctx context.Context, arg CreateTapologyParams) error {
|
||||
_, err := q.db.ExecContext(ctx, createTapology, arg.Name, arg.Url)
|
||||
return err
|
||||
}
|
||||
|
||||
const getEvent = `-- name: GetEvent :one
|
||||
SELECT
|
||||
url, slug, name, location, organization, image, date, fights, history
|
||||
FROM
|
||||
event
|
||||
WHERE
|
||||
slug = ?
|
||||
LIMIT
|
||||
1
|
||||
`
|
||||
|
||||
func (q *Queries) GetEvent(ctx context.Context, slug string) (Event, error) {
|
||||
row := q.db.QueryRowContext(ctx, getEvent, slug)
|
||||
var i Event
|
||||
err := row.Scan(
|
||||
&i.Url,
|
||||
&i.Slug,
|
||||
&i.Name,
|
||||
&i.Location,
|
||||
&i.Organization,
|
||||
&i.Image,
|
||||
&i.Date,
|
||||
&i.Fights,
|
||||
&i.History,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getTapology = `-- name: GetTapology :one
|
||||
SELECT
|
||||
name, url
|
||||
FROM
|
||||
tapology
|
||||
WHERE
|
||||
name = ?
|
||||
LIMIT
|
||||
1
|
||||
`
|
||||
|
||||
func (q *Queries) GetTapology(ctx context.Context, name string) (Tapology, error) {
|
||||
row := q.db.QueryRowContext(ctx, getTapology, name)
|
||||
var i Tapology
|
||||
err := row.Scan(&i.Name, &i.Url)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getUpcomingEvent = `-- name: GetUpcomingEvent :one
|
||||
SELECT
|
||||
slug
|
||||
FROM
|
||||
event
|
||||
WHERE
|
||||
date >= ?
|
||||
ORDER BY
|
||||
date ASC
|
||||
LIMIT
|
||||
1
|
||||
`
|
||||
|
||||
func (q *Queries) GetUpcomingEvent(ctx context.Context, date int64) (string, error) {
|
||||
row := q.db.QueryRowContext(ctx, getUpcomingEvent, date)
|
||||
var slug string
|
||||
err := row.Scan(&slug)
|
||||
return slug, err
|
||||
}
|
||||
|
||||
const listEvents = `-- name: ListEvents :many
|
||||
SELECT
|
||||
url, slug, name, location, organization, image, date, fights, history
|
||||
FROM
|
||||
event
|
||||
`
|
||||
|
||||
func (q *Queries) ListEvents(ctx context.Context) ([]Event, error) {
|
||||
rows, err := q.db.QueryContext(ctx, listEvents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Event
|
||||
for rows.Next() {
|
||||
var i Event
|
||||
if err := rows.Scan(
|
||||
&i.Url,
|
||||
&i.Slug,
|
||||
&i.Name,
|
||||
&i.Location,
|
||||
&i.Organization,
|
||||
&i.Image,
|
||||
&i.Date,
|
||||
&i.Fights,
|
||||
&i.History,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listUpcomingEvents = `-- name: ListUpcomingEvents :many
|
||||
SELECT
|
||||
name,
|
||||
slug,
|
||||
date
|
||||
FROM
|
||||
event
|
||||
WHERE
|
||||
date >= ?
|
||||
ORDER BY
|
||||
date ASC
|
||||
`
|
||||
|
||||
type ListUpcomingEventsRow struct {
|
||||
Name string `json:"name"`
|
||||
Slug string `json:"slug"`
|
||||
Date int64 `json:"date"`
|
||||
}
|
||||
|
||||
func (q *Queries) ListUpcomingEvents(ctx context.Context, date int64) ([]ListUpcomingEventsRow, error) {
|
||||
rows, err := q.db.QueryContext(ctx, listUpcomingEvents, date)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListUpcomingEventsRow
|
||||
for rows.Next() {
|
||||
var i ListUpcomingEventsRow
|
||||
if err := rows.Scan(&i.Name, &i.Slug, &i.Date); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
module mmaschedule-go
|
||||
|
||||
go 1.23
|
||||
|
||||
toolchain go1.23.5
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.10.1
|
||||
github.com/adrg/strutil v0.3.1
|
||||
github.com/gocolly/colly v1.2.0
|
||||
github.com/mattn/go-sqlite3 v1.14.23
|
||||
github.com/pressly/goose/v3 v3.22.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/a-h/templ v0.3.833 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/antchfx/htmlquery v1.3.1 // indirect
|
||||
github.com/antchfx/xmlquery v1.4.0 // indirect
|
||||
github.com/antchfx/xpath v1.3.0 // indirect
|
||||
github.com/gobwas/glob v0.2.3 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
|
||||
github.com/sethvargo/go-retry v0.3.0 // indirect
|
||||
github.com/temoto/robotstxt v1.1.2 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/net v0.33.0 // indirect
|
||||
golang.org/x/sync v0.10.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
google.golang.org/protobuf v1.33.0 // indirect
|
||||
)
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
|
||||
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
||||
github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU=
|
||||
github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY=
|
||||
github.com/a-h/templ v0.3.833 h1:L/KOk/0VvVTBegtE0fp2RJQiBm7/52Zxv5fqlEHiQUU=
|
||||
github.com/a-h/templ v0.3.833/go.mod h1:cAu4AiZhtJfBjMY0HASlyzvkrtjnHWPeEsyGK2YYmfk=
|
||||
github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4=
|
||||
github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/antchfx/htmlquery v1.3.1 h1:wm0LxjLMsZhRHfQKKZscDf2COyH4vDYA3wyH+qZ+Ylc=
|
||||
github.com/antchfx/htmlquery v1.3.1/go.mod h1:PTj+f1V2zksPlwNt7uVvZPsxpKNa7mlVliCRxLX6Nx8=
|
||||
github.com/antchfx/xmlquery v1.4.0 h1:xg2HkfcRK2TeTbdb0m1jxCYnvsPaGY/oeZWTGqX/0hA=
|
||||
github.com/antchfx/xmlquery v1.4.0/go.mod h1:Ax2aeaeDjfIw3CwXKDQ0GkwZ6QlxoChlIBP+mGnDFjI=
|
||||
github.com/antchfx/xpath v1.3.0 h1:nTMlzGAK3IJ0bPpME2urTuFL76o4A96iYvoKFHRXJgc=
|
||||
github.com/antchfx/xpath v1.3.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
|
||||
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0=
|
||||
github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw=
|
||||
github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
|
||||
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
|
||||
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
|
||||
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
|
||||
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
|
||||
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
|
||||
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
|
||||
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
|
||||
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
|
||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
||||
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
|
||||
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
|
||||
modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s=
|
||||
modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA=
|
||||
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
|
||||
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/adrg/strutil"
|
||||
"github.com/adrg/strutil/metrics"
|
||||
)
|
||||
|
||||
type Values = url.Values
|
||||
|
||||
func TextContent(doc *goquery.Selection, selector string) string {
|
||||
return CleanupWhitespace(doc.Find(selector).First().Text())
|
||||
}
|
||||
|
||||
var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
|
||||
|
||||
func CleanupWhitespace(s string) string {
|
||||
return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
|
||||
}
|
||||
|
||||
func PrintJson(v any) {
|
||||
out, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
log.Println(string(out))
|
||||
}
|
||||
}
|
||||
|
||||
var hamming *metrics.Hamming = metrics.NewHamming()
|
||||
|
||||
func HammingScore(a, b string) float64 {
|
||||
return strutil.Similarity(a, b, hamming)
|
||||
}
|
||||
|
||||
type ScraperClient struct {
|
||||
headers map[string]string
|
||||
client http.Client
|
||||
}
|
||||
|
||||
type RequestOption func(*http.Request)
|
||||
|
||||
func (c *ScraperClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) {
|
||||
resp, err := c.GetResponse(url, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
selection, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return selection.Find("html").First(), nil
|
||||
}
|
||||
|
||||
func (c *ScraperClient) GetResponse(url string, options ...RequestOption) (*http.Response, error) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for key, value := range c.headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
for _, option := range options {
|
||||
if option != nil {
|
||||
option(req)
|
||||
}
|
||||
}
|
||||
return c.client.Do(req)
|
||||
}
|
||||
|
||||
func (c *ScraperClient) AddHeader(key string, value string) {
|
||||
c.headers[key] = value
|
||||
}
|
||||
func (c *ScraperClient) HasHeader(key string) bool {
|
||||
_, exists := c.headers[key]
|
||||
return exists
|
||||
}
|
||||
|
||||
func NewScraperClient() *ScraperClient {
|
||||
client := ScraperClient{
|
||||
headers: map[string]string{
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
},
|
||||
client: http.Client{},
|
||||
}
|
||||
return &client
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"embed"
|
||||
"fmt"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
"mmaschedule-go/event"
|
||||
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrations embed.FS
|
||||
|
||||
func main() {
|
||||
queries, err := InitDb("db.sqlite")
|
||||
if err != nil {
|
||||
fmt.Println("Error initializing database:", err)
|
||||
return
|
||||
}
|
||||
|
||||
err = RunServer("0.0.0.0:3001", queries)
|
||||
if err != nil {
|
||||
fmt.Println("Error starting web server:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func InitDb(name string) (*event.Queries, error) {
|
||||
db, err := sql.Open("sqlite3", name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = db.Exec("PRAGMA journal_mode = WAL")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
goose.SetBaseFS(migrations)
|
||||
|
||||
if err := goose.SetDialect("sqlite3"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := goose.Up(db, "migrations"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
queries := event.New(db)
|
||||
|
||||
return queries, nil
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
--PRAGMA journal_mode = WAL;
|
||||
|
||||
CREATE TABLE event (
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
location TEXT NOT NULL,
|
||||
organization TEXT NOT NULL,
|
||||
image TEXT NOT NULL,
|
||||
date INTEGER NOT NULL,
|
||||
fights TEXT NOT NULL,
|
||||
history TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE tapology (
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
url TEXT NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"mmaschedule-go/event"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Scraper func(ClientScraper) (*[]*event.Event, error)
|
||||
|
||||
func ScrapeEvents(q *event.Queries, client ClientScraper) {
|
||||
events := []*event.Event{}
|
||||
scrapers := []Scraper{
|
||||
ScrapeONE,
|
||||
ScrapeUFC,
|
||||
}
|
||||
|
||||
for _, scraper := range scrapers {
|
||||
e, err := scraper(client)
|
||||
if err != nil {
|
||||
fmt.Println("Error scraping events:", err)
|
||||
} else {
|
||||
events = append(events, *e...)
|
||||
}
|
||||
}
|
||||
|
||||
UpdateTapology(q, client, &events)
|
||||
|
||||
if len(events) > 0 {
|
||||
err := q.UpsertEvents(context.Background(), events)
|
||||
if err != nil {
|
||||
fmt.Println("Error updating events in database:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func UpdateAllTapology(q *event.Queries, client ClientScraper) error {
|
||||
events_, err := q.ListEvents(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
events := []*event.Event{}
|
||||
for _, e := range events_ {
|
||||
events = append(events, &e)
|
||||
}
|
||||
|
||||
UpdateTapology(q, client, &events)
|
||||
|
||||
err = q.UpsertEvents(context.Background(), events)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Event) {
|
||||
err := SetTapologyCSRF(client)
|
||||
if err != nil {
|
||||
fmt.Println("Error settings tapology CSRF:", err)
|
||||
}
|
||||
|
||||
for _, e := range *events {
|
||||
fights := e.UnmarshalFights()
|
||||
for _, f := range fights {
|
||||
fighters := []*event.Fighter{
|
||||
f.FighterA,
|
||||
f.FighterB,
|
||||
}
|
||||
for _, ff := range fighters {
|
||||
fmt.Println("Getting tapology link for", ff.Name)
|
||||
tapology, err := q.GetTapology(context.Background(), ff.Name)
|
||||
if err != nil {
|
||||
fmt.Println("Error getting tapology from database:", err)
|
||||
link, err := GetTapologyLink(client, ff.Name)
|
||||
if err != nil {
|
||||
fmt.Println("Error getting tapology from site:", err)
|
||||
} else if len(link) > 0 {
|
||||
err := q.CreateTapology(context.Background(), event.CreateTapologyParams{Name: ff.Name, Url: link})
|
||||
if err != nil {
|
||||
fmt.Println("Error creating tapology in database:", err)
|
||||
}
|
||||
ff.Link = link
|
||||
}
|
||||
time.Sleep(time.Duration(5_000_000_000))
|
||||
} else {
|
||||
ff.Link = tapology.Url
|
||||
}
|
||||
}
|
||||
}
|
||||
e.MarshalFights(fights)
|
||||
}
|
||||
}
|
||||
+305
@@ -0,0 +1,305 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"mmaschedule-go/event"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
type ClientScraper interface {
|
||||
Get(url string, options ...RequestOption) (*goquery.Selection, error)
|
||||
AddHeader(key string, value string)
|
||||
HasHeader(key string) bool
|
||||
}
|
||||
|
||||
func ScrapeONE(client ClientScraper) (*[]*event.Event, error) {
|
||||
events := []*event.Event{}
|
||||
page, err := client.Get("https://www.onefc.com/events/")
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
page.Find("#upcoming-events-section .is-event").Each(func(i int, s *goquery.Selection) {
|
||||
_, exists := s.Find("a.smart-link").Attr("href")
|
||||
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
url, exists := s.Find("a.is-image-zoom[href]").Attr("href")
|
||||
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
e := event.Event{
|
||||
Url: url,
|
||||
Organization: "ONE",
|
||||
Slug: strings.Trim(
|
||||
strings.Replace(url, "https://www.onefc.com/events/", "", 1),
|
||||
"/",
|
||||
),
|
||||
}
|
||||
|
||||
err := ScrapeONEEvent(client, &e)
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
if e.Fights == "null" {
|
||||
return
|
||||
}
|
||||
|
||||
events = append(events, &e)
|
||||
})
|
||||
|
||||
return &events, nil
|
||||
}
|
||||
|
||||
type ONEEventLocation struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type ONEEventData struct {
|
||||
Name string `json:"name"`
|
||||
StartDate string `json:"startDate"`
|
||||
Image []string `json:"image"`
|
||||
Location ONEEventLocation `json:"location"`
|
||||
}
|
||||
|
||||
func ScrapeONEEvent(client ClientScraper, e *event.Event) error {
|
||||
page, err := client.Get(e.Url)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data := ONEEventData{}
|
||||
_ = json.Unmarshal([]byte(page.Find("#site-main script[type=\"application/ld+json\"]").Text()), &data)
|
||||
|
||||
e.Name = data.Name
|
||||
e.Location = data.Location.Name
|
||||
|
||||
if len(data.Image) > 2 {
|
||||
e.Image = data.Image[2]
|
||||
}
|
||||
|
||||
date, err := time.Parse(time.RFC3339, data.StartDate)
|
||||
if err == nil {
|
||||
e.Date = date.Unix()
|
||||
}
|
||||
|
||||
var fights []*event.Fight
|
||||
page.Find(".event-matchup").Each(func(i int, s *goquery.Selection) {
|
||||
fights = append(fights, ScrapeONEFight(s))
|
||||
})
|
||||
e.MarshalFights(fights)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func ScrapeONEFight(s *goquery.Selection) *event.Fight {
|
||||
fight := event.Fight{}
|
||||
|
||||
fight.Weight = TextContent(s, ".title")
|
||||
fight.FighterA = ScrapeONEFighter(s, true)
|
||||
fight.FighterB = ScrapeONEFighter(s, false)
|
||||
|
||||
return &fight
|
||||
}
|
||||
|
||||
func ScrapeONEFighter(s *goquery.Selection, is_a bool) *event.Fighter {
|
||||
fighter := event.Fighter{}
|
||||
nth_child := "1"
|
||||
face := "1"
|
||||
|
||||
if is_a {
|
||||
nth_child = "3"
|
||||
face = "2"
|
||||
}
|
||||
|
||||
fighter.Name = TextContent(s, fmt.Sprintf("tr.vs :nth-child(%s) a", nth_child))
|
||||
fighter.Country = TextContent(s, fmt.Sprintf("tr.vs + tr :nth-child(%s)", nth_child))
|
||||
|
||||
image, _ := s.Find(fmt.Sprintf(".face%s img[src]", face)).Attr("src")
|
||||
fighter.Image = image
|
||||
|
||||
return &fighter
|
||||
}
|
||||
|
||||
func ScrapeUFC(client ClientScraper) (*[]*event.Event, error) {
|
||||
events := []*event.Event{}
|
||||
page, err := client.Get("https://www.ufc.com/events")
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
page.Find("#events-list-upcoming .c-card-event--result__headline a").Each(func(i int, s *goquery.Selection) {
|
||||
slug, exists := s.Attr("href")
|
||||
if exists {
|
||||
e := event.Event{
|
||||
Url: "https://www.ufc.com" + slug,
|
||||
Organization: "UFC",
|
||||
Slug: strings.Replace(slug, "/event/", "", 1),
|
||||
}
|
||||
err := ScrapeUFCEvent(client, &e)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else if e.Fights != "null" {
|
||||
events = append(events, &e)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return &events, nil
|
||||
}
|
||||
|
||||
func ScrapeUFCEvent(client ClientScraper, e *event.Event) error {
|
||||
page, err := client.Get(e.Url)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.Name = strings.Join(
|
||||
page.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
|
||||
return CleanupWhitespace(s.Text())
|
||||
}),
|
||||
": ",
|
||||
)
|
||||
e.Location = TextContent(page, ".field--name-venue")
|
||||
|
||||
image, exists := page.Find(".c-hero__image img").First().Attr("src")
|
||||
if exists {
|
||||
e.Image = image
|
||||
}
|
||||
|
||||
datestr, exists := page.Find(`
|
||||
|
||||
#early-prelims .c-event-fight-card-broadcaster__time,
|
||||
#prelims-card .c-event-fight-card-broadcaster__time,
|
||||
.c-hero__headline-suffix
|
||||
|
||||
`).Last().Attr("data-timestamp")
|
||||
|
||||
if exists {
|
||||
date, err := strconv.Atoi(datestr)
|
||||
if err == nil {
|
||||
e.Date = int64(date)
|
||||
}
|
||||
}
|
||||
|
||||
var fights []*event.Fight
|
||||
page.Find(".l-listing__item").Each(func(i int, s *goquery.Selection) {
|
||||
fights = append(fights, ScrapeUFCFight(s))
|
||||
})
|
||||
e.MarshalFights(fights)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func ScrapeUFCFight(s *goquery.Selection) *event.Fight {
|
||||
fight := event.Fight{}
|
||||
|
||||
fight.Weight = strings.Replace(TextContent(s, ".c-listing-fight__class-text"), " Bout", "", -1)
|
||||
fight.FighterA = ScrapeUFCFighter(s, "red")
|
||||
fight.FighterB = ScrapeUFCFighter(s, "blue")
|
||||
|
||||
return &fight
|
||||
}
|
||||
|
||||
func ScrapeUFCFighter(s *goquery.Selection, color string) *event.Fighter {
|
||||
fighter := event.Fighter{}
|
||||
|
||||
fighter.Name = TextContent(s, ".c-listing-fight__corner-name--"+color)
|
||||
fighter.Country = TextContent(s, ".c-listing-fight__country--"+color+" .c-listing-fight__country-text")
|
||||
|
||||
image, exists := s.Find(".c-listing-fight__corner-image--" + color + " img").First().Attr("src")
|
||||
if exists {
|
||||
fighter.Image = image
|
||||
}
|
||||
|
||||
return &fighter
|
||||
}
|
||||
|
||||
const TAPOLOGY_URL = "https://www.tapology.com"
|
||||
|
||||
func SetTapologyCSRF(client ClientScraper) error {
|
||||
selection, err := client.Get(TAPOLOGY_URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
token, exists := selection.Find("meta[name=\"csrf-token\"]").First().Attr("content")
|
||||
if !exists {
|
||||
return fmt.Errorf("Could not parse CSRF token.")
|
||||
}
|
||||
client.AddHeader("X-CSRF-Token", token)
|
||||
return nil
|
||||
}
|
||||
|
||||
var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`)
|
||||
|
||||
type TapologyResult struct {
|
||||
Name string `json:"name"`
|
||||
Url string `json:"url"`
|
||||
}
|
||||
|
||||
func GetTapologyLink(client ClientScraper, name string) (string, error) {
|
||||
if !client.HasHeader("X-CSRF-Token") {
|
||||
return "", fmt.Errorf("CSRF token not set.")
|
||||
}
|
||||
|
||||
selection, err := client.Get(TAPOLOGY_URL+"/search/nav", func(r *http.Request) {
|
||||
query := url.Values{}
|
||||
query.Set("ajax", "true")
|
||||
query.Set("model", "fighters")
|
||||
query.Set("term", name)
|
||||
r.URL.RawQuery = query.Encode()
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
results := []TapologyResult{}
|
||||
|
||||
selection.Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
|
||||
url, _ := s.Attr("href")
|
||||
name := CleanupWhitespace(nickname.ReplaceAllString(s.Text(), ""))
|
||||
results = append(results, TapologyResult{Name: name, Url: url})
|
||||
})
|
||||
|
||||
if len(results) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
slices.SortStableFunc(results, func(a, b TapologyResult) int {
|
||||
score_a := HammingScore(name, a.Name)
|
||||
score_b := HammingScore(name, b.Name)
|
||||
|
||||
if score_a > score_b {
|
||||
return -1
|
||||
} else if score_a < score_b {
|
||||
return 1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
return TAPOLOGY_URL + results[0].Url, nil
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"testing"
|
||||
"net/http"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
//go:embed testdata/ufc-event-list.html
|
||||
var UFCEventList []byte
|
||||
|
||||
//go:embed testdata/ufc-312.html
|
||||
var UFC312 []byte
|
||||
|
||||
//go:embed testdata/ufc-fn-feb-15.html
|
||||
var UFCFNFeb15 []byte
|
||||
|
||||
//go:embed testdata/ufc-fn-feb-22.html
|
||||
var UFCFNFeb22 []byte
|
||||
|
||||
//go:embed testdata/ufc-fn-mar-1.html
|
||||
var UFCFNMar1 []byte
|
||||
|
||||
//go:embed testdata/ufc-313.html
|
||||
var UFC313 []byte
|
||||
|
||||
//go:embed testdata/ufc-fn-mar-15.html
|
||||
var UFCFNMar15 []byte
|
||||
|
||||
//go:embed testdata/ufc-fn-mar-22.html
|
||||
var UFCFNMar22 []byte
|
||||
|
||||
//go:embed testdata/ufc-fn-mar-29.html
|
||||
var UFCFNMar29 []byte
|
||||
|
||||
//go:embed testdata/one-event-list.html
|
||||
var ONEEventList []byte
|
||||
|
||||
//go:embed testdata/one-friday-fights-97.html
|
||||
var ONEFridayFights97 []byte
|
||||
|
||||
//go:embed testdata/one-171.html
|
||||
var ONE171 []byte
|
||||
|
||||
//go:embed testdata/one-fn-29.html
|
||||
var ONEFN29 []byte
|
||||
|
||||
//go:embed testdata/one-172.html
|
||||
var ONE172 []byte
|
||||
|
||||
//go:embed testdata/tapology-index.html
|
||||
var TapologyIndex []byte
|
||||
|
||||
var HTMLContent map[string][]byte = map[string][]byte{
|
||||
"https://www.ufc.com/events": UFCEventList,
|
||||
"https://www.ufc.com/event/ufc-312": UFC312,
|
||||
"https://www.ufc.com/event/ufc-fight-night-february-15-2025": UFCFNFeb15,
|
||||
"https://www.ufc.com/event/ufc-fight-night-february-22-2025": UFCFNFeb22,
|
||||
"https://www.ufc.com/event/ufc-fight-night-march-01-2025": UFCFNMar1,
|
||||
"https://www.ufc.com/event/ufc-313": UFC313,
|
||||
"https://www.ufc.com/event/ufc-fight-night-march-15-2025": UFCFNMar15,
|
||||
"https://www.ufc.com/event/ufc-fight-night-march-22-2025": UFCFNMar22,
|
||||
"https://www.ufc.com/event/ufc-fight-night-march-29-2025": UFCFNMar29,
|
||||
"https://www.onefc.com/events/": ONEEventList,
|
||||
"https://www.onefc.com/events/one-friday-fights-97/": ONEFridayFights97,
|
||||
"https://www.onefc.com/events/one171/": ONE171,
|
||||
"https://www.onefc.com/events/onefightnight29/": ONEFN29,
|
||||
"https://www.onefc.com/events/one172/": ONE172,
|
||||
"https://www.tapology.com": TapologyIndex,
|
||||
}
|
||||
|
||||
type DummyClient struct {
|
||||
headers map[string]string
|
||||
}
|
||||
|
||||
func (c *DummyClient) Get(url string, options ...RequestOption) (*goquery.Selection, error) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
for _, option := range options {
|
||||
if option != nil {
|
||||
option(req)
|
||||
}
|
||||
}
|
||||
url = req.URL.String()
|
||||
htmlstring, exists := HTMLContent[url]
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("Invalid URL: %s", url)
|
||||
}
|
||||
document, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlstring))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return document.Find("html").First(), nil
|
||||
}
|
||||
|
||||
func (c *DummyClient) AddHeader(key string, value string) {
|
||||
c.headers[key] = value
|
||||
}
|
||||
func (c *DummyClient) HasHeader(key string) bool {
|
||||
_, exists := c.headers[key]
|
||||
return exists
|
||||
}
|
||||
|
||||
func NewDummyClient() *DummyClient {
|
||||
return &DummyClient{
|
||||
headers: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
func TestUFC(t *testing.T) {
|
||||
client := NewDummyClient()
|
||||
events, err := ScrapeUFC(client)
|
||||
if err != nil {
|
||||
t.Errorf("Scraping UFC failed: %s", err)
|
||||
}
|
||||
if len(*events) == 0 {
|
||||
t.Fatal("Scraping UFC failed: Events are empty")
|
||||
}
|
||||
if len(*events) < 7 {
|
||||
t.Fatal("Scraping UFC failed: Did not get all events")
|
||||
}
|
||||
//PrintJson(events)
|
||||
}
|
||||
|
||||
func TestONE(t *testing.T) {
|
||||
client := NewDummyClient()
|
||||
events, err := ScrapeONE(client)
|
||||
if err != nil {
|
||||
t.Errorf("Scraping ONE failed: %s", err)
|
||||
}
|
||||
if len(*events) == 0 {
|
||||
t.Fatal("Scraping ONE failed: Events are empty")
|
||||
}
|
||||
//PrintJson(events)
|
||||
}
|
||||
|
||||
func TestScrapeEvents(t *testing.T) {
|
||||
client := NewDummyClient()
|
||||
q, err := InitDb("test-db.sqlite")
|
||||
if err != nil {
|
||||
t.Error("Error initializing database: ", err)
|
||||
}
|
||||
ScrapeEvents(q, client)
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
"mmaschedule-go/event"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
//go:generate bun run build
|
||||
//go:generate templ generate
|
||||
|
||||
func RunServer(addr string, queries *event.Queries) error {
|
||||
mux := http.NewServeMux()
|
||||
state := ServerState{queries: queries}
|
||||
static, err := StaticHandler("/static/")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mux.HandleFunc("GET /", state.HandleFunc(RouteIndex))
|
||||
mux.HandleFunc("GET /events/{slug}/", state.HandleFunc(RouteEvent))
|
||||
mux.Handle("GET /static/", static)
|
||||
err = http.ListenAndServe(addr, mux)
|
||||
return err
|
||||
}
|
||||
|
||||
//go:embed static/*
|
||||
var staticFiles embed.FS
|
||||
|
||||
func StaticHandler(prefix string) (http.Handler, error) {
|
||||
static, err := fs.Sub(staticFiles, "static")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return http.StripPrefix(prefix, http.FileServer(http.FS(static))), nil
|
||||
}
|
||||
|
||||
func RouteIndex(w http.ResponseWriter, r *http.Request, q *event.Queries) {
|
||||
slug, err := q.GetUpcomingEvent(r.Context(), time.Now().Unix())
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
} else {
|
||||
http.Redirect(w, r, "/events/"+slug+"/", http.StatusSeeOther)
|
||||
}
|
||||
}
|
||||
|
||||
func RouteEvent(w http.ResponseWriter, r *http.Request, q *event.Queries) {
|
||||
slug := r.PathValue("slug")
|
||||
|
||||
if len(slug) == 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
event, err := q.GetEvent(r.Context(), slug)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
upcoming, err := q.ListUpcomingEvents(r.Context(), time.Now().Unix())
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
TemplEventPage(event, upcoming).Render(r.Context(), w)
|
||||
}
|
||||
|
||||
type StateHandler func(w http.ResponseWriter, r *http.Request, q *event.Queries)
|
||||
|
||||
type ServerState struct {
|
||||
queries *event.Queries
|
||||
}
|
||||
|
||||
func (state *ServerState) HandleFunc(handler StateHandler) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
handler(w, r, state.queries)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
-- name: GetEvent :one
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
event
|
||||
WHERE
|
||||
slug = ?
|
||||
LIMIT
|
||||
1;
|
||||
|
||||
-- name: ListEvents :many
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
event;
|
||||
|
||||
-- name: GetUpcomingEvent :one
|
||||
SELECT
|
||||
slug
|
||||
FROM
|
||||
event
|
||||
WHERE
|
||||
date >= ?
|
||||
ORDER BY
|
||||
date ASC
|
||||
LIMIT
|
||||
1;
|
||||
|
||||
-- name: ListUpcomingEvents :many
|
||||
SELECT
|
||||
name,
|
||||
slug,
|
||||
date
|
||||
FROM
|
||||
event
|
||||
WHERE
|
||||
date >= ?
|
||||
ORDER BY
|
||||
date ASC;
|
||||
|
||||
-- name: GetTapology :one
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
tapology
|
||||
WHERE
|
||||
name = ?
|
||||
LIMIT
|
||||
1;
|
||||
|
||||
-- name: CreateTapology :exec
|
||||
INSERT INTO
|
||||
tapology (name, url)
|
||||
VALUES
|
||||
(?, ?);
|
||||
Reference in New Issue
Block a user