guy176251/mmaschedule
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tapology now uses hamming distance to get approximately correct link

Browse Source
This commit is contained in:
long
2024-07-03 15:28:13 -05:00
parent ad7b487954
commit 02f27034d5
7 changed files with 87 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1
View File
@@ -4,6 +4,7 @@ go 1.22.4
require (
github.com/PuerkitoBio/goquery v1.9.2 // indirect
github.com/adrg/strutil v0.3.1 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/antchfx/htmlquery v1.3.1 // indirect
github.com/antchfx/xmlquery v1.4.0 // indirect
go.sum
+11
View File
@@ -1,5 +1,7 @@
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4=
github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/antchfx/htmlquery v1.3.1 h1:wm0LxjLMsZhRHfQKKZscDf2COyH4vDYA3wyH+qZ+Ylc=
@@ -9,6 +11,7 @@ github.com/antchfx/xmlquery v1.4.0/go.mod h1:Ax2aeaeDjfIw3CwXKDQ0GkwZ6QlxoChlIBP
github.com/antchfx/xpath v1.3.0 h1:nTMlzGAK3IJ0bPpME2urTuFL76o4A96iYvoKFHRXJgc=
github.com/antchfx/xpath v1.3.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
@@ -25,7 +28,12 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
@@ -74,3 +82,6 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
scraper/helpers.go
+10 -2
View File
@@ -8,15 +8,17 @@ import (
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/adrg/strutil"
"github.com/adrg/strutil/metrics"
)
func text_content(doc *goquery.Selection, selector string) string {
return cleanup_string(doc.Find(selector).First().Text())
return cleanup_whitespace(doc.Find(selector).First().Text())
}
var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
func cleanup_string(s string) string {
func cleanup_whitespace(s string) string {
return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
}
@@ -33,3 +35,9 @@ func print_json(v any) {
log.Println(string(out))
}
}
var hamming *metrics.Hamming = metrics.NewHamming()
func hamming_score(a, b string) float64 {
return strutil.Similarity(a, b, hamming)
}
scraper/scraper.go
+14 -1
View File
@@ -31,5 +31,18 @@ const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36
var scrapers []EventScraper = []EventScraper{
scrape_ufc,
scrape_one,
scrape_one,
}
func ScrapeEvents(callback EventCallback) {
get_tapology := tapology_getter()
for _, scraper := range scrapers {
scraper(func(e *Event) {
for _, fight := range e.Fights {
fight.FighterA.Link = get_tapology(fight.FighterA.Name)
}
callback(e)
})
}
}
scraper/tapology.go
+37 -14
View File
@@ -3,6 +3,8 @@ package scraper
import (
"log"
"net/url"
"regexp"
"slices"
"time"
"github.com/PuerkitoBio/goquery"
@@ -16,10 +18,12 @@ type TapologyResult struct {
Url string `json:"url"`
}
func tapology_getter() func(n string) []TapologyResult {
type TapologyGetter func(n string) string
func tapology_getter() TapologyGetter {
csrf_token := ""
name := ""
results := []TapologyResult{}
result := ""
index := tapology_collector()
index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) {
@@ -27,6 +31,10 @@ func tapology_getter() func(n string) []TapologyResult {
time.Sleep(5 * time.Second)
})
if err := index.Visit(tapology_url); err != nil {
log.Println(err)
}
search := tapology_collector()
search.AllowURLRevisit = true
search.OnRequest(func(r *colly.Request) {
@@ -41,39 +49,54 @@ func tapology_getter() func(n string) []TapologyResult {
log.Printf("Making tapology request to %s", r.URL)
})
search.OnResponse(func(r *colly.Response) {
results = parse_tapology_results(r.Body)
result = parse_tapology_results(name, r.Body)
time.Sleep(5 * time.Second)
})
if err := index.Visit(tapology_url); err != nil {
log.Println(err)
}
return func(n string) []TapologyResult {
return func(n string) string {
if csrf_token == "" {
return nil
return ""
}
name = n
if err := search.Visit(tapology_url + "/search/nav"); err != nil {
log.Println(err)
return nil
return ""
}
return results
return result
}
}
func parse_tapology_results(b []byte) []TapologyResult {
var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`)
func parse_tapology_results(name string, b []byte) string {
results := []TapologyResult{}
document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
url, _ := s.Attr("href")
name := s.Text()
name := cleanup_whitespace(nickname.ReplaceAllString(s.Text(), ""))
results = append(results, TapologyResult{Name: name, Url: url})
})
return results
slices.SortStableFunc(results, func(a, b TapologyResult) int {
score_a := hamming_score(name, a.Name)
score_b := hamming_score(name, b.Name)
if score_a > score_b {
return -1
} else if score_a < score_b {
return 1
} else {
return 0
}
})
if len(results) < 1 {
return ""
}
return tapology_url + results[0].Url
}
func tapology_collector() *colly.Collector {
scraper/tapology_test.go
+13 -5
View File
@@ -10,11 +10,19 @@ var search_result []byte
func testTapologyGetter(t *testing.T) {
get_tapology_for := tapology_getter()
print_json(get_tapology_for("Dustin Poirier"))
print_json(get_tapology_for("Jon Jones"))
print_json(get_tapology_for("Conor Mcgregor"))
names := []string{
"Dustin Poirier",
"Jon Jones",
"Conor Mcgregor",
"Justin Gaethje",
"Benoit Saint Denis",
}
for _, name := range names {
print_json(get_tapology_for(name))
}
}
func TestTapologyParseFragment(t *testing.T) {
print_json(parse_tapology_results(search_result))
func TestParseTapologyResults(t *testing.T) {
print_json(parse_tapology_results("Jon Jones", search_result))
}
scraper/ufc.go
+1 -1
View File
@@ -53,7 +53,7 @@ func parse_ufc_event(url string, b []byte) *Event {
doc := document_from_bytes(b)
parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
return cleanup_string(s.Text())
return cleanup_whitespace(s.Text())
})
event.Name = strings.Join(parts, ": ")
event.Location = text_content(doc, ".field--name-venue")