From 02f27034d5aae3d27314d155d783c56019d723a8 Mon Sep 17 00:00:00 2001 From: long Date: Wed, 3 Jul 2024 15:28:13 -0500 Subject: [PATCH] tapology now uses hamming distance to get approximately correct link --- go.mod | 1 + go.sum | 11 +++++++++ scraper/helpers.go | 12 ++++++++-- scraper/scraper.go | 15 +++++++++++- scraper/tapology.go | 51 +++++++++++++++++++++++++++++----------- scraper/tapology_test.go | 18 ++++++++++---- scraper/ufc.go | 2 +- 7 files changed, 87 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index ed08928..df3a875 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.22.4 require ( github.com/PuerkitoBio/goquery v1.9.2 // indirect + github.com/adrg/strutil v0.3.1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/antchfx/htmlquery v1.3.1 // indirect github.com/antchfx/xmlquery v1.4.0 // indirect diff --git a/go.sum b/go.sum index cf66d2f..e8a2f00 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= +github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4= +github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antchfx/htmlquery v1.3.1 h1:wm0LxjLMsZhRHfQKKZscDf2COyH4vDYA3wyH+qZ+Ylc= @@ -9,6 +11,7 @@ github.com/antchfx/xmlquery v1.4.0/go.mod h1:Ax2aeaeDjfIw3CwXKDQ0GkwZ6QlxoChlIBP github.com/antchfx/xpath v1.3.0 h1:nTMlzGAK3IJ0bPpME2urTuFL76o4A96iYvoKFHRXJgc= github.com/antchfx/xpath v1.3.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= @@ -25,7 +28,12 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -74,3 +82,6 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scraper/helpers.go b/scraper/helpers.go index 1b35cbe..12b1502 100644 --- a/scraper/helpers.go +++ b/scraper/helpers.go @@ -8,15 +8,17 @@ import ( "strings" "github.com/PuerkitoBio/goquery" + "github.com/adrg/strutil" + "github.com/adrg/strutil/metrics" ) func text_content(doc *goquery.Selection, selector string) string { - return cleanup_string(doc.Find(selector).First().Text()) + return cleanup_whitespace(doc.Find(selector).First().Text()) } var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`) -func cleanup_string(s string) string { +func cleanup_whitespace(s string) string { return strings.TrimSpace(whitespace.ReplaceAllString(s, " ")) } @@ -33,3 +35,9 @@ func print_json(v any) { log.Println(string(out)) } } + +var hamming *metrics.Hamming = metrics.NewHamming() + +func hamming_score(a, b string) float64 { + return strutil.Similarity(a, b, hamming) +} diff --git a/scraper/scraper.go b/scraper/scraper.go index 3208bcb..240fcfe 100644 --- a/scraper/scraper.go +++ b/scraper/scraper.go @@ -31,5 +31,18 @@ const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 var scrapers []EventScraper = []EventScraper{ scrape_ufc, - scrape_one, + scrape_one, +} + +func ScrapeEvents(callback EventCallback) { + get_tapology := tapology_getter() + + for _, scraper := range scrapers { + scraper(func(e *Event) { + for _, fight := range e.Fights { + fight.FighterA.Link = get_tapology(fight.FighterA.Name) + } + callback(e) + }) + } } diff --git a/scraper/tapology.go b/scraper/tapology.go index 8cef823..95cd096 100644 --- a/scraper/tapology.go +++ b/scraper/tapology.go @@ -3,6 +3,8 @@ package scraper import ( "log" "net/url" + "regexp" + "slices" "time" "github.com/PuerkitoBio/goquery" @@ -16,10 +18,12 @@ type TapologyResult struct { Url string `json:"url"` } -func tapology_getter() func(n string) []TapologyResult { +type TapologyGetter func(n string) string + +func tapology_getter() TapologyGetter { csrf_token := "" name := "" - results := []TapologyResult{} + result := "" index := tapology_collector() index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) { @@ -27,6 +31,10 @@ func tapology_getter() func(n string) []TapologyResult { time.Sleep(5 * time.Second) }) + if err := index.Visit(tapology_url); err != nil { + log.Println(err) + } + search := tapology_collector() search.AllowURLRevisit = true search.OnRequest(func(r *colly.Request) { @@ -41,39 +49,54 @@ func tapology_getter() func(n string) []TapologyResult { log.Printf("Making tapology request to %s", r.URL) }) search.OnResponse(func(r *colly.Response) { - results = parse_tapology_results(r.Body) + result = parse_tapology_results(name, r.Body) time.Sleep(5 * time.Second) }) - if err := index.Visit(tapology_url); err != nil { - log.Println(err) - } - - return func(n string) []TapologyResult { + return func(n string) string { if csrf_token == "" { - return nil + return "" } name = n if err := search.Visit(tapology_url + "/search/nav"); err != nil { log.Println(err) - return nil + return "" } - return results + return result } } -func parse_tapology_results(b []byte) []TapologyResult { +var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`) + +func parse_tapology_results(name string, b []byte) string { results := []TapologyResult{} document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) { url, _ := s.Attr("href") - name := s.Text() + name := cleanup_whitespace(nickname.ReplaceAllString(s.Text(), "")) results = append(results, TapologyResult{Name: name, Url: url}) }) - return results + slices.SortStableFunc(results, func(a, b TapologyResult) int { + score_a := hamming_score(name, a.Name) + score_b := hamming_score(name, b.Name) + + if score_a > score_b { + return -1 + } else if score_a < score_b { + return 1 + } else { + return 0 + } + }) + + if len(results) < 1 { + return "" + } + + return tapology_url + results[0].Url } func tapology_collector() *colly.Collector { diff --git a/scraper/tapology_test.go b/scraper/tapology_test.go index aadce12..79803ba 100644 --- a/scraper/tapology_test.go +++ b/scraper/tapology_test.go @@ -10,11 +10,19 @@ var search_result []byte func testTapologyGetter(t *testing.T) { get_tapology_for := tapology_getter() - print_json(get_tapology_for("Dustin Poirier")) - print_json(get_tapology_for("Jon Jones")) - print_json(get_tapology_for("Conor Mcgregor")) + names := []string{ + "Dustin Poirier", + "Jon Jones", + "Conor Mcgregor", + "Justin Gaethje", + "Benoit Saint Denis", + } + + for _, name := range names { + print_json(get_tapology_for(name)) + } } -func TestTapologyParseFragment(t *testing.T) { - print_json(parse_tapology_results(search_result)) +func TestParseTapologyResults(t *testing.T) { + print_json(parse_tapology_results("Jon Jones", search_result)) } diff --git a/scraper/ufc.go b/scraper/ufc.go index f6ab0c3..29e1f14 100644 --- a/scraper/ufc.go +++ b/scraper/ufc.go @@ -53,7 +53,7 @@ func parse_ufc_event(url string, b []byte) *Event { doc := document_from_bytes(b) parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string { - return cleanup_string(s.Text()) + return cleanup_whitespace(s.Text()) }) event.Name = strings.Join(parts, ": ") event.Location = text_content(doc, ".field--name-venue")