tapology now uses hamming distance to get approximately correct link

go.mod

@@ -4,6 +4,7 @@ go 1.22.4
 
 require (
 	github.com/PuerkitoBio/goquery v1.9.2 // indirect
+	github.com/adrg/strutil v0.3.1 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/antchfx/htmlquery v1.3.1 // indirect
 	github.com/antchfx/xmlquery v1.4.0 // indirect

go.sum

@@ -1,5 +1,7 @@
 github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
 github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
+github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4=
+github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA=
 github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/antchfx/htmlquery v1.3.1 h1:wm0LxjLMsZhRHfQKKZscDf2COyH4vDYA3wyH+qZ+Ylc=
@@ -9,6 +11,7 @@ github.com/antchfx/xmlquery v1.4.0/go.mod h1:Ax2aeaeDjfIw3CwXKDQ0GkwZ6QlxoChlIBP
 github.com/antchfx/xpath v1.3.0 h1:nTMlzGAK3IJ0bPpME2urTuFL76o4A96iYvoKFHRXJgc=
 github.com/antchfx/xpath v1.3.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
@@ -25,7 +28,12 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
 github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
@@ -74,3 +82,6 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

scraper/helpers.go

@@ -8,15 +8,17 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
+	"github.com/adrg/strutil"
+	"github.com/adrg/strutil/metrics"
 )
 
 func text_content(doc *goquery.Selection, selector string) string {
-	return cleanup_string(doc.Find(selector).First().Text())
+	return cleanup_whitespace(doc.Find(selector).First().Text())
 }
 
 var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
 
-func cleanup_string(s string) string {
+func cleanup_whitespace(s string) string {
 	return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
 }
 
@@ -33,3 +35,9 @@ func print_json(v any) {
 		log.Println(string(out))
 	}
 }
+
+var hamming *metrics.Hamming = metrics.NewHamming()
+
+func hamming_score(a, b string) float64 {
+    return strutil.Similarity(a, b, hamming)
+}

scraper/scraper.go

@@ -31,5 +31,18 @@ const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36
 
 var scrapers []EventScraper = []EventScraper{
 	scrape_ufc,
-    scrape_one,
+	scrape_one,
+}
+
+func ScrapeEvents(callback EventCallback) {
+	get_tapology := tapology_getter()
+
+	for _, scraper := range scrapers {
+		scraper(func(e *Event) {
+			for _, fight := range e.Fights {
+				fight.FighterA.Link = get_tapology(fight.FighterA.Name)
+			}
+            callback(e)
+		})
+	}
 }

scraper/tapology.go

@@ -3,6 +3,8 @@ package scraper
 import (
 	"log"
 	"net/url"
+	"regexp"
+	"slices"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
@@ -16,10 +18,12 @@ type TapologyResult struct {
 	Url  string `json:"url"`
 }
 
-func tapology_getter() func(n string) []TapologyResult {
+type TapologyGetter func(n string) string
+
+func tapology_getter() TapologyGetter {
 	csrf_token := ""
 	name := ""
-	results := []TapologyResult{}
+	result := ""
 
 	index := tapology_collector()
 	index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) {
@@ -27,6 +31,10 @@ func tapology_getter() func(n string) []TapologyResult {
 		time.Sleep(5 * time.Second)
 	})
 
+	if err := index.Visit(tapology_url); err != nil {
+		log.Println(err)
+	}
+
 	search := tapology_collector()
 	search.AllowURLRevisit = true
 	search.OnRequest(func(r *colly.Request) {
@@ -41,39 +49,54 @@ func tapology_getter() func(n string) []TapologyResult {
 		log.Printf("Making tapology request to %s", r.URL)
 	})
 	search.OnResponse(func(r *colly.Response) {
-		results = parse_tapology_results(r.Body)
+		result = parse_tapology_results(name, r.Body)
 		time.Sleep(5 * time.Second)
 	})
 
-	if err := index.Visit(tapology_url); err != nil {
+	return func(n string) string {
-		log.Println(err)
-	}
-
-	return func(n string) []TapologyResult {
 		if csrf_token == "" {
-			return nil
+			return ""
 		}
 
 		name = n
 		if err := search.Visit(tapology_url + "/search/nav"); err != nil {
 			log.Println(err)
-			return nil
+			return ""
 		}
 
-		return results
+		return result
 	}
 }
 
-func parse_tapology_results(b []byte) []TapologyResult {
+var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`)
+
+func parse_tapology_results(name string, b []byte) string {
 	results := []TapologyResult{}
 
 	document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
 		url, _ := s.Attr("href")
-		name := s.Text()
+		name := cleanup_whitespace(nickname.ReplaceAllString(s.Text(), ""))
 		results = append(results, TapologyResult{Name: name, Url: url})
 	})
 
-	return results
+	slices.SortStableFunc(results, func(a, b TapologyResult) int {
+		score_a := hamming_score(name, a.Name)
+		score_b := hamming_score(name, b.Name)
+
+		if score_a > score_b {
+			return -1
+		} else if score_a < score_b {
+			return 1
+		} else {
+			return 0
+		}
+	})
+
+	if len(results) < 1 {
+		return ""
+	}
+
+	return tapology_url + results[0].Url
 }
 
 func tapology_collector() *colly.Collector {

scraper/tapology_test.go

@@ -10,11 +10,19 @@ var search_result []byte
 
 func testTapologyGetter(t *testing.T) {
 	get_tapology_for := tapology_getter()
-	print_json(get_tapology_for("Dustin Poirier"))
+	names := []string{
-	print_json(get_tapology_for("Jon Jones"))
+		"Dustin Poirier",
-	print_json(get_tapology_for("Conor Mcgregor"))
+		"Jon Jones",
+		"Conor Mcgregor",
+        "Justin Gaethje",
+        "Benoit Saint Denis",
+	}
+
+	for _, name := range names {
+		print_json(get_tapology_for(name))
+	}
 }
 
-func TestTapologyParseFragment(t *testing.T) {
+func TestParseTapologyResults(t *testing.T) {
-    print_json(parse_tapology_results(search_result))
+	print_json(parse_tapology_results("Jon Jones", search_result))
 }

scraper/ufc.go

@@ -53,7 +53,7 @@ func parse_ufc_event(url string, b []byte) *Event {
 	doc := document_from_bytes(b)
 
 	parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
-		return cleanup_string(s.Text())
+		return cleanup_whitespace(s.Text())
 	})
 	event.Name = strings.Join(parts, ": ")
 	event.Location = text_content(doc, ".field--name-venue")