tapology now uses hamming distance to get approximately correct link

scraper/helpers.go

@@ -8,15 +8,17 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
+	"github.com/adrg/strutil"
+	"github.com/adrg/strutil/metrics"
 )
 
 func text_content(doc *goquery.Selection, selector string) string {
-	return cleanup_string(doc.Find(selector).First().Text())
+	return cleanup_whitespace(doc.Find(selector).First().Text())
 }
 
 var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
 
-func cleanup_string(s string) string {
+func cleanup_whitespace(s string) string {
 	return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
 }
 
@@ -33,3 +35,9 @@ func print_json(v any) {
 		log.Println(string(out))
 	}
 }
+
+var hamming *metrics.Hamming = metrics.NewHamming()
+
+func hamming_score(a, b string) float64 {
+    return strutil.Similarity(a, b, hamming)
+}

scraper/scraper.go

@@ -31,5 +31,18 @@ const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36
 
 var scrapers []EventScraper = []EventScraper{
 	scrape_ufc,
-    scrape_one,
+	scrape_one,
+}
+
+func ScrapeEvents(callback EventCallback) {
+	get_tapology := tapology_getter()
+
+	for _, scraper := range scrapers {
+		scraper(func(e *Event) {
+			for _, fight := range e.Fights {
+				fight.FighterA.Link = get_tapology(fight.FighterA.Name)
+			}
+            callback(e)
+		})
+	}
 }

scraper/tapology.go

@@ -3,6 +3,8 @@ package scraper
 import (
 	"log"
 	"net/url"
+	"regexp"
+	"slices"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
@@ -16,10 +18,12 @@ type TapologyResult struct {
 	Url  string `json:"url"`
 }
 
-func tapology_getter() func(n string) []TapologyResult {
+type TapologyGetter func(n string) string
+
+func tapology_getter() TapologyGetter {
 	csrf_token := ""
 	name := ""
-	results := []TapologyResult{}
+	result := ""
 
 	index := tapology_collector()
 	index.OnHTML("meta[name=\"csrf-token\"]", func(h *colly.HTMLElement) {
@@ -27,6 +31,10 @@ func tapology_getter() func(n string) []TapologyResult {
 		time.Sleep(5 * time.Second)
 	})
 
+	if err := index.Visit(tapology_url); err != nil {
+		log.Println(err)
+	}
+
 	search := tapology_collector()
 	search.AllowURLRevisit = true
 	search.OnRequest(func(r *colly.Request) {
@@ -41,39 +49,54 @@ func tapology_getter() func(n string) []TapologyResult {
 		log.Printf("Making tapology request to %s", r.URL)
 	})
 	search.OnResponse(func(r *colly.Response) {
-		results = parse_tapology_results(r.Body)
+		result = parse_tapology_results(name, r.Body)
 		time.Sleep(5 * time.Second)
 	})
 
-	if err := index.Visit(tapology_url); err != nil {
-		log.Println(err)
-	}
-
-	return func(n string) []TapologyResult {
+	return func(n string) string {
 		if csrf_token == "" {
-			return nil
+			return ""
 		}
 
 		name = n
 		if err := search.Visit(tapology_url + "/search/nav"); err != nil {
 			log.Println(err)
-			return nil
+			return ""
 		}
 
-		return results
+		return result
 	}
 }
 
-func parse_tapology_results(b []byte) []TapologyResult {
+var nickname *regexp.Regexp = regexp.MustCompile(`"(\w| )+"`)
+
+func parse_tapology_results(name string, b []byte) string {
 	results := []TapologyResult{}
 
 	document_from_bytes(b).Find("span.star a[href]").Each(func(i int, s *goquery.Selection) {
 		url, _ := s.Attr("href")
-		name := s.Text()
+		name := cleanup_whitespace(nickname.ReplaceAllString(s.Text(), ""))
 		results = append(results, TapologyResult{Name: name, Url: url})
 	})
 
-	return results
+	slices.SortStableFunc(results, func(a, b TapologyResult) int {
+		score_a := hamming_score(name, a.Name)
+		score_b := hamming_score(name, b.Name)
+
+		if score_a > score_b {
+			return -1
+		} else if score_a < score_b {
+			return 1
+		} else {
+			return 0
+		}
+	})
+
+	if len(results) < 1 {
+		return ""
+	}
+
+	return tapology_url + results[0].Url
 }
 
 func tapology_collector() *colly.Collector {

scraper/tapology_test.go

@@ -10,11 +10,19 @@ var search_result []byte
 
 func testTapologyGetter(t *testing.T) {
 	get_tapology_for := tapology_getter()
-	print_json(get_tapology_for("Dustin Poirier"))
-	print_json(get_tapology_for("Jon Jones"))
-	print_json(get_tapology_for("Conor Mcgregor"))
+	names := []string{
+		"Dustin Poirier",
+		"Jon Jones",
+		"Conor Mcgregor",
+        "Justin Gaethje",
+        "Benoit Saint Denis",
+	}
+
+	for _, name := range names {
+		print_json(get_tapology_for(name))
+	}
 }
 
-func TestTapologyParseFragment(t *testing.T) {
-    print_json(parse_tapology_results(search_result))
+func TestParseTapologyResults(t *testing.T) {
+	print_json(parse_tapology_results("Jon Jones", search_result))
 }

scraper/ufc.go

@@ -53,7 +53,7 @@ func parse_ufc_event(url string, b []byte) *Event {
 	doc := document_from_bytes(b)
 
 	parts := doc.Find(".c-hero__headline-prefix, .c-hero__headline").Map(func(i int, s *goquery.Selection) string {
-		return cleanup_string(s.Text())
+		return cleanup_whitespace(s.Text())
 	})
 	event.Name = strings.Join(parts, ": ")
 	event.Location = text_content(doc, ".field--name-venue")