made stuff private

This commit is contained in:
2024-07-02 14:14:38 -05:00
parent eaf229abde
commit ad7b487954
9 changed files with 81 additions and 91 deletions
+5 -16
View File
@@ -3,7 +3,6 @@ package scraper
import (
"bytes"
"encoding/json"
"hash/fnv"
"log"
"regexp"
"strings"
@@ -11,32 +10,22 @@ import (
"github.com/PuerkitoBio/goquery"
)
func TextContent(doc *goquery.Selection, selector string) string {
return CleanupString(doc.Find(selector).First().Text())
func text_content(doc *goquery.Selection, selector string) string {
return cleanup_string(doc.Find(selector).First().Text())
}
var whitespace *regexp.Regexp = regexp.MustCompile(`\s+`)
func CleanupString(s string) string {
func cleanup_string(s string) string {
return strings.TrimSpace(whitespace.ReplaceAllString(s, " "))
}
func DocumentFromBytes(b []byte) *goquery.Selection {
func document_from_bytes(b []byte) *goquery.Selection {
doc, _ := goquery.NewDocumentFromReader(bytes.NewReader(b))
return doc.Find("html").First()
}
func HashJSON(v any) uint32 {
out, err := json.Marshal(v)
if err != nil {
return 0
}
h := fnv.New32a()
h.Write(out)
return h.Sum32()
}
func PrintJSON(v any) {
func print_json(v any) {
out, err := json.MarshalIndent(v, "", " ")
if err != nil {
log.Println(err)