guy176251/mmaschedule-rust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- moved dummy client

Browse Source 
- added ufc scraper
This commit is contained in:
long
2026-07-14 15:25:44 -05:00
parent f70aaa7492
commit 19e0090ff5
3 changed files with 245 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/mmaschedule_scraper/src/lib.rs
+31 -22
View File
@@ -11,6 +11,7 @@ mod one;
mod ufc;
use crate::one::ONE;
use crate::ufc::UFC;
#[derive(Debug, Default, Deserialize, Serialize)]
pub struct Event {
@@ -49,33 +50,13 @@ trait ClientScraper {
async fn get<T: ToString>(&self, url: T) -> Result<Html>;
}
struct DummyClient<'client> {
url_to_html: HashMap<&'client str, &'client str>,
}
impl<'client> DummyClient<'client> {
fn new(url_to_html: HashMap<&'client str, &'client str>) -> DummyClient<'client> {
DummyClient { url_to_html }
}
}
impl ClientScraper for DummyClient<'_> {
async fn get<T: ToString>(&self, url: T) -> Result<Html> {
let doc = self
.url_to_html
.get(url.to_string().as_str())
.ok_or(StrError::new("Invalid URL"))?;
Ok(Html::parse_document(doc))
}
}
trait EventScraper {
async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>>;
}
pub async fn do_scraping<C: ClientScraper>(client: &C) {
ONE::scrape(client).await;
UFC::scrape(client).await;
todo!()
}
@@ -95,7 +76,14 @@ fn text_content(elem: ElementRef<'_>, selector: &Selector) -> Option<String> {
elem.select(selector).next().map(|e| {
let tokens: Vec<String> = e
.text()
.map(|t| WHITESPACE.replace_all(t, " ").to_string())
.map(|t| {
WHITESPACE
.replace_all(t, " ")
.to_string()
.trim()
.to_string()
})
.filter(|s| s != "")
.collect();
tokens.join(" ").trim().to_string()
@@ -127,4 +115,25 @@ impl std::error::Error for StrError {
#[cfg(test)]
mod tests {
use super::*;
pub struct DummyClient<'client> {
url_to_html: HashMap<&'client str, &'client str>,
}
impl<'client> DummyClient<'client> {
pub fn new(url_to_html: HashMap<&'client str, &'client str>) -> DummyClient<'client> {
DummyClient { url_to_html }
}
}
impl ClientScraper for DummyClient<'_> {
async fn get<T: ToString>(&self, url: T) -> Result<Html> {
let doc = self
.url_to_html
.get(url.to_string().as_str())
.ok_or(StrError::new("Invalid URL"))?;
Ok(Html::parse_document(doc))
}
}
}
crates/mmaschedule_scraper/src/one.rs
+1 -1
View File
@@ -174,7 +174,7 @@ mod tests {
use std::{collections::HashMap, sync::LazyLock};
use crate::DummyClient;
use crate::tests::DummyClient;
static CLIENT: LazyLock<DummyClient> = LazyLock::new(|| {
DummyClient::new(HashMap::from([
crates/mmaschedule_scraper/src/ufc.rs
+213
View File
@@ -0,0 +1,213 @@
use std::sync::LazyLock;
use anyhow::{Ok, Result};
use scraper::{ElementRef, Selector};
use crate::{ClientScraper, Event, EventScraper, Fight, Fighter, text_content};
pub enum UFC {}
const URL: &str = "https://www.ufc.com";
static EVENT_LIST: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("#events-list-upcoming .c-card-event--result__headline a").unwrap()
});
impl EventScraper for UFC {
async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>> {
let page = client.get(URL.to_string() + "/events").await?;
let mut events = Vec::new();
for elem in page.select(&EVENT_LIST) {
if let Some(event) = get_event(client, elem).await {
events.push(event);
};
}
Ok(events)
}
}
static EVENT_PREFIX: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".c-hero__headline-prefix").unwrap());
static EVENT_HEADLINE: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".c-hero__headline").unwrap());
static EVENT_LOCATION: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".field--name-venue").unwrap());
static EVENT_IMAGE: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".c-hero__image img").unwrap());
static EVENT_DATE: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse(
r#"
#early-prelims .c-event-fight-card-broadcaster__time,
#prelims-card .c-event-fight-card-broadcaster__time,
.c-hero__headline-suffix
"#,
)
.unwrap()
});
static EVENT_FIGHTS: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".l-listing__item").unwrap());
async fn get_event<C: ClientScraper>(client: &C, elem: ElementRef<'_>) -> Option<Event> {
let slug = elem.attr("href")?.to_string();
let url = URL.to_string() + &slug;
let page = client.get(&url).await.ok()?;
let prefix = text_content(page.root_element(), &EVENT_PREFIX)?;
let headline = text_content(page.root_element(), &EVENT_HEADLINE)?;
let name = prefix + ": " + &headline;
let location = text_content(page.root_element(), &EVENT_LOCATION).unwrap_or("Unknown".into());
let image = page
.select(&EVENT_IMAGE)
.next()
.map(|e| e.attr("src"))
.flatten()?
.to_string();
let date: i64 = page
.select(&EVENT_DATE)
.last()
.map(|e| e.attr("data-timestamp"))
.flatten()?
.parse()
.ok()?;
let mut fights = Vec::new();
for elem in page.select(&EVENT_FIGHTS) {
if let Some(fight) = get_fight(elem) {
fights.push(fight);
}
}
if fights.len() == 0 {
return None;
}
Some(Event {
url,
slug,
name,
location,
organization: "UFC".into(),
image,
date,
fights,
})
}
static FIGHT_WEIGHT: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".c-listing-fight__class-text").unwrap());
fn get_fight(elem: ElementRef<'_>) -> Option<Fight> {
let weight = text_content(elem, &FIGHT_WEIGHT)?.replace(" Bout", "");
let fighter_a = get_fighter::<true>(elem)?;
let fighter_b = get_fighter::<false>(elem)?;
Some(Fight {
weight,
fighter_a,
fighter_b,
})
}
static FIGHTER_A_NAME: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".c-listing-fight__corner-name--red").unwrap());
static FIGHTER_A_COUNTRY: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse(".c-listing-fight__country--red .c-listing-fight__country-text").unwrap()
});
static FIGHTER_A_IMAGE: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".c-listing-fight__corner-image--red img").unwrap());
static FIGHTER_B_NAME: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".c-listing-fight__corner-name--blue").unwrap());
static FIGHTER_B_COUNTRY: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse(".c-listing-fight__country--blue .c-listing-fight__country-text").unwrap()
});
static FIGHTER_B_IMAGE: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".c-listing-fight__corner-image--blue img").unwrap());
fn get_fighter<const IS_A: bool>(elem: ElementRef<'_>) -> Option<Fighter> {
let name = text_content(
elem,
if IS_A {
&FIGHTER_A_NAME
} else {
&FIGHTER_B_NAME
},
)?;
let country = text_content(
elem,
if IS_A {
&FIGHTER_A_COUNTRY
} else {
&FIGHTER_B_COUNTRY
},
)?;
let image = elem
.select(if IS_A {
&FIGHTER_A_IMAGE
} else {
&FIGHTER_B_IMAGE
})
.next()
.map(|e| e.attr("src"))
.flatten()?
.to_string();
Some(Fighter {
name,
image,
country,
link: "".into(),
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::{collections::HashMap, sync::LazyLock};
use crate::tests::DummyClient;
static CLIENT: LazyLock<DummyClient> = LazyLock::new(|| {
DummyClient::new(HashMap::from([
(
"https://www.ufc.com/events",
include_str!("./testdata/ufc-event-list.html"),
),
(
"https://www.ufc.com/event/ufc-312",
include_str!("./testdata/ufc-312.html"),
),
(
"https://www.ufc.com/event/ufc-fight-night-february-15-2025",
include_str!("./testdata/ufc-fn-feb-15.html"),
),
(
"https://www.ufc.com/event/ufc-fight-night-february-22-2025",
include_str!("./testdata/ufc-fn-feb-22.html"),
),
(
"https://www.ufc.com/event/ufc-fight-night-march-01-2025",
include_str!("./testdata/ufc-fn-mar-1.html"),
),
(
"https://www.ufc.com/event/ufc-313",
include_str!("./testdata/ufc-313.html"),
),
(
"https://www.ufc.com/event/ufc-fight-night-march-15-2025",
include_str!("./testdata/ufc-fn-mar-15.html"),
),
(
"https://www.ufc.com/event/ufc-fight-night-march-22-2025",
include_str!("./testdata/ufc-fn-mar-22.html"),
),
(
"https://www.ufc.com/event/ufc-fight-night-march-29-2025",
include_str!("./testdata/ufc-fn-mar-29.html"),
),
]))
});
#[tokio::test]
async fn test_event_list() {
let events = UFC::scrape(&*CLIENT).await.unwrap();
assert!(events.len() == 7);
println!("{}", serde_json::to_string_pretty(&events).unwrap());
}
}