diff --git a/crates/mmaschedule_scraper/src/lib.rs b/crates/mmaschedule_scraper/src/lib.rs index ab087aa..5d0bdec 100644 --- a/crates/mmaschedule_scraper/src/lib.rs +++ b/crates/mmaschedule_scraper/src/lib.rs @@ -11,6 +11,7 @@ mod one; mod ufc; use crate::one::ONE; +use crate::ufc::UFC; #[derive(Debug, Default, Deserialize, Serialize)] pub struct Event { @@ -49,33 +50,13 @@ trait ClientScraper { async fn get(&self, url: T) -> Result; } -struct DummyClient<'client> { - url_to_html: HashMap<&'client str, &'client str>, -} - -impl<'client> DummyClient<'client> { - fn new(url_to_html: HashMap<&'client str, &'client str>) -> DummyClient<'client> { - DummyClient { url_to_html } - } -} - -impl ClientScraper for DummyClient<'_> { - async fn get(&self, url: T) -> Result { - let doc = self - .url_to_html - .get(url.to_string().as_str()) - .ok_or(StrError::new("Invalid URL"))?; - - Ok(Html::parse_document(doc)) - } -} - trait EventScraper { async fn scrape(client: &C) -> Result>; } pub async fn do_scraping(client: &C) { ONE::scrape(client).await; + UFC::scrape(client).await; todo!() } @@ -95,7 +76,14 @@ fn text_content(elem: ElementRef<'_>, selector: &Selector) -> Option { elem.select(selector).next().map(|e| { let tokens: Vec = e .text() - .map(|t| WHITESPACE.replace_all(t, " ").to_string()) + .map(|t| { + WHITESPACE + .replace_all(t, " ") + .to_string() + .trim() + .to_string() + }) + .filter(|s| s != "") .collect(); tokens.join(" ").trim().to_string() @@ -127,4 +115,25 @@ impl std::error::Error for StrError { #[cfg(test)] mod tests { use super::*; + + pub struct DummyClient<'client> { + url_to_html: HashMap<&'client str, &'client str>, + } + + impl<'client> DummyClient<'client> { + pub fn new(url_to_html: HashMap<&'client str, &'client str>) -> DummyClient<'client> { + DummyClient { url_to_html } + } + } + + impl ClientScraper for DummyClient<'_> { + async fn get(&self, url: T) -> Result { + let doc = self + .url_to_html + .get(url.to_string().as_str()) + .ok_or(StrError::new("Invalid URL"))?; + + Ok(Html::parse_document(doc)) + } + } } diff --git a/crates/mmaschedule_scraper/src/one.rs b/crates/mmaschedule_scraper/src/one.rs index 518f1ba..9eb8dda 100644 --- a/crates/mmaschedule_scraper/src/one.rs +++ b/crates/mmaschedule_scraper/src/one.rs @@ -174,7 +174,7 @@ mod tests { use std::{collections::HashMap, sync::LazyLock}; - use crate::DummyClient; + use crate::tests::DummyClient; static CLIENT: LazyLock = LazyLock::new(|| { DummyClient::new(HashMap::from([ diff --git a/crates/mmaschedule_scraper/src/ufc.rs b/crates/mmaschedule_scraper/src/ufc.rs new file mode 100644 index 0000000..eb3265f --- /dev/null +++ b/crates/mmaschedule_scraper/src/ufc.rs @@ -0,0 +1,213 @@ +use std::sync::LazyLock; + +use anyhow::{Ok, Result}; +use scraper::{ElementRef, Selector}; + +use crate::{ClientScraper, Event, EventScraper, Fight, Fighter, text_content}; + +pub enum UFC {} + +const URL: &str = "https://www.ufc.com"; +static EVENT_LIST: LazyLock = LazyLock::new(|| { + Selector::parse("#events-list-upcoming .c-card-event--result__headline a").unwrap() +}); + +impl EventScraper for UFC { + async fn scrape(client: &C) -> Result> { + let page = client.get(URL.to_string() + "/events").await?; + let mut events = Vec::new(); + + for elem in page.select(&EVENT_LIST) { + if let Some(event) = get_event(client, elem).await { + events.push(event); + }; + } + + Ok(events) + } +} + +static EVENT_PREFIX: LazyLock = + LazyLock::new(|| Selector::parse(".c-hero__headline-prefix").unwrap()); +static EVENT_HEADLINE: LazyLock = + LazyLock::new(|| Selector::parse(".c-hero__headline").unwrap()); +static EVENT_LOCATION: LazyLock = + LazyLock::new(|| Selector::parse(".field--name-venue").unwrap()); +static EVENT_IMAGE: LazyLock = + LazyLock::new(|| Selector::parse(".c-hero__image img").unwrap()); +static EVENT_DATE: LazyLock = LazyLock::new(|| { + Selector::parse( + r#" + #early-prelims .c-event-fight-card-broadcaster__time, + #prelims-card .c-event-fight-card-broadcaster__time, + .c-hero__headline-suffix + "#, + ) + .unwrap() +}); +static EVENT_FIGHTS: LazyLock = + LazyLock::new(|| Selector::parse(".l-listing__item").unwrap()); + +async fn get_event(client: &C, elem: ElementRef<'_>) -> Option { + let slug = elem.attr("href")?.to_string(); + let url = URL.to_string() + &slug; + let page = client.get(&url).await.ok()?; + let prefix = text_content(page.root_element(), &EVENT_PREFIX)?; + let headline = text_content(page.root_element(), &EVENT_HEADLINE)?; + let name = prefix + ": " + &headline; + let location = text_content(page.root_element(), &EVENT_LOCATION).unwrap_or("Unknown".into()); + let image = page + .select(&EVENT_IMAGE) + .next() + .map(|e| e.attr("src")) + .flatten()? + .to_string(); + let date: i64 = page + .select(&EVENT_DATE) + .last() + .map(|e| e.attr("data-timestamp")) + .flatten()? + .parse() + .ok()?; + let mut fights = Vec::new(); + for elem in page.select(&EVENT_FIGHTS) { + if let Some(fight) = get_fight(elem) { + fights.push(fight); + } + } + if fights.len() == 0 { + return None; + } + Some(Event { + url, + slug, + name, + location, + organization: "UFC".into(), + image, + date, + fights, + }) +} + +static FIGHT_WEIGHT: LazyLock = + LazyLock::new(|| Selector::parse(".c-listing-fight__class-text").unwrap()); + +fn get_fight(elem: ElementRef<'_>) -> Option { + let weight = text_content(elem, &FIGHT_WEIGHT)?.replace(" Bout", ""); + let fighter_a = get_fighter::(elem)?; + let fighter_b = get_fighter::(elem)?; + Some(Fight { + weight, + fighter_a, + fighter_b, + }) +} + +static FIGHTER_A_NAME: LazyLock = + LazyLock::new(|| Selector::parse(".c-listing-fight__corner-name--red").unwrap()); +static FIGHTER_A_COUNTRY: LazyLock = LazyLock::new(|| { + Selector::parse(".c-listing-fight__country--red .c-listing-fight__country-text").unwrap() +}); +static FIGHTER_A_IMAGE: LazyLock = + LazyLock::new(|| Selector::parse(".c-listing-fight__corner-image--red img").unwrap()); + +static FIGHTER_B_NAME: LazyLock = + LazyLock::new(|| Selector::parse(".c-listing-fight__corner-name--blue").unwrap()); +static FIGHTER_B_COUNTRY: LazyLock = LazyLock::new(|| { + Selector::parse(".c-listing-fight__country--blue .c-listing-fight__country-text").unwrap() +}); +static FIGHTER_B_IMAGE: LazyLock = + LazyLock::new(|| Selector::parse(".c-listing-fight__corner-image--blue img").unwrap()); + +fn get_fighter(elem: ElementRef<'_>) -> Option { + let name = text_content( + elem, + if IS_A { + &FIGHTER_A_NAME + } else { + &FIGHTER_B_NAME + }, + )?; + let country = text_content( + elem, + if IS_A { + &FIGHTER_A_COUNTRY + } else { + &FIGHTER_B_COUNTRY + }, + )?; + let image = elem + .select(if IS_A { + &FIGHTER_A_IMAGE + } else { + &FIGHTER_B_IMAGE + }) + .next() + .map(|e| e.attr("src")) + .flatten()? + .to_string(); + Some(Fighter { + name, + image, + country, + link: "".into(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::{collections::HashMap, sync::LazyLock}; + + use crate::tests::DummyClient; + + static CLIENT: LazyLock = LazyLock::new(|| { + DummyClient::new(HashMap::from([ + ( + "https://www.ufc.com/events", + include_str!("./testdata/ufc-event-list.html"), + ), + ( + "https://www.ufc.com/event/ufc-312", + include_str!("./testdata/ufc-312.html"), + ), + ( + "https://www.ufc.com/event/ufc-fight-night-february-15-2025", + include_str!("./testdata/ufc-fn-feb-15.html"), + ), + ( + "https://www.ufc.com/event/ufc-fight-night-february-22-2025", + include_str!("./testdata/ufc-fn-feb-22.html"), + ), + ( + "https://www.ufc.com/event/ufc-fight-night-march-01-2025", + include_str!("./testdata/ufc-fn-mar-1.html"), + ), + ( + "https://www.ufc.com/event/ufc-313", + include_str!("./testdata/ufc-313.html"), + ), + ( + "https://www.ufc.com/event/ufc-fight-night-march-15-2025", + include_str!("./testdata/ufc-fn-mar-15.html"), + ), + ( + "https://www.ufc.com/event/ufc-fight-night-march-22-2025", + include_str!("./testdata/ufc-fn-mar-22.html"), + ), + ( + "https://www.ufc.com/event/ufc-fight-night-march-29-2025", + include_str!("./testdata/ufc-fn-mar-29.html"), + ), + ])) + }); + + #[tokio::test] + async fn test_event_list() { + let events = UFC::scrape(&*CLIENT).await.unwrap(); + assert!(events.len() == 7); + println!("{}", serde_json::to_string_pretty(&events).unwrap()); + } +}