19e0090ff5
- added ufc scraper
214 lines
6.6 KiB
Rust
214 lines
6.6 KiB
Rust
use std::sync::LazyLock;
|
|
|
|
use anyhow::{Ok, Result};
|
|
use scraper::{ElementRef, Selector};
|
|
|
|
use crate::{ClientScraper, Event, EventScraper, Fight, Fighter, text_content};
|
|
|
|
pub enum UFC {}
|
|
|
|
const URL: &str = "https://www.ufc.com";
|
|
static EVENT_LIST: LazyLock<Selector> = LazyLock::new(|| {
|
|
Selector::parse("#events-list-upcoming .c-card-event--result__headline a").unwrap()
|
|
});
|
|
|
|
impl EventScraper for UFC {
|
|
async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>> {
|
|
let page = client.get(URL.to_string() + "/events").await?;
|
|
let mut events = Vec::new();
|
|
|
|
for elem in page.select(&EVENT_LIST) {
|
|
if let Some(event) = get_event(client, elem).await {
|
|
events.push(event);
|
|
};
|
|
}
|
|
|
|
Ok(events)
|
|
}
|
|
}
|
|
|
|
static EVENT_PREFIX: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".c-hero__headline-prefix").unwrap());
|
|
static EVENT_HEADLINE: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".c-hero__headline").unwrap());
|
|
static EVENT_LOCATION: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".field--name-venue").unwrap());
|
|
static EVENT_IMAGE: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".c-hero__image img").unwrap());
|
|
static EVENT_DATE: LazyLock<Selector> = LazyLock::new(|| {
|
|
Selector::parse(
|
|
r#"
|
|
#early-prelims .c-event-fight-card-broadcaster__time,
|
|
#prelims-card .c-event-fight-card-broadcaster__time,
|
|
.c-hero__headline-suffix
|
|
"#,
|
|
)
|
|
.unwrap()
|
|
});
|
|
static EVENT_FIGHTS: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".l-listing__item").unwrap());
|
|
|
|
async fn get_event<C: ClientScraper>(client: &C, elem: ElementRef<'_>) -> Option<Event> {
|
|
let slug = elem.attr("href")?.to_string();
|
|
let url = URL.to_string() + &slug;
|
|
let page = client.get(&url).await.ok()?;
|
|
let prefix = text_content(page.root_element(), &EVENT_PREFIX)?;
|
|
let headline = text_content(page.root_element(), &EVENT_HEADLINE)?;
|
|
let name = prefix + ": " + &headline;
|
|
let location = text_content(page.root_element(), &EVENT_LOCATION).unwrap_or("Unknown".into());
|
|
let image = page
|
|
.select(&EVENT_IMAGE)
|
|
.next()
|
|
.map(|e| e.attr("src"))
|
|
.flatten()?
|
|
.to_string();
|
|
let date: i64 = page
|
|
.select(&EVENT_DATE)
|
|
.last()
|
|
.map(|e| e.attr("data-timestamp"))
|
|
.flatten()?
|
|
.parse()
|
|
.ok()?;
|
|
let mut fights = Vec::new();
|
|
for elem in page.select(&EVENT_FIGHTS) {
|
|
if let Some(fight) = get_fight(elem) {
|
|
fights.push(fight);
|
|
}
|
|
}
|
|
if fights.len() == 0 {
|
|
return None;
|
|
}
|
|
Some(Event {
|
|
url,
|
|
slug,
|
|
name,
|
|
location,
|
|
organization: "UFC".into(),
|
|
image,
|
|
date,
|
|
fights,
|
|
})
|
|
}
|
|
|
|
static FIGHT_WEIGHT: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".c-listing-fight__class-text").unwrap());
|
|
|
|
fn get_fight(elem: ElementRef<'_>) -> Option<Fight> {
|
|
let weight = text_content(elem, &FIGHT_WEIGHT)?.replace(" Bout", "");
|
|
let fighter_a = get_fighter::<true>(elem)?;
|
|
let fighter_b = get_fighter::<false>(elem)?;
|
|
Some(Fight {
|
|
weight,
|
|
fighter_a,
|
|
fighter_b,
|
|
})
|
|
}
|
|
|
|
static FIGHTER_A_NAME: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".c-listing-fight__corner-name--red").unwrap());
|
|
static FIGHTER_A_COUNTRY: LazyLock<Selector> = LazyLock::new(|| {
|
|
Selector::parse(".c-listing-fight__country--red .c-listing-fight__country-text").unwrap()
|
|
});
|
|
static FIGHTER_A_IMAGE: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".c-listing-fight__corner-image--red img").unwrap());
|
|
|
|
static FIGHTER_B_NAME: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".c-listing-fight__corner-name--blue").unwrap());
|
|
static FIGHTER_B_COUNTRY: LazyLock<Selector> = LazyLock::new(|| {
|
|
Selector::parse(".c-listing-fight__country--blue .c-listing-fight__country-text").unwrap()
|
|
});
|
|
static FIGHTER_B_IMAGE: LazyLock<Selector> =
|
|
LazyLock::new(|| Selector::parse(".c-listing-fight__corner-image--blue img").unwrap());
|
|
|
|
fn get_fighter<const IS_A: bool>(elem: ElementRef<'_>) -> Option<Fighter> {
|
|
let name = text_content(
|
|
elem,
|
|
if IS_A {
|
|
&FIGHTER_A_NAME
|
|
} else {
|
|
&FIGHTER_B_NAME
|
|
},
|
|
)?;
|
|
let country = text_content(
|
|
elem,
|
|
if IS_A {
|
|
&FIGHTER_A_COUNTRY
|
|
} else {
|
|
&FIGHTER_B_COUNTRY
|
|
},
|
|
)?;
|
|
let image = elem
|
|
.select(if IS_A {
|
|
&FIGHTER_A_IMAGE
|
|
} else {
|
|
&FIGHTER_B_IMAGE
|
|
})
|
|
.next()
|
|
.map(|e| e.attr("src"))
|
|
.flatten()?
|
|
.to_string();
|
|
Some(Fighter {
|
|
name,
|
|
image,
|
|
country,
|
|
link: "".into(),
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
use std::{collections::HashMap, sync::LazyLock};
|
|
|
|
use crate::tests::DummyClient;
|
|
|
|
static CLIENT: LazyLock<DummyClient> = LazyLock::new(|| {
|
|
DummyClient::new(HashMap::from([
|
|
(
|
|
"https://www.ufc.com/events",
|
|
include_str!("./testdata/ufc-event-list.html"),
|
|
),
|
|
(
|
|
"https://www.ufc.com/event/ufc-312",
|
|
include_str!("./testdata/ufc-312.html"),
|
|
),
|
|
(
|
|
"https://www.ufc.com/event/ufc-fight-night-february-15-2025",
|
|
include_str!("./testdata/ufc-fn-feb-15.html"),
|
|
),
|
|
(
|
|
"https://www.ufc.com/event/ufc-fight-night-february-22-2025",
|
|
include_str!("./testdata/ufc-fn-feb-22.html"),
|
|
),
|
|
(
|
|
"https://www.ufc.com/event/ufc-fight-night-march-01-2025",
|
|
include_str!("./testdata/ufc-fn-mar-1.html"),
|
|
),
|
|
(
|
|
"https://www.ufc.com/event/ufc-313",
|
|
include_str!("./testdata/ufc-313.html"),
|
|
),
|
|
(
|
|
"https://www.ufc.com/event/ufc-fight-night-march-15-2025",
|
|
include_str!("./testdata/ufc-fn-mar-15.html"),
|
|
),
|
|
(
|
|
"https://www.ufc.com/event/ufc-fight-night-march-22-2025",
|
|
include_str!("./testdata/ufc-fn-mar-22.html"),
|
|
),
|
|
(
|
|
"https://www.ufc.com/event/ufc-fight-night-march-29-2025",
|
|
include_str!("./testdata/ufc-fn-mar-29.html"),
|
|
),
|
|
]))
|
|
});
|
|
|
|
#[tokio::test]
|
|
async fn test_event_list() {
|
|
let events = UFC::scrape(&*CLIENT).await.unwrap();
|
|
assert!(events.len() == 7);
|
|
println!("{}", serde_json::to_string_pretty(&events).unwrap());
|
|
}
|
|
}
|