guy176251/mmaschedule-rust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scraper changes

Browse Source 
- ported ONE scraper
- added html test data
This commit is contained in:
long
2026-07-13 15:24:37 -05:00
parent d69955ea39
commit f70aaa7492
21 changed files with 58077 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/mmaschedule_scraper/src/one.rs
+163 -40
View File
@@ -1,44 +1,30 @@
use anyhow::Result;
use std::sync::LazyLock;
use anyhow::{Ok, Result};
use scraper::{ElementRef, Selector, selectable::Selectable};
use serde::{Deserialize, Serialize};
use serde::Deserialize;
use time_rs::OffsetDateTime;
use crate::{ClientScraper, Event, EventScraper, Fight, Fighter};
use crate::{ClientScraper, Event, EventScraper, Fight, Fighter, text_content};
pub enum ONE {}
const URL: &str = "https://www.onefc.com/events/";
static EVENT_LIST: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("#upcoming-events-section .is-event").unwrap());
impl EventScraper for ONE {
async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>> {
let page = client.get("https://www.onefc.com/events/").await?;
let page = client.get(URL).await?;
let mut events = Vec::new();
// should panic if selectors are invalid
let selector_event = Selector::parse("#upcoming-events-section .is-event").unwrap();
let selector_check = Selector::parse("a.smart-link").unwrap();
let selector_event_url = Selector::parse("a.is-image-zoom[href]").unwrap();
for event_fragment in page.select(&selector_event) {
do_thing(client, event_fragment).await;
if event_fragment
.select(&selector_check)
.next()
.map(|e| e.attr("href"))
.flatten()
.is_none()
{
continue;
}
let Some(url) = event_fragment
.select(&selector_event_url)
.next()
.map(|e| e.attr("href"))
.flatten()
else {
continue;
for elem in page.select(&EVENT_LIST) {
if let Some(event) = get_event(client, elem).await {
events.push(event);
};
}
todo!()
Ok(events)
}
}
@@ -58,30 +44,167 @@ struct EventData {
start_date: OffsetDateTime,
}
async fn do_thing<C: ClientScraper>(client: &C, elem: ElementRef<'_>) -> Option<Vec<Event>> {
let selector_check = Selector::parse("a.smart-link").unwrap();
let selector_event_url = Selector::parse("a.is-image-zoom[href]").unwrap();
let selector_data = Selector::parse("#site-main script[type=\"application/ld+json\"]").unwrap();
static EVENT_CHECK: LazyLock<Selector> = LazyLock::new(|| Selector::parse("a.smart-link").unwrap());
static EVENT_URL: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("a.is-image-zoom[href]").unwrap());
static EVENT_DATA: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("#site-main script[type=\"application/ld+json\"]").unwrap());
static EVENT_FIGHTS: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".event-matchup").unwrap());
let mut events = Vec::new();
elem.select(&selector_check)
async fn get_event<C: ClientScraper>(client: &C, elem: ElementRef<'_>) -> Option<Event> {
elem.select(&EVENT_CHECK)
.next()
.map(|e| e.attr("href"))
.flatten()?;
let url = elem
.select(&selector_event_url)
.select(&EVENT_URL)
.next()
.map(|e| e.attr("href"))
.flatten()?;
let page = client.get(url).await.ok()?;
let data_elem = page.select(&selector_data).next()?;
let data_elem = page.select(&EVENT_DATA).next()?;
let data_str: String = data_elem.text().collect();
let data: EventData = serde_json::from_str(&data_str).ok()?;
data.start_date.unix_timestamp();
Some(events)
let mut fights = Vec::new();
for elem in page.select(&EVENT_FIGHTS) {
if let Some(fight) = get_fight(elem) {
fights.push(fight);
}
}
if fights.len() == 0 {
return None;
}
let event = Event {
organization: "ONE".into(),
name: data.name,
location: data.location.name,
date: data.start_date.unix_timestamp(),
image: data
.image
.get(2)
.map(|s| s.clone())
.unwrap_or_else(String::default),
url: url.to_string(),
slug: url.replace(URL, "").replace("/", ""),
fights,
};
Some(event)
}
static FIGHT_WEIGHT: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".title").unwrap());
fn get_fight(elem: ElementRef<'_>) -> Option<Fight> {
let fighter_a = get_fighter::<true>(elem)?;
let fighter_b = get_fighter::<false>(elem)?;
let weight = text_content(elem, &FIGHT_WEIGHT)?;
Some(Fight {
weight,
fighter_a,
fighter_b,
})
}
static FIGHTER_A_NAME: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("tr.vs :nth-child(1) a").unwrap());
static FIGHTER_A_COUNTRY: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("tr.vs + tr :nth-child(1)").unwrap());
static FIGHTER_A_IMAGE: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".face1 img[src]").unwrap());
static FIGHTER_B_NAME: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("tr.vs :nth-child(3) a").unwrap());
static FIGHTER_B_COUNTRY: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("tr.vs + tr :nth-child(3)").unwrap());
static FIGHTER_B_IMAGE: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".face2 img[src]").unwrap());
fn get_fighter<const IS_A: bool>(elem: ElementRef<'_>) -> Option<Fighter> {
let name = text_content(
elem,
if IS_A {
&FIGHTER_A_NAME
} else {
&FIGHTER_B_NAME
},
)?;
let country = text_content(
elem,
if IS_A {
&FIGHTER_A_COUNTRY
} else {
&FIGHTER_B_COUNTRY
},
)?;
let image = elem
.select(if IS_A {
&FIGHTER_A_IMAGE
} else {
&FIGHTER_B_IMAGE
})
.next()
.map(|e| e.attr("src"))
.flatten()
.map(|s| s.to_string())?;
Some(Fighter {
name,
image,
country,
link: "".into(),
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::{collections::HashMap, sync::LazyLock};
use crate::DummyClient;
static CLIENT: LazyLock<DummyClient> = LazyLock::new(|| {
DummyClient::new(HashMap::from([
(
"https://www.onefc.com/events/",
include_str!("./testdata/one-event-list.html"),
),
(
"https://www.onefc.com/events/one-friday-fights-97/",
include_str!("./testdata/one-friday-fights-97.html"),
),
(
"https://www.onefc.com/events/one171/",
include_str!("./testdata/one-171.html"),
),
(
"https://www.onefc.com/events/onefightnight29/",
include_str!("./testdata/one-fn-29.html"),
),
(
"https://www.onefc.com/events/one172/",
include_str!("./testdata/one-172.html"),
),
]))
});
#[tokio::test]
async fn event_list() {
let events = ONE::scrape(&*CLIENT).await.unwrap();
assert!(events.len() == 4);
println!("{}", serde_json::to_string_pretty(&events).unwrap());
}
}