start

2026-07-01 14:13:32 -05:00
commit d69955ea39
10 changed files with 2145 additions and 0 deletions
@@ -0,0 +1,13 @@
+[package]
+name = "mmaschedule_scraper"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = "1.0.103"
+scraper = "0.27.0"
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = "1.0.150"
+time_rs = { package = "time", version = "0.3.51", features = ["formatting", "parsing", "serde"] }
+wreq = "5.3.0"
+wreq-util = "2.2.6"
@@ -0,0 +1,97 @@
+use anyhow::Result;
+use scraper::{Html, Selector};
+use wreq::Client;
+use wreq_util::Emulation;
+
+mod one;
+
+use crate::one::ONE;
+
+pub struct Event {
+    pub url: String,
+    pub slug: String,
+    pub name: String,
+    pub location: String,
+    pub organization: String,
+    pub image: String,
+    pub date: i64,
+    pub fights: Vec<Fight>,
+}
+
+pub struct Fight {
+    pub weight: String,
+    pub fighter_a: Fighter,
+    pub fighter_b: Fighter,
+}
+
+pub struct Fighter {
+    pub name: String,
+    pub image: String,
+    pub country: String,
+    pub link: String,
+}
+
+pub struct Tapology {
+    pub name: String,
+    pub url: String,
+}
+
+trait ClientScraper {
+    async fn get(&self, url: &str) -> Result<Html>;
+}
+
+trait EventScraper {
+    async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>>;
+}
+
+enum UFC {}
+
+impl EventScraper for UFC {
+    async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>> {
+        todo!()
+    }
+}
+
+pub async fn hello() -> wreq::Result<()> {
+    // Build a client
+    let client = Client::builder().emulation(Emulation::Firefox139).build()?;
+
+    // Use the API you're already familiar with
+    let resp = client.get("https://tls.peet.ws/api/all").send().await?;
+    println!("{}", resp.text().await?);
+    Ok(())
+}
+
+pub fn add(left: u64, right: u64) -> u64 {
+    left + right
+}
+
+fn yes() {
+    let html = r#"
+    <ul>
+        <li>Foo</li>
+        <li>Bar</li>
+        <li>Baz</li>
+    </ul>
+"#;
+
+    let fragment = Html::parse_fragment(html);
+    let ul_selector = Selector::parse("ul").unwrap();
+    let li_selector = Selector::parse("li").unwrap();
+
+    let ul = fragment.select(&ul_selector).next().unwrap();
+    for element in ul.select(&li_selector) {
+        assert_eq!("li", element.value().name());
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let result = add(2, 2);
+        assert_eq!(result, 4);
+    }
+}
@@ -0,0 +1,87 @@
+use anyhow::Result;
+use scraper::{ElementRef, Selector, selectable::Selectable};
+use serde::{Deserialize, Serialize};
+use time_rs::OffsetDateTime;
+
+use crate::{ClientScraper, Event, EventScraper, Fight, Fighter};
+
+pub enum ONE {}
+
+impl EventScraper for ONE {
+    async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>> {
+        let page = client.get("https://www.onefc.com/events/").await?;
+
+        // should panic if selectors are invalid
+        let selector_event = Selector::parse("#upcoming-events-section .is-event").unwrap();
+        let selector_check = Selector::parse("a.smart-link").unwrap();
+        let selector_event_url = Selector::parse("a.is-image-zoom[href]").unwrap();
+
+        for event_fragment in page.select(&selector_event) {
+            do_thing(client, event_fragment).await;
+            if event_fragment
+                .select(&selector_check)
+                .next()
+                .map(|e| e.attr("href"))
+                .flatten()
+                .is_none()
+            {
+                continue;
+            }
+
+            let Some(url) = event_fragment
+                .select(&selector_event_url)
+                .next()
+                .map(|e| e.attr("href"))
+                .flatten()
+            else {
+                continue;
+            };
+        }
+
+        todo!()
+    }
+}
+
+#[derive(Deserialize)]
+struct Location {
+    name: String,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct EventData {
+    name: String,
+    image: Vec<String>,
+    location: Location,
+
+    #[serde(with = "time_rs::serde::rfc3339")]
+    start_date: OffsetDateTime,
+}
+
+async fn do_thing<C: ClientScraper>(client: &C, elem: ElementRef<'_>) -> Option<Vec<Event>> {
+    let selector_check = Selector::parse("a.smart-link").unwrap();
+    let selector_event_url = Selector::parse("a.is-image-zoom[href]").unwrap();
+    let selector_data = Selector::parse("#site-main script[type=\"application/ld+json\"]").unwrap();
+
+    let mut events = Vec::new();
+
+    elem.select(&selector_check)
+        .next()
+        .map(|e| e.attr("href"))
+        .flatten()?;
+
+    let url = elem
+        .select(&selector_event_url)
+        .next()
+        .map(|e| e.attr("href"))
+        .flatten()?;
+
+    let page = client.get(url).await.ok()?;
+
+    let data_elem = page.select(&selector_data).next()?;
+    let data_str: String = data_elem.text().collect();
+    let data: EventData = serde_json::from_str(&data_str).ok()?;
+    data.start_date.unix_timestamp();
+
+    Some(events)
+}