Scraper changes
- ported ONE scraper - added html test data
This commit is contained in:
Generated
+68
-55
@@ -159,15 +159,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.12.0"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
|
||||
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.65"
|
||||
version = "1.2.67"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
|
||||
checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -361,6 +361,16 @@ version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.4.1"
|
||||
@@ -491,14 +501,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.3.4"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
||||
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
"wasip2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -535,9 +544,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "1.0.1"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
|
||||
checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http",
|
||||
@@ -545,9 +554,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "http-body-util"
|
||||
version = "0.1.3"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
|
||||
checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -736,9 +745,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.34"
|
||||
version = "0.1.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
|
||||
checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"libc",
|
||||
@@ -815,9 +824,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.2"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
@@ -843,9 +852,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.2.1"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
|
||||
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"wasi",
|
||||
@@ -865,10 +874,12 @@ name = "mmaschedule_scraper"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"regex",
|
||||
"scraper",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
"tokio",
|
||||
"wreq",
|
||||
"wreq-util",
|
||||
]
|
||||
@@ -1041,9 +1052,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "5.3.0"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
@@ -1056,9 +1067,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.4"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
|
||||
checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -1068,9 +1079,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.14"
|
||||
version = "0.4.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
||||
checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -1085,9 +1096,9 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.2"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
||||
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
@@ -1106,9 +1117,9 @@ checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.22"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
@@ -1238,6 +1249,16 @@ version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
|
||||
dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.9"
|
||||
@@ -1366,19 +1387,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4790fc369d5a530f4b544b094e31388b9b3a37c0f4652ade4505945f5660d24"
|
||||
checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.52"
|
||||
version = "0.3.53"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e48db7b415311b615f910b3dcaa4557bcd4bf1982379c95c223fd8c2a20e210"
|
||||
checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"num-conv",
|
||||
@@ -1423,8 +1443,11 @@ dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
"mio",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"socket2 0.6.4",
|
||||
"tokio-macros",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
@@ -1439,6 +1462,17 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.18"
|
||||
@@ -1529,12 +1563,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf-8"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@@ -1556,15 +1584,6 @@ version = "0.11.1+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||
|
||||
[[package]]
|
||||
name = "wasip2"
|
||||
version = "1.0.4+wasi-0.2.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
|
||||
dependencies = [
|
||||
"wit-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web_atoms"
|
||||
version = "0.2.5"
|
||||
@@ -1740,12 +1759,6 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
version = "0.57.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
|
||||
|
||||
[[package]]
|
||||
name = "wreq"
|
||||
version = "5.3.0"
|
||||
@@ -1882,9 +1895,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
|
||||
@@ -5,9 +5,11 @@ edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.103"
|
||||
regex = "1.13.0"
|
||||
scraper = "0.27.0"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.150"
|
||||
time_rs = { package = "time", version = "0.3.51", features = ["formatting", "parsing", "serde"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
wreq = "5.3.0"
|
||||
wreq-util = "2.2.6"
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
use std::{collections::HashMap, fmt::Display, sync::LazyLock};
|
||||
|
||||
use anyhow::Result;
|
||||
use scraper::{Html, Selector};
|
||||
use regex::Regex;
|
||||
use scraper::{ElementRef, Html, Selector};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use wreq::Client;
|
||||
use wreq_util::Emulation;
|
||||
|
||||
mod one;
|
||||
mod ufc;
|
||||
|
||||
use crate::one::ONE;
|
||||
|
||||
#[derive(Debug, Default, Deserialize, Serialize)]
|
||||
pub struct Event {
|
||||
pub url: String,
|
||||
pub slug: String,
|
||||
@@ -18,12 +24,14 @@ pub struct Event {
|
||||
pub fights: Vec<Fight>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize, Serialize)]
|
||||
pub struct Fight {
|
||||
pub weight: String,
|
||||
pub fighter_a: Fighter,
|
||||
pub fighter_b: Fighter,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize, Serialize)]
|
||||
pub struct Fighter {
|
||||
pub name: String,
|
||||
pub image: String,
|
||||
@@ -31,26 +39,45 @@ pub struct Fighter {
|
||||
pub link: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize, Serialize)]
|
||||
pub struct Tapology {
|
||||
pub name: String,
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
trait ClientScraper {
|
||||
async fn get(&self, url: &str) -> Result<Html>;
|
||||
async fn get<T: ToString>(&self, url: T) -> Result<Html>;
|
||||
}
|
||||
|
||||
struct DummyClient<'client> {
|
||||
url_to_html: HashMap<&'client str, &'client str>,
|
||||
}
|
||||
|
||||
impl<'client> DummyClient<'client> {
|
||||
fn new(url_to_html: HashMap<&'client str, &'client str>) -> DummyClient<'client> {
|
||||
DummyClient { url_to_html }
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientScraper for DummyClient<'_> {
|
||||
async fn get<T: ToString>(&self, url: T) -> Result<Html> {
|
||||
let doc = self
|
||||
.url_to_html
|
||||
.get(url.to_string().as_str())
|
||||
.ok_or(StrError::new("Invalid URL"))?;
|
||||
|
||||
Ok(Html::parse_document(doc))
|
||||
}
|
||||
}
|
||||
|
||||
trait EventScraper {
|
||||
async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>>;
|
||||
}
|
||||
|
||||
enum UFC {}
|
||||
|
||||
impl EventScraper for UFC {
|
||||
async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>> {
|
||||
pub async fn do_scraping<C: ClientScraper>(client: &C) {
|
||||
ONE::scrape(client).await;
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn hello() -> wreq::Result<()> {
|
||||
// Build a client
|
||||
@@ -62,36 +89,42 @@ pub async fn hello() -> wreq::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add(left: u64, right: u64) -> u64 {
|
||||
left + right
|
||||
static WHITESPACE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"\s+"#).unwrap());
|
||||
|
||||
fn text_content(elem: ElementRef<'_>, selector: &Selector) -> Option<String> {
|
||||
elem.select(selector).next().map(|e| {
|
||||
let tokens: Vec<String> = e
|
||||
.text()
|
||||
.map(|t| WHITESPACE.replace_all(t, " ").to_string())
|
||||
.collect();
|
||||
|
||||
tokens.join(" ").trim().to_string()
|
||||
})
|
||||
}
|
||||
|
||||
fn yes() {
|
||||
let html = r#"
|
||||
<ul>
|
||||
<li>Foo</li>
|
||||
<li>Bar</li>
|
||||
<li>Baz</li>
|
||||
</ul>
|
||||
"#;
|
||||
#[derive(Debug)]
|
||||
struct StrError(String);
|
||||
|
||||
let fragment = Html::parse_fragment(html);
|
||||
let ul_selector = Selector::parse("ul").unwrap();
|
||||
let li_selector = Selector::parse("li").unwrap();
|
||||
impl StrError {
|
||||
fn new<T: Into<String>>(value: T) -> StrError {
|
||||
StrError(value.into())
|
||||
}
|
||||
}
|
||||
|
||||
let ul = fragment.select(&ul_selector).next().unwrap();
|
||||
for element in ul.select(&li_selector) {
|
||||
assert_eq!("li", element.value().name());
|
||||
impl Display for StrError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.0);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for StrError {
|
||||
fn description(&self) -> &str {
|
||||
self.0.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn it_works() {
|
||||
let result = add(2, 2);
|
||||
assert_eq!(result, 4);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,44 +1,30 @@
|
||||
use anyhow::Result;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use anyhow::{Ok, Result};
|
||||
use scraper::{ElementRef, Selector, selectable::Selectable};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::Deserialize;
|
||||
use time_rs::OffsetDateTime;
|
||||
|
||||
use crate::{ClientScraper, Event, EventScraper, Fight, Fighter};
|
||||
use crate::{ClientScraper, Event, EventScraper, Fight, Fighter, text_content};
|
||||
|
||||
pub enum ONE {}
|
||||
|
||||
const URL: &str = "https://www.onefc.com/events/";
|
||||
static EVENT_LIST: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse("#upcoming-events-section .is-event").unwrap());
|
||||
|
||||
impl EventScraper for ONE {
|
||||
async fn scrape<C: ClientScraper>(client: &C) -> Result<Vec<Event>> {
|
||||
let page = client.get("https://www.onefc.com/events/").await?;
|
||||
let page = client.get(URL).await?;
|
||||
let mut events = Vec::new();
|
||||
|
||||
// should panic if selectors are invalid
|
||||
let selector_event = Selector::parse("#upcoming-events-section .is-event").unwrap();
|
||||
let selector_check = Selector::parse("a.smart-link").unwrap();
|
||||
let selector_event_url = Selector::parse("a.is-image-zoom[href]").unwrap();
|
||||
|
||||
for event_fragment in page.select(&selector_event) {
|
||||
do_thing(client, event_fragment).await;
|
||||
if event_fragment
|
||||
.select(&selector_check)
|
||||
.next()
|
||||
.map(|e| e.attr("href"))
|
||||
.flatten()
|
||||
.is_none()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(url) = event_fragment
|
||||
.select(&selector_event_url)
|
||||
.next()
|
||||
.map(|e| e.attr("href"))
|
||||
.flatten()
|
||||
else {
|
||||
continue;
|
||||
for elem in page.select(&EVENT_LIST) {
|
||||
if let Some(event) = get_event(client, elem).await {
|
||||
events.push(event);
|
||||
};
|
||||
}
|
||||
|
||||
todo!()
|
||||
Ok(events)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,30 +44,167 @@ struct EventData {
|
||||
start_date: OffsetDateTime,
|
||||
}
|
||||
|
||||
async fn do_thing<C: ClientScraper>(client: &C, elem: ElementRef<'_>) -> Option<Vec<Event>> {
|
||||
let selector_check = Selector::parse("a.smart-link").unwrap();
|
||||
let selector_event_url = Selector::parse("a.is-image-zoom[href]").unwrap();
|
||||
let selector_data = Selector::parse("#site-main script[type=\"application/ld+json\"]").unwrap();
|
||||
static EVENT_CHECK: LazyLock<Selector> = LazyLock::new(|| Selector::parse("a.smart-link").unwrap());
|
||||
static EVENT_URL: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse("a.is-image-zoom[href]").unwrap());
|
||||
static EVENT_DATA: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse("#site-main script[type=\"application/ld+json\"]").unwrap());
|
||||
static EVENT_FIGHTS: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse(".event-matchup").unwrap());
|
||||
|
||||
let mut events = Vec::new();
|
||||
|
||||
elem.select(&selector_check)
|
||||
async fn get_event<C: ClientScraper>(client: &C, elem: ElementRef<'_>) -> Option<Event> {
|
||||
elem.select(&EVENT_CHECK)
|
||||
.next()
|
||||
.map(|e| e.attr("href"))
|
||||
.flatten()?;
|
||||
|
||||
let url = elem
|
||||
.select(&selector_event_url)
|
||||
.select(&EVENT_URL)
|
||||
.next()
|
||||
.map(|e| e.attr("href"))
|
||||
.flatten()?;
|
||||
|
||||
let page = client.get(url).await.ok()?;
|
||||
|
||||
let data_elem = page.select(&selector_data).next()?;
|
||||
let data_elem = page.select(&EVENT_DATA).next()?;
|
||||
let data_str: String = data_elem.text().collect();
|
||||
let data: EventData = serde_json::from_str(&data_str).ok()?;
|
||||
data.start_date.unix_timestamp();
|
||||
|
||||
Some(events)
|
||||
let mut fights = Vec::new();
|
||||
|
||||
for elem in page.select(&EVENT_FIGHTS) {
|
||||
if let Some(fight) = get_fight(elem) {
|
||||
fights.push(fight);
|
||||
}
|
||||
}
|
||||
|
||||
if fights.len() == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let event = Event {
|
||||
organization: "ONE".into(),
|
||||
|
||||
name: data.name,
|
||||
location: data.location.name,
|
||||
date: data.start_date.unix_timestamp(),
|
||||
image: data
|
||||
.image
|
||||
.get(2)
|
||||
.map(|s| s.clone())
|
||||
.unwrap_or_else(String::default),
|
||||
|
||||
url: url.to_string(),
|
||||
slug: url.replace(URL, "").replace("/", ""),
|
||||
|
||||
fights,
|
||||
};
|
||||
|
||||
Some(event)
|
||||
}
|
||||
|
||||
static FIGHT_WEIGHT: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".title").unwrap());
|
||||
|
||||
fn get_fight(elem: ElementRef<'_>) -> Option<Fight> {
|
||||
let fighter_a = get_fighter::<true>(elem)?;
|
||||
let fighter_b = get_fighter::<false>(elem)?;
|
||||
let weight = text_content(elem, &FIGHT_WEIGHT)?;
|
||||
Some(Fight {
|
||||
weight,
|
||||
fighter_a,
|
||||
fighter_b,
|
||||
})
|
||||
}
|
||||
|
||||
static FIGHTER_A_NAME: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse("tr.vs :nth-child(1) a").unwrap());
|
||||
static FIGHTER_A_COUNTRY: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse("tr.vs + tr :nth-child(1)").unwrap());
|
||||
static FIGHTER_A_IMAGE: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse(".face1 img[src]").unwrap());
|
||||
|
||||
static FIGHTER_B_NAME: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse("tr.vs :nth-child(3) a").unwrap());
|
||||
static FIGHTER_B_COUNTRY: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse("tr.vs + tr :nth-child(3)").unwrap());
|
||||
static FIGHTER_B_IMAGE: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse(".face2 img[src]").unwrap());
|
||||
|
||||
fn get_fighter<const IS_A: bool>(elem: ElementRef<'_>) -> Option<Fighter> {
|
||||
let name = text_content(
|
||||
elem,
|
||||
if IS_A {
|
||||
&FIGHTER_A_NAME
|
||||
} else {
|
||||
&FIGHTER_B_NAME
|
||||
},
|
||||
)?;
|
||||
|
||||
let country = text_content(
|
||||
elem,
|
||||
if IS_A {
|
||||
&FIGHTER_A_COUNTRY
|
||||
} else {
|
||||
&FIGHTER_B_COUNTRY
|
||||
},
|
||||
)?;
|
||||
|
||||
let image = elem
|
||||
.select(if IS_A {
|
||||
&FIGHTER_A_IMAGE
|
||||
} else {
|
||||
&FIGHTER_B_IMAGE
|
||||
})
|
||||
.next()
|
||||
.map(|e| e.attr("src"))
|
||||
.flatten()
|
||||
.map(|s| s.to_string())?;
|
||||
|
||||
Some(Fighter {
|
||||
name,
|
||||
image,
|
||||
country,
|
||||
link: "".into(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use std::{collections::HashMap, sync::LazyLock};
|
||||
|
||||
use crate::DummyClient;
|
||||
|
||||
static CLIENT: LazyLock<DummyClient> = LazyLock::new(|| {
|
||||
DummyClient::new(HashMap::from([
|
||||
(
|
||||
"https://www.onefc.com/events/",
|
||||
include_str!("./testdata/one-event-list.html"),
|
||||
),
|
||||
(
|
||||
"https://www.onefc.com/events/one-friday-fights-97/",
|
||||
include_str!("./testdata/one-friday-fights-97.html"),
|
||||
),
|
||||
(
|
||||
"https://www.onefc.com/events/one171/",
|
||||
include_str!("./testdata/one-171.html"),
|
||||
),
|
||||
(
|
||||
"https://www.onefc.com/events/onefightnight29/",
|
||||
include_str!("./testdata/one-fn-29.html"),
|
||||
),
|
||||
(
|
||||
"https://www.onefc.com/events/one172/",
|
||||
include_str!("./testdata/one-172.html"),
|
||||
),
|
||||
]))
|
||||
});
|
||||
|
||||
#[tokio::test]
|
||||
async fn event_list() {
|
||||
let events = ONE::scrape(&*CLIENT).await.unwrap();
|
||||
assert!(events.len() == 4);
|
||||
println!("{}", serde_json::to_string_pretty(&events).unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
+2427
File diff suppressed because it is too large
Load Diff
+2186
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+1114
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
+5600
File diff suppressed because it is too large
Load Diff
+4123
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user