From 9c2921c7fa829b8de04965fd167850b21d7d11a4 Mon Sep 17 00:00:00 2001 From: long Date: Fri, 15 May 2026 23:44:01 -0500 Subject: [PATCH] - refactored scraper filter function - added ability to specify default scrapers in environment variables --- data/config.php | 7 ++ lib/frontend.php | 162 ++++------------------------------------------- lib/scrapers.php | 115 +++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 151 deletions(-) create mode 100644 lib/scrapers.php diff --git a/data/config.php b/data/config.php index 161458f..62aed90 100644 --- a/data/config.php +++ b/data/config.php @@ -22,6 +22,13 @@ class config{ // Enable the API? const API_ENABLED = true; + + // Default scrapers + const SCRAPER_WEB = "ddg"; + const SCRAPER_IMAGES = "ddg"; + const SCRAPER_NEWS = "ddg"; + const SCRAPER_VIDEOS = "yt"; + const SCRAPER_MUSIC = "sc"; // // BOT PROTECTION diff --git a/lib/frontend.php b/lib/frontend.php index 8f2f20b..6d4a7d1 100644 --- a/lib/frontend.php +++ b/lib/frontend.php @@ -593,7 +593,7 @@ class frontend{ public function getscraperfilters($page){ - $get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : null; + $get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : ""; if( isset($_GET["scraper"]) && @@ -619,163 +619,23 @@ class frontend{ } } + include "lib/scrapers.php"; + // add search field $filters = [ "s" => [ "option" => "_SEARCH" - ] + ], + "scraper" => [ + "display" => "Scraper", + "option" => AVAILABLE_SCRAPERS[$page], + ], ]; - - // define default scrapers - switch($page){ - - case "web": - $filters["scraper"] = [ - "display" => "Scraper", - "option" => [ - "ddg" => "DuckDuckGo", - //"yahoo" => "Yahoo!", - "brave" => "Brave", - "yandex" => "Yandex", - "google" => "Google", - "google_api" => "Google API", - "google_cse" => "Google CSE", - "yahoo_japan" => "Yahoo! JAPAN", - "startpage" => "Startpage", - "qwant" => "Qwant", - "ghostery" => "Ghostery", - "yep" => "Yep", - "mwmbl" => "Mwmbl", - "mojeek" => "Mojeek", - "baidu" => "Baidu", - "coccoc" => "Cốc Cốc", - "solofield" => "Solofield", - "marginalia" => "Marginalia", - "wiby" => "wiby" - ] - ]; - break; - - case "images": - $filters["scraper"] = [ - "display" => "Scraper", - "option" => [ - "ddg" => "DuckDuckGo", - "yandex" => "Yandex", - "brave" => "Brave", - "google" => "Google", - "google_api" => "Google API", - "google_cse" => "Google CSE", - "yahoo_japan" => "Yahoo! JAPAN", - "startpage" => "Startpage", - "qwant" => "Qwant", - "baidu" => "Baidu", - "solofield" => "Solofield", - "pinterest" => "Pinterest", - "cara" => "Cara", - "flickr" => "Flickr", - "pexels" => "Pexels", - "pixabay" => "Pixabay", - "unsplash" => "Unsplash", - "fivehpx" => "500px", - "vsco" => "VSCO", - "imgur" => "Imgur", - "ftm" => "FindThatMeme" - ] - ]; - break; - - case "videos": - $filters["scraper"] = [ - "display" => "Scraper", - "option" => [ - "yt" => "YouTube", - //"archiveorg" => "Archive.org", - "vimeo" => "Vimeo", - //"odysee" => "Odysee", - "sepiasearch" => "Sepia Search", - //"fb" => "Facebook videos", - "ddg" => "DuckDuckGo", - "brave" => "Brave", - "yandex" => "Yandex", - "google" => "Google", - "yahoo_japan" => "Yahoo! JAPAN", - "startpage" => "Startpage", - "qwant" => "Qwant", - "baidu" => "Baidu", - "coccoc" => "Cốc Cốc", - "solofield" => "Solofield" - ] - ]; - break; - - case "news": - $filters["scraper"] = [ - "display" => "Scraper", - "option" => [ - "ddg" => "DuckDuckGo", - "brave" => "Brave", - "google" => "Google", - "yahoo_japan" => "Yahoo! JAPAN", - "startpage" => "Startpage", - "qwant" => "Qwant", - "mojeek" => "Mojeek", - "baidu" => "Baidu" - ] - ]; - break; - - case "music": - $filters["scraper"] = [ - "display" => "Scraper", - "option" => [ - "sc" => "SoundCloud", - "swisscows" => "Swisscows (SoundCloud)" - //"spotify" => "Spotify" - ] - ]; - break; - - case "booru": - $filters["scraper"] = [ - "display" => "Scraper", - "option" => [ - "safebooru" => "Safebooru", - "konachan" => "Konachan", - "tbib" => "The Big Imageboard", - "gelbooru" => "Gelbooru", - "yandere" => "Yande.re", - "tbib" => "The Big Imageboard", - "sankakucomplex" => "SankakuComplex", - "soybooru" => "SoyBooru" - ] - ]; - break; - } - + // get scraper name from user input, or default out to preferred scraper - $scraper_out = null; - $first = true; - - foreach($filters["scraper"]["option"] as $scraper_name => $scraper_pretty){ - - if($first === true){ - - $first = $scraper_name; - } - - if($scraper_name == $get_scraper){ - - $scraper_out = $scraper_name; - } - } - - if($scraper_out === null){ - - $scraper_out = $first; - } - + $scraper_out = get_scraper($page, $get_scraper); + include "scraper/$scraper_out.php"; $lib = new $scraper_out(); diff --git a/lib/scrapers.php b/lib/scrapers.php new file mode 100644 index 0000000..e3fa298 --- /dev/null +++ b/lib/scrapers.php @@ -0,0 +1,115 @@ + config::SCRAPER_WEB, + "images" => config::SCRAPER_IMAGES, + "news" => config::SCRAPER_NEWS, + "videos" => config::SCRAPER_VIDEOS, + "music" => config::SCRAPER_MUSIC, +]; + +const AVAILABLE_SCRAPERS = [ + "web" => [ + "ddg" => "DuckDuckGo", + //"yahoo" => "Yahoo!", + "brave" => "Brave", + "yandex" => "Yandex", + "google" => "Google", + "google_api" => "Google API", + "google_cse" => "Google CSE", + "yahoo_japan" => "Yahoo! JAPAN", + "startpage" => "Startpage", + "qwant" => "Qwant", + "ghostery" => "Ghostery", + "yep" => "Yep", + "mwmbl" => "Mwmbl", + "mojeek" => "Mojeek", + "baidu" => "Baidu", + "coccoc" => "Cốc Cốc", + "solofield" => "Solofield", + "marginalia" => "Marginalia", + "wiby" => "wiby" + ], + "images" => [ + "ddg" => "DuckDuckGo", + "yandex" => "Yandex", + "brave" => "Brave", + "google" => "Google", + "google_api" => "Google API", + "google_cse" => "Google CSE", + "yahoo_japan" => "Yahoo! JAPAN", + "startpage" => "Startpage", + "qwant" => "Qwant", + "baidu" => "Baidu", + "solofield" => "Solofield", + "pinterest" => "Pinterest", + "cara" => "Cara", + "flickr" => "Flickr", + "pexels" => "Pexels", + "pixabay" => "Pixabay", + "unsplash" => "Unsplash", + "fivehpx" => "500px", + "vsco" => "VSCO", + "imgur" => "Imgur", + "ftm" => "FindThatMeme" + ], + + "videos" => [ + "yt" => "YouTube", + //"archiveorg" => "Archive.org", + "vimeo" => "Vimeo", + //"odysee" => "Odysee", + "sepiasearch" => "Sepia Search", + //"fb" => "Facebook videos", + "ddg" => "DuckDuckGo", + "brave" => "Brave", + "yandex" => "Yandex", + "google" => "Google", + "yahoo_japan" => "Yahoo! JAPAN", + "startpage" => "Startpage", + "qwant" => "Qwant", + "baidu" => "Baidu", + "coccoc" => "Cốc Cốc", + "solofield" => "Solofield" + ], + + "news" => [ + "ddg" => "DuckDuckGo", + "brave" => "Brave", + "google" => "Google", + "yahoo_japan" => "Yahoo! JAPAN", + "startpage" => "Startpage", + "qwant" => "Qwant", + "mojeek" => "Mojeek", + "baidu" => "Baidu" + ], + + "music" => [ + "sc" => "SoundCloud", + "swisscows" => "Swisscows (SoundCloud)", + //"spotify" => "Spotify" + ], + + "booru" => [ + "safebooru" => "Safebooru", + "konachan" => "Konachan", + "tbib" => "The Big Imageboard", + "gelbooru" => "Gelbooru", + "yandere" => "Yande.re", + "sankakucomplex" => "SankakuComplex", + "soybooru" => "SoyBooru", + ], +]; + +function get_scraper(string $page, string $selected): string +{ + $available = AVAILABLE_SCRAPERS[$page]; + if (!$available) { + return "__NONEXISTENT__"; + } + if ($available[$selected]) { + return $selected; + } + $default = DEFAULT_SCRAPERS[$page]; + return $available[$default] ? $default : array_key_first($available); +}