guy176251/4get
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- refactored scraper filter function

Browse Source 
- added ability to specify default scrapers in environment variables
This commit is contained in:
long
2026-05-15 23:44:01 -05:00
parent b13dceaeb5
commit 9c2921c7fa
3 changed files with 133 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/config.php
+7
View File
@@ -23,6 +23,13 @@ class config{
// Enable the API?
const API_ENABLED = true;
// Default scrapers
const SCRAPER_WEB = "ddg";
const SCRAPER_IMAGES = "ddg";
const SCRAPER_NEWS = "ddg";
const SCRAPER_VIDEOS = "yt";
const SCRAPER_MUSIC = "sc";
//
// BOT PROTECTION
//
lib/frontend.php
+9 -149
View File
@@ -593,7 +593,7 @@ class frontend{
public function getscraperfilters($page){
$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : null;
$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : "";
if(
isset($_GET["scraper"]) &&
@@ -619,162 +619,22 @@ class frontend{
}
}
include "lib/scrapers.php";
// add search field
$filters =
[
"s" => [
"option" => "_SEARCH"
]
],
"scraper" => [
"display" => "Scraper",
"option" => AVAILABLE_SCRAPERS[$page],
],
];
// define default scrapers
switch($page){
case "web":
$filters["scraper"] = [
"display" => "Scraper",
"option" => [
"ddg" => "DuckDuckGo",
//"yahoo" => "Yahoo!",
"brave" => "Brave",
"yandex" => "Yandex",
"google" => "Google",
"google_api" => "Google API",
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"ghostery" => "Ghostery",
"yep" => "Yep",
"mwmbl" => "Mwmbl",
"mojeek" => "Mojeek",
"baidu" => "Baidu",
"coccoc" => "Cốc Cốc",
"solofield" => "Solofield",
"marginalia" => "Marginalia",
"wiby" => "wiby"
]
];
break;
case "images":
$filters["scraper"] = [
"display" => "Scraper",
"option" => [
"ddg" => "DuckDuckGo",
"yandex" => "Yandex",
"brave" => "Brave",
"google" => "Google",
"google_api" => "Google API",
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"baidu" => "Baidu",
"solofield" => "Solofield",
"pinterest" => "Pinterest",
"cara" => "Cara",
"flickr" => "Flickr",
"pexels" => "Pexels",
"pixabay" => "Pixabay",
"unsplash" => "Unsplash",
"fivehpx" => "500px",
"vsco" => "VSCO",
"imgur" => "Imgur",
"ftm" => "FindThatMeme"
]
];
break;
case "videos":
$filters["scraper"] = [
"display" => "Scraper",
"option" => [
"yt" => "YouTube",
//"archiveorg" => "Archive.org",
"vimeo" => "Vimeo",
//"odysee" => "Odysee",
"sepiasearch" => "Sepia Search",
//"fb" => "Facebook videos",
"ddg" => "DuckDuckGo",
"brave" => "Brave",
"yandex" => "Yandex",
"google" => "Google",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"baidu" => "Baidu",
"coccoc" => "Cốc Cốc",
"solofield" => "Solofield"
]
];
break;
case "news":
$filters["scraper"] = [
"display" => "Scraper",
"option" => [
"ddg" => "DuckDuckGo",
"brave" => "Brave",
"google" => "Google",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"mojeek" => "Mojeek",
"baidu" => "Baidu"
]
];
break;
case "music":
$filters["scraper"] = [
"display" => "Scraper",
"option" => [
"sc" => "SoundCloud",
"swisscows" => "Swisscows (SoundCloud)"
//"spotify" => "Spotify"
]
];
break;
case "booru":
$filters["scraper"] = [
"display" => "Scraper",
"option" => [
"safebooru" => "Safebooru",
"konachan" => "Konachan",
"tbib" => "The Big Imageboard",
"gelbooru" => "Gelbooru",
"yandere" => "Yande.re",
"tbib" => "The Big Imageboard",
"sankakucomplex" => "SankakuComplex",
"soybooru" => "SoyBooru"
]
];
break;
}
// get scraper name from user input, or default out to preferred scraper
$scraper_out = null;
$first = true;
foreach($filters["scraper"]["option"] as $scraper_name => $scraper_pretty){
if($first === true){
$first = $scraper_name;
}
if($scraper_name == $get_scraper){
$scraper_out = $scraper_name;
}
}
if($scraper_out === null){
$scraper_out = $first;
}
$scraper_out = get_scraper($page, $get_scraper);
include "scraper/$scraper_out.php";
$lib = new $scraper_out();
lib/scrapers.php
+115
View File
@@ -0,0 +1,115 @@
<?php
const DEFAULT_SCRAPERS = [
"web" => config::SCRAPER_WEB,
"images" => config::SCRAPER_IMAGES,
"news" => config::SCRAPER_NEWS,
"videos" => config::SCRAPER_VIDEOS,
"music" => config::SCRAPER_MUSIC,
];
const AVAILABLE_SCRAPERS = [
"web" => [
"ddg" => "DuckDuckGo",
//"yahoo" => "Yahoo!",
"brave" => "Brave",
"yandex" => "Yandex",
"google" => "Google",
"google_api" => "Google API",
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"ghostery" => "Ghostery",
"yep" => "Yep",
"mwmbl" => "Mwmbl",
"mojeek" => "Mojeek",
"baidu" => "Baidu",
"coccoc" => "Cốc Cốc",
"solofield" => "Solofield",
"marginalia" => "Marginalia",
"wiby" => "wiby"
],
"images" => [
"ddg" => "DuckDuckGo",
"yandex" => "Yandex",
"brave" => "Brave",
"google" => "Google",
"google_api" => "Google API",
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"baidu" => "Baidu",
"solofield" => "Solofield",
"pinterest" => "Pinterest",
"cara" => "Cara",
"flickr" => "Flickr",
"pexels" => "Pexels",
"pixabay" => "Pixabay",
"unsplash" => "Unsplash",
"fivehpx" => "500px",
"vsco" => "VSCO",
"imgur" => "Imgur",
"ftm" => "FindThatMeme"
],
"videos" => [
"yt" => "YouTube",
//"archiveorg" => "Archive.org",
"vimeo" => "Vimeo",
//"odysee" => "Odysee",
"sepiasearch" => "Sepia Search",
//"fb" => "Facebook videos",
"ddg" => "DuckDuckGo",
"brave" => "Brave",
"yandex" => "Yandex",
"google" => "Google",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"baidu" => "Baidu",
"coccoc" => "Cốc Cốc",
"solofield" => "Solofield"
],
"news" => [
"ddg" => "DuckDuckGo",
"brave" => "Brave",
"google" => "Google",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"mojeek" => "Mojeek",
"baidu" => "Baidu"
],
"music" => [
"sc" => "SoundCloud",
"swisscows" => "Swisscows (SoundCloud)",
//"spotify" => "Spotify"
],
"booru" => [
"safebooru" => "Safebooru",
"konachan" => "Konachan",
"tbib" => "The Big Imageboard",
"gelbooru" => "Gelbooru",
"yandere" => "Yande.re",
"sankakucomplex" => "SankakuComplex",
"soybooru" => "SoyBooru",
],
];
function get_scraper(string $page, string $selected): string
{
$available = AVAILABLE_SCRAPERS[$page];
if (!$available) {
return "__NONEXISTENT__";
}
if ($available[$selected]) {
return $selected;
}
$default = DEFAULT_SCRAPERS[$page];
return $available[$default] ? $default : array_key_first($available);
}