- refactored scraper filter function
- added ability to specify default scrapers in environment variables
This commit is contained in:
@@ -23,6 +23,13 @@ class config{
|
|||||||
// Enable the API?
|
// Enable the API?
|
||||||
const API_ENABLED = true;
|
const API_ENABLED = true;
|
||||||
|
|
||||||
|
// Default scrapers
|
||||||
|
const SCRAPER_WEB = "ddg";
|
||||||
|
const SCRAPER_IMAGES = "ddg";
|
||||||
|
const SCRAPER_NEWS = "ddg";
|
||||||
|
const SCRAPER_VIDEOS = "yt";
|
||||||
|
const SCRAPER_MUSIC = "sc";
|
||||||
|
|
||||||
//
|
//
|
||||||
// BOT PROTECTION
|
// BOT PROTECTION
|
||||||
//
|
//
|
||||||
|
|||||||
+9
-149
@@ -593,7 +593,7 @@ class frontend{
|
|||||||
|
|
||||||
public function getscraperfilters($page){
|
public function getscraperfilters($page){
|
||||||
|
|
||||||
$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : null;
|
$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : "";
|
||||||
|
|
||||||
if(
|
if(
|
||||||
isset($_GET["scraper"]) &&
|
isset($_GET["scraper"]) &&
|
||||||
@@ -619,162 +619,22 @@ class frontend{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
include "lib/scrapers.php";
|
||||||
|
|
||||||
// add search field
|
// add search field
|
||||||
$filters =
|
$filters =
|
||||||
[
|
[
|
||||||
"s" => [
|
"s" => [
|
||||||
"option" => "_SEARCH"
|
"option" => "_SEARCH"
|
||||||
]
|
],
|
||||||
|
"scraper" => [
|
||||||
|
"display" => "Scraper",
|
||||||
|
"option" => AVAILABLE_SCRAPERS[$page],
|
||||||
|
],
|
||||||
];
|
];
|
||||||
|
|
||||||
// define default scrapers
|
|
||||||
switch($page){
|
|
||||||
|
|
||||||
case "web":
|
|
||||||
$filters["scraper"] = [
|
|
||||||
"display" => "Scraper",
|
|
||||||
"option" => [
|
|
||||||
"ddg" => "DuckDuckGo",
|
|
||||||
//"yahoo" => "Yahoo!",
|
|
||||||
"brave" => "Brave",
|
|
||||||
"yandex" => "Yandex",
|
|
||||||
"google" => "Google",
|
|
||||||
"google_api" => "Google API",
|
|
||||||
"google_cse" => "Google CSE",
|
|
||||||
"yahoo_japan" => "Yahoo! JAPAN",
|
|
||||||
"startpage" => "Startpage",
|
|
||||||
"qwant" => "Qwant",
|
|
||||||
"ghostery" => "Ghostery",
|
|
||||||
"yep" => "Yep",
|
|
||||||
"mwmbl" => "Mwmbl",
|
|
||||||
"mojeek" => "Mojeek",
|
|
||||||
"baidu" => "Baidu",
|
|
||||||
"coccoc" => "Cốc Cốc",
|
|
||||||
"solofield" => "Solofield",
|
|
||||||
"marginalia" => "Marginalia",
|
|
||||||
"wiby" => "wiby"
|
|
||||||
]
|
|
||||||
];
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "images":
|
|
||||||
$filters["scraper"] = [
|
|
||||||
"display" => "Scraper",
|
|
||||||
"option" => [
|
|
||||||
"ddg" => "DuckDuckGo",
|
|
||||||
"yandex" => "Yandex",
|
|
||||||
"brave" => "Brave",
|
|
||||||
"google" => "Google",
|
|
||||||
"google_api" => "Google API",
|
|
||||||
"google_cse" => "Google CSE",
|
|
||||||
"yahoo_japan" => "Yahoo! JAPAN",
|
|
||||||
"startpage" => "Startpage",
|
|
||||||
"qwant" => "Qwant",
|
|
||||||
"baidu" => "Baidu",
|
|
||||||
"solofield" => "Solofield",
|
|
||||||
"pinterest" => "Pinterest",
|
|
||||||
"cara" => "Cara",
|
|
||||||
"flickr" => "Flickr",
|
|
||||||
"pexels" => "Pexels",
|
|
||||||
"pixabay" => "Pixabay",
|
|
||||||
"unsplash" => "Unsplash",
|
|
||||||
"fivehpx" => "500px",
|
|
||||||
"vsco" => "VSCO",
|
|
||||||
"imgur" => "Imgur",
|
|
||||||
"ftm" => "FindThatMeme"
|
|
||||||
]
|
|
||||||
];
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "videos":
|
|
||||||
$filters["scraper"] = [
|
|
||||||
"display" => "Scraper",
|
|
||||||
"option" => [
|
|
||||||
"yt" => "YouTube",
|
|
||||||
//"archiveorg" => "Archive.org",
|
|
||||||
"vimeo" => "Vimeo",
|
|
||||||
//"odysee" => "Odysee",
|
|
||||||
"sepiasearch" => "Sepia Search",
|
|
||||||
//"fb" => "Facebook videos",
|
|
||||||
"ddg" => "DuckDuckGo",
|
|
||||||
"brave" => "Brave",
|
|
||||||
"yandex" => "Yandex",
|
|
||||||
"google" => "Google",
|
|
||||||
"yahoo_japan" => "Yahoo! JAPAN",
|
|
||||||
"startpage" => "Startpage",
|
|
||||||
"qwant" => "Qwant",
|
|
||||||
"baidu" => "Baidu",
|
|
||||||
"coccoc" => "Cốc Cốc",
|
|
||||||
"solofield" => "Solofield"
|
|
||||||
]
|
|
||||||
];
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "news":
|
|
||||||
$filters["scraper"] = [
|
|
||||||
"display" => "Scraper",
|
|
||||||
"option" => [
|
|
||||||
"ddg" => "DuckDuckGo",
|
|
||||||
"brave" => "Brave",
|
|
||||||
"google" => "Google",
|
|
||||||
"yahoo_japan" => "Yahoo! JAPAN",
|
|
||||||
"startpage" => "Startpage",
|
|
||||||
"qwant" => "Qwant",
|
|
||||||
"mojeek" => "Mojeek",
|
|
||||||
"baidu" => "Baidu"
|
|
||||||
]
|
|
||||||
];
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "music":
|
|
||||||
$filters["scraper"] = [
|
|
||||||
"display" => "Scraper",
|
|
||||||
"option" => [
|
|
||||||
"sc" => "SoundCloud",
|
|
||||||
"swisscows" => "Swisscows (SoundCloud)"
|
|
||||||
//"spotify" => "Spotify"
|
|
||||||
]
|
|
||||||
];
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "booru":
|
|
||||||
$filters["scraper"] = [
|
|
||||||
"display" => "Scraper",
|
|
||||||
"option" => [
|
|
||||||
"safebooru" => "Safebooru",
|
|
||||||
"konachan" => "Konachan",
|
|
||||||
"tbib" => "The Big Imageboard",
|
|
||||||
"gelbooru" => "Gelbooru",
|
|
||||||
"yandere" => "Yande.re",
|
|
||||||
"tbib" => "The Big Imageboard",
|
|
||||||
"sankakucomplex" => "SankakuComplex",
|
|
||||||
"soybooru" => "SoyBooru"
|
|
||||||
]
|
|
||||||
];
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
// get scraper name from user input, or default out to preferred scraper
|
// get scraper name from user input, or default out to preferred scraper
|
||||||
$scraper_out = null;
|
$scraper_out = get_scraper($page, $get_scraper);
|
||||||
$first = true;
|
|
||||||
|
|
||||||
foreach($filters["scraper"]["option"] as $scraper_name => $scraper_pretty){
|
|
||||||
|
|
||||||
if($first === true){
|
|
||||||
|
|
||||||
$first = $scraper_name;
|
|
||||||
}
|
|
||||||
|
|
||||||
if($scraper_name == $get_scraper){
|
|
||||||
|
|
||||||
$scraper_out = $scraper_name;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if($scraper_out === null){
|
|
||||||
|
|
||||||
$scraper_out = $first;
|
|
||||||
}
|
|
||||||
|
|
||||||
include "scraper/$scraper_out.php";
|
include "scraper/$scraper_out.php";
|
||||||
$lib = new $scraper_out();
|
$lib = new $scraper_out();
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
const DEFAULT_SCRAPERS = [
|
||||||
|
"web" => config::SCRAPER_WEB,
|
||||||
|
"images" => config::SCRAPER_IMAGES,
|
||||||
|
"news" => config::SCRAPER_NEWS,
|
||||||
|
"videos" => config::SCRAPER_VIDEOS,
|
||||||
|
"music" => config::SCRAPER_MUSIC,
|
||||||
|
];
|
||||||
|
|
||||||
|
const AVAILABLE_SCRAPERS = [
|
||||||
|
"web" => [
|
||||||
|
"ddg" => "DuckDuckGo",
|
||||||
|
//"yahoo" => "Yahoo!",
|
||||||
|
"brave" => "Brave",
|
||||||
|
"yandex" => "Yandex",
|
||||||
|
"google" => "Google",
|
||||||
|
"google_api" => "Google API",
|
||||||
|
"google_cse" => "Google CSE",
|
||||||
|
"yahoo_japan" => "Yahoo! JAPAN",
|
||||||
|
"startpage" => "Startpage",
|
||||||
|
"qwant" => "Qwant",
|
||||||
|
"ghostery" => "Ghostery",
|
||||||
|
"yep" => "Yep",
|
||||||
|
"mwmbl" => "Mwmbl",
|
||||||
|
"mojeek" => "Mojeek",
|
||||||
|
"baidu" => "Baidu",
|
||||||
|
"coccoc" => "Cốc Cốc",
|
||||||
|
"solofield" => "Solofield",
|
||||||
|
"marginalia" => "Marginalia",
|
||||||
|
"wiby" => "wiby"
|
||||||
|
],
|
||||||
|
"images" => [
|
||||||
|
"ddg" => "DuckDuckGo",
|
||||||
|
"yandex" => "Yandex",
|
||||||
|
"brave" => "Brave",
|
||||||
|
"google" => "Google",
|
||||||
|
"google_api" => "Google API",
|
||||||
|
"google_cse" => "Google CSE",
|
||||||
|
"yahoo_japan" => "Yahoo! JAPAN",
|
||||||
|
"startpage" => "Startpage",
|
||||||
|
"qwant" => "Qwant",
|
||||||
|
"baidu" => "Baidu",
|
||||||
|
"solofield" => "Solofield",
|
||||||
|
"pinterest" => "Pinterest",
|
||||||
|
"cara" => "Cara",
|
||||||
|
"flickr" => "Flickr",
|
||||||
|
"pexels" => "Pexels",
|
||||||
|
"pixabay" => "Pixabay",
|
||||||
|
"unsplash" => "Unsplash",
|
||||||
|
"fivehpx" => "500px",
|
||||||
|
"vsco" => "VSCO",
|
||||||
|
"imgur" => "Imgur",
|
||||||
|
"ftm" => "FindThatMeme"
|
||||||
|
],
|
||||||
|
|
||||||
|
"videos" => [
|
||||||
|
"yt" => "YouTube",
|
||||||
|
//"archiveorg" => "Archive.org",
|
||||||
|
"vimeo" => "Vimeo",
|
||||||
|
//"odysee" => "Odysee",
|
||||||
|
"sepiasearch" => "Sepia Search",
|
||||||
|
//"fb" => "Facebook videos",
|
||||||
|
"ddg" => "DuckDuckGo",
|
||||||
|
"brave" => "Brave",
|
||||||
|
"yandex" => "Yandex",
|
||||||
|
"google" => "Google",
|
||||||
|
"yahoo_japan" => "Yahoo! JAPAN",
|
||||||
|
"startpage" => "Startpage",
|
||||||
|
"qwant" => "Qwant",
|
||||||
|
"baidu" => "Baidu",
|
||||||
|
"coccoc" => "Cốc Cốc",
|
||||||
|
"solofield" => "Solofield"
|
||||||
|
],
|
||||||
|
|
||||||
|
"news" => [
|
||||||
|
"ddg" => "DuckDuckGo",
|
||||||
|
"brave" => "Brave",
|
||||||
|
"google" => "Google",
|
||||||
|
"yahoo_japan" => "Yahoo! JAPAN",
|
||||||
|
"startpage" => "Startpage",
|
||||||
|
"qwant" => "Qwant",
|
||||||
|
"mojeek" => "Mojeek",
|
||||||
|
"baidu" => "Baidu"
|
||||||
|
],
|
||||||
|
|
||||||
|
"music" => [
|
||||||
|
"sc" => "SoundCloud",
|
||||||
|
"swisscows" => "Swisscows (SoundCloud)",
|
||||||
|
//"spotify" => "Spotify"
|
||||||
|
],
|
||||||
|
|
||||||
|
"booru" => [
|
||||||
|
"safebooru" => "Safebooru",
|
||||||
|
"konachan" => "Konachan",
|
||||||
|
"tbib" => "The Big Imageboard",
|
||||||
|
"gelbooru" => "Gelbooru",
|
||||||
|
"yandere" => "Yande.re",
|
||||||
|
"sankakucomplex" => "SankakuComplex",
|
||||||
|
"soybooru" => "SoyBooru",
|
||||||
|
],
|
||||||
|
];
|
||||||
|
|
||||||
|
function get_scraper(string $page, string $selected): string
|
||||||
|
{
|
||||||
|
$available = AVAILABLE_SCRAPERS[$page];
|
||||||
|
if (!$available) {
|
||||||
|
return "__NONEXISTENT__";
|
||||||
|
}
|
||||||
|
if ($available[$selected]) {
|
||||||
|
return $selected;
|
||||||
|
}
|
||||||
|
$default = DEFAULT_SCRAPERS[$page];
|
||||||
|
return $available[$default] ? $default : array_key_first($available);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user