diff --git a/README.md b/README.md index 57a7569..5f68843 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ _NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._ ## Totally unbiased comparison between alternatives -| | 4get | searx(ng) | whoogle | degoog | -|----------------------------|-------------------------|-----------|------------|--------------------------------------| -| RAM usage | 100-400mb~ | 400mb-1GB | 100mb | 200mb-1GB | -| Does it suck | no (debunked by snopes) | yes | kind of? | its kinda cool but no search filters | -| Does it work | ye | lmao | shits dead | works right now... | +| | 4get | searx(ng) | whoogle | degoog | +|----------------------------|-------------------------|-----------|------------|-------------------------------------------| +| RAM usage | 100-400mb~ | 400mb-1GB | 100mb | 100mb-250mb | +| Does it suck | no (debunked by snopes) | yes | kind of? | hit and miss with search filters | +| Does it work | ye | lmao | shits dead | works $rightNow, it's actually kinda cool | ## Features 1. Rotating proxies on a per-scraper basis @@ -62,4 +62,4 @@ Refer to the will (at) lolcat.ca or create an issue. ## License -AGPL +AGPLv3-only diff --git a/audio/linear.php b/audio/linear.php deleted file mode 100644 index b6a848f..0000000 --- a/audio/linear.php +++ /dev/null @@ -1,20 +0,0 @@ -stream_linear_audio($_GET["s"]); -}catch(Exception $error){ - - header("X-Error: " . $error->getMessage()); -} diff --git a/audio/sc.php b/audio/sc.php deleted file mode 100644 index 6dfe66e..0000000 --- a/audio/sc.php +++ /dev/null @@ -1,224 +0,0 @@ -proxy = new proxy(); - - if(isset($_GET["u"])){ - - /* - we're now proxying audio - */ - $viewkey = $_GET["u"]; - - if(!isset($_GET["r"])){ - - $this->do404("Ranges(r) are missing"); - } - - $ranges = explode(",", $_GET["r"]); - - // sanitize ranges - foreach($ranges as &$range){ - - if(!is_numeric($range)){ - - $this->do404("Invalid range specified"); - } - - $range = (int)$range; - } - - // sort ranges (just to make sure) - sort($ranges); - - // convert ranges to pairs - $last = -1; - foreach($ranges as &$r){ - - $tmp = $r; - $r = [$last + 1, $r]; - - $last = $tmp; - } - - $browser_headers = getallheaders(); - - // get the requested range from client - $client_range = 0; - foreach($browser_headers as $key => $value){ - - if(strtolower($key) == "range"){ - - preg_match( - '/bytes=([0-9]+)/', - $value, - $client_regex - ); - - if(isset($client_regex[1])){ - - $client_range = (int)$client_regex[1]; - }else{ - - $client_range = 0; - } - break; - } - } - - if( - $client_range < 0 || - $client_range > $ranges[count($ranges) - 1][1] - ){ - - // range is not satisfiable - http_response_code(416); - header("Content-Type: text/plain"); - die(); - } - - $rng = null; - for($i=0; $iproxy->stream_linear_audio( - $viewkey - ); - }catch(Exception $error){ - - $this->do404("Could not read stream"); - } - - die(); - } - - /* - redirect user to correct resource - we need to scrape and store the byte positions in the result URL - */ - if(!isset($_GET["s"])){ - - $this->do404("The URL(s) parameter is missing"); - } - - $viewkey = $_GET["s"]; - - if( - preg_match( - '/soundcloud\.com$/', - parse_url($viewkey, PHP_URL_HOST) - ) === false - ){ - - $this->do404("This endpoint can only be used for soundcloud streams"); - } - - try{ - - $json = $this->proxy->get($viewkey)["body"]; - }catch(Exception $error){ - - $this->do404("Curl error: " . $error->getMessage()); - } - - $json = json_decode($json, true); - - if(!isset($json["url"])){ - - $this->do404("Could not get URL from JSON"); - } - - $viewkey = $json["url"]; - - $m3u8 = $this->proxy->get($viewkey)["body"]; - - $m3u8 = explode("\n", $m3u8); - - $lineout = null; - $streampos_arr = []; - foreach($m3u8 as $line){ - - $line = trim($line); - if($line[0] == "#"){ - - continue; - } - - if($lineout === null){ - $lineout = $line; - } - - preg_match( - '/\/media\/[0-9]+\/([0-9]+)\/([0-9]+)/', - $line, - $matches - ); - - if(isset($matches[0])){ - - $streampos_arr[] = [ - (int)$matches[1], - (int)$matches[2] - ]; - } - } - - if($lineout === null){ - - $this->do404("Could not get stream URL"); - } - - $lineout = - preg_replace( - '/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/', - '/media/$1/0/0', - $lineout - ); - - $streampos = []; - - foreach($streampos_arr as $pos){ - - $streampos[] = $pos[1]; - } - - $streampos = implode(",", $streampos); - - header("Location: /audio/sc?u=" . urlencode($lineout) . "&r=$streampos"); - header("Accept-Ranges: bytes"); - } - - private function do404($error){ - - http_response_code(404); - header("Content-Type: text/plain"); - header("X-Error: $error"); - die(); - } -} diff --git a/audio/seekable.php b/audio/seekable.php deleted file mode 100644 index b6a848f..0000000 --- a/audio/seekable.php +++ /dev/null @@ -1,20 +0,0 @@ -stream_linear_audio($_GET["s"]); -}catch(Exception $error){ - - header("X-Error: " . $error->getMessage()); -} diff --git a/audio/spotify.php b/audio/spotify.php deleted file mode 100644 index dc8fae6..0000000 --- a/audio/spotify.php +++ /dev/null @@ -1,214 +0,0 @@ -fuckhtml = new fuckhtml(); - - if( - !isset($_GET["s"]) || - !preg_match( - '/^(track|episode)\.([A-Za-z0-9]{22})$/', - $_GET["s"], - $matches - ) - ){ - - $this->do404("The track ID(s) parameter is missing or invalid"); - } - - try{ - - if($matches[1] == "episode"){ - - $uri = "show"; - }else{ - - $uri = $matches[1]; - } - - $embed = - $this->get("https://embed.spotify.com/{$uri}/" . $matches[2]); - }catch(Exception $error){ - - $this->do404("Failed to fetch embed data"); - } - - $this->fuckhtml->load($embed); - - $json = - $this->fuckhtml - ->getElementById( - "__NEXT_DATA__", - "script" - ); - - if($json === null){ - - $this->do404("Failed to extract JSON"); - } - - $json = - json_decode($json["innerHTML"], true); - - if($json === null){ - - $this->do404("Failed to decode JSON"); - } - - switch($matches[1]){ - - case "track": - if( - isset( - $json - ["props"] - ["pageProps"] - ["state"] - ["data"] - ["entity"] - ["audioPreview"] - ["url"] - ) - ){ - - header("Content-type: audio/mpeg"); - header( - "Location: /audio/linear?s=" . - urlencode( - $json - ["props"] - ["pageProps"] - ["state"] - ["data"] - ["entity"] - ["audioPreview"] - ["url"] - ) - ); - }else{ - - $this->do404("Could not extract playback URL"); - } - break; - - case "episode": - if( - isset( - $json - ["props"] - ["pageProps"] - ["state"] - ["data"] - ["entity"] - ["id"] - ) - ){ - - try{ - $json = - $this->get( - "https://spclient.wg.spotify.com/soundfinder/v1/unauth/episode/" . - $json - ["props"] - ["pageProps"] - ["state"] - ["data"] - ["entity"] - ["id"] . - "/com.widevine.alpha" - ); - }catch(Exception $error){ - - $this->do404("Failed to fetch audio resource"); - } - - $json = json_decode($json, true); - - if($json === null){ - - $this->do404("Failed to decode audio resource JSON"); - } - - if( - isset($json["passthrough"]) && - $json["passthrough"] == "ALLOWED" && - isset($json["passthroughUrl"]) - ){ - - header( - "Location:" . - "/audio/linear.php?s=" . - urlencode( - str_replace( - "http://", - "https://", - $json["passthroughUrl"] - ) - ) - ); - }else{ - - $this->do404("Failed to find passthroughUrl"); - } - - }else{ - - $this->do404("Failed to find episode ID"); - } - break; - } - } - - private function get($url){ - - $headers = [ - "User-Agent: " . config::USER_AGENT, - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", - "DNT: 1", - "Connection: keep-alive", - "Upgrade-Insecure-Requests: 1", - "Sec-Fetch-Dest: document", - "Sec-Fetch-Mode: navigate", - "Sec-Fetch-Site: none", - "Sec-Fetch-User: ?1" - ]; - - $curlproc = curl_init(); - - curl_setopt($curlproc, CURLOPT_URL, $url); - - curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding - curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); - - curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); - curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); - curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); - - $data = curl_exec($curlproc); - - if(curl_errno($curlproc)){ - throw new Exception(curl_error($curlproc)); - } - - curl_close($curlproc); - return $data; - } - - private function do404($error){ - - http_response_code(404); - header("Content-Type: text/plain"); - header("X-Error: $error"); - die(); - } -} diff --git a/data/api_keys/yep.txt b/data/api_keys/yep.txt new file mode 100644 index 0000000..80f6458 --- /dev/null +++ b/data/api_keys/yep.txt @@ -0,0 +1 @@ +# Paste Yep API keys here diff --git a/data/config.php b/data/config.php index 62aed90..21a51de 100644 --- a/data/config.php +++ b/data/config.php @@ -30,6 +30,16 @@ class config{ const SCRAPER_VIDEOS = "yt"; const SCRAPER_MUSIC = "sc"; + // + // 4play (session provider) + // + // Enable 4play API? + const FPLAY_ENABLE_API = true; + + // 4play password. Please set this to something secure if you enable the 4play API. + // This password is used to POST sessions to /api/v2/provide_sesh + const FPLAY_PASSWORD = "1234"; + // // BOT PROTECTION // @@ -125,10 +135,10 @@ class config{ // Default user agent to use for scraper requests. Sometimes ignored to get specific webpages // Changing this might break things. - const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0"; + const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:151.0) Gecko/20100101 Firefox/151.0"; // User agent to use with 4get-friendly APIs - const USER_AGENT_FRIENDLY = "4get-scrapist"; + const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)"; // Proxy pool assignments for each scraper // false = Use server's raw IP @@ -138,7 +148,6 @@ class config{ const PROXY_YAHOO = false; const PROXY_YAHOO_JAPAN = false; const PROXY_BRAVE = false; - const PROXY_FB = false; // facebook const PROXY_GOOGLE = false; const PROXY_GOOGLE_API = false; const PROXY_GOOGLE_CSE = false; @@ -162,7 +171,6 @@ class config{ const PROXY_VIMEO = false; const PROXY_YEP = false; const PROXY_PINTEREST = false; - const PROXY_SANKAKUCOMPLEX = false; const PROXY_FLICKR = false; const PROXY_PIXABAY = false; const PROXY_UNSPLASH = false; @@ -171,8 +179,6 @@ class config{ const PROXY_VSCO = false; const PROXY_SEZNAM = false; const PROXY_NAVER = false; - const PROXY_GREPPR = false; - const PROXY_CROWDVIEW = false; const PROXY_MWMBL = false; const PROXY_FTM = false; // findthatmeme const PROXY_IMGUR = false; @@ -180,6 +186,11 @@ class config{ const PROXY_YANDEX_W = false; // yandex web const PROXY_YANDEX_I = false; // yandex images const PROXY_YANDEX_V = false; // yandex videos + const PROXY_SAFEBOORU = false; + const PROXY_KONACHAN = false; + const PROXY_YANDERE = false; + const PROXY_TBIB = false; + const PROXY_GELBOORU = false; // // Scraper-specific parameters @@ -192,4 +203,7 @@ class config{ // Use "null" to default out to HTML scraping OR specify a string to // use the API (Eg: "public"). API has less filters. const MARGINALIA_API_KEY = null; + + // Yep + const YEP_USE_API = false; } diff --git a/docs/apache2-example.md b/docs/apache2-example.md index 09f5c1d..a6115b1 100644 --- a/docs/apache2-example.md +++ b/docs/apache2-example.md @@ -49,9 +49,13 @@ Looking for the apache2 guide? + Require all denied + + + # remove directory listing from /icons + + Options -Indexes diff --git a/docs/apache2.md b/docs/apache2.md index ea337d6..7e545b3 100644 --- a/docs/apache2.md +++ b/docs/apache2.md @@ -104,9 +104,13 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r # deny access to private resources - - Order Deny,allow - Deny from all + + Require all denied + + + # remove directory listing from /icons + + Options -Indexes ``` diff --git a/docs/develop.md b/docs/develop.md new file mode 100644 index 0000000..95b8517 --- /dev/null +++ b/docs/develop.md @@ -0,0 +1,7 @@ +# Develop + +Start the PHP webserver like this, assuming you have curl-impersonate installed already + +```sh +LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000 +``` diff --git a/scraper/baidu.php b/scraper/baidu.php index efb14ca..b81c224 100644 --- a/scraper/baidu.php +++ b/scraper/baidu.php @@ -666,14 +666,24 @@ class baidu{ // // Don't parse as a search result if it's a card // - $card = + $tpl_probe = $this->fuckhtml - ->getElementsByClassName( - "cosc-card", - $div + ->getElementsByAttributeValue( + "tpl", + "www_index", + [$datafield] ); - if(count($card) !== 0){ + if( + count($tpl_probe) === 0 && + count( + $this->fuckhtml + ->getElementsByClassName( + "cosc-card", + $div + ) + ) !== 0 + ){ // // Parse chinese youtube shorts @@ -870,10 +880,13 @@ class baidu{ // class:FYB_RD -> News garbage, IGNORE $result = - $this->fuckhtml - ->getElementsByClassName( - "result", - [$datafield] + array_merge( + $tpl_probe, + $this->fuckhtml + ->getElementsByClassName( + "result", + [$datafield] + ) ); if(count($result) !== 0){ @@ -885,7 +898,7 @@ class baidu{ $title = $this->fuckhtml ->getElementsByClassName( - "sc-link", + "cosc-title-a", "a" ); @@ -905,8 +918,9 @@ class baidu{ $description = $this->fuckhtml - ->getElementsByClassName( - "c-color", + ->getElementsByAttributeValue( + "data-sanssr-cmpt", + "card/www-summary-1", $div ); diff --git a/scraper/ddg.php b/scraper/ddg.php index c59b423..8f68d42 100644 --- a/scraper/ddg.php +++ b/scraper/ddg.php @@ -54,8 +54,8 @@ class ddg{ $headers = ["User-Agent: " . config::USER_AGENT, "Accept: */*", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", + "Accept-Language: en-US,en;q=0.9", + "Accept-Encoding: gzip, deflate, br, zstd", "Referer: https://duckduckgo.com/", "DNT: 1", "Sec-GPC: 1", @@ -492,6 +492,17 @@ class ddg{ "form" ); + foreach($forms as $form){ + + if( + isset($form["attributes"]["id"]) && + $form["attributes"]["id"] == "challenge-form" + ){ + + throw new Exception("DuckDuckGo returned an image captcha"); + } + } + foreach(array_reverse($forms) as $form){ $this->fuckhtml->load($form); @@ -909,112 +920,6 @@ class ddg{ ){ throw new Exception("DuckDuckGo returned a JSA challenge"); - - // get JSA initial token - if( - !preg_match( - '/let jsa *= *([0-9]+)/', - $js, - $jsa - ) - ){ - - $jsa = 0; - }else{ - - $jsa = (int)$jsa[1]; - } - - // get function bodies - preg_match_all( - '/let *([A-Za-z0-9]+) *= *function\(.*\) *{(.*)};/sU', - $js, - $functions - ); - - $parsed_functions = []; - - for($i=0; $i "multiplication", - "num" => (int)$num[1] - ]; - continue; - } - - if( - preg_match( - '/innerHTML *= *`([^`]+)`/i', - $functions[2][$i], - $challenge - ) - ){ - - $challenge[1] = - preg_replace( - '/<\/(br)>/', - '<$1>', - $challenge[1] - ); - - $parsed_functions[$functions[1][$i]] = [ - "type" => "challenge", - "text" => $challenge[1] - ]; - } - } - - // get function call order - preg_match_all( - '/jsa *= *([A-Za-z0-9]+)\(jsa\)/i', - $js, - $call_order - ); - - foreach($call_order[1] as $order){ - - if(!isset($parsed_functions[$order])){ - - throw new Exception("JS challenge solve failure: DuckDuckGo called an unknown function"); - } - - if($parsed_functions[$order]["type"] == "multiplication"){ - - $jsa = $jsa * $parsed_functions[$order]["num"]; - continue; - } - - if($parsed_functions[$order]["type"] == "challenge"){ - - // @TODO get parsed length - //$parsed_functions[$order]["text"] - - $jsa = $jsa + strlen($parsed_functions[$order]["text"]); - } - } - - try{ - $js = $this->get( - $proxy, - "https://links.duckduckgo.com" . $challenge_url[1] . $jsa, - [], - ddg::req_xhr - ); - }catch(Exception $error){ - - throw new Exception("Failed to get challenged d.js"); - } } // diff --git a/scraper/ghostery.php b/scraper/ghostery.php deleted file mode 100644 index 394756e..0000000 --- a/scraper/ghostery.php +++ /dev/null @@ -1,320 +0,0 @@ -backend = new backend("ghostery"); - - include "lib/fuckhtml.php"; - $this->fuckhtml = new fuckhtml(); - } - - public function getfilters($page){ - - if($page != "web"){ - - return []; - } - - return [ - "country" => [ - "display" => "Country", - "option" => [ - "any" => "All regions", - "AR" => "Argentina", - "AU" => "Australia", - "AT" => "Austria", - "BE" => "Belgium", - "BR" => "Brazil", - "CA" => "Canada", - "CL" => "Chile", - "DK" => "Denmark", - "FI" => "Finland", - "FR" => "France", - "DE" => "Germany", - "HK" => "Hong Kong", - "IN" => "India", - "ID" => "Indonesia", - "IT" => "Italy", - "JP" => "Japan", - "KR" => "Korea", - "MY" => "Malaysia", - "MX" => "Mexico", - "NL" => "Netherlands", - "NZ" => "New Zealand", - "NO" => "Norway", - "CN" => "People's Republic of China", - "PL" => "Poland", - "PT" => "Portugal", - "PH" => "Republic of the Philippines", - "RU" => "Russia", - "SA" => "Saudi Arabia", - "ZA" => "South Africa", - "ES" => "Spain", - "SE" => "Sweden", - "CH" => "Switzerland", - "TW" => "Taiwan", - "TR" => "Turkey", - "GB" => "United Kingdom", - "US" => "United States" - ] - ] - ]; - } - - private function get($proxy, $url, $get = [], $country){ - - $curlproc = curl_init(); - - if($get !== []){ - $get = http_build_query($get); - $url .= "?" . $get; - } - - curl_setopt($curlproc, CURLOPT_URL, $url); - - curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding - curl_setopt($curlproc, CURLOPT_HTTPHEADER, - ["User-Agent: " . config::USER_AGENT, - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", - "Referer: https://ghosterysearch.com", - "DNT: 1", - "Sec-GPC: 1", - "Connection: keep-alive", - "Cookie: ctry=" . ($country == "any" ? "--" : $country) . "; noads=true", - "Upgrade-Insecure-Requests: 1", - "Sec-Fetch-Dest: document", - "Sec-Fetch-Mode: navigate", - "Sec-Fetch-Site: same-origin", - "Sec-Fetch-User: ?1", - "Priority: u=0, i"] - ); - - // http2 bypass - curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); - - curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); - curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); - curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); - - $this->backend->assign_proxy($curlproc, $proxy); - - $data = curl_exec($curlproc); - - if(curl_errno($curlproc)){ - - throw new Exception(curl_error($curlproc)); - } - - curl_close($curlproc); - return $data; - } - - public function web($get){ - - if($get["npt"]){ - - [$query, $proxy] = $this->backend->get($get["npt"], "web"); - - parse_str($query, $query); - - // country - $country = $query["c"]; - unset($query["c"]); - - $query = http_build_query($query); - - try{ - - $html = - $this->get( - $proxy, - "https://ghosterysearch.com/search?" . $query, - [], - $country - ); - }catch(Exception $error){ - - throw new Exception("Failed to fetch search page"); - } - }else{ - - $proxy = $this->backend->get_ip(); - - try{ - - $html = - $this->get( - $proxy, - "https://ghosterysearch.com/search", - [ - "q" => $get["s"] - ], - $get["country"] - ); - }catch(Exception $error){ - - throw new Exception("Failed to fetch search page"); - } - } - - $out = [ - "status" => "ok", - "spelling" => [ - "type" => "no_correction", - "using" => null, - "correction" => null - ], - "npt" => null, - "answer" => [], - "web" => [], - "image" => [], - "video" => [], - "news" => [], - "related" => [] - ]; - - $this->fuckhtml->load($html); - - $results_wrapper = - $this->fuckhtml - ->getElementsByClassName( - "results", - "section" - ); - - if(count($results_wrapper) === 0){ - - throw new Exception("Failed to grep result section"); - } - - $this->fuckhtml->load($results_wrapper[0]); - - // get search results - $results = - $this->fuckhtml - ->getElementsByClassName( - "result", - "li" - ); - - if(count($results) === 0){ - - return $out; - } - - foreach($results as $result){ - - $this->fuckhtml->load($result); - - $a = - $this->fuckhtml - ->getElementsByClassName( - "url", - "a" - ); - - if(count($a) === 0){ - - continue; - } - - $a = $a[0]; - - $out["web"][] = [ - "title" => - $this->titledots( - $this->fuckhtml - ->getTextContent( - $this->fuckhtml - ->getElementsByTagName( - "h2" - )[0] - ) - ), - "description" => - $this->titledots( - $this->fuckhtml - ->getTextContent( - $this->fuckhtml - ->getElementsByTagName( - "p" - )[0] - ) - ), - "url" => - $this->fuckhtml - ->getTextContent( - $a - ["attributes"] - ["href"] - ), - "date" => null, - "type" => "web", - "thumb" => [ - "url" => null, - "ratio" => null - ], - "sublink" => [], - "table" => [] - ]; - } - - $this->fuckhtml->load($html); - - // get pagination token - $pagination_wrapper = - $this->fuckhtml - ->getElementsByClassName( - "pagination", - "div" - ); - - if(count($pagination_wrapper) !== 0){ - - // found next page! - $this->fuckhtml->load($pagination_wrapper[0]); - - $a = - $this->fuckhtml - ->getElementsByTagName( - "a" - ); - - if(count($a) !== 0){ - - $q = - parse_url( - $this->fuckhtml - ->getTextContent( - $a[count($a) - 1] - ["attributes"] - ["href"] - ), - PHP_URL_QUERY - ); - - $out["npt"] = - $this->backend - ->store( - $q . "&c=" . $get["country"], - "web", - $proxy - ); - } - } - - return $out; - } - - private function titledots($title){ - - return trim($title, " .\t\n\r\0\x0B…"); - } -} diff --git a/scraper/marginalia.php b/scraper/marginalia.php index f5be5e1..ee18cc0 100644 --- a/scraper/marginalia.php +++ b/scraper/marginalia.php @@ -289,8 +289,18 @@ class marginalia{ throw new Exception("The API key used is rate limited. Please try again in a few minutes."); } + if(strpos($json, "Bad Gateway (502)")){ + + throw new Exception("Marginalia returned a 502 error"); + } + $json = json_decode($json, true); + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + foreach($json["results"] as $result){ $out["web"][] = [ @@ -490,6 +500,37 @@ class marginalia{ return $this->web($get, $redirect); } + // detect internal error + $infobox = + $this->fuckhtml + ->getElementsByClassName("infobox"); + + if(count($infobox) !== 0){ + + foreach($infobox as $i){ + + $this->fuckhtml->load($i); + + $h2 = + $this->fuckhtml + ->getElementsByTagName("h2"); + + if( + count($h2) !== 0 && + $this->fuckhtml + ->getTextContent( + $h2[0] + ) == "Internal error" + ){ + + throw new Exception("Marginalia returned an internal server error"); + } + } + + // reset + $this->fuckhtml->load($html); + } + $sections = $this->fuckhtml ->getElementsByClassName( diff --git a/scraper/mojeek.php b/scraper/mojeek.php index 2939be5..f123b69 100644 --- a/scraper/mojeek.php +++ b/scraper/mojeek.php @@ -340,7 +340,7 @@ class mojeek{ "zm" => "Zambia", "zw" => "Zimbabwe" ] - ], + ],/* "region" => [ "display" => "Region", "option" => [ @@ -350,7 +350,7 @@ class mojeek{ "fr" => "France", "uk" => "United Kingdom" ] - ], + ],*/ "domain" => [ "display" => "Results per domain", "option" => [ @@ -458,7 +458,7 @@ class mojeek{ "date" => 1, // show date "tlen" => 128, // max length of title //"dlen" => 511, // max length of description - "arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect! + //"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect! ]; switch($focus){ @@ -503,6 +503,11 @@ class mojeek{ } } + if(strlen($html) === 0){ + + throw new Exception("Mojeek returned an empty page (probably a homepage redirect)"); + } + $out = [ "status" => "ok", "spelling" => [ diff --git a/scraper/naver.php b/scraper/naver.php new file mode 100644 index 0000000..492488a --- /dev/null +++ b/scraper/naver.php @@ -0,0 +1,1107 @@ +backend = new backend("naver"); + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); + } + + public function getfilters($page){ + + $base = [ + "time" => [ + "display" => "Time", + "option" => [ + "any" => "Any time", + "1h" => "Last hour", + "1d" => "Last day", + "1w" => "Last week", + "1m" => "Last month", + "3m" => "Last 3 months", + "6m" => "Last 6 months", + "1y" => "Last year", + ] + ] + ]; + + switch($page){ + + case "web": + return + array_merge([ + "sort" => [ + "display" => "Sort by", + "option" => [ + "relevance" => "Relevance", // r + "most_recent" => "Most recent" // dd + ] + ] + ], $base); + break; + + case "images": + return + array_merge( + $base, + [ + "size" => [ + "display" => "Size", + "option" => [ + "any" => "Any size", + "highdef" => "High definition" // &res_fr=786432&res_to=100000000 + ] + ], + "color" => [ // &color= + "display" => "Color", + "option" => [ + "any" => "Any color", + "orange" => "Orange", + "yellow" => "Yellow", + "lime" => "Lime", + "green" => "Green", + "cyan" => "Cyan", + "blue" => "Blue", + "purple" => "Purple", + "pink" => "Pink", + "apricot" => "Apricot", + "ocher" => "Ocher", + "sepia" => "Sepia", + "black" => "Black", + "gray" => "Gray", + "white" => "White" + ] + ], + "license" => [ // &ccl= + "display" => "License", + "option" => [ + "any" => "Any license", + "1" => "CCL Total", + "2" => "Commercial use", + "4" => "Modifications permitted" + ] + ] + ] + ); + break; + + case "videos": + return + [ + "time" => [ // done + "display" => "Time", + "option" => [ + "any" => "Any time", + "1day" => "Last day", + "1week" => "Last week", + "1month" => "Last month", + "3month" => "Last 3 months", + "6month" => "Last 6 months", + "1year" => "Last year" + ] + ], + "sort" => [ // done + "display" => "Sort by", + "option" => [ + "rel" => "Relevance", + "date" => "Most recent", // &sort=date + "playcount" => "Most views", // &sort=playcount + ] + ], + "type" => [ // done + "display" => "Type", + "option" => [ + "any" => "Any videos", + "shorts" => "Shorts" // dtype=shorts + ] + ], + "duration" => [ + "display" => "Duration", // &playtime= + "option" => [ + "any" => "Any duration", + "0:600" => "10 minutes", + "601:1800" => "10-30 minutes", + "1801:3600" => "30-60 minutes", + "3601:65535" => "More than 1 hour" + ] + ] + ]; + break; + } + } + + private function get($proxy, $url, $get = [], $is_xhr = false){ + + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + // use http2 + curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + + if($is_xhr === false){ + + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["User-Agent: " . config::USER_AGENT, + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "DNT: 1", + "Sec-GPC: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: same-origin", + "Priority: u=0, i", + "Sec-Fetch-User: ?1"] + ); + }else{ + + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["User-Agent: " . config::USER_AGENT, + "Accept: */*", + "Accept-Language: en-US,en;q=0.9", + "Accept-Encoding: gzip, deflate, br, zstd", + "Referer: https://search.naver.com/", + "DNT: 1", + "Sec-GPC: 1", + "Alt-Used: s.search.naver.com", + "Connection: keep-alive", + "Sec-Fetch-Dest: script", + "Sec-Fetch-Mode: no-cors", + "Sec-Fetch-Site: same-site", + "TE: trailers"] + ); + } + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $this->backend->assign_proxy($curlproc, $proxy); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + + + public function web($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + if($get["npt"]){ + + [$d, $proxy] = $this->backend->get($get["npt"], "web"); + + try{ + + $html = + $this->get( + $proxy, + "https://search.naver.com/search.naver" . $d, + [] + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + }else{ + + // parse filters + // https://search.naver.com + // /search.naver + // ?nso= + // &page=1 + // &query=nisekoi + // &sm=tab_pge + // &start=1 + // &where=web + + $filters = [ + "nso" => "", + "query" => $search, + "sm" => "tab_pge", + "where" => "web", + "start" => 1 // increment by number of results each time (16??) + ]; + + $options = []; + + if($get["sort"] != "relevance"){ + + $options[] = "so:dd"; + } + + if($get["time"] != "any"){ + + $options[] = "p:" . $get["time"]; + } + + if(count($options) !== 0){ + + $filters["nso"] = implode(",", $options); + } + + //$html = file_get_contents("scraper/naver.html"); + + $proxy = $this->backend->get_ip(); + + try{ + $html = + $this->get( + $proxy, + "https://search.naver.com/search.naver", + $filters + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + } + + $this->fuckhtml->load($html); + + $results = + preg_split( + '/entry\.bootstrap\(document\.getElementById\("[a-f0-9-r]+"\), ?/', + $html + ); + + if(count($results) !== 2){ + + // this is thrown when no results are found + + $nsfw_probe = + $this->fuckhtml + ->getElementsByClassName( + "dsc_adult", + "div" + ); + + if(count($nsfw_probe) !== 0){ + + $out["answer"][] = [ + "title" => "NSFW results", + "description" => [ + [ + "type" => "text", + "value" => "Naver blocks logged-out NSFW searches." + ] + ], + "url" => null, + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + } + + return $out; + //throw new Exception("Failed to grep results entrypoint"); + } + + $json = + json_decode( + $this->fuckhtml + ->extract_json( + $results[1] + ), + true + ); + + if(!isset($json["body"]["props"]["children"][0]["props"]["children"])){ + + throw new Exception("Failed to access nested children"); + } + + foreach($json["body"]["props"]["children"][0]["props"]["children"] as $result){ + + if( + !isset($result["templateId"]) || + $result["templateId"] != "webItem" + ){ + + // should not happen + continue; + } + + $result = $result["props"]; + + // get sublinks + $sublinks = []; + + if(isset($result["subLinks"])){ + + foreach($result["subLinks"] as $s){ + + $sublinks[] = [ + "title" => $s["text"], + "description" => null, + "url" => $s["href"], + "date" => null + ]; + } + } + + if(isset($result["linkBtns"])){ + + foreach($result["linkBtns"] as $s){ + + $sublinks[] = [ + "title" => $s["text"], + "description" => null, + "url" => $s["href"], + "date" => null + ]; + } + } + + // get image (thumbnail, i guess) + if(isset($result["images"][0]["imageSrc"])){ + + $thumb = [ + "ratio" => "16:9", + "url" => $this->unshit_thumb($result["images"][0]["imageSrc"]) + ]; + }else{ + + $thumb = [ + "ratio" => null, + "url" => null + ]; + } + + // get table elements + $table = []; + + if(isset($result["keyValue"]["contents"])){ + + foreach($result["keyValue"]["contents"] as $s){ + + if(!isset($s["valueData"]["text"])){ continue; } + + $table[$s["key"]] = $s["valueData"]["text"]; + } + } + + // get date + $time = null; + + if(isset($result["bodyPrefixes"][0]["text"])){ + + $date = + strtotime( + substr( + $result["bodyPrefixes"][0]["text"], + -1 + ) + ); + + if($date !== false){ + + $time = $date; + } + } + + $out["web"][] = [ + "title" => $this->decode_html($result["title"]), + "description" => $this->decode_html($result["bodyText"]), + "url" => $result["href"], + "date" => $time, + "type" => "web", + "thumb" => $thumb, + "sublink" => $sublinks, + "table" => $table + ]; + } + + // get next page + $npt = + $this->fuckhtml + ->getElementsByClassName( + "btn_next", + "a" + ); + + if(count($npt) !== 0){ + + $out["npt"] = + $this->backend->store( + $this->fuckhtml + ->getTextContent( + $npt[0]["attributes"]["href"] + ), + "web", + $proxy + ); + } + + return $out; + } + + + public function image($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $out = [ + "status" => "ok", + "npt" => null, + "image" => [] + ]; + + if($get["npt"]){ + + [$url, $proxy] = $this->backend->get($get["npt"], "images"); + + try{ + + $json = + $this->get( + $proxy, + $url, + [], + true + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + }else{ + + $filters = [ + "ac" => "0", + "api_type" => "pc_tab_more", + "aq" => "0", + "display" => 100, + "logStart" => 1, + "mode" => "column", + "nso" => "so:r,p:all", + "nx_search_query" => $search, + "query" => $search, + "section" => "image", + "sm" => "tab_opt", + "ssc" => "tab.image.all", + "start" => 1, + "where" => "image" + // no callback, returns raw json lol + ]; + + $options = [ + "so:r" + ]; + + if($get["time"] != "any"){ + + $options[] = "p:" . $get["time"]; + } + + if(count($options) !== 0){ + + $filters["nso"] = implode(",", $options); + } + + if($get["size"] != "any"){ + + $filters["res_fr"] = 786432; + $filters["res_to"] = 100000000; + } + + if($get["color"] != "any"){ + + $filters["color"] = $get["color"]; + } + + if($get["license"] != "any"){ + + $filters["ccl"] = $get["license"]; + } + + //$json = file_get_contents("scraper/naver.html"); + + $proxy = $this->backend->get_ip(); + + try{ + $json = + $this->get( + $proxy, + "https://s.search.naver.com/p/c/image/46/search.naver", + $filters, + true + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + } + + $json = json_decode($json, true); + + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + + if(!isset($json["items"])){ + + // no results returned :( + return $out; + //throw new Exception("Naver did not return an items object"); + } + + foreach($json["items"] as $image){ + + // why does it fucking do that + if($image["orgWidth"] === 0){ continue; } + + $out["image"][] = [ + "title" => trim($image["title"], "."), + "source" => [ + [ + "url" => $image["originalUrl"], + "width" => (int)$image["orgWidth"], + "height" => (int)$image["orgHeight"] + ], + [ + "url" => $image["thumb"], + "width" => (int)$image["thumbWidth"], + "height" => (int)$image["thumbHeight"] + ] + ], + "url" => $image["link"] + ]; + } + + // get npt + if( + isset($json["url"]) && + $json["url"] != "" && + $json["url"] != null + ){ + + $out["npt"] = + $this->backend->store( + $json["url"], + "images", + $proxy + ); + } + + return $out; + } + + + public function video($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $out = [ + "status" => "ok", + "npt" => null, + "video" => [], + "author" => [], + "livestream" => [], + "playlist" => [], + "reel" => [] + ]; + + if($get["npt"]){ + + [$url, $proxy] = $this->backend->get($get["npt"], "images"); + + try{ + + $json = + $this->get( + $proxy, + $url, + [], + true + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + }else{ + + // https://s.search.naver.com/p/video/48/search.naver?ac=0&aq=0&crbase=63&display=48&dtype=&last_block_type=recom&nlu_query=&nq=&nqx_theme={"theme":{"main":{"name":"encyclopedia","source":"TOS"}}}&nx_and_query=&nx_search_hlquery=&nx_search_query=&nx_sub_query=&page=2&period=&playtime=&ptype=&query=asmr&selected_channel=&selected_cp=&sm=mtb_pge&sort=rel&ssc=tab.video.all&start=49&video_more=1 + // https://s.search.naver.com/p/video/48/search.naver + // ?ac=0 + // &aq=0 + // &crbase=63 + // &display=48 + // &dtype= + // &last_block_type=recom + // &nlu_query= + // &nq= + // &nqx_theme={"theme":{"main":{"name":"encyclopedia","source":"TOS"}}} + // &nx_and_query= + // &nx_search_hlquery= + // &nx_search_query= + // &nx_sub_query= + // &page=2 + // &period= + // &playtime= + // &ptype= + // &query=asmr + // &selected_channel= + // &selected_cp= + // &sm=mtb_pge + // &sort=rel + // &ssc=tab.video.all + // &start=49 + // &video_more=1 + + $filters = [ + "ac" => "0", + "aq" => "0", + "crbase" => "78", + "display" => 48, + "dtype" => "", + "last_block_type" => "recom", + "nlu_query" => "", + "nq" => "", + "nx_and_query" => "", + "nx_search_hlquery" => "", + "nx_search_query" => "", + "nx_sub_query" => "", + "page" => 1, + "period" => "", + "playtime" => "", + "ptype" => "", + "query" => $search, + "selected_channel" => "", + "selected_cp" => "", + "sm" => "mtb_pge", + "sort" => "rel", + "ssc" => "tab.video.all", + "start" => 1, + "video_more" => 1 + ]; + + if($get["type"] != "any"){ + + $filters["dtype"] = $get["type"]; + } + + if($get["time"] != "any"){ + + $filters["period"] = $get["time"]; + } + + if($get["sort"] != "rel"){ + + $filters["sort"] = $get["sort"]; + } + + if($get["duration"] != "any"){ + + $filters["playtime"] = $get["duration"]; + } + + //$json = file_get_contents("scraper/naver.html"); + + $proxy = $this->backend->get_ip(); + + try{ + $json = + $this->get( + $proxy, + "https://s.search.naver.com/p/video/48/search.naver", + $filters, + true + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + } + + $json = json_decode($json, true); + + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + + if(!isset($json["collection"])){ + + return $out; + //throw new Exception("Naver did not return a collection HTML element"); + } + + foreach($json["collection"] as $snippet){ + + if(!isset($snippet["html"])){ continue; } + + $this->fuckhtml->load($snippet["html"]); + + $div = + $this->fuckhtml + ->getElementsByTagName( + "div" + ); + + $items = + $this->fuckhtml + ->getElementsByAttributeValue( + "data-template-id", + "videoItem", + $div + ); + + // parse normal videos + foreach($items as $item){ + + if($item["level"] === 6){ continue; } + + $this->fuckhtml->load($item); + + // get url + $as = + $this->fuckhtml + ->getElementsByAttributeName( + "data-heatmap-target", + "a" + ); + + if(count($as) === 0){ + + // should not happen + continue; + } + + // get thumbnail + $thumb = + $this->fuckhtml + ->getElementsByAttributeValue( + "loading", + "lazy", + "img" + ); + + if(count($thumb) !== 0){ + + $thumb = [ + "url" => + $this->unshit_thumb( + $this->fuckhtml + ->getTextContent( + $thumb[0]["attributes"]["src"] + ) + ), + "ratio" => "16:9" + ]; + }else{ + + $thumb = [ + "url" => null, + "ratio" => null + ]; + } + + // get timestamp + $timestamp_probe = + $this->fuckhtml + ->getElementsByClassName( + "sds-comps-text-type-footnote", + "span" + ); + + if(count($timestamp_probe) !== 0){ + + $timestamp = + $this->hms2int( + $this->fuckhtml + ->getTextContent( + $timestamp_probe[0] + ) + ); + }else{ + + $timestamp = null; + } + + $out["video"][] = [ + "title" => + $this->fuckhtml + ->getTextContent( + $as[0] + ), + "description" => null, + "author" => [ + "name" => + isset($item["attributes"]["profileimagealt"]) ? + $this->fuckhtml + ->getTextContent( + $item["attributes"]["profileimagealt"] + ) : null, + "url" => + isset($item["attributes"]["profileimagehref"]) ? + $this->fuckhtml + ->getTextContent( + $item["attributes"]["profileimagehref"] + ) : null, + "avatar" => + isset($item["attributes"]["profileimagesrc"]) ? + $this->fuckhtml + ->getTextContent( + $item["attributes"]["profileimagesrc"] + ) : null + ], + "date" => null, + "duration" => $timestamp, + "views" => null, + "thumb" => $thumb, + "url" => + $this->fuckhtml + ->getTextContent( + $as[0]["attributes"]["href"] + ) + ]; + } + + // reset + $this->fuckhtml->load($snippet["html"]); + + // parse reels + $carousels = + array_merge( + $this->fuckhtml // for the reels only tab + ->getElementsByClassName( + "fds-video-tab-shortform-desk-filter", + $div + ), + $this->fuckhtml // for the normal tab with reels inbetween + ->getElementsByClassName( + "fds-video-tab-shortform-desk", + $div + ) + ); + + foreach($carousels as $carousel){ + + $this->fuckhtml->load($carousel); + + $as = + $this->fuckhtml + ->getElementsByTagName( + "a" + ); + + foreach($as as $reel){ + + $this->fuckhtml->load($reel); + + $spans = + $this->fuckhtml + ->getElementsByTagName( + "span" + ); + + $title = + $this->fuckhtml + ->getTextContent( + $spans[0] + ); + + // get thumbnail + $thumb = + $this->fuckhtml + ->getElementsByAttributeValue( + "loading", + "lazy", + "img" + ); + + if(count($thumb) !== 0){ + + $thumb = [ + "url" => + $this->unshit_thumb( + $this->fuckhtml + ->getTextContent( + $thumb[0]["attributes"]["src"] + ) + ), + "ratio" => "16:9" + ]; + }else{ + + $thumb = [ + "url" => null, + "ratio" => null + ]; + } + + $name = + $this->fuckhtml + ->getElementsByClassName( + "sds-comps-profile-info-title-text", + $spans + ); + + if(count($name) === 0){ + + $name = null; + }else{ + + $name = + $this->fuckhtml + ->getTextContent( + $name[0] + ); + } + + $out["reel"][] = [ + "title" => $title, + "description" => null, + "author" => [ + "name" => $name, + "url" => null, + "avatar" => null + ], + "date" => null, + "duration" => null, + "views" => null, + "thumb" => $thumb, + "url" => + $this->fuckhtml + ->getTextContent( + $reel["attributes"]["href"] + ) + ]; + } + } + } + + // get npt + if( + isset($json["url"]) && + $json["url"] != "" && + $json["url"] != null + ){ + + $out["npt"] = + $this->backend->store( + $json["url"], + "images", + $proxy + ); + } + + return $out; + } + + + private function unshit_thumb($url){ + + $parts = parse_url($url); + + if($parts["host"] == "search.pstatic.net"){ + + parse_str($parts["query"], $str); + + if(isset($str["src"])){ + + return $str["src"]; + } + } + + return $url; + } + + + private function decode_html($html){ + + return + trim( + html_entity_decode( + strip_tags( + $html + ) + ), + "." + ); + } + + + private function hms2int($time){ + + $parts = explode(":", $time, 3); + $time = 0; + + if(count($parts) === 3){ + + // hours + $time = $time + ((int)$parts[0] * 3600); + array_shift($parts); + } + + if(count($parts) === 2){ + + // minutes + $time = $time + ((int)$parts[0] * 60); + array_shift($parts); + } + + // seconds + $time = $time + (int)$parts[0]; + + return $time; + } +} diff --git a/scraper/qwant.php b/scraper/qwant.php index c242de7..322182e 100644 --- a/scraper/qwant.php +++ b/scraper/qwant.php @@ -212,28 +212,58 @@ class qwant{ return $base; } - private function get($proxy, $url, $get = []){ - - $headers = [ - "User-Agent: " . config::USER_AGENT, - "Accept: application/json, text/plain, */*", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", - "DNT: 1", - "Connection: keep-alive", - "Origin: https://www.qwant.com", - "Referer: https://www.qwant.com/", - "Sec-Fetch-Dest: empty", - "Sec-Fetch-Mode: cors", - "Sec-Fetch-Site: same-site", - "TE: trailers" - ]; + private function get($proxy, $url, $get = [], $is_post = false, $cookie = null){ $curlproc = curl_init(); - if($get !== []){ - $get = http_build_query($get); - $url .= "?" . $get; + if($is_post){ + + $headers = [ + "User-Agent: " . config::USER_AGENT, + "Accept: */*", + "Accept-Language: en-US,en;q=0.9", + "Accept-Encoding: gzip, deflate, br, zstd", + "Referer: https://www.qwant.com/", + "Content-type: application/x-www-form-urlencoded", + "Origin: https://www.qwant.com", + "DNT: 1", + "Sec-GPC: 1", + "Connection: keep-alive", + "Sec-Fetch-Dest: empty", + "Sec-Fetch-Mode: cors", + "Sec-Fetch-Site: same-site", + "Pragma: no-cache", + "Cache-Control: no-cache" + ]; + + curl_setopt($curlproc, CURLOPT_POST, true); + curl_setopt($curlproc, CURLOPT_POSTFIELDS, http_build_query($get)); + }else{ + + $headers = [ + "User-Agent: " . config::USER_AGENT, + "Accept: application/json, text/plain, */*", + "Accept-Language: en-US,en;q=0.9", + "Accept-Encoding: gzip, deflate, br, zstd", + "Referer: https://www.qwant.com/", + "Origin: https://www.qwant.com", + "DNT: 1", + ($cookie !== null ? "Cookie: {$cookie}" : ""), + "Sec-GPC: 1", + "Connection: keep-alive", + "Sec-Fetch-Dest: empty", + "Sec-Fetch-Mode: cors", + "Sec-Fetch-Site: same-site", + "Priority: u=4", + "Pragma: no-cache", + "Cache-Control: no-cache", + "TE: trailers" + ]; + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } } curl_setopt($curlproc, CURLOPT_URL, $url); @@ -271,6 +301,8 @@ class qwant{ [$params, $proxy] = $this->backend->get($get["npt"], "web"); $params = json_decode($params, true); + $cookie = $params["cookie"]; + unset($params["cookie"]); }else{ @@ -297,7 +329,8 @@ class qwant{ "device" => "desktop", "tgp" => 3, "safesearch" => 0, - "displayed" => "true" + "displayed" => "true", + "llm" => "false" ]; switch($get["nsfw"]){ @@ -306,18 +339,24 @@ class qwant{ case "maybe": $params["safesearch"] = 1; break; case "no": $params["safesearch"] = 2; break; } + + $cookie = null; } /* $handle = fopen("scraper/qwant_web.json", "r"); $json = fread($handle, filesize("scraper/qwant_web.json")); fclose($handle);*/ + $cookie = $this->fuck_datadome($proxy, $search); + try{ $json = $this->get( $proxy, - "https://fdn.qwant.com/v3/search/web", - $params + "https://api.qwant.com/v3/search/web", + $params, + false, + $cookie ); }catch(Exception $error){ @@ -353,6 +392,8 @@ class qwant{ "related" => [] ]; + $this->detect_errors($json); + if( $json["status"] != "success" && $json["data"]["error_code"] === 5 @@ -362,8 +403,6 @@ class qwant{ return $out; } - $this->detect_errors($json); - if(!isset($json["data"]["result"]["items"]["mainline"])){ throw new Exception("Server did not return a result object"); @@ -436,6 +475,7 @@ class qwant{ if($json["data"]["result"]["lastPage"] === false){ $params["offset"] = $params["offset"] + 10; + $params["cookie"] = $cookie; $out["npt"] = $this->backend->store( @@ -577,6 +617,9 @@ class qwant{ ); $params = json_decode($params, true); + $cookie = $params["cookie"]; + unset($params["cookie"]); + }else{ $search = $get["s"]; @@ -617,13 +660,19 @@ class qwant{ case "maybe": $params["safesearch"] = 1; break; case "no": $params["safesearch"] = 2; break; } + + $cookie = null; } + $cookie = $this->fuck_datadome($proxy, $search); + try{ $json = $this->get( $proxy, "https://api.qwant.com/v3/search/images", $params, + false, + $cookie ); }catch(Exception $err){ @@ -1001,4 +1050,53 @@ class qwant{ return "https://" . $image["host"] . "/th?id=" . rawurlencode($id); } + + private function fuck_datadome($proxy, $search){ + + if($cookie === null){ + // get datadome bullshit + // yeah go ahead and patch this you code monkey + + // ... you know, instead of fixing this shit, can you make the second page of results return more than 1 result? + // i search shit like "higurashi" and i only get 1 result and then just a bunch of related searches ad finitum + // also, stop pretending you guys have your own index, this is clearly all bing results + try{ + $datadome = + $this->get( + $proxy, + "https://dd.qwant.com/js/", + [ + "jspl" => "-oirHDeP7G3G9LXY3rhxGDYf6t_ZlYPzaiL2dRhQ34uNUc4dO2Td0SmmKu_VJp7-jrAIoOMrTqKgCtV4cPzntMyNuOYN7QQ1q0Fi2c-679vYpJr9ub5dRFXMhCMQmjZsOWXUa8yafIXf85UA5Goct-n4hVT5dFfBGM-dE6gwZEFkHQ1j1195sLybZFTNA1IfX26MBMxlVHoyT-aHtkd6YjGXbvOPMMsObVHDa_Cf4BmApRGo-eehsAttsCqDRwkl0M2E1b0OXCIwMVnCK-qWZ81MGaJGjqxdj_LaxhAjE9JI_4oNedAg4Yj4m_vkQBKdmWJT_bTUsllAZXqX5IxYQHJ8WvlknOTr6sR3wgoSOa_JNVHkUQxqXBbQcoDdy6VciL5rIqwWl4Y2caVYzeXa8YNXxwsfrPDegH3dQ6vOiIiCDT3bexkcS_y7GAioPPv9qDu7M9y4sI8t0VkPgMbWY6MIu5u9WAv6lEiKIYrtlyU-iiXnNdkd9EnpC1TD8_s12PtCuLg5bGgLVmI1dw8VXVZxmVyFvfBtuqXkzM-kZOyv-YIRcf-wPY8DUtnHox1d6t64oYhGv-pYV5dmZ0rGs99LC6ThiifeF_oHz5ZK46Ms68WsiAjc1HLAbFPFxWw8X6lyLlpT5aHgFdDBeCTExKXyItlc9PBxkgK8UnXHQFiWX3HTen8rziwkI31AbkMU7xGP6cvuwuLG7MA7sH3yLF1uimQlhr6EuTrImWQNV7X81DqyZ0OE7I-uT9bp_xISa5it-6AJpFtGcwz04_OuO2zvZazBeiKJI5Fw0X2r3RHTHdA29SXqTIe4y2LRAAp0MSAZUIJAMIbbrz9BG-3D5BHfQUA1Bbuklxv-tNWiuanWbbTmAiuxTAo9P-D2ZMrWHwdN8sCb3nINLur38z1xviSs2qEEjKo9kzQIWoHgwm_amV-jjDtNBXwpCQnoaqt3FfL8e59ecYWVg61Gnl3YCwq6rubFdkIv6UStcWSlj4pRFlFZSjQxzznvGTAc5Nz1JJWVeBHJItei4wqDUYEw2BNTS_fU9keV3LFyKDH_x248AxHLiXt583DhZZwPAKlpmMo0E7odzfOgVQGM5b9l64NL3RQRq8cDxI8KgmOFnPHdI7Myq1A3nW1uoBB8m9_GR_9orboKl5tAQrT0VTMRd6JyNwOOQ71ZoQpMwsD5Xd9YDZd618TNE9dA38h9OyBco_-hwKCYayyEqib1atLLBO_gnO4InR471AklOokDOKx1oY_xgNhlFrZskE4nQQOqeCQ_47oysob9s3kblA8UqDrL2vM3i2KPLWwlQzMAPrQAIv7Amp6rO-wYwBQdbP7XghYympxx9-hNxc1Duq-nS4R3pjPtZg3LoCvG1N_0JLf9jfPBqw_rdPM6DnjEOvpCdvrfv4wBlEpgqbaRQCZBPEz-O5ZlY24OyPvaNX0krPeXkbi-KYuwZSAbfW0zsksXb8cbRmz6tvh25IRYP1XNDPVd8NFrYOdDH9GzhV5MYZziYMurQlVwgumGLz85Y6qTHZU2D5qkMH7o5C5bzptReO96889oxkReS_LDhzdxH0tAQ5a9GgETJTHAXJ9Lt7X037i0qQlatbPLizOl9fo4vUBGJ7w-uSUmIzg1yIwOo8xhLu39WghXfoJ_EX5ptNi7arpAo11UoJIGEgWgq7bS40q2dtnG5HYpp9ef37iFgztM7PV5dlJf_W-n9uXJyCk2nhxZeuGiIGhMTzjP_8oeyiMFelmoyLlqK8wOs6hvWqgDD4NwtZc-XBkEAKsxuPrirb_sszp0z6BWsh3D6fBxAW_lTrNVkxZ1EBtoEN72HvO1aeX8PVuJP3CKwkkUcAhJ5Zvajo8t4Ai8ave2MTZul4QhDIe7ZALJMo71u-hNSj8YG9JLPqF1jcnbkl8ykt_XMMxX8wSZ_AfTsfmAuOWBfEzxv8Fl2Ne-oxcEtfuYnYNJIjQDHXXdtYbWKBezE41wonQu7FWh3wSSF3DDm9EocWMsUZ610nrIviOFdtTqjrY7KnOZp0hS9Hm2V04z-06Db-To71L5O5poYKuOlDAP_ULfW0hgMgYaPnt4Nive9gtue6ZQ5yNFylKM11zCWcF8vmuEPGx4NIZjvNFB-QSkdxZtBTQOgO6r4BqHJNZIDqiD9A6PyMemVtQEo-JhooX6bwHFHx6C84r-kKq1lg0RaDelbu1h6347kocxOczdgHWlagKPFrUvHpPHvCZeFmzLLmPWbDEpUhhfkbEZlNe47svO5WN1TYZMCAlE0g01pe6b_NB9-twQ4FJB1SkUQRA4o5DxiIHSgXBBfw1X98XPEKcJgCU2zRbYuU69mvyPcsxoPsLo8Cl2OAXj2U1tcLUxVhR_vu3I0SG_Ee8hxBp5b6QbpIu2sE0bgpqfi88XqUXf3qvxGbkoZDZ8dPvEvCVmpDZOOGJY5Db4MnpKBByW79lUnpbLCbsVOoxYu8SVmVQ63eGY9yBlACuWxlXA_PhCpnKYwMppjxKvKqalNJGMMg5WQ8cvhsIftE5wgaj21n0A15GwRO1WgUcsO_dLe1MqO88hTHWHV0Q3gNut3sdsmVr_n-0SniLtVAaw4IEtDZWNUfhpaoC7iyKxoJ86t8r697EYPlWUu4nTQhb3TOXAjHF22rFsJJMldKHVxjxjd086ky3OY3L_7cbjVof60-y4gTJdumw5hch-mhaT93nVnaTuPLGCAC48dZSgOitW8Dw62mPS05fTPAfKUY1HpyIWr9ACk7U2KVAijpzFB-bXvqJeCyBeUHr4cB61K2XxTmkNA8Oo9RC32TeYDrThFzoUyYZ_D6zPe1lLtQaFsPRvDBSsYCcr5fcABlST4DyUgnkgqtx-0-D1QEwtWiH3xiHLQGa82ldjqNIXL9LqwxYoCRBKuljHyqEylZD26A9Jp6SL9rk_Ol6TJlYa3zx44PHyMoA3Coy-HIZW5NMwVKdV7egUDDOchvuFnc1Zv2QpSFKc0JY8lYFtSm1HZFwjq7hf2hBqkU0Xneh8AT4Ty9QVb4mJ8rpzc8m_NBZNrpZVE-pIzmv2yGjnRwKSrwsjat7b2iY34UPFqGBlDk5FX6EfRlN9bq-nztWdextLc47ndr1gxsjC-vr6tbdXl5FxMVhbsnLH80wOC6hHubxCysr5C7Ft59s3MLhVQCc3IIOPMBhLOjjjVsu6K8kAJvvamIpj-iYUQj9T8JztAqfobytzbiwPsQ3mywUGkJpZxTjXjqkKdT6yfFdW5ksR8p3VlKGAAKft9TKKcc5IgErmIRtiZF4hGhB45nwIO5XHA59V94QuV3b0oJrleuDCBj0iDFqWYpsUVHyKl3RjrO86Yq1IbNRwXIIMrlsVYos9CS1I75k9kW4yQC6ni3a2ZitDkiEB-SoQuTlBsvfTs892Wt5q3CWH6_7srZO6l2QEAlWqyQp_P4i8j8uiEbZksGsgghVIgOkriblDP9ml7PdkpRX7S3OpY7jjQgTNR8XHaZ-SEw0-zPiqIIt5VioFK8JljPn-nMIDx-ew_vTDKqmKEOF39ezQcc5oQNTQNUvTAe8qCbK4ghS3fagItezoHtB5zm3qwhx9Iu6JMJy0UhywUZBaInY6NokiVZGDgtoqlVjOk7K4299MVZgsOr2P9QHRDo41lYnCeWWFUf50d-ouEOUO_ngteDpgG1DfV64eycrUOWAnpfI8XOUZ7mxS89SqJriw3T3tcljowHiOFw9QjYsjJi0vuHL5jfoBZVdYAUH6KiLfIScO-KgqitdUTpFwks8Qk7xfd2HcEfArlzSiAt3Hmb2Ty-5mhvtQj2RbPehgoI7gqa60b6Qffth-5tbIjXoKlZCQsaKVMAggF9C3w0Rv5uJ3ExCTGsfKO9IVOf--AfxaA8cNoO0W3NTaMH2iWftY0O1-1x3VJEcerp2v9BcZ0iIL3VFJ9eyOPYNHNwTYrI3S6OrVZxuQXFbu2E_Us32wxYqXvnMx-DEOk7bUUFIanL69Cbn_mR_9Wm1sFixBQ15diJoe1Oyl3XWHWL6_YF4XnHzZlMWySt3Awdeynwu99Ter8WMGXl-6K30novKRWC9jDVOfAtxPvXBkkRtpm8nnJT8fwkmIQoWOEPT2MCg3VYzzHzVEH-AueuRiCZj_o7L-fRqbtFrWPyVnGuetUYS5u0D3n7Rr6kCVqWhLfwjqAwV_calzIQs8-J0-AJe6GGaaONPYfXwXjYan0Sl11CS64lXvx7i3MgeOZuH210T6g6hj54dSLw0upO1L8k0FGli3I5OhLMekeGMLT_N2akyHktjbz8Odpk2ms0pJP_PdqvUegjXgH81F5lr40HAXZs7jfC9ytgcWaEjwlmnoFHSCTdlCIgJqEBzItXiz6ilk5zP35VwxdQTNoms7Meqg-ZrgRuLRaqXNtpQT9qWBmiFP6zoYLwymqytZ_P1SV9DCM-KIZQ2Cq-ew8PsYSrwl8GmfuegoKEMBiON1I4mC9ChDgzJmbo5P9te60T0URS5g9ExiQuSo-JXs0MKC57LLiNo3MPZvqR1wRJOTg0_FqS4JFxzPwN5OqhBzfs25f6kK-axf0wWp-LetUx3U1CTNaxfbCTK5tb8KnLOIs3lDwJw5H5wIIRgXnfNLBJT8A2srfXkdpA", + "eventCounters" => [], + "jsType" => "ch", + "cid" => "3i4vPGHCl74uIBumWaxF2a0PjFkyCJxcSvFnn~heX7keA6sGHgRbnzSYcMr_RzJFlfeeBdg9HeufzFPxnvSHbyWowSvjm6mNCSPhkx6MwDqptANoj93ZgGVbrzxGtcYe", + "ddk" => "78B13B7513D180B7AB6D6FF9EB0A51", + "Referer" => "https://www.qwant.com/?q={$search}&t=web", + "request" => "/?q={$search}&t=web", + "responsePage" => "origin", + "ddv" => "5.8.0" + ], + true + ); + }catch(Exception $error){ + + throw new Exception("Failed to fetch JS challenge token"); + } + + $datadome = json_decode($datadome, true); + if($datadome === null){ + + throw new Exception("Failed to decode JS challenge JSON"); + } + + if(!isset($datadome["cookie"])){ + + throw new Exception("Failed to get cookie from JS challenge endpoint"); + } + + $cookie = explode(";", $datadome["cookie"])[0]; + } + + return $cookie; + } } diff --git a/scraper/startpage.php b/scraper/startpage.php index a555dc5..0ce0e64 100644 --- a/scraper/startpage.php +++ b/scraper/startpage.php @@ -408,6 +408,8 @@ class startpage{ //$html = file_get_contents("scraper/startpage.html"); } + //echo htmlspecialchars($html); + $this->detect_captcha($html); if( @@ -570,8 +572,8 @@ class startpage{ $out["spelling"] = [ "type" => "including", - "using" => $json["render"]["query"], - "correction" => urldecode($category["results"][0]["query"]) + "using" => rawurldecode($json["render"]["query"]), + "correction" => html_entity_decode($category["results"][0]["query"]) ]; } break; @@ -1105,29 +1107,20 @@ class startpage{ if($page["name"] == "Next"){ - parse_str( - explode( - "?", - $page["url"], - 2 - )[1], - $str - ); - return $this->backend->store( http_build_query( [ "lui" => "english", "language" => "english", - "sc" => $str["sc"], + "sc" => $json["render"]["search_sc"], "t" => "device", "cat" => $pagetype, - "segment" => "startpage.udog", - "abd" => 0, - "abe" => 0, - "query" => $str["q"], - "page" => $str["page"], + "segment" => "organic", + "abd" => "0", + "abe" => "0", + "query" => rawurldecode($json["render"]["query"]), + "page" => $page["number"], "qsr" => "all", "qadf" => "none" // @ todo fix (??) ] diff --git a/scraper/vsco.php b/scraper/vsco.php index 8a7f057..083c362 100644 --- a/scraper/vsco.php +++ b/scraper/vsco.php @@ -30,19 +30,18 @@ class vsco{ curl_setopt($curlproc, CURLOPT_HTTPHEADER, ["User-Agent: " . config::USER_AGENT, - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", + "Accept-Encoding: gzip, deflate, br, zstd", "DNT: 1", "Sec-GPC: 1", "Connection: keep-alive", "Upgrade-Insecure-Requests: 1", "Sec-Fetch-Dest: document", "Sec-Fetch-Mode: navigate", - "Sec-Fetch-Site: same-origin", + "Sec-Fetch-Site: none", "Sec-Fetch-User: ?1", - "Priority: u=0, i", - "TE: trailers"] + "Priority: u=0, i"] ); }else{ @@ -73,8 +72,8 @@ class vsco{ curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); - // http2 bypass - curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + // http1 bypass?!?!?! + curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); $this->backend->assign_proxy($curlproc, $proxy); diff --git a/scraper/yep.php b/scraper/yep.php index 6e661bd..9624cc9 100644 --- a/scraper/yep.php +++ b/scraper/yep.php @@ -216,7 +216,7 @@ class yep{ ]; } - private function get($proxy, $url, $get = []){ + private function get($proxy, $url, $get = [], $use_api = false, $post_data = null, $bearer = null){ $curlproc = curl_init(); @@ -227,25 +227,41 @@ class yep{ curl_setopt($curlproc, CURLOPT_URL, $url); - // use http2 - curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + // use http1 lol + curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding - curl_setopt($curlproc, CURLOPT_HTTPHEADER, - ["User-Agent: " . config::USER_AGENT, - "Accept: */*", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip, deflate, br, zstd", - "Referer: https://yep.com/", - "Origin: https://yep.com", - "DNT: 1", - "Connection: keep-alive", - "Sec-Fetch-Dest: empty", - "Sec-Fetch-Mode: cors", - "Sec-Fetch-Site: same-site", - "Priority: u=4", - "TE: trailers"] - ); + + if($use_api){ + + $post_data = json_encode($post_data); + + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["Content-Type: application/json", + "Authorization: Bearer $bearer", + "Content-Length: " . strlen($post_data)] + ); + + curl_setopt($curlproc, CURLOPT_POST, true); + curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data); + }else{ + + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["User-Agent: " . config::USER_AGENT, + "Accept: */*", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br, zstd", + "Referer: https://yep.com/", + "Origin: https://yep.com", + "DNT: 1", + "Connection: keep-alive", + "Sec-Fetch-Dest: empty", + "Sec-Fetch-Mode: cors", + "Sec-Fetch-Site: same-site", + "Priority: u=4", + "TE: trailers"] + ); + } curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); @@ -270,6 +286,11 @@ class yep{ public function web($get){ + if(config::YEP_USE_API){ + + return $this->web_api($get); + } + $search = $get["s"]; if(strlen($search) === 0){ @@ -303,11 +324,11 @@ class yep{ try{ - // https://api.yep.com/fs/2/search?limit=20&query=asmr + // https://api.yep.com/search?limit=20&query=asmr $json = $this->get( $this->backend->get_ip(), - "https://api.yep.com/fs/2/search", + "https://api.yep.com/search", $filters ); @@ -392,6 +413,123 @@ class yep{ } + private function web_api($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + // parse filters + $filters = [ + "query" => $search, + "limit" => 99 // for some reason they dont show anything sometimes at 100 results + ]; + + if($get["nsfw"] == "no"){ $filters["safe_search"] = true; } + if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; } + + // add api key + $key_data = $this->backend->get_key(); + + try{ + + $json = + $this->get( + $this->backend->get_ip($key_data["increment"]), + "https://platform.yep.com/api/search", + [], + true, + $filters, + $key_data["key"] + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch JSON"); + } + + // should never happen + //$this->detect_cf($json); + + $json = json_decode($json, true); + //$json = json_decode(file_get_contents("scraper/yep.json"), true); + + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + + if(isset($json["error"])){ + + throw new Exception("Yep API returned an error: " . $json["error"]); + } + + if(isset($json["errors"])){ + + throw new Exception("Yep API returned the following errors: {$json["message"]}"); + } + + if( + isset($json["success"]) && + $json["success"] !== true + ){ + + throw new Exception("Yep API returned a false-y success value"); + } + + if(!isset($json["results"])){ + + throw new Exception("Yep API did not return a results object"); + } + + foreach($json["results"] as $item){ + + if( + $item["url"] === null || + $item["url"] == "" + ){ + + // sometimes API fucks up + continue; + } + + $out["web"][] = [ + "title" => $item["title"], + "description" => $item["description"], + "url" => $item["url"], + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + } + + return $out; + } + + private function detect_cf($payload){ // detect cloudflare page diff --git a/settings.php b/settings.php index 8b34425..11146f3 100644 --- a/settings.php +++ b/settings.php @@ -91,10 +91,6 @@ $settings = [ "value" => "qwant", "text" => "Qwant" ], - [ - "value" => "ghostery", - "text" => "Ghostery" - ], [ "value" => "yep", "text" => "Yep" @@ -169,6 +165,10 @@ $settings = [ "value" => "mojeek", "text" => "Mojeek" ], + [ + "value" => "naver", + "text" => "Naver" + ], [ "value" => "baidu", "text" => "Baidu" @@ -231,6 +231,10 @@ $settings = [ "value" => "qwant", "text" => "Qwant" ], + [ + "value" => "naver", + "text" => "Naver" + ], [ "value" => "baidu", "text" => "Baidu" @@ -325,6 +329,10 @@ $settings = [ "value" => "qwant", "text" => "Qwant" ], + [ + "value" => "naver", + "text" => "Naver" + ], [ "value" => "baidu", "text" => "Baidu" diff --git a/static/serverping.js b/static/serverping.js index 6b680d5..992e025 100644 --- a/static/serverping.js +++ b/static/serverping.js @@ -80,6 +80,15 @@ document.body.insertBefore(table, quote.nextSibling); // handle sorting clicks var tbody = table.getElementsByTagName("tbody")[0]; + +// server popup without inline javascript +tbody.addEventListener('click', function(event){ + var row = event.target.closest('.show_server'); + if(!row) return; + + show_server(row.id); +}) + var th = table.getElementsByTagName("th"); for(var i=0; i'; + html += ''; for(var i=0; i<7; i++){ @@ -406,6 +415,7 @@ function render_list(){ console.log(html); tbody.innerHTML = html; + } var popup_bg = document.getElementById("popup-bg"); diff --git a/static/themes/Rosé Pine.css b/static/themes/Rosé Pine.css new file mode 100644 index 0000000..cbc11fb --- /dev/null +++ b/static/themes/Rosé Pine.css @@ -0,0 +1,20 @@ +:root{ + /* background */ + --1d2021: #191724; + --282828: #1f1d2e; + --3c3836: #26233a; + --504945: #524f67; + + /* font */ + --928374: #e0def4; + --a89984: #908caa; + --bdae93: #6e6a86; + --8ec07c: #9ccfd8; + --ebdbb2: #f6c177; + + /* code highlighter */ + --comment: #6e6a86; + --default: #c4a7e7; + --keyword: #31748f; + --string: #f6c177; +}