Merge remote-tracking branch 'upstream/master' into upstream-merge

Reviewed-on: https://git.lolcat.ca/lolcat/4get/pulls/94

README.md

@@ -13,11 +13,11 @@ _NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._
 
 ## Totally unbiased comparison between alternatives
 
-|                            | 4get                    | searx(ng) | whoogle    | degoog                               |
+|                            | 4get                    | searx(ng) | whoogle    | degoog                                    |
-|----------------------------|-------------------------|-----------|------------|--------------------------------------|
+|----------------------------|-------------------------|-----------|------------|-------------------------------------------|
-| RAM usage                  | 100-400mb~              | 400mb-1GB | 100mb      | 200mb-1GB                            |
+| RAM usage                  | 100-400mb~              | 400mb-1GB | 100mb      | 100mb-250mb                               |
-| Does it suck               | no (debunked by snopes) | yes       | kind of?   | its kinda cool but no search filters |
+| Does it suck               | no (debunked by snopes) | yes       | kind of?   | hit and miss with search filters          |
-| Does it work               | ye                      | lmao      | shits dead | works right now...                   |
+| Does it work               | ye                      | lmao      | shits dead | works $rightNow, it's actually kinda cool |
 
 ## Features
 1. Rotating proxies on a per-scraper basis
@@ -62,4 +62,4 @@ Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/"
 Shit breaks all the time but I repair it all the time too. Email me here: <b>will (at) lolcat.ca</b> or create an issue.
 
 ## License
-AGPL
+AGPLv3-only

audio/linear.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/sc.php

@@ -1,224 +0,0 @@
-<?php
-
-new sc_audio();
-
-class sc_audio{
-	
-	public function __construct(){
-		
-		include "../data/config.php";
-		include "../lib/curlproxy.php";
-		$this->proxy = new proxy();
-		
-		if(isset($_GET["u"])){
-			
-			/*
-				we're now proxying audio
-			*/
-			$viewkey = $_GET["u"];
-			
-			if(!isset($_GET["r"])){
-				
-				$this->do404("Ranges(r) are missing");
-			}
-			
-			$ranges = explode(",", $_GET["r"]);
-			
-			// sanitize ranges
-			foreach($ranges as &$range){
-				
-				if(!is_numeric($range)){
-					
-					$this->do404("Invalid range specified");
-				}
-				
-				$range = (int)$range;
-			}
-			
-			// sort ranges (just to make sure)
-			sort($ranges);
-			
-			// convert ranges to pairs
-			$last = -1;
-			foreach($ranges as &$r){
-				
-				$tmp = $r;
-				$r = [$last + 1, $r];
-				
-				$last = $tmp;
-			}
-			
-			$browser_headers = getallheaders();
-			
-			// get the requested range from client
-			$client_range = 0;
-			foreach($browser_headers as $key => $value){
-				
-				if(strtolower($key) == "range"){
-					
-					preg_match(
-						'/bytes=([0-9]+)/',
-						$value,
-						$client_regex
-					);
-					
-					if(isset($client_regex[1])){
-						
-						$client_range = (int)$client_regex[1];
-					}else{
-						
-						$client_range = 0;
-					}
-					break;
-				}
-			}
-			
-			if(
-				$client_range < 0 ||
-				$client_range > $ranges[count($ranges) - 1][1]
-			){
-				
-				// range is not satisfiable
-				http_response_code(416);
-				header("Content-Type: text/plain");
-				die();
-			}
-			
-			$rng = null;
-			for($i=0; $i<count($ranges); $i++){
-				
-				if($ranges[$i][0] <= $client_range){
-					
-					$rng = $ranges[$i];
-				}
-			}
-			
-			// proxy data!
-			http_response_code(206); // partial content
-			header("Accept-Ranges: bytes");
-			header("Content-Range: bytes {$rng[0]}-{$rng[1]}/" . ($ranges[count($ranges) - 1][1] + 1));
-			
-			$viewkey =
-				preg_replace(
-					'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-					'/media/$1/' . $rng[0] . '/' . $rng[1],
-					$viewkey
-				);
-			
-			try{
-				
-				$this->proxy->stream_linear_audio(
-					$viewkey
-				);
-			}catch(Exception $error){
-				
-				$this->do404("Could not read stream");
-			}
-			
-			die();
-		}
-		
-		/*
-			redirect user to correct resource
-			we need to scrape and store the byte positions in the result URL
-		*/
-		if(!isset($_GET["s"])){
-			
-			$this->do404("The URL(s) parameter is missing");
-		}
-		
-		$viewkey = $_GET["s"];
-		
-		if(
-			preg_match(
-				'/soundcloud\.com$/',
-				parse_url($viewkey, PHP_URL_HOST)
-			) === false
-		){
-			
-			$this->do404("This endpoint can only be used for soundcloud streams");
-		}
-		
-		try{
-			
-			$json = $this->proxy->get($viewkey)["body"];
-		}catch(Exception $error){
-			
-			$this->do404("Curl error: " . $error->getMessage());
-		}
-		
-		$json = json_decode($json, true);
-		
-		if(!isset($json["url"])){
-			
-			$this->do404("Could not get URL from JSON");
-		}
-		
-		$viewkey = $json["url"];
-		
-		$m3u8 = $this->proxy->get($viewkey)["body"];
-		
-		$m3u8 = explode("\n", $m3u8);
-		
-		$lineout = null;
-		$streampos_arr = [];
-		foreach($m3u8 as $line){
-			
-			$line = trim($line);
-			if($line[0] == "#"){
-				
-				continue;
-			}
-			
-			if($lineout === null){
-				$lineout = $line;
-			}
-			
-			preg_match(
-				'/\/media\/[0-9]+\/([0-9]+)\/([0-9]+)/',
-				$line,
-				$matches
-			);
-			
-			if(isset($matches[0])){
-				
-				$streampos_arr[] = [
-					(int)$matches[1],
-					(int)$matches[2]
-				];
-			}
-		}
-		
-		if($lineout === null){
-			
-			$this->do404("Could not get stream URL");
-		}
-		
-		$lineout =
-			preg_replace(
-				'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-				'/media/$1/0/0',
-				$lineout
-			);
-		
-		$streampos = [];
-		
-		foreach($streampos_arr as $pos){
-			
-			$streampos[] = $pos[1];
-		}
-		
-		$streampos = implode(",", $streampos);
-		
-		header("Location: /audio/sc?u=" . urlencode($lineout) . "&r=$streampos");
-		header("Accept-Ranges: bytes");
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

audio/seekable.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/spotify.php

@@ -1,214 +0,0 @@
-<?php
-
-include "../data/config.php";
-new spotify();
-
-class spotify{
-	
-	public function __construct(){
-		
-		include "../lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-		
-		if(
-			!isset($_GET["s"]) ||
-			!preg_match(
-				'/^(track|episode)\.([A-Za-z0-9]{22})$/',
-				$_GET["s"],
-				$matches
-			)
-		){
-			
-			$this->do404("The track ID(s) parameter is missing or invalid");
-		}
-		
-		try{
-			
-			if($matches[1] == "episode"){
-				
-				$uri = "show";
-			}else{
-				
-				$uri = $matches[1];
-			}
-			
-			$embed =
-				$this->get("https://embed.spotify.com/{$uri}/" . $matches[2]);
-		}catch(Exception $error){
-			
-			$this->do404("Failed to fetch embed data");
-		}
-		
-		$this->fuckhtml->load($embed);
-		
-		$json =
-			$this->fuckhtml
-			->getElementById(
-				"__NEXT_DATA__",
-				"script"
-			);
-		
-		if($json === null){
-			
-			$this->do404("Failed to extract JSON");
-		}
-		
-		$json =
-			json_decode($json["innerHTML"], true);
-		
-		if($json === null){
-			
-			$this->do404("Failed to decode JSON");
-		}
-		
-		switch($matches[1]){
-			
-			case "track":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["audioPreview"]
-						["url"]
-					)
-				){
-					
-					header("Content-type: audio/mpeg");
-					header(
-						"Location: /audio/linear?s=" .
-						urlencode(
-							$json
-							["props"]
-							["pageProps"]
-							["state"]
-							["data"]
-							["entity"]
-							["audioPreview"]
-							["url"]
-						)
-					);
-				}else{
-					
-					$this->do404("Could not extract playback URL");
-				}
-				break;
-			
-			case "episode":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["id"]
-					)
-				){
-					
-					try{
-						$json =
-							$this->get(
-								"https://spclient.wg.spotify.com/soundfinder/v1/unauth/episode/" .
-								$json
-								["props"]
-								["pageProps"]
-								["state"]
-								["data"]
-								["entity"]
-								["id"] .
-								"/com.widevine.alpha"
-							);
-					}catch(Exception $error){
-						
-						$this->do404("Failed to fetch audio resource");
-					}
-					
-					$json = json_decode($json, true);
-					
-					if($json === null){
-						
-						$this->do404("Failed to decode audio resource JSON");
-					}
-					
-					if(
-						isset($json["passthrough"]) &&
-						$json["passthrough"] == "ALLOWED" &&
-						isset($json["passthroughUrl"])
-					){
-						
-						header(
-							"Location:" .
-							"/audio/linear.php?s=" .
-							urlencode(
-								str_replace(
-									"http://",
-									"https://",
-									$json["passthroughUrl"]
-								)
-							)
-						);
-					}else{
-						
-						$this->do404("Failed to find passthroughUrl");
-					}
-					
-				}else{
-					
-					$this->do404("Failed to find episode ID");
-				}
-				break;
-			}
-	}
-	
-	private function get($url){
-		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"
-		];
-		
-		$curlproc = curl_init();
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

data/api_keys/yep.txt

@@ -0,0 +1 @@
+# Paste Yep API keys here

data/config.php

@@ -30,6 +30,16 @@ class config{
     const SCRAPER_VIDEOS = "yt";
     const SCRAPER_MUSIC = "sc";
 	
+	//
+	// 4play (session provider)
+	//
+	// Enable 4play API?
+	const FPLAY_ENABLE_API = true;
+	
+	// 4play password. Please set this to something secure if you enable the 4play API.
+	// This password is used to POST sessions to /api/v2/provide_sesh
+	const FPLAY_PASSWORD = "1234";
+	
 	//
 	// BOT PROTECTION
 	//
@@ -125,10 +135,10 @@ class config{
 	
 	// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
 	// Changing this might break things.
-	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0";
+	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:151.0) Gecko/20100101 Firefox/151.0";
 	
 	// User agent to use with 4get-friendly APIs
-	const USER_AGENT_FRIENDLY = "4get-scrapist";
+	const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
 	
 	// Proxy pool assignments for each scraper
 	// false = Use server's raw IP
@@ -138,7 +148,6 @@ class config{
 	const PROXY_YAHOO = false;
 	const PROXY_YAHOO_JAPAN = false;
 	const PROXY_BRAVE = false;
-	const PROXY_FB = false; // facebook
 	const PROXY_GOOGLE = false;
 	const PROXY_GOOGLE_API = false;
 	const PROXY_GOOGLE_CSE = false;
@@ -162,7 +171,6 @@ class config{
 	const PROXY_VIMEO = false;
 	const PROXY_YEP = false;
 	const PROXY_PINTEREST = false;
-	const PROXY_SANKAKUCOMPLEX = false;
 	const PROXY_FLICKR = false;
 	const PROXY_PIXABAY = false;
 	const PROXY_UNSPLASH = false;
@@ -171,8 +179,6 @@ class config{
 	const PROXY_VSCO = false;
 	const PROXY_SEZNAM = false;
 	const PROXY_NAVER = false;
-	const PROXY_GREPPR = false;
-	const PROXY_CROWDVIEW = false;
 	const PROXY_MWMBL = false;
 	const PROXY_FTM = false; // findthatmeme
 	const PROXY_IMGUR = false;
@@ -180,6 +186,11 @@ class config{
 	const PROXY_YANDEX_W = false; // yandex web
 	const PROXY_YANDEX_I = false; // yandex images
 	const PROXY_YANDEX_V = false; // yandex videos
+	const PROXY_SAFEBOORU = false;
+	const PROXY_KONACHAN = false;
+	const PROXY_YANDERE = false;
+	const PROXY_TBIB = false;
+	const PROXY_GELBOORU = false;
 	
 	//
 	// Scraper-specific parameters
@@ -192,4 +203,7 @@ class config{
 	// Use "null" to default out to HTML scraping OR specify a string to
 	// use the API (Eg: "public"). API has less filters.
 	const MARGINALIA_API_KEY = null;
+	
+	// Yep
+	const YEP_USE_API = false;
 }

docs/apache2-example.md

@@ -49,9 +49,13 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
 	</Directory>
 
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
-		Order Deny,allow
+		Require all denied
-		Deny from all
+	</DirectoryMatch>
+
+	# remove directory listing from /icons
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
 

docs/apache2.md

@@ -104,9 +104,13 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
 	</Directory>
 	
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
-		Order Deny,allow
+		Require all denied
-		Deny from all
+	</DirectoryMatch>
+
+	# remove directory listing from /icons
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
 ```

docs/develop.md

@@ -0,0 +1,7 @@
+# Develop
+
+Start the PHP webserver like this, assuming you have curl-impersonate installed already
+
+```sh
+LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000
+```

scraper/baidu.php

@@ -666,14 +666,24 @@ class baidu{
 			//
 			// Don't parse as a search result if it's a card
 			//
-			$card =
+			$tpl_probe =
 				$this->fuckhtml
-				->getElementsByClassName(
+				->getElementsByAttributeValue(
-					"cosc-card",
+					"tpl",
-					$div
+					"www_index",
+					[$datafield]
 				);
 			
-			if(count($card) !== 0){
+			if(
+				count($tpl_probe) === 0 &&
+				count(
+					$this->fuckhtml
+					->getElementsByClassName(
+						"cosc-card",
+						$div
+					)
+				) !== 0
+			){
 				
 				//
 				// Parse chinese youtube shorts
@@ -870,10 +880,13 @@ class baidu{
 			// class:FYB_RD -> News garbage, IGNORE
 			
 			$result =
-				$this->fuckhtml
+				array_merge(
-				->getElementsByClassName(
+					$tpl_probe,
-					"result",
+					$this->fuckhtml
-					[$datafield]
+					->getElementsByClassName(
+						"result",
+						[$datafield]
+					)
 				);
 			
 			if(count($result) !== 0){
@@ -885,7 +898,7 @@ class baidu{
 				$title =
 					$this->fuckhtml
 					->getElementsByClassName(
-						"sc-link",
+						"cosc-title-a",
 						"a"
 					);
 				
@@ -905,8 +918,9 @@ class baidu{
 				
 				$description =
 					$this->fuckhtml
-					->getElementsByClassName(
+					->getElementsByAttributeValue(
-						"c-color",
+						"data-sanssr-cmpt",
+						"card/www-summary-1",
 						$div
 					);
 				

scraper/ddg.php

@@ -54,8 +54,8 @@ class ddg{
 				$headers =
 					["User-Agent: " . config::USER_AGENT,
 					"Accept: */*",
-					"Accept-Language: en-US,en;q=0.5",
+					"Accept-Language: en-US,en;q=0.9",
-					"Accept-Encoding: gzip",
+					"Accept-Encoding: gzip, deflate, br, zstd",
 					"Referer: https://duckduckgo.com/",
 					"DNT: 1",
 					"Sec-GPC: 1",
@@ -492,6 +492,17 @@ class ddg{
 				"form"
 			);
 		
+		foreach($forms as $form){
+			
+			if(
+				isset($form["attributes"]["id"]) &&
+				$form["attributes"]["id"] == "challenge-form"
+			){
+			
+				throw new Exception("DuckDuckGo returned an image captcha");
+			}
+		}
+		
 		foreach(array_reverse($forms) as $form){
 			
 			$this->fuckhtml->load($form);
@@ -909,112 +920,6 @@ class ddg{
 			){
 				
 				throw new Exception("DuckDuckGo returned a JSA challenge");
-				
-				// get JSA initial token
-				if(
-					!preg_match(
-						'/let jsa *= *([0-9]+)/',
-						$js,
-						$jsa
-					)
-				){
-					
-					$jsa = 0;
-				}else{
-					
-					$jsa = (int)$jsa[1];
-				}
-				
-				// get function bodies
-				preg_match_all(
-					'/let *([A-Za-z0-9]+) *= *function\(.*\) *{(.*)};/sU',
-					$js,
-					$functions
-				);
-				
-				$parsed_functions = [];
-				
-				for($i=0; $i<count($functions[0]); $i++){
-					
-					$functions[2][$i] = trim($functions[2][$i]);
-					
-					if(
-						preg_match(
-							'/return num *\* *([0-9]+)/i',
-							$functions[2][$i],
-							$num
-						)
-					){
-						
-						$parsed_functions[$functions[1][$i]] = [
-							"type" => "multiplication",
-							"num" => (int)$num[1]
-						];
-						continue;
-					}
-					
-					if(
-						preg_match(
-							'/innerHTML *= *`([^`]+)`/i',
-							$functions[2][$i],
-							$challenge
-						)
-					){
-						
-						$challenge[1] =
-							preg_replace(
-								'/<\/(br)>/',
-								'<$1>',
-								$challenge[1]
-							);
-						
-						$parsed_functions[$functions[1][$i]] = [
-							"type" => "challenge",
-							"text" => $challenge[1]
-						];
-					}
-				}
-				
-				// get function call order
-				preg_match_all(
-					'/jsa *= *([A-Za-z0-9]+)\(jsa\)/i',
-					$js,
-					$call_order
-				);
-				
-				foreach($call_order[1] as $order){
-					
-					if(!isset($parsed_functions[$order])){
-						
-						throw new Exception("JS challenge solve failure: DuckDuckGo called an unknown function");
-					}
-					
-					if($parsed_functions[$order]["type"] == "multiplication"){
-						
-						$jsa = $jsa * $parsed_functions[$order]["num"];
-						continue;
-					}
-					
-					if($parsed_functions[$order]["type"] == "challenge"){
-						
-						// @TODO get parsed length
-						//$parsed_functions[$order]["text"]
-						
-						$jsa = $jsa + strlen($parsed_functions[$order]["text"]);
-					}
-				}
-				
-				try{
-					$js = $this->get(
-						$proxy,
-						"https://links.duckduckgo.com" . $challenge_url[1] . $jsa,
-						[],
-						ddg::req_xhr
-					);
-				}catch(Exception $error){
-					
-					throw new Exception("Failed to get challenged d.js");
-				}
 			}
 			
 			//

scraper/ghostery.php

@@ -1,320 +0,0 @@
-<?php
-
-class ghostery{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("ghostery");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		if($page != "web"){
-			
-			return [];
-		}
-		
-		return [
-			"country" => [
-				"display" => "Country",
-				"option" => [
-					"any" => "All regions",
-					"AR" => "Argentina",
-					"AU" => "Australia",
-					"AT" => "Austria",
-					"BE" => "Belgium",
-					"BR" => "Brazil",
-					"CA" => "Canada",
-					"CL" => "Chile",
-					"DK" => "Denmark",
-					"FI" => "Finland",
-					"FR" => "France",
-					"DE" => "Germany",
-					"HK" => "Hong Kong",
-					"IN" => "India",
-					"ID" => "Indonesia",
-					"IT" => "Italy",
-					"JP" => "Japan",
-					"KR" => "Korea",
-					"MY" => "Malaysia",
-					"MX" => "Mexico",
-					"NL" => "Netherlands",
-					"NZ" => "New Zealand",
-					"NO" => "Norway",
-					"CN" => "People's Republic of China",
-					"PL" => "Poland",
-					"PT" => "Portugal",
-					"PH" => "Republic of the Philippines",
-					"RU" => "Russia",
-					"SA" => "Saudi Arabia",
-					"ZA" => "South Africa",
-					"ES" => "Spain",
-					"SE" => "Sweden",
-					"CH" => "Switzerland",
-					"TW" => "Taiwan",
-					"TR" => "Turkey",
-					"GB" => "United Kingdom",
-					"US" => "United States"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $country){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"Referer: https://ghosterysearch.com",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Cookie: ctry=" . ($country == "any" ? "--" : $country) . "; noads=true",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: same-origin",
-			"Sec-Fetch-User: ?1",
-			"Priority: u=0, i"]
-		);
-		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$query, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			parse_str($query, $query);
-			
-			// country
-			$country = $query["c"];
-			unset($query["c"]);
-			
-			$query = http_build_query($query);
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search?" . $query,
-						[],
-						$country
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}else{
-			
-			$proxy = $this->backend->get_ip();
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search",
-						[
-							"q" => $get["s"]
-						],
-						$get["country"]
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$this->fuckhtml->load($html);
-		
-		$results_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"results",
-				"section"
-			);
-		
-		if(count($results_wrapper) === 0){
-			
-			throw new Exception("Failed to grep result section");
-		}
-		
-		$this->fuckhtml->load($results_wrapper[0]);
-		
-		// get search results
-		$results =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"result",
-				"li"
-			);
-		
-		if(count($results) === 0){
-			
-			return $out;
-		}
-		
-		foreach($results as $result){
-			
-			$this->fuckhtml->load($result);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"url",
-					"a"
-				);
-			
-			if(count($a) === 0){
-				
-				continue;
-			}
-			
-			$a = $a[0];
-			
-			$out["web"][] = [
-				"title" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"h2"
-							)[0]
-						)
-					),
-				"description" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"p"
-							)[0]
-						)
-					),
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a
-						["attributes"]
-						["href"]
-					),
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		$this->fuckhtml->load($html);
-		
-		// get pagination token
-		$pagination_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"pagination",
-				"div"
-			);
-		
-		if(count($pagination_wrapper) !== 0){
-			
-			// found next page!
-			$this->fuckhtml->load($pagination_wrapper[0]);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"a"
-				);
-			
-			if(count($a) !== 0){
-				
-				$q =
-					parse_url(
-						$this->fuckhtml
-						->getTextContent(
-							$a[count($a) - 1]
-							["attributes"]
-							["href"]
-						),
-						PHP_URL_QUERY
-					);
-				
-				$out["npt"] =
-					$this->backend
-					->store(
-						$q . "&c=" . $get["country"],
-						"web",
-						$proxy
-					);
-			}
-		}
-		
-		return $out;
-	}
-	
-	private function titledots($title){
-		
-		return trim($title, " .\t\n\r\0\x0B…");
-	}
-}

scraper/marginalia.php

@@ -289,8 +289,18 @@ class marginalia{
 				throw new Exception("The API key used is rate limited. Please try again in a few minutes.");
 			}
 			
+			if(strpos($json, "<title>Bad Gateway (502)</title>")){
+				
+				throw new Exception("Marginalia returned a 502 error");
+			}
+			
 			$json = json_decode($json, true);
 			
+			if($json === null){
+				
+				throw new Exception("Failed to decode JSON");
+			}
+			
 			foreach($json["results"] as $result){
 				
 				$out["web"][] = [
@@ -490,6 +500,37 @@ class marginalia{
 			return $this->web($get, $redirect);
 		}
 		
+		// detect internal error
+		$infobox =
+			$this->fuckhtml
+			->getElementsByClassName("infobox");
+		
+		if(count($infobox) !== 0){
+			
+			foreach($infobox as $i){
+				
+				$this->fuckhtml->load($i);
+				
+				$h2 =
+					$this->fuckhtml
+					->getElementsByTagName("h2");
+				
+				if(
+					count($h2) !== 0 &&
+					$this->fuckhtml
+					->getTextContent(
+						$h2[0]
+					) == "Internal error"
+				){
+					
+					throw new Exception("Marginalia returned an internal server error");
+				}
+			}
+						
+			// reset
+			$this->fuckhtml->load($html);
+		}
+		
 		$sections =
 			$this->fuckhtml
 			->getElementsByClassName(

scraper/mojeek.php

@@ -340,7 +340,7 @@ class mojeek{
 							"zm" => "Zambia",
 							"zw" => "Zimbabwe"
 						]
-					],
+					],/*
 					"region" => [
 						"display" => "Region",
 						"option" => [
@@ -350,7 +350,7 @@ class mojeek{
 							"fr" => "France",
 							"uk" => "United Kingdom"
 						]
-					],
+					],*/
 					"domain" => [
 						"display" => "Results per domain",
 						"option" => [
@@ -458,7 +458,7 @@ class mojeek{
 				"date" => 1, // show date
 				"tlen" => 128, // max length of title
 				//"dlen" => 511, // max length of description
-				"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
+				//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
 			];
 			
 			switch($focus){
@@ -503,6 +503,11 @@ class mojeek{
 			}
 		}
 		
+		if(strlen($html) === 0){
+			
+			throw new Exception("Mojeek returned an empty page (probably a homepage redirect)");
+		}
+		
 		$out = [
 			"status" => "ok",
 			"spelling" => [

scraper/qwant.php

@@ -212,28 +212,58 @@ class qwant{
 		return $base;
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $is_post = false, $cookie = null){
-		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: application/json, text/plain, */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Origin: https://www.qwant.com",
-			"Referer: https://www.qwant.com/",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-site",
-			"TE: trailers"
-		];
 		
 		$curlproc = curl_init();
 		
-		if($get !== []){
+		if($is_post){
-			$get = http_build_query($get);
+			
-			$url .= "?" . $get;
+			$headers = [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://www.qwant.com/",
+				"Content-type: application/x-www-form-urlencoded",
+				"Origin: https://www.qwant.com",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache"
+			];
+			
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, http_build_query($get));
+		}else{
+			
+			$headers = [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: application/json, text/plain, */*",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://www.qwant.com/",
+				"Origin: https://www.qwant.com",
+				"DNT: 1",
+				($cookie !== null ? "Cookie: {$cookie}" : ""),
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Priority: u=4",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache",
+				"TE: trailers"
+			];
+						
+			if($get !== []){
+				$get = http_build_query($get);
+				$url .= "?" . $get;
+			}
 		}
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
@@ -271,6 +301,8 @@ class qwant{
 			[$params, $proxy] = $this->backend->get($get["npt"], "web");
 			
 			$params = json_decode($params, true);
+			$cookie = $params["cookie"];
+			unset($params["cookie"]);
 			
 		}else{
 			
@@ -297,7 +329,8 @@ class qwant{
 				"device" => "desktop",
 				"tgp" => 3,
 				"safesearch" => 0,
-				"displayed" => "true"
+				"displayed" => "true",
+				"llm" => "false"
 			];
 			
 			switch($get["nsfw"]){
@@ -306,18 +339,24 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
+			
+			$cookie = null;
 		}
 		/*
 		$handle = fopen("scraper/qwant_web.json", "r");
 		$json = fread($handle, filesize("scraper/qwant_web.json"));
 		fclose($handle);*/
 		
+		$cookie = $this->fuck_datadome($proxy, $search);
+		
 		try{
 			$json =
 				$this->get(
 					$proxy,
-					"https://fdn.qwant.com/v3/search/web",
+					"https://api.qwant.com/v3/search/web",
-					$params
+					$params,
+					false,
+					$cookie
 				);
 			
 		}catch(Exception $error){
@@ -353,6 +392,8 @@ class qwant{
 			"related" => []
 		];
 		
+		$this->detect_errors($json);
+		
 		if(
 			$json["status"] != "success" &&
 			$json["data"]["error_code"] === 5
@@ -362,8 +403,6 @@ class qwant{
 			return $out;
 		}
 		
-		$this->detect_errors($json);
-		
 		if(!isset($json["data"]["result"]["items"]["mainline"])){
 			
 			throw new Exception("Server did not return a result object");
@@ -436,6 +475,7 @@ class qwant{
 		if($json["data"]["result"]["lastPage"] === false){
 			
 			$params["offset"] = $params["offset"] + 10;
+			$params["cookie"] = $cookie;
 			
 			$out["npt"] =
 				$this->backend->store(
@@ -577,6 +617,9 @@ class qwant{
 				);
 			
 			$params = json_decode($params, true);
+			$cookie = $params["cookie"];
+			unset($params["cookie"]);
+			
 		}else{
 			
 			$search = $get["s"];
@@ -617,13 +660,19 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
+			
+			$cookie = null;
 		}
 		
+		$cookie = $this->fuck_datadome($proxy, $search);
+		
 		try{
 			$json = $this->get(
 				$proxy,
 				"https://api.qwant.com/v3/search/images",
 				$params,
+				false,
+				$cookie
 			);
 		}catch(Exception $err){
 			
@@ -1001,4 +1050,53 @@ class qwant{
 		
 		return "https://" . $image["host"] . "/th?id=" . rawurlencode($id);
 	}
+	
+	private function fuck_datadome($proxy, $search){
+		
+		if($cookie === null){
+			// get datadome bullshit
+			// yeah go ahead and patch this you code monkey
+			
+			// ... you know, instead of fixing this shit, can you make the second page of results return more than 1 result?
+			// i search shit like "higurashi" and i only get 1 result and then just a bunch of related searches ad finitum
+			// also, stop pretending you guys have your own index, this is clearly all bing results
+			try{
+				$datadome =
+					$this->get(
+						$proxy,
+						"https://dd.qwant.com/js/",
+						[
+							"jspl" => "-oirHDeP7G3G9LXY3rhxGDYf6t_ZlYPzaiL2dRhQ34uNUc4dO2Td0SmmKu_VJp7-jrAIoOMrTqKgCtV4cPzntMyNuOYN7QQ1q0Fi2c-679vYpJr9ub5dRFXMhCMQmjZsOWXUa8yafIXf85UA5Goct-n4hVT5dFfBGM-dE6gwZEFkHQ1j1195sLybZFTNA1IfX26MBMxlVHoyT-aHtkd6YjGXbvOPMMsObVHDa_Cf4BmApRGo-eehsAttsCqDRwkl0M2E1b0OXCIwMVnCK-qWZ81MGaJGjqxdj_LaxhAjE9JI_4oNedAg4Yj4m_vkQBKdmWJT_bTUsllAZXqX5IxYQHJ8WvlknOTr6sR3wgoSOa_JNVHkUQxqXBbQcoDdy6VciL5rIqwWl4Y2caVYzeXa8YNXxwsfrPDegH3dQ6vOiIiCDT3bexkcS_y7GAioPPv9qDu7M9y4sI8t0VkPgMbWY6MIu5u9WAv6lEiKIYrtlyU-iiXnNdkd9EnpC1TD8_s12PtCuLg5bGgLVmI1dw8VXVZxmVyFvfBtuqXkzM-kZOyv-YIRcf-wPY8DUtnHox1d6t64oYhGv-pYV5dmZ0rGs99LC6ThiifeF_oHz5ZK46Ms68WsiAjc1HLAbFPFxWw8X6lyLlpT5aHgFdDBeCTExKXyItlc9PBxkgK8UnXHQFiWX3HTen8rziwkI31AbkMU7xGP6cvuwuLG7MA7sH3yLF1uimQlhr6EuTrImWQNV7X81DqyZ0OE7I-uT9bp_xISa5it-6AJpFtGcwz04_OuO2zvZazBeiKJI5Fw0X2r3RHTHdA29SXqTIe4y2LRAAp0MSAZUIJAMIbbrz9BG-3D5BHfQUA1Bbuklxv-tNWiuanWbbTmAiuxTAo9P-D2ZMrWHwdN8sCb3nINLur38z1xviSs2qEEjKo9kzQIWoHgwm_amV-jjDtNBXwpCQnoaqt3FfL8e59ecYWVg61Gnl3YCwq6rubFdkIv6UStcWSlj4pRFlFZSjQxzznvGTAc5Nz1JJWVeBHJItei4wqDUYEw2BNTS_fU9keV3LFyKDH_x248AxHLiXt583DhZZwPAKlpmMo0E7odzfOgVQGM5b9l64NL3RQRq8cDxI8KgmOFnPHdI7Myq1A3nW1uoBB8m9_GR_9orboKl5tAQrT0VTMRd6JyNwOOQ71ZoQpMwsD5Xd9YDZd618TNE9dA38h9OyBco_-hwKCYayyEqib1atLLBO_gnO4InR471AklOokDOKx1oY_xgNhlFrZskE4nQQOqeCQ_47oysob9s3kblA8UqDrL2vM3i2KPLWwlQzMAPrQAIv7Amp6rO-wYwBQdbP7XghYympxx9-hNxc1Duq-nS4R3pjPtZg3LoCvG1N_0JLf9jfPBqw_rdPM6DnjEOvpCdvrfv4wBlEpgqbaRQCZBPEz-O5ZlY24OyPvaNX0krPeXkbi-KYuwZSAbfW0zsksXb8cbRmz6tvh25IRYP1XNDPVd8NFrYOdDH9GzhV5MYZziYMurQlVwgumGLz85Y6qTHZU2D5qkMH7o5C5bzptReO96889oxkReS_LDhzdxH0tAQ5a9GgETJTHAXJ9Lt7X037i0qQlatbPLizOl9fo4vUBGJ7w-uSUmIzg1yIwOo8xhLu39WghXfoJ_EX5ptNi7arpAo11UoJIGEgWgq7bS40q2dtnG5HYpp9ef37iFgztM7PV5dlJf_W-n9uXJyCk2nhxZeuGiIGhMTzjP_8oeyiMFelmoyLlqK8wOs6hvWqgDD4NwtZc-XBkEAKsxuPrirb_sszp0z6BWsh3D6fBxAW_lTrNVkxZ1EBtoEN72HvO1aeX8PVuJP3CKwkkUcAhJ5Zvajo8t4Ai8ave2MTZul4QhDIe7ZALJMo71u-hNSj8YG9JLPqF1jcnbkl8ykt_XMMxX8wSZ_AfTsfmAuOWBfEzxv8Fl2Ne-oxcEtfuYnYNJIjQDHXXdtYbWKBezE41wonQu7FWh3wSSF3DDm9EocWMsUZ610nrIviOFdtTqjrY7KnOZp0hS9Hm2V04z-06Db-To71L5O5poYKuOlDAP_ULfW0hgMgYaPnt4Nive9gtue6ZQ5yNFylKM11zCWcF8vmuEPGx4NIZjvNFB-QSkdxZtBTQOgO6r4BqHJNZIDqiD9A6PyMemVtQEo-JhooX6bwHFHx6C84r-kKq1lg0RaDelbu1h6347kocxOczdgHWlagKPFrUvHpPHvCZeFmzLLmPWbDEpUhhfkbEZlNe47svO5WN1TYZMCAlE0g01pe6b_NB9-twQ4FJB1SkUQRA4o5DxiIHSgXBBfw1X98XPEKcJgCU2zRbYuU69mvyPcsxoPsLo8Cl2OAXj2U1tcLUxVhR_vu3I0SG_Ee8hxBp5b6QbpIu2sE0bgpqfi88XqUXf3qvxGbkoZDZ8dPvEvCVmpDZOOGJY5Db4MnpKBByW79lUnpbLCbsVOoxYu8SVmVQ63eGY9yBlACuWxlXA_PhCpnKYwMppjxKvKqalNJGMMg5WQ8cvhsIftE5wgaj21n0A15GwRO1WgUcsO_dLe1MqO88hTHWHV0Q3gNut3sdsmVr_n-0SniLtVAaw4IEtDZWNUfhpaoC7iyKxoJ86t8r697EYPlWUu4nTQhb3TOXAjHF22rFsJJMldKHVxjxjd086ky3OY3L_7cbjVof60-y4gTJdumw5hch-mhaT93nVnaTuPLGCAC48dZSgOitW8Dw62mPS05fTPAfKUY1HpyIWr9ACk7U2KVAijpzFB-bXvqJeCyBeUHr4cB61K2XxTmkNA8Oo9RC32TeYDrThFzoUyYZ_D6zPe1lLtQaFsPRvDBSsYCcr5fcABlST4DyUgnkgqtx-0-D1QEwtWiH3xiHLQGa82ldjqNIXL9LqwxYoCRBKuljHyqEylZD26A9Jp6SL9rk_Ol6TJlYa3zx44PHyMoA3Coy-HIZW5NMwVKdV7egUDDOchvuFnc1Zv2QpSFKc0JY8lYFtSm1HZFwjq7hf2hBqkU0Xneh8AT4Ty9QVb4mJ8rpzc8m_NBZNrpZVE-pIzmv2yGjnRwKSrwsjat7b2iY34UPFqGBlDk5FX6EfRlN9bq-nztWdextLc47ndr1gxsjC-vr6tbdXl5FxMVhbsnLH80wOC6hHubxCysr5C7Ft59s3MLhVQCc3IIOPMBhLOjjjVsu6K8kAJvvamIpj-iYUQj9T8JztAqfobytzbiwPsQ3mywUGkJpZxTjXjqkKdT6yfFdW5ksR8p3VlKGAAKft9TKKcc5IgErmIRtiZF4hGhB45nwIO5XHA59V94QuV3b0oJrleuDCBj0iDFqWYpsUVHyKl3RjrO86Yq1IbNRwXIIMrlsVYos9CS1I75k9kW4yQC6ni3a2ZitDkiEB-SoQuTlBsvfTs892Wt5q3CWH6_7srZO6l2QEAlWqyQp_P4i8j8uiEbZksGsgghVIgOkriblDP9ml7PdkpRX7S3OpY7jjQgTNR8XHaZ-SEw0-zPiqIIt5VioFK8JljPn-nMIDx-ew_vTDKqmKEOF39ezQcc5oQNTQNUvTAe8qCbK4ghS3fagItezoHtB5zm3qwhx9Iu6JMJy0UhywUZBaInY6NokiVZGDgtoqlVjOk7K4299MVZgsOr2P9QHRDo41lYnCeWWFUf50d-ouEOUO_ngteDpgG1DfV64eycrUOWAnpfI8XOUZ7mxS89SqJriw3T3tcljowHiOFw9QjYsjJi0vuHL5jfoBZVdYAUH6KiLfIScO-KgqitdUTpFwks8Qk7xfd2HcEfArlzSiAt3Hmb2Ty-5mhvtQj2RbPehgoI7gqa60b6Qffth-5tbIjXoKlZCQsaKVMAggF9C3w0Rv5uJ3ExCTGsfKO9IVOf--AfxaA8cNoO0W3NTaMH2iWftY0O1-1x3VJEcerp2v9BcZ0iIL3VFJ9eyOPYNHNwTYrI3S6OrVZxuQXFbu2E_Us32wxYqXvnMx-DEOk7bUUFIanL69Cbn_mR_9Wm1sFixBQ15diJoe1Oyl3XWHWL6_YF4XnHzZlMWySt3Awdeynwu99Ter8WMGXl-6K30novKRWC9jDVOfAtxPvXBkkRtpm8nnJT8fwkmIQoWOEPT2MCg3VYzzHzVEH-AueuRiCZj_o7L-fRqbtFrWPyVnGuetUYS5u0D3n7Rr6kCVqWhLfwjqAwV_calzIQs8-J0-AJe6GGaaONPYfXwXjYan0Sl11CS64lXvx7i3MgeOZuH210T6g6hj54dSLw0upO1L8k0FGli3I5OhLMekeGMLT_N2akyHktjbz8Odpk2ms0pJP_PdqvUegjXgH81F5lr40HAXZs7jfC9ytgcWaEjwlmnoFHSCTdlCIgJqEBzItXiz6ilk5zP35VwxdQTNoms7Meqg-ZrgRuLRaqXNtpQT9qWBmiFP6zoYLwymqytZ_P1SV9DCM-KIZQ2Cq-ew8PsYSrwl8GmfuegoKEMBiON1I4mC9ChDgzJmbo5P9te60T0URS5g9ExiQuSo-JXs0MKC57LLiNo3MPZvqR1wRJOTg0_FqS4JFxzPwN5OqhBzfs25f6kK-axf0wWp-LetUx3U1CTNaxfbCTK5tb8KnLOIs3lDwJw5H5wIIRgXnfNLBJT8A2srfXkdpA",
+							"eventCounters" => [],
+							"jsType" => "ch",
+							"cid" => "3i4vPGHCl74uIBumWaxF2a0PjFkyCJxcSvFnn~heX7keA6sGHgRbnzSYcMr_RzJFlfeeBdg9HeufzFPxnvSHbyWowSvjm6mNCSPhkx6MwDqptANoj93ZgGVbrzxGtcYe",
+							"ddk" => "78B13B7513D180B7AB6D6FF9EB0A51",
+							"Referer" => "https://www.qwant.com/?q={$search}&t=web",
+							"request" => "/?q={$search}&t=web",
+							"responsePage" => "origin",
+							"ddv" => "5.8.0"
+						],
+						true
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JS challenge token");
+			}
+			
+			$datadome = json_decode($datadome, true);
+			if($datadome === null){
+				
+				throw new Exception("Failed to decode JS challenge JSON");
+			}
+			
+			if(!isset($datadome["cookie"])){
+				
+				throw new Exception("Failed to get cookie from JS challenge endpoint");
+			}
+			
+			$cookie = explode(";", $datadome["cookie"])[0];
+		}
+		
+		return $cookie;
+	}
 }

scraper/startpage.php

@@ -408,6 +408,8 @@ class startpage{
 			//$html = file_get_contents("scraper/startpage.html");
 		}
 		
+		//echo htmlspecialchars($html);
+		
 		$this->detect_captcha($html);
 		
 		if(
@@ -570,8 +572,8 @@ class startpage{
 						$out["spelling"] =
 							[
 								"type" => "including",
-								"using" => $json["render"]["query"],
+								"using" => rawurldecode($json["render"]["query"]),
-								"correction" => urldecode($category["results"][0]["query"])
+								"correction" => html_entity_decode($category["results"][0]["query"])
 							];
 					}
 					break;
@@ -1105,29 +1107,20 @@ class startpage{
 			
 			if($page["name"] == "Next"){
 				
-				parse_str(
-					explode(
-						"?",
-						$page["url"],
-						2
-					)[1],
-					$str
-				);
-				
 				return
 					$this->backend->store(
 						http_build_query(
 							[
 								"lui" => "english",
 								"language" => "english",
-								"sc" => $str["sc"],
+								"sc" => $json["render"]["search_sc"],
 								"t" => "device",
 								"cat" => $pagetype,
-								"segment" => "startpage.udog",
+								"segment" => "organic",
-								"abd" => 0,
+								"abd" => "0",
-								"abe" => 0,
+								"abe" => "0",
-								"query" => $str["q"],
+								"query" => rawurldecode($json["render"]["query"]),
-								"page" => $str["page"],
+								"page" => $page["number"],
 								"qsr" => "all",
 								"qadf" => "none" // @ todo fix (??)
 							]

scraper/vsco.php

@@ -30,19 +30,18 @@ class vsco{
 			
 			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
 				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
 				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
+				"Accept-Encoding: gzip, deflate, br, zstd",
 				"DNT: 1",
 				"Sec-GPC: 1",
 				"Connection: keep-alive",
 				"Upgrade-Insecure-Requests: 1",
 				"Sec-Fetch-Dest: document",
 				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: same-origin",
+				"Sec-Fetch-Site: none",
 				"Sec-Fetch-User: ?1",
-				"Priority: u=0, i",
+				"Priority: u=0, i"]
-				"TE: trailers"]
 			);
 		}else{
 			
@@ -73,8 +72,8 @@ class vsco{
 		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
 		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
 		
-		// http2 bypass
+		// http1 bypass?!?!?!
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 		
 		$this->backend->assign_proxy($curlproc, $proxy);
 		

scraper/yep.php

@@ -216,7 +216,7 @@ class yep{
 		];
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $use_api = false, $post_data = null, $bearer = null){
 		
 		$curlproc = curl_init();
 		
@@ -227,25 +227,41 @@ class yep{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
-		// use http2
+		// use http1 lol
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+		
-			["User-Agent: " . config::USER_AGENT,
+		if($use_api){
-			"Accept: */*",
+			
-			"Accept-Language: en-US,en;q=0.5",
+			$post_data = json_encode($post_data);
-			"Accept-Encoding: gzip, deflate, br, zstd",
+			
-			"Referer: https://yep.com/",
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			"Origin: https://yep.com",
+				["Content-Type: application/json",
-			"DNT: 1",
+				"Authorization: Bearer $bearer",
-			"Connection: keep-alive",
+				"Content-Length: " . strlen($post_data)]
-			"Sec-Fetch-Dest: empty",
+			);
-			"Sec-Fetch-Mode: cors",
+			
-			"Sec-Fetch-Site: same-site",
+			curl_setopt($curlproc, CURLOPT_POST, true);
-			"Priority: u=4",
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data);
-			"TE: trailers"]
+		}else{
-		);
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://yep.com/",
+				"Origin: https://yep.com",
+				"DNT: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Priority: u=4",
+				"TE: trailers"]
+			);
+		}
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -270,6 +286,11 @@ class yep{
 	
 	public function web($get){
 		
+		if(config::YEP_USE_API){
+			
+			return $this->web_api($get);
+		}
+		
 		$search = $get["s"];
 		if(strlen($search) === 0){
 			
@@ -303,11 +324,11 @@ class yep{
 		
 		try{
 			
-			// https://api.yep.com/fs/2/search?limit=20&query=asmr
+			// https://api.yep.com/search?limit=20&query=asmr
 			$json =
 				$this->get(
 					$this->backend->get_ip(),
-					"https://api.yep.com/fs/2/search",
+					"https://api.yep.com/search",
 					$filters
 				);
 			
@@ -392,6 +413,123 @@ class yep{
 	}
 	
 	
+	private function web_api($get){
+		
+		$search = $get["s"];
+		if(strlen($search) === 0){
+			
+			throw new Exception("Search term is empty!");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		// parse filters
+		$filters = [
+			"query" => $search,
+			"limit" => 99 // for some reason they dont show anything sometimes at 100 results
+		];
+		
+		if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
+		if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
+		
+		// add api key
+		$key_data = $this->backend->get_key();
+		
+		try{
+			
+			$json =
+				$this->get(
+					$this->backend->get_ip($key_data["increment"]),
+					"https://platform.yep.com/api/search",
+					[],
+					true,
+					$filters,
+					$key_data["key"]
+				);
+			
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		// should never happen
+		//$this->detect_cf($json);
+		
+		$json = json_decode($json, true);
+		//$json = json_decode(file_get_contents("scraper/yep.json"), true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(isset($json["error"])){
+			
+			throw new Exception("Yep API returned an error: " . $json["error"]);
+		}
+		
+		if(isset($json["errors"])){
+			
+			throw new Exception("Yep API returned the following errors: {$json["message"]}");
+		}
+		
+		if(
+			isset($json["success"]) &&
+			$json["success"] !== true
+		){
+			
+			throw new Exception("Yep API returned a false-y success value");
+		}
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Yep API did not return a results object");
+		}
+		
+		foreach($json["results"] as $item){
+			
+			if(
+				$item["url"] === null ||
+				$item["url"] == ""
+			){
+				
+				// sometimes API fucks up
+				continue;
+			}
+			
+			$out["web"][] = [
+				"title" => $item["title"],
+				"description" => $item["description"],
+				"url" => $item["url"],
+				"date" => null,
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		return $out;
+	}
+	
+	
 	private function detect_cf($payload){
 		
 		// detect cloudflare page

settings.php

@@ -91,10 +91,6 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
-					[
-						"value" => "ghostery",
-						"text" => "Ghostery"
-					],
 					[
 						"value" => "yep",
 						"text" => "Yep"
@@ -169,6 +165,10 @@ $settings = [
 						"value" => "mojeek",
 						"text" => "Mojeek"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"
@@ -231,6 +231,10 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"
@@ -325,6 +329,10 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"

static/serverping.js

@@ -80,6 +80,15 @@ document.body.insertBefore(table, quote.nextSibling);
 
 // handle sorting clicks
 var tbody = table.getElementsByTagName("tbody")[0];
+
+// server popup without inline javascript
+tbody.addEventListener('click', function(event){
+	var row = event.target.closest('.show_server');
+	if(!row) return;
+
+	show_server(row.id);
+})
+
 var th = table.getElementsByTagName("th");
 
 for(var i=0; i<th.length; i++){
@@ -343,7 +352,7 @@ function render_list(){
 	
 	for(var k=0; k<sorted_list.length; k++){
 		
-		html += '<tr onclick="show_server(' + sorted_list[k].index + ');">';
+		html += '<tr class="show_server" id="' + sorted_list[k].index + '">';
 		
 		for(var i=0; i<7; i++){
 			
@@ -406,6 +415,7 @@ function render_list(){
 	console.log(html);
 	
 	tbody.innerHTML = html;
+
 }
 
 var popup_bg = document.getElementById("popup-bg");

static/themes/Rosé Pine.css

@@ -0,0 +1,20 @@
+:root{
+	/* background */
+	--1d2021: #191724;
+	--282828: #1f1d2e;
+	--3c3836: #26233a;
+	--504945: #524f67;
+
+	/* font */
+	--928374: #e0def4;
+	--a89984: #908caa;
+	--bdae93: #6e6a86;
+	--8ec07c: #9ccfd8;
+	--ebdbb2: #f6c177;
+
+	/* code highlighter */
+	--comment: #6e6a86;
+	--default: #c4a7e7;
+	--keyword: #31748f;
+	--string: #f6c177;
+}