Merge remote-tracking branch 'upstream/master' into upstream-merge

Reviewed-on: https://git.lolcat.ca/lolcat/4get/pulls/94

Dockerfile

@@ -2,16 +2,22 @@ FROM alpine:3.21
 WORKDIR /var/www/html/4get
 
 RUN apk update && apk upgrade
-RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg
+RUN apk add php84 apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg libstdc++
 
 COPY . .
 
+RUN mv ./shared_objects/libcurl-impersonate.so /usr/local/lib
 RUN chmod 777 /var/www/html/4get/icons
+RUN ln -s /usr/bin/php84 /usr/bin/php
 
 EXPOSE 80
 EXPOSE 443
 
+ENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate.so
+ENV CURL_IMPERSONATE=chrome146
+
 ENV FOURGET_PROTO=http
+ENV FOURGET_USER_AGENT='Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.7680.80 Safari/537.36'
 
 ENTRYPOINT  ["./docker/docker-entrypoint.sh"]
-CMD ["start"]
+CMD ["start"]

README.md

@@ -13,11 +13,11 @@ _NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._
 
 ## Totally unbiased comparison between alternatives
 
-|                            | 4get                    | searx(ng) | libreY      | araa      | hearch.co         |
-|----------------------------|-------------------------|-----------|-------------|-----------|-------------------|
-| RAM usage                  | 200-400mb~              | 2GB~      | 200-400mb~  | 2GB~      | idk               |
-| Does it suck               | no (debunked by snopes) | yes       | yes         | a little  | better than searx |
-| Does it work               | ye                      | sometimes | sometimes   | sometimes | yes               |
+|                            | 4get                    | searx(ng) | whoogle    | degoog                                    |
+|----------------------------|-------------------------|-----------|------------|-------------------------------------------|
+| RAM usage                  | 100-400mb~              | 400mb-1GB | 100mb      | 100mb-250mb                               |
+| Does it suck               | no (debunked by snopes) | yes       | kind of?   | hit and miss with search filters          |
+| Does it work               | ye                      | lmao      | shits dead | works $rightNow, it's actually kinda cool |
 
 ## Features
 1. Rotating proxies on a per-scraper basis
@@ -31,25 +31,29 @@ tl;dr 4get is the best way to browse for shit.
 
 # Supported websites
 
-| Web        | Images       | Videos       | News       | Music      | Autocompleter |
-|------------|--------------|--------------|------------|------------|---------------|
-| DuckDuckGo | DuckDuckGo   | YouTube      | DuckDuckGo | Soundcloud | Brave         |
-| Brave      | Brave        | Sepia Search | Brave      |            | DuckDuckGo    |
-| Yandex     | Yandex       | DuckDuckGo   | Google     |            | Yandex        |
-| Google     | Google       | Brave        | Startpage  |            | Google        |
-| Startpage  | Startpage    | Yandex       | Qwant      |            | Startpage     |
-| Qwant      | Qwant        | Google       | Mojeek     |            | Kagi          |
-| Ghostery   | Yep          | Startpage    | Baidu      |            | Qwant         |
-| Yep        | Baidu        | Qwant        |            |            | Ghostery      |
-| Greppr     | Pinterest    | Baidu        |            |            | Yep           |
-| Crowdview  | 500px        | Coc Coc      |            |            | Marginalia    |
-| Mwmbl      | VSCO         |              |            |            | YouTube       |
-| Mojeek     | Imgur        |              |            |            | Soundcloud    |
-| Baidu      | FindThatMeme |              |            |            |               |
-| Coc Coc    |              |              |            |            |               |
-| Marginalia |              |              |            |            |               |
-| wiby       |              |              |            |            |               |
-| Curlie     |              |              |            |            |               |
+| web          | images       | videos       | news         | music      | autocomplete |
+|--------------|--------------|--------------|--------------|------------|--------------|
+| DuckDuckGo   | DuckDuckGo   | YouTube      | DuckDuckGo   | SoundCloud | Brave        |
+| Brave        | Yandex       | Vimeo        | Brave        | Swisscows  | DuckDuckGo   |
+| Yandex       | Brave        | Sepia Search | Google       |            | Yandex       |
+| Google       | Google       | DuckDuckGo   | Yahoo! JAPAN |            | Google       |
+| Google API   | Google API   | Brave        | Startpage    |            | Startpage    |
+| Google CSE   | Google CSE   | Yandex       | Qwant        |            | Kagi         |
+| Yahoo! JAPAN | Yahoo! JAPAN | Google       | Mojeek       |            | Qwant        |
+| Startpage    | Startpage    | Yahoo! JAPAN | Baidu        |            | Ghostery     |
+| Qwant        | Qwant        | Startpage    |              |            | Yep          |
+| Ghostery     | Baidu        | Qwant        |              |            | Marginalia   |
+| Yep          | Solofield    | Baidu        |              |            | YouTube      |
+| Mwmbl        | Pinterest    | Coc Coc      |              |            | SoundCloud   |
+| Mojeek       | Cara         | Solofield    |              |            |              |
+| Baidu        | Flickr       |              |              |            |              |
+| Coc Coc      | Pexels       |              |              |            |              |
+| Solofield    | Pixabay      |              |              |            |              |
+| Marginalia   | Unsplash     |              |              |            |              |
+| wiby         | 500px        |              |              |            |              |
+|              | VSCO         |              |              |            |              |
+|              | Imgur        |              |              |            |              |
+|              | FindThatMeme |              |              |            |              |
 
 # Installation
 Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/">documentation index</a>. I recommend following the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">apache2 guide</a>.
@@ -58,4 +62,4 @@ Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/"
 Shit breaks all the time but I repair it all the time too. Email me here: <b>will (at) lolcat.ca</b> or create an issue.
 
 ## License
-AGPL
+AGPLv3-only

audio/linear.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/sc.php

@@ -1,224 +0,0 @@
-<?php
-
-new sc_audio();
-
-class sc_audio{
-	
-	public function __construct(){
-		
-		include "../data/config.php";
-		include "../lib/curlproxy.php";
-		$this->proxy = new proxy();
-		
-		if(isset($_GET["u"])){
-			
-			/*
-				we're now proxying audio
-			*/
-			$viewkey = $_GET["u"];
-			
-			if(!isset($_GET["r"])){
-				
-				$this->do404("Ranges(r) are missing");
-			}
-			
-			$ranges = explode(",", $_GET["r"]);
-			
-			// sanitize ranges
-			foreach($ranges as &$range){
-				
-				if(!is_numeric($range)){
-					
-					$this->do404("Invalid range specified");
-				}
-				
-				$range = (int)$range;
-			}
-			
-			// sort ranges (just to make sure)
-			sort($ranges);
-			
-			// convert ranges to pairs
-			$last = -1;
-			foreach($ranges as &$r){
-				
-				$tmp = $r;
-				$r = [$last + 1, $r];
-				
-				$last = $tmp;
-			}
-			
-			$browser_headers = getallheaders();
-			
-			// get the requested range from client
-			$client_range = 0;
-			foreach($browser_headers as $key => $value){
-				
-				if(strtolower($key) == "range"){
-					
-					preg_match(
-						'/bytes=([0-9]+)/',
-						$value,
-						$client_regex
-					);
-					
-					if(isset($client_regex[1])){
-						
-						$client_range = (int)$client_regex[1];
-					}else{
-						
-						$client_range = 0;
-					}
-					break;
-				}
-			}
-			
-			if(
-				$client_range < 0 ||
-				$client_range > $ranges[count($ranges) - 1][1]
-			){
-				
-				// range is not satisfiable
-				http_response_code(416);
-				header("Content-Type: text/plain");
-				die();
-			}
-			
-			$rng = null;
-			for($i=0; $i<count($ranges); $i++){
-				
-				if($ranges[$i][0] <= $client_range){
-					
-					$rng = $ranges[$i];
-				}
-			}
-			
-			// proxy data!
-			http_response_code(206); // partial content
-			header("Accept-Ranges: bytes");
-			header("Content-Range: bytes {$rng[0]}-{$rng[1]}/" . ($ranges[count($ranges) - 1][1] + 1));
-			
-			$viewkey =
-				preg_replace(
-					'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-					'/media/$1/' . $rng[0] . '/' . $rng[1],
-					$viewkey
-				);
-			
-			try{
-				
-				$this->proxy->stream_linear_audio(
-					$viewkey
-				);
-			}catch(Exception $error){
-				
-				$this->do404("Could not read stream");
-			}
-			
-			die();
-		}
-		
-		/*
-			redirect user to correct resource
-			we need to scrape and store the byte positions in the result URL
-		*/
-		if(!isset($_GET["s"])){
-			
-			$this->do404("The URL(s) parameter is missing");
-		}
-		
-		$viewkey = $_GET["s"];
-		
-		if(
-			preg_match(
-				'/soundcloud\.com$/',
-				parse_url($viewkey, PHP_URL_HOST)
-			) === false
-		){
-			
-			$this->do404("This endpoint can only be used for soundcloud streams");
-		}
-		
-		try{
-			
-			$json = $this->proxy->get($viewkey)["body"];
-		}catch(Exception $error){
-			
-			$this->do404("Curl error: " . $error->getMessage());
-		}
-		
-		$json = json_decode($json, true);
-		
-		if(!isset($json["url"])){
-			
-			$this->do404("Could not get URL from JSON");
-		}
-		
-		$viewkey = $json["url"];
-		
-		$m3u8 = $this->proxy->get($viewkey)["body"];
-		
-		$m3u8 = explode("\n", $m3u8);
-		
-		$lineout = null;
-		$streampos_arr = [];
-		foreach($m3u8 as $line){
-			
-			$line = trim($line);
-			if($line[0] == "#"){
-				
-				continue;
-			}
-			
-			if($lineout === null){
-				$lineout = $line;
-			}
-			
-			preg_match(
-				'/\/media\/[0-9]+\/([0-9]+)\/([0-9]+)/',
-				$line,
-				$matches
-			);
-			
-			if(isset($matches[0])){
-				
-				$streampos_arr[] = [
-					(int)$matches[1],
-					(int)$matches[2]
-				];
-			}
-		}
-		
-		if($lineout === null){
-			
-			$this->do404("Could not get stream URL");
-		}
-		
-		$lineout =
-			preg_replace(
-				'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-				'/media/$1/0/0',
-				$lineout
-			);
-		
-		$streampos = [];
-		
-		foreach($streampos_arr as $pos){
-			
-			$streampos[] = $pos[1];
-		}
-		
-		$streampos = implode(",", $streampos);
-		
-		header("Location: /audio/sc?u=" . urlencode($lineout) . "&r=$streampos");
-		header("Accept-Ranges: bytes");
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

audio/seekable.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/spotify.php

@@ -1,214 +0,0 @@
-<?php
-
-include "../data/config.php";
-new spotify();
-
-class spotify{
-	
-	public function __construct(){
-		
-		include "../lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-		
-		if(
-			!isset($_GET["s"]) ||
-			!preg_match(
-				'/^(track|episode)\.([A-Za-z0-9]{22})$/',
-				$_GET["s"],
-				$matches
-			)
-		){
-			
-			$this->do404("The track ID(s) parameter is missing or invalid");
-		}
-		
-		try{
-			
-			if($matches[1] == "episode"){
-				
-				$uri = "show";
-			}else{
-				
-				$uri = $matches[1];
-			}
-			
-			$embed =
-				$this->get("https://embed.spotify.com/{$uri}/" . $matches[2]);
-		}catch(Exception $error){
-			
-			$this->do404("Failed to fetch embed data");
-		}
-		
-		$this->fuckhtml->load($embed);
-		
-		$json =
-			$this->fuckhtml
-			->getElementById(
-				"__NEXT_DATA__",
-				"script"
-			);
-		
-		if($json === null){
-			
-			$this->do404("Failed to extract JSON");
-		}
-		
-		$json =
-			json_decode($json["innerHTML"], true);
-		
-		if($json === null){
-			
-			$this->do404("Failed to decode JSON");
-		}
-		
-		switch($matches[1]){
-			
-			case "track":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["audioPreview"]
-						["url"]
-					)
-				){
-					
-					header("Content-type: audio/mpeg");
-					header(
-						"Location: /audio/linear?s=" .
-						urlencode(
-							$json
-							["props"]
-							["pageProps"]
-							["state"]
-							["data"]
-							["entity"]
-							["audioPreview"]
-							["url"]
-						)
-					);
-				}else{
-					
-					$this->do404("Could not extract playback URL");
-				}
-				break;
-			
-			case "episode":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["id"]
-					)
-				){
-					
-					try{
-						$json =
-							$this->get(
-								"https://spclient.wg.spotify.com/soundfinder/v1/unauth/episode/" .
-								$json
-								["props"]
-								["pageProps"]
-								["state"]
-								["data"]
-								["entity"]
-								["id"] .
-								"/com.widevine.alpha"
-							);
-					}catch(Exception $error){
-						
-						$this->do404("Failed to fetch audio resource");
-					}
-					
-					$json = json_decode($json, true);
-					
-					if($json === null){
-						
-						$this->do404("Failed to decode audio resource JSON");
-					}
-					
-					if(
-						isset($json["passthrough"]) &&
-						$json["passthrough"] == "ALLOWED" &&
-						isset($json["passthroughUrl"])
-					){
-						
-						header(
-							"Location:" .
-							"/audio/linear.php?s=" .
-							urlencode(
-								str_replace(
-									"http://",
-									"https://",
-									$json["passthroughUrl"]
-								)
-							)
-						);
-					}else{
-						
-						$this->do404("Failed to find passthroughUrl");
-					}
-					
-				}else{
-					
-					$this->do404("Failed to find episode ID");
-				}
-				break;
-			}
-	}
-	
-	private function get($url){
-		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"
-		];
-		
-		$curlproc = curl_init();
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

data/api_keys/yep.txt

@@ -0,0 +1 @@
+# Paste Yep API keys here

data/config.php

@@ -22,6 +22,23 @@ class config{
 	
 	// Enable the API?
 	const API_ENABLED = true;
+
+    // Default scrapers
+    const SCRAPER_WEB = "ddg";
+    const SCRAPER_IMAGES = "ddg";
+    const SCRAPER_NEWS = "ddg";
+    const SCRAPER_VIDEOS = "yt";
+    const SCRAPER_MUSIC = "sc";
+	
+	//
+	// 4play (session provider)
+	//
+	// Enable 4play API?
+	const FPLAY_ENABLE_API = true;
+	
+	// 4play password. Please set this to something secure if you enable the 4play API.
+	// This password is used to POST sessions to /api/v2/provide_sesh
+	const FPLAY_PASSWORD = "1234";
 	
 	//
 	// BOT PROTECTION
@@ -118,10 +135,10 @@ class config{
 	
 	// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
 	// Changing this might break things.
-	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0";
+	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:151.0) Gecko/20100101 Firefox/151.0";
 	
 	// User agent to use with 4get-friendly APIs
-	const USER_AGENT_FRIENDLY = "4get-scrapist";
+	const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
 	
 	// Proxy pool assignments for each scraper
 	// false = Use server's raw IP
@@ -131,7 +148,6 @@ class config{
 	const PROXY_YAHOO = false;
 	const PROXY_YAHOO_JAPAN = false;
 	const PROXY_BRAVE = false;
-	const PROXY_FB = false; // facebook
 	const PROXY_GOOGLE = false;
 	const PROXY_GOOGLE_API = false;
 	const PROXY_GOOGLE_CSE = false;
@@ -155,7 +171,6 @@ class config{
 	const PROXY_VIMEO = false;
 	const PROXY_YEP = false;
 	const PROXY_PINTEREST = false;
-	const PROXY_SANKAKUCOMPLEX = false;
 	const PROXY_FLICKR = false;
 	const PROXY_PIXABAY = false;
 	const PROXY_UNSPLASH = false;
@@ -164,8 +179,6 @@ class config{
 	const PROXY_VSCO = false;
 	const PROXY_SEZNAM = false;
 	const PROXY_NAVER = false;
-	const PROXY_GREPPR = false;
-	const PROXY_CROWDVIEW = false;
 	const PROXY_MWMBL = false;
 	const PROXY_FTM = false; // findthatmeme
 	const PROXY_IMGUR = false;
@@ -173,6 +186,11 @@ class config{
 	const PROXY_YANDEX_W = false; // yandex web
 	const PROXY_YANDEX_I = false; // yandex images
 	const PROXY_YANDEX_V = false; // yandex videos
+	const PROXY_SAFEBOORU = false;
+	const PROXY_KONACHAN = false;
+	const PROXY_YANDERE = false;
+	const PROXY_TBIB = false;
+	const PROXY_GELBOORU = false;
 	
 	//
 	// Scraper-specific parameters
@@ -185,4 +203,7 @@ class config{
 	// Use "null" to default out to HTML scraping OR specify a string to
 	// use the API (Eg: "public"). API has less filters.
 	const MARGINALIA_API_KEY = null;
+	
+	// Yep
+	const YEP_USE_API = false;
 }

docker/docker-entrypoint.sh

@@ -23,9 +23,8 @@ else
         cp -r ${FOURGET_SRC}/docker/apache/http/conf.d/* /etc/apache2/conf.d
 fi
 
-php ./docker/gen_config.php
-
 if [ "$@" = "start" ]; then
+        php ./docker/gen_config.php
         echo "4get is running"
         exec httpd -DFOREGROUND
 else 

docs/apache2-example.md

@@ -49,9 +49,13 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
 	</Directory>
 
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
-		Order Deny,allow
-		Deny from all
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+		Require all denied
+	</DirectoryMatch>
+
+	# remove directory listing from /icons
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
 

docs/apache2.md

@@ -104,9 +104,13 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
 	</Directory>
 	
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
-		Order Deny,allow
-		Deny from all
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+		Require all denied
+	</DirectoryMatch>
+
+	# remove directory listing from /icons
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
 ```

docs/develop.md

@@ -0,0 +1,7 @@
+# Develop
+
+Start the PHP webserver like this, assuming you have curl-impersonate installed already
+
+```sh
+LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000
+```

lib/frontend.php

@@ -276,7 +276,7 @@ class frontend{
 				
 				$payload .=
 					'">' .
-						'<img class="thumb" src="' . $this->htmlimage($site["thumb"]["url"], $size) . '" alt="thumb">';
+						'<img loading="lazy" class="thumb" src="' . $this->htmlimage($site["thumb"]["url"], $size) . '" alt="thumb">';
 				
 				if($duration !== null){
 					
@@ -499,7 +499,7 @@ class frontend{
 			$payload =
 				'<div class="url">' .
 					'<button class="favicon" tabindex="-1">' .
-						'<img src="/favicon?s=404" alt="xx">' .
+						'<img loading="lazy" src="/favicon?s=404" alt="xx">' .
 					'</button>';
 		}else{
 			
@@ -528,18 +528,18 @@ class frontend{
 			$payload =
 				'<div class="url">' .
 					'<button class="favicon" tabindex="-1">' .
-						'<img src="/favicon?s=' . htmlspecialchars($host["scheme"] . "://" . $host["host"]) . '" alt="' . htmlspecialchars($esc) . '">' .
-						//'<img src="/404.php" alt="' . htmlspecialchars($esc) . '">' .
+						'<img loading="lazy" src="/favicon?s=' . htmlspecialchars($host["scheme"] . "://" . $host["host"]) . '" alt="' . htmlspecialchars($esc) . '">' .
+						//'<img loading="lazy" src="/404.php" alt="' . htmlspecialchars($esc) . '">' .
 					'</button>' .
 					'<div class="favicon-dropdown">';
 			
 			$payload .=
-					'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
-					'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
-					'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
-					'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
-					'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
-					'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
+					'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
+					'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
+					'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
+					'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
+					'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
+					'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
 				'</div>';
 		}
 		
@@ -593,7 +593,7 @@ class frontend{
 	
 	public function getscraperfilters($page){
 		
-		$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : null;
+		$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : "";
 		
 		if(
 			isset($_GET["scraper"]) &&
@@ -619,163 +619,23 @@ class frontend{
 			}
 		}
 		
+        include "lib/scrapers.php";
+
 		// add search field
 		$filters =
 			[
 				"s" => [
 					"option" => "_SEARCH"
-				]
+				],
+				"scraper" => [
+					"display" => "Scraper",
+					"option" => AVAILABLE_SCRAPERS[$page],
+				],
 			];
-		
-		// define default scrapers
-		switch($page){
-			
-			case "web":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"ddg" => "DuckDuckGo",
-						//"yahoo" => "Yahoo!",
-						"brave" => "Brave",
-						"yandex" => "Yandex",
-						"google" => "Google",
-						"google_api" => "Google API",
-						"google_cse" => "Google CSE",
-						"yahoo_japan" => "Yahoo! JAPAN",
-						"startpage" => "Startpage",
-						"qwant" => "Qwant",
-						"ghostery" => "Ghostery",
-						"yep" => "Yep",
-						"mwmbl" => "Mwmbl",
-						"mojeek" => "Mojeek",
-						"baidu" => "Baidu",
-						"coccoc" => "Cốc Cốc",
-						"solofield" => "Solofield",
-						"marginalia" => "Marginalia",
-						"wiby" => "wiby"
-					]
-				];
-				break;
-			
-			case "images":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"ddg" => "DuckDuckGo",
-						"yandex" => "Yandex",
-						"brave" => "Brave",
-						"google" => "Google",
-						"google_api" => "Google API",
-						"google_cse" => "Google CSE",
-						"yahoo_japan" => "Yahoo! JAPAN",
-						"startpage" => "Startpage",
-						"qwant" => "Qwant",
-						"baidu" => "Baidu",
-						"solofield" => "Solofield",
-						"pinterest" => "Pinterest",
-						"cara" => "Cara",
-						"flickr" => "Flickr",
-						"pexels" => "Pexels",
-						"pixabay" => "Pixabay",
-						"unsplash" => "Unsplash",
-						"fivehpx" => "500px",
-						"vsco" => "VSCO",
-						"imgur" => "Imgur",
-						"ftm" => "FindThatMeme"
-					]
-				];
-				break;
-			
-			case "videos":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"yt" => "YouTube",
-						//"archiveorg" => "Archive.org",
-						"vimeo" => "Vimeo",
-						//"odysee" => "Odysee",
-						"sepiasearch" => "Sepia Search",
-						//"fb" => "Facebook videos",
-						"ddg" => "DuckDuckGo",
-						"brave" => "Brave",
-						"yandex" => "Yandex",
-						"google" => "Google",
-						"yahoo_japan" => "Yahoo! JAPAN",
-						"startpage" => "Startpage",
-						"qwant" => "Qwant",
-						"baidu" => "Baidu",
-						"coccoc" => "Cốc Cốc",
-						"solofield" => "Solofield"
-					]
-				];
-				break;
-			
-			case "news":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"ddg" => "DuckDuckGo",
-						"brave" => "Brave",
-						"google" => "Google",
-						"yahoo_japan" => "Yahoo! JAPAN",
-						"startpage" => "Startpage",
-						"qwant" => "Qwant",
-						"mojeek" => "Mojeek",
-						"baidu" => "Baidu"
-					]
-				];
-				break;
-			
-			case "music":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"sc" => "SoundCloud",
-						"swisscows" => "Swisscows (SoundCloud)"
-						//"spotify" => "Spotify"
-					]
-				];
-				break;
-			
-			case "booru":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"safebooru" => "Safebooru",
-						"konachan" => "Konachan",
-						"tbib" => "The Big Imageboard",
-						"gelbooru" => "Gelbooru",
-						"yandere" => "Yande.re",
-						"tbib" => "The Big Imageboard",
-						"sankakucomplex" => "SankakuComplex",
-						"soybooru" => "SoyBooru"
-					]
-				];
-				break;
-		}
-		
+
 		// get scraper name from user input, or default out to preferred scraper
-		$scraper_out = null;
-		$first = true;
-		
-		foreach($filters["scraper"]["option"] as $scraper_name => $scraper_pretty){
-			
-			if($first === true){
-				
-				$first = $scraper_name;
-			}
-			
-			if($scraper_name == $get_scraper){
-				
-				$scraper_out = $scraper_name;
-			}
-		}
-		
-		if($scraper_out === null){
-			
-			$scraper_out = $first;
-		}
-		
+		$scraper_out = get_scraper($page, $get_scraper);
+
 		include "scraper/$scraper_out.php";
 		$lib = new $scraper_out();
 		

lib/fuckhtml.php

@@ -553,28 +553,21 @@ class fuckhtml{
 				
 				case "\"":
 				case "'":
-					if(
-						$i !== 0 && // only check if a quote could be there
-						(
-							(
-								$json[$i - 1] === "\\" &&
-								(
-									$i === 2 ||
-									$json[$i - 2] === "\\"
-								)
-							) ||
-							$json[$i - 1] !== "\\"
-						)
-					){
-						// found a non-escaped quote
-						
+					// count preceding backslashes
+					$bsCount = 0;
+					$j = $i - 1;
+
+					while($j >= 0 && $json[$j] === "\\"){
+						$bsCount++;
+						$j--;
+					}
+
+					// quote is NOT escaped if even number of backslashes
+					if($bsCount % 2 === 0){
 						if($in_quote === null){
-							
 							// open quote
 							$in_quote = $json[$i];
-							
 						}elseif($in_quote === $json[$i]){
-							
 							// close quote
 							$in_quote = null;
 						}

lib/scrapers.php

@@ -0,0 +1,115 @@
+<?php
+
+const DEFAULT_SCRAPERS = [
+    "web" => config::SCRAPER_WEB,
+    "images" => config::SCRAPER_IMAGES,
+    "news" => config::SCRAPER_NEWS,
+    "videos" => config::SCRAPER_VIDEOS,
+    "music" => config::SCRAPER_MUSIC,
+];
+
+const AVAILABLE_SCRAPERS = [
+    "web" => [
+        "ddg" => "DuckDuckGo",
+        //"yahoo" => "Yahoo!",
+        "brave" => "Brave",
+        "yandex" => "Yandex",
+        "google" => "Google",
+        "google_api" => "Google API",
+        "google_cse" => "Google CSE",
+        "yahoo_japan" => "Yahoo! JAPAN",
+        "startpage" => "Startpage",
+        "qwant" => "Qwant",
+        "ghostery" => "Ghostery",
+        "yep" => "Yep",
+        "mwmbl" => "Mwmbl",
+        "mojeek" => "Mojeek",
+        "baidu" => "Baidu",
+        "coccoc" => "Cốc Cốc",
+        "solofield" => "Solofield",
+        "marginalia" => "Marginalia",
+        "wiby" => "wiby"
+    ],
+    "images" => [
+        "ddg" => "DuckDuckGo",
+        "yandex" => "Yandex",
+        "brave" => "Brave",
+        "google" => "Google",
+        "google_api" => "Google API",
+        "google_cse" => "Google CSE",
+        "yahoo_japan" => "Yahoo! JAPAN",
+        "startpage" => "Startpage",
+        "qwant" => "Qwant",
+        "baidu" => "Baidu",
+        "solofield" => "Solofield",
+        "pinterest" => "Pinterest",
+        "cara" => "Cara",
+        "flickr" => "Flickr",
+        "pexels" => "Pexels",
+        "pixabay" => "Pixabay",
+        "unsplash" => "Unsplash",
+        "fivehpx" => "500px",
+        "vsco" => "VSCO",
+        "imgur" => "Imgur",
+        "ftm" => "FindThatMeme"
+    ],
+
+    "videos" => [
+        "yt" => "YouTube",
+        //"archiveorg" => "Archive.org",
+        "vimeo" => "Vimeo",
+        //"odysee" => "Odysee",
+        "sepiasearch" => "Sepia Search",
+        //"fb" => "Facebook videos",
+        "ddg" => "DuckDuckGo",
+        "brave" => "Brave",
+        "yandex" => "Yandex",
+        "google" => "Google",
+        "yahoo_japan" => "Yahoo! JAPAN",
+        "startpage" => "Startpage",
+        "qwant" => "Qwant",
+        "baidu" => "Baidu",
+        "coccoc" => "Cốc Cốc",
+        "solofield" => "Solofield"
+    ],
+
+    "news" => [
+        "ddg" => "DuckDuckGo",
+        "brave" => "Brave",
+        "google" => "Google",
+        "yahoo_japan" => "Yahoo! JAPAN",
+        "startpage" => "Startpage",
+        "qwant" => "Qwant",
+        "mojeek" => "Mojeek",
+        "baidu" => "Baidu"
+    ],
+
+    "music" => [
+        "sc" => "SoundCloud",
+        "swisscows" => "Swisscows (SoundCloud)",
+        //"spotify" => "Spotify"
+    ],
+
+    "booru" => [
+        "safebooru" => "Safebooru",
+        "konachan" => "Konachan",
+        "tbib" => "The Big Imageboard",
+        "gelbooru" => "Gelbooru",
+        "yandere" => "Yande.re",
+        "sankakucomplex" => "SankakuComplex",
+        "soybooru" => "SoyBooru",
+    ],
+];
+
+function get_scraper(string $page, string $selected): string
+{
+    $available = AVAILABLE_SCRAPERS[$page];
+    if (!$available) {
+        return "__NONEXISTENT__";
+    }
+    if ($available[$selected]) {
+        return $selected;
+    }
+    $default = DEFAULT_SCRAPERS[$page];
+    return $available[$default] ? $default : array_key_first($available);
+}

scraper/baidu.php

@@ -666,14 +666,24 @@ class baidu{
 			//
 			// Don't parse as a search result if it's a card
 			//
-			$card =
+			$tpl_probe =
 				$this->fuckhtml
-				->getElementsByClassName(
-					"cosc-card",
-					$div
+				->getElementsByAttributeValue(
+					"tpl",
+					"www_index",
+					[$datafield]
 				);
 			
-			if(count($card) !== 0){
+			if(
+				count($tpl_probe) === 0 &&
+				count(
+					$this->fuckhtml
+					->getElementsByClassName(
+						"cosc-card",
+						$div
+					)
+				) !== 0
+			){
 				
 				//
 				// Parse chinese youtube shorts
@@ -870,10 +880,13 @@ class baidu{
 			// class:FYB_RD -> News garbage, IGNORE
 			
 			$result =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"result",
-					[$datafield]
+				array_merge(
+					$tpl_probe,
+					$this->fuckhtml
+					->getElementsByClassName(
+						"result",
+						[$datafield]
+					)
 				);
 			
 			if(count($result) !== 0){
@@ -885,7 +898,7 @@ class baidu{
 				$title =
 					$this->fuckhtml
 					->getElementsByClassName(
-						"sc-link",
+						"cosc-title-a",
 						"a"
 					);
 				
@@ -905,8 +918,9 @@ class baidu{
 				
 				$description =
 					$this->fuckhtml
-					->getElementsByClassName(
-						"c-color",
+					->getElementsByAttributeValue(
+						"data-sanssr-cmpt",
+						"card/www-summary-1",
 						$div
 					);
 				

scraper/brave.php

@@ -347,11 +347,8 @@ class brave{
 				$q["spellcheck"] = "0";
 			}
 		}
-		/*
-		$handle = fopen("scraper/brave.html", "r");
-		$html = fread($handle, filesize("scraper/brave.html"));
-		fclose($handle);*/
 		
+		//$html = file_get_contents("scraper/brave.html");
 		try{
 			$html =
 				$this->get(

scraper/ddg.php

@@ -54,8 +54,8 @@ class ddg{
 				$headers =
 					["User-Agent: " . config::USER_AGENT,
 					"Accept: */*",
-					"Accept-Language: en-US,en;q=0.5",
-					"Accept-Encoding: gzip",
+					"Accept-Language: en-US,en;q=0.9",
+					"Accept-Encoding: gzip, deflate, br, zstd",
 					"Referer: https://duckduckgo.com/",
 					"DNT: 1",
 					"Sec-GPC: 1",
@@ -492,6 +492,17 @@ class ddg{
 				"form"
 			);
 		
+		foreach($forms as $form){
+			
+			if(
+				isset($form["attributes"]["id"]) &&
+				$form["attributes"]["id"] == "challenge-form"
+			){
+			
+				throw new Exception("DuckDuckGo returned an image captcha");
+			}
+		}
+		
 		foreach(array_reverse($forms) as $form){
 			
 			$this->fuckhtml->load($form);
@@ -909,112 +920,6 @@ class ddg{
 			){
 				
 				throw new Exception("DuckDuckGo returned a JSA challenge");
-				
-				// get JSA initial token
-				if(
-					!preg_match(
-						'/let jsa *= *([0-9]+)/',
-						$js,
-						$jsa
-					)
-				){
-					
-					$jsa = 0;
-				}else{
-					
-					$jsa = (int)$jsa[1];
-				}
-				
-				// get function bodies
-				preg_match_all(
-					'/let *([A-Za-z0-9]+) *= *function\(.*\) *{(.*)};/sU',
-					$js,
-					$functions
-				);
-				
-				$parsed_functions = [];
-				
-				for($i=0; $i<count($functions[0]); $i++){
-					
-					$functions[2][$i] = trim($functions[2][$i]);
-					
-					if(
-						preg_match(
-							'/return num *\* *([0-9]+)/i',
-							$functions[2][$i],
-							$num
-						)
-					){
-						
-						$parsed_functions[$functions[1][$i]] = [
-							"type" => "multiplication",
-							"num" => (int)$num[1]
-						];
-						continue;
-					}
-					
-					if(
-						preg_match(
-							'/innerHTML *= *`([^`]+)`/i',
-							$functions[2][$i],
-							$challenge
-						)
-					){
-						
-						$challenge[1] =
-							preg_replace(
-								'/<\/(br)>/',
-								'<$1>',
-								$challenge[1]
-							);
-						
-						$parsed_functions[$functions[1][$i]] = [
-							"type" => "challenge",
-							"text" => $challenge[1]
-						];
-					}
-				}
-				
-				// get function call order
-				preg_match_all(
-					'/jsa *= *([A-Za-z0-9]+)\(jsa\)/i',
-					$js,
-					$call_order
-				);
-				
-				foreach($call_order[1] as $order){
-					
-					if(!isset($parsed_functions[$order])){
-						
-						throw new Exception("JS challenge solve failure: DuckDuckGo called an unknown function");
-					}
-					
-					if($parsed_functions[$order]["type"] == "multiplication"){
-						
-						$jsa = $jsa * $parsed_functions[$order]["num"];
-						continue;
-					}
-					
-					if($parsed_functions[$order]["type"] == "challenge"){
-						
-						// @TODO get parsed length
-						//$parsed_functions[$order]["text"]
-						
-						$jsa = $jsa + strlen($parsed_functions[$order]["text"]);
-					}
-				}
-				
-				try{
-					$js = $this->get(
-						$proxy,
-						"https://links.duckduckgo.com" . $challenge_url[1] . $jsa,
-						[],
-						ddg::req_xhr
-					);
-				}catch(Exception $error){
-					
-					throw new Exception("Failed to get challenged d.js");
-				}
 			}
 			
 			//

scraper/ghostery.php

@@ -1,320 +0,0 @@
-<?php
-
-class ghostery{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("ghostery");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		if($page != "web"){
-			
-			return [];
-		}
-		
-		return [
-			"country" => [
-				"display" => "Country",
-				"option" => [
-					"any" => "All regions",
-					"AR" => "Argentina",
-					"AU" => "Australia",
-					"AT" => "Austria",
-					"BE" => "Belgium",
-					"BR" => "Brazil",
-					"CA" => "Canada",
-					"CL" => "Chile",
-					"DK" => "Denmark",
-					"FI" => "Finland",
-					"FR" => "France",
-					"DE" => "Germany",
-					"HK" => "Hong Kong",
-					"IN" => "India",
-					"ID" => "Indonesia",
-					"IT" => "Italy",
-					"JP" => "Japan",
-					"KR" => "Korea",
-					"MY" => "Malaysia",
-					"MX" => "Mexico",
-					"NL" => "Netherlands",
-					"NZ" => "New Zealand",
-					"NO" => "Norway",
-					"CN" => "People's Republic of China",
-					"PL" => "Poland",
-					"PT" => "Portugal",
-					"PH" => "Republic of the Philippines",
-					"RU" => "Russia",
-					"SA" => "Saudi Arabia",
-					"ZA" => "South Africa",
-					"ES" => "Spain",
-					"SE" => "Sweden",
-					"CH" => "Switzerland",
-					"TW" => "Taiwan",
-					"TR" => "Turkey",
-					"GB" => "United Kingdom",
-					"US" => "United States"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $country){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"Referer: https://ghosterysearch.com",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Cookie: ctry=" . ($country == "any" ? "--" : $country) . "; noads=true",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: same-origin",
-			"Sec-Fetch-User: ?1",
-			"Priority: u=0, i"]
-		);
-		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$query, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			parse_str($query, $query);
-			
-			// country
-			$country = $query["c"];
-			unset($query["c"]);
-			
-			$query = http_build_query($query);
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search?" . $query,
-						[],
-						$country
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}else{
-			
-			$proxy = $this->backend->get_ip();
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search",
-						[
-							"q" => $get["s"]
-						],
-						$get["country"]
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$this->fuckhtml->load($html);
-		
-		$results_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"results",
-				"section"
-			);
-		
-		if(count($results_wrapper) === 0){
-			
-			throw new Exception("Failed to grep result section");
-		}
-		
-		$this->fuckhtml->load($results_wrapper[0]);
-		
-		// get search results
-		$results =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"result",
-				"li"
-			);
-		
-		if(count($results) === 0){
-			
-			return $out;
-		}
-		
-		foreach($results as $result){
-			
-			$this->fuckhtml->load($result);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"url",
-					"a"
-				);
-			
-			if(count($a) === 0){
-				
-				continue;
-			}
-			
-			$a = $a[0];
-			
-			$out["web"][] = [
-				"title" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"h2"
-							)[0]
-						)
-					),
-				"description" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"p"
-							)[0]
-						)
-					),
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a
-						["attributes"]
-						["href"]
-					),
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		$this->fuckhtml->load($html);
-		
-		// get pagination token
-		$pagination_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"pagination",
-				"div"
-			);
-		
-		if(count($pagination_wrapper) !== 0){
-			
-			// found next page!
-			$this->fuckhtml->load($pagination_wrapper[0]);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"a"
-				);
-			
-			if(count($a) !== 0){
-				
-				$q =
-					parse_url(
-						$this->fuckhtml
-						->getTextContent(
-							$a[count($a) - 1]
-							["attributes"]
-							["href"]
-						),
-						PHP_URL_QUERY
-					);
-				
-				$out["npt"] =
-					$this->backend
-					->store(
-						$q . "&c=" . $get["country"],
-						"web",
-						$proxy
-					);
-			}
-		}
-		
-		return $out;
-	}
-	
-	private function titledots($title){
-		
-		return trim($title, " .\t\n\r\0\x0B…");
-	}
-}

scraper/marginalia.php

@@ -289,8 +289,18 @@ class marginalia{
 				throw new Exception("The API key used is rate limited. Please try again in a few minutes.");
 			}
 			
+			if(strpos($json, "<title>Bad Gateway (502)</title>")){
+				
+				throw new Exception("Marginalia returned a 502 error");
+			}
+			
 			$json = json_decode($json, true);
 			
+			if($json === null){
+				
+				throw new Exception("Failed to decode JSON");
+			}
+			
 			foreach($json["results"] as $result){
 				
 				$out["web"][] = [
@@ -490,6 +500,37 @@ class marginalia{
 			return $this->web($get, $redirect);
 		}
 		
+		// detect internal error
+		$infobox =
+			$this->fuckhtml
+			->getElementsByClassName("infobox");
+		
+		if(count($infobox) !== 0){
+			
+			foreach($infobox as $i){
+				
+				$this->fuckhtml->load($i);
+				
+				$h2 =
+					$this->fuckhtml
+					->getElementsByTagName("h2");
+				
+				if(
+					count($h2) !== 0 &&
+					$this->fuckhtml
+					->getTextContent(
+						$h2[0]
+					) == "Internal error"
+				){
+					
+					throw new Exception("Marginalia returned an internal server error");
+				}
+			}
+						
+			// reset
+			$this->fuckhtml->load($html);
+		}
+		
 		$sections =
 			$this->fuckhtml
 			->getElementsByClassName(

scraper/mojeek.php

@@ -340,7 +340,7 @@ class mojeek{
 							"zm" => "Zambia",
 							"zw" => "Zimbabwe"
 						]
-					],
+					],/*
 					"region" => [
 						"display" => "Region",
 						"option" => [
@@ -350,7 +350,7 @@ class mojeek{
 							"fr" => "France",
 							"uk" => "United Kingdom"
 						]
-					],
+					],*/
 					"domain" => [
 						"display" => "Results per domain",
 						"option" => [
@@ -458,7 +458,7 @@ class mojeek{
 				"date" => 1, // show date
 				"tlen" => 128, // max length of title
 				//"dlen" => 511, // max length of description
-				"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
+				//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
 			];
 			
 			switch($focus){
@@ -503,6 +503,11 @@ class mojeek{
 			}
 		}
 		
+		if(strlen($html) === 0){
+			
+			throw new Exception("Mojeek returned an empty page (probably a homepage redirect)");
+		}
+		
 		$out = [
 			"status" => "ok",
 			"spelling" => [

scraper/pinterest.php

@@ -297,7 +297,13 @@ class pinterest{
 			throw new Exception("Failed to decode JSON");
 		}
 		
-		print_r($json);
+		if(
+			isset($json["client_context"]["is_bad_bot"]) &&
+			(int)$json["client_context"]["is_bad_bot"] === 1
+		){
+			
+			throw new Exception("Pinterest blocked this instance or request proxy.");
+		}
 		
 		$out = [
 			"status" => "ok",

scraper/qwant.php

@@ -212,28 +212,58 @@ class qwant{
 		return $base;
 	}
 	
-	private function get($proxy, $url, $get = []){
-		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: application/json, text/plain, */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Origin: https://www.qwant.com",
-			"Referer: https://www.qwant.com/",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-site",
-			"TE: trailers"
-		];
+	private function get($proxy, $url, $get = [], $is_post = false, $cookie = null){
 		
 		$curlproc = curl_init();
 		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
+		if($is_post){
+			
+			$headers = [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://www.qwant.com/",
+				"Content-type: application/x-www-form-urlencoded",
+				"Origin: https://www.qwant.com",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache"
+			];
+			
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, http_build_query($get));
+		}else{
+			
+			$headers = [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: application/json, text/plain, */*",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://www.qwant.com/",
+				"Origin: https://www.qwant.com",
+				"DNT: 1",
+				($cookie !== null ? "Cookie: {$cookie}" : ""),
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Priority: u=4",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache",
+				"TE: trailers"
+			];
+						
+			if($get !== []){
+				$get = http_build_query($get);
+				$url .= "?" . $get;
+			}
 		}
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
@@ -271,6 +301,8 @@ class qwant{
 			[$params, $proxy] = $this->backend->get($get["npt"], "web");
 			
 			$params = json_decode($params, true);
+			$cookie = $params["cookie"];
+			unset($params["cookie"]);
 			
 		}else{
 			
@@ -297,7 +329,8 @@ class qwant{
 				"device" => "desktop",
 				"tgp" => 3,
 				"safesearch" => 0,
-				"displayed" => "true"
+				"displayed" => "true",
+				"llm" => "false"
 			];
 			
 			switch($get["nsfw"]){
@@ -306,18 +339,24 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
+			
+			$cookie = null;
 		}
 		/*
 		$handle = fopen("scraper/qwant_web.json", "r");
 		$json = fread($handle, filesize("scraper/qwant_web.json"));
 		fclose($handle);*/
 		
+		$cookie = $this->fuck_datadome($proxy, $search);
+		
 		try{
 			$json =
 				$this->get(
 					$proxy,
-					"https://fdn.qwant.com/v3/search/web",
-					$params
+					"https://api.qwant.com/v3/search/web",
+					$params,
+					false,
+					$cookie
 				);
 			
 		}catch(Exception $error){
@@ -353,6 +392,8 @@ class qwant{
 			"related" => []
 		];
 		
+		$this->detect_errors($json);
+		
 		if(
 			$json["status"] != "success" &&
 			$json["data"]["error_code"] === 5
@@ -362,8 +403,6 @@ class qwant{
 			return $out;
 		}
 		
-		$this->detect_errors($json);
-		
 		if(!isset($json["data"]["result"]["items"]["mainline"])){
 			
 			throw new Exception("Server did not return a result object");
@@ -436,6 +475,7 @@ class qwant{
 		if($json["data"]["result"]["lastPage"] === false){
 			
 			$params["offset"] = $params["offset"] + 10;
+			$params["cookie"] = $cookie;
 			
 			$out["npt"] =
 				$this->backend->store(
@@ -577,6 +617,9 @@ class qwant{
 				);
 			
 			$params = json_decode($params, true);
+			$cookie = $params["cookie"];
+			unset($params["cookie"]);
+			
 		}else{
 			
 			$search = $get["s"];
@@ -617,13 +660,19 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
+			
+			$cookie = null;
 		}
 		
+		$cookie = $this->fuck_datadome($proxy, $search);
+		
 		try{
 			$json = $this->get(
 				$proxy,
 				"https://api.qwant.com/v3/search/images",
 				$params,
+				false,
+				$cookie
 			);
 		}catch(Exception $err){
 			
@@ -1001,4 +1050,53 @@ class qwant{
 		
 		return "https://" . $image["host"] . "/th?id=" . rawurlencode($id);
 	}
+	
+	private function fuck_datadome($proxy, $search){
+		
+		if($cookie === null){
+			// get datadome bullshit
+			// yeah go ahead and patch this you code monkey
+			
+			// ... you know, instead of fixing this shit, can you make the second page of results return more than 1 result?
+			// i search shit like "higurashi" and i only get 1 result and then just a bunch of related searches ad finitum
+			// also, stop pretending you guys have your own index, this is clearly all bing results
+			try{
+				$datadome =
+					$this->get(
+						$proxy,
+						"https://dd.qwant.com/js/",
+						[
+							"jspl" => "-oirHDeP7G3G9LXY3rhxGDYf6t_ZlYPzaiL2dRhQ34uNUc4dO2Td0SmmKu_VJp7-jrAIoOMrTqKgCtV4cPzntMyNuOYN7QQ1q0Fi2c-679vYpJr9ub5dRFXMhCMQmjZsOWXUa8yafIXf85UA5Goct-n4hVT5dFfBGM-dE6gwZEFkHQ1j1195sLybZFTNA1IfX26MBMxlVHoyT-aHtkd6YjGXbvOPMMsObVHDa_Cf4BmApRGo-eehsAttsCqDRwkl0M2E1b0OXCIwMVnCK-qWZ81MGaJGjqxdj_LaxhAjE9JI_4oNedAg4Yj4m_vkQBKdmWJT_bTUsllAZXqX5IxYQHJ8WvlknOTr6sR3wgoSOa_JNVHkUQxqXBbQcoDdy6VciL5rIqwWl4Y2caVYzeXa8YNXxwsfrPDegH3dQ6vOiIiCDT3bexkcS_y7GAioPPv9qDu7M9y4sI8t0VkPgMbWY6MIu5u9WAv6lEiKIYrtlyU-iiXnNdkd9EnpC1TD8_s12PtCuLg5bGgLVmI1dw8VXVZxmVyFvfBtuqXkzM-kZOyv-YIRcf-wPY8DUtnHox1d6t64oYhGv-pYV5dmZ0rGs99LC6ThiifeF_oHz5ZK46Ms68WsiAjc1HLAbFPFxWw8X6lyLlpT5aHgFdDBeCTExKXyItlc9PBxkgK8UnXHQFiWX3HTen8rziwkI31AbkMU7xGP6cvuwuLG7MA7sH3yLF1uimQlhr6EuTrImWQNV7X81DqyZ0OE7I-uT9bp_xISa5it-6AJpFtGcwz04_OuO2zvZazBeiKJI5Fw0X2r3RHTHdA29SXqTIe4y2LRAAp0MSAZUIJAMIbbrz9BG-3D5BHfQUA1Bbuklxv-tNWiuanWbbTmAiuxTAo9P-D2ZMrWHwdN8sCb3nINLur38z1xviSs2qEEjKo9kzQIWoHgwm_amV-jjDtNBXwpCQnoaqt3FfL8e59ecYWVg61Gnl3YCwq6rubFdkIv6UStcWSlj4pRFlFZSjQxzznvGTAc5Nz1JJWVeBHJItei4wqDUYEw2BNTS_fU9keV3LFyKDH_x248AxHLiXt583DhZZwPAKlpmMo0E7odzfOgVQGM5b9l64NL3RQRq8cDxI8KgmOFnPHdI7Myq1A3nW1uoBB8m9_GR_9orboKl5tAQrT0VTMRd6JyNwOOQ71ZoQpMwsD5Xd9YDZd618TNE9dA38h9OyBco_-hwKCYayyEqib1atLLBO_gnO4InR471AklOokDOKx1oY_xgNhlFrZskE4nQQOqeCQ_47oysob9s3kblA8UqDrL2vM3i2KPLWwlQzMAPrQAIv7Amp6rO-wYwBQdbP7XghYympxx9-hNxc1Duq-nS4R3pjPtZg3LoCvG1N_0JLf9jfPBqw_rdPM6DnjEOvpCdvrfv4wBlEpgqbaRQCZBPEz-O5ZlY24OyPvaNX0krPeXkbi-KYuwZSAbfW0zsksXb8cbRmz6tvh25IRYP1XNDPVd8NFrYOdDH9GzhV5MYZziYMurQlVwgumGLz85Y6qTHZU2D5qkMH7o5C5bzptReO96889oxkReS_LDhzdxH0tAQ5a9GgETJTHAXJ9Lt7X037i0qQlatbPLizOl9fo4vUBGJ7w-uSUmIzg1yIwOo8xhLu39WghXfoJ_EX5ptNi7arpAo11UoJIGEgWgq7bS40q2dtnG5HYpp9ef37iFgztM7PV5dlJf_W-n9uXJyCk2nhxZeuGiIGhMTzjP_8oeyiMFelmoyLlqK8wOs6hvWqgDD4NwtZc-XBkEAKsxuPrirb_sszp0z6BWsh3D6fBxAW_lTrNVkxZ1EBtoEN72HvO1aeX8PVuJP3CKwkkUcAhJ5Zvajo8t4Ai8ave2MTZul4QhDIe7ZALJMo71u-hNSj8YG9JLPqF1jcnbkl8ykt_XMMxX8wSZ_AfTsfmAuOWBfEzxv8Fl2Ne-oxcEtfuYnYNJIjQDHXXdtYbWKBezE41wonQu7FWh3wSSF3DDm9EocWMsUZ610nrIviOFdtTqjrY7KnOZp0hS9Hm2V04z-06Db-To71L5O5poYKuOlDAP_ULfW0hgMgYaPnt4Nive9gtue6ZQ5yNFylKM11zCWcF8vmuEPGx4NIZjvNFB-QSkdxZtBTQOgO6r4BqHJNZIDqiD9A6PyMemVtQEo-JhooX6bwHFHx6C84r-kKq1lg0RaDelbu1h6347kocxOczdgHWlagKPFrUvHpPHvCZeFmzLLmPWbDEpUhhfkbEZlNe47svO5WN1TYZMCAlE0g01pe6b_NB9-twQ4FJB1SkUQRA4o5DxiIHSgXBBfw1X98XPEKcJgCU2zRbYuU69mvyPcsxoPsLo8Cl2OAXj2U1tcLUxVhR_vu3I0SG_Ee8hxBp5b6QbpIu2sE0bgpqfi88XqUXf3qvxGbkoZDZ8dPvEvCVmpDZOOGJY5Db4MnpKBByW79lUnpbLCbsVOoxYu8SVmVQ63eGY9yBlACuWxlXA_PhCpnKYwMppjxKvKqalNJGMMg5WQ8cvhsIftE5wgaj21n0A15GwRO1WgUcsO_dLe1MqO88hTHWHV0Q3gNut3sdsmVr_n-0SniLtVAaw4IEtDZWNUfhpaoC7iyKxoJ86t8r697EYPlWUu4nTQhb3TOXAjHF22rFsJJMldKHVxjxjd086ky3OY3L_7cbjVof60-y4gTJdumw5hch-mhaT93nVnaTuPLGCAC48dZSgOitW8Dw62mPS05fTPAfKUY1HpyIWr9ACk7U2KVAijpzFB-bXvqJeCyBeUHr4cB61K2XxTmkNA8Oo9RC32TeYDrThFzoUyYZ_D6zPe1lLtQaFsPRvDBSsYCcr5fcABlST4DyUgnkgqtx-0-D1QEwtWiH3xiHLQGa82ldjqNIXL9LqwxYoCRBKuljHyqEylZD26A9Jp6SL9rk_Ol6TJlYa3zx44PHyMoA3Coy-HIZW5NMwVKdV7egUDDOchvuFnc1Zv2QpSFKc0JY8lYFtSm1HZFwjq7hf2hBqkU0Xneh8AT4Ty9QVb4mJ8rpzc8m_NBZNrpZVE-pIzmv2yGjnRwKSrwsjat7b2iY34UPFqGBlDk5FX6EfRlN9bq-nztWdextLc47ndr1gxsjC-vr6tbdXl5FxMVhbsnLH80wOC6hHubxCysr5C7Ft59s3MLhVQCc3IIOPMBhLOjjjVsu6K8kAJvvamIpj-iYUQj9T8JztAqfobytzbiwPsQ3mywUGkJpZxTjXjqkKdT6yfFdW5ksR8p3VlKGAAKft9TKKcc5IgErmIRtiZF4hGhB45nwIO5XHA59V94QuV3b0oJrleuDCBj0iDFqWYpsUVHyKl3RjrO86Yq1IbNRwXIIMrlsVYos9CS1I75k9kW4yQC6ni3a2ZitDkiEB-SoQuTlBsvfTs892Wt5q3CWH6_7srZO6l2QEAlWqyQp_P4i8j8uiEbZksGsgghVIgOkriblDP9ml7PdkpRX7S3OpY7jjQgTNR8XHaZ-SEw0-zPiqIIt5VioFK8JljPn-nMIDx-ew_vTDKqmKEOF39ezQcc5oQNTQNUvTAe8qCbK4ghS3fagItezoHtB5zm3qwhx9Iu6JMJy0UhywUZBaInY6NokiVZGDgtoqlVjOk7K4299MVZgsOr2P9QHRDo41lYnCeWWFUf50d-ouEOUO_ngteDpgG1DfV64eycrUOWAnpfI8XOUZ7mxS89SqJriw3T3tcljowHiOFw9QjYsjJi0vuHL5jfoBZVdYAUH6KiLfIScO-KgqitdUTpFwks8Qk7xfd2HcEfArlzSiAt3Hmb2Ty-5mhvtQj2RbPehgoI7gqa60b6Qffth-5tbIjXoKlZCQsaKVMAggF9C3w0Rv5uJ3ExCTGsfKO9IVOf--AfxaA8cNoO0W3NTaMH2iWftY0O1-1x3VJEcerp2v9BcZ0iIL3VFJ9eyOPYNHNwTYrI3S6OrVZxuQXFbu2E_Us32wxYqXvnMx-DEOk7bUUFIanL69Cbn_mR_9Wm1sFixBQ15diJoe1Oyl3XWHWL6_YF4XnHzZlMWySt3Awdeynwu99Ter8WMGXl-6K30novKRWC9jDVOfAtxPvXBkkRtpm8nnJT8fwkmIQoWOEPT2MCg3VYzzHzVEH-AueuRiCZj_o7L-fRqbtFrWPyVnGuetUYS5u0D3n7Rr6kCVqWhLfwjqAwV_calzIQs8-J0-AJe6GGaaONPYfXwXjYan0Sl11CS64lXvx7i3MgeOZuH210T6g6hj54dSLw0upO1L8k0FGli3I5OhLMekeGMLT_N2akyHktjbz8Odpk2ms0pJP_PdqvUegjXgH81F5lr40HAXZs7jfC9ytgcWaEjwlmnoFHSCTdlCIgJqEBzItXiz6ilk5zP35VwxdQTNoms7Meqg-ZrgRuLRaqXNtpQT9qWBmiFP6zoYLwymqytZ_P1SV9DCM-KIZQ2Cq-ew8PsYSrwl8GmfuegoKEMBiON1I4mC9ChDgzJmbo5P9te60T0URS5g9ExiQuSo-JXs0MKC57LLiNo3MPZvqR1wRJOTg0_FqS4JFxzPwN5OqhBzfs25f6kK-axf0wWp-LetUx3U1CTNaxfbCTK5tb8KnLOIs3lDwJw5H5wIIRgXnfNLBJT8A2srfXkdpA",
+							"eventCounters" => [],
+							"jsType" => "ch",
+							"cid" => "3i4vPGHCl74uIBumWaxF2a0PjFkyCJxcSvFnn~heX7keA6sGHgRbnzSYcMr_RzJFlfeeBdg9HeufzFPxnvSHbyWowSvjm6mNCSPhkx6MwDqptANoj93ZgGVbrzxGtcYe",
+							"ddk" => "78B13B7513D180B7AB6D6FF9EB0A51",
+							"Referer" => "https://www.qwant.com/?q={$search}&t=web",
+							"request" => "/?q={$search}&t=web",
+							"responsePage" => "origin",
+							"ddv" => "5.8.0"
+						],
+						true
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JS challenge token");
+			}
+			
+			$datadome = json_decode($datadome, true);
+			if($datadome === null){
+				
+				throw new Exception("Failed to decode JS challenge JSON");
+			}
+			
+			if(!isset($datadome["cookie"])){
+				
+				throw new Exception("Failed to get cookie from JS challenge endpoint");
+			}
+			
+			$cookie = explode(";", $datadome["cookie"])[0];
+		}
+		
+		return $cookie;
+	}
 }

scraper/startpage.php

@@ -408,6 +408,8 @@ class startpage{
 			//$html = file_get_contents("scraper/startpage.html");
 		}
 		
+		//echo htmlspecialchars($html);
+		
 		$this->detect_captcha($html);
 		
 		if(
@@ -564,12 +566,16 @@ class startpage{
 					break;
 				
 				case "spellsuggest-google":
-					$out["spelling"] =
-						[
-							"type" => "including",
-							"using" => $json["render"]["query"],
-							"correction" => $category["results"][0]["query"]
-						];
+					
+					if(isset($category["results"][0]["query"])){
+						
+						$out["spelling"] =
+							[
+								"type" => "including",
+								"using" => rawurldecode($json["render"]["query"]),
+								"correction" => html_entity_decode($category["results"][0]["query"])
+							];
+					}
 					break;
 				
 				case "dictionary-qi":
@@ -645,318 +651,6 @@ class startpage{
 			}
 		}
 		
-		// parse instant answers
-		if(
-			$get["extendedsearch"] == "yes" &&
-			$get_instant_answer === true
-		){
-			
-			// https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=BqZ3inqrAgF701&sr=1
-			try{
-				$post = [
-					"se" => "n0vze2y9dqwy",
-					"q" => $json["render"]["query"],
-					"results" => [], // populate
-					"enableKnowledgePanel" => true,
-					"enableMediaThumbBar" => false,
-					"enableSearchSuggestions" => false,
-					"enableTripadvisorProperties" => [],
-					"enableTripadvisorPlaces" => [],
-					"enableTripadvisorPlacesForLocations" => [],
-					"enableWebProducts" => false,
-					"tripadvisorPartnerId" => null,
-					"tripadvisorMapColorMode" => "light",
-					"tripadvisorDisablesKnowledgePanel" => false,
-					"instantAnswers" => [
-						"smartAnswers",
-						"youtube",
-						"tripadvisor"
-					],
-					"iaType" => null,
-					"forceEnhancedKnowledgePanel" => false,
-					"shoppingOnly" => false,
-					"allowAdultProducts" => true,
-					"lang" => "en",
-					"browserLang" => "en-US",
-					"browserTimezone" => "America/New_York",
-					"market" => null,
-					"userLocation" => null,
-					"userDate" => date("Y-m-d"),
-					"userAgentType" => "unknown"
-				];
-				
-				foreach($out["web"] as $result){
-					
-					$post["results"][] = [
-						"url" => $result["url"],
-						"title" => $result["title"]
-					];
-				}
-				
-				$post = json_encode($post, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE);
-				
-				$additional_data =
-					$this->get(
-						$proxy,
-						"https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=" . $json["render"]["callback_sc"] . "&sr=1",
-						$post,
-						true,
-						true
-					);
-				
-				$additional_data = json_decode($additional_data, true);
-				
-				if($additional_data === null){
-					
-					throw new Exception("Failed to decode JSON"); // just break out, dont fail completely
-				}
-				
-				if(!isset($additional_data["knowledgePanel"])){
-					
-					throw new Exception("Response has missing data (knowledgePanel)");
-				}
-				
-				$additional_data = $additional_data["knowledgePanel"];
-				
-				$answer = [
-					"title" => $additional_data["meta"]["title"],
-					"description" => [
-						[
-							"type" => "quote",
-							"value" => $additional_data["meta"]["description"]
-						]
-					],
-					"url" => $additional_data["meta"]["origWikiUrl"],
-					"thumb" => $additional_data["meta"]["image"],
-					"table" => [],
-					"sublink" => []
-				];
-				
-				// parse html for instant answer
-				$this->fuckhtml->load($additional_data["html"]);
-				
-				$div =
-					$this->fuckhtml
-					->getElementsByTagName(
-						"div"
-					);
-				
-				// get description
-				$description =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-kp-short-extract sx-kp-short-extract-complete",
-						$div
-					);
-				
-				if(count($description) !== 0){
-					
-					$answer["description"][] = [
-						"type" => "text",
-						"value" =>
-							html_entity_decode(
-								$this->fuckhtml
-								->getTextContent(
-									$description[0]
-								)
-							)
-					];
-				}
-				
-				// get socials
-				$socials =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-wiki-social-link",
-						"a"
-					);
-				
-				foreach($socials as $social){
-					
-					$title =
-						$this->fuckhtml
-						->getTextContent(
-							$social["attributes"]["title"]
-						);
-					
-					$url =
-						$this->fuckhtml
-						->getTextContent(
-							$social["attributes"]["href"]
-						);
-					
-					switch($title){
-						
-						case "Official Website":
-							$title = "Website";
-							break;
-					}
-					
-					$answer["sublink"][$title] = $url;
-				}
-				
-				// get videos
-				$videos =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-kp-video-grid-item",
-						$div
-					);
-				
-				foreach($videos as $video){
-					
-					$this->fuckhtml->load($video);
-					
-					$as =
-						$this->fuckhtml
-						->getElementsByTagName(
-							"a"
-						);
-					
-					if(count($as) === 0){
-						
-						// ?? invalid
-						continue;
-					}
-					
-					$image =
-						$this->fuckhtml
-						->getElementsByAttributeName(
-							"data-sx-src",
-							"img"
-						);
-					
-					if(count($image) !== 0){
-						
-						$thumb = [
-							"ratio" => "16:9",
-							"url" =>
-								$this->fuckhtml
-								->getTextContent(
-									$image[0]["attributes"]["data-sx-src"]
-								)
-						];
-					}else{
-
-						$thumb = [
-							"ratio" => null,
-							"url" => null
-						];
-					}
-					
-					$out["video"][] = [
-						"title" =>
-							$this->fuckhtml
-							->getTextContent(
-								$as[0]["attributes"]["title"]
-							),
-						"description" => null,
-						"date" => null,
-						"duration" => null,
-						"views" => null,
-						"thumb" => $thumb,
-						"url" =>
-							$this->fuckhtml
-							->getTextContent(
-								$as[0]["attributes"]["href"]
-							)
-					];
-				}
-				
-				// reset
-				$this->fuckhtml->load($additional_data["html"]);
-				
-				// get table elements
-				$table =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-infobox",
-						"table"
-					);
-				
-				if(count($table) !== 0){
-					
-					$trs =
-						$this->fuckhtml
-						->getElementsByTagName(
-							"tr"
-						);
-					
-					foreach($trs as $tr){
-						
-						$this->fuckhtml->load($tr);
-						
-						// ok so startpage devs cant fucking code a table
-						// td = content
-						// th (AAAHH) = title
-						$tds =
-							$this->fuckhtml
-							->getElementsByTagName(
-								"td"	
-							);
-						
-						$ths =
-							$this->fuckhtml
-							->getElementsByTagName(
-								"th"
-							);
-						
-						if(
-							count($ths) === 1 &&
-							count($tds) === 1
-						){
-							
-							$title =
-								$this->fuckhtml
-								->getTextContent(
-									$ths[0]
-								);
-							
-							$description = [];
-							
-							$this->fuckhtml->load($tds[0]);
-							
-							$lis =
-								$this->fuckhtml
-								->getElementsByTagName(
-									"li"
-								);
-							
-							if(count($lis) !== 0){
-								
-								foreach($lis as $li){
-									
-									$description[] =
-										$this->fuckhtml
-										->getTextContent(
-											$li
-										);
-								}
-								
-								$description = implode(", ", $description);
-							}else{
-								
-								$description =
-									$this->fuckhtml
-									->getTextContent(
-										$tds[0]
-									);
-							}
-							
-							$answer["table"][$title] = $description;
-						}
-					}
-				}
-				
-				$out["answer"][] = $answer;
-				
-			}catch(Exception $error){
-				
-				// do nothing
-				//echo "error!";
-			}
-		}
-		
 		return $out;
 	}
 	
@@ -1413,27 +1107,22 @@ class startpage{
 			
 			if($page["name"] == "Next"){
 				
-				parse_str(
-					explode(
-						"?",
-						$page["url"],
-						2
-					)[1],
-					$str
-				);
-				
 				return
 					$this->backend->store(
 						http_build_query(
 							[
 								"lui" => "english",
 								"language" => "english",
-								"query" => $str["q"],
-								"cat" => $pagetype,
-								"sc" => $str["sc"],
+								"sc" => $json["render"]["search_sc"],
 								"t" => "device",
-								"segment" => "startpage.udog",
-								"page" => $str["page"]
+								"cat" => $pagetype,
+								"segment" => "organic",
+								"abd" => "0",
+								"abe" => "0",
+								"query" => rawurldecode($json["render"]["query"]),
+								"page" => $page["number"],
+								"qsr" => "all",
+								"qadf" => "none" // @ todo fix (??)
 							]
 						),
 						$pagetype,

scraper/vsco.php

@@ -30,19 +30,18 @@ class vsco{
 			
 			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
 				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
 				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
+				"Accept-Encoding: gzip, deflate, br, zstd",
 				"DNT: 1",
 				"Sec-GPC: 1",
 				"Connection: keep-alive",
 				"Upgrade-Insecure-Requests: 1",
 				"Sec-Fetch-Dest: document",
 				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: same-origin",
+				"Sec-Fetch-Site: none",
 				"Sec-Fetch-User: ?1",
-				"Priority: u=0, i",
-				"TE: trailers"]
+				"Priority: u=0, i"]
 			);
 		}else{
 			
@@ -73,8 +72,8 @@ class vsco{
 		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
 		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
 		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		// http1 bypass?!?!?!
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 		
 		$this->backend->assign_proxy($curlproc, $proxy);
 		

scraper/yandex.php

@@ -868,123 +868,71 @@ class yandex{
 		
 		if($get["npt"]){
 			
-			[$params, $proxy] =
+			[$get, $proxy] =
 				$this->backend->get(
 					$get["npt"],
 					"video"
 				);
 			
-			$params = json_decode($params, true);
-			
-			$nsfw = $params["nsfw"];
-			unset($params["nsfw"]);
+			$get = json_decode($get, true);
 		}else{
 			
-			$search = $get["s"];
-			if(strlen($search) === 0){
+			if(strlen($get["s"]) === 0){
 				
 				throw new Exception("Search term is empty!");
 			}
 			
 			$proxy = $this->backend->get_ip();
-			$nsfw = $get["nsfw"];
-			$time = $get["time"];
-			$duration = $get["duration"];
-			
-			// https://yandex.com/video/search
-			// ?tmpl_version=releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63
-			// &format=json
-			// &request=
-			// {
-			//		"blocks":[
-			//			{"block":"extra-content","params":{},"version":2},
-			//			{"block":"i-global__params:ajax","params":{},"version":2},
-			//			{"block":"search2:ajax","params":{},"version":2},
-			//			{"block":"vital-incut","params":{},"version":2},
-			//			{"block":"content_type_search","params":{},"version":2},
-			//			{"block":"serp-controller","params":{},"version":2},
-			//			{"block":"cookies_ajax","params":{},"version":2}
-			//		],
-			//		"metadata":{
-			//			"bundles":{"lb":"^G]!q<X120"},
-			//			"assets":{"las":"react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"},
-			//			"extraContent":{"names":["i-react-ajax-adapter"]}
-			//		}
-			// }
-			// &yu=4861394161661655015
-			// &from=tabbar
-			// &reqid=1693106278500184-6825210746979814879-balancer-l7leveler-kubr-yp-sas-7-BAL-4237
-			// &suggest_reqid=486139416166165501562797413447032
-			// &text=minecraft
-			
-			$params = [
-				"tmpl_version" => "releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63",
-				"format" => "json",
-				"request" => json_encode([
-					"blocks" => [
-						(object)[
-							"block" => "extra-content",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "i-global__params:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "search2:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "vital-incut",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "content_type_search",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "serp-controller",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "cookies_ajax",
-							"params" => (object)[],
-							"version" => 2
-						]
-					],
-					"metadata" => (object)[
-						"bundles" => (object)[
-							"lb" => "^G]!q<X120"
-						],
-						"assets" => (object)[
-							"las" => "react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"
-						],
-						"extraContent" => (object)[
-							"names" => [
-								"i-react-ajax-adapter"
-							]
-						]
-					]
-				]),
-				"text" => $search
-			];
-			
-			if($duration != "any"){
-				
-				$params["duration"] = $duration;
-			}
-			
-			if($time != "any"){
-				
-				$params["within"] = $time;
-			}
 		}
+		
+		// https://yandex.com/video/search?text=skycamefalling&from=tabbar&format=json&ncrnd=7271&p=0&parent-reqid=&request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]}&serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL&yu=3091577281773194415&tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		// https://yandex.com/video/search
+		// ?text=skycamefalling
+		// &from=tabbar
+		// &format=json
+		// &ncrnd=7271
+		// &p=0
+		// &parent-reqid=
+		// &request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]} {"blocks":[{"block":"video-app","params":{}}]}
+		// &serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL
+		// &yu=3091577281773194415
+		// &tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		
+		$params = [
+			"text" => $get["s"],
+			"from" => "tabbar",
+			"format" => "json",
+			"ncrnd" => 7271,
+			"p" => 0,
+			"parent-reqid" => "",
+			"request" => json_encode((object)[
+				"blocks" => [
+					(object)[
+						"block" => "video-app",
+						"params" => (object)[]
+					]
+				]
+			]),
+			"serpid" => "1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL",
+			"yu" => 3091577281773194415,
+			"tmpl_version" => "releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a"
+		];
+		
+		if(isset($get["p"])){
+			
+			$params["p"] = $get["p"];
+		}
+		
+		if($get["duration"] != "any"){
+			
+			$params["duration"] = $get["duration"];
+		}
+		
+		if($get["time"] != "any"){
+			
+			$params["within"] = $get["time"];
+		}
+		
 		/*
 		$handle = fopen("scraper/yandex-video.json", "r");
 		$json = fread($handle, filesize("scraper/yandex-video.json"));
@@ -996,7 +944,7 @@ class yandex{
 					$proxy,
 					"https://yandex.com/video/search",
 					$params,
-					$nsfw,
+					$get["nsfw"],
 					"yandex_v"
 				);
 		}catch(Exception $error){
@@ -1011,7 +959,7 @@ class yandex{
 			throw new Exception("Could not parse JSON");
 		}
 		
-		if(!isset($json["blocks"])){
+		if(!isset($json["results"]["clips"]["items"])){
 			
 			throw new Exception("Yandex blocked this 4get instance. Please try again in 7~ minutes.");
 		}
@@ -1026,209 +974,120 @@ class yandex{
 			"reel" => []
 		];
 		
-		$html = null;
-		foreach($json["blocks"] as $block){
+		foreach($json["results"]["clips"]["items"] as $k => $data){
 			
-			if(isset($block["html"])){
+			if(isset($data["preview"]["posterSrc"])){
 				
-				$html .= $block["html"];
+				$poster = $data["preview"]["posterSrc"];
+				
+				if(
+					preg_match(
+						'/^\/\//',
+						$data["preview"]["posterSrc"]
+					)
+				){
+					
+					$poster = "https:" . $poster;
+				}
+				
+				$thumb = [
+					"ratio" => "16:9",
+					"url" => $poster
+				];
+			}else{
+				
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
 			}
+			
+			$out["video"][] = [
+				"title" => $data["relatedParams"]["text"],
+				"description" => $this->titledots($data["description"]),
+				"author" => [
+					"name" =>
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"] : null,
+					"url" => 
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"] : null,
+					"avatar" => null
+				],
+				"date" =>
+					isset($json["results"]["clips"]["dups"][$k]["date"]) ?
+					strtotime($json["results"]["clips"]["dups"][$k]["date"]) : null,
+				"duration" =>
+					isset($json["results"]["clips"]["dups"][$k]["duration"]["value"]) ?
+					(int)$json["results"]["clips"]["dups"][$k]["duration"]["value"] : null,
+				"views" =>
+					isset($json["results"]["clips"]["dups"][$k]["views"]["text"]) ?
+					$this->parseviews($json["results"]["clips"]["dups"][$k]["views"]["text"]) : null,
+				"thumb" => $thumb,
+				"url" =>
+					preg_replace(
+						'/^http:\/\//',
+						"https://",
+						$data["relatedParams"]["related_url"]
+					)
+			];
 		}
 		
-		$this->fuckhtml->load($html);
-		
-		$div =
-			$this->fuckhtml
-			->getElementsByTagName("div");
-		
-		/*
-			Get nextpage
-		*/
-		$npt =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"more more_direction_next i-bem",
-				$div
-			);
-		
-		if(count($npt) !== 0){
+		// get npt
+		if($json["results"]["search"]["hasNextPage"]){
+			
+			$get["p"] = (int)$json["results"]["search"]["currentPage"] + 1;
 			
-			$params["p"] = "1";
-			$params["nsfw"] = $nsfw;
 			$out["npt"] =
 				$this->backend->store(
-					json_encode($params),
+					json_encode($get),
 					"video",
 					$proxy
 				);
 		}
 		
-		$items =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"serp-item",
-				$div
-			);
-		
-		foreach($items as $item){
-			
-			$data =
-				json_decode(
-					$this->fuckhtml
-					->getTextContent(
-						$item["attributes"]["data-video"]
-					),
-					true
-				);
-			
-			$this->fuckhtml->load($item);
-			
-			$thumb =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"thumb-image__image",
-					"img"
-				);
-			
-			$c = 1;
-			if(count($thumb) === 0){
-				
-				$thumb = [
-					"url" => null,
-					"ratio" => null
-				];
-			}else{
-				
-				$thumb = [
-					"url" =>
-						str_replace(
-							"//",
-							"https://",
-							$this->fuckhtml
-							->getTextContent(
-								$thumb
-								[0]
-								["attributes"]
-								["src"]
-							),
-							$c
-						),
-					"ratio" => "16:9"
-				];
-			}
-			
-			$smallinfos =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__sitelinks-item",
-					"div"
-				);
-			
-			$date = null;
-			$views = null;
-			$first = true;
-			
-			foreach($smallinfos as $info){
-				
-				if($first){
-					
-					$first = false;
-					continue;
-				}
-				
-				$info =
-					$this->fuckhtml
-					->getTextContent(
-						$info
-					);
-				
-				if($temp_date = strtotime($info)){
-					
-					$date = $temp_date;
-				}else{
-					
-					$views = $this->parseviews($info);
-				}
-			}
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__text serp-item__text_visibleText_always",
-					"div"
-				);
-			
-			if(count($description) === 0){
-				
-				$description = null;
-			}else{
-				
-				$description =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-			}
-			
-			$out["video"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$this->titledots(
-							$data["title"]
-						)
-					),
-				"description" => $description,
-				"author" => [
-					"name" => null,
-					"url" => null,
-					"avatar" => null
-				],
-				"date" => $date,
-				"duration" =>
-					(int)$data
-					["counters"]
-					["toHostingLoaded"]
-					["stredParams"]
-					["duration"],
-				"views" => $views,
-				"thumb" => $thumb,
-				"url" =>
-					str_replace(
-						"http://",
-						"https://",
-						$this->fuckhtml
-						->getTextContent(
-							$data["counters"]
-							["toHostingLoaded"]
-							["postfix"]
-							["href"]
-						),
-						$c
-					)
-			];
-		}
-		
 		return $out;
 	}
 	
-	private function parseviews($text){
+	private function parseviews($number){
 		
-		$text = explode(" ", $text);
+		// decimal should always be 1 number long
+		$number = explode(" ", $number, 2);
+		$number = $number[0];
 		
-		$num = (float)$text[0];
-		$mod = $text[1];
+		$unit = strtolower($number[strlen($number) - 1]);
 		
-		switch($mod){
+		$tmp = explode(".", $number, 2);
+		$number = (int)$number;
+		
+		if(count($tmp) === 2){
 			
-			case "bln.": $num = $num * 1000000000; break;
-			case "mln.": $num = $num * 1000000; break;
-			case "thsd.": $num = $num * 1000; break;
+			$decimal = (int)$tmp[1];
+		}else{
+			
+			$decimal = 0;
 		}
 		
-		return $num;
+		switch($unit){
+			
+			case "k":
+				$exponant = 1000;
+				break;
+			
+			case "m":
+				$exponant = 1000000;
+				break;
+			
+			case "b";
+				$exponant = 1000000000;
+				break;
+			
+			default:
+				$exponant = 1;
+				break;
+		}
+		
+		return ($number * $exponant) + ($decimal * ($exponant / 10));
 	}
 	
 	private function titledots($title){

scraper/yep.php

@@ -216,7 +216,7 @@ class yep{
 		];
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $use_api = false, $post_data = null, $bearer = null){
 		
 		$curlproc = curl_init();
 		
@@ -227,25 +227,41 @@ class yep{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
-		// use http2
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		// use http1 lol
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"Referer: https://yep.com/",
-			"Origin: https://yep.com",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-site",
-			"Priority: u=4",
-			"TE: trailers"]
-		);
+		
+		if($use_api){
+			
+			$post_data = json_encode($post_data);
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["Content-Type: application/json",
+				"Authorization: Bearer $bearer",
+				"Content-Length: " . strlen($post_data)]
+			);
+			
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://yep.com/",
+				"Origin: https://yep.com",
+				"DNT: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Priority: u=4",
+				"TE: trailers"]
+			);
+		}
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -270,6 +286,11 @@ class yep{
 	
 	public function web($get){
 		
+		if(config::YEP_USE_API){
+			
+			return $this->web_api($get);
+		}
+		
 		$search = $get["s"];
 		if(strlen($search) === 0){
 			
@@ -303,11 +324,11 @@ class yep{
 		
 		try{
 			
-			// https://api.yep.com/fs/2/search?limit=20&query=asmr
+			// https://api.yep.com/search?limit=20&query=asmr
 			$json =
 				$this->get(
 					$this->backend->get_ip(),
-					"https://api.yep.com/fs/2/search",
+					"https://api.yep.com/search",
 					$filters
 				);
 			
@@ -392,6 +413,123 @@ class yep{
 	}
 	
 	
+	private function web_api($get){
+		
+		$search = $get["s"];
+		if(strlen($search) === 0){
+			
+			throw new Exception("Search term is empty!");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		// parse filters
+		$filters = [
+			"query" => $search,
+			"limit" => 99 // for some reason they dont show anything sometimes at 100 results
+		];
+		
+		if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
+		if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
+		
+		// add api key
+		$key_data = $this->backend->get_key();
+		
+		try{
+			
+			$json =
+				$this->get(
+					$this->backend->get_ip($key_data["increment"]),
+					"https://platform.yep.com/api/search",
+					[],
+					true,
+					$filters,
+					$key_data["key"]
+				);
+			
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		// should never happen
+		//$this->detect_cf($json);
+		
+		$json = json_decode($json, true);
+		//$json = json_decode(file_get_contents("scraper/yep.json"), true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(isset($json["error"])){
+			
+			throw new Exception("Yep API returned an error: " . $json["error"]);
+		}
+		
+		if(isset($json["errors"])){
+			
+			throw new Exception("Yep API returned the following errors: {$json["message"]}");
+		}
+		
+		if(
+			isset($json["success"]) &&
+			$json["success"] !== true
+		){
+			
+			throw new Exception("Yep API returned a false-y success value");
+		}
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Yep API did not return a results object");
+		}
+		
+		foreach($json["results"] as $item){
+			
+			if(
+				$item["url"] === null ||
+				$item["url"] == ""
+			){
+				
+				// sometimes API fucks up
+				continue;
+			}
+			
+			$out["web"][] = [
+				"title" => $item["title"],
+				"description" => $item["description"],
+				"url" => $item["url"],
+				"date" => null,
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		return $out;
+	}
+	
+	
 	private function detect_cf($payload){
 		
 		// detect cloudflare page

settings.php

@@ -91,10 +91,6 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
-					[
-						"value" => "ghostery",
-						"text" => "Ghostery"
-					],
 					[
 						"value" => "yep",
 						"text" => "Yep"
@@ -169,6 +165,10 @@ $settings = [
 						"value" => "mojeek",
 						"text" => "Mojeek"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"
@@ -231,6 +231,10 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"
@@ -325,6 +329,10 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"

static/serverping.js

@@ -80,6 +80,15 @@ document.body.insertBefore(table, quote.nextSibling);
 
 // handle sorting clicks
 var tbody = table.getElementsByTagName("tbody")[0];
+
+// server popup without inline javascript
+tbody.addEventListener('click', function(event){
+	var row = event.target.closest('.show_server');
+	if(!row) return;
+
+	show_server(row.id);
+})
+
 var th = table.getElementsByTagName("th");
 
 for(var i=0; i<th.length; i++){
@@ -343,7 +352,7 @@ function render_list(){
 	
 	for(var k=0; k<sorted_list.length; k++){
 		
-		html += '<tr onclick="show_server(' + sorted_list[k].index + ');">';
+		html += '<tr class="show_server" id="' + sorted_list[k].index + '">';
 		
 		for(var i=0; i<7; i++){
 			
@@ -406,6 +415,7 @@ function render_list(){
 	console.log(html);
 	
 	tbody.innerHTML = html;
+
 }
 
 var popup_bg = document.getElementById("popup-bg");

static/themes/Rosé Pine.css

@@ -0,0 +1,20 @@
+:root{
+	/* background */
+	--1d2021: #191724;
+	--282828: #1f1d2e;
+	--3c3836: #26233a;
+	--504945: #524f67;
+
+	/* font */
+	--928374: #e0def4;
+	--a89984: #908caa;
+	--bdae93: #6e6a86;
+	--8ec07c: #9ccfd8;
+	--ebdbb2: #f6c177;
+
+	/* code highlighter */
+	--comment: #6e6a86;
+	--default: #c4a7e7;
+	--keyword: #31748f;
+	--string: #f6c177;
+}

web.php

@@ -147,7 +147,7 @@ if(count($results["image"]) !== 0){
 		
 		$right["image"] .=
 			'<a class="image" href="' . htmlspecialchars($image["url"]) . '" rel="noreferrer nofollow" title="' . htmlspecialchars($image["title"]) . '" data-json="' . htmlspecialchars(json_encode($image["source"])) . '" tabindex="-1">' .
-				'<img src="' . $frontend->htmlimage($image["source"][count($image["source"]) - 1]["url"], "square") . '" alt="thumb">';
+				'<img loading="lazy" src="' . $frontend->htmlimage($image["source"][count($image["source"]) - 1]["url"], "square") . '" alt="thumb">';
 		
 		if(
 			$image["source"][0]["width"] !== null &&
@@ -301,7 +301,7 @@ if(count($results["answer"]) !== 0){
 			
 			$right["answer"] .=
 				'<a href="' . htmlspecialchars($answer["thumb"]) . '" rel="noreferrer nofollow" class="photo">' .
-					'<img src="' . $frontend->htmlimage($answer["thumb"], "cover") . '" alt="thumb" class="openimg">' .
+					'<img loading="lazy" src="' . $frontend->htmlimage($answer["thumb"], "cover") . '" alt="thumb" class="openimg">' .
 				'</a>';
 		}
 		
@@ -355,7 +355,7 @@ if(count($results["answer"]) !== 0){
 				
 				case "image":
 					$right["answer"] .=
-						'<a href="' . htmlspecialchars($description["url"]) . '" rel="noreferrer nofollow" tabindex="-1"><img src="' . $frontend->htmlimage($description["url"], "thumb") . '" alt="image" class="fullimg openimg"></a>';
+						'<a href="' . htmlspecialchars($description["url"]) . '" rel="noreferrer nofollow" tabindex="-1"><img loading="lazy" src="' . $frontend->htmlimage($description["url"], "thumb") . '" alt="image" class="fullimg openimg"></a>';
 					break;
 				
 				case "audio":
@@ -410,7 +410,7 @@ if(count($results["answer"]) !== 0){
 				$right["answer"] .=
 					'<a href="' . htmlspecialchars($url) . '" rel="noreferrer nofollow" tabindex="-1">' .
 						'<div class="center">' .
-							'<img src="/static/icon/' . $icon . '.png" alt="icon">' .
+							'<img loading="lazy" src="/static/icon/' . $icon . '.png" alt="icon">' .
 							'<div class="title">' . $website . '</div>' .
 						'</div>' .
 					'</a>';