added lazy image loading for search page

- added ability to specify default scrapers in environment variables

Dockerfile

@@ -2,16 +2,22 @@ FROM alpine:3.21
 WORKDIR /var/www/html/4get
 
 RUN apk update && apk upgrade
-RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg
+RUN apk add php84 apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg libstdc++
 
 COPY . .
 
+RUN mv ./shared_objects/libcurl-impersonate.so /usr/local/lib
 RUN chmod 777 /var/www/html/4get/icons
+RUN ln -s /usr/bin/php84 /usr/bin/php
 
 EXPOSE 80
 EXPOSE 443
 
+ENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate.so
+ENV CURL_IMPERSONATE=chrome146
+
 ENV FOURGET_PROTO=http
+ENV FOURGET_USER_AGENT='Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.7680.80 Safari/537.36'
 
 ENTRYPOINT  ["./docker/docker-entrypoint.sh"]
-CMD ["start"]
+CMD ["start"]

README.md

@@ -13,11 +13,11 @@ _NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._
 
 ## Totally unbiased comparison between alternatives
 
-|                            | 4get                    | searx(ng) | libreY      | araa      | hearch.co         |
+|                            | 4get                    | searx(ng) | whoogle    | degoog                               |
-|----------------------------|-------------------------|-----------|-------------|-----------|-------------------|
+|----------------------------|-------------------------|-----------|------------|--------------------------------------|
-| RAM usage                  | 200-400mb~              | 2GB~      | 200-400mb~  | 2GB~      | idk               |
+| RAM usage                  | 100-400mb~              | 400mb-1GB | 100mb      | 200mb-1GB                            |
-| Does it suck               | no (debunked by snopes) | yes       | yes         | a little  | better than searx |
+| Does it suck               | no (debunked by snopes) | yes       | kind of?   | its kinda cool but no search filters |
-| Does it work               | ye                      | sometimes | sometimes   | sometimes | yes               |
+| Does it work               | ye                      | lmao      | shits dead | works right now...                   |
 
 ## Features
 1. Rotating proxies on a per-scraper basis
@@ -31,25 +31,29 @@ tl;dr 4get is the best way to browse for shit.
 
 # Supported websites
 
-| Web        | Images       | Videos       | News       | Music      | Autocompleter |
+| web          | images       | videos       | news         | music      | autocomplete |
-|------------|--------------|--------------|------------|------------|---------------|
+|--------------|--------------|--------------|--------------|------------|--------------|
-| DuckDuckGo | DuckDuckGo   | YouTube      | DuckDuckGo | Soundcloud | Brave         |
+| DuckDuckGo   | DuckDuckGo   | YouTube      | DuckDuckGo   | SoundCloud | Brave        |
-| Brave      | Brave        | Sepia Search | Brave      |            | DuckDuckGo    |
+| Brave        | Yandex       | Vimeo        | Brave        | Swisscows  | DuckDuckGo   |
-| Yandex     | Yandex       | DuckDuckGo   | Google     |            | Yandex        |
+| Yandex       | Brave        | Sepia Search | Google       |            | Yandex       |
-| Google     | Google       | Brave        | Startpage  |            | Google        |
+| Google       | Google       | DuckDuckGo   | Yahoo! JAPAN |            | Google       |
-| Startpage  | Startpage    | Yandex       | Qwant      |            | Startpage     |
+| Google API   | Google API   | Brave        | Startpage    |            | Startpage    |
-| Qwant      | Qwant        | Google       | Mojeek     |            | Kagi          |
+| Google CSE   | Google CSE   | Yandex       | Qwant        |            | Kagi         |
-| Ghostery   | Yep          | Startpage    | Baidu      |            | Qwant         |
+| Yahoo! JAPAN | Yahoo! JAPAN | Google       | Mojeek       |            | Qwant        |
-| Yep        | Baidu        | Qwant        |            |            | Ghostery      |
+| Startpage    | Startpage    | Yahoo! JAPAN | Baidu        |            | Ghostery     |
-| Greppr     | Pinterest    | Baidu        |            |            | Yep           |
+| Qwant        | Qwant        | Startpage    |              |            | Yep          |
-| Crowdview  | 500px        | Coc Coc      |            |            | Marginalia    |
+| Ghostery     | Baidu        | Qwant        |              |            | Marginalia   |
-| Mwmbl      | VSCO         |              |            |            | YouTube       |
+| Yep          | Solofield    | Baidu        |              |            | YouTube      |
-| Mojeek     | Imgur        |              |            |            | Soundcloud    |
+| Mwmbl        | Pinterest    | Coc Coc      |              |            | SoundCloud   |
-| Baidu      | FindThatMeme |              |            |            |               |
+| Mojeek       | Cara         | Solofield    |              |            |              |
-| Coc Coc    |              |              |            |            |               |
+| Baidu        | Flickr       |              |              |            |              |
-| Marginalia |              |              |            |            |               |
+| Coc Coc      | Pexels       |              |              |            |              |
-| wiby       |              |              |            |            |               |
+| Solofield    | Pixabay      |              |              |            |              |
-| Curlie     |              |              |            |            |               |
+| Marginalia   | Unsplash     |              |              |            |              |
+| wiby         | 500px        |              |              |            |              |
+|              | VSCO         |              |              |            |              |
+|              | Imgur        |              |              |            |              |
+|              | FindThatMeme |              |              |            |              |
 
 # Installation
 Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/">documentation index</a>. I recommend following the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">apache2 guide</a>.

data/config.php

@@ -22,6 +22,13 @@ class config{
 	
 	// Enable the API?
 	const API_ENABLED = true;
+
+    // Default scrapers
+    const SCRAPER_WEB = "ddg";
+    const SCRAPER_IMAGES = "ddg";
+    const SCRAPER_NEWS = "ddg";
+    const SCRAPER_VIDEOS = "yt";
+    const SCRAPER_MUSIC = "sc";
 	
 	//
 	// BOT PROTECTION

docker/docker-entrypoint.sh

@@ -23,9 +23,8 @@ else
         cp -r ${FOURGET_SRC}/docker/apache/http/conf.d/* /etc/apache2/conf.d
 fi
 
-php ./docker/gen_config.php
-
 if [ "$@" = "start" ]; then
+        php ./docker/gen_config.php
         echo "4get is running"
         exec httpd -DFOREGROUND
 else 

lib/frontend.php

@@ -276,7 +276,7 @@ class frontend{
 				
 				$payload .=
 					'">' .
-						'<img class="thumb" src="' . $this->htmlimage($site["thumb"]["url"], $size) . '" alt="thumb">';
+						'<img loading="lazy" class="thumb" src="' . $this->htmlimage($site["thumb"]["url"], $size) . '" alt="thumb">';
 				
 				if($duration !== null){
 					
@@ -499,7 +499,7 @@ class frontend{
 			$payload =
 				'<div class="url">' .
 					'<button class="favicon" tabindex="-1">' .
-						'<img src="/favicon?s=404" alt="xx">' .
+						'<img loading="lazy" src="/favicon?s=404" alt="xx">' .
 					'</button>';
 		}else{
 			
@@ -528,18 +528,18 @@ class frontend{
 			$payload =
 				'<div class="url">' .
 					'<button class="favicon" tabindex="-1">' .
-						'<img src="/favicon?s=' . htmlspecialchars($host["scheme"] . "://" . $host["host"]) . '" alt="' . htmlspecialchars($esc) . '">' .
+						'<img loading="lazy" src="/favicon?s=' . htmlspecialchars($host["scheme"] . "://" . $host["host"]) . '" alt="' . htmlspecialchars($esc) . '">' .
-						//'<img src="/404.php" alt="' . htmlspecialchars($esc) . '">' .
+						//'<img loading="lazy" src="/404.php" alt="' . htmlspecialchars($esc) . '">' .
 					'</button>' .
 					'<div class="favicon-dropdown">';
 			
 			$payload .=
-					'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
+					'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
-					'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
+					'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
-					'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
+					'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
-					'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
+					'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
-					'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
+					'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
-					'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
+					'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img loading="lazy" src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
 				'</div>';
 		}
 		
@@ -593,7 +593,7 @@ class frontend{
 	
 	public function getscraperfilters($page){
 		
-		$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : null;
+		$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : "";
 		
 		if(
 			isset($_GET["scraper"]) &&
@@ -619,163 +619,23 @@ class frontend{
 			}
 		}
 		
+        include "lib/scrapers.php";
+
 		// add search field
 		$filters =
 			[
 				"s" => [
 					"option" => "_SEARCH"
-				]
+				],
+				"scraper" => [
+					"display" => "Scraper",
+					"option" => AVAILABLE_SCRAPERS[$page],
+				],
 			];
-		
+
-		// define default scrapers
-		switch($page){
-			
-			case "web":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"ddg" => "DuckDuckGo",
-						//"yahoo" => "Yahoo!",
-						"brave" => "Brave",
-						"yandex" => "Yandex",
-						"google" => "Google",
-						"google_api" => "Google API",
-						"google_cse" => "Google CSE",
-						"yahoo_japan" => "Yahoo! JAPAN",
-						"startpage" => "Startpage",
-						"qwant" => "Qwant",
-						"ghostery" => "Ghostery",
-						"yep" => "Yep",
-						"mwmbl" => "Mwmbl",
-						"mojeek" => "Mojeek",
-						"baidu" => "Baidu",
-						"coccoc" => "Cốc Cốc",
-						"solofield" => "Solofield",
-						"marginalia" => "Marginalia",
-						"wiby" => "wiby"
-					]
-				];
-				break;
-			
-			case "images":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"ddg" => "DuckDuckGo",
-						"yandex" => "Yandex",
-						"brave" => "Brave",
-						"google" => "Google",
-						"google_api" => "Google API",
-						"google_cse" => "Google CSE",
-						"yahoo_japan" => "Yahoo! JAPAN",
-						"startpage" => "Startpage",
-						"qwant" => "Qwant",
-						"baidu" => "Baidu",
-						"solofield" => "Solofield",
-						"pinterest" => "Pinterest",
-						"cara" => "Cara",
-						"flickr" => "Flickr",
-						"pexels" => "Pexels",
-						"pixabay" => "Pixabay",
-						"unsplash" => "Unsplash",
-						"fivehpx" => "500px",
-						"vsco" => "VSCO",
-						"imgur" => "Imgur",
-						"ftm" => "FindThatMeme"
-					]
-				];
-				break;
-			
-			case "videos":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"yt" => "YouTube",
-						//"archiveorg" => "Archive.org",
-						"vimeo" => "Vimeo",
-						//"odysee" => "Odysee",
-						"sepiasearch" => "Sepia Search",
-						//"fb" => "Facebook videos",
-						"ddg" => "DuckDuckGo",
-						"brave" => "Brave",
-						"yandex" => "Yandex",
-						"google" => "Google",
-						"yahoo_japan" => "Yahoo! JAPAN",
-						"startpage" => "Startpage",
-						"qwant" => "Qwant",
-						"baidu" => "Baidu",
-						"coccoc" => "Cốc Cốc",
-						"solofield" => "Solofield"
-					]
-				];
-				break;
-			
-			case "news":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"ddg" => "DuckDuckGo",
-						"brave" => "Brave",
-						"google" => "Google",
-						"yahoo_japan" => "Yahoo! JAPAN",
-						"startpage" => "Startpage",
-						"qwant" => "Qwant",
-						"mojeek" => "Mojeek",
-						"baidu" => "Baidu"
-					]
-				];
-				break;
-			
-			case "music":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"sc" => "SoundCloud",
-						"swisscows" => "Swisscows (SoundCloud)"
-						//"spotify" => "Spotify"
-					]
-				];
-				break;
-			
-			case "booru":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"safebooru" => "Safebooru",
-						"konachan" => "Konachan",
-						"tbib" => "The Big Imageboard",
-						"gelbooru" => "Gelbooru",
-						"yandere" => "Yande.re",
-						"tbib" => "The Big Imageboard",
-						"sankakucomplex" => "SankakuComplex",
-						"soybooru" => "SoyBooru"
-					]
-				];
-				break;
-		}
-		
 		// get scraper name from user input, or default out to preferred scraper
-		$scraper_out = null;
+		$scraper_out = get_scraper($page, $get_scraper);
-		$first = true;
+
-		
-		foreach($filters["scraper"]["option"] as $scraper_name => $scraper_pretty){
-			
-			if($first === true){
-				
-				$first = $scraper_name;
-			}
-			
-			if($scraper_name == $get_scraper){
-				
-				$scraper_out = $scraper_name;
-			}
-		}
-		
-		if($scraper_out === null){
-			
-			$scraper_out = $first;
-		}
-		
 		include "scraper/$scraper_out.php";
 		$lib = new $scraper_out();
 		

lib/fuckhtml.php

@@ -553,28 +553,21 @@ class fuckhtml{
 				
 				case "\"":
 				case "'":
-					if(
+					// count preceding backslashes
-						$i !== 0 && // only check if a quote could be there
+					$bsCount = 0;
-						(
+					$j = $i - 1;
-							(
+
-								$json[$i - 1] === "\\" &&
+					while($j >= 0 && $json[$j] === "\\"){
-								(
+						$bsCount++;
-									$i === 2 ||
+						$j--;
-									$json[$i - 2] === "\\"
+					}
-								)
+
-							) ||
+					// quote is NOT escaped if even number of backslashes
-							$json[$i - 1] !== "\\"
+					if($bsCount % 2 === 0){
-						)
-					){
-						// found a non-escaped quote
-						
 						if($in_quote === null){
-							
 							// open quote
 							$in_quote = $json[$i];
-							
 						}elseif($in_quote === $json[$i]){
-							
 							// close quote
 							$in_quote = null;
 						}

lib/scrapers.php

@@ -0,0 +1,115 @@
+<?php
+
+const DEFAULT_SCRAPERS = [
+    "web" => config::SCRAPER_WEB,
+    "images" => config::SCRAPER_IMAGES,
+    "news" => config::SCRAPER_NEWS,
+    "videos" => config::SCRAPER_VIDEOS,
+    "music" => config::SCRAPER_MUSIC,
+];
+
+const AVAILABLE_SCRAPERS = [
+    "web" => [
+        "ddg" => "DuckDuckGo",
+        //"yahoo" => "Yahoo!",
+        "brave" => "Brave",
+        "yandex" => "Yandex",
+        "google" => "Google",
+        "google_api" => "Google API",
+        "google_cse" => "Google CSE",
+        "yahoo_japan" => "Yahoo! JAPAN",
+        "startpage" => "Startpage",
+        "qwant" => "Qwant",
+        "ghostery" => "Ghostery",
+        "yep" => "Yep",
+        "mwmbl" => "Mwmbl",
+        "mojeek" => "Mojeek",
+        "baidu" => "Baidu",
+        "coccoc" => "Cốc Cốc",
+        "solofield" => "Solofield",
+        "marginalia" => "Marginalia",
+        "wiby" => "wiby"
+    ],
+    "images" => [
+        "ddg" => "DuckDuckGo",
+        "yandex" => "Yandex",
+        "brave" => "Brave",
+        "google" => "Google",
+        "google_api" => "Google API",
+        "google_cse" => "Google CSE",
+        "yahoo_japan" => "Yahoo! JAPAN",
+        "startpage" => "Startpage",
+        "qwant" => "Qwant",
+        "baidu" => "Baidu",
+        "solofield" => "Solofield",
+        "pinterest" => "Pinterest",
+        "cara" => "Cara",
+        "flickr" => "Flickr",
+        "pexels" => "Pexels",
+        "pixabay" => "Pixabay",
+        "unsplash" => "Unsplash",
+        "fivehpx" => "500px",
+        "vsco" => "VSCO",
+        "imgur" => "Imgur",
+        "ftm" => "FindThatMeme"
+    ],
+
+    "videos" => [
+        "yt" => "YouTube",
+        //"archiveorg" => "Archive.org",
+        "vimeo" => "Vimeo",
+        //"odysee" => "Odysee",
+        "sepiasearch" => "Sepia Search",
+        //"fb" => "Facebook videos",
+        "ddg" => "DuckDuckGo",
+        "brave" => "Brave",
+        "yandex" => "Yandex",
+        "google" => "Google",
+        "yahoo_japan" => "Yahoo! JAPAN",
+        "startpage" => "Startpage",
+        "qwant" => "Qwant",
+        "baidu" => "Baidu",
+        "coccoc" => "Cốc Cốc",
+        "solofield" => "Solofield"
+    ],
+
+    "news" => [
+        "ddg" => "DuckDuckGo",
+        "brave" => "Brave",
+        "google" => "Google",
+        "yahoo_japan" => "Yahoo! JAPAN",
+        "startpage" => "Startpage",
+        "qwant" => "Qwant",
+        "mojeek" => "Mojeek",
+        "baidu" => "Baidu"
+    ],
+
+    "music" => [
+        "sc" => "SoundCloud",
+        "swisscows" => "Swisscows (SoundCloud)",
+        //"spotify" => "Spotify"
+    ],
+
+    "booru" => [
+        "safebooru" => "Safebooru",
+        "konachan" => "Konachan",
+        "tbib" => "The Big Imageboard",
+        "gelbooru" => "Gelbooru",
+        "yandere" => "Yande.re",
+        "sankakucomplex" => "SankakuComplex",
+        "soybooru" => "SoyBooru",
+    ],
+];
+
+function get_scraper(string $page, string $selected): string
+{
+    $available = AVAILABLE_SCRAPERS[$page];
+    if (!$available) {
+        return "__NONEXISTENT__";
+    }
+    if ($available[$selected]) {
+        return $selected;
+    }
+    $default = DEFAULT_SCRAPERS[$page];
+    return $available[$default] ? $default : array_key_first($available);
+}

scraper/brave.php

@@ -347,11 +347,8 @@ class brave{
 				$q["spellcheck"] = "0";
 			}
 		}
-		/*
-		$handle = fopen("scraper/brave.html", "r");
-		$html = fread($handle, filesize("scraper/brave.html"));
-		fclose($handle);*/
 		
+		//$html = file_get_contents("scraper/brave.html");
 		try{
 			$html =
 				$this->get(

scraper/pinterest.php

@@ -297,7 +297,13 @@ class pinterest{
 			throw new Exception("Failed to decode JSON");
 		}
 		
-		print_r($json);
+		if(
+			isset($json["client_context"]["is_bad_bot"]) &&
+			(int)$json["client_context"]["is_bad_bot"] === 1
+		){
+			
+			throw new Exception("Pinterest blocked this instance or request proxy.");
+		}
 		
 		$out = [
 			"status" => "ok",

scraper/startpage.php

@@ -564,12 +564,16 @@ class startpage{
 					break;
 				
 				case "spellsuggest-google":
-					$out["spelling"] =
+					
-						[
+					if(isset($category["results"][0]["query"])){
-							"type" => "including",
+						
-							"using" => $json["render"]["query"],
+						$out["spelling"] =
-							"correction" => $category["results"][0]["query"]
+							[
-						];
+								"type" => "including",
+								"using" => $json["render"]["query"],
+								"correction" => urldecode($category["results"][0]["query"])
+							];
+					}
 					break;
 				
 				case "dictionary-qi":
@@ -645,318 +649,6 @@ class startpage{
 			}
 		}
 		
-		// parse instant answers
-		if(
-			$get["extendedsearch"] == "yes" &&
-			$get_instant_answer === true
-		){
-			
-			// https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=BqZ3inqrAgF701&sr=1
-			try{
-				$post = [
-					"se" => "n0vze2y9dqwy",
-					"q" => $json["render"]["query"],
-					"results" => [], // populate
-					"enableKnowledgePanel" => true,
-					"enableMediaThumbBar" => false,
-					"enableSearchSuggestions" => false,
-					"enableTripadvisorProperties" => [],
-					"enableTripadvisorPlaces" => [],
-					"enableTripadvisorPlacesForLocations" => [],
-					"enableWebProducts" => false,
-					"tripadvisorPartnerId" => null,
-					"tripadvisorMapColorMode" => "light",
-					"tripadvisorDisablesKnowledgePanel" => false,
-					"instantAnswers" => [
-						"smartAnswers",
-						"youtube",
-						"tripadvisor"
-					],
-					"iaType" => null,
-					"forceEnhancedKnowledgePanel" => false,
-					"shoppingOnly" => false,
-					"allowAdultProducts" => true,
-					"lang" => "en",
-					"browserLang" => "en-US",
-					"browserTimezone" => "America/New_York",
-					"market" => null,
-					"userLocation" => null,
-					"userDate" => date("Y-m-d"),
-					"userAgentType" => "unknown"
-				];
-				
-				foreach($out["web"] as $result){
-					
-					$post["results"][] = [
-						"url" => $result["url"],
-						"title" => $result["title"]
-					];
-				}
-				
-				$post = json_encode($post, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE);
-				
-				$additional_data =
-					$this->get(
-						$proxy,
-						"https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=" . $json["render"]["callback_sc"] . "&sr=1",
-						$post,
-						true,
-						true
-					);
-				
-				$additional_data = json_decode($additional_data, true);
-				
-				if($additional_data === null){
-					
-					throw new Exception("Failed to decode JSON"); // just break out, dont fail completely
-				}
-				
-				if(!isset($additional_data["knowledgePanel"])){
-					
-					throw new Exception("Response has missing data (knowledgePanel)");
-				}
-				
-				$additional_data = $additional_data["knowledgePanel"];
-				
-				$answer = [
-					"title" => $additional_data["meta"]["title"],
-					"description" => [
-						[
-							"type" => "quote",
-							"value" => $additional_data["meta"]["description"]
-						]
-					],
-					"url" => $additional_data["meta"]["origWikiUrl"],
-					"thumb" => $additional_data["meta"]["image"],
-					"table" => [],
-					"sublink" => []
-				];
-				
-				// parse html for instant answer
-				$this->fuckhtml->load($additional_data["html"]);
-				
-				$div =
-					$this->fuckhtml
-					->getElementsByTagName(
-						"div"
-					);
-				
-				// get description
-				$description =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-kp-short-extract sx-kp-short-extract-complete",
-						$div
-					);
-				
-				if(count($description) !== 0){
-					
-					$answer["description"][] = [
-						"type" => "text",
-						"value" =>
-							html_entity_decode(
-								$this->fuckhtml
-								->getTextContent(
-									$description[0]
-								)
-							)
-					];
-				}
-				
-				// get socials
-				$socials =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-wiki-social-link",
-						"a"
-					);
-				
-				foreach($socials as $social){
-					
-					$title =
-						$this->fuckhtml
-						->getTextContent(
-							$social["attributes"]["title"]
-						);
-					
-					$url =
-						$this->fuckhtml
-						->getTextContent(
-							$social["attributes"]["href"]
-						);
-					
-					switch($title){
-						
-						case "Official Website":
-							$title = "Website";
-							break;
-					}
-					
-					$answer["sublink"][$title] = $url;
-				}
-				
-				// get videos
-				$videos =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-kp-video-grid-item",
-						$div
-					);
-				
-				foreach($videos as $video){
-					
-					$this->fuckhtml->load($video);
-					
-					$as =
-						$this->fuckhtml
-						->getElementsByTagName(
-							"a"
-						);
-					
-					if(count($as) === 0){
-						
-						// ?? invalid
-						continue;
-					}
-					
-					$image =
-						$this->fuckhtml
-						->getElementsByAttributeName(
-							"data-sx-src",
-							"img"
-						);
-					
-					if(count($image) !== 0){
-						
-						$thumb = [
-							"ratio" => "16:9",
-							"url" =>
-								$this->fuckhtml
-								->getTextContent(
-									$image[0]["attributes"]["data-sx-src"]
-								)
-						];
-					}else{
-
-						$thumb = [
-							"ratio" => null,
-							"url" => null
-						];
-					}
-					
-					$out["video"][] = [
-						"title" =>
-							$this->fuckhtml
-							->getTextContent(
-								$as[0]["attributes"]["title"]
-							),
-						"description" => null,
-						"date" => null,
-						"duration" => null,
-						"views" => null,
-						"thumb" => $thumb,
-						"url" =>
-							$this->fuckhtml
-							->getTextContent(
-								$as[0]["attributes"]["href"]
-							)
-					];
-				}
-				
-				// reset
-				$this->fuckhtml->load($additional_data["html"]);
-				
-				// get table elements
-				$table =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-infobox",
-						"table"
-					);
-				
-				if(count($table) !== 0){
-					
-					$trs =
-						$this->fuckhtml
-						->getElementsByTagName(
-							"tr"
-						);
-					
-					foreach($trs as $tr){
-						
-						$this->fuckhtml->load($tr);
-						
-						// ok so startpage devs cant fucking code a table
-						// td = content
-						// th (AAAHH) = title
-						$tds =
-							$this->fuckhtml
-							->getElementsByTagName(
-								"td"	
-							);
-						
-						$ths =
-							$this->fuckhtml
-							->getElementsByTagName(
-								"th"
-							);
-						
-						if(
-							count($ths) === 1 &&
-							count($tds) === 1
-						){
-							
-							$title =
-								$this->fuckhtml
-								->getTextContent(
-									$ths[0]
-								);
-							
-							$description = [];
-							
-							$this->fuckhtml->load($tds[0]);
-							
-							$lis =
-								$this->fuckhtml
-								->getElementsByTagName(
-									"li"
-								);
-							
-							if(count($lis) !== 0){
-								
-								foreach($lis as $li){
-									
-									$description[] =
-										$this->fuckhtml
-										->getTextContent(
-											$li
-										);
-								}
-								
-								$description = implode(", ", $description);
-							}else{
-								
-								$description =
-									$this->fuckhtml
-									->getTextContent(
-										$tds[0]
-									);
-							}
-							
-							$answer["table"][$title] = $description;
-						}
-					}
-				}
-				
-				$out["answer"][] = $answer;
-				
-			}catch(Exception $error){
-				
-				// do nothing
-				//echo "error!";
-			}
-		}
-		
 		return $out;
 	}
 	
@@ -1428,12 +1120,16 @@ class startpage{
 							[
 								"lui" => "english",
 								"language" => "english",
-								"query" => $str["q"],
-								"cat" => $pagetype,
 								"sc" => $str["sc"],
 								"t" => "device",
+								"cat" => $pagetype,
 								"segment" => "startpage.udog",
-								"page" => $str["page"]
+								"abd" => 0,
+								"abe" => 0,
+								"query" => $str["q"],
+								"page" => $str["page"],
+								"qsr" => "all",
+								"qadf" => "none" // @ todo fix (??)
 							]
 						),
 						$pagetype,

scraper/yandex.php

@@ -868,123 +868,71 @@ class yandex{
 		
 		if($get["npt"]){
 			
-			[$params, $proxy] =
+			[$get, $proxy] =
 				$this->backend->get(
 					$get["npt"],
 					"video"
 				);
 			
-			$params = json_decode($params, true);
+			$get = json_decode($get, true);
-			
-			$nsfw = $params["nsfw"];
-			unset($params["nsfw"]);
 		}else{
 			
-			$search = $get["s"];
+			if(strlen($get["s"]) === 0){
-			if(strlen($search) === 0){
 				
 				throw new Exception("Search term is empty!");
 			}
 			
 			$proxy = $this->backend->get_ip();
-			$nsfw = $get["nsfw"];
-			$time = $get["time"];
-			$duration = $get["duration"];
-			
-			// https://yandex.com/video/search
-			// ?tmpl_version=releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63
-			// &format=json
-			// &request=
-			// {
-			//		"blocks":[
-			//			{"block":"extra-content","params":{},"version":2},
-			//			{"block":"i-global__params:ajax","params":{},"version":2},
-			//			{"block":"search2:ajax","params":{},"version":2},
-			//			{"block":"vital-incut","params":{},"version":2},
-			//			{"block":"content_type_search","params":{},"version":2},
-			//			{"block":"serp-controller","params":{},"version":2},
-			//			{"block":"cookies_ajax","params":{},"version":2}
-			//		],
-			//		"metadata":{
-			//			"bundles":{"lb":"^G]!q<X120"},
-			//			"assets":{"las":"react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"},
-			//			"extraContent":{"names":["i-react-ajax-adapter"]}
-			//		}
-			// }
-			// &yu=4861394161661655015
-			// &from=tabbar
-			// &reqid=1693106278500184-6825210746979814879-balancer-l7leveler-kubr-yp-sas-7-BAL-4237
-			// &suggest_reqid=486139416166165501562797413447032
-			// &text=minecraft
-			
-			$params = [
-				"tmpl_version" => "releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63",
-				"format" => "json",
-				"request" => json_encode([
-					"blocks" => [
-						(object)[
-							"block" => "extra-content",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "i-global__params:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "search2:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "vital-incut",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "content_type_search",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "serp-controller",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "cookies_ajax",
-							"params" => (object)[],
-							"version" => 2
-						]
-					],
-					"metadata" => (object)[
-						"bundles" => (object)[
-							"lb" => "^G]!q<X120"
-						],
-						"assets" => (object)[
-							"las" => "react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"
-						],
-						"extraContent" => (object)[
-							"names" => [
-								"i-react-ajax-adapter"
-							]
-						]
-					]
-				]),
-				"text" => $search
-			];
-			
-			if($duration != "any"){
-				
-				$params["duration"] = $duration;
-			}
-			
-			if($time != "any"){
-				
-				$params["within"] = $time;
-			}
 		}
+		
+		// https://yandex.com/video/search?text=skycamefalling&from=tabbar&format=json&ncrnd=7271&p=0&parent-reqid=&request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]}&serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL&yu=3091577281773194415&tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		// https://yandex.com/video/search
+		// ?text=skycamefalling
+		// &from=tabbar
+		// &format=json
+		// &ncrnd=7271
+		// &p=0
+		// &parent-reqid=
+		// &request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]} {"blocks":[{"block":"video-app","params":{}}]}
+		// &serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL
+		// &yu=3091577281773194415
+		// &tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		
+		$params = [
+			"text" => $get["s"],
+			"from" => "tabbar",
+			"format" => "json",
+			"ncrnd" => 7271,
+			"p" => 0,
+			"parent-reqid" => "",
+			"request" => json_encode((object)[
+				"blocks" => [
+					(object)[
+						"block" => "video-app",
+						"params" => (object)[]
+					]
+				]
+			]),
+			"serpid" => "1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL",
+			"yu" => 3091577281773194415,
+			"tmpl_version" => "releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a"
+		];
+		
+		if(isset($get["p"])){
+			
+			$params["p"] = $get["p"];
+		}
+		
+		if($get["duration"] != "any"){
+			
+			$params["duration"] = $get["duration"];
+		}
+		
+		if($get["time"] != "any"){
+			
+			$params["within"] = $get["time"];
+		}
+		
 		/*
 		$handle = fopen("scraper/yandex-video.json", "r");
 		$json = fread($handle, filesize("scraper/yandex-video.json"));
@@ -996,7 +944,7 @@ class yandex{
 					$proxy,
 					"https://yandex.com/video/search",
 					$params,
-					$nsfw,
+					$get["nsfw"],
 					"yandex_v"
 				);
 		}catch(Exception $error){
@@ -1011,7 +959,7 @@ class yandex{
 			throw new Exception("Could not parse JSON");
 		}
 		
-		if(!isset($json["blocks"])){
+		if(!isset($json["results"]["clips"]["items"])){
 			
 			throw new Exception("Yandex blocked this 4get instance. Please try again in 7~ minutes.");
 		}
@@ -1026,209 +974,120 @@ class yandex{
 			"reel" => []
 		];
 		
-		$html = null;
+		foreach($json["results"]["clips"]["items"] as $k => $data){
-		foreach($json["blocks"] as $block){
 			
-			if(isset($block["html"])){
+			if(isset($data["preview"]["posterSrc"])){
 				
-				$html .= $block["html"];
+				$poster = $data["preview"]["posterSrc"];
+				
+				if(
+					preg_match(
+						'/^\/\//',
+						$data["preview"]["posterSrc"]
+					)
+				){
+					
+					$poster = "https:" . $poster;
+				}
+				
+				$thumb = [
+					"ratio" => "16:9",
+					"url" => $poster
+				];
+			}else{
+				
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
 			}
+			
+			$out["video"][] = [
+				"title" => $data["relatedParams"]["text"],
+				"description" => $this->titledots($data["description"]),
+				"author" => [
+					"name" =>
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"] : null,
+					"url" => 
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"] : null,
+					"avatar" => null
+				],
+				"date" =>
+					isset($json["results"]["clips"]["dups"][$k]["date"]) ?
+					strtotime($json["results"]["clips"]["dups"][$k]["date"]) : null,
+				"duration" =>
+					isset($json["results"]["clips"]["dups"][$k]["duration"]["value"]) ?
+					(int)$json["results"]["clips"]["dups"][$k]["duration"]["value"] : null,
+				"views" =>
+					isset($json["results"]["clips"]["dups"][$k]["views"]["text"]) ?
+					$this->parseviews($json["results"]["clips"]["dups"][$k]["views"]["text"]) : null,
+				"thumb" => $thumb,
+				"url" =>
+					preg_replace(
+						'/^http:\/\//',
+						"https://",
+						$data["relatedParams"]["related_url"]
+					)
+			];
 		}
 		
-		$this->fuckhtml->load($html);
+		// get npt
-		
+		if($json["results"]["search"]["hasNextPage"]){
-		$div =
+			
-			$this->fuckhtml
+			$get["p"] = (int)$json["results"]["search"]["currentPage"] + 1;
-			->getElementsByTagName("div");
-		
-		/*
-			Get nextpage
-		*/
-		$npt =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"more more_direction_next i-bem",
-				$div
-			);
-		
-		if(count($npt) !== 0){
 			
-			$params["p"] = "1";
-			$params["nsfw"] = $nsfw;
 			$out["npt"] =
 				$this->backend->store(
-					json_encode($params),
+					json_encode($get),
 					"video",
 					$proxy
 				);
 		}
 		
-		$items =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"serp-item",
-				$div
-			);
-		
-		foreach($items as $item){
-			
-			$data =
-				json_decode(
-					$this->fuckhtml
-					->getTextContent(
-						$item["attributes"]["data-video"]
-					),
-					true
-				);
-			
-			$this->fuckhtml->load($item);
-			
-			$thumb =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"thumb-image__image",
-					"img"
-				);
-			
-			$c = 1;
-			if(count($thumb) === 0){
-				
-				$thumb = [
-					"url" => null,
-					"ratio" => null
-				];
-			}else{
-				
-				$thumb = [
-					"url" =>
-						str_replace(
-							"//",
-							"https://",
-							$this->fuckhtml
-							->getTextContent(
-								$thumb
-								[0]
-								["attributes"]
-								["src"]
-							),
-							$c
-						),
-					"ratio" => "16:9"
-				];
-			}
-			
-			$smallinfos =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__sitelinks-item",
-					"div"
-				);
-			
-			$date = null;
-			$views = null;
-			$first = true;
-			
-			foreach($smallinfos as $info){
-				
-				if($first){
-					
-					$first = false;
-					continue;
-				}
-				
-				$info =
-					$this->fuckhtml
-					->getTextContent(
-						$info
-					);
-				
-				if($temp_date = strtotime($info)){
-					
-					$date = $temp_date;
-				}else{
-					
-					$views = $this->parseviews($info);
-				}
-			}
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__text serp-item__text_visibleText_always",
-					"div"
-				);
-			
-			if(count($description) === 0){
-				
-				$description = null;
-			}else{
-				
-				$description =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-			}
-			
-			$out["video"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$this->titledots(
-							$data["title"]
-						)
-					),
-				"description" => $description,
-				"author" => [
-					"name" => null,
-					"url" => null,
-					"avatar" => null
-				],
-				"date" => $date,
-				"duration" =>
-					(int)$data
-					["counters"]
-					["toHostingLoaded"]
-					["stredParams"]
-					["duration"],
-				"views" => $views,
-				"thumb" => $thumb,
-				"url" =>
-					str_replace(
-						"http://",
-						"https://",
-						$this->fuckhtml
-						->getTextContent(
-							$data["counters"]
-							["toHostingLoaded"]
-							["postfix"]
-							["href"]
-						),
-						$c
-					)
-			];
-		}
-		
 		return $out;
 	}
 	
-	private function parseviews($text){
+	private function parseviews($number){
 		
-		$text = explode(" ", $text);
+		// decimal should always be 1 number long
+		$number = explode(" ", $number, 2);
+		$number = $number[0];
 		
-		$num = (float)$text[0];
+		$unit = strtolower($number[strlen($number) - 1]);
-		$mod = $text[1];
 		
-		switch($mod){
+		$tmp = explode(".", $number, 2);
+		$number = (int)$number;
+		
+		if(count($tmp) === 2){
 			
-			case "bln.": $num = $num * 1000000000; break;
+			$decimal = (int)$tmp[1];
-			case "mln.": $num = $num * 1000000; break;
+		}else{
-			case "thsd.": $num = $num * 1000; break;
+			
+			$decimal = 0;
 		}
 		
-		return $num;
+		switch($unit){
+			
+			case "k":
+				$exponant = 1000;
+				break;
+			
+			case "m":
+				$exponant = 1000000;
+				break;
+			
+			case "b";
+				$exponant = 1000000000;
+				break;
+			
+			default:
+				$exponant = 1;
+				break;
+		}
+		
+		return ($number * $exponant) + ($decimal * ($exponant / 10));
 	}
 	
 	private function titledots($title){

web.php

@@ -147,7 +147,7 @@ if(count($results["image"]) !== 0){
 		
 		$right["image"] .=
 			'<a class="image" href="' . htmlspecialchars($image["url"]) . '" rel="noreferrer nofollow" title="' . htmlspecialchars($image["title"]) . '" data-json="' . htmlspecialchars(json_encode($image["source"])) . '" tabindex="-1">' .
-				'<img src="' . $frontend->htmlimage($image["source"][count($image["source"]) - 1]["url"], "square") . '" alt="thumb">';
+				'<img loading="lazy" src="' . $frontend->htmlimage($image["source"][count($image["source"]) - 1]["url"], "square") . '" alt="thumb">';
 		
 		if(
 			$image["source"][0]["width"] !== null &&
@@ -301,7 +301,7 @@ if(count($results["answer"]) !== 0){
 			
 			$right["answer"] .=
 				'<a href="' . htmlspecialchars($answer["thumb"]) . '" rel="noreferrer nofollow" class="photo">' .
-					'<img src="' . $frontend->htmlimage($answer["thumb"], "cover") . '" alt="thumb" class="openimg">' .
+					'<img loading="lazy" src="' . $frontend->htmlimage($answer["thumb"], "cover") . '" alt="thumb" class="openimg">' .
 				'</a>';
 		}
 		
@@ -355,7 +355,7 @@ if(count($results["answer"]) !== 0){
 				
 				case "image":
 					$right["answer"] .=
-						'<a href="' . htmlspecialchars($description["url"]) . '" rel="noreferrer nofollow" tabindex="-1"><img src="' . $frontend->htmlimage($description["url"], "thumb") . '" alt="image" class="fullimg openimg"></a>';
+						'<a href="' . htmlspecialchars($description["url"]) . '" rel="noreferrer nofollow" tabindex="-1"><img loading="lazy" src="' . $frontend->htmlimage($description["url"], "thumb") . '" alt="image" class="fullimg openimg"></a>';
 					break;
 				
 				case "audio":
@@ -410,7 +410,7 @@ if(count($results["answer"]) !== 0){
 				$right["answer"] .=
 					'<a href="' . htmlspecialchars($url) . '" rel="noreferrer nofollow" tabindex="-1">' .
 						'<div class="center">' .
-							'<img src="/static/icon/' . $icon . '.png" alt="icon">' .
+							'<img loading="lazy" src="/static/icon/' . $icon . '.png" alt="icon">' .
 							'<div class="title">' . $website . '</div>' .
 						'</div>' .
 					'</a>';