From cf3c77ed04545dc310f1692c86ac7e017ff1193a Mon Sep 17 00:00:00 2001 From: throwaway Date: Tue, 19 May 2026 17:08:10 -0700 Subject: [PATCH 01/30] add lwthiker/curl-impersonate --- Dockerfile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1017d8..867cc2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,17 @@ +FROM lwthiker/curl-impersonate:0.6.1-ff-alpine AS curl-impersonate + FROM alpine:3.21 WORKDIR /var/www/html/4get RUN apk update && apk upgrade -RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg +RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates + +COPY --from=curl-impersonate /usr/local/bin /usr/local/bin +COPY --from=curl-impersonate /usr/local/lib /usr/local/lib + +ENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so +ENV CURL_IMPERSONATE=ff117 +ENV CURL_IMPERSONATE_HEADERS=no COPY . . @@ -14,4 +23,4 @@ EXPOSE 443 ENV FOURGET_PROTO=http ENTRYPOINT ["./docker/docker-entrypoint.sh"] -CMD ["start"] \ No newline at end of file +CMD ["start"] From 25e8095d0df761db91d537e78058c905b04eb554 Mon Sep 17 00:00:00 2001 From: lolcat Date: Wed, 20 May 2026 00:59:27 -0400 Subject: [PATCH 02/30] yep scraper fix? still getting tons of CF errors... --- scraper/yep.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scraper/yep.php b/scraper/yep.php index 6e661bd..ad6a4b0 100644 --- a/scraper/yep.php +++ b/scraper/yep.php @@ -303,11 +303,11 @@ class yep{ try{ - // https://api.yep.com/fs/2/search?limit=20&query=asmr + // https://api.yep.com/search?limit=20&query=asmr $json = $this->get( $this->backend->get_ip(), - "https://api.yep.com/fs/2/search", + "https://api.yep.com/search", $filters ); From e1e92d715ec22570f1bdd5a4cdf6ceecb5426428 Mon Sep 17 00:00:00 2001 From: lolcat Date: Wed, 20 May 2026 11:01:18 -0400 Subject: [PATCH 03/30] add support for yep api --- data/api_keys/yep.txt | 1 + data/config.php | 26 +++++-- scraper/yep.php | 163 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 168 insertions(+), 22 deletions(-) create mode 100644 data/api_keys/yep.txt diff --git a/data/api_keys/yep.txt b/data/api_keys/yep.txt new file mode 100644 index 0000000..80f6458 --- /dev/null +++ b/data/api_keys/yep.txt @@ -0,0 +1 @@ +# Paste Yep API keys here diff --git a/data/config.php b/data/config.php index 161458f..c9b9fb3 100644 --- a/data/config.php +++ b/data/config.php @@ -23,6 +23,16 @@ class config{ // Enable the API? const API_ENABLED = true; + // + // 4play (session provider) + // + // Enable 4play API? + const FPLAY_ENABLE_API = true; + + // 4play password. Please set this to something secure if you enable the 4play API. + // This password is used to POST sessions to /api/v2/provide_sesh + const FPLAY_PASSWORD = "1234"; + // // BOT PROTECTION // @@ -118,10 +128,10 @@ class config{ // Default user agent to use for scraper requests. Sometimes ignored to get specific webpages // Changing this might break things. - const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0"; + const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:151.0) Gecko/20100101 Firefox/151.0"; // User agent to use with 4get-friendly APIs - const USER_AGENT_FRIENDLY = "4get-scrapist"; + const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)"; // Proxy pool assignments for each scraper // false = Use server's raw IP @@ -131,7 +141,6 @@ class config{ const PROXY_YAHOO = false; const PROXY_YAHOO_JAPAN = false; const PROXY_BRAVE = false; - const PROXY_FB = false; // facebook const PROXY_GOOGLE = false; const PROXY_GOOGLE_API = false; const PROXY_GOOGLE_CSE = false; @@ -155,7 +164,6 @@ class config{ const PROXY_VIMEO = false; const PROXY_YEP = false; const PROXY_PINTEREST = false; - const PROXY_SANKAKUCOMPLEX = false; const PROXY_FLICKR = false; const PROXY_PIXABAY = false; const PROXY_UNSPLASH = false; @@ -164,8 +172,6 @@ class config{ const PROXY_VSCO = false; const PROXY_SEZNAM = false; const PROXY_NAVER = false; - const PROXY_GREPPR = false; - const PROXY_CROWDVIEW = false; const PROXY_MWMBL = false; const PROXY_FTM = false; // findthatmeme const PROXY_IMGUR = false; @@ -173,6 +179,11 @@ class config{ const PROXY_YANDEX_W = false; // yandex web const PROXY_YANDEX_I = false; // yandex images const PROXY_YANDEX_V = false; // yandex videos + const PROXY_SAFEBOORU = false; + const PROXY_KONACHAN = false; + const PROXY_YANDERE = false; + const PROXY_TBIB = false; + const PROXY_GELBOORU = false; // // Scraper-specific parameters @@ -185,4 +196,7 @@ class config{ // Use "null" to default out to HTML scraping OR specify a string to // use the API (Eg: "public"). API has less filters. const MARGINALIA_API_KEY = null; + + // Yep + const YEP_USE_API = false; } diff --git a/scraper/yep.php b/scraper/yep.php index ad6a4b0..c1855fb 100644 --- a/scraper/yep.php +++ b/scraper/yep.php @@ -216,7 +216,7 @@ class yep{ ]; } - private function get($proxy, $url, $get = []){ + private function get($proxy, $url, $get = [], $use_api = false, $post_data = null, $bearer = null){ $curlproc = curl_init(); @@ -231,21 +231,37 @@ class yep{ curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding - curl_setopt($curlproc, CURLOPT_HTTPHEADER, - ["User-Agent: " . config::USER_AGENT, - "Accept: */*", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip, deflate, br, zstd", - "Referer: https://yep.com/", - "Origin: https://yep.com", - "DNT: 1", - "Connection: keep-alive", - "Sec-Fetch-Dest: empty", - "Sec-Fetch-Mode: cors", - "Sec-Fetch-Site: same-site", - "Priority: u=4", - "TE: trailers"] - ); + + if($use_api){ + + $post_data = json_encode($post_data); + + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["Content-Type: application/json", + "Authorization: Bearer $bearer", + "Content-Length: " . strlen($post_data)] + ); + + curl_setopt($curlproc, CURLOPT_POST, true); + curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data); + }else{ + + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["User-Agent: " . config::USER_AGENT, + "Accept: */*", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br, zstd", + "Referer: https://yep.com/", + "Origin: https://yep.com", + "DNT: 1", + "Connection: keep-alive", + "Sec-Fetch-Dest: empty", + "Sec-Fetch-Mode: cors", + "Sec-Fetch-Site: same-site", + "Priority: u=4", + "TE: trailers"] + ); + } curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); @@ -270,6 +286,11 @@ class yep{ public function web($get){ + if(config::YEP_USE_API){ + + return $this->web_api($get); + } + $search = $get["s"]; if(strlen($search) === 0){ @@ -392,6 +413,116 @@ class yep{ } + private function web_api($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + // parse filters + $filters = [ + "query" => $search, + "limit" => 100 + ]; + + if($get["nsfw"] == "no"){ $filters["safe_search"] = true; } + if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; } + + // add api key + $key_data = $this->backend->get_key(); + + print_r($filters); + + try{ + + $json = + $this->get( + $this->backend->get_ip($key_data["increment"]), + "https://platform.yep.com/api/search", + [], + true, + $filters, + $key_data["key"] + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch JSON"); + } + + // should never happen + //$this->detect_cf($json); + + $json = json_decode($json, true); + //$json = json_decode(file_get_contents("scraper/yep.json"), true); + + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + + if(isset($json["error"])){ + + throw new Exception("Yep API returned an error: " . $json["error"]); + } + + if(isset($json["errors"])){ + + throw new Exception("Yep API returned the following errors: {$json["message"]}"); + } + + if( + isset($json["success"]) && + $json["success"] !== true + ){ + + throw new Exception("Yep API returned a false-y success value"); + } + + if(!isset($json["results"])){ + + throw new Exception("Yep API did not return a results object"); + } + + foreach($json["results"] as $item){ + + $out["web"][] = [ + "title" => $item["title"], + "description" => $item["description"], + "url" => $item["url"], + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + } + + return $out; + } + + private function detect_cf($payload){ // detect cloudflare page From 95819bfe52ee4d7a79daf091b8d0e619a29ec352 Mon Sep 17 00:00:00 2001 From: lolcat Date: Wed, 20 May 2026 11:05:41 -0400 Subject: [PATCH 04/30] yep api fix --- scraper/yep.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scraper/yep.php b/scraper/yep.php index c1855fb..682dd6f 100644 --- a/scraper/yep.php +++ b/scraper/yep.php @@ -449,8 +449,6 @@ class yep{ // add api key $key_data = $this->backend->get_key(); - print_r($filters); - try{ $json = @@ -504,6 +502,15 @@ class yep{ foreach($json["results"] as $item){ + if( + $item["url"] === null || + $item["url"] == "" + ){ + + // sometimes API fucks up + continue; + } + $out["web"][] = [ "title" => $item["title"], "description" => $item["description"], From 1e3f3d82fdca5ced58db35910afb682d2b4ea995 Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 24 May 2026 22:59:20 -0400 Subject: [PATCH 05/30] remove baseless degoog slander lolololo --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 57a7569..1f60880 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ _NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._ ## Totally unbiased comparison between alternatives -| | 4get | searx(ng) | whoogle | degoog | -|----------------------------|-------------------------|-----------|------------|--------------------------------------| -| RAM usage | 100-400mb~ | 400mb-1GB | 100mb | 200mb-1GB | -| Does it suck | no (debunked by snopes) | yes | kind of? | its kinda cool but no search filters | -| Does it work | ye | lmao | shits dead | works right now... | +| | 4get | searx(ng) | whoogle | degoog | +|----------------------------|-------------------------|-----------|------------|-------------------------------------------| +| RAM usage | 100-400mb~ | 400mb-1GB | 100mb | 100mb-250mb | +| Does it suck | no (debunked by snopes) | yes | kind of? | hit and miss with search filters | +| Does it work | ye | lmao | shits dead | works $rightNow, it's actually kinda cool | ## Features 1. Rotating proxies on a per-scraper basis From 5f112737b863e5fa629cc7621e4d97cc50ab9e64 Mon Sep 17 00:00:00 2001 From: lolcat Date: Wed, 27 May 2026 21:54:36 -0400 Subject: [PATCH 06/30] detect ddg captcha --- scraper/ddg.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scraper/ddg.php b/scraper/ddg.php index c59b423..188ad92 100644 --- a/scraper/ddg.php +++ b/scraper/ddg.php @@ -492,6 +492,17 @@ class ddg{ "form" ); + foreach($forms as $form){ + + if( + isset($form["attributes"]["id"]) && + $form["attributes"]["id"] == "challenge-form" + ) + }{ + + throw new Exception("DuckDuckGo returned an image captcha"); + } + foreach(array_reverse($forms) as $form){ $this->fuckhtml->load($form); From d7d35f02f30f0b113e841ec1278ea446001d095e Mon Sep 17 00:00:00 2001 From: lolcat Date: Wed, 27 May 2026 21:55:56 -0400 Subject: [PATCH 07/30] holy shit im retarded --- scraper/ddg.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scraper/ddg.php b/scraper/ddg.php index 188ad92..d752cb8 100644 --- a/scraper/ddg.php +++ b/scraper/ddg.php @@ -497,10 +497,10 @@ class ddg{ if( isset($form["attributes"]["id"]) && $form["attributes"]["id"] == "challenge-form" - ) - }{ + ){ - throw new Exception("DuckDuckGo returned an image captcha"); + throw new Exception("DuckDuckGo returned an image captcha"); + } } foreach(array_reverse($forms) as $form){ From ba2c6f6bb165104365fcc1a5fcbeeb1ec559da10 Mon Sep 17 00:00:00 2001 From: lolcat Date: Fri, 29 May 2026 21:48:06 -0400 Subject: [PATCH 08/30] fix startpage pagination, again --- scraper/startpage.php | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/scraper/startpage.php b/scraper/startpage.php index a555dc5..d1f2b41 100644 --- a/scraper/startpage.php +++ b/scraper/startpage.php @@ -1105,29 +1105,20 @@ class startpage{ if($page["name"] == "Next"){ - parse_str( - explode( - "?", - $page["url"], - 2 - )[1], - $str - ); - return $this->backend->store( http_build_query( [ "lui" => "english", "language" => "english", - "sc" => $str["sc"], + "sc" => $json["render"]["search_sc"], "t" => "device", "cat" => $pagetype, - "segment" => "startpage.udog", - "abd" => 0, - "abe" => 0, - "query" => $str["q"], - "page" => $str["page"], + "segment" => "organic", + "abd" => "0", + "abe" => "0", + "query" => $json["render"]["query"], + "page" => $page["number"], "qsr" => "all", "qadf" => "none" // @ todo fix (??) ] From 72fc1ae0803b388b2d9ddb26148d8b94ba404318 Mon Sep 17 00:00:00 2001 From: lolcat Date: Fri, 29 May 2026 21:54:16 -0400 Subject: [PATCH 09/30] stop changing the fucking encoding of the motherfucking word corrector --- scraper/startpage.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scraper/startpage.php b/scraper/startpage.php index d1f2b41..ec70834 100644 --- a/scraper/startpage.php +++ b/scraper/startpage.php @@ -570,8 +570,8 @@ class startpage{ $out["spelling"] = [ "type" => "including", - "using" => $json["render"]["query"], - "correction" => urldecode($category["results"][0]["query"]) + "using" => rawurldecode($json["render"]["query"]), + "correction" => html_entity_decode($category["results"][0]["query"]) ]; } break; From 572730415a25a02875e2d2572dea5e44f4cceded Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 31 May 2026 13:42:50 -0400 Subject: [PATCH 10/30] startpage pagination fix, fucking again --- scraper/startpage.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scraper/startpage.php b/scraper/startpage.php index ec70834..0ce0e64 100644 --- a/scraper/startpage.php +++ b/scraper/startpage.php @@ -408,6 +408,8 @@ class startpage{ //$html = file_get_contents("scraper/startpage.html"); } + //echo htmlspecialchars($html); + $this->detect_captcha($html); if( @@ -1117,7 +1119,7 @@ class startpage{ "segment" => "organic", "abd" => "0", "abe" => "0", - "query" => $json["render"]["query"], + "query" => rawurldecode($json["render"]["query"]), "page" => $page["number"], "qsr" => "all", "qadf" => "none" // @ todo fix (??) From caa9a6c51615ae61e83d36ddd0cf8c79e718e225 Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 31 May 2026 20:15:09 -0400 Subject: [PATCH 11/30] yep api fix --- scraper/yep.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraper/yep.php b/scraper/yep.php index 682dd6f..b715676 100644 --- a/scraper/yep.php +++ b/scraper/yep.php @@ -440,7 +440,7 @@ class yep{ // parse filters $filters = [ "query" => $search, - "limit" => 100 + "limit" => 99 // for some reason they dont show anything sometimes at 100 results ]; if($get["nsfw"] == "no"){ $filters["safe_search"] = true; } From fc8b41bfaddbe0402677c29cc6e94968b858de8a Mon Sep 17 00:00:00 2001 From: lolcat Date: Mon, 1 Jun 2026 02:48:27 -0400 Subject: [PATCH 12/30] added naver kansanmidaaaa --- lib/frontend.php | 3 + scraper/naver.php | 1104 +++++++++++++++++++++++++++++++++++++++++++++ settings.php | 12 + 3 files changed, 1119 insertions(+) create mode 100644 scraper/naver.php diff --git a/lib/frontend.php b/lib/frontend.php index 8f2f20b..86acf20 100644 --- a/lib/frontend.php +++ b/lib/frontend.php @@ -648,6 +648,7 @@ class frontend{ "yep" => "Yep", "mwmbl" => "Mwmbl", "mojeek" => "Mojeek", + "naver" => "Naver", "baidu" => "Baidu", "coccoc" => "Cốc Cốc", "solofield" => "Solofield", @@ -670,6 +671,7 @@ class frontend{ "yahoo_japan" => "Yahoo! JAPAN", "startpage" => "Startpage", "qwant" => "Qwant", + "naver" => "Naver", "baidu" => "Baidu", "solofield" => "Solofield", "pinterest" => "Pinterest", @@ -703,6 +705,7 @@ class frontend{ "yahoo_japan" => "Yahoo! JAPAN", "startpage" => "Startpage", "qwant" => "Qwant", + "naver" => "Naver", "baidu" => "Baidu", "coccoc" => "Cốc Cốc", "solofield" => "Solofield" diff --git a/scraper/naver.php b/scraper/naver.php new file mode 100644 index 0000000..652241c --- /dev/null +++ b/scraper/naver.php @@ -0,0 +1,1104 @@ +backend = new backend("naver"); + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); + } + + public function getfilters($page){ + + $base = [ + "time" => [ + "display" => "Time", + "option" => [ + "any" => "Any time", + "1h" => "Last hour", + "1d" => "Last day", + "1w" => "Last week", + "1m" => "Last month", + "3m" => "Last 3 months", + "6m" => "Last 6 months", + "1y" => "Last year", + ] + ] + ]; + + switch($page){ + + case "web": + return + array_merge([ + "sort" => [ + "display" => "Sort by", + "option" => [ + "relevance" => "Relevance", // r + "most_recent" => "Most recent" // dd + ] + ] + ], $base); + break; + + case "images": + return + array_merge( + $base, + [ + "size" => [ + "display" => "Size", + "option" => [ + "any" => "Any size", + "highdef" => "High definition" // &res_fr=786432&res_to=100000000 + ] + ], + "color" => [ // &color= + "display" => "Color", + "option" => [ + "any" => "Any color", + "orange" => "Orange", + "yellow" => "Yellow", + "lime" => "Lime", + "green" => "Green", + "cyan" => "Cyan", + "blue" => "Blue", + "purple" => "Purple", + "pink" => "Pink", + "apricot" => "Apricot", + "ocher" => "Ocher", + "sepia" => "Sepia", + "black" => "Black", + "gray" => "Gray", + "white" => "White" + ] + ], + "license" => [ // &ccl= + "display" => "License", + "option" => [ + "any" => "Any license", + "1" => "CCL Total", + "2" => "Commercial use", + "4" => "Modifications permitted" + ] + ] + ] + ); + break; + + case "videos": + return + [ + "time" => [ // done + "display" => "Time", + "option" => [ + "any" => "Any time", + "1day" => "Last day", + "1week" => "Last week", + "1month" => "Last month", + "3month" => "Last 3 months", + "6month" => "Last 6 months", + "1year" => "Last year" + ] + ], + "sort" => [ // done + "display" => "Sort by", + "option" => [ + "rel" => "Relevance", + "date" => "Most recent", // &sort=date + "playcount" => "Most views", // &sort=playcount + ] + ], + "type" => [ // done + "display" => "Type", + "option" => [ + "any" => "Any videos", + "shorts" => "Shorts" // dtype=shorts + ] + ], + "duration" => [ + "display" => "Duration", // &playtime= + "option" => [ + "any" => "Any duration", + "0:600" => "10 minutes", + "601:1800" => "10-30 minutes", + "1801:3600" => "30-60 minutes", + "3601:65535" => "More than 1 hour" + ] + ] + ]; + break; + } + } + + private function get($proxy, $url, $get = [], $is_xhr = false){ + + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + // use http2 + curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + + if($is_xhr === false){ + + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["User-Agent: " . config::USER_AGENT, + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "DNT: 1", + "Sec-GPC: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: same-origin", + "Priority: u=0, i", + "Sec-Fetch-User: ?1"] + ); + }else{ + + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["User-Agent: " . config::USER_AGENT, + "Accept: */*", + "Accept-Language: en-US,en;q=0.9", + "Accept-Encoding: gzip, deflate, br, zstd", + "Referer: https://search.naver.com/", + "DNT: 1", + "Sec-GPC: 1", + "Alt-Used: s.search.naver.com", + "Connection: keep-alive", + "Sec-Fetch-Dest: script", + "Sec-Fetch-Mode: no-cors", + "Sec-Fetch-Site: same-site", + "TE: trailers"] + ); + } + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $this->backend->assign_proxy($curlproc, $proxy); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + + + public function web($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + if($get["npt"]){ + + [$d, $proxy] = $this->backend->get($get["npt"], "web"); + + try{ + + $html = + $this->get( + $proxy, + "https://search.naver.com/search.naver" . $d, + [] + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + }else{ + + // parse filters + // https://search.naver.com + // /search.naver + // ?nso= + // &page=1 + // &query=nisekoi + // &sm=tab_pge + // &start=1 + // &where=web + + $filters = [ + "nso" => "", + "query" => $search, + "sm" => "tab_pge", + "where" => "web", + "start" => 1 // increment by number of results each time (16??) + ]; + + $options = []; + + if($get["sort"] != "relevance"){ + + $options[] = "so:dd"; + } + + if($get["time"] != "any"){ + + $options[] = "p:" . $get["time"]; + } + + if(count($options) !== 0){ + + $filters["nso"] = implode(",", $options); + } + + //$html = file_get_contents("scraper/naver.html"); + + $proxy = $this->backend->get_ip(); + + try{ + $html = + $this->get( + $proxy, + "https://search.naver.com/search.naver", + $filters + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + } + + $this->fuckhtml->load($html); + + $results = + preg_split( + '/entry\.bootstrap\(document\.getElementById\("[a-f0-9-r]+"\), ?/', + $html + ); + + if(count($results) !== 2){ + + // this is thrown when no results are found + + $nsfw_probe = + $this->fuckhtml + ->getElementsByClassName( + "dsc_adult", + "div" + ); + + if(count($nsfw_probe) !== 0){ + + $out["answer"][] = [ + "title" => "NSFW results", + "description" => [ + [ + "type" => "text", + "value" => "Naver blocks logged-out NSFW searches." + ] + ], + "url" => null, + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + } + + return $out; + //throw new Exception("Failed to grep results entrypoint"); + } + + $json = + json_decode( + $this->fuckhtml + ->extract_json( + $results[1] + ), + true + ); + + if(!isset($json["body"]["props"]["children"][0]["props"]["children"])){ + + throw new Exception("Failed to access nested children"); + } + + foreach($json["body"]["props"]["children"][0]["props"]["children"] as $result){ + + if( + !isset($result["templateId"]) || + $result["templateId"] != "webItem" + ){ + + // should not happen + continue; + } + + $result = $result["props"]; + + // get sublinks + $sublinks = []; + + if(isset($result["subLinks"])){ + + foreach($result["subLinks"] as $s){ + + $sublinks[] = [ + "title" => $s["text"], + "description" => null, + "url" => $s["href"], + "date" => null + ]; + } + } + + if(isset($result["linkBtns"])){ + + foreach($result["linkBtns"] as $s){ + + $sublinks[] = [ + "title" => $s["text"], + "description" => null, + "url" => $s["href"], + "date" => null + ]; + } + } + + // get image (thumbnail, i guess) + if(isset($result["images"][0]["imageSrc"])){ + + $thumb = [ + "ratio" => "16:9", + "url" => $this->unshit_thumb($result["images"][0]["imageSrc"]) + ]; + }else{ + + $thumb = [ + "ratio" => null, + "url" => null + ]; + } + + // get table elements + $table = []; + + if(isset($result["keyValue"]["contents"])){ + + foreach($result["keyValue"]["contents"] as $s){ + + if(!isset($s["valueData"]["text"])){ continue; } + + $table[$s["key"]] = $s["valueData"]["text"]; + } + } + + // get date + $time = null; + + if(isset($result["bodyPrefixes"][0]["text"])){ + + $date = + strtotime( + substr( + $result["bodyPrefixes"][0]["text"], + -1 + ) + ); + + if($date !== false){ + + $time = $date; + } + } + + $out["web"][] = [ + "title" => $this->decode_html($result["title"]), + "description" => $this->decode_html($result["bodyText"]), + "url" => $result["href"], + "date" => $time, + "type" => "web", + "thumb" => $thumb, + "sublink" => $sublinks, + "table" => $table + ]; + } + + // get next page + $npt = + $this->fuckhtml + ->getElementsByClassName( + "btn_next", + "a" + ); + + if(count($npt) !== 0){ + + $out["npt"] = + $this->backend->store( + $this->fuckhtml + ->getTextContent( + $npt[0]["attributes"]["href"] + ), + "web", + $proxy + ); + } + + return $out; + } + + + public function image($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $out = [ + "status" => "ok", + "npt" => null, + "image" => [] + ]; + + if($get["npt"]){ + + [$url, $proxy] = $this->backend->get($get["npt"], "images"); + + try{ + + $json = + $this->get( + $proxy, + $url, + [], + true + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + }else{ + + $filters = [ + "ac" => "0", + "api_type" => "pc_tab_more", + "aq" => "0", + "display" => 100, + "logStart" => 1, + "mode" => "column", + "nso" => "so:r,p:all", + "nx_search_query" => $search, + "query" => $search, + "section" => "image", + "sm" => "tab_opt", + "ssc" => "tab.image.all", + "start" => 1, + "where" => "image" + // no callback, returns raw json lol + ]; + + $options = [ + "so:r" + ]; + + if($get["time"] != "any"){ + + $options[] = "p:" . $get["time"]; + } + + if(count($options) !== 0){ + + $filters["nso"] = implode(",", $options); + } + + if($get["size"] != "any"){ + + $filters["res_fr"] = 786432; + $filters["res_to"] = 100000000; + } + + if($get["color"] != "any"){ + + $filters["color"] = $get["color"]; + } + + if($get["license"] != "any"){ + + $filters["ccl"] = $get["license"]; + } + + //$json = file_get_contents("scraper/naver.html"); + + $proxy = $this->backend->get_ip(); + + try{ + $json = + $this->get( + $proxy, + "https://s.search.naver.com/p/c/image/46/search.naver", + $filters, + true + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + } + + $json = json_decode($json, true); + + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + + if(!isset($json["items"])){ + + // no results returned :( + return $out; + //throw new Exception("Naver did not return an items object"); + } + + foreach($json["items"] as $image){ + + $out["image"][] = [ + "title" => trim($image["title"], "."), + "source" => [ + [ + "url" => $image["originalUrl"], + "width" => (int)$image["orgWidth"], + "height" => (int)$image["orgHeight"] + ], + [ + "url" => $image["thumb"], + "width" => (int)$image["thumbWidth"], + "height" => (int)$image["thumbHeight"] + ] + ], + "url" => $image["link"] + ]; + } + + // get npt + if( + isset($json["url"]) && + $json["url"] != "" && + $json["url"] != null + ){ + + $out["npt"] = + $this->backend->store( + $json["url"], + "images", + $proxy + ); + } + + return $out; + } + + + public function video($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $out = [ + "status" => "ok", + "npt" => null, + "video" => [], + "author" => [], + "livestream" => [], + "playlist" => [], + "reel" => [] + ]; + + if($get["npt"]){ + + [$url, $proxy] = $this->backend->get($get["npt"], "images"); + + try{ + + $json = + $this->get( + $proxy, + $url, + [], + true + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + }else{ + + // https://s.search.naver.com/p/video/48/search.naver?ac=0&aq=0&crbase=63&display=48&dtype=&last_block_type=recom&nlu_query=&nq=&nqx_theme={"theme":{"main":{"name":"encyclopedia","source":"TOS"}}}&nx_and_query=&nx_search_hlquery=&nx_search_query=&nx_sub_query=&page=2&period=&playtime=&ptype=&query=asmr&selected_channel=&selected_cp=&sm=mtb_pge&sort=rel&ssc=tab.video.all&start=49&video_more=1 + // https://s.search.naver.com/p/video/48/search.naver + // ?ac=0 + // &aq=0 + // &crbase=63 + // &display=48 + // &dtype= + // &last_block_type=recom + // &nlu_query= + // &nq= + // &nqx_theme={"theme":{"main":{"name":"encyclopedia","source":"TOS"}}} + // &nx_and_query= + // &nx_search_hlquery= + // &nx_search_query= + // &nx_sub_query= + // &page=2 + // &period= + // &playtime= + // &ptype= + // &query=asmr + // &selected_channel= + // &selected_cp= + // &sm=mtb_pge + // &sort=rel + // &ssc=tab.video.all + // &start=49 + // &video_more=1 + + $filters = [ + "ac" => "0", + "aq" => "0", + "crbase" => "78", + "display" => 48, + "dtype" => "", + "last_block_type" => "recom", + "nlu_query" => "", + "nq" => "", + "nx_and_query" => "", + "nx_search_hlquery" => "", + "nx_search_query" => "", + "nx_sub_query" => "", + "page" => 1, + "period" => "", + "playtime" => "", + "ptype" => "", + "query" => $search, + "selected_channel" => "", + "selected_cp" => "", + "sm" => "mtb_pge", + "sort" => "rel", + "ssc" => "tab.video.all", + "start" => 1, + "video_more" => 1 + ]; + + if($get["type"] != "any"){ + + $filters["dtype"] = $get["type"]; + } + + if($get["time"] != "any"){ + + $filters["period"] = $get["time"]; + } + + if($get["sort"] != "rel"){ + + $filters["sort"] = $get["sort"]; + } + + if($get["duration"] != "any"){ + + $filters["playtime"] = $get["duration"]; + } + + //$json = file_get_contents("scraper/naver.html"); + + $proxy = $this->backend->get_ip(); + + try{ + $json = + $this->get( + $proxy, + "https://s.search.naver.com/p/video/48/search.naver", + $filters, + true + ); + + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + } + + $json = json_decode($json, true); + + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + + if(!isset($json["collection"])){ + + return $out; + //throw new Exception("Naver did not return a collection HTML element"); + } + + foreach($json["collection"] as $snippet){ + + if(!isset($snippet["html"])){ continue; } + + $this->fuckhtml->load($snippet["html"]); + + $div = + $this->fuckhtml + ->getElementsByTagName( + "div" + ); + + $items = + $this->fuckhtml + ->getElementsByAttributeValue( + "data-template-id", + "videoItem", + $div + ); + + // parse normal videos + foreach($items as $item){ + + if($item["level"] === 6){ continue; } + + $this->fuckhtml->load($item); + + // get url + $as = + $this->fuckhtml + ->getElementsByAttributeName( + "data-heatmap-target", + "a" + ); + + if(count($as) === 0){ + + // should not happen + continue; + } + + // get thumbnail + $thumb = + $this->fuckhtml + ->getElementsByAttributeValue( + "loading", + "lazy", + "img" + ); + + if(count($thumb) !== 0){ + + $thumb = [ + "url" => + $this->unshit_thumb( + $this->fuckhtml + ->getTextContent( + $thumb[0]["attributes"]["src"] + ) + ), + "ratio" => "16:9" + ]; + }else{ + + $thumb = [ + "url" => null, + "ratio" => null + ]; + } + + // get timestamp + $timestamp_probe = + $this->fuckhtml + ->getElementsByClassName( + "sds-comps-text-type-footnote", + "span" + ); + + if(count($timestamp_probe) !== 0){ + + $timestamp = + $this->hms2int( + $this->fuckhtml + ->getTextContent( + $timestamp_probe[0] + ) + ); + }else{ + + $timestamp = null; + } + + $out["video"][] = [ + "title" => + $this->fuckhtml + ->getTextContent( + $as[0] + ), + "description" => null, + "author" => [ + "name" => + isset($item["attributes"]["profileimagealt"]) ? + $this->fuckhtml + ->getTextContent( + $item["attributes"]["profileimagealt"] + ) : null, + "url" => + isset($item["attributes"]["profileimagehref"]) ? + $this->fuckhtml + ->getTextContent( + $item["attributes"]["profileimagehref"] + ) : null, + "avatar" => + isset($item["attributes"]["profileimagesrc"]) ? + $this->fuckhtml + ->getTextContent( + $item["attributes"]["profileimagesrc"] + ) : null + ], + "date" => null, + "duration" => $timestamp, + "views" => null, + "thumb" => $thumb, + "url" => + $this->fuckhtml + ->getTextContent( + $as[0]["attributes"]["href"] + ) + ]; + } + + // reset + $this->fuckhtml->load($snippet["html"]); + + // parse reels + $carousels = + array_merge( + $this->fuckhtml // for the reels only tab + ->getElementsByClassName( + "fds-video-tab-shortform-desk-filter", + $div + ), + $this->fuckhtml // for the normal tab with reels inbetween + ->getElementsByClassName( + "fds-video-tab-shortform-desk", + $div + ) + ); + + foreach($carousels as $carousel){ + + $this->fuckhtml->load($carousel); + + $as = + $this->fuckhtml + ->getElementsByTagName( + "a" + ); + + foreach($as as $reel){ + + $this->fuckhtml->load($reel); + + $spans = + $this->fuckhtml + ->getElementsByTagName( + "span" + ); + + $title = + $this->fuckhtml + ->getTextContent( + $spans[0] + ); + + // get thumbnail + $thumb = + $this->fuckhtml + ->getElementsByAttributeValue( + "loading", + "lazy", + "img" + ); + + if(count($thumb) !== 0){ + + $thumb = [ + "url" => + $this->unshit_thumb( + $this->fuckhtml + ->getTextContent( + $thumb[0]["attributes"]["src"] + ) + ), + "ratio" => "16:9" + ]; + }else{ + + $thumb = [ + "url" => null, + "ratio" => null + ]; + } + + $name = + $this->fuckhtml + ->getElementsByClassName( + "sds-comps-profile-info-title-text", + $spans + ); + + if(count($name) === 0){ + + $name = null; + }else{ + + $name = + $this->fuckhtml + ->getTextContent( + $name[0] + ); + } + + $out["reel"][] = [ + "title" => $title, + "description" => null, + "author" => [ + "name" => $name, + "url" => null, + "avatar" => null + ], + "date" => null, + "duration" => null, + "views" => null, + "thumb" => $thumb, + "url" => + $this->fuckhtml + ->getTextContent( + $reel["attributes"]["href"] + ) + ]; + } + } + } + + // get npt + if( + isset($json["url"]) && + $json["url"] != "" && + $json["url"] != null + ){ + + $out["npt"] = + $this->backend->store( + $json["url"], + "images", + $proxy + ); + } + + return $out; + } + + + private function unshit_thumb($url){ + + $parts = parse_url($url); + + if($parts["host"] == "search.pstatic.net"){ + + parse_str($parts["query"], $str); + + if(isset($str["src"])){ + + return $str["src"]; + } + } + + return $url; + } + + + private function decode_html($html){ + + return + trim( + html_entity_decode( + strip_tags( + $html + ) + ), + "." + ); + } + + + private function hms2int($time){ + + $parts = explode(":", $time, 3); + $time = 0; + + if(count($parts) === 3){ + + // hours + $time = $time + ((int)$parts[0] * 3600); + array_shift($parts); + } + + if(count($parts) === 2){ + + // minutes + $time = $time + ((int)$parts[0] * 60); + array_shift($parts); + } + + // seconds + $time = $time + (int)$parts[0]; + + return $time; + } +} diff --git a/settings.php b/settings.php index 8b34425..5c68f4a 100644 --- a/settings.php +++ b/settings.php @@ -169,6 +169,10 @@ $settings = [ "value" => "mojeek", "text" => "Mojeek" ], + [ + "value" => "naver", + "text" => "Naver" + ], [ "value" => "baidu", "text" => "Baidu" @@ -231,6 +235,10 @@ $settings = [ "value" => "qwant", "text" => "Qwant" ], + [ + "value" => "naver", + "text" => "Naver" + ], [ "value" => "baidu", "text" => "Baidu" @@ -325,6 +333,10 @@ $settings = [ "value" => "qwant", "text" => "Qwant" ], + [ + "value" => "naver", + "text" => "Naver" + ], [ "value" => "baidu", "text" => "Baidu" From 50349d23909842275c9c31cc5b34cebfdf5ad4e8 Mon Sep 17 00:00:00 2001 From: lolcat Date: Mon, 1 Jun 2026 02:53:48 -0400 Subject: [PATCH 13/30] why do you fucking do that naver --- scraper/naver.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scraper/naver.php b/scraper/naver.php index 652241c..492488a 100644 --- a/scraper/naver.php +++ b/scraper/naver.php @@ -599,6 +599,9 @@ class naver{ foreach($json["items"] as $image){ + // why does it fucking do that + if($image["orgWidth"] === 0){ continue; } + $out["image"][] = [ "title" => trim($image["title"], "."), "source" => [ From 7ffbc5f67b337a3ce81215ab2303a0025f4d76c7 Mon Sep 17 00:00:00 2001 From: lolcat Date: Fri, 5 Jun 2026 00:18:55 -0400 Subject: [PATCH 14/30] why does http1 work jej --- scraper/vsco.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scraper/vsco.php b/scraper/vsco.php index 8a7f057..083c362 100644 --- a/scraper/vsco.php +++ b/scraper/vsco.php @@ -30,19 +30,18 @@ class vsco{ curl_setopt($curlproc, CURLOPT_HTTPHEADER, ["User-Agent: " . config::USER_AGENT, - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", + "Accept-Encoding: gzip, deflate, br, zstd", "DNT: 1", "Sec-GPC: 1", "Connection: keep-alive", "Upgrade-Insecure-Requests: 1", "Sec-Fetch-Dest: document", "Sec-Fetch-Mode: navigate", - "Sec-Fetch-Site: same-origin", + "Sec-Fetch-Site: none", "Sec-Fetch-User: ?1", - "Priority: u=0, i", - "TE: trailers"] + "Priority: u=0, i"] ); }else{ @@ -73,8 +72,8 @@ class vsco{ curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); - // http2 bypass - curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + // http1 bypass?!?!?! + curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); $this->backend->assign_proxy($curlproc, $proxy); From 76f7d0de198a53de6b5e089aaa9b4301bbf4f632 Mon Sep 17 00:00:00 2001 From: lolcat Date: Fri, 5 Jun 2026 00:22:24 -0400 Subject: [PATCH 15/30] wtf it works for yep too --- scraper/yep.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scraper/yep.php b/scraper/yep.php index b715676..9624cc9 100644 --- a/scraper/yep.php +++ b/scraper/yep.php @@ -227,8 +227,8 @@ class yep{ curl_setopt($curlproc, CURLOPT_URL, $url); - // use http2 - curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + // use http1 lol + curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding From 02cab7632dbc89f95abb954c771c486b1742e510 Mon Sep 17 00:00:00 2001 From: lolcat Date: Sat, 6 Jun 2026 02:18:30 -0400 Subject: [PATCH 16/30] license clarification --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f60880..5f68843 100644 --- a/README.md +++ b/README.md @@ -62,4 +62,4 @@ Refer to the will (at) lolcat.ca or create an issue. ## License -AGPL +AGPLv3-only From a430ef5a124b0f3a362f185d61d526ed73aad12f Mon Sep 17 00:00:00 2001 From: lolcat Date: Sat, 6 Jun 2026 04:14:50 -0400 Subject: [PATCH 17/30] remove ghostery search, discontinued --- lib/frontend.php | 1 - scraper/ghostery.php | 320 ------------------------------------------- settings.php | 4 - 3 files changed, 325 deletions(-) delete mode 100644 scraper/ghostery.php diff --git a/lib/frontend.php b/lib/frontend.php index 86acf20..9509ae7 100644 --- a/lib/frontend.php +++ b/lib/frontend.php @@ -644,7 +644,6 @@ class frontend{ "yahoo_japan" => "Yahoo! JAPAN", "startpage" => "Startpage", "qwant" => "Qwant", - "ghostery" => "Ghostery", "yep" => "Yep", "mwmbl" => "Mwmbl", "mojeek" => "Mojeek", diff --git a/scraper/ghostery.php b/scraper/ghostery.php deleted file mode 100644 index 394756e..0000000 --- a/scraper/ghostery.php +++ /dev/null @@ -1,320 +0,0 @@ -backend = new backend("ghostery"); - - include "lib/fuckhtml.php"; - $this->fuckhtml = new fuckhtml(); - } - - public function getfilters($page){ - - if($page != "web"){ - - return []; - } - - return [ - "country" => [ - "display" => "Country", - "option" => [ - "any" => "All regions", - "AR" => "Argentina", - "AU" => "Australia", - "AT" => "Austria", - "BE" => "Belgium", - "BR" => "Brazil", - "CA" => "Canada", - "CL" => "Chile", - "DK" => "Denmark", - "FI" => "Finland", - "FR" => "France", - "DE" => "Germany", - "HK" => "Hong Kong", - "IN" => "India", - "ID" => "Indonesia", - "IT" => "Italy", - "JP" => "Japan", - "KR" => "Korea", - "MY" => "Malaysia", - "MX" => "Mexico", - "NL" => "Netherlands", - "NZ" => "New Zealand", - "NO" => "Norway", - "CN" => "People's Republic of China", - "PL" => "Poland", - "PT" => "Portugal", - "PH" => "Republic of the Philippines", - "RU" => "Russia", - "SA" => "Saudi Arabia", - "ZA" => "South Africa", - "ES" => "Spain", - "SE" => "Sweden", - "CH" => "Switzerland", - "TW" => "Taiwan", - "TR" => "Turkey", - "GB" => "United Kingdom", - "US" => "United States" - ] - ] - ]; - } - - private function get($proxy, $url, $get = [], $country){ - - $curlproc = curl_init(); - - if($get !== []){ - $get = http_build_query($get); - $url .= "?" . $get; - } - - curl_setopt($curlproc, CURLOPT_URL, $url); - - curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding - curl_setopt($curlproc, CURLOPT_HTTPHEADER, - ["User-Agent: " . config::USER_AGENT, - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", - "Referer: https://ghosterysearch.com", - "DNT: 1", - "Sec-GPC: 1", - "Connection: keep-alive", - "Cookie: ctry=" . ($country == "any" ? "--" : $country) . "; noads=true", - "Upgrade-Insecure-Requests: 1", - "Sec-Fetch-Dest: document", - "Sec-Fetch-Mode: navigate", - "Sec-Fetch-Site: same-origin", - "Sec-Fetch-User: ?1", - "Priority: u=0, i"] - ); - - // http2 bypass - curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); - - curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); - curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); - curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); - - $this->backend->assign_proxy($curlproc, $proxy); - - $data = curl_exec($curlproc); - - if(curl_errno($curlproc)){ - - throw new Exception(curl_error($curlproc)); - } - - curl_close($curlproc); - return $data; - } - - public function web($get){ - - if($get["npt"]){ - - [$query, $proxy] = $this->backend->get($get["npt"], "web"); - - parse_str($query, $query); - - // country - $country = $query["c"]; - unset($query["c"]); - - $query = http_build_query($query); - - try{ - - $html = - $this->get( - $proxy, - "https://ghosterysearch.com/search?" . $query, - [], - $country - ); - }catch(Exception $error){ - - throw new Exception("Failed to fetch search page"); - } - }else{ - - $proxy = $this->backend->get_ip(); - - try{ - - $html = - $this->get( - $proxy, - "https://ghosterysearch.com/search", - [ - "q" => $get["s"] - ], - $get["country"] - ); - }catch(Exception $error){ - - throw new Exception("Failed to fetch search page"); - } - } - - $out = [ - "status" => "ok", - "spelling" => [ - "type" => "no_correction", - "using" => null, - "correction" => null - ], - "npt" => null, - "answer" => [], - "web" => [], - "image" => [], - "video" => [], - "news" => [], - "related" => [] - ]; - - $this->fuckhtml->load($html); - - $results_wrapper = - $this->fuckhtml - ->getElementsByClassName( - "results", - "section" - ); - - if(count($results_wrapper) === 0){ - - throw new Exception("Failed to grep result section"); - } - - $this->fuckhtml->load($results_wrapper[0]); - - // get search results - $results = - $this->fuckhtml - ->getElementsByClassName( - "result", - "li" - ); - - if(count($results) === 0){ - - return $out; - } - - foreach($results as $result){ - - $this->fuckhtml->load($result); - - $a = - $this->fuckhtml - ->getElementsByClassName( - "url", - "a" - ); - - if(count($a) === 0){ - - continue; - } - - $a = $a[0]; - - $out["web"][] = [ - "title" => - $this->titledots( - $this->fuckhtml - ->getTextContent( - $this->fuckhtml - ->getElementsByTagName( - "h2" - )[0] - ) - ), - "description" => - $this->titledots( - $this->fuckhtml - ->getTextContent( - $this->fuckhtml - ->getElementsByTagName( - "p" - )[0] - ) - ), - "url" => - $this->fuckhtml - ->getTextContent( - $a - ["attributes"] - ["href"] - ), - "date" => null, - "type" => "web", - "thumb" => [ - "url" => null, - "ratio" => null - ], - "sublink" => [], - "table" => [] - ]; - } - - $this->fuckhtml->load($html); - - // get pagination token - $pagination_wrapper = - $this->fuckhtml - ->getElementsByClassName( - "pagination", - "div" - ); - - if(count($pagination_wrapper) !== 0){ - - // found next page! - $this->fuckhtml->load($pagination_wrapper[0]); - - $a = - $this->fuckhtml - ->getElementsByTagName( - "a" - ); - - if(count($a) !== 0){ - - $q = - parse_url( - $this->fuckhtml - ->getTextContent( - $a[count($a) - 1] - ["attributes"] - ["href"] - ), - PHP_URL_QUERY - ); - - $out["npt"] = - $this->backend - ->store( - $q . "&c=" . $get["country"], - "web", - $proxy - ); - } - } - - return $out; - } - - private function titledots($title){ - - return trim($title, " .\t\n\r\0\x0B…"); - } -} diff --git a/settings.php b/settings.php index 5c68f4a..11146f3 100644 --- a/settings.php +++ b/settings.php @@ -91,10 +91,6 @@ $settings = [ "value" => "qwant", "text" => "Qwant" ], - [ - "value" => "ghostery", - "text" => "Ghostery" - ], [ "value" => "yep", "text" => "Yep" From 7d8b12ba9dc949bde1305d303088ac957d325cc0 Mon Sep 17 00:00:00 2001 From: lolcat Date: Mon, 8 Jun 2026 00:48:34 -0400 Subject: [PATCH 18/30] cleaning up useless shit --- audio/linear.php | 20 ---- audio/sc.php | 224 --------------------------------------------- audio/seekable.php | 20 ---- audio/spotify.php | 214 ------------------------------------------- 4 files changed, 478 deletions(-) delete mode 100644 audio/linear.php delete mode 100644 audio/sc.php delete mode 100644 audio/seekable.php delete mode 100644 audio/spotify.php diff --git a/audio/linear.php b/audio/linear.php deleted file mode 100644 index b6a848f..0000000 --- a/audio/linear.php +++ /dev/null @@ -1,20 +0,0 @@ -stream_linear_audio($_GET["s"]); -}catch(Exception $error){ - - header("X-Error: " . $error->getMessage()); -} diff --git a/audio/sc.php b/audio/sc.php deleted file mode 100644 index 6dfe66e..0000000 --- a/audio/sc.php +++ /dev/null @@ -1,224 +0,0 @@ -proxy = new proxy(); - - if(isset($_GET["u"])){ - - /* - we're now proxying audio - */ - $viewkey = $_GET["u"]; - - if(!isset($_GET["r"])){ - - $this->do404("Ranges(r) are missing"); - } - - $ranges = explode(",", $_GET["r"]); - - // sanitize ranges - foreach($ranges as &$range){ - - if(!is_numeric($range)){ - - $this->do404("Invalid range specified"); - } - - $range = (int)$range; - } - - // sort ranges (just to make sure) - sort($ranges); - - // convert ranges to pairs - $last = -1; - foreach($ranges as &$r){ - - $tmp = $r; - $r = [$last + 1, $r]; - - $last = $tmp; - } - - $browser_headers = getallheaders(); - - // get the requested range from client - $client_range = 0; - foreach($browser_headers as $key => $value){ - - if(strtolower($key) == "range"){ - - preg_match( - '/bytes=([0-9]+)/', - $value, - $client_regex - ); - - if(isset($client_regex[1])){ - - $client_range = (int)$client_regex[1]; - }else{ - - $client_range = 0; - } - break; - } - } - - if( - $client_range < 0 || - $client_range > $ranges[count($ranges) - 1][1] - ){ - - // range is not satisfiable - http_response_code(416); - header("Content-Type: text/plain"); - die(); - } - - $rng = null; - for($i=0; $iproxy->stream_linear_audio( - $viewkey - ); - }catch(Exception $error){ - - $this->do404("Could not read stream"); - } - - die(); - } - - /* - redirect user to correct resource - we need to scrape and store the byte positions in the result URL - */ - if(!isset($_GET["s"])){ - - $this->do404("The URL(s) parameter is missing"); - } - - $viewkey = $_GET["s"]; - - if( - preg_match( - '/soundcloud\.com$/', - parse_url($viewkey, PHP_URL_HOST) - ) === false - ){ - - $this->do404("This endpoint can only be used for soundcloud streams"); - } - - try{ - - $json = $this->proxy->get($viewkey)["body"]; - }catch(Exception $error){ - - $this->do404("Curl error: " . $error->getMessage()); - } - - $json = json_decode($json, true); - - if(!isset($json["url"])){ - - $this->do404("Could not get URL from JSON"); - } - - $viewkey = $json["url"]; - - $m3u8 = $this->proxy->get($viewkey)["body"]; - - $m3u8 = explode("\n", $m3u8); - - $lineout = null; - $streampos_arr = []; - foreach($m3u8 as $line){ - - $line = trim($line); - if($line[0] == "#"){ - - continue; - } - - if($lineout === null){ - $lineout = $line; - } - - preg_match( - '/\/media\/[0-9]+\/([0-9]+)\/([0-9]+)/', - $line, - $matches - ); - - if(isset($matches[0])){ - - $streampos_arr[] = [ - (int)$matches[1], - (int)$matches[2] - ]; - } - } - - if($lineout === null){ - - $this->do404("Could not get stream URL"); - } - - $lineout = - preg_replace( - '/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/', - '/media/$1/0/0', - $lineout - ); - - $streampos = []; - - foreach($streampos_arr as $pos){ - - $streampos[] = $pos[1]; - } - - $streampos = implode(",", $streampos); - - header("Location: /audio/sc?u=" . urlencode($lineout) . "&r=$streampos"); - header("Accept-Ranges: bytes"); - } - - private function do404($error){ - - http_response_code(404); - header("Content-Type: text/plain"); - header("X-Error: $error"); - die(); - } -} diff --git a/audio/seekable.php b/audio/seekable.php deleted file mode 100644 index b6a848f..0000000 --- a/audio/seekable.php +++ /dev/null @@ -1,20 +0,0 @@ -stream_linear_audio($_GET["s"]); -}catch(Exception $error){ - - header("X-Error: " . $error->getMessage()); -} diff --git a/audio/spotify.php b/audio/spotify.php deleted file mode 100644 index dc8fae6..0000000 --- a/audio/spotify.php +++ /dev/null @@ -1,214 +0,0 @@ -fuckhtml = new fuckhtml(); - - if( - !isset($_GET["s"]) || - !preg_match( - '/^(track|episode)\.([A-Za-z0-9]{22})$/', - $_GET["s"], - $matches - ) - ){ - - $this->do404("The track ID(s) parameter is missing or invalid"); - } - - try{ - - if($matches[1] == "episode"){ - - $uri = "show"; - }else{ - - $uri = $matches[1]; - } - - $embed = - $this->get("https://embed.spotify.com/{$uri}/" . $matches[2]); - }catch(Exception $error){ - - $this->do404("Failed to fetch embed data"); - } - - $this->fuckhtml->load($embed); - - $json = - $this->fuckhtml - ->getElementById( - "__NEXT_DATA__", - "script" - ); - - if($json === null){ - - $this->do404("Failed to extract JSON"); - } - - $json = - json_decode($json["innerHTML"], true); - - if($json === null){ - - $this->do404("Failed to decode JSON"); - } - - switch($matches[1]){ - - case "track": - if( - isset( - $json - ["props"] - ["pageProps"] - ["state"] - ["data"] - ["entity"] - ["audioPreview"] - ["url"] - ) - ){ - - header("Content-type: audio/mpeg"); - header( - "Location: /audio/linear?s=" . - urlencode( - $json - ["props"] - ["pageProps"] - ["state"] - ["data"] - ["entity"] - ["audioPreview"] - ["url"] - ) - ); - }else{ - - $this->do404("Could not extract playback URL"); - } - break; - - case "episode": - if( - isset( - $json - ["props"] - ["pageProps"] - ["state"] - ["data"] - ["entity"] - ["id"] - ) - ){ - - try{ - $json = - $this->get( - "https://spclient.wg.spotify.com/soundfinder/v1/unauth/episode/" . - $json - ["props"] - ["pageProps"] - ["state"] - ["data"] - ["entity"] - ["id"] . - "/com.widevine.alpha" - ); - }catch(Exception $error){ - - $this->do404("Failed to fetch audio resource"); - } - - $json = json_decode($json, true); - - if($json === null){ - - $this->do404("Failed to decode audio resource JSON"); - } - - if( - isset($json["passthrough"]) && - $json["passthrough"] == "ALLOWED" && - isset($json["passthroughUrl"]) - ){ - - header( - "Location:" . - "/audio/linear.php?s=" . - urlencode( - str_replace( - "http://", - "https://", - $json["passthroughUrl"] - ) - ) - ); - }else{ - - $this->do404("Failed to find passthroughUrl"); - } - - }else{ - - $this->do404("Failed to find episode ID"); - } - break; - } - } - - private function get($url){ - - $headers = [ - "User-Agent: " . config::USER_AGENT, - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", - "DNT: 1", - "Connection: keep-alive", - "Upgrade-Insecure-Requests: 1", - "Sec-Fetch-Dest: document", - "Sec-Fetch-Mode: navigate", - "Sec-Fetch-Site: none", - "Sec-Fetch-User: ?1" - ]; - - $curlproc = curl_init(); - - curl_setopt($curlproc, CURLOPT_URL, $url); - - curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding - curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); - - curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); - curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); - curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); - - $data = curl_exec($curlproc); - - if(curl_errno($curlproc)){ - throw new Exception(curl_error($curlproc)); - } - - curl_close($curlproc); - return $data; - } - - private function do404($error){ - - http_response_code(404); - header("Content-Type: text/plain"); - header("X-Error: $error"); - die(); - } -} From b28767d8f05977e84e3e00c7d1adc2525bc6748e Mon Sep 17 00:00:00 2001 From: lolcat Date: Mon, 8 Jun 2026 01:06:51 -0400 Subject: [PATCH 19/30] fix ACL rules for apache2 --- docs/apache2-example.md | 10 +++++++--- docs/apache2.md | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/apache2-example.md b/docs/apache2-example.md index 09f5c1d..a6115b1 100644 --- a/docs/apache2-example.md +++ b/docs/apache2-example.md @@ -49,9 +49,13 @@ Looking for the apache2 guide? + Require all denied + + + # remove directory listing from /icons + + Options -Indexes diff --git a/docs/apache2.md b/docs/apache2.md index ea337d6..7e545b3 100644 --- a/docs/apache2.md +++ b/docs/apache2.md @@ -104,9 +104,13 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r # deny access to private resources - - Order Deny,allow - Deny from all + + Require all denied + + + # remove directory listing from /icons + + Options -Indexes ``` From d54e4dfcddf5625527188d6a687742b699a33a54 Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 14 Jun 2026 15:10:55 -0400 Subject: [PATCH 20/30] qwant fix --- scraper/qwant.php | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/scraper/qwant.php b/scraper/qwant.php index c242de7..e65fc73 100644 --- a/scraper/qwant.php +++ b/scraper/qwant.php @@ -217,15 +217,19 @@ class qwant{ $headers = [ "User-Agent: " . config::USER_AGENT, "Accept: application/json, text/plain, */*", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", - "DNT: 1", - "Connection: keep-alive", - "Origin: https://www.qwant.com", + "Accept-Language: en-US,en;q=0.9", + "Accept-Encoding: gzip, deflate, br, zstd", "Referer: https://www.qwant.com/", + "Origin: https://www.qwant.com", + "DNT: 1", + "Sec-GPC: 1", + "Connection: keep-alive", "Sec-Fetch-Dest: empty", "Sec-Fetch-Mode: cors", "Sec-Fetch-Site: same-site", + "Priority: u=4", + "Pragma: no-cache", + "Cache-Control: no-cache", "TE: trailers" ]; @@ -297,7 +301,8 @@ class qwant{ "device" => "desktop", "tgp" => 3, "safesearch" => 0, - "displayed" => "true" + "displayed" => "true", + "llm" => "false" ]; switch($get["nsfw"]){ @@ -316,7 +321,7 @@ class qwant{ $json = $this->get( $proxy, - "https://fdn.qwant.com/v3/search/web", + "https://api.qwant.com/v3/search/web", $params ); @@ -353,6 +358,8 @@ class qwant{ "related" => [] ]; + $this->detect_errors($json); + if( $json["status"] != "success" && $json["data"]["error_code"] === 5 @@ -362,8 +369,6 @@ class qwant{ return $out; } - $this->detect_errors($json); - if(!isset($json["data"]["result"]["items"]["mainline"])){ throw new Exception("Server did not return a result object"); From d87bac94b5290fb49ba97e97d805caa3a93ef569 Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 14 Jun 2026 16:01:56 -0400 Subject: [PATCH 21/30] handle marginalia internal error --- scraper/marginalia.php | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scraper/marginalia.php b/scraper/marginalia.php index f5be5e1..90b7f4c 100644 --- a/scraper/marginalia.php +++ b/scraper/marginalia.php @@ -490,6 +490,37 @@ class marginalia{ return $this->web($get, $redirect); } + // detect internal error + $infobox = + $this->fuckhtml + ->getElementsByClassName("infobox"); + + if(count($infobox) !== 0){ + + foreach($infobox as $i){ + + $this->fuckhtml->load($i); + + $h2 = + $this->fuckhtml + ->getElementsByTagName("h2"); + + if( + count($h2) !== 0 && + $this->fuckhtml + ->getTextContent( + $h2[0] + ) == "Internal error" + ){ + + throw new Exception("Marginalia returned an internal server error"); + } + } + + // reset + $this->fuckhtml->load($html); + } + $sections = $this->fuckhtml ->getElementsByClassName( From 5a7cecef11d4b728ba202e1a80f5a77b7aee88fe Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 14 Jun 2026 16:07:33 -0400 Subject: [PATCH 22/30] marginalia api 502 --- scraper/marginalia.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scraper/marginalia.php b/scraper/marginalia.php index 90b7f4c..ee18cc0 100644 --- a/scraper/marginalia.php +++ b/scraper/marginalia.php @@ -289,8 +289,18 @@ class marginalia{ throw new Exception("The API key used is rate limited. Please try again in a few minutes."); } + if(strpos($json, "Bad Gateway (502)")){ + + throw new Exception("Marginalia returned a 502 error"); + } + $json = json_decode($json, true); + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + foreach($json["results"] as $result){ $out["web"][] = [ From 347300cc26390917cbd3a0c1a1c8954e2e305ae6 Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 21 Jun 2026 22:15:27 -0400 Subject: [PATCH 23/30] for some reason mojeek doesnt want me using region settings anymore lol --- scraper/mojeek.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scraper/mojeek.php b/scraper/mojeek.php index 2939be5..f123b69 100644 --- a/scraper/mojeek.php +++ b/scraper/mojeek.php @@ -340,7 +340,7 @@ class mojeek{ "zm" => "Zambia", "zw" => "Zimbabwe" ] - ], + ],/* "region" => [ "display" => "Region", "option" => [ @@ -350,7 +350,7 @@ class mojeek{ "fr" => "France", "uk" => "United Kingdom" ] - ], + ],*/ "domain" => [ "display" => "Results per domain", "option" => [ @@ -458,7 +458,7 @@ class mojeek{ "date" => 1, // show date "tlen" => 128, // max length of title //"dlen" => 511, // max length of description - "arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect! + //"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect! ]; switch($focus){ @@ -503,6 +503,11 @@ class mojeek{ } } + if(strlen($html) === 0){ + + throw new Exception("Mojeek returned an empty page (probably a homepage redirect)"); + } + $out = [ "status" => "ok", "spelling" => [ From f037ad1d7fb70811b9d67d221fc879ef8bfd136a Mon Sep 17 00:00:00 2001 From: lolcat Date: Thu, 25 Jun 2026 23:20:06 -0400 Subject: [PATCH 24/30] serverping inlineJS fix (ty ethan) --- static/serverping.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/static/serverping.js b/static/serverping.js index 6b680d5..992e025 100644 --- a/static/serverping.js +++ b/static/serverping.js @@ -80,6 +80,15 @@ document.body.insertBefore(table, quote.nextSibling); // handle sorting clicks var tbody = table.getElementsByTagName("tbody")[0]; + +// server popup without inline javascript +tbody.addEventListener('click', function(event){ + var row = event.target.closest('.show_server'); + if(!row) return; + + show_server(row.id); +}) + var th = table.getElementsByTagName("th"); for(var i=0; i'; + html += ''; for(var i=0; i<7; i++){ @@ -406,6 +415,7 @@ function render_list(){ console.log(html); tbody.innerHTML = html; + } var popup_bg = document.getElementById("popup-bg"); From abdb041a640e6560bdc57fac17c9931f95a974fa Mon Sep 17 00:00:00 2001 From: NECOtype Date: Sat, 27 Jun 2026 16:15:54 +0000 Subject: [PATCH 25/30] =?UTF-8?q?add=20Ros=C3=A9=20Pine=20theme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- static/themes/Rosé Pine.css | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 static/themes/Rosé Pine.css diff --git a/static/themes/Rosé Pine.css b/static/themes/Rosé Pine.css new file mode 100644 index 0000000..cbc11fb --- /dev/null +++ b/static/themes/Rosé Pine.css @@ -0,0 +1,20 @@ +:root{ + /* background */ + --1d2021: #191724; + --282828: #1f1d2e; + --3c3836: #26233a; + --504945: #524f67; + + /* font */ + --928374: #e0def4; + --a89984: #908caa; + --bdae93: #6e6a86; + --8ec07c: #9ccfd8; + --ebdbb2: #f6c177; + + /* code highlighter */ + --comment: #6e6a86; + --default: #c4a7e7; + --keyword: #31748f; + --string: #f6c177; +} From 8328d93b17de34c255673c2b98be7eb0f828f6c5 Mon Sep 17 00:00:00 2001 From: lolcat Date: Sat, 4 Jul 2026 17:16:35 -0400 Subject: [PATCH 26/30] duckduckgo fix --- scraper/ddg.php | 110 +----------------------------------------------- 1 file changed, 2 insertions(+), 108 deletions(-) diff --git a/scraper/ddg.php b/scraper/ddg.php index d752cb8..8f68d42 100644 --- a/scraper/ddg.php +++ b/scraper/ddg.php @@ -54,8 +54,8 @@ class ddg{ $headers = ["User-Agent: " . config::USER_AGENT, "Accept: */*", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", + "Accept-Language: en-US,en;q=0.9", + "Accept-Encoding: gzip, deflate, br, zstd", "Referer: https://duckduckgo.com/", "DNT: 1", "Sec-GPC: 1", @@ -920,112 +920,6 @@ class ddg{ ){ throw new Exception("DuckDuckGo returned a JSA challenge"); - - // get JSA initial token - if( - !preg_match( - '/let jsa *= *([0-9]+)/', - $js, - $jsa - ) - ){ - - $jsa = 0; - }else{ - - $jsa = (int)$jsa[1]; - } - - // get function bodies - preg_match_all( - '/let *([A-Za-z0-9]+) *= *function\(.*\) *{(.*)};/sU', - $js, - $functions - ); - - $parsed_functions = []; - - for($i=0; $i "multiplication", - "num" => (int)$num[1] - ]; - continue; - } - - if( - preg_match( - '/innerHTML *= *`([^`]+)`/i', - $functions[2][$i], - $challenge - ) - ){ - - $challenge[1] = - preg_replace( - '/<\/(br)>/', - '<$1>', - $challenge[1] - ); - - $parsed_functions[$functions[1][$i]] = [ - "type" => "challenge", - "text" => $challenge[1] - ]; - } - } - - // get function call order - preg_match_all( - '/jsa *= *([A-Za-z0-9]+)\(jsa\)/i', - $js, - $call_order - ); - - foreach($call_order[1] as $order){ - - if(!isset($parsed_functions[$order])){ - - throw new Exception("JS challenge solve failure: DuckDuckGo called an unknown function"); - } - - if($parsed_functions[$order]["type"] == "multiplication"){ - - $jsa = $jsa * $parsed_functions[$order]["num"]; - continue; - } - - if($parsed_functions[$order]["type"] == "challenge"){ - - // @TODO get parsed length - //$parsed_functions[$order]["text"] - - $jsa = $jsa + strlen($parsed_functions[$order]["text"]); - } - } - - try{ - $js = $this->get( - $proxy, - "https://links.duckduckgo.com" . $challenge_url[1] . $jsa, - [], - ddg::req_xhr - ); - }catch(Exception $error){ - - throw new Exception("Failed to get challenged d.js"); - } } // From 46411a6c83bb387b6c89d791a5225383a996960f Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 5 Jul 2026 01:57:56 -0400 Subject: [PATCH 27/30] i overengineered the fuck out of that baidu scraper jesus --- scraper/baidu.php | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/scraper/baidu.php b/scraper/baidu.php index efb14ca..b81c224 100644 --- a/scraper/baidu.php +++ b/scraper/baidu.php @@ -666,14 +666,24 @@ class baidu{ // // Don't parse as a search result if it's a card // - $card = + $tpl_probe = $this->fuckhtml - ->getElementsByClassName( - "cosc-card", - $div + ->getElementsByAttributeValue( + "tpl", + "www_index", + [$datafield] ); - if(count($card) !== 0){ + if( + count($tpl_probe) === 0 && + count( + $this->fuckhtml + ->getElementsByClassName( + "cosc-card", + $div + ) + ) !== 0 + ){ // // Parse chinese youtube shorts @@ -870,10 +880,13 @@ class baidu{ // class:FYB_RD -> News garbage, IGNORE $result = - $this->fuckhtml - ->getElementsByClassName( - "result", - [$datafield] + array_merge( + $tpl_probe, + $this->fuckhtml + ->getElementsByClassName( + "result", + [$datafield] + ) ); if(count($result) !== 0){ @@ -885,7 +898,7 @@ class baidu{ $title = $this->fuckhtml ->getElementsByClassName( - "sc-link", + "cosc-title-a", "a" ); @@ -905,8 +918,9 @@ class baidu{ $description = $this->fuckhtml - ->getElementsByClassName( - "c-color", + ->getElementsByAttributeValue( + "data-sanssr-cmpt", + "card/www-summary-1", $div ); From c1e9d527754b090c347f7af946eb338770ae581f Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 5 Jul 2026 18:40:43 -0400 Subject: [PATCH 28/30] notes for me --- docs/develop.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/develop.md diff --git a/docs/develop.md b/docs/develop.md new file mode 100644 index 0000000..95b8517 --- /dev/null +++ b/docs/develop.md @@ -0,0 +1,7 @@ +# Develop + +Start the PHP webserver like this, assuming you have curl-impersonate installed already + +```sh +LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000 +``` From e4c7250cade2ffc1b81a296243269b44e1501944 Mon Sep 17 00:00:00 2001 From: lolcat Date: Fri, 10 Jul 2026 21:42:10 -0400 Subject: [PATCH 29/30] qwant fix, datadome is such a joke lol --- scraper/qwant.php | 145 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 121 insertions(+), 24 deletions(-) diff --git a/scraper/qwant.php b/scraper/qwant.php index e65fc73..bd26aeb 100644 --- a/scraper/qwant.php +++ b/scraper/qwant.php @@ -1,5 +1,25 @@ backend->get($get["npt"], "web"); $params = json_decode($params, true); + $cookie = $params["cookie"]; + unset($params["cookie"]); }else{ @@ -311,18 +359,66 @@ class qwant{ case "maybe": $params["safesearch"] = 1; break; case "no": $params["safesearch"] = 2; break; } + + $cookie = null; } /* $handle = fopen("scraper/qwant_web.json", "r"); $json = fread($handle, filesize("scraper/qwant_web.json")); fclose($handle);*/ + if($cookie === null){ + // get datadome bullshit + // yeah go ahead and patch this you code monkey + + // ... you know, instead of fixing this shit, can you make the second page of results return more than 1 result? + // i search shit like "higurashi" and i only get 1 result and then just a bunch of related searches ad finitum + // also, stop pretending you guys have your own index, this is clearly all bing results + try{ + $datadome = + $this->get( + $proxy, + "https://dd.qwant.com/js/", + [ + "jspl" => "-oirHDeP7G3G9LXY3rhxGDYf6t_ZlYPzaiL2dRhQ34uNUc4dO2Td0SmmKu_VJp7-jrAIoOMrTqKgCtV4cPzntMyNuOYN7QQ1q0Fi2c-679vYpJr9ub5dRFXMhCMQmjZsOWXUa8yafIXf85UA5Goct-n4hVT5dFfBGM-dE6gwZEFkHQ1j1195sLybZFTNA1IfX26MBMxlVHoyT-aHtkd6YjGXbvOPMMsObVHDa_Cf4BmApRGo-eehsAttsCqDRwkl0M2E1b0OXCIwMVnCK-qWZ81MGaJGjqxdj_LaxhAjE9JI_4oNedAg4Yj4m_vkQBKdmWJT_bTUsllAZXqX5IxYQHJ8WvlknOTr6sR3wgoSOa_JNVHkUQxqXBbQcoDdy6VciL5rIqwWl4Y2caVYzeXa8YNXxwsfrPDegH3dQ6vOiIiCDT3bexkcS_y7GAioPPv9qDu7M9y4sI8t0VkPgMbWY6MIu5u9WAv6lEiKIYrtlyU-iiXnNdkd9EnpC1TD8_s12PtCuLg5bGgLVmI1dw8VXVZxmVyFvfBtuqXkzM-kZOyv-YIRcf-wPY8DUtnHox1d6t64oYhGv-pYV5dmZ0rGs99LC6ThiifeF_oHz5ZK46Ms68WsiAjc1HLAbFPFxWw8X6lyLlpT5aHgFdDBeCTExKXyItlc9PBxkgK8UnXHQFiWX3HTen8rziwkI31AbkMU7xGP6cvuwuLG7MA7sH3yLF1uimQlhr6EuTrImWQNV7X81DqyZ0OE7I-uT9bp_xISa5it-6AJpFtGcwz04_OuO2zvZazBeiKJI5Fw0X2r3RHTHdA29SXqTIe4y2LRAAp0MSAZUIJAMIbbrz9BG-3D5BHfQUA1Bbuklxv-tNWiuanWbbTmAiuxTAo9P-D2ZMrWHwdN8sCb3nINLur38z1xviSs2qEEjKo9kzQIWoHgwm_amV-jjDtNBXwpCQnoaqt3FfL8e59ecYWVg61Gnl3YCwq6rubFdkIv6UStcWSlj4pRFlFZSjQxzznvGTAc5Nz1JJWVeBHJItei4wqDUYEw2BNTS_fU9keV3LFyKDH_x248AxHLiXt583DhZZwPAKlpmMo0E7odzfOgVQGM5b9l64NL3RQRq8cDxI8KgmOFnPHdI7Myq1A3nW1uoBB8m9_GR_9orboKl5tAQrT0VTMRd6JyNwOOQ71ZoQpMwsD5Xd9YDZd618TNE9dA38h9OyBco_-hwKCYayyEqib1atLLBO_gnO4InR471AklOokDOKx1oY_xgNhlFrZskE4nQQOqeCQ_47oysob9s3kblA8UqDrL2vM3i2KPLWwlQzMAPrQAIv7Amp6rO-wYwBQdbP7XghYympxx9-hNxc1Duq-nS4R3pjPtZg3LoCvG1N_0JLf9jfPBqw_rdPM6DnjEOvpCdvrfv4wBlEpgqbaRQCZBPEz-O5ZlY24OyPvaNX0krPeXkbi-KYuwZSAbfW0zsksXb8cbRmz6tvh25IRYP1XNDPVd8NFrYOdDH9GzhV5MYZziYMurQlVwgumGLz85Y6qTHZU2D5qkMH7o5C5bzptReO96889oxkReS_LDhzdxH0tAQ5a9GgETJTHAXJ9Lt7X037i0qQlatbPLizOl9fo4vUBGJ7w-uSUmIzg1yIwOo8xhLu39WghXfoJ_EX5ptNi7arpAo11UoJIGEgWgq7bS40q2dtnG5HYpp9ef37iFgztM7PV5dlJf_W-n9uXJyCk2nhxZeuGiIGhMTzjP_8oeyiMFelmoyLlqK8wOs6hvWqgDD4NwtZc-XBkEAKsxuPrirb_sszp0z6BWsh3D6fBxAW_lTrNVkxZ1EBtoEN72HvO1aeX8PVuJP3CKwkkUcAhJ5Zvajo8t4Ai8ave2MTZul4QhDIe7ZALJMo71u-hNSj8YG9JLPqF1jcnbkl8ykt_XMMxX8wSZ_AfTsfmAuOWBfEzxv8Fl2Ne-oxcEtfuYnYNJIjQDHXXdtYbWKBezE41wonQu7FWh3wSSF3DDm9EocWMsUZ610nrIviOFdtTqjrY7KnOZp0hS9Hm2V04z-06Db-To71L5O5poYKuOlDAP_ULfW0hgMgYaPnt4Nive9gtue6ZQ5yNFylKM11zCWcF8vmuEPGx4NIZjvNFB-QSkdxZtBTQOgO6r4BqHJNZIDqiD9A6PyMemVtQEo-JhooX6bwHFHx6C84r-kKq1lg0RaDelbu1h6347kocxOczdgHWlagKPFrUvHpPHvCZeFmzLLmPWbDEpUhhfkbEZlNe47svO5WN1TYZMCAlE0g01pe6b_NB9-twQ4FJB1SkUQRA4o5DxiIHSgXBBfw1X98XPEKcJgCU2zRbYuU69mvyPcsxoPsLo8Cl2OAXj2U1tcLUxVhR_vu3I0SG_Ee8hxBp5b6QbpIu2sE0bgpqfi88XqUXf3qvxGbkoZDZ8dPvEvCVmpDZOOGJY5Db4MnpKBByW79lUnpbLCbsVOoxYu8SVmVQ63eGY9yBlACuWxlXA_PhCpnKYwMppjxKvKqalNJGMMg5WQ8cvhsIftE5wgaj21n0A15GwRO1WgUcsO_dLe1MqO88hTHWHV0Q3gNut3sdsmVr_n-0SniLtVAaw4IEtDZWNUfhpaoC7iyKxoJ86t8r697EYPlWUu4nTQhb3TOXAjHF22rFsJJMldKHVxjxjd086ky3OY3L_7cbjVof60-y4gTJdumw5hch-mhaT93nVnaTuPLGCAC48dZSgOitW8Dw62mPS05fTPAfKUY1HpyIWr9ACk7U2KVAijpzFB-bXvqJeCyBeUHr4cB61K2XxTmkNA8Oo9RC32TeYDrThFzoUyYZ_D6zPe1lLtQaFsPRvDBSsYCcr5fcABlST4DyUgnkgqtx-0-D1QEwtWiH3xiHLQGa82ldjqNIXL9LqwxYoCRBKuljHyqEylZD26A9Jp6SL9rk_Ol6TJlYa3zx44PHyMoA3Coy-HIZW5NMwVKdV7egUDDOchvuFnc1Zv2QpSFKc0JY8lYFtSm1HZFwjq7hf2hBqkU0Xneh8AT4Ty9QVb4mJ8rpzc8m_NBZNrpZVE-pIzmv2yGjnRwKSrwsjat7b2iY34UPFqGBlDk5FX6EfRlN9bq-nztWdextLc47ndr1gxsjC-vr6tbdXl5FxMVhbsnLH80wOC6hHubxCysr5C7Ft59s3MLhVQCc3IIOPMBhLOjjjVsu6K8kAJvvamIpj-iYUQj9T8JztAqfobytzbiwPsQ3mywUGkJpZxTjXjqkKdT6yfFdW5ksR8p3VlKGAAKft9TKKcc5IgErmIRtiZF4hGhB45nwIO5XHA59V94QuV3b0oJrleuDCBj0iDFqWYpsUVHyKl3RjrO86Yq1IbNRwXIIMrlsVYos9CS1I75k9kW4yQC6ni3a2ZitDkiEB-SoQuTlBsvfTs892Wt5q3CWH6_7srZO6l2QEAlWqyQp_P4i8j8uiEbZksGsgghVIgOkriblDP9ml7PdkpRX7S3OpY7jjQgTNR8XHaZ-SEw0-zPiqIIt5VioFK8JljPn-nMIDx-ew_vTDKqmKEOF39ezQcc5oQNTQNUvTAe8qCbK4ghS3fagItezoHtB5zm3qwhx9Iu6JMJy0UhywUZBaInY6NokiVZGDgtoqlVjOk7K4299MVZgsOr2P9QHRDo41lYnCeWWFUf50d-ouEOUO_ngteDpgG1DfV64eycrUOWAnpfI8XOUZ7mxS89SqJriw3T3tcljowHiOFw9QjYsjJi0vuHL5jfoBZVdYAUH6KiLfIScO-KgqitdUTpFwks8Qk7xfd2HcEfArlzSiAt3Hmb2Ty-5mhvtQj2RbPehgoI7gqa60b6Qffth-5tbIjXoKlZCQsaKVMAggF9C3w0Rv5uJ3ExCTGsfKO9IVOf--AfxaA8cNoO0W3NTaMH2iWftY0O1-1x3VJEcerp2v9BcZ0iIL3VFJ9eyOPYNHNwTYrI3S6OrVZxuQXFbu2E_Us32wxYqXvnMx-DEOk7bUUFIanL69Cbn_mR_9Wm1sFixBQ15diJoe1Oyl3XWHWL6_YF4XnHzZlMWySt3Awdeynwu99Ter8WMGXl-6K30novKRWC9jDVOfAtxPvXBkkRtpm8nnJT8fwkmIQoWOEPT2MCg3VYzzHzVEH-AueuRiCZj_o7L-fRqbtFrWPyVnGuetUYS5u0D3n7Rr6kCVqWhLfwjqAwV_calzIQs8-J0-AJe6GGaaONPYfXwXjYan0Sl11CS64lXvx7i3MgeOZuH210T6g6hj54dSLw0upO1L8k0FGli3I5OhLMekeGMLT_N2akyHktjbz8Odpk2ms0pJP_PdqvUegjXgH81F5lr40HAXZs7jfC9ytgcWaEjwlmnoFHSCTdlCIgJqEBzItXiz6ilk5zP35VwxdQTNoms7Meqg-ZrgRuLRaqXNtpQT9qWBmiFP6zoYLwymqytZ_P1SV9DCM-KIZQ2Cq-ew8PsYSrwl8GmfuegoKEMBiON1I4mC9ChDgzJmbo5P9te60T0URS5g9ExiQuSo-JXs0MKC57LLiNo3MPZvqR1wRJOTg0_FqS4JFxzPwN5OqhBzfs25f6kK-axf0wWp-LetUx3U1CTNaxfbCTK5tb8KnLOIs3lDwJw5H5wIIRgXnfNLBJT8A2srfXkdpA", + "eventCounters" => [], + "jsType" => "ch", + "cid" => "3i4vPGHCl74uIBumWaxF2a0PjFkyCJxcSvFnn~heX7keA6sGHgRbnzSYcMr_RzJFlfeeBdg9HeufzFPxnvSHbyWowSvjm6mNCSPhkx6MwDqptANoj93ZgGVbrzxGtcYe", + "ddk" => "78B13B7513D180B7AB6D6FF9EB0A51", + "Referer" => "https://www.qwant.com/?q={$search}&t=web", + "request" => "/?q={$search}&t=web", + "responsePage" => "origin", + "ddv" => "5.8.0" + ], + true + ); + }catch(Exception $error){ + + throw new Exception("Failed to fetch JS challenge token"); + } + + $datadome = json_decode($datadome, true); + if($datadome === null){ + + throw new Exception("Failed to decode JS challenge JSON"); + } + + if(!isset($datadome["cookie"])){ + + throw new Exception("Failed to get cookie from JS challenge endpoint"); + } + + $cookie = explode(";", $datadome["cookie"])[0]; + } + try{ $json = $this->get( $proxy, "https://api.qwant.com/v3/search/web", - $params + $params, + false, + $cookie ); }catch(Exception $error){ @@ -441,6 +537,7 @@ class qwant{ if($json["data"]["result"]["lastPage"] === false){ $params["offset"] = $params["offset"] + 10; + $params["cookie"] = $cookie; $out["npt"] = $this->backend->store( From 3e4c7c16262406bb8f7f7d4c550bd2efb58d2a8f Mon Sep 17 00:00:00 2001 From: lolcat Date: Fri, 10 Jul 2026 21:49:25 -0400 Subject: [PATCH 30/30] forgot to fix qwant images lllllooooooollllllll --- scraper/qwant.php | 122 ++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 63 deletions(-) diff --git a/scraper/qwant.php b/scraper/qwant.php index bd26aeb..322182e 100644 --- a/scraper/qwant.php +++ b/scraper/qwant.php @@ -1,25 +1,5 @@ get( - $proxy, - "https://dd.qwant.com/js/", - [ - "jspl" => "-oirHDeP7G3G9LXY3rhxGDYf6t_ZlYPzaiL2dRhQ34uNUc4dO2Td0SmmKu_VJp7-jrAIoOMrTqKgCtV4cPzntMyNuOYN7QQ1q0Fi2c-679vYpJr9ub5dRFXMhCMQmjZsOWXUa8yafIXf85UA5Goct-n4hVT5dFfBGM-dE6gwZEFkHQ1j1195sLybZFTNA1IfX26MBMxlVHoyT-aHtkd6YjGXbvOPMMsObVHDa_Cf4BmApRGo-eehsAttsCqDRwkl0M2E1b0OXCIwMVnCK-qWZ81MGaJGjqxdj_LaxhAjE9JI_4oNedAg4Yj4m_vkQBKdmWJT_bTUsllAZXqX5IxYQHJ8WvlknOTr6sR3wgoSOa_JNVHkUQxqXBbQcoDdy6VciL5rIqwWl4Y2caVYzeXa8YNXxwsfrPDegH3dQ6vOiIiCDT3bexkcS_y7GAioPPv9qDu7M9y4sI8t0VkPgMbWY6MIu5u9WAv6lEiKIYrtlyU-iiXnNdkd9EnpC1TD8_s12PtCuLg5bGgLVmI1dw8VXVZxmVyFvfBtuqXkzM-kZOyv-YIRcf-wPY8DUtnHox1d6t64oYhGv-pYV5dmZ0rGs99LC6ThiifeF_oHz5ZK46Ms68WsiAjc1HLAbFPFxWw8X6lyLlpT5aHgFdDBeCTExKXyItlc9PBxkgK8UnXHQFiWX3HTen8rziwkI31AbkMU7xGP6cvuwuLG7MA7sH3yLF1uimQlhr6EuTrImWQNV7X81DqyZ0OE7I-uT9bp_xISa5it-6AJpFtGcwz04_OuO2zvZazBeiKJI5Fw0X2r3RHTHdA29SXqTIe4y2LRAAp0MSAZUIJAMIbbrz9BG-3D5BHfQUA1Bbuklxv-tNWiuanWbbTmAiuxTAo9P-D2ZMrWHwdN8sCb3nINLur38z1xviSs2qEEjKo9kzQIWoHgwm_amV-jjDtNBXwpCQnoaqt3FfL8e59ecYWVg61Gnl3YCwq6rubFdkIv6UStcWSlj4pRFlFZSjQxzznvGTAc5Nz1JJWVeBHJItei4wqDUYEw2BNTS_fU9keV3LFyKDH_x248AxHLiXt583DhZZwPAKlpmMo0E7odzfOgVQGM5b9l64NL3RQRq8cDxI8KgmOFnPHdI7Myq1A3nW1uoBB8m9_GR_9orboKl5tAQrT0VTMRd6JyNwOOQ71ZoQpMwsD5Xd9YDZd618TNE9dA38h9OyBco_-hwKCYayyEqib1atLLBO_gnO4InR471AklOokDOKx1oY_xgNhlFrZskE4nQQOqeCQ_47oysob9s3kblA8UqDrL2vM3i2KPLWwlQzMAPrQAIv7Amp6rO-wYwBQdbP7XghYympxx9-hNxc1Duq-nS4R3pjPtZg3LoCvG1N_0JLf9jfPBqw_rdPM6DnjEOvpCdvrfv4wBlEpgqbaRQCZBPEz-O5ZlY24OyPvaNX0krPeXkbi-KYuwZSAbfW0zsksXb8cbRmz6tvh25IRYP1XNDPVd8NFrYOdDH9GzhV5MYZziYMurQlVwgumGLz85Y6qTHZU2D5qkMH7o5C5bzptReO96889oxkReS_LDhzdxH0tAQ5a9GgETJTHAXJ9Lt7X037i0qQlatbPLizOl9fo4vUBGJ7w-uSUmIzg1yIwOo8xhLu39WghXfoJ_EX5ptNi7arpAo11UoJIGEgWgq7bS40q2dtnG5HYpp9ef37iFgztM7PV5dlJf_W-n9uXJyCk2nhxZeuGiIGhMTzjP_8oeyiMFelmoyLlqK8wOs6hvWqgDD4NwtZc-XBkEAKsxuPrirb_sszp0z6BWsh3D6fBxAW_lTrNVkxZ1EBtoEN72HvO1aeX8PVuJP3CKwkkUcAhJ5Zvajo8t4Ai8ave2MTZul4QhDIe7ZALJMo71u-hNSj8YG9JLPqF1jcnbkl8ykt_XMMxX8wSZ_AfTsfmAuOWBfEzxv8Fl2Ne-oxcEtfuYnYNJIjQDHXXdtYbWKBezE41wonQu7FWh3wSSF3DDm9EocWMsUZ610nrIviOFdtTqjrY7KnOZp0hS9Hm2V04z-06Db-To71L5O5poYKuOlDAP_ULfW0hgMgYaPnt4Nive9gtue6ZQ5yNFylKM11zCWcF8vmuEPGx4NIZjvNFB-QSkdxZtBTQOgO6r4BqHJNZIDqiD9A6PyMemVtQEo-JhooX6bwHFHx6C84r-kKq1lg0RaDelbu1h6347kocxOczdgHWlagKPFrUvHpPHvCZeFmzLLmPWbDEpUhhfkbEZlNe47svO5WN1TYZMCAlE0g01pe6b_NB9-twQ4FJB1SkUQRA4o5DxiIHSgXBBfw1X98XPEKcJgCU2zRbYuU69mvyPcsxoPsLo8Cl2OAXj2U1tcLUxVhR_vu3I0SG_Ee8hxBp5b6QbpIu2sE0bgpqfi88XqUXf3qvxGbkoZDZ8dPvEvCVmpDZOOGJY5Db4MnpKBByW79lUnpbLCbsVOoxYu8SVmVQ63eGY9yBlACuWxlXA_PhCpnKYwMppjxKvKqalNJGMMg5WQ8cvhsIftE5wgaj21n0A15GwRO1WgUcsO_dLe1MqO88hTHWHV0Q3gNut3sdsmVr_n-0SniLtVAaw4IEtDZWNUfhpaoC7iyKxoJ86t8r697EYPlWUu4nTQhb3TOXAjHF22rFsJJMldKHVxjxjd086ky3OY3L_7cbjVof60-y4gTJdumw5hch-mhaT93nVnaTuPLGCAC48dZSgOitW8Dw62mPS05fTPAfKUY1HpyIWr9ACk7U2KVAijpzFB-bXvqJeCyBeUHr4cB61K2XxTmkNA8Oo9RC32TeYDrThFzoUyYZ_D6zPe1lLtQaFsPRvDBSsYCcr5fcABlST4DyUgnkgqtx-0-D1QEwtWiH3xiHLQGa82ldjqNIXL9LqwxYoCRBKuljHyqEylZD26A9Jp6SL9rk_Ol6TJlYa3zx44PHyMoA3Coy-HIZW5NMwVKdV7egUDDOchvuFnc1Zv2QpSFKc0JY8lYFtSm1HZFwjq7hf2hBqkU0Xneh8AT4Ty9QVb4mJ8rpzc8m_NBZNrpZVE-pIzmv2yGjnRwKSrwsjat7b2iY34UPFqGBlDk5FX6EfRlN9bq-nztWdextLc47ndr1gxsjC-vr6tbdXl5FxMVhbsnLH80wOC6hHubxCysr5C7Ft59s3MLhVQCc3IIOPMBhLOjjjVsu6K8kAJvvamIpj-iYUQj9T8JztAqfobytzbiwPsQ3mywUGkJpZxTjXjqkKdT6yfFdW5ksR8p3VlKGAAKft9TKKcc5IgErmIRtiZF4hGhB45nwIO5XHA59V94QuV3b0oJrleuDCBj0iDFqWYpsUVHyKl3RjrO86Yq1IbNRwXIIMrlsVYos9CS1I75k9kW4yQC6ni3a2ZitDkiEB-SoQuTlBsvfTs892Wt5q3CWH6_7srZO6l2QEAlWqyQp_P4i8j8uiEbZksGsgghVIgOkriblDP9ml7PdkpRX7S3OpY7jjQgTNR8XHaZ-SEw0-zPiqIIt5VioFK8JljPn-nMIDx-ew_vTDKqmKEOF39ezQcc5oQNTQNUvTAe8qCbK4ghS3fagItezoHtB5zm3qwhx9Iu6JMJy0UhywUZBaInY6NokiVZGDgtoqlVjOk7K4299MVZgsOr2P9QHRDo41lYnCeWWFUf50d-ouEOUO_ngteDpgG1DfV64eycrUOWAnpfI8XOUZ7mxS89SqJriw3T3tcljowHiOFw9QjYsjJi0vuHL5jfoBZVdYAUH6KiLfIScO-KgqitdUTpFwks8Qk7xfd2HcEfArlzSiAt3Hmb2Ty-5mhvtQj2RbPehgoI7gqa60b6Qffth-5tbIjXoKlZCQsaKVMAggF9C3w0Rv5uJ3ExCTGsfKO9IVOf--AfxaA8cNoO0W3NTaMH2iWftY0O1-1x3VJEcerp2v9BcZ0iIL3VFJ9eyOPYNHNwTYrI3S6OrVZxuQXFbu2E_Us32wxYqXvnMx-DEOk7bUUFIanL69Cbn_mR_9Wm1sFixBQ15diJoe1Oyl3XWHWL6_YF4XnHzZlMWySt3Awdeynwu99Ter8WMGXl-6K30novKRWC9jDVOfAtxPvXBkkRtpm8nnJT8fwkmIQoWOEPT2MCg3VYzzHzVEH-AueuRiCZj_o7L-fRqbtFrWPyVnGuetUYS5u0D3n7Rr6kCVqWhLfwjqAwV_calzIQs8-J0-AJe6GGaaONPYfXwXjYan0Sl11CS64lXvx7i3MgeOZuH210T6g6hj54dSLw0upO1L8k0FGli3I5OhLMekeGMLT_N2akyHktjbz8Odpk2ms0pJP_PdqvUegjXgH81F5lr40HAXZs7jfC9ytgcWaEjwlmnoFHSCTdlCIgJqEBzItXiz6ilk5zP35VwxdQTNoms7Meqg-ZrgRuLRaqXNtpQT9qWBmiFP6zoYLwymqytZ_P1SV9DCM-KIZQ2Cq-ew8PsYSrwl8GmfuegoKEMBiON1I4mC9ChDgzJmbo5P9te60T0URS5g9ExiQuSo-JXs0MKC57LLiNo3MPZvqR1wRJOTg0_FqS4JFxzPwN5OqhBzfs25f6kK-axf0wWp-LetUx3U1CTNaxfbCTK5tb8KnLOIs3lDwJw5H5wIIRgXnfNLBJT8A2srfXkdpA", - "eventCounters" => [], - "jsType" => "ch", - "cid" => "3i4vPGHCl74uIBumWaxF2a0PjFkyCJxcSvFnn~heX7keA6sGHgRbnzSYcMr_RzJFlfeeBdg9HeufzFPxnvSHbyWowSvjm6mNCSPhkx6MwDqptANoj93ZgGVbrzxGtcYe", - "ddk" => "78B13B7513D180B7AB6D6FF9EB0A51", - "Referer" => "https://www.qwant.com/?q={$search}&t=web", - "request" => "/?q={$search}&t=web", - "responsePage" => "origin", - "ddv" => "5.8.0" - ], - true - ); - }catch(Exception $error){ - - throw new Exception("Failed to fetch JS challenge token"); - } - - $datadome = json_decode($datadome, true); - if($datadome === null){ - - throw new Exception("Failed to decode JS challenge JSON"); - } - - if(!isset($datadome["cookie"])){ - - throw new Exception("Failed to get cookie from JS challenge endpoint"); - } - - $cookie = explode(";", $datadome["cookie"])[0]; - } + $cookie = $this->fuck_datadome($proxy, $search); try{ $json = @@ -679,6 +617,9 @@ class qwant{ ); $params = json_decode($params, true); + $cookie = $params["cookie"]; + unset($params["cookie"]); + }else{ $search = $get["s"]; @@ -719,13 +660,19 @@ class qwant{ case "maybe": $params["safesearch"] = 1; break; case "no": $params["safesearch"] = 2; break; } + + $cookie = null; } + $cookie = $this->fuck_datadome($proxy, $search); + try{ $json = $this->get( $proxy, "https://api.qwant.com/v3/search/images", $params, + false, + $cookie ); }catch(Exception $err){ @@ -1103,4 +1050,53 @@ class qwant{ return "https://" . $image["host"] . "/th?id=" . rawurlencode($id); } + + private function fuck_datadome($proxy, $search){ + + if($cookie === null){ + // get datadome bullshit + // yeah go ahead and patch this you code monkey + + // ... you know, instead of fixing this shit, can you make the second page of results return more than 1 result? + // i search shit like "higurashi" and i only get 1 result and then just a bunch of related searches ad finitum + // also, stop pretending you guys have your own index, this is clearly all bing results + try{ + $datadome = + $this->get( + $proxy, + "https://dd.qwant.com/js/", + [ + "jspl" => "-oirHDeP7G3G9LXY3rhxGDYf6t_ZlYPzaiL2dRhQ34uNUc4dO2Td0SmmKu_VJp7-jrAIoOMrTqKgCtV4cPzntMyNuOYN7QQ1q0Fi2c-679vYpJr9ub5dRFXMhCMQmjZsOWXUa8yafIXf85UA5Goct-n4hVT5dFfBGM-dE6gwZEFkHQ1j1195sLybZFTNA1IfX26MBMxlVHoyT-aHtkd6YjGXbvOPMMsObVHDa_Cf4BmApRGo-eehsAttsCqDRwkl0M2E1b0OXCIwMVnCK-qWZ81MGaJGjqxdj_LaxhAjE9JI_4oNedAg4Yj4m_vkQBKdmWJT_bTUsllAZXqX5IxYQHJ8WvlknOTr6sR3wgoSOa_JNVHkUQxqXBbQcoDdy6VciL5rIqwWl4Y2caVYzeXa8YNXxwsfrPDegH3dQ6vOiIiCDT3bexkcS_y7GAioPPv9qDu7M9y4sI8t0VkPgMbWY6MIu5u9WAv6lEiKIYrtlyU-iiXnNdkd9EnpC1TD8_s12PtCuLg5bGgLVmI1dw8VXVZxmVyFvfBtuqXkzM-kZOyv-YIRcf-wPY8DUtnHox1d6t64oYhGv-pYV5dmZ0rGs99LC6ThiifeF_oHz5ZK46Ms68WsiAjc1HLAbFPFxWw8X6lyLlpT5aHgFdDBeCTExKXyItlc9PBxkgK8UnXHQFiWX3HTen8rziwkI31AbkMU7xGP6cvuwuLG7MA7sH3yLF1uimQlhr6EuTrImWQNV7X81DqyZ0OE7I-uT9bp_xISa5it-6AJpFtGcwz04_OuO2zvZazBeiKJI5Fw0X2r3RHTHdA29SXqTIe4y2LRAAp0MSAZUIJAMIbbrz9BG-3D5BHfQUA1Bbuklxv-tNWiuanWbbTmAiuxTAo9P-D2ZMrWHwdN8sCb3nINLur38z1xviSs2qEEjKo9kzQIWoHgwm_amV-jjDtNBXwpCQnoaqt3FfL8e59ecYWVg61Gnl3YCwq6rubFdkIv6UStcWSlj4pRFlFZSjQxzznvGTAc5Nz1JJWVeBHJItei4wqDUYEw2BNTS_fU9keV3LFyKDH_x248AxHLiXt583DhZZwPAKlpmMo0E7odzfOgVQGM5b9l64NL3RQRq8cDxI8KgmOFnPHdI7Myq1A3nW1uoBB8m9_GR_9orboKl5tAQrT0VTMRd6JyNwOOQ71ZoQpMwsD5Xd9YDZd618TNE9dA38h9OyBco_-hwKCYayyEqib1atLLBO_gnO4InR471AklOokDOKx1oY_xgNhlFrZskE4nQQOqeCQ_47oysob9s3kblA8UqDrL2vM3i2KPLWwlQzMAPrQAIv7Amp6rO-wYwBQdbP7XghYympxx9-hNxc1Duq-nS4R3pjPtZg3LoCvG1N_0JLf9jfPBqw_rdPM6DnjEOvpCdvrfv4wBlEpgqbaRQCZBPEz-O5ZlY24OyPvaNX0krPeXkbi-KYuwZSAbfW0zsksXb8cbRmz6tvh25IRYP1XNDPVd8NFrYOdDH9GzhV5MYZziYMurQlVwgumGLz85Y6qTHZU2D5qkMH7o5C5bzptReO96889oxkReS_LDhzdxH0tAQ5a9GgETJTHAXJ9Lt7X037i0qQlatbPLizOl9fo4vUBGJ7w-uSUmIzg1yIwOo8xhLu39WghXfoJ_EX5ptNi7arpAo11UoJIGEgWgq7bS40q2dtnG5HYpp9ef37iFgztM7PV5dlJf_W-n9uXJyCk2nhxZeuGiIGhMTzjP_8oeyiMFelmoyLlqK8wOs6hvWqgDD4NwtZc-XBkEAKsxuPrirb_sszp0z6BWsh3D6fBxAW_lTrNVkxZ1EBtoEN72HvO1aeX8PVuJP3CKwkkUcAhJ5Zvajo8t4Ai8ave2MTZul4QhDIe7ZALJMo71u-hNSj8YG9JLPqF1jcnbkl8ykt_XMMxX8wSZ_AfTsfmAuOWBfEzxv8Fl2Ne-oxcEtfuYnYNJIjQDHXXdtYbWKBezE41wonQu7FWh3wSSF3DDm9EocWMsUZ610nrIviOFdtTqjrY7KnOZp0hS9Hm2V04z-06Db-To71L5O5poYKuOlDAP_ULfW0hgMgYaPnt4Nive9gtue6ZQ5yNFylKM11zCWcF8vmuEPGx4NIZjvNFB-QSkdxZtBTQOgO6r4BqHJNZIDqiD9A6PyMemVtQEo-JhooX6bwHFHx6C84r-kKq1lg0RaDelbu1h6347kocxOczdgHWlagKPFrUvHpPHvCZeFmzLLmPWbDEpUhhfkbEZlNe47svO5WN1TYZMCAlE0g01pe6b_NB9-twQ4FJB1SkUQRA4o5DxiIHSgXBBfw1X98XPEKcJgCU2zRbYuU69mvyPcsxoPsLo8Cl2OAXj2U1tcLUxVhR_vu3I0SG_Ee8hxBp5b6QbpIu2sE0bgpqfi88XqUXf3qvxGbkoZDZ8dPvEvCVmpDZOOGJY5Db4MnpKBByW79lUnpbLCbsVOoxYu8SVmVQ63eGY9yBlACuWxlXA_PhCpnKYwMppjxKvKqalNJGMMg5WQ8cvhsIftE5wgaj21n0A15GwRO1WgUcsO_dLe1MqO88hTHWHV0Q3gNut3sdsmVr_n-0SniLtVAaw4IEtDZWNUfhpaoC7iyKxoJ86t8r697EYPlWUu4nTQhb3TOXAjHF22rFsJJMldKHVxjxjd086ky3OY3L_7cbjVof60-y4gTJdumw5hch-mhaT93nVnaTuPLGCAC48dZSgOitW8Dw62mPS05fTPAfKUY1HpyIWr9ACk7U2KVAijpzFB-bXvqJeCyBeUHr4cB61K2XxTmkNA8Oo9RC32TeYDrThFzoUyYZ_D6zPe1lLtQaFsPRvDBSsYCcr5fcABlST4DyUgnkgqtx-0-D1QEwtWiH3xiHLQGa82ldjqNIXL9LqwxYoCRBKuljHyqEylZD26A9Jp6SL9rk_Ol6TJlYa3zx44PHyMoA3Coy-HIZW5NMwVKdV7egUDDOchvuFnc1Zv2QpSFKc0JY8lYFtSm1HZFwjq7hf2hBqkU0Xneh8AT4Ty9QVb4mJ8rpzc8m_NBZNrpZVE-pIzmv2yGjnRwKSrwsjat7b2iY34UPFqGBlDk5FX6EfRlN9bq-nztWdextLc47ndr1gxsjC-vr6tbdXl5FxMVhbsnLH80wOC6hHubxCysr5C7Ft59s3MLhVQCc3IIOPMBhLOjjjVsu6K8kAJvvamIpj-iYUQj9T8JztAqfobytzbiwPsQ3mywUGkJpZxTjXjqkKdT6yfFdW5ksR8p3VlKGAAKft9TKKcc5IgErmIRtiZF4hGhB45nwIO5XHA59V94QuV3b0oJrleuDCBj0iDFqWYpsUVHyKl3RjrO86Yq1IbNRwXIIMrlsVYos9CS1I75k9kW4yQC6ni3a2ZitDkiEB-SoQuTlBsvfTs892Wt5q3CWH6_7srZO6l2QEAlWqyQp_P4i8j8uiEbZksGsgghVIgOkriblDP9ml7PdkpRX7S3OpY7jjQgTNR8XHaZ-SEw0-zPiqIIt5VioFK8JljPn-nMIDx-ew_vTDKqmKEOF39ezQcc5oQNTQNUvTAe8qCbK4ghS3fagItezoHtB5zm3qwhx9Iu6JMJy0UhywUZBaInY6NokiVZGDgtoqlVjOk7K4299MVZgsOr2P9QHRDo41lYnCeWWFUf50d-ouEOUO_ngteDpgG1DfV64eycrUOWAnpfI8XOUZ7mxS89SqJriw3T3tcljowHiOFw9QjYsjJi0vuHL5jfoBZVdYAUH6KiLfIScO-KgqitdUTpFwks8Qk7xfd2HcEfArlzSiAt3Hmb2Ty-5mhvtQj2RbPehgoI7gqa60b6Qffth-5tbIjXoKlZCQsaKVMAggF9C3w0Rv5uJ3ExCTGsfKO9IVOf--AfxaA8cNoO0W3NTaMH2iWftY0O1-1x3VJEcerp2v9BcZ0iIL3VFJ9eyOPYNHNwTYrI3S6OrVZxuQXFbu2E_Us32wxYqXvnMx-DEOk7bUUFIanL69Cbn_mR_9Wm1sFixBQ15diJoe1Oyl3XWHWL6_YF4XnHzZlMWySt3Awdeynwu99Ter8WMGXl-6K30novKRWC9jDVOfAtxPvXBkkRtpm8nnJT8fwkmIQoWOEPT2MCg3VYzzHzVEH-AueuRiCZj_o7L-fRqbtFrWPyVnGuetUYS5u0D3n7Rr6kCVqWhLfwjqAwV_calzIQs8-J0-AJe6GGaaONPYfXwXjYan0Sl11CS64lXvx7i3MgeOZuH210T6g6hj54dSLw0upO1L8k0FGli3I5OhLMekeGMLT_N2akyHktjbz8Odpk2ms0pJP_PdqvUegjXgH81F5lr40HAXZs7jfC9ytgcWaEjwlmnoFHSCTdlCIgJqEBzItXiz6ilk5zP35VwxdQTNoms7Meqg-ZrgRuLRaqXNtpQT9qWBmiFP6zoYLwymqytZ_P1SV9DCM-KIZQ2Cq-ew8PsYSrwl8GmfuegoKEMBiON1I4mC9ChDgzJmbo5P9te60T0URS5g9ExiQuSo-JXs0MKC57LLiNo3MPZvqR1wRJOTg0_FqS4JFxzPwN5OqhBzfs25f6kK-axf0wWp-LetUx3U1CTNaxfbCTK5tb8KnLOIs3lDwJw5H5wIIRgXnfNLBJT8A2srfXkdpA", + "eventCounters" => [], + "jsType" => "ch", + "cid" => "3i4vPGHCl74uIBumWaxF2a0PjFkyCJxcSvFnn~heX7keA6sGHgRbnzSYcMr_RzJFlfeeBdg9HeufzFPxnvSHbyWowSvjm6mNCSPhkx6MwDqptANoj93ZgGVbrzxGtcYe", + "ddk" => "78B13B7513D180B7AB6D6FF9EB0A51", + "Referer" => "https://www.qwant.com/?q={$search}&t=web", + "request" => "/?q={$search}&t=web", + "responsePage" => "origin", + "ddv" => "5.8.0" + ], + true + ); + }catch(Exception $error){ + + throw new Exception("Failed to fetch JS challenge token"); + } + + $datadome = json_decode($datadome, true); + if($datadome === null){ + + throw new Exception("Failed to decode JS challenge JSON"); + } + + if(!isset($datadome["cookie"])){ + + throw new Exception("Failed to get cookie from JS challenge endpoint"); + } + + $cookie = explode(";", $datadome["cookie"])[0]; + } + + return $cookie; + } }