From a8c4f4e6090b0bec609d39d4fbb8614ddee2ecab Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 13:50:01 +0100 Subject: [PATCH 01/15] Use caching for all requests --- engines/ahmia/hidden_service.php | 3 +- engines/bittorrent/1337x.php | 4 +-- engines/bittorrent/merge.php | 2 +- engines/bittorrent/nyaa.php | 3 +- engines/bittorrent/rutor.php | 3 +- engines/bittorrent/thepiratebay.php | 3 +- engines/bittorrent/torrentgalaxy.php | 3 +- engines/bittorrent/yts.php | 3 +- engines/invidious/video.php | 3 +- engines/librex/fallback.php | 4 +-- engines/qwant/image.php | 4 +-- engines/special/currency.php | 4 +-- engines/special/definition.php | 4 +-- engines/special/ip.php | 2 +- engines/special/tor.php | 4 +-- engines/special/user_agent.php | 2 +- engines/special/weather.php | 3 +- engines/special/wikipedia.php | 4 +-- engines/text/duckduckgo.php | 4 +-- engines/text/google.php | 4 +-- engines/text/text.php | 2 +- misc/cooldowns.php | 23 ++++++++++++++++ misc/search_engine.php | 41 ++++++++++++++++++++++++---- 23 files changed, 83 insertions(+), 49 deletions(-) diff --git a/engines/ahmia/hidden_service.php b/engines/ahmia/hidden_service.php index f950ba8..5ef9e39 100644 --- a/engines/ahmia/hidden_service.php +++ b/engines/ahmia/hidden_service.php @@ -6,8 +6,7 @@ return "https://ahmia.fi/search/?q=" . urlencode($this->query); } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $results = array(); $xpath = get_xpath($response); diff --git a/engines/bittorrent/1337x.php b/engines/bittorrent/1337x.php index f5390ec..fab7615 100644 --- a/engines/bittorrent/1337x.php +++ b/engines/bittorrent/1337x.php @@ -5,9 +5,7 @@ return "https://1337x.to/search/$query/1/"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); - + public function parse_results($response) { $xpath = get_xpath($response); $results = array(); diff --git a/engines/bittorrent/merge.php b/engines/bittorrent/merge.php index 31dcc24..7d5caae 100644 --- a/engines/bittorrent/merge.php +++ b/engines/bittorrent/merge.php @@ -21,7 +21,7 @@ ); } - public function get_results() { + public function parse_results($response) { $results = array(); foreach ($this->requests as $request) { if ($request->successful()) diff --git a/engines/bittorrent/nyaa.php b/engines/bittorrent/nyaa.php index d3cc22a..9d7b021 100644 --- a/engines/bittorrent/nyaa.php +++ b/engines/bittorrent/nyaa.php @@ -6,8 +6,7 @@ return "https://$this->SOURCE/?q=" . urlencode($this->query); } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $xpath = get_xpath($response); $results = array(); diff --git a/engines/bittorrent/rutor.php b/engines/bittorrent/rutor.php index e575bac..2ee863f 100644 --- a/engines/bittorrent/rutor.php +++ b/engines/bittorrent/rutor.php @@ -4,8 +4,7 @@ return "http://rutor.info/search/" . urlencode($this->query); } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $xpath = get_xpath($response); $results = array(); diff --git a/engines/bittorrent/thepiratebay.php b/engines/bittorrent/thepiratebay.php index 49f3f7a..b674274 100644 --- a/engines/bittorrent/thepiratebay.php +++ b/engines/bittorrent/thepiratebay.php @@ -4,8 +4,7 @@ return "https://apibay.org/q.php?q=" . urlencode($this->query); } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $results = array(); $json_response = json_decode($response, true); diff --git a/engines/bittorrent/torrentgalaxy.php b/engines/bittorrent/torrentgalaxy.php index 2f6c606..48a5222 100644 --- a/engines/bittorrent/torrentgalaxy.php +++ b/engines/bittorrent/torrentgalaxy.php @@ -5,8 +5,7 @@ return "https://torrentgalaxy.to/torrents.php?search=$query#results"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $xpath = get_xpath($response); $results = array(); diff --git a/engines/bittorrent/yts.php b/engines/bittorrent/yts.php index d146ba7..9f7b515 100644 --- a/engines/bittorrent/yts.php +++ b/engines/bittorrent/yts.php @@ -4,9 +4,8 @@ return "https://yts.mx/api/v2/list_movies.json?query_term=" . urlencode($this->query); } - public function get_results() { + public function parse_results($response) { $response = curl_multi_getcontent($this->ch); - global $config; $results = array(); $json_response = json_decode($response, true); diff --git a/engines/invidious/video.php b/engines/invidious/video.php index afd51b2..230bc19 100644 --- a/engines/invidious/video.php +++ b/engines/invidious/video.php @@ -6,9 +6,8 @@ return "$this->instance_url/api/v1/search?q=$query"; } - public function get_results() { + public function parse_results($response) { $results = array(); - $response = curl_multi_getcontent($this->ch); $json_response = json_decode($response, true); foreach ($json_response as $response) { diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 4ff13b0..ed5a9a8 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -1,6 +1,7 @@ instance = $instance; parent::__construct($opts, $mh); @@ -10,8 +11,7 @@ return $this->instance . "api.php?" . opts_to_params($this->opts); } - public function get_results() { - $response = curl_exec($this->ch); + public function parse_results($response) { $response = json_decode($response, true); if (!$response) return array(); diff --git a/engines/qwant/image.php b/engines/qwant/image.php index 7459b18..9dd9db0 100644 --- a/engines/qwant/image.php +++ b/engines/qwant/image.php @@ -7,9 +7,9 @@ return "https://lite.qwant.com/?q=$query&t=images&p=$page"; } - public function get_results() { + public function parse_results($response) { $results = array(); - $xpath = get_xpath(curl_multi_getcontent($this->ch)); + $xpath = get_xpath($response); if (!$xpath) return $results; diff --git a/engines/special/currency.php b/engines/special/currency.php index ba53598..fcff1fa 100644 --- a/engines/special/currency.php +++ b/engines/special/currency.php @@ -4,9 +4,7 @@ return "https://cdn.moneyconvert.net/api/latest.json"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); - + public function parse_results($response) { $split_query = explode(" ", $this->query); $base_currency = strtoupper($split_query[1]); diff --git a/engines/special/definition.php b/engines/special/definition.php index b862b32..1ca5479 100644 --- a/engines/special/definition.php +++ b/engines/special/definition.php @@ -8,9 +8,7 @@ return "https://api.dictionaryapi.dev/api/v2/entries/en/$word_to_define"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); - + public function parse_results($response) { $json_response = json_decode($response, true); if (!array_key_exists("title", $json_response)) diff --git a/engines/special/ip.php b/engines/special/ip.php index 6fbb684..3f7ab12 100644 --- a/engines/special/ip.php +++ b/engines/special/ip.php @@ -1,6 +1,6 @@ array( "response" => $_SERVER["REMOTE_ADDR"], diff --git a/engines/special/tor.php b/engines/special/tor.php index d3172d0..b461532 100644 --- a/engines/special/tor.php +++ b/engines/special/tor.php @@ -5,9 +5,7 @@ return "https://check.torproject.org/torbulkexitlist"; } - public function get_results() { - $response = curl_multi_getcontent($ch); - + public function parse_results($response) { $formatted_response = strpos($response, $_SERVER["REMOTE_ADDR"]) ? "It seems like you are using Tor" : "It seems like you are not using Tor"; $source = "https://check.torproject.org"; diff --git a/engines/special/user_agent.php b/engines/special/user_agent.php index 64f5e8a..90503bc 100644 --- a/engines/special/user_agent.php +++ b/engines/special/user_agent.php @@ -1,6 +1,6 @@ array( "response" => $_SERVER["HTTP_USER_AGENT"], diff --git a/engines/special/weather.php b/engines/special/weather.php index 2d1d0fb..2980963 100644 --- a/engines/special/weather.php +++ b/engines/special/weather.php @@ -4,8 +4,7 @@ return "https://wttr.in/@" . $_SERVER["REMOTE_ADDR"] . "?format=j1"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $json_response = json_decode($response, true); if (!$json_response) diff --git a/engines/special/wikipedia.php b/engines/special/wikipedia.php index 436deb4..a5b2a1f 100644 --- a/engines/special/wikipedia.php +++ b/engines/special/wikipedia.php @@ -10,9 +10,7 @@ return "https://$this->wikipedia_language.wikipedia.org/w/api.php?format=json&action=query&prop=extracts%7Cpageimages&exintro&explaintext&redirects=1&pithumbsize=500&titles=$query_encoded"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); - + public function parse_results($response) { $json_response = json_decode($response, true); $first_page = array_values($json_response["query"]["pages"])[0]; diff --git a/engines/text/duckduckgo.php b/engines/text/duckduckgo.php index 378ad4e..2ae46aa 100644 --- a/engines/text/duckduckgo.php +++ b/engines/text/duckduckgo.php @@ -22,9 +22,9 @@ return $url; } - public function get_results() { + public function parse_results($response) { $results = array(); - $xpath = get_xpath(curl_multi_getcontent($this->ch)); + $xpath = get_xpath(response); foreach($xpath->query("/html/body/div[1]/div[". count($xpath->query('/html/body/div[1]/div')) ."]/div/div/div/div") as $result) { diff --git a/engines/text/google.php b/engines/text/google.php index 45c871d..911f320 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -26,9 +26,9 @@ } - public function get_results() { + public function parse_results($response) { $results = array(); - $xpath = get_xpath(curl_multi_getcontent($this->ch)); + $xpath = get_xpath($response); if (!$xpath) return $results; diff --git a/engines/text/text.php b/engines/text/text.php index 26af3ae..7b134e9 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -30,7 +30,7 @@ $this->special_request = get_special_search_request($opts, $mh); } - public function get_results() { + public function parse_results($response) { if (!$this->engine_request) return array(); diff --git a/misc/cooldowns.php b/misc/cooldowns.php index 0285f47..78ba4fb 100644 --- a/misc/cooldowns.php +++ b/misc/cooldowns.php @@ -1,4 +1,8 @@ time(); } + + function has_cached_results($url) { + if (function_exists("apcu_exists")) + return apcu_exists("cached:$url"); + + return false; + } + + function store_cached_results($url, $results) { + if (function_exists("apcu_store") && !empty($results)) + return apcu_store("cached:$url", $results); + } + + function fetch_cached_results($url) { + if (function_exists("apcu_fetch")) + return apcu_fetch("cached:$url"); + + return array(); + } ?> diff --git a/misc/search_engine.php b/misc/search_engine.php index b76cdcb..cde1419 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -1,15 +1,20 @@ query = $opts->query; $this->page = $opts->page; $this->opts = $opts; - $url = $this->get_request_url(); - if (!$url) + $this->url = $this->get_request_url(); + if (!$this->url) return; - $this->ch = curl_init($url); + if (has_cached_results($this->url)) + return; + + $this->ch = curl_init($this->url); if ($opts->curl_settings) curl_setopt_array($this->ch, $opts->curl_settings); @@ -23,10 +28,35 @@ } public function successful() { - return curl_getinfo($this->ch)['http_code'] == '200'; + return (isset($this->ch) && curl_getinfo($this->ch)['http_code'] == '200') + || has_cached_results($this->url); + } + + abstract function parse_results($response); + + public function get_results() { + if (!isset($this->url)) + return $this->parse_results(null); + + if ($this->DO_CACHING && has_cached_results($this->url)) { + error_log("used cache for $this->url"); + return fetch_cached_results($this->url); + } + + if (!isset($this->ch)) + return $this->parse_results(null); + + $response = curl_multi_getcontent($this->ch); + $results = $this->parse_results($response) ?? array(); + + if ($this->DO_CACHING) { + store_cached_results($this->url, $results); + error_log("caching $this->url in cache"); + } + + return $results; } - abstract function get_results(); static public function print_results($results){} } @@ -113,7 +143,6 @@ } function fetch_search_results($opts, $do_print) { - require "misc/cooldowns.php"; $opts->cooldowns = load_cooldowns(); $start_time = microtime(true); From 4488200d225f0dc1ab05b256168079008c2636d5 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 13:58:47 +0100 Subject: [PATCH 02/15] Add ttl on caches --- config.php.example | 3 +++ misc/cooldowns.php | 4 ++-- misc/search_engine.php | 11 ++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/config.php.example b/config.php.example index 567e839..bd95e80 100644 --- a/config.php.example +++ b/config.php.example @@ -23,6 +23,9 @@ // how long in minutes to put google/other instances on cooldown if they aren't responding "request_cooldown" => 25, + // how long in minutes to store results for in the cache + "cache_time" => 20, + /* Preset privacy friendly frontends for users, these can be overwritten by users in the settings e.g.: Preset the invidious instance URL: "instance_url" => "https://yewtu.be", diff --git a/misc/cooldowns.php b/misc/cooldowns.php index 78ba4fb..ec55fb7 100644 --- a/misc/cooldowns.php +++ b/misc/cooldowns.php @@ -31,9 +31,9 @@ return false; } - function store_cached_results($url, $results) { + function store_cached_results($url, $results, $ttl = 0) { if (function_exists("apcu_store") && !empty($results)) - return apcu_store("cached:$url", $results); + return apcu_store("cached:$url", $results, $ttl); } function fetch_cached_results($url) { diff --git a/misc/search_engine.php b/misc/search_engine.php index cde1419..5851b38 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -38,10 +38,8 @@ if (!isset($this->url)) return $this->parse_results(null); - if ($this->DO_CACHING && has_cached_results($this->url)) { - error_log("used cache for $this->url"); + if ($this->DO_CACHING && has_cached_results($this->url)) return fetch_cached_results($this->url); - } if (!isset($this->ch)) return $this->parse_results(null); @@ -49,10 +47,8 @@ $response = curl_multi_getcontent($this->ch); $results = $this->parse_results($response) ?? array(); - if ($this->DO_CACHING) { - store_cached_results($this->url, $results); - error_log("caching $this->url in cache"); - } + if ($this->DO_CACHING) + store_cached_results($this->url, $results, $this->opts->cache_time * 60); return $results; } @@ -64,6 +60,7 @@ $opts = require "config.php"; $opts->request_cooldown ??= 25; + $opts->cache_time ??= 25; $opts->query = trim($_REQUEST["q"] ?? ""); $opts->type = (int) ($_REQUEST["t"] ?? 0); From 4da025948b3f7791aa27840efcea1667f0dcbf05 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 14:00:44 +0100 Subject: [PATCH 03/15] fix missing $ --- engines/text/duckduckgo.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/text/duckduckgo.php b/engines/text/duckduckgo.php index 2ae46aa..b672d7c 100644 --- a/engines/text/duckduckgo.php +++ b/engines/text/duckduckgo.php @@ -24,7 +24,7 @@ public function parse_results($response) { $results = array(); - $xpath = get_xpath(response); + $xpath = get_xpath($response); foreach($xpath->query("/html/body/div[1]/div[". count($xpath->query('/html/body/div[1]/div')) ."]/div/div/div/div") as $result) { From aaf111abf72fb20e9896dc25135b61bb6f99d595 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 16:05:31 +0100 Subject: [PATCH 04/15] Always do no fallback on fallback requests --- engines/librex/fallback.php | 4 ++-- engines/text/text.php | 2 +- misc/search_engine.php | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index ed5a9a8..0af3e2d 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -6,9 +6,9 @@ $this->instance = $instance; parent::__construct($opts, $mh); } - +) public function get_request_url() { - return $this->instance . "api.php?" . opts_to_params($this->opts); + return $this->instance . "api.php?" . opts_to_params($this->opts) . "&nfb=1"; } public function parse_results($response) { diff --git a/engines/text/text.php b/engines/text/text.php index 7b134e9..e818708 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -31,7 +31,7 @@ } public function parse_results($response) { - if (!$this->engine_request) + if (!isset($this->engine_request)) return array(); $results = $this->engine_request->get_results(); diff --git a/misc/search_engine.php b/misc/search_engine.php index 5851b38..87b39bd 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -97,7 +97,6 @@ $params .= "p=$opts->page"; $params .= "&q=$query"; $params .= "&t=$opts->type"; - $params .= "&nfb=" . ($opts->do_fallback ? 0 : 1); $params .= "&safe=" . ($opts->safe_search ? 1 : 0); $params .= "&nf=" . ($opts->disable_frontends ? 1 : 0); $params .= "&ns=" . ($opts->disable_special ? 1 : 0); From a00da6eb65749125e45444eed52e752f648741a3 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 16:07:35 +0100 Subject: [PATCH 05/15] Fix typo --- engines/librex/fallback.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 0af3e2d..d29f046 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -6,7 +6,7 @@ $this->instance = $instance; parent::__construct($opts, $mh); } -) + public function get_request_url() { return $this->instance . "api.php?" . opts_to_params($this->opts) . "&nfb=1"; } From 12f02198f9561a4fb48eea3f37227161da41210b Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 17:26:34 +0100 Subject: [PATCH 06/15] Add no results messages --- engines/text/google.php | 6 ++++++ engines/text/text.php | 26 +++++++++++++++++--------- misc/search_engine.php | 3 ++- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/engines/text/google.php b/engines/text/google.php index 911f320..eaff127 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -70,6 +70,12 @@ ); } + if (empty($results) && !str_contains($response, "Our systems have detected unusual traffic from your computer network.")) { + $results["error"] = array( + "message" => "There are no results. Please try different keywords!" + ); + } + return $results; } } diff --git a/engines/text/text.php b/engines/text/text.php index e818708..c43be94 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -36,23 +36,31 @@ $results = $this->engine_request->get_results(); - if ($this->special_request) { - $special_result = $this->special_request->get_results(); - - if ($special_result) - $results = array_merge(array($special_result), $results); - } - - if (count($results) <= 1) + if (empty(count($results))) { set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns); + } else { + if ($this->special_request) { + $special_result = $this->special_request->get_results(); + + if ($special_result) + $results = array_merge(array($special_result), $results); + } + } return $results; } public static function print_results($results) { - if (empty($results)) + if (empty($results)) { + echo "

An error occured fetching results

"; return; + } + + if (array_key_exists("error", $results)) { + echo "

" . $results["error"]["message"] . "

"; + return; + } $special = $results[0]; diff --git a/misc/search_engine.php b/misc/search_engine.php index 87b39bd..b203d03 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -152,8 +152,9 @@ } while ($running); $results = $search_category->get_results(); + error_log(print_r($results, true)); - if (count($results) <= 1) { + if (empty($results)) { require "engines/librex/fallback.php"; $results = get_librex_results($opts); } From a33f26a63805feb08bf785dafcd448a029fb3120 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 17:50:56 +0100 Subject: [PATCH 07/15] Add no instances message --- engines/librex/fallback.php | 6 +++++- misc/search_engine.php | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index d29f046..0be02cc 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -62,7 +62,11 @@ } while (!empty($instances)); - return array(); + return array( + "error" => array( + "message" => "No results found. Unable to fallback to other instances." + ) + ); } ?> diff --git a/misc/search_engine.php b/misc/search_engine.php index b203d03..6eb791a 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -152,7 +152,6 @@ } while ($running); $results = $search_category->get_results(); - error_log(print_r($results, true)); if (empty($results)) { require "engines/librex/fallback.php"; From ad747d51a123f1926de2b2f0cec7064023a31e04 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 18:11:17 +0100 Subject: [PATCH 08/15] Make fallback work when no results found --- engines/librex/fallback.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 0be02cc..27798ef 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -53,7 +53,7 @@ $librex_request = new LibreXFallback($instance, $opts, null); $results = $librex_request->get_results(); - if (count($results) > 1) + if (count($results) > 0) return $results; // on fail then do this From 9d431ae8c090d0e8e789f6d2758b46402f7f375c Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 20:50:23 +0100 Subject: [PATCH 09/15] Actually exec fallback requests --- engines/librex/fallback.php | 7 ++++++- misc/search_engine.php | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 27798ef..f9612ca 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -47,13 +47,18 @@ $instance = array_pop($instances); + if (!$instance) + break; + if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"]) continue; $librex_request = new LibreXFallback($instance, $opts, null); + error_log($librex_request->url); + $results = $librex_request->get_results(); - if (count($results) > 0) + if (!empty($results)) return $results; // on fail then do this diff --git a/misc/search_engine.php b/misc/search_engine.php index 6eb791a..d653c6c 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -5,6 +5,7 @@ function __construct($opts, $mh) { $this->query = $opts->query; $this->page = $opts->page; + $this->mh = $mh; $this->opts = $opts; $this->url = $this->get_request_url(); @@ -44,10 +45,10 @@ if (!isset($this->ch)) return $this->parse_results(null); - $response = curl_multi_getcontent($this->ch); + $response = $this->mh ? curl_multi_getcontent($this->ch) : curl_exec($this->ch); $results = $this->parse_results($response) ?? array(); - if ($this->DO_CACHING) + if ($this->DO_CACHING && !empty($results)) store_cached_results($this->url, $results, $this->opts->cache_time * 60); return $results; From 73c5b4b98e33f45eba61bb1f13a61efb716b65a8 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 21:00:00 +0100 Subject: [PATCH 10/15] fix not respecting cooldown --- engines/text/text.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/text/text.php b/engines/text/text.php index c43be94..75ab847 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -36,7 +36,7 @@ $results = $this->engine_request->get_results(); - if (empty(count($results))) { + if (empty($results)) { set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns); } else { if ($this->special_request) { From 01fe9830098a7eb72e67e02eecdebe9b06992b44 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 21:06:09 +0100 Subject: [PATCH 11/15] curl follow location by default --- misc/search_engine.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/search_engine.php b/misc/search_engine.php index d653c6c..9ee7d3b 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -88,6 +88,8 @@ $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"]; } + $opts->curl_settings[CURLOPT_FOLLOWLOCATION] ??= true; + return $opts; } From f8f30323eea53d12f25cd5f13c51cba481f9780c Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 21:17:30 +0100 Subject: [PATCH 12/15] do not avoid request if cached --- engines/librex/fallback.php | 4 +--- engines/text/text.php | 10 +++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index f9612ca..d3b6c34 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -1,7 +1,6 @@ instance = $instance; parent::__construct($opts, $mh); @@ -54,8 +53,7 @@ continue; $librex_request = new LibreXFallback($instance, $opts, null); - error_log($librex_request->url); - + $results = $librex_request->get_results(); if (!empty($results)) diff --git a/engines/text/text.php b/engines/text/text.php index 75ab847..2ea76cc 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -12,9 +12,6 @@ if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") check_ddg_bang($this->query, $opts); - if (has_cooldown($this->engine, $this->opts->cooldowns)) - return; - if ($this->engine == "google") { require "engines/text/google.php"; @@ -26,6 +23,13 @@ $this->engine_request = new DuckDuckGoRequest($opts, $mh); } + if (has_cooldown($this->engine, $this->opts->cooldowns) && !has_cached_results($this->engine_request->url)) { + // TODO dont add it in the first place + curl_multi_remove_handle($mh, $this->engine_request->ch); + return; + } + + require "engines/special/special.php"; $this->special_request = get_special_search_request($opts, $mh); } From bfe699d5ad1341db10eb378ed454c8cdd19ee7b8 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 21:40:30 +0100 Subject: [PATCH 13/15] Do not throw error if duckduck go query fails --- engines/text/duckduckgo.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/engines/text/duckduckgo.php b/engines/text/duckduckgo.php index b672d7c..ddee34c 100644 --- a/engines/text/duckduckgo.php +++ b/engines/text/duckduckgo.php @@ -25,6 +25,9 @@ public function parse_results($response) { $results = array(); $xpath = get_xpath($response); + + if (!$xpath) + return $results; foreach($xpath->query("/html/body/div[1]/div[". count($xpath->query('/html/body/div[1]/div')) ."]/div/div/div/div") as $result) { From 37f77e5e8d7a02534809e607e8e33dd5d6dca389 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 21:47:22 +0100 Subject: [PATCH 14/15] set default language --- engines/text/duckduckgo.php | 3 ++- misc/search_engine.php | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/engines/text/duckduckgo.php b/engines/text/duckduckgo.php index ddee34c..2c03833 100644 --- a/engines/text/duckduckgo.php +++ b/engines/text/duckduckgo.php @@ -19,6 +19,7 @@ if (isset($_COOKIE["safe_search"])) $url .= "&safe=medium"; + return $url; } @@ -60,7 +61,7 @@ ) ); } - return $results; + return $results; } } diff --git a/misc/search_engine.php b/misc/search_engine.php index 9ee7d3b..79398cf 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -75,7 +75,7 @@ $opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]); - $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language)); + $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language ?? "en")); $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; if (!$opts->instance_fallback) { From 4a98228a90d3159a80a4dc7d300af1415ce3b96c Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 30 Aug 2023 22:06:36 +0100 Subject: [PATCH 15/15] Properly remove google request if on cooldown --- engines/text/text.php | 1 + 1 file changed, 1 insertion(+) diff --git a/engines/text/text.php b/engines/text/text.php index 2ea76cc..7d97df7 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -26,6 +26,7 @@ if (has_cooldown($this->engine, $this->opts->cooldowns) && !has_cached_results($this->engine_request->url)) { // TODO dont add it in the first place curl_multi_remove_handle($mh, $this->engine_request->ch); + $this->engine_request = null; return; }