diff --git a/config.php.example b/config.php.example index 567e839..bd95e80 100644 --- a/config.php.example +++ b/config.php.example @@ -23,6 +23,9 @@ // how long in minutes to put google/other instances on cooldown if they aren't responding "request_cooldown" => 25, + // how long in minutes to store results for in the cache + "cache_time" => 20, + /* Preset privacy friendly frontends for users, these can be overwritten by users in the settings e.g.: Preset the invidious instance URL: "instance_url" => "https://yewtu.be", diff --git a/engines/ahmia/hidden_service.php b/engines/ahmia/hidden_service.php index f950ba8..5ef9e39 100644 --- a/engines/ahmia/hidden_service.php +++ b/engines/ahmia/hidden_service.php @@ -6,8 +6,7 @@ return "https://ahmia.fi/search/?q=" . urlencode($this->query); } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $results = array(); $xpath = get_xpath($response); diff --git a/engines/bittorrent/1337x.php b/engines/bittorrent/1337x.php index f5390ec..fab7615 100644 --- a/engines/bittorrent/1337x.php +++ b/engines/bittorrent/1337x.php @@ -5,9 +5,7 @@ return "https://1337x.to/search/$query/1/"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); - + public function parse_results($response) { $xpath = get_xpath($response); $results = array(); diff --git a/engines/bittorrent/merge.php b/engines/bittorrent/merge.php index 31dcc24..7d5caae 100644 --- a/engines/bittorrent/merge.php +++ b/engines/bittorrent/merge.php @@ -21,7 +21,7 @@ ); } - public function get_results() { + public function parse_results($response) { $results = array(); foreach ($this->requests as $request) { if ($request->successful()) diff --git a/engines/bittorrent/nyaa.php b/engines/bittorrent/nyaa.php index d3cc22a..9d7b021 100644 --- a/engines/bittorrent/nyaa.php +++ b/engines/bittorrent/nyaa.php @@ -6,8 +6,7 @@ return "https://$this->SOURCE/?q=" . urlencode($this->query); } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $xpath = get_xpath($response); $results = array(); diff --git a/engines/bittorrent/rutor.php b/engines/bittorrent/rutor.php index e575bac..2ee863f 100644 --- a/engines/bittorrent/rutor.php +++ b/engines/bittorrent/rutor.php @@ -4,8 +4,7 @@ return "http://rutor.info/search/" . urlencode($this->query); } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $xpath = get_xpath($response); $results = array(); diff --git a/engines/bittorrent/thepiratebay.php b/engines/bittorrent/thepiratebay.php index 49f3f7a..b674274 100644 --- a/engines/bittorrent/thepiratebay.php +++ b/engines/bittorrent/thepiratebay.php @@ -4,8 +4,7 @@ return "https://apibay.org/q.php?q=" . urlencode($this->query); } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $results = array(); $json_response = json_decode($response, true); diff --git a/engines/bittorrent/torrentgalaxy.php b/engines/bittorrent/torrentgalaxy.php index 2f6c606..48a5222 100644 --- a/engines/bittorrent/torrentgalaxy.php +++ b/engines/bittorrent/torrentgalaxy.php @@ -5,8 +5,7 @@ return "https://torrentgalaxy.to/torrents.php?search=$query#results"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $xpath = get_xpath($response); $results = array(); diff --git a/engines/bittorrent/yts.php b/engines/bittorrent/yts.php index d146ba7..9f7b515 100644 --- a/engines/bittorrent/yts.php +++ b/engines/bittorrent/yts.php @@ -4,9 +4,8 @@ return "https://yts.mx/api/v2/list_movies.json?query_term=" . urlencode($this->query); } - public function get_results() { + public function parse_results($response) { $response = curl_multi_getcontent($this->ch); - global $config; $results = array(); $json_response = json_decode($response, true); diff --git a/engines/invidious/video.php b/engines/invidious/video.php index afd51b2..230bc19 100644 --- a/engines/invidious/video.php +++ b/engines/invidious/video.php @@ -6,9 +6,8 @@ return "$this->instance_url/api/v1/search?q=$query"; } - public function get_results() { + public function parse_results($response) { $results = array(); - $response = curl_multi_getcontent($this->ch); $json_response = json_decode($response, true); foreach ($json_response as $response) { diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 4ff13b0..d3b6c34 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -7,11 +7,10 @@ } public function get_request_url() { - return $this->instance . "api.php?" . opts_to_params($this->opts); + return $this->instance . "api.php?" . opts_to_params($this->opts) . "&nfb=1"; } - public function get_results() { - $response = curl_exec($this->ch); + public function parse_results($response) { $response = json_decode($response, true); if (!$response) return array(); @@ -47,13 +46,17 @@ $instance = array_pop($instances); + if (!$instance) + break; + if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"]) continue; $librex_request = new LibreXFallback($instance, $opts, null); + $results = $librex_request->get_results(); - if (count($results) > 1) + if (!empty($results)) return $results; // on fail then do this @@ -62,7 +65,11 @@ } while (!empty($instances)); - return array(); + return array( + "error" => array( + "message" => "No results found. Unable to fallback to other instances." + ) + ); } ?> diff --git a/engines/qwant/image.php b/engines/qwant/image.php index 7459b18..9dd9db0 100644 --- a/engines/qwant/image.php +++ b/engines/qwant/image.php @@ -7,9 +7,9 @@ return "https://lite.qwant.com/?q=$query&t=images&p=$page"; } - public function get_results() { + public function parse_results($response) { $results = array(); - $xpath = get_xpath(curl_multi_getcontent($this->ch)); + $xpath = get_xpath($response); if (!$xpath) return $results; diff --git a/engines/special/currency.php b/engines/special/currency.php index ba53598..fcff1fa 100644 --- a/engines/special/currency.php +++ b/engines/special/currency.php @@ -4,9 +4,7 @@ return "https://cdn.moneyconvert.net/api/latest.json"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); - + public function parse_results($response) { $split_query = explode(" ", $this->query); $base_currency = strtoupper($split_query[1]); diff --git a/engines/special/definition.php b/engines/special/definition.php index b862b32..1ca5479 100644 --- a/engines/special/definition.php +++ b/engines/special/definition.php @@ -8,9 +8,7 @@ return "https://api.dictionaryapi.dev/api/v2/entries/en/$word_to_define"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); - + public function parse_results($response) { $json_response = json_decode($response, true); if (!array_key_exists("title", $json_response)) diff --git a/engines/special/ip.php b/engines/special/ip.php index 6fbb684..3f7ab12 100644 --- a/engines/special/ip.php +++ b/engines/special/ip.php @@ -1,6 +1,6 @@ array( "response" => $_SERVER["REMOTE_ADDR"], diff --git a/engines/special/tor.php b/engines/special/tor.php index d3172d0..b461532 100644 --- a/engines/special/tor.php +++ b/engines/special/tor.php @@ -5,9 +5,7 @@ return "https://check.torproject.org/torbulkexitlist"; } - public function get_results() { - $response = curl_multi_getcontent($ch); - + public function parse_results($response) { $formatted_response = strpos($response, $_SERVER["REMOTE_ADDR"]) ? "It seems like you are using Tor" : "It seems like you are not using Tor"; $source = "https://check.torproject.org"; diff --git a/engines/special/user_agent.php b/engines/special/user_agent.php index 64f5e8a..90503bc 100644 --- a/engines/special/user_agent.php +++ b/engines/special/user_agent.php @@ -1,6 +1,6 @@ array( "response" => $_SERVER["HTTP_USER_AGENT"], diff --git a/engines/special/weather.php b/engines/special/weather.php index 2d1d0fb..2980963 100644 --- a/engines/special/weather.php +++ b/engines/special/weather.php @@ -4,8 +4,7 @@ return "https://wttr.in/@" . $_SERVER["REMOTE_ADDR"] . "?format=j1"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); + public function parse_results($response) { $json_response = json_decode($response, true); if (!$json_response) diff --git a/engines/special/wikipedia.php b/engines/special/wikipedia.php index 436deb4..a5b2a1f 100644 --- a/engines/special/wikipedia.php +++ b/engines/special/wikipedia.php @@ -10,9 +10,7 @@ return "https://$this->wikipedia_language.wikipedia.org/w/api.php?format=json&action=query&prop=extracts%7Cpageimages&exintro&explaintext&redirects=1&pithumbsize=500&titles=$query_encoded"; } - public function get_results() { - $response = curl_multi_getcontent($this->ch); - + public function parse_results($response) { $json_response = json_decode($response, true); $first_page = array_values($json_response["query"]["pages"])[0]; diff --git a/engines/text/duckduckgo.php b/engines/text/duckduckgo.php index 9993f5d..5fa8264 100644 --- a/engines/text/duckduckgo.php +++ b/engines/text/duckduckgo.php @@ -19,12 +19,16 @@ if (isset($_COOKIE["safe_search"])) $url .= "&safe=medium"; + return $url; } - public function get_results() { + public function parse_results($response) { $results = array(); - $xpath = get_xpath(curl_multi_getcontent($this->ch)); + $xpath = get_xpath($response); + + if (!$xpath) + return $results; foreach($xpath->query("/html/body/div[1]/div[". count($xpath->query('/html/body/div[1]/div')) ."]/div/div/div[contains(@class, 'web-result')]/div") as $result) { @@ -57,7 +61,7 @@ ) ); } - return $results; + return $results; } } diff --git a/engines/text/google.php b/engines/text/google.php index 45c871d..eaff127 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -26,9 +26,9 @@ } - public function get_results() { + public function parse_results($response) { $results = array(); - $xpath = get_xpath(curl_multi_getcontent($this->ch)); + $xpath = get_xpath($response); if (!$xpath) return $results; @@ -70,6 +70,12 @@ ); } + if (empty($results) && !str_contains($response, "Our systems have detected unusual traffic from your computer network.")) { + $results["error"] = array( + "message" => "There are no results. Please try different keywords!" + ); + } + return $results; } } diff --git a/engines/text/text.php b/engines/text/text.php index 26af3ae..7d97df7 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -12,9 +12,6 @@ if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") check_ddg_bang($this->query, $opts); - if (has_cooldown($this->engine, $this->opts->cooldowns)) - return; - if ($this->engine == "google") { require "engines/text/google.php"; @@ -26,33 +23,49 @@ $this->engine_request = new DuckDuckGoRequest($opts, $mh); } + if (has_cooldown($this->engine, $this->opts->cooldowns) && !has_cached_results($this->engine_request->url)) { + // TODO dont add it in the first place + curl_multi_remove_handle($mh, $this->engine_request->ch); + $this->engine_request = null; + return; + } + + require "engines/special/special.php"; $this->special_request = get_special_search_request($opts, $mh); } - public function get_results() { - if (!$this->engine_request) + public function parse_results($response) { + if (!isset($this->engine_request)) return array(); $results = $this->engine_request->get_results(); - if ($this->special_request) { - $special_result = $this->special_request->get_results(); - - if ($special_result) - $results = array_merge(array($special_result), $results); - } - - if (count($results) <= 1) + if (empty($results)) { set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns); + } else { + if ($this->special_request) { + $special_result = $this->special_request->get_results(); + + if ($special_result) + $results = array_merge(array($special_result), $results); + } + } return $results; } public static function print_results($results) { - if (empty($results)) + if (empty($results)) { + echo "

An error occured fetching results

"; return; + } + + if (array_key_exists("error", $results)) { + echo "

" . $results["error"]["message"] . "

"; + return; + } $special = $results[0]; diff --git a/misc/cooldowns.php b/misc/cooldowns.php index 0285f47..ec55fb7 100644 --- a/misc/cooldowns.php +++ b/misc/cooldowns.php @@ -1,4 +1,8 @@ time(); } + + function has_cached_results($url) { + if (function_exists("apcu_exists")) + return apcu_exists("cached:$url"); + + return false; + } + + function store_cached_results($url, $results, $ttl = 0) { + if (function_exists("apcu_store") && !empty($results)) + return apcu_store("cached:$url", $results, $ttl); + } + + function fetch_cached_results($url) { + if (function_exists("apcu_fetch")) + return apcu_fetch("cached:$url"); + + return array(); + } ?> diff --git a/misc/search_engine.php b/misc/search_engine.php index b76cdcb..79398cf 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -1,15 +1,21 @@ query = $opts->query; $this->page = $opts->page; + $this->mh = $mh; $this->opts = $opts; - $url = $this->get_request_url(); - if (!$url) + $this->url = $this->get_request_url(); + if (!$this->url) return; - $this->ch = curl_init($url); + if (has_cached_results($this->url)) + return; + + $this->ch = curl_init($this->url); if ($opts->curl_settings) curl_setopt_array($this->ch, $opts->curl_settings); @@ -23,10 +29,31 @@ } public function successful() { - return curl_getinfo($this->ch)['http_code'] == '200'; + return (isset($this->ch) && curl_getinfo($this->ch)['http_code'] == '200') + || has_cached_results($this->url); + } + + abstract function parse_results($response); + + public function get_results() { + if (!isset($this->url)) + return $this->parse_results(null); + + if ($this->DO_CACHING && has_cached_results($this->url)) + return fetch_cached_results($this->url); + + if (!isset($this->ch)) + return $this->parse_results(null); + + $response = $this->mh ? curl_multi_getcontent($this->ch) : curl_exec($this->ch); + $results = $this->parse_results($response) ?? array(); + + if ($this->DO_CACHING && !empty($results)) + store_cached_results($this->url, $results, $this->opts->cache_time * 60); + + return $results; } - abstract function get_results(); static public function print_results($results){} } @@ -34,6 +61,7 @@ $opts = require "config.php"; $opts->request_cooldown ??= 25; + $opts->cache_time ??= 25; $opts->query = trim($_REQUEST["q"] ?? ""); $opts->type = (int) ($_REQUEST["t"] ?? 0); @@ -47,7 +75,7 @@ $opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]); - $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language)); + $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language ?? "en")); $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; if (!$opts->instance_fallback) { @@ -60,6 +88,8 @@ $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"]; } + $opts->curl_settings[CURLOPT_FOLLOWLOCATION] ??= true; + return $opts; } @@ -70,7 +100,6 @@ $params .= "p=$opts->page"; $params .= "&q=$query"; $params .= "&t=$opts->type"; - $params .= "&nfb=" . ($opts->do_fallback ? 0 : 1); $params .= "&safe=" . ($opts->safe_search ? 1 : 0); $params .= "&nf=" . ($opts->disable_frontends ? 1 : 0); $params .= "&ns=" . ($opts->disable_special ? 1 : 0); @@ -113,7 +142,6 @@ } function fetch_search_results($opts, $do_print) { - require "misc/cooldowns.php"; $opts->cooldowns = load_cooldowns(); $start_time = microtime(true); @@ -128,7 +156,7 @@ $results = $search_category->get_results(); - if (count($results) <= 1) { + if (empty($results)) { require "engines/librex/fallback.php"; $results = get_librex_results($opts); }