mirror of https://github.com/supernets/LibreY.git
feat: cache results to combat spammers (merge pull request #34 from davidovski/results_caching)
Results caching
This commit is contained in:
commit
dd8df71abd
|
@ -23,6 +23,9 @@
|
|||
// how long in minutes to put google/other instances on cooldown if they aren't responding
|
||||
"request_cooldown" => 25,
|
||||
|
||||
// how long in minutes to store results for in the cache
|
||||
"cache_time" => 20,
|
||||
|
||||
/*
|
||||
Preset privacy friendly frontends for users, these can be overwritten by users in the settings
|
||||
e.g.: Preset the invidious instance URL: "instance_url" => "https://yewtu.be",
|
||||
|
|
|
@ -6,8 +6,7 @@
|
|||
return "https://ahmia.fi/search/?q=" . urlencode($this->query);
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
public function parse_results($response) {
|
||||
$results = array();
|
||||
$xpath = get_xpath($response);
|
||||
|
||||
|
|
|
@ -5,9 +5,7 @@
|
|||
return "https://1337x.to/search/$query/1/";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
|
||||
public function parse_results($response) {
|
||||
$xpath = get_xpath($response);
|
||||
$results = array();
|
||||
|
||||
|
|
|
@ -21,7 +21,7 @@
|
|||
);
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
public function parse_results($response) {
|
||||
$results = array();
|
||||
foreach ($this->requests as $request) {
|
||||
if ($request->successful())
|
||||
|
|
|
@ -6,8 +6,7 @@
|
|||
return "https://$this->SOURCE/?q=" . urlencode($this->query);
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
public function parse_results($response) {
|
||||
$xpath = get_xpath($response);
|
||||
$results = array();
|
||||
|
||||
|
|
|
@ -4,8 +4,7 @@
|
|||
return "http://rutor.info/search/" . urlencode($this->query);
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
public function parse_results($response) {
|
||||
$xpath = get_xpath($response);
|
||||
$results = array();
|
||||
|
||||
|
|
|
@ -4,8 +4,7 @@
|
|||
return "https://apibay.org/q.php?q=" . urlencode($this->query);
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
public function parse_results($response) {
|
||||
$results = array();
|
||||
$json_response = json_decode($response, true);
|
||||
|
||||
|
|
|
@ -5,8 +5,7 @@
|
|||
return "https://torrentgalaxy.to/torrents.php?search=$query#results";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
public function parse_results($response) {
|
||||
$xpath = get_xpath($response);
|
||||
$results = array();
|
||||
|
||||
|
|
|
@ -4,9 +4,8 @@
|
|||
return "https://yts.mx/api/v2/list_movies.json?query_term=" . urlencode($this->query);
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
public function parse_results($response) {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
global $config;
|
||||
$results = array();
|
||||
$json_response = json_decode($response, true);
|
||||
|
||||
|
|
|
@ -6,9 +6,8 @@
|
|||
return "$this->instance_url/api/v1/search?q=$query";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
public function parse_results($response) {
|
||||
$results = array();
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
$json_response = json_decode($response, true);
|
||||
|
||||
foreach ($json_response as $response) {
|
||||
|
|
|
@ -7,11 +7,10 @@
|
|||
}
|
||||
|
||||
public function get_request_url() {
|
||||
return $this->instance . "api.php?" . opts_to_params($this->opts);
|
||||
return $this->instance . "api.php?" . opts_to_params($this->opts) . "&nfb=1";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_exec($this->ch);
|
||||
public function parse_results($response) {
|
||||
$response = json_decode($response, true);
|
||||
if (!$response)
|
||||
return array();
|
||||
|
@ -47,13 +46,17 @@
|
|||
|
||||
$instance = array_pop($instances);
|
||||
|
||||
if (!$instance)
|
||||
break;
|
||||
|
||||
if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"])
|
||||
continue;
|
||||
|
||||
$librex_request = new LibreXFallback($instance, $opts, null);
|
||||
|
||||
$results = $librex_request->get_results();
|
||||
|
||||
if (count($results) > 1)
|
||||
if (!empty($results))
|
||||
return $results;
|
||||
|
||||
// on fail then do this
|
||||
|
@ -62,7 +65,11 @@
|
|||
|
||||
} while (!empty($instances));
|
||||
|
||||
return array();
|
||||
return array(
|
||||
"error" => array(
|
||||
"message" => "No results found. Unable to fallback to other instances."
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
?>
|
||||
|
|
|
@ -7,9 +7,9 @@
|
|||
return "https://lite.qwant.com/?q=$query&t=images&p=$page";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
public function parse_results($response) {
|
||||
$results = array();
|
||||
$xpath = get_xpath(curl_multi_getcontent($this->ch));
|
||||
$xpath = get_xpath($response);
|
||||
|
||||
if (!$xpath)
|
||||
return $results;
|
||||
|
|
|
@ -4,9 +4,7 @@
|
|||
return "https://cdn.moneyconvert.net/api/latest.json";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
|
||||
public function parse_results($response) {
|
||||
$split_query = explode(" ", $this->query);
|
||||
|
||||
$base_currency = strtoupper($split_query[1]);
|
||||
|
|
|
@ -8,9 +8,7 @@
|
|||
return "https://api.dictionaryapi.dev/api/v2/entries/en/$word_to_define";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
|
||||
public function parse_results($response) {
|
||||
$json_response = json_decode($response, true);
|
||||
|
||||
if (!array_key_exists("title", $json_response))
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
<?php
|
||||
class IPRequest extends EngineRequest {
|
||||
function get_results() {
|
||||
public function parse_results($response) {
|
||||
return array(
|
||||
"special_response" => array(
|
||||
"response" => $_SERVER["REMOTE_ADDR"],
|
||||
|
|
|
@ -5,9 +5,7 @@
|
|||
return "https://check.torproject.org/torbulkexitlist";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($ch);
|
||||
|
||||
public function parse_results($response) {
|
||||
$formatted_response = strpos($response, $_SERVER["REMOTE_ADDR"]) ? "It seems like you are using Tor" : "It seems like you are not using Tor";
|
||||
$source = "https://check.torproject.org";
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
<?php
|
||||
class UserAgentRequest extends EngineRequest {
|
||||
function get_results() {
|
||||
public function parse_results($response) {
|
||||
return array(
|
||||
"special_response" => array(
|
||||
"response" => $_SERVER["HTTP_USER_AGENT"],
|
||||
|
|
|
@ -4,8 +4,7 @@
|
|||
return "https://wttr.in/@" . $_SERVER["REMOTE_ADDR"] . "?format=j1";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
public function parse_results($response) {
|
||||
$json_response = json_decode($response, true);
|
||||
|
||||
if (!$json_response)
|
||||
|
|
|
@ -10,9 +10,7 @@
|
|||
return "https://$this->wikipedia_language.wikipedia.org/w/api.php?format=json&action=query&prop=extracts%7Cpageimages&exintro&explaintext&redirects=1&pithumbsize=500&titles=$query_encoded";
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
$response = curl_multi_getcontent($this->ch);
|
||||
|
||||
public function parse_results($response) {
|
||||
$json_response = json_decode($response, true);
|
||||
|
||||
$first_page = array_values($json_response["query"]["pages"])[0];
|
||||
|
|
|
@ -19,12 +19,16 @@
|
|||
|
||||
if (isset($_COOKIE["safe_search"]))
|
||||
$url .= "&safe=medium";
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
public function parse_results($response) {
|
||||
$results = array();
|
||||
$xpath = get_xpath(curl_multi_getcontent($this->ch));
|
||||
$xpath = get_xpath($response);
|
||||
|
||||
if (!$xpath)
|
||||
return $results;
|
||||
|
||||
foreach($xpath->query("/html/body/div[1]/div[". count($xpath->query('/html/body/div[1]/div')) ."]/div/div/div[contains(@class, 'web-result')]/div") as $result)
|
||||
{
|
||||
|
@ -57,7 +61,7 @@
|
|||
)
|
||||
);
|
||||
}
|
||||
return $results;
|
||||
return $results;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -26,9 +26,9 @@
|
|||
}
|
||||
|
||||
|
||||
public function get_results() {
|
||||
public function parse_results($response) {
|
||||
$results = array();
|
||||
$xpath = get_xpath(curl_multi_getcontent($this->ch));
|
||||
$xpath = get_xpath($response);
|
||||
|
||||
if (!$xpath)
|
||||
return $results;
|
||||
|
@ -70,6 +70,12 @@
|
|||
);
|
||||
}
|
||||
|
||||
if (empty($results) && !str_contains($response, "Our systems have detected unusual traffic from your computer network.")) {
|
||||
$results["error"] = array(
|
||||
"message" => "There are no results. Please try different keywords!"
|
||||
);
|
||||
}
|
||||
|
||||
return $results;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -12,9 +12,6 @@
|
|||
if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!")
|
||||
check_ddg_bang($this->query, $opts);
|
||||
|
||||
if (has_cooldown($this->engine, $this->opts->cooldowns))
|
||||
return;
|
||||
|
||||
if ($this->engine == "google") {
|
||||
|
||||
require "engines/text/google.php";
|
||||
|
@ -26,33 +23,49 @@
|
|||
$this->engine_request = new DuckDuckGoRequest($opts, $mh);
|
||||
}
|
||||
|
||||
if (has_cooldown($this->engine, $this->opts->cooldowns) && !has_cached_results($this->engine_request->url)) {
|
||||
// TODO dont add it in the first place
|
||||
curl_multi_remove_handle($mh, $this->engine_request->ch);
|
||||
$this->engine_request = null;
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
require "engines/special/special.php";
|
||||
$this->special_request = get_special_search_request($opts, $mh);
|
||||
}
|
||||
|
||||
public function get_results() {
|
||||
if (!$this->engine_request)
|
||||
public function parse_results($response) {
|
||||
if (!isset($this->engine_request))
|
||||
return array();
|
||||
|
||||
$results = $this->engine_request->get_results();
|
||||
|
||||
if ($this->special_request) {
|
||||
$special_result = $this->special_request->get_results();
|
||||
|
||||
if ($special_result)
|
||||
$results = array_merge(array($special_result), $results);
|
||||
}
|
||||
|
||||
if (count($results) <= 1)
|
||||
if (empty($results)) {
|
||||
set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns);
|
||||
} else {
|
||||
if ($this->special_request) {
|
||||
$special_result = $this->special_request->get_results();
|
||||
|
||||
if ($special_result)
|
||||
$results = array_merge(array($special_result), $results);
|
||||
}
|
||||
}
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
public static function print_results($results) {
|
||||
|
||||
if (empty($results))
|
||||
if (empty($results)) {
|
||||
echo "<div class=\"text-result-container\"><p>An error occured fetching results</p></div>";
|
||||
return;
|
||||
}
|
||||
|
||||
if (array_key_exists("error", $results)) {
|
||||
echo "<div class=\"text-result-container\"><p>" . $results["error"]["message"] . "</p></div>";
|
||||
return;
|
||||
}
|
||||
|
||||
$special = $results[0];
|
||||
|
||||
|
|
|
@ -1,4 +1,8 @@
|
|||
<?php
|
||||
if (!function_exists("apcu_fetch"))
|
||||
error_log("apcu is not installed! Please consider installing php-pecl-apcu for significant performance improvements");
|
||||
|
||||
|
||||
function load_cooldowns() {
|
||||
if (function_exists("apcu_fetch"))
|
||||
return apcu_exists("cooldowns") ? apcu_fetch("cooldowns") : array();
|
||||
|
@ -19,4 +23,23 @@
|
|||
function has_cooldown($instance, $cooldowns) {
|
||||
return ($cooldowns[$instance] ?? 0) > time();
|
||||
}
|
||||
|
||||
function has_cached_results($url) {
|
||||
if (function_exists("apcu_exists"))
|
||||
return apcu_exists("cached:$url");
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function store_cached_results($url, $results, $ttl = 0) {
|
||||
if (function_exists("apcu_store") && !empty($results))
|
||||
return apcu_store("cached:$url", $results, $ttl);
|
||||
}
|
||||
|
||||
function fetch_cached_results($url) {
|
||||
if (function_exists("apcu_fetch"))
|
||||
return apcu_fetch("cached:$url");
|
||||
|
||||
return array();
|
||||
}
|
||||
?>
|
||||
|
|
|
@ -1,15 +1,21 @@
|
|||
<?php
|
||||
require "misc/cooldowns.php";
|
||||
abstract class EngineRequest {
|
||||
protected $DO_CACHING = true;
|
||||
function __construct($opts, $mh) {
|
||||
$this->query = $opts->query;
|
||||
$this->page = $opts->page;
|
||||
$this->mh = $mh;
|
||||
$this->opts = $opts;
|
||||
|
||||
$url = $this->get_request_url();
|
||||
if (!$url)
|
||||
$this->url = $this->get_request_url();
|
||||
if (!$this->url)
|
||||
return;
|
||||
|
||||
$this->ch = curl_init($url);
|
||||
if (has_cached_results($this->url))
|
||||
return;
|
||||
|
||||
$this->ch = curl_init($this->url);
|
||||
|
||||
if ($opts->curl_settings)
|
||||
curl_setopt_array($this->ch, $opts->curl_settings);
|
||||
|
@ -23,10 +29,31 @@
|
|||
}
|
||||
|
||||
public function successful() {
|
||||
return curl_getinfo($this->ch)['http_code'] == '200';
|
||||
return (isset($this->ch) && curl_getinfo($this->ch)['http_code'] == '200')
|
||||
|| has_cached_results($this->url);
|
||||
}
|
||||
|
||||
abstract function parse_results($response);
|
||||
|
||||
public function get_results() {
|
||||
if (!isset($this->url))
|
||||
return $this->parse_results(null);
|
||||
|
||||
if ($this->DO_CACHING && has_cached_results($this->url))
|
||||
return fetch_cached_results($this->url);
|
||||
|
||||
if (!isset($this->ch))
|
||||
return $this->parse_results(null);
|
||||
|
||||
$response = $this->mh ? curl_multi_getcontent($this->ch) : curl_exec($this->ch);
|
||||
$results = $this->parse_results($response) ?? array();
|
||||
|
||||
if ($this->DO_CACHING && !empty($results))
|
||||
store_cached_results($this->url, $results, $this->opts->cache_time * 60);
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
abstract function get_results();
|
||||
static public function print_results($results){}
|
||||
}
|
||||
|
||||
|
@ -34,6 +61,7 @@
|
|||
$opts = require "config.php";
|
||||
|
||||
$opts->request_cooldown ??= 25;
|
||||
$opts->cache_time ??= 25;
|
||||
|
||||
$opts->query = trim($_REQUEST["q"] ?? "");
|
||||
$opts->type = (int) ($_REQUEST["t"] ?? 0);
|
||||
|
@ -47,7 +75,7 @@
|
|||
|
||||
$opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]);
|
||||
|
||||
$opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language));
|
||||
$opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language ?? "en"));
|
||||
|
||||
$opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0;
|
||||
if (!$opts->instance_fallback) {
|
||||
|
@ -60,6 +88,8 @@
|
|||
$opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"];
|
||||
}
|
||||
|
||||
$opts->curl_settings[CURLOPT_FOLLOWLOCATION] ??= true;
|
||||
|
||||
return $opts;
|
||||
}
|
||||
|
||||
|
@ -70,7 +100,6 @@
|
|||
$params .= "p=$opts->page";
|
||||
$params .= "&q=$query";
|
||||
$params .= "&t=$opts->type";
|
||||
$params .= "&nfb=" . ($opts->do_fallback ? 0 : 1);
|
||||
$params .= "&safe=" . ($opts->safe_search ? 1 : 0);
|
||||
$params .= "&nf=" . ($opts->disable_frontends ? 1 : 0);
|
||||
$params .= "&ns=" . ($opts->disable_special ? 1 : 0);
|
||||
|
@ -113,7 +142,6 @@
|
|||
}
|
||||
|
||||
function fetch_search_results($opts, $do_print) {
|
||||
require "misc/cooldowns.php";
|
||||
$opts->cooldowns = load_cooldowns();
|
||||
|
||||
$start_time = microtime(true);
|
||||
|
@ -128,7 +156,7 @@
|
|||
|
||||
$results = $search_category->get_results();
|
||||
|
||||
if (count($results) <= 1) {
|
||||
if (empty($results)) {
|
||||
require "engines/librex/fallback.php";
|
||||
$results = get_librex_results($opts);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue