Commit 40bc1b99 authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch '864-optimize-request-times' into 'development'

Resolve "Optimize request times"

Closes #864

See merge request !1386
parents c502f47f 11147134
Pipeline #2706 passed with stage
in 1 minute and 6 seconds
......@@ -4,7 +4,10 @@ namespace App\Http\Controllers;
use App;
use App\MetaGer;
use Cache;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use View;
const TIP_SERVER = 'http://metager3.de:63825/tips.xml';
......@@ -19,13 +22,6 @@ class MetaGerSearch extends Controller
return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
}
/*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
$metager->parseFormData($request);
return $metager->createView();
}*/
#die($request->header('User-Agent'));
$time = microtime();
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
......@@ -39,19 +35,132 @@ class MetaGerSearch extends Controller
# auf Ergebnisse warten und die Ergebnisse laden
$metager->createSearchEngines($request);
$metager->startSearch();
$metager->waitForMainResults();
$metager->retrieveResults();
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
# Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll();
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults();
# Save the results in Redis
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
foreach ($metager->getResults() as $result) {
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
# Die Ausgabe erstellen:
return $metager->createView($quicktipResults);
}
public function loadMore(Request $request)
{
/**
* There are three forms of requests to the resultpage
* 1. Initial Request: Loads the fastest searchengines and sends them to the user
* 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
* 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
*/
if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
return $this->loadMoreJS($request);
}
}
private function loadMoreJS(Request $request)
{
# Create a MetaGer Instance with the supplied hash
$hash = $request->input('loadMore', '');
$metager = new MetaGer($hash);
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$result = [];
# Check if there should be more results
$stats = $redis->hgetall($metager->getRedisEngineResult() . "status");
$stats["startTime"] = floatval($stats["startTime"]);
$stats["engineCount"] = intval($stats["engineCount"]);
$stats["engineAnswered"] = intval($stats["engineAnswered"]);
$stats["engineDelivered"] = intval($stats["engineDelivered"]);
$result["finished"] = true;
$result["engineCount"] = $stats["engineCount"];
$result["engineAnswered"] = $stats["engineAnswered"];
$result["engineDelivered"] = $stats["engineDelivered"];
$result["timeWaiting"] = microtime(true) - $stats["startTime"];
# Check if we can abort
if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
$metager->parseFormData($request);
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
# Read which search engines are new
$newEngines = [];
while (($engine = $redis->lpop($metager->getRedisResultWaitingKey())) != null) {
$result["engineDelivered"]++;
$newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
}
$cache = Cache::get($hash);
if ($cache != null) {
$metager->setNext(unserialize($cache)["engines"]);
}
# Check if this request is not for page one
$metager->setEngines($request, $newEngines);
# Add the results already delivered to the user
$results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1);
foreach ($results as $index => $oldResult) {
$results[$index] = unserialize(base64_decode($oldResult));
$results[$index]->new = false;
}
$metager->setResults($results);
$metager->retrieveResults();
$metager->rankAll();
$metager->prepareResults();
$result["nextSearchLink"] = $metager->nextSearchLink();
$results = $metager->getResults();
foreach ($results as $index => $resultTmp) {
if ($resultTmp->new) {
if ($metager->getFokus() !== "bilder") {
$view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
$result["imagesearch"] = false;
} else {
$view = View::make('layouts.image_result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
$result["imagesearch"] = true;
}
}
}
# Save the results in Redis
$pipeline = $redis->pipeline();
$pipeline->hincrby($metager->getRedisEngineResult() . "status", "engineDelivered", sizeof($newEngines));
$pipeline->hset($metager->getRedisEngineResult() . "status", "nextSearchLink", $result["nextSearchLink"]);
foreach ($metager->getResults() as $resultTmp) {
$resultTmp->new = false;
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($resultTmp)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
}
return response()->json($result);
}
public function botProtection($redirect)
{
$hash = md5(date('YmdHi'));
......
......@@ -85,8 +85,7 @@ class Searcher implements ShouldQueue
$url = base64_decode($mission[1]); // The url to fetch
$timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected");
Redis::hset('search.' . $hashValue . ".results." . $this->name, "status", "connected");
$result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue);
......@@ -99,7 +98,7 @@ class Searcher implements ShouldQueue
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if (getenv("QUEUE_DRIVER") === "sync"
if (getenv("QUEUE_CONNECTION") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true) - $this->startTime) > $this->MAX_TIME) {
break;
......@@ -161,16 +160,24 @@ class Searcher implements ShouldQueue
// Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch);
$this->connectionInfo = curl_getinfo($this->ch);
return $result;
}
private function storeResult($result, $poptime, $hashValue)
{
Redis::hset('search.' . $hashValue, $this->name, $result);
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0");
$pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
Redis::expire('search.' . $hashValue, 60);
$pipeline->expire('search.' . $hashValue . ".results." . $this->name, env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
$pipeline->expire('search.' . $hashValue . ".ready", env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
$pipeline->expire('search.' . $hashValue . ".engines", env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
$this->lastTime = microtime(true);
}
......
This diff is collapsed.
......@@ -3,7 +3,6 @@
namespace App\Models\Quicktips;
use App\Jobs\Searcher;
use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Log;
......@@ -12,8 +11,8 @@ class Quicktips
{
use DispatchesJobs;
const QUICKTIP_URL = "http://localhost:63825/1.1/quicktips.xml";
const QUICKTIP_NAME = "quicktips";
const QUICKTIP_URL = "http://localhost:63825/1.1/quicktips.xml";
const QUICKTIP_NAME = "quicktips";
const CACHE_DURATION = 60;
private $hash;
......@@ -26,18 +25,17 @@ class Quicktips
public function startSearch($search, $locale, $max_time)
{
$url = self::QUICKTIP_URL . "?search=" . $this->normalize_search($search) . "&locale=" . $locale;
$hash = md5($url);
# TODO anders weitergeben
$this->hash = $hash;
$this->hash = md5($url);
# TODO cache wieder einbauen (eventuell)
if ( /*!Cache::has($hash)*/true) {
Redis::hset("search." . $hash, self::QUICKTIP_NAME, "waiting");
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$redis->hset("search." . $this->hash . ".results." . self::QUICKTIP_NAME, "status", "waiting");
// Queue this search
$mission = $hash . ";" . base64_encode($url) . ";" . $max_time;
$mission = $this->hash . ";" . base64_encode($url) . ";" . $max_time;
Redis::rpush(self::QUICKTIP_NAME . ".queue", $mission);
// Check the current status of Searchers for QUICKTIP_NAME
......@@ -85,14 +83,11 @@ class Quicktips
public function retrieveResults($hash)
{
$body = "";
#if (Cache::has($hash)) {
$body = Cache::get($hash);
#} elseif (Redis::hexists('search.' . $hash, self::QUICKTIP_NAME)) {
$body = Redis::hget('search.' . $hash, self::QUICKTIP_NAME);
Redis::hdel('search.' . $hash, self::QUICKTIP_NAME);
Cache::put($hash, $body, self::CACHE_DURATION);
#}
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$body = $redis->hget('search.' . $hash . ".results." . self::QUICKTIP_NAME, "response");
$redis->del('search.' . $hash . ".results." . self::QUICKTIP_NAME);
$redis->del('search.' . $hash . ".ready");
if ($body !== "") {
return $body;
} else {
......@@ -157,14 +152,14 @@ class Quicktips
$descr = $quicktip_xml->content->__toString();
// Details
$details = [];
$details = [];
$details_xpath = $quicktip_xml->xpath('mg:details');
if (sizeof($details_xpath) > 0) {
foreach ($details_xpath[0] as $detail_xml) {
$details_title = $detail_xml->title->__toString();
$details_link = $detail_xml->url->__toString();
$details_link = $detail_xml->url->__toString();
$details_descr = $detail_xml->text->__toString();
$details[] = new \App\Models\Quicktips\Quicktip_detail(
$details[] = new \App\Models\Quicktips\Quicktip_detail(
$details_title,
$details_link,
$details_descr
......
......@@ -27,6 +27,7 @@ class Result
public $strippedDomain; # Die Domain in Form "bar.de"
public $strippedLink; # Der Link in Form "foo.bar.de/test"
public $rank; # Das Ranking für das Ergebnis
public $new = true;
# Erstellt ein neues Ergebnis
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
......@@ -67,6 +68,7 @@ class Result
$this->price = isset($additionalInformation["price"]) ? $additionalInformation["price"] : 0;
$this->price_text = $this->price_to_text($this->price);
$this->additionalInformation = $additionalInformation;
$this->hash = spl_object_hash($this);
}
private function price_to_text($price)
......
......@@ -87,7 +87,7 @@ abstract class Searchengine
$this->getString = $this->generateGetString($q);
$this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name);
$this->resultHash = $metager->getHashCode();
$this->resultHash = $metager->getSearchUid();
$this->canCache = $metager->canCache();
}
......@@ -102,13 +102,15 @@ abstract class Searchengine
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
public function startSearch(\App\MetaGer $metager)
{
if ($this->canCache && Cache::has($this->hash)) {
$this->cached = true;
$this->retrieveResults($metager);
} else {
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
// We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
$redis->hset($metager->getRedisEngineResult() . $this->name, "status", "waiting");
$redis->expire($metager->getRedisEngineResult() . $this->name, env('REDIS_RESULT_CACHE_DURATION'));
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
......@@ -190,10 +192,6 @@ abstract class Searchengine
{
foreach ($this->results as $result) {
$result->rank($eingabe);
if (str_contains($this->engine->{"display-name"}, "Yahoo")) {
#die(var_dump($result));
}
}
}
......@@ -210,12 +208,12 @@ abstract class Searchengine
}
$body = "";
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
$body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);
Redis::hdel('search.' . $this->resultHash, $this->name);
} elseif ($redis->hexists($metager->getRedisEngineResult() . $this->name, "response")) {
$body = $redis->hget($metager->getRedisEngineResult() . $this->name, "response");
if ($this->canCache && $this->cacheDuration > 0) {
Cache::put($this->hash, $body, $this->cacheDuration);
}
......@@ -251,13 +249,7 @@ abstract class Searchengine
# Append the Query String
$getString .= "&" . $this->engine->{"query-parameter"} . "=" . $this->urlEncode($query);
/*
die(var_dump($getString));
# Affildata
if (strpos($getString, "<<AFFILDATA>>")) {
$getString = str_replace("<<AFFILDATA>>", $this->getOvertureAffilData($url), $getString);
}*/
return $getString;
}
......
#!/usr/bin/perl
use Lingua::Identify qw(:language_identification);
use JSON;
use warnings;
use strict;
binmode STDOUT, ":utf8";
binmode STDIN, ":utf8";
use utf8;
chomp(my $filename = <STDIN>);
# Lets open the given file:
open(my $fh, "<", $filename)
or die "Can't open < $filename: $!";
my $json = <$fh>;
close $fh;
# Decode the JSON String
my $data = JSON->new->utf8->decode($json);
# Wir durchlaufen den Hash:
foreach my $key (keys %{$data}){
$data->{$key} = langof($data->{$key});
}
$data = encode_json($data);
# Nur noch die temporäre Datei löschen:
unlink($filename);
print $data;
......@@ -18,9 +18,6 @@
* php-gd
* sqlite3
* redis-server
* Die Perl-Pakete
* Lingua::Identify (http://search.cpan.org/~ambs/Lingua-Identify-0.56/lib/Lingua/Identify.pm)
* JSON (http://search.cpan.org/~makamaka/JSON-2.90/lib/JSON.pm)
---
[<img src="public/img/Browserstack-logo_2x.png" width="250px" alt="Browserstack Logo" />](https://www.browserstack.com) <br />
......
$(document).ready(function () {
botProtection();
enableFormResetter();
loadMoreResults();
});
function botProtection() {
......@@ -50,4 +50,72 @@ function enableFormResetter() {
timeout = null;
}, 500);
});
}
function loadMoreResults() {
var searchKey = $("meta[name=searchkey]").attr("content");
var updateUrl = document.location.href;
updateUrl += "&loadMore=" + searchKey + "&script=yes";
updateUrl = updateUrl.replace("/meta.ger3", "/loadMore");
var expectedEngines = -1;
var deliveredEngines = -1;
var currentlyLoading = false;
// Regularily check for not yet delivered Results
var resultLoader = window.setInterval(function () {
if (!currentlyLoading) {
currentlyLoading = true;
$.getJSON(updateUrl, function (data) {
// Check if we can clear the interval (once every searchengine has answered)
if (data.engineDelivered == data.engineCount || data.timeWaiting > 15) {
clearInterval(resultLoader);
}
// If there are new results we can add them
if (typeof data.newResults != "undefined") {
for (var key in data.newResults) {
var value = data.newResults[key];
// If there are more results than the given index we will prepend otherwise we will append the result
if (!data.imagesearch) {
var results = $(".result:not(.ad)");
if (key == 0) {
if ($(".result.ad").length > 0) {
$(value).insertAfter($($(".result.ad")[$(".result.ad").length - 1]));
} else {
$("#results").prepend(value);
}
} else if (typeof results[key] != "undefined") {
$(value).insertBefore($(results[key]));
} else if (typeof results[key - 1] != "undefined") {
$(value).insertAfter($(results[key - 1]));
}
} else {
var results = $(".image-container > .image");
if (key == 0) {
$(".image-container").prepend(value);
} else if (typeof results[key] != "undefined") {
$(value).insertBefore($(results[key]));
} else if (typeof results[key - 1] != "undefined") {
$(value).insertAfter($(results[key - 1]));
}
}
}
if ($(".no-results-error").length > 0 && ($(".image-container > .image").length > 0) || $(".result:not(.ad)").length > 0) {
$(".no-results-error").remove();
if ($(".alert.alert-danger > ul").children().length == 0) {
$(".alert.alert-danger").remove();
}
}
console.log(data);
}
currentlyLoading = false;
});
}
}, 1000);
//clearInterval(resultLoader);
console.log(updateUrl);
}
\ No newline at end of file
<div class="image">
<a href="{{ $result->link }}" target="_blank">
<div title="{{ $result->titel }}">
<img src="{{ $metager->getImageProxyLink($result->image)}}" alt="{{ $result->titel }}"/>
<div>{{ $result->gefVon }}</div>
</div>
</a>
</div>
<div class="result" data-count="{{ $result->number }}">
<div class="result" data-count="{{ $result->hash }}">
<div class="result-header">
<div class="result-headline">
<h2 class="result-title" title="{{ $result->titel }}">
......@@ -47,7 +47,7 @@
</div>
@endif
</div>
<input type="checkbox" id="result-toggle-{{$result->number}}" class="result-toggle" style="display: none">
<input type="checkbox" id="result-toggle-{{$result->hash}}" class="result-toggle" style="display: none">
<div class="result-footer">
<a class="result-open" href="{{ $result->link }}" target="_self" rel="noopener">
{!! trans('result.options.7') !!}
......@@ -58,10 +58,10 @@
<a class="result-open-proxy" onmouseover="$(this).popover('show');" onmouseout="$(this).popover('hide');" data-toggle="popover" data-placement="auto right" data-container="body" data-content="@lang('result.proxytext')" href="{{ $result->proxyLink }}" target="{{ $metager->getNewtab() }}" rel="noopener">
{!! trans('result.options.5') !!}
</a>
<label class="open-result-options navigation-element" for="result-toggle-{{$result->number}}">
<label class="open-result-options navigation-element" for="result-toggle-{{$result->hash}}">
{{ trans('result.options.more')}}
</label>
<label class="close-result-options navigation-element" for="result-toggle-{{$result->number}}">
<label class="close-result-options navigation-element" for="result-toggle-{{$result->hash}}">
{{ trans('result.options.less')}}
</label>
</div>
......@@ -69,7 +69,7 @@
<div class="options">
<ul class="option-list list-unstyled small">
<li class="js-only">
<a href="javascript:resultSaver({{ $result->number }});" class="saver">
<a href="javascript:resultSaver("{{ $result->hash }}");" class="saver">
<nobr><i class="fa fa-floppy-o"></i> {!! trans('result.options.savetab') !!}</nobr>
</a>
</li>
......
......@@ -11,6 +11,7 @@
<meta name="l" content="{{ LaravelLocalization::getCurrentLocale() }}" />
<meta name="mm" content="{{ $metager->getVerificationId() }}" />
<meta name="mn" content="{{ $metager->getVerificationCount() }}" />
<meta name="searchkey" content="{{ $metager->getSearchUid() }}" />
<link rel="search" type="application/opensearchdescription+xml" title="{!! trans('resultPage.opensearch') !!}" href="{{ LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), action('StartpageController@loadPlugin', ['params' => base64_encode(serialize(Request::all()))])) }}">
<link type="text/css" rel="stylesheet" href="{{ mix('css/fontawesome.css') }}" />
<link type="text/css" rel="stylesheet" href="{{ mix('css/fontawesome-solid.css') }}" />
......
......@@ -3,7 +3,7 @@
<div class="alert alert-danger">
<ul>
@foreach($errors as $error)
<li>{!! $error !!}</li>
<li @if($error === trans('metaGer.results.failed')) class="no-results-error" @endif>{!! $error !!}</li>
@endforeach
</ul>
</div>
......
......@@ -3,7 +3,7 @@
<div @if($metager->getPage() === 1) class="disabled" @endif>
<a @if($metager->getPage() !== 1) href="javascript:history.back()" @endif>{{ trans('results.zurueck') }}</a>
</div>
<div @if($metager->nextSearchLink() === "#") class="disabled" @endif>
<div id="next-search-link" @if($metager->nextSearchLink() === "#") class="disabled" @endif>
<a @if($metager->nextSearchLink() !== "#") href="{{ $metager->nextSearchLink() }}" @endif>{{ trans('results.weiter') }}</a>
</div>
</nav>
\ No newline at end of file
</nav>
......@@ -7,17 +7,17 @@
@endfor
@endif
{{-- Create results and ongoing ads --}}
@foreach($metager->getResults() as $result)
@foreach($metager->getResults() as $index => $result)
@if($mobile)
@if($result->number % 4 === 0)
@if($index % 4 === 0)
@include('layouts.ad', ['ad' => $metager->popAd()])
@endif
@else
@if($result->number % 7 === 0)
@if($index % 7 === 0)
@include('layouts.ad', ['ad' => $metager->popAd()])
@endif
@endif
@include('layouts.result', ['result' => $result])
@endforeach
@include('parts.pager')
</div>
\ No newline at end of file
</div>
@include('parts.errors')
@include('parts.warnings')
<div id="container" class="image-container">
@foreach($metager->getResults() as $result)
<div class="image">
<a href="{{ $result->link }}" target="_blank">
<div title="{{ $result->titel }}">
<img src="{{ $metager->getImageProxyLink($result->image)}}" alt="{{ $result->titel }}"/>
<div>{{ $result->gefVon }}</div>
</div>
</a>
</div>
@include('layouts.image_result', ['result' => $result])
@endforeach
</div>
@include('parts.pager')
......@@ -179,6 +179,7 @@ Route::group(
Route::get('settings', 'StartpageController@loadSettings');
Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification');
Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
Route::post('img/cat.jpg', 'HumanVerification@remove');
Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);
Route::post('img/dog.jpg', 'HumanVerification@whitelist');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment