Commit 37fe69f8 authored by Dominik Hebeler's avatar Dominik Hebeler

We can now generate json responses for missing results

parent 7d16f746
......@@ -5,6 +5,8 @@ namespace App\Http\Controllers;
use App;
use App\MetaGer;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use View;
const TIP_SERVER = 'http://metager3.de:63825/tips.xml';
......@@ -19,13 +21,6 @@ class MetaGerSearch extends Controller
return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
}
/*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
$metager->parseFormData($request);
return $metager->createView();
}*/
#die($request->header('User-Agent'));
$time = microtime(true);
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
......@@ -39,6 +34,12 @@ class MetaGerSearch extends Controller
# auf Ergebnisse warten und die Ergebnisse laden
$metager->createSearchEngines($request);
$metager->startSearch();
$metager->waitForMainResults();
$metager->retrieveResults();
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
# Alle Ergebnisse vor der Zusammenführung ranken:
......@@ -47,10 +48,93 @@ class MetaGerSearch extends Controller
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults();
# Save the results in Redis
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
foreach ($metager->getResults() as $result) {
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), 6000);
$pipeline->execute();
# Die Ausgabe erstellen:
return $metager->createView($quicktipResults);
}
public function loadMore(Request $request)
{
/**
* There are three forms of requests to the resultpage
* 1. Initial Request: Loads the fastest searchengines and sends them to the user
* 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
* 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
*/
if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
return $this->loadMoreJS($request);
}
}
private function loadMoreJS(Request $request)
{
# Create a MetaGer Instance with the supplied hash
$hash = $request->input('loadMore', '');
$metager = new MetaGer($hash);
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$result = [];
# Check if there should be more results
$stats = $redis->hgetall($metager->getRedisEngineResult() . "status");
$stats["startTime"] = floatval($stats["startTime"]);
$stats["engineCount"] = intval($stats["engineCount"]);
$stats["engineAnswered"] = intval($stats["engineAnswered"]);
$stats["engineDelivered"] = intval($stats["engineDelivered"]);
$result["finished"] = true;
$result["engineCount"] = $stats["engineCount"];
$result["engineAnswered"] = $stats["engineAnswered"];
$result["engineDelivered"] = $stats["engineDelivered"];
$result["timeWaiting"] = microtime(true) - $stats["startTime"];
# Check if we can abort
if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
$metager->parseFormData($request);
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
# Read which search engines are new
$newEngines = [];
foreach ($redis->lrange($metager->getRedisResultWaitingKey(), 0, -1) as $engine) {
$result["engineDelivered"]++;
$newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
}
$metager->actuallyCreateSearchEngines($newEngines);
# Add the results already delivered to the suer
$results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1);
foreach ($results as $index => $oldResult) {
$results[$index] = unserialize(base64_decode($oldResult));
$results[$index]->new = false;
}
$metager->setResults($results);
$metager->retrieveResults();
$metager->rankAll();
$metager->prepareResults();
$results = $metager->getResults();
foreach ($results as $index => $resultTmp) {
if ($resultTmp->new) {
$view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
}
}
}
return response()->json($result);
}
public function botProtection($redirect)
{
$hash = md5(date('YmdHi'));
......
......@@ -171,12 +171,14 @@ class Searcher implements ShouldQueue
}
$pipeline = Redis::pipeline();
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0");
$pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
$pipeline->expire('search.' . $hashValue . ".results." . $this->name, 60);
$pipeline->expire('search.' . $hashValue . ".results." . $this->name, 6000);
$pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
$pipeline->expire('search.' . $hashValue . ".ready", 60);
$pipeline->expire('search.' . $hashValue . ".ready", 6000);
$pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
$pipeline->expire('search.' . $hashValue . ".engines", 60);
$pipeline->expire('search.' . $hashValue . ".engines", 6000);
$pipeline->execute();
$this->lastTime = microtime(true);
}
......
This diff is collapsed.
......@@ -27,6 +27,7 @@ class Result
public $strippedDomain; # Die Domain in Form "bar.de"
public $strippedLink; # Der Link in Form "foo.bar.de/test"
public $rank; # Das Ranking für das Ergebnis
public $new = true;
# Erstellt ein neues Ergebnis
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
......@@ -67,6 +68,7 @@ class Result
$this->price = isset($additionalInformation["price"]) ? $additionalInformation["price"] : 0;
$this->price_text = $this->price_to_text($this->price);
$this->additionalInformation = $additionalInformation;
$this->hash = spl_object_hash($this);
}
private function price_to_text($price)
......
#!/usr/bin/perl
use Lingua::Identify qw(:language_identification);
use JSON;
use warnings;
use strict;
binmode STDOUT, ":utf8";
binmode STDIN, ":utf8";
use utf8;
chomp(my $filename = <STDIN>);
# Lets open the given file:
open(my $fh, "<", $filename)
or die "Can't open < $filename: $!";
my $json = <$fh>;
close $fh;
# Decode the JSON String
my $data = JSON->new->utf8->decode($json);
# Wir durchlaufen den Hash:
foreach my $key (keys %{$data}){
$data->{$key} = langof($data->{$key});
}
$data = encode_json($data);
# Nur noch die temporäre Datei löschen:
unlink($filename);
print $data;
......@@ -18,9 +18,6 @@
* php-gd
* sqlite3
* redis-server
* Die Perl-Pakete
* Lingua::Identify (http://search.cpan.org/~ambs/Lingua-Identify-0.56/lib/Lingua/Identify.pm)
* JSON (http://search.cpan.org/~makamaka/JSON-2.90/lib/JSON.pm)
---
[<img src="public/img/Browserstack-logo_2x.png" width="250px" alt="Browserstack Logo" />](https://www.browserstack.com) <br />
......
......@@ -53,5 +53,48 @@ function enableFormResetter() {
}
function loadMoreResults() {
var searchKey = $("meta[name=searchkey]").attr("content");
var updateUrl = document.location.href;
updateUrl += "&loadMore=" + searchKey + "&script=yes";
updateUrl = updateUrl.replace("/meta.ger3", "/loadMore");
var expectedEngines = -1;
var deliveredEngines = -1;
var currentlyLoading = false;
// Regularily check for not yet delivered Results
var resultLoader = window.setInterval(function () {
if (!currentlyLoading) {
currentlyLoading = true;
$.getJSON(updateUrl, function (data) {
// Check if we can clear the interval (once every searchengine has answered)
if (data.engineDelivered == data.engineCount || data.timeWaiting > 5) {
clearInterval(resultLoader);
}
// If there are new results we can add them
if (typeof data.newResults != "undefined") {
for (var key in data.newResults) {
var value = data.newResults[key];
// If there are more results than the given index we will prepend otherwise we will append the result
var results = $(".result:not(.ad)");
if (typeof results[key] != "undefined") {
console.log("inserting before " + key);
$(results[key]).insertBefore(value);
} else {
$(results[key - 1]).insertAfter(value);
}
console.log(key + "=>" + value);
}
}
currentlyLoading = false;
});
}
}, 1000);
console.log(updateUrl);
}
\ No newline at end of file
<div class="result" data-count="{{ $result->number }}">
<div class="result" data-count="{{ $result->hash }}">
<div class="result-header">
<div class="result-headline">
<h2 class="result-title" title="{{ $result->titel }}">
......@@ -47,7 +47,7 @@
</div>
@endif
</div>
<input type="checkbox" id="result-toggle-{{$result->number}}" class="result-toggle" style="display: none">
<input type="checkbox" id="result-toggle-{{$result->hash}}" class="result-toggle" style="display: none">
<div class="result-footer">
<a class="result-open" href="{{ $result->link }}" target="_self" rel="noopener">
{!! trans('result.options.7') !!}
......@@ -58,10 +58,10 @@
<a class="result-open-proxy" onmouseover="$(this).popover('show');" onmouseout="$(this).popover('hide');" data-toggle="popover" data-placement="auto right" data-container="body" data-content="@lang('result.proxytext')" href="{{ $result->proxyLink }}" target="{{ $metager->getNewtab() }}" rel="noopener">
{!! trans('result.options.5') !!}
</a>
<label class="open-result-options navigation-element" for="result-toggle-{{$result->number}}">
<label class="open-result-options navigation-element" for="result-toggle-{{$result->hash}}">
{{ trans('result.options.more')}}
</label>
<label class="close-result-options navigation-element" for="result-toggle-{{$result->number}}">
<label class="close-result-options navigation-element" for="result-toggle-{{$result->hash}}">
{{ trans('result.options.less')}}
</label>
</div>
......@@ -69,7 +69,7 @@
<div class="options">
<ul class="option-list list-unstyled small">
<li class="js-only">
<a href="javascript:resultSaver({{ $result->number }});" class="saver">
<a href="javascript:resultSaver("{{ $result->hash }}");" class="saver">
<nobr><i class="fa fa-floppy-o"></i> {!! trans('result.options.savetab') !!}</nobr>
</a>
</li>
......
......@@ -7,17 +7,17 @@
@endfor
@endif
{{-- Create results and ongoing ads --}}
@foreach($metager->getResults() as $result)
@foreach($metager->getResults() as $index => $result)
@if($mobile)
@if($result->number % 4 === 0)
@if($index % 4 === 0)
@include('layouts.ad', ['ad' => $metager->popAd()])
@endif
@else
@if($result->number % 7 === 0)
@if($index % 7 === 0)
@include('layouts.ad', ['ad' => $metager->popAd()])
@endif
@endif
@include('layouts.result', ['result' => $result])
@endforeach
@include('parts.pager')
</div>
\ No newline at end of file
</div>
......@@ -179,6 +179,7 @@ Route::group(
Route::get('settings', 'StartpageController@loadSettings');
Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification');
Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
Route::post('img/cat.jpg', 'HumanVerification@remove');
Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);
Route::post('img/dog.jpg', 'HumanVerification@whitelist');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment