Commit 5ade5d6f authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch 'development' into 485-ux-bei-such-timeout

parents 9e0014ed 340fcc86
......@@ -5,8 +5,6 @@ namespace App\Exceptions;
use Exception;
use Illuminate\Auth\AuthenticationException;
use Illuminate\Foundation\Exceptions\Handler as ExceptionHandler;
use Illuminate\Http\Response;
use Predis\Connection\ConnectionException;
class Handler extends ExceptionHandler
{
......
......@@ -11,6 +11,14 @@ class MetaGerSearch extends Controller
{
public function search(Request $request, MetaGer $metager)
{
$focus = $request->input("focus", "web");
if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
$metager->parseFormData($request);
if ($metager->doBotProtection($request->input('bot', ""))) {
return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), url("/noaccess", ['redirect' => base64_encode(url()->full())])));
}
return $metager->createView();
}
#die($request->header('User-Agent'));
$time = microtime();
# Mit gelieferte Formulardaten parsen und abspeichern:
......@@ -255,4 +263,10 @@ class MetaGerSearch extends Controller
{
return file_get_contents($url);
}
private function startsWith($haystack, $needle)
{
$length = strlen($needle);
return (substr($haystack, 0, $length) === $needle);
}
}
......@@ -45,14 +45,13 @@ class StartpageController extends Controller
->with('theme', $theme)
->with('autocomplete', $request->input('param_autocomplete', 'on'))
->with('foki', $this->loadFoki())
->with('focus', $request->input('focus', 'web'))
->with('lang', $request->input('param_lang', 'all'))
->with('resultCount', $request->input('param_resultCount', '20'))
->with('time', $request->input('param_time', '1000'))
->with('sprueche', $request->input('param_sprueche', 'off'))
->with('time', $request->input('param_time', '1500'))
->with('sprueche', $request->input('param_sprueche', 'on'))
->with('newtab', $request->input('param_newtab', 'on'))
->with('maps', $maps = $request->input('param_maps', 'on'));
->with('maps', $maps = $request->input('param_maps', 'off'));
}
public function loadPage($subpage)
......
......@@ -8,7 +8,6 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Predis\Connection\ConnectionException;
use Log;
class Searcher implements ShouldQueue
......@@ -107,12 +106,6 @@ class Searcher implements ShouldQueue
$result = curl_exec($this->ch);
if(!empty(curl_error($this->ch))){
// Es gab einen Fehler beim Abruf des Ergebnisses
// diesen sollten wir protokollieren
Log::error("Fehler in der Searcher Klasse, beim Abruf des Ergebnisses: \n" . curl_error($this->ch));
}
return $result;
}
......
......@@ -563,42 +563,35 @@ class MetaGer
}
# Wir starten alle Suchen
$success = true;
foreach ($engines as $engine) {
if(!$engine->startSearch($this)){
$success = false;
}
$engine->startSearch($this);
}
if($success){
// Derzeit deaktiviert, da es die eigene Suche gibt
// $this->adjustFocus($sumas, $enabledSearchengines);
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$enginesToLoad = [];
$canBreak = false;
foreach ($engines as $engine) {
if ($engine->cached) {
if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
$canBreak = true;
}
} else {
$enginesToLoad[$engine->name] = false;
// Derzeit deaktiviert, da es die eigene Suche gibt
// $this->adjustFocus($sumas, $enabledSearchengines);
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$enginesToLoad = [];
$canBreak = false;
foreach ($engines as $engine) {
if ($engine->cached) {
if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
$canBreak = true;
}
} else {
$enginesToLoad[$engine->name] = false;
}
}
$this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
$this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
$this->retrieveResults($engines);
}else{
Log::error('Fehler beim Verbindungsaufbau zum lokalen Redis Server!');
$this->errors[] = trans('metager.redis.error');
}
$this->retrieveResults($engines);
}
# Spezielle Suchen und Sumas
......@@ -822,13 +815,9 @@ class MetaGer
{
$timeStart = microtime(true);
$results = array();
$results = null;
while (true) {
try{
$results = Redis::hgetall('search.' . $this->getHashCode());
}catch(ConnectionException $e){
break;
}
$results = Redis::hgetall('search.' . $this->getHashCode());
$ready = true;
// When every
......
......@@ -8,7 +8,6 @@ use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Log;
use Illuminate\Support\Facades\Redis;
use Predis\Connection\ConnectionException;
abstract class Searchengine
{
......@@ -110,73 +109,67 @@ abstract class Searchengine
$this->cached = true;
$this->retrieveResults($metager);
} else {
try{
// We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if($this->port === "443"){
$url = "https://";
}else{
$url = "http://";
// We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if($this->port === "443"){
$url = "https://";
}else{
$url = "http://";
}
$url .= $this->host . $this->getString;
$mission = $this->resultHash . ";" . $url;
// Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher = false;
$searcherData = Redis::hgetall($this->name . ".stats");
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if(sizeof($searcherData) === 0){
$needSearcher = true;
}else{
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
// searches or if it needs help.
// Let's hardcode a minimum of 100ms between every search job.
// First calculate the median of all Times
$median = 0;
foreach($searcherData as $pid => $data){
$data = explode(";", $data);
$median += floatval($data[1]);
}
$url .= $this->host . $this->getString;
$mission = $this->resultHash . ";" . $url;
// Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher = false;
$searcherData = Redis::hgetall($this->name . ".stats");
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if(sizeof($searcherData) === 0){
$median /= sizeof($searcherData);
if($median < 100){
$needSearcher = true;
}else{
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
// searches or if it needs help.
// Let's hardcode a minimum of 100ms between every search job.
// First calculate the median of all Times
$median = 0;
foreach($searcherData as $pid => $data){
$data = explode(";", $data);
$median += floatval($data[1]);
}
$median /= sizeof($searcherData);
if($median < .1){ // Median is given as a float in seconds
$needSearcher = true;
}
}
if($needSearcher && Redis::get($this->name) !== "locked"){
Redis::set($this->name, "locked");
$this->dispatch(new Searcher($this->name));
}
return true;
}catch(ConnectionException $e){
return false;
}
if($needSearcher && Redis::get($this->name) !== "locked"){
Redis::set($this->name, "locked");
$this->dispatch(new Searcher($this->name));
}
}
}
......@@ -229,33 +222,28 @@ abstract class Searchengine
# Fragt die Ergebnisse von Redis ab und lädt Sie
public function retrieveResults(MetaGer $metager)
{
try{
if ($this->loaded) {
return true;
}
$body = "";
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
$body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);
if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
Cache::put($this->hash, $body, $this->cacheDuration);
}
if ($this->loaded) {
return true;
}
$body = "";
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
$body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);
if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
Cache::put($this->hash, $body, $this->cacheDuration);
}
if ($body !== "") {
$this->loadResults($body);
$this->getNext($metager, $body);
$this->loaded = true;
Redis::hdel('search.' . $this->hash, $this->name);
return true;
} else {
return false;
}
}catch(ConnectionException $e){
}
if ($body !== "") {
$this->loadResults($body);
$this->getNext($metager, $body);
$this->loaded = true;
Redis::hdel('search.' . $this->hash, $this->name);
return true;
} else {
return false;
}
}
......
......@@ -17,7 +17,40 @@ class AppServiceProvider extends ServiceProvider
*/
public function boot()
{
//
# Wir loggen im Redis-System für jede Sekunde des Tages, wie viele Worker aktiv am Laufen waren.
# Dies ist notwendig, damit wir mitbekommen können, ab welchem Zeitpunkt wir zu wenig Worker zur Verfügung haben.
Queue::before(function (JobProcessing $event) {
$this->begin = strtotime(date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))));
});
Queue::after(function (JobProcessed $event) {
$today = strtotime(date(DATE_RFC822, mktime(0, 0, 0, date("m"), date("d"), date("Y"))));
$end = strtotime(date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y")))) - $today;
$expireAt = strtotime(date(DATE_RFC822, mktime(0, 0, 0, date("m"), date("d") + 1, date("Y"))));
try {
$redis = Redis::connection('redisLogs');
if (!$redis) {
return;
}
$p = getmypid();
$host = gethostname();
$begin = $this->begin - $today;
$redis->pipeline(function ($pipe) use ($p, $expireAt, $host, $begin, $end) {
for ($i = $begin; $i <= $end; $i++) {
$pipe->sadd("logs.worker.$host.$i", strval($p));
$pipe->expire("logs.worker.$host.$i", 10);
$pipe->eval("redis.call('hset', 'logs.worker.$host', '$i', redis.call('scard', 'logs.worker.$host.$i'))", 0);
$pipe->sadd("logs.worker", $host);
if (date("H") !== 0) {
$pipe->expire("logs.worker.$host", $expireAt);
$pipe->expire("logs.worker", $expireAt);
}
}
});
} catch (\Exception $e) {
return;
}
});
}
/**
......
This diff is collapsed.
$(document).ready(function () {
createCustomFocuses()
var focus = $('#foki > li.active > a').attr('aria-controls')
var custom = $('#foki > li.active').hasClass('custom-focus-tab-selector')
getDocumentReadyForUse(focus, custom)
})
function tabs () {
$('#foki > li.tab-selector > a').each(function () {
if ($(this).attr('target') != '_blank') {
$(this).attr('href', '#' + $(this).attr('aria-controls'))
$(this).attr('role', 'tab')
$(this).attr('data-toggle', 'tab')
}
})
$('#foki > li.tab-selector > a').off()
$('#foki > li.tab-selector > a').on('show.bs.tab', function (e) {
var fokus = $(this).attr('aria-controls')
var link = $('#' + fokus + 'TabSelector a').attr('data-href')
if ($('#' + fokus + 'TabSelector').attr('data-loaded') != '1') {
$.get(link, function (data) {
$('#' + fokus + 'TabSelector').attr('data-loaded', '1')
$('#' + fokus).html(data)
$('input[name=focus]').val($('#foki li.active a').attr('aria-controls'))
getDocumentReadyForUse(fokus)
})
} else {
getDocumentReadyForUse(fokus)
}
})
}
function getDocumentReadyForUse (fokus, custom = false) {
clickLog()
popovers()
if (fokus === 'bilder') imageLoader()
if (custom) initialLoadContent(fokus)
// pagination()
tabs()
theme()
fokiChanger()
pluginInfo()
productWidget()
$('iframe:not(.resized)').iFrameResize()
$('iframe').addClass('resized')
}
function pluginInfo () {
if (localStorage) {
if (localStorage.getItem('pluginInfo') == 'off') $('#searchplugin').css('display', 'none')
$('#searchplugin').on('close.bs.alert', function () {
$.get('/pluginClose')
localStorage.setItem('pluginInfo', 'off')
})
$('#searchplugin a.btn').click(function () {
$.get('/pluginInstall')
})
}
}
function theme () {
if (localStorage) {
var theme = localStorage.getItem('theme')
if (theme != null) {
if ((theme.match(/,/g) || []).length != 3) {
localStorage.removeItem('theme')
} else {
theme = theme.split(',')
$('#theme').attr('href', '/css/theme.css.php?r=' + theme[0] + '&g=' + theme[1] + '&b=' + theme[2] + '&a=' + theme[3])
}
}
}
}
function clickLog () {
$('.result a.title, .result div.link-link a').off()
$('.result a.title, .result div.link-link a').click(function () {
$.get('/clickstats', {
i: $('meta[name=p]').attr('content'),
s: $(this).attr('data-hoster'),
q: $('meta[name=q]').attr('content'),
p: $(this).attr('data-count'),
url: $(this).attr('href')
})
})
}
function popovers () {
$('[data-toggle=popover]').each(function (e) {
$(this).popover('destroy')
$(this).popover({
// html : true,
// title : "<i class="fa fa-cog" aria-hidden="true"></i> Optionen",
content: $(this).parent().find('.content').html()
})
})
}
function pagination () {
$('.pagination li:not(.active) > a').attr('href', '#')
$('.pagination li.disabled > a').removeAttr('href')
$('.pagination li:not(.active) > a').off()
$('.pagination li:not(.active) > a').click(paginationHandler)
}
function paginationHandler () {
var link = $(this).attr('data-href')
if (link.length == 0) {
return
}
var tabPane = $('.tab-pane.active')
$(tabPane).html('<div class="loader"><img src="/img/ajax-loader.gif" alt="" /></div>')
$.get(link, function (data) {
$(tabPane).html(data)
$('.pagination li:not(.active) > a').attr('href', '#')
$('.pagination li.disabled > a').removeAttr('href')
$('.pagination li:not(.active) > a').off()
$('.pagination li:not(.active) > a').click(paginationHandler)
getDocumentReadyForUse()
})
}
function imageLoader () {
if (typeof $('#container').masonry == 'undefined') {
return
}
var $grid = $('#container').masonry({
columnWidth: 150,
itemSelector: '.item',
gutter: 10,
isFitWidth: true
})
$grid.imagesLoaded().progress(function (instance, image) {
$grid.masonry('layout')
})
}
function eliminateHost (host) {
$('.result:not(.ad)').each(function (e) {
var host2 = $(this).find('.link-link > a').attr('data-host')
if (host2.indexOf(host) === 0) {
$(this).css('display', 'none')
}
})
}
function fokiChanger () {
$('#fokiChanger ul > li').click(function () {
document.location.href = $(this).attr('data-href')
})
}
// Polyfill for form attribute
(function ($) {
/**
* polyfill for html5 form attr
*/
// detect if browser supports this
var sampleElement = $('[form]').get(0)
var isIE11 = !(window.ActiveXObject) && 'ActiveXObject' in window
if (sampleElement && window.HTMLFormElement && sampleElement.form instanceof HTMLFormElement && !isIE11) {
// browser supports it, no need to fix
return
}
/**
* Append a field to a form
*
*/
$.fn.appendField = function (data) {
// for form only
if (!this.is('form')) return
// wrap data
if (!$.isArray(data) && data.name && data.value) {
data = [data]
}
var $form = this
// attach new params
$.each(data, function (i, item) {
$('<input/>').attr('type', 'hidden').attr('name', item.name).val(item.value).appendTo($form)
})
return $form
}
/**
* Find all input fields with form attribute point to jQuery object
*
*/
$('form[id]').submit(function (e) {
var $form = $(this)
// serialize data
var data = $('[form=' + $form.attr('id') + ']').serializeArray()
// append data to form
$form.appendField(data)
}).each(function () {
var form = this,
$form = $(form),
$fields = $('[form=' + $form.attr('id') + ']')
$fields.filter('button, input').filter('[type=reset],[type=submit]').click(function () {
var type = this.type.toLowerCase()