Commit a8438db8 authored by Aria Givi's avatar Aria Givi
Browse files

Merge branch 'development' into 408-arbeit-am-ubersetzungstool

parents ff3c3e70 b56cfcb9
......@@ -5,3 +5,4 @@
Homestead.json
Homestead.yaml
.env
.orig
......@@ -38,13 +38,32 @@ class Search implements ShouldQueue
*/
public function handle()
{
$this->fp = $this->getFreeSocket();
if ($this->fp) {
if ($this->writeRequest()) {
$this->readAnswer();
}
$url = "";
if($this->port === "443"){
$url = "https://";
}else{
$url = "http://";
}
$url .= $this->host . $this->getString;
$ch = curl_init($url);
curl_setopt_array($ch, array(
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_URL => $url,
CURLOPT_USERAGENT => $this->useragent,
CURLOPT_FOLLOWLOCATION => TRUE,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_MAXCONNECTS => 50,
CURLOPT_LOW_SPEED_LIMIT => 500,
CURLOPT_LOW_SPEED_TIME => 5,
CURLOPT_TIMEOUT => 10
));
$result = curl_exec($ch);
curl_close($ch);
Redis::hset('search.' . $this->hash, $this->name, $result);
}
private function readAnswer()
......
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Log;
class Searcher implements ShouldQueue
{
use InteractsWithQueue, Queueable, SerializesModels;
protected $name, $ch, $pid, $counter, $lastTime;
protected $MAX_REQUESTS = 500;
/**
* Create a new job instance.
* This is our new Worker/Searcher Class
* It will take it's name from the sumas.xml as constructor argument
* Each Searcher is dedicated to one remote server from our supported Searchengines
* It will listen to a queue in the Redis Database within the handle() method and
* answer requests to this specific search engine.
* The curl handle will be left initialized and opened so that we can make full use of
* keep-alive requests.
* @return void
*/
public function __construct($name)
{
$this->name = $name;
$this->pid = getmypid();
// Submit this worker to the Redis System
Redis::set($this->name, "running");
Redis::expire($this->name, 5);
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
// This Searches is freshly called so we need to initialize the curl handle $ch
$this->ch = $this->initCurlHandle();
$this->counter = 0; // Counts the number of answered jobs
$time = microtime(true);
while(true){
// Update the expire
Redis::expire($this->name, 5);
// One Searcher can handle a ton of requests to the same server
// Each search to the server of this Searcher will be submitted to a queue
// stored in redis which has the same name as this searchengine appended by a ".queue"
// We will perform a blocking pop on this queue so the queue can remain empty for a while
// without killing this searcher directly.
$mission = Redis::blpop($this->name . ".queue", 4);
// The mission can be empty when blpop hit the timeout
if(empty($mission)){
continue;
}else{
$mission = $mission[1];
$this->counter++;#
$poptime = microtime(true) - $time;
$time = microtime(true);
}
// The mission is a String which can be divided to retrieve two informations:
// 1. The Hash Value where the result should be stored
// 2. The Url to Retrieve
// These two informations are divided by a ";" in the mission string
$hashValue = substr($mission, 0, strpos($mission, ";"));
$url = substr($mission, strpos($mission, ";") + 1);
Redis::hset('search.' . $hashValue, $this->name, "connected");
$result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue);
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if(getenv("QUEUE_DRIVER") === "sync" || $this->counter > $this->MAX_REQUESTS){
break;
}
}
// When we reach this point, time has come for this Searcher to retire
$this->exit();
}
private function retrieveUrl($url){
// Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch);
return $result;
}
private function storeResult($result, $poptime, $hashValue){
Redis::hset('search.' . $hashValue, $this->name, $result);
$connectionInfo = base64_encode(json_encode(curl_getinfo($this->ch), true));
Redis::hset($this->name . ".stats", $this->pid, $connectionInfo . ";" . $poptime);
$this->lastTime = microtime(true);
}
private function exit(){
Redis::hdel($this->name . ".stats", $this->pid);
if(sizeof(Redis::hgetall($this->name . ".stats")) === 0){
Redis::del($this->name);
}
// We should close our curl handle before we do so
curl_close($this->ch);
}
private function initCurlHandle(){
$ch = curl_init();
curl_setopt_array($ch, array(
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
CURLOPT_FOLLOWLOCATION => TRUE,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 500,
CURLOPT_LOW_SPEED_TIME => 5,
CURLOPT_TIMEOUT => 10
));
return $ch;
}
}
......@@ -813,39 +813,41 @@ class MetaGer
public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
{
$loadedEngines = 0;
$timeStart = microtime(true);
# Auf wie viele Suchmaschinen warten wir?
$engineCount = count($enginesToLoad);
$timeStart = microtime(true);
$results = null;
while (true) {
$time = (microtime(true) - $timeStart) * 1000;
$loadedEngines = intval(Redis::hlen('search.' . $this->getHashCode()));
if ($overtureEnabled && (Redis::hexists('search.' . $this->getHashCode(), 'overture') || Redis::hexists('search.' . $this->getHashCode(), 'overtureAds'))) {
$canBreak = true;
}
# Abbruchbedingung
if ($time < 500) {
if (($engineCount === 0 || $loadedEngines >= $engineCount) && $canBreak) {
break;
$results = Redis::hgetall('search.' . $this->getHashCode());
$ready = true;
// When every
$connected = true;
foreach($results as $key => $value){
if($value === "waiting" || $value === "connected"){
$ready = false;
}
} elseif ($time >= 500 && $time < $this->time) {
if (($engineCount === 0 || ($loadedEngines / ($engineCount * 1.0)) >= 0.8) && $canBreak) {
break;
if($value === "waiting"){
$connected = false;
}
}
} else {
// If $ready is false at this point, we're waiting for more searchengines
// But we have to check for the timeout, too
if(!$connected) $timeStart = microtime(true);
$time = (microtime(true) - $timeStart) * 1000;
// We will apply the timeout only if it's not Yahoo we're waiting for since they are one the most
// important search engines.
$canTimeout = !((isset($results["overture"]) && $results["overture"] === "waiting") || (isset($results["overtureAds"]) && $results["overtureAds"] === "waiting"));
if($time > $this->time && $canTimeout) $ready = true;
if($ready){
break;
}
usleep(50000);
}
# Wir haben nun so lange wie möglich gewartet. Wir registrieren nun noch die Suchmaschinen, die geanwortet haben.
$answered = Redis::hgetall('search.' . $this->getHashCode());
foreach ($answered as $key => $value) {
foreach ($results as $key => $value) {
$enginesToLoad[$key] = true;
}
$this->enginesToLoad = $enginesToLoad;
......@@ -882,7 +884,7 @@ class MetaGer
{
$this->request = $request;
# Sichert, dass der request in UTF-8 formatiert ist
if ($request->input('encoding', '') !== "utf8") {
if ($request->input('encoding', 'utf8') !== "utf8") {
# In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
$input = $request->all();
foreach ($input as $key => $value) {
......@@ -921,7 +923,7 @@ class MetaGer
# Category
$this->category = $request->input('category', '');
# Request Times
$this->time = $request->input('time', 1000);
$this->time = $request->input('time', 1500);
# Page
$this->page = 1;
# Lang
......
......@@ -2,7 +2,7 @@
namespace App\Models;
use App\Jobs\Search;
use App\Jobs\Searcher;
use App\MetaGer;
use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
......@@ -105,17 +105,63 @@ abstract class Searchengine
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
public function startSearch(\App\MetaGer $metager)
{
if ($this->canCache && Cache::has($this->hash)) {
if ($this->canCache && Cache::has($this->hash) && 0 == 1) {
$this->cached = true;
$this->retrieveResults($metager);
} else {
/* Die Anfragen an die Suchmaschinen werden nun von der Laravel-Queue bearbeitet:
* Hinweis: solange in der .env der QUEUE_DRIVER auf "sync" gestellt ist, werden die Abfragen
* nacheinander abgeschickt.
* Sollen diese Parallel verarbeitet werden, muss ein anderer QUEUE_DRIVER verwendet werden.
* siehe auch: https://laravel.com/docs/5.2/queues
*/
$this->dispatch(new Search($this->resultHash, $this->host, $this->port, $this->name, $this->getString, $this->useragent, $this->additionalHeaders));
// We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if($this->port === "443"){
$url = "https://";
}else{
$url = "http://";
}
$url .= $this->host . $this->getString;
$mission = $this->resultHash . ";" . $url;
// Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher = false;
$searcherData = Redis::hgetall($this->name . ".stats");
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if(sizeof($searcherData) === 0){
$needSearcher = true;
}else{
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
// searches or if it needs help.
// Let's hardcode a minimum of 100ms between every search job.
die(var_dump($searcherData));
}
if($needSearcher){
$this->dispatch(new Searcher($this->name));
}
}
}
......@@ -172,11 +218,11 @@ abstract class Searchengine
}
$body = "";
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
$body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);
if ($this->canCache && $this->cacheDuration > 0) {
if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
Cache::put($this->hash, $body, $this->cacheDuration);
}
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -90,7 +90,7 @@ function popovers() {
$(this).popover("destroy");
$(this).popover({
//html : true,
//title : "<span class='glyphicon glyphicon-cog'></span> Optionen",
//title : "<i class="fa fa-cog" aria-hidden="true"></i> Optionen",
content: $(this).parent().find(".content").html()
});
});
......@@ -229,8 +229,8 @@ function productWidget() {
easing: 'cubic-bezier(0.25, 0, 0.25, 1)',
speed: 600,
pager: false,
prevHtml: '<span class="glyphicon glyphicon-chevron-left" aria-hidden="true"></span><span class="sr-only">Previous</span>',
nextHtml: '<span class="glyphicon glyphicon-chevron-right" aria-hidden="true"></span><span class="sr-only">Next</span>',
prevHtml: '<i class="fa fa-chevron-left" aria-hidden="true"></i></span><span class="sr-only">Previous</span>',
nextHtml: '<i class="fa fa-chevron-right" aria-hidden="true"></i><span class="sr-only">Next</span>',
responsive: [{
breakpoint: 1400,
settings: {
......@@ -273,8 +273,8 @@ function productWidget() {
pager: false,
enableTouch: false,
enableDrag: false,
prevHtml: '<span class="glyphicon glyphicon-chevron-left" aria-hidden="true"></span><span class="sr-only">Previous</span>',
nextHtml: '<span class="glyphicon glyphicon-chevron-right" aria-hidden="true"></span><span class="sr-only">Next</span>',
prevHtml: '<a class="fa fa-chevron-left" aria-hidden="true"></a><span class="sr-only">Previous</span>',
nextHtml: '<a class="fa fa-chevron-right" aria-hidden="true"></a><span class="sr-only">Next</span>',
responsive: [{
breakpoint: 1400,
settings: {
......@@ -332,14 +332,14 @@ function createCustomFocuses() {
* @if( $metager->getFokus() === "produktsuche" )
* <li id="produktsucheTabSelector" class="active tab-selector" role="presentation" data-loaded="1">
* <a aria-controls="produktsuche" data-href="#produktsuche" href="#produktsuche">
* <span class='glyphicon glyphicon-shopping-cart'></span>
* <i class="fa fa-shopping-cart" aria-hidden="true"></i>
* <span class="hidden-xs">{{ trans('index.foki.produkte') }}</span>
* </a>
* </li>
* @else
* <li id="produktsucheTabSelector" class="tab-selector" role="presentation" data-loaded="0">
* <a aria-controls="produktsuche" data-href="{!! $metager->generateSearchLink('produktsuche') !!}" href="{!! $metager->generateSearchLink('produktsuche', false) !!}">
* <span class='glyphicon glyphicon-shopping-cart'></span>
* <i class="fa fa-shopping-cart" aria-hidden="true"></i>
* <span class="hidden-xs">{{ trans('index.foki.produkte') }}</span>
* </a>
* </li>
......@@ -366,10 +366,11 @@ function addFocus(focus, active = false) {
var searchLink = generateSearchLinkForFocus(focus)
focusElementLink.setAttribute("data-href", searchLink);
focusElementLink.setAttribute("href", searchLink);
// create <span> glyphicon
var focusElementIcon = document.createElement("span");
focusElementIcon.classList.add("glyphicon");
focusElementIcon.classList.add("glyphicon-cog");
// create <a> icon
var focusElementIcon = document.createElement("a");
focusElementIcon.classList.add("fa");
focusElementIcon.classList.add("fa-cog");
focusElementIcon.setAttribute("aria-hidden", "true");
// create <span> focusname
var focusElementName = document.createElement("span");
focusElementName.classList.add("hidden-xs");
......
$(document).ready(function() {
// checkPlugin();
if (location.href.indexOf("#plugin-modal") > -1) {
$("#plugin-modal").modal("show");
}
$('#addFocusBtn').removeClass('hide');
$("button").popover();
if (localStorage) {
var theme = localStorage.getItem("theme");
if (theme != null) {
if ((theme.match(/,/g) || []).length != 3) {
localStorage.removeItem("theme");
} else {
theme = theme.split(",");
$("#theme").attr("href", "/css/theme.css.php?r=" + theme[0] + "&g=" + theme[1] + "&b=" + theme[2] + "&a=" + theme[3]);
}
}
if (localStorage.getItem("pers") && !isUseOnce()) {
setSettings();
}
}
setActionListeners();
loadInitialCustomFocuses()
});
function setActionListeners() {
$("button").on("shown.bs.popover", function() {
$("#color-chooser a").click(function() {
var theme = $(this).attr("data-rgba");
if (localStorage) {
localStorage.setItem("theme", theme);
location.href = "/";
}
});
});
$("#mobileFoki").change(function() {
var focus = $("#mobileFoki > option:selected").val();
if (focus == "angepasst") {
window.location = "./settings/";
} else {
window.location = "./?focus=" + focus;
}
});
if ($("fieldset#foki.mobile").length) {
$("fieldset#foki.mobile label#anpassen-label").click(function() {
window.location = "./settings/";
});
}
$("#addFocusBtn").click(function() {
showFocusCreateDialog("");
});
$("#save-focus-btn").click(saveFocus);
$("#delete-focus-btn").click(deleteFocus);
}
function setSettings() {
for (var i = 0; i < localStorage.length; i++) {
var key = localStorage.key(i);
var value = localStorage.getItem(key);
if (key.startsWith("param_") && !key.endsWith("lang") && !key.endsWith('autocomplete')) {
key = key.substring(key.indexOf("param_") + 6);
$("#searchForm").append("<input type=\"hidden\" name=\"" + key + "\" value=\"" + value + "\">");
}
$("#foki input[type=radio]#angepasst").attr("checked", true);
}
if (localStorage.getItem("param_lang") !== null) {
var value = localStorage.getItem("param_lang");
// Change the value of the lang input field to the given parameter
$("input[name=lang]").val(value);
}
if (localStorage.getItem("param_autocomplete") !== null) {
var value = localStorage.getItem("param_autocomplete");
// Change the value of the lang input field to the given parameter
$("input[name=eingabe]").attr("autocomplete", value);
}
if ($("fieldset#foki.mobile").length) {
$("fieldset.mobile input#bilder").val("angepasst");
$("fieldset.mobile input#bilder").prop("checked", true);
$("fieldset.mobile input#bilder").attr("id", "angepasst");
$("fieldset.mobile label#bilder-label").attr("id", "anpassen-label");
$("fieldset.mobile label#anpassen-label").attr("for", "angepasst");
$("fieldset.mobile label#anpassen-label span.glyphicon").attr("class", "glyphicon glyphicon-cog");
$("fieldset.mobile label#anpassen-label span.content").html("angepasst");
}
}
//Polyfill for form attribute
(function($) {
/**
* polyfill for html5 form attr
*/
// detect if browser supports this
var sampleElement = $('[form]').get(0);
var isIE11 = !(window.ActiveXObject) && "ActiveXObject" in window;
if (sampleElement && window.HTMLFormElement && sampleElement.form instanceof HTMLFormElement && !isIE11) {
// browser supports it, no need to fix
return;
}
/**
* Append a field to a form
*
*/
$.fn.appendField = function(data) {
// for form only
if (!this.is('form')) return;
// wrap data
if (!$.isArray(data) && data.name && data.value) {
data = [data];
}
var $form = this;
// attach new params
$.each(data, function(i, item) {
$('<input/>').attr('type', 'hidden').attr('name', item.name).val(item.value).appendTo($form);
});
return $form;
};
/**
* Find all input fields with form attribute point to jQuery object
*
*/
$('form[id]').submit(function(e) {
var $form = $(this);
// serialize data
var data = $('[form=' + $form.attr('id') + ']').serializeArray();
// append data to form
$form.appendField(data);
}).each(function() {
var form = this,
$form = $(form),
$fields = $('[form=' + $form.attr('id') + ']');
$fields.filter('button, input').filter('[type=reset],[type=submit]').click(function() {
var type = this.type.toLowerCase();
if (type === 'reset') {
// reset form
form.reset();
// for elements outside form
$fields.each(function() {
this.value = this.defaultValue;
this.checked = this.defaultChecked;
}).filter('select').each(function() {
$(this).find('option').each(function() {
this.selected = this.defaultSelected;
});
});
} else if (type.match(/^submit|image$/i)) {
$(form).appendField({
name: this.name,
value: this.value
}).submit();
}
});
});
})(jQuery);
// Opera 8.0+
var isOpera = (!!window.opr && !!opr.addons) || !!window.opera || navigator.userAgent.indexOf(' OPR/') >= 0;
// Firefox 1.0+
var isFirefox = typeof InstallTrigger !== 'undefined';
// At least Safari 3+: "[object HTMLElementConstructor]"
var isSafari = Object.prototype.toString.call(window.HTMLElement).indexOf('Constructor') > 0;
// Internet Explorer 6-11
var isIE = /*@cc_on!@*/ false || !!document.documentMode;
// Edge 20+
var isEdge = !isIE && !!window.StyleMedia;