Merge branch 'development' into 408-arbeit-am-ubersetzungstool

.gitignore

@@ -5,3 +5,4 @@
 Homestead.json
 Homestead.yaml
 .env
+.orig

app/Jobs/Search.php

@@ -38,13 +38,32 @@ class Search implements ShouldQueue
      */
     public function handle()
     {
-        $this->fp = $this->getFreeSocket();
-
-        if ($this->fp) {
-            if ($this->writeRequest()) {
-                $this->readAnswer();
-            }
+        $url = "";
+        if($this->port === "443"){
+            $url = "https://";
+        }else{
+            $url = "http://";
         }
+        $url .= $this->host . $this->getString;
+
+
+        $ch = curl_init($url);
+        curl_setopt_array($ch, array(
+                CURLOPT_RETURNTRANSFER => 1,
+                CURLOPT_URL => $url,
+                CURLOPT_USERAGENT => $this->useragent,
+                CURLOPT_FOLLOWLOCATION => TRUE,
+                CURLOPT_CONNECTTIMEOUT => 10,
+                CURLOPT_MAXCONNECTS => 50,
+                CURLOPT_LOW_SPEED_LIMIT => 500,
+                CURLOPT_LOW_SPEED_TIME => 5,
+                CURLOPT_TIMEOUT => 10
+        ));
+
+        $result = curl_exec($ch);
+
+        curl_close($ch);
+        Redis::hset('search.' . $this->hash, $this->name, $result);
     }
 
     private function readAnswer()

app/Jobs/Searcher.php

+<?php
+
+namespace App\Jobs;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\DispatchesJobs;
+use Illuminate\Support\Facades\Redis;
+use Log;
+
+class Searcher implements ShouldQueue
+{
+    use InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $name, $ch, $pid, $counter, $lastTime;
+    protected $MAX_REQUESTS = 500;
+
+    /**
+     * Create a new job instance.
+     * This is our new Worker/Searcher Class
+     * It will take it's name from the sumas.xml as constructor argument
+     * Each Searcher is dedicated to one remote server from our supported Searchengines
+     * It will listen to a queue in the Redis Database within the handle() method and
+     * answer requests to this specific search engine.
+     * The curl handle will be left initialized and opened so that we can make full use of
+     * keep-alive requests.
+     * @return void
+     */
+    public function __construct($name)
+    {
+        $this->name = $name;
+        $this->pid = getmypid();
+        // Submit this worker to the Redis System
+        Redis::set($this->name, "running");
+        Redis::expire($this->name, 5);
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        // This Searches is freshly called so we need to initialize the curl handle $ch
+        $this->ch = $this->initCurlHandle();
+        $this->counter = 0;                 // Counts the number of answered jobs
+        $time = microtime(true);
+        while(true){
+            // Update the expire
+            Redis::expire($this->name, 5);
+            // One Searcher can handle a ton of requests to the same server
+            // Each search to the server of this Searcher will be submitted to a queue
+            // stored in redis which has the same name as this searchengine appended by a ".queue"
+            // We will perform a blocking pop on this queue so the queue can remain empty for a while 
+            // without killing this searcher directly.
+            $mission = Redis::blpop($this->name . ".queue", 4);
+
+            // The mission can be empty when blpop hit the timeout
+            if(empty($mission)){
+                continue;
+            }else{
+                $mission = $mission[1];
+                $this->counter++;#
+                $poptime = microtime(true) - $time;
+                $time = microtime(true);
+            }
+
+            // The mission is a String which can be divided to retrieve two informations:
+            // 1. The Hash Value where the result should be stored
+            // 2. The Url to Retrieve
+            // These two informations are divided by a ";" in the mission string
+            $hashValue = substr($mission, 0, strpos($mission, ";"));
+            $url = substr($mission, strpos($mission, ";") + 1);
+
+            Redis::hset('search.' . $hashValue, $this->name, "connected");
+
+            $result = $this->retrieveUrl($url);
+
+            $this->storeResult($result, $poptime, $hashValue);
+
+            // In sync mode every Searcher may only retrieve one result because it would block
+            // the execution of the remaining code otherwise:
+            if(getenv("QUEUE_DRIVER") === "sync" || $this->counter > $this->MAX_REQUESTS){
+                break;
+            } 
+        }
+        // When we reach this point, time has come for this Searcher to retire
+        $this->exit();
+    }
+
+    private function retrieveUrl($url){
+        // Set this URL to the Curl handle
+        curl_setopt($this->ch, CURLOPT_URL, $url);
+
+        $result = curl_exec($this->ch);
+
+        return $result;
+    }
+
+    private function storeResult($result, $poptime, $hashValue){
+        Redis::hset('search.' . $hashValue, $this->name, $result);
+        $connectionInfo = base64_encode(json_encode(curl_getinfo($this->ch), true));
+        Redis::hset($this->name . ".stats", $this->pid, $connectionInfo . ";" . $poptime);
+        $this->lastTime = microtime(true);
+    }
+
+    private function exit(){
+        Redis::hdel($this->name . ".stats", $this->pid);
+        if(sizeof(Redis::hgetall($this->name . ".stats")) === 0){
+            Redis::del($this->name);
+        }
+        // We should close our curl handle before we do so
+        curl_close($this->ch);
+    }
+
+    private function initCurlHandle(){
+        $ch = curl_init();
+
+        curl_setopt_array($ch, array(
+                CURLOPT_RETURNTRANSFER => 1,
+                CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
+                CURLOPT_FOLLOWLOCATION => TRUE,
+                CURLOPT_CONNECTTIMEOUT => 10,
+                CURLOPT_MAXCONNECTS => 500,
+                CURLOPT_LOW_SPEED_LIMIT => 500,
+                CURLOPT_LOW_SPEED_TIME => 5,
+                CURLOPT_TIMEOUT => 10
+        ));
+
+        return $ch;
+    }
+}

app/MetaGer.php

@@ -813,39 +813,41 @@ class MetaGer
 
     public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
     {
-        $loadedEngines = 0;
-        $timeStart     = microtime(true);
-
-        # Auf wie viele Suchmaschinen warten wir?
-        $engineCount = count($enginesToLoad);
 
+        $timeStart     = microtime(true);
+        $results = null;
         while (true) {
-            $time          = (microtime(true) - $timeStart) * 1000;
-            $loadedEngines = intval(Redis::hlen('search.' . $this->getHashCode()));
-            if ($overtureEnabled && (Redis::hexists('search.' . $this->getHashCode(), 'overture') || Redis::hexists('search.' . $this->getHashCode(), 'overtureAds'))) {
-                $canBreak = true;
-            }
-
-            # Abbruchbedingung
-            if ($time < 500) {
-                if (($engineCount === 0 || $loadedEngines >= $engineCount) && $canBreak) {
-                    break;
+            $results = Redis::hgetall('search.' . $this->getHashCode());
+
+            $ready = true;
+            // When every 
+            $connected = true;
+            foreach($results as $key => $value){
+                if($value === "waiting" || $value === "connected"){
+                    $ready = false;
                 }
-
-            } elseif ($time >= 500 && $time < $this->time) {
-                if (($engineCount === 0 || ($loadedEngines / ($engineCount * 1.0)) >= 0.8) && $canBreak) {
-                    break;
+                if($value === "waiting"){
+                    $connected = false;
                 }
+            }
 
-            } else {
+            // If $ready is false at this point, we're waiting for more searchengines
+            // But we have to check for the timeout, too
+            if(!$connected) $timeStart = microtime(true);
+            $time = (microtime(true) - $timeStart) * 1000;
+            // We will apply the timeout only if it's not Yahoo we're waiting for since they are one the most
+            // important search engines.
+            $canTimeout = !((isset($results["overture"]) && $results["overture"] === "waiting") || (isset($results["overtureAds"]) && $results["overtureAds"] === "waiting"));
+            if($time > $this->time && $canTimeout) $ready = true;
+
+            if($ready){
                 break;
             }
             usleep(50000);
         }
 
         # Wir haben nun so lange wie möglich gewartet. Wir registrieren nun noch die Suchmaschinen, die geanwortet haben.
-        $answered = Redis::hgetall('search.' . $this->getHashCode());
-        foreach ($answered as $key => $value) {
+        foreach ($results as $key => $value) {
             $enginesToLoad[$key] = true;
         }
         $this->enginesToLoad = $enginesToLoad;
@@ -882,7 +884,7 @@ class MetaGer
     {
         $this->request = $request;
         # Sichert, dass der request in UTF-8 formatiert ist
-        if ($request->input('encoding', '') !== "utf8") {
+        if ($request->input('encoding', 'utf8') !== "utf8") {
             # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
             $input = $request->all();
             foreach ($input as $key => $value) {
@@ -921,7 +923,7 @@ class MetaGer
         # Category
         $this->category = $request->input('category', '');
         # Request Times
-        $this->time = $request->input('time', 1000);
+        $this->time = $request->input('time', 1500);
         # Page
         $this->page = 1;
         # Lang

app/Models/Searchengine.php

@@ -2,7 +2,7 @@
 
 namespace App\Models;
 
-use App\Jobs\Search;
+use App\Jobs\Searcher;
 use App\MetaGer;
 use Cache;
 use Illuminate\Foundation\Bus\DispatchesJobs;
@@ -105,17 +105,63 @@ abstract class Searchengine
     # Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
     public function startSearch(\App\MetaGer $metager)
     {
-        if ($this->canCache && Cache::has($this->hash)) {
+        if ($this->canCache && Cache::has($this->hash) && 0 == 1) {
             $this->cached = true;
             $this->retrieveResults($metager);
         } else {
-            /* Die Anfragen an die Suchmaschinen werden nun von der Laravel-Queue bearbeitet:
-             *  Hinweis: solange in der .env der QUEUE_DRIVER auf "sync" gestellt ist, werden die Abfragen
-             *  nacheinander abgeschickt.
-             *  Sollen diese Parallel verarbeitet werden, muss ein anderer QUEUE_DRIVER verwendet werden.
-             *  siehe auch: https://laravel.com/docs/5.2/queues
-             */
-            $this->dispatch(new Search($this->resultHash, $this->host, $this->port, $this->name, $this->getString, $this->useragent, $this->additionalHeaders));
+            // We will push the confirmation of the submission to the Result Hash
+            Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
+            // We need to submit a action that one of our workers can understand
+            // The missions are submitted to a redis queue in the following string format
+            // <ResultHash>;<URL to fetch>
+            // With <ResultHash> being the Hash Value where the fetcher will store the result.
+            // and <URL to fetch> being the full URL to the searchengine
+            $url = "";
+            if($this->port === "443"){
+                $url = "https://";
+            }else{
+                $url = "http://";
+            }
+            $url .= $this->host . $this->getString;
+            $mission = $this->resultHash . ";" . $url;
+            // Submit this mission to the corresponding Redis Queue
+            // Since each Searcher is dedicated to one specific search engine
+            // each Searcher has it's own queue lying under the redis key <name>.queue
+            Redis::rpush($this->name . ".queue", $mission);
+
+            /**
+            * We have Searcher processes running for MetaGer
+            * Each Searcher is dedicated to one specific Searchengine and fetches it's results.
+            * We can have multiple Searchers for each engine, if needed.
+            * At this point we need to decide, whether we need to start a new Searcher process or
+            * if we have enough of them running.
+            * The information for that is provided through the redis system. Each running searcher 
+            * gives information how long it has waited to be given the last fetcher job.
+            * The longer this time value is, the less frequent the search engine is used and the less
+            * searcher of that type we need.
+            * But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
+            **/
+            $needSearcher = false;
+            $searcherData = Redis::hgetall($this->name . ".stats");
+
+            // We now have an array of statistical data from the searchers
+            // Each searcher has one entry in it.
+            // So if it's empty, then we have currently no searcher running and 
+            // of course need to spawn a new one.
+            if(sizeof($searcherData) === 0){
+                $needSearcher = true;
+            }else{
+                // There we go:
+                // There's at least one Fetcher running for this search engine.
+                // Now we have to check if the current count is enough to fetch all the
+                // searches or if it needs help.
+                // Let's hardcode a minimum of 100ms between every search job.
+                die(var_dump($searcherData));
+            }
+            
+            if($needSearcher){
+                $this->dispatch(new Searcher($this->name));
+            }
         }
     }
 
@@ -172,11 +218,11 @@ abstract class Searchengine
         }
 
         $body = "";
-        if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
+        if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
             $body = Cache::get($this->hash);
         } elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
             $body = Redis::hget('search.' . $this->resultHash, $this->name);
-            if ($this->canCache && $this->cacheDuration > 0) {
+            if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
                 Cache::put($this->hash, $body, $this->cacheDuration);
             }
 

public/build/js/scriptResultPage-95d76cf16b.js → public/build/js/scriptResultPage-357d27fe23.js

@@ -90,7 +90,7 @@ function popovers() {
         $(this).popover("destroy");
         $(this).popover({
             //html          :   true,
-            //title         :   "<span class='glyphicon glyphicon-cog'></span> Optionen",
+            //title         :   "<i class="fa fa-cog" aria-hidden="true"></i> Optionen",
             content: $(this).parent().find(".content").html()
         });
     });
@@ -229,8 +229,8 @@ function productWidget() {
             easing: 'cubic-bezier(0.25, 0, 0.25, 1)',
             speed: 600,
             pager: false,
-            prevHtml: '<span class="glyphicon glyphicon-chevron-left" aria-hidden="true"></span><span class="sr-only">Previous</span>',
-            nextHtml: '<span class="glyphicon glyphicon-chevron-right" aria-hidden="true"></span><span class="sr-only">Next</span>',
+            prevHtml: '<i class="fa fa-chevron-left" aria-hidden="true"></i></span><span class="sr-only">Previous</span>',
+            nextHtml: '<i class="fa fa-chevron-right" aria-hidden="true"></i><span class="sr-only">Next</span>',
             responsive: [{
                 breakpoint: 1400,
                 settings: {
@@ -273,8 +273,8 @@ function productWidget() {
             pager: false,
             enableTouch: false,
             enableDrag: false,
-            prevHtml: '<span class="glyphicon glyphicon-chevron-left" aria-hidden="true"></span><span class="sr-only">Previous</span>',
-            nextHtml: '<span class="glyphicon glyphicon-chevron-right" aria-hidden="true"></span><span class="sr-only">Next</span>',
+            prevHtml: '<a class="fa fa-chevron-left" aria-hidden="true"></a><span class="sr-only">Previous</span>',
+            nextHtml: '<a class="fa fa-chevron-right" aria-hidden="true"></a><span class="sr-only">Next</span>',
             responsive: [{
                 breakpoint: 1400,
                 settings: {
@@ -332,14 +332,14 @@ function createCustomFocuses() {
  * @if( $metager->getFokus() === "produktsuche" )
  *     <li id="produktsucheTabSelector" class="active tab-selector" role="presentation" data-loaded="1">
  *        <a aria-controls="produktsuche" data-href="#produktsuche" href="#produktsuche">
- *             <span class='glyphicon glyphicon-shopping-cart'></span>
+ *             <i class="fa fa-shopping-cart" aria-hidden="true"></i>
  *             <span class="hidden-xs">{{ trans('index.foki.produkte') }}</span>
  *         </a>
  *     </li>
  * @else
  *     <li id="produktsucheTabSelector" class="tab-selector" role="presentation" data-loaded="0">
  *         <a aria-controls="produktsuche" data-href="{!! $metager->generateSearchLink('produktsuche') !!}" href="{!! $metager->generateSearchLink('produktsuche', false) !!}">
- *             <span class='glyphicon glyphicon-shopping-cart'></span>
+ *             <i class="fa fa-shopping-cart" aria-hidden="true"></i>
  *             <span class="hidden-xs">{{ trans('index.foki.produkte') }}</span>
  *         </a>
  *     </li>
@@ -366,10 +366,11 @@ function addFocus(focus, active = false) {
     var searchLink = generateSearchLinkForFocus(focus)
     focusElementLink.setAttribute("data-href", searchLink);
     focusElementLink.setAttribute("href", searchLink);
-    // create <span> glyphicon
-    var focusElementIcon = document.createElement("span");
-    focusElementIcon.classList.add("glyphicon");
-    focusElementIcon.classList.add("glyphicon-cog");
+    // create <a> icon
+    var focusElementIcon = document.createElement("a");
+    focusElementIcon.classList.add("fa");
+    focusElementIcon.classList.add("fa-cog");
+    focusElementIcon.setAttribute("aria-hidden", "true");
     // create <span> focusname
     var focusElementName = document.createElement("span");
     focusElementName.classList.add("hidden-xs");

public/build/js/scriptStartPage-330e649107.js

-$(document).ready(function() {
-    // checkPlugin();
-    if (location.href.indexOf("#plugin-modal") > -1) {
-        $("#plugin-modal").modal("show");
-    }
-    $('#addFocusBtn').removeClass('hide');
-    $("button").popover();
-    if (localStorage) {
-        var theme = localStorage.getItem("theme");
-        if (theme != null) {
-            if ((theme.match(/,/g) || []).length != 3) {
-                localStorage.removeItem("theme");
-            } else {
-                theme = theme.split(",");
-                $("#theme").attr("href", "/css/theme.css.php?r=" + theme[0] + "&g=" + theme[1] + "&b=" + theme[2] + "&a=" + theme[3]);
-            }
-        }
-        if (localStorage.getItem("pers") && !isUseOnce()) {
-            setSettings();
-        }
-    }
-    setActionListeners();
-    loadInitialCustomFocuses()
-});
-
-function setActionListeners() {
-    $("button").on("shown.bs.popover", function() {
-        $("#color-chooser a").click(function() {
-            var theme = $(this).attr("data-rgba");
-            if (localStorage) {
-                localStorage.setItem("theme", theme);
-                location.href = "/";
-            }
-        });
-    });
-    $("#mobileFoki").change(function() {
-        var focus = $("#mobileFoki > option:selected").val();
-        if (focus == "angepasst") {
-            window.location = "./settings/";
-        } else {
-            window.location = "./?focus=" + focus;
-        }
-    });
-    if ($("fieldset#foki.mobile").length) {
-        $("fieldset#foki.mobile label#anpassen-label").click(function() {
-            window.location = "./settings/";
-        });
-    }
-    $("#addFocusBtn").click(function() {
-        showFocusCreateDialog("");
-    });
-    $("#save-focus-btn").click(saveFocus);
-    $("#delete-focus-btn").click(deleteFocus);
-}
-
-function setSettings() {
-    for (var i = 0; i < localStorage.length; i++) {
-        var key = localStorage.key(i);
-        var value = localStorage.getItem(key);
-        if (key.startsWith("param_") && !key.endsWith("lang") && !key.endsWith('autocomplete')) {
-            key = key.substring(key.indexOf("param_") + 6);
-            $("#searchForm").append("<input type=\"hidden\" name=\"" + key + "\" value=\"" + value + "\">");
-        }
-        $("#foki input[type=radio]#angepasst").attr("checked", true);
-    }
-    if (localStorage.getItem("param_lang") !== null) {
-        var value = localStorage.getItem("param_lang");
-        // Change the value of the lang input field to the given parameter
-        $("input[name=lang]").val(value);
-    }
-    if (localStorage.getItem("param_autocomplete") !== null) {
-        var value = localStorage.getItem("param_autocomplete");
-        // Change the value of the lang input field to the given parameter
-        $("input[name=eingabe]").attr("autocomplete", value);
-    }
-    if ($("fieldset#foki.mobile").length) {
-        $("fieldset.mobile input#bilder").val("angepasst");
-        $("fieldset.mobile input#bilder").prop("checked", true);
-        $("fieldset.mobile input#bilder").attr("id", "angepasst");
-        $("fieldset.mobile label#bilder-label").attr("id", "anpassen-label");
-        $("fieldset.mobile label#anpassen-label").attr("for", "angepasst");
-        $("fieldset.mobile label#anpassen-label span.glyphicon").attr("class", "glyphicon glyphicon-cog");
-        $("fieldset.mobile label#anpassen-label span.content").html("angepasst");
-    }
-}
-//Polyfill for form attribute
-(function($) {
-    /**
-     * polyfill for html5 form attr
-     */
-    // detect if browser supports this
-    var sampleElement = $('[form]').get(0);
-    var isIE11 = !(window.ActiveXObject) && "ActiveXObject" in window;
-    if (sampleElement && window.HTMLFormElement && sampleElement.form instanceof HTMLFormElement && !isIE11) {
-        // browser supports it, no need to fix
-        return;
-    }
-    /**
-     * Append a field to a form
-     *
-     */
-    $.fn.appendField = function(data) {
-        // for form only
-        if (!this.is('form')) return;
-        // wrap data
-        if (!$.isArray(data) && data.name && data.value) {
-            data = [data];
-        }
-        var $form = this;
-        // attach new params
-        $.each(data, function(i, item) {
-            $('<input/>').attr('type', 'hidden').attr('name', item.name).val(item.value).appendTo($form);
-        });
-        return $form;
-    };
-    /**
-     * Find all input fields with form attribute point to jQuery object
-     * 
-     */
-    $('form[id]').submit(function(e) {
-        var $form = $(this);
-        // serialize data
-        var data = $('[form=' + $form.attr('id') + ']').serializeArray();
-        // append data to form
-        $form.appendField(data);
-    }).each(function() {
-        var form = this,
-            $form = $(form),
-            $fields = $('[form=' + $form.attr('id') + ']');
-        $fields.filter('button, input').filter('[type=reset],[type=submit]').click(function() {
-            var type = this.type.toLowerCase();
-            if (type === 'reset') {
-                // reset form
-                form.reset();
-                // for elements outside form
-                $fields.each(function() {
-                    this.value = this.defaultValue;
-                    this.checked = this.defaultChecked;
-                }).filter('select').each(function() {
-                    $(this).find('option').each(function() {
-                        this.selected = this.defaultSelected;
-                    });
-                });
-            } else if (type.match(/^submit|image$/i)) {
-                $(form).appendField({
-                    name: this.name,
-                    value: this.value
-                }).submit();
-            }
-        });
-    });
-})(jQuery);
-// Opera 8.0+
-var isOpera = (!!window.opr && !!opr.addons) || !!window.opera || navigator.userAgent.indexOf(' OPR/') >= 0;
-// Firefox 1.0+
-var isFirefox = typeof InstallTrigger !== 'undefined';
-// At least Safari 3+: "[object HTMLElementConstructor]"
-var isSafari = Object.prototype.toString.call(window.HTMLElement).indexOf('Constructor') > 0;
-// Internet Explorer 6-11
-var isIE = /*@cc_on!@*/ false || !!document.documentMode;
-// Edge 20+
-var isEdge = !isIE && !!window.StyleMedia;