Merge branch 'MetaGer-Redesign' into 'development'

Meta ger redesign

See merge request !1198

.gitignore

@@ -16,3 +16,5 @@ langfiles.zip
 **/*.map
 /.buildpath
 /.project
+
+composer.lock
\ No newline at end of file

.gitlab-ci.yml

@@ -29,8 +29,8 @@ update(144.76.113.134):
     - chmod -R 777 storage
     - chmod -R 777 bootstrap/cache
     - npm install
+    - npm run prodution
     - php artisan migrate --force
-    - npm run prod
     - if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
     - cd ~/
     - while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
@@ -69,8 +69,8 @@ update(metager2):
     - chmod -R 777 storage
     - chmod -R 777 bootstrap/cache
     - npm install
+    - npm run prodution
     - php artisan migrate --force
-    - npm run prod
     - if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
     - cd ~/
     - while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
@@ -109,8 +109,8 @@ update(metager3.de):
     - chmod -R 777 storage
     - chmod -R 777 bootstrap/cache
     - npm install
+    - npm run prodution
     - php artisan migrate --force
-    - npm run prod
     - if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
     - cd ~/
     - while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done

app/Console/Kernel.php

@@ -35,5 +35,6 @@ class Kernel extends ConsoleKernel
     protected function commands()
     {
         require base_path('routes/console.php');
+        $this->load(__DIR__.'/Commands');
     }
 }

app/Http/Controllers/FokiLoader.php

+<?php
+
+namespace App\Http\Controllers;
+
+use App;
+
+class FokiLoader
+{
+    public static function loadFoki()
+    {
+        $sumaFile = "";
+        if (App::isLocale('en')) {
+            $sumaFile = config_path() . "/sumas.xml";
+        } else {
+            $sumaFile = config_path() . "/sumas.xml";
+        }
+
+        $xml   = simplexml_load_file($sumaFile);
+        $sumas = $xml->xpath("suma");
+
+        $foki = [];
+        foreach ($sumas as $suma) {
+            if ((!isset($suma['disabled']) || $suma['disabled'] === "") && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
+                if (isset($suma['type'])) {
+                    $f = explode(",", $suma['type']->__toString());
+                    foreach ($f as $tmp) {
+                        $displayName                             = $suma['displayName']->__toString();
+                        $url                                     = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
+                        $foki[$tmp][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
+                    }
+                } else {
+                    $displayName                                 = $suma['displayName']->__toString();
+                    $url                                         = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
+                    $foki["andere"][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
+                }
+            }
+        }
+
+        return $foki;
+    }
+}

app/Http/Controllers/MailController.php

@@ -49,7 +49,7 @@ class MailController extends Controller
             $replyTo = $request->input('email');
         }
 
-        if (!$request->has('message') || !$request->has('subject')) {
+        if (!$request->filled('message') || !$request->filled('subject')) {
             $messageType   = "error";
             $returnMessage = "Tut uns leid, aber leider haben wir mit Ihrer Kontaktanfrage keine Daten erhalten. Die Nachricht wurde nicht versandt.";
         } else {
@@ -65,7 +65,7 @@ class MailController extends Controller
 
         return view('kontakt.kontakt')
             ->with('title', 'Kontakt')
-            ->with('js', ['openpgp.min.js', 'kontakt.js'])
+            ->with('js', ['lib.js'])
             ->with($messageType, $returnMessage);
     }
 
@@ -95,7 +95,7 @@ class MailController extends Controller
             # Kontonummer ( IBAN )
             # Bankleitzahl ( BIC )
             # Nachricht
-            if (!$request->has('Kontonummer') || !$request->has('Bankleitzahl') || !$request->has('Betrag')) {
+            if (!$request->filled('Kontonummer') || !$request->filled('Bankleitzahl') || !$request->filled('Betrag')) {
                 $messageToUser = "Sie haben eins der folgenden Felder nicht ausgefüllt: IBAN, BIC, Nachricht. Bitte korrigieren Sie Ihre Eingabe und versuchen es erneut.\n";
                 $messageType   = "error";
             } else {

app/Http/Controllers/MetaGerSearch.php

@@ -11,6 +11,7 @@ class MetaGerSearch extends Controller
 {
     public function search(Request $request, MetaGer $metager)
     {
+
         $focus = $request->input("focus", "web");
         
         if ($focus === "maps") {
@@ -18,10 +19,11 @@ class MetaGerSearch extends Controller
             return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
         }
 
-        if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
+        /*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
             $metager->parseFormData($request);
             return $metager->createView();
-        }
+        }*/
+
         #die($request->header('User-Agent'));
         $time = microtime();
         # Mit gelieferte Formulardaten parsen und abspeichern:
@@ -30,9 +32,16 @@ class MetaGerSearch extends Controller
         # Nach Spezialsuchen überprüfen:
         $metager->checkSpecialSearches($request);
 
-        # Alle Suchmaschinen erstellen
+        # Die Quicktips als Job erstellen
+        $quicktips = $metager->createQuicktips();
+
+        # Suche für alle zu verwendenden Suchmaschinen als Job erstellen, 
+        # auf Ergebnisse warten und die Ergebnisse laden
         $metager->createSearchEngines($request);
 
+        # Versuchen die Ergebnisse der Quicktips zu laden
+        $quicktipResults = $quicktips->loadResults();
+
         # Alle Ergebnisse vor der Zusammenführung ranken:
         $metager->rankAll();
 
@@ -40,7 +49,7 @@ class MetaGerSearch extends Controller
         $metager->prepareResults();
 
         # Die Ausgabe erstellen:
-        return $metager->createView();
+        return $metager->createView($quicktipResults);
     }
 
     public function botProtection($redirect)

app/Http/Controllers/Pictureproxy.php

@@ -10,7 +10,7 @@ class Pictureproxy extends Controller
 {
     public function get(Request $request)
     {
-        if ($request->has('url')) {
+        if ($request->filled('url')) {
             try {
                 $arrContextOptions = array(
                     "ssl" => array(

app/Http/Controllers/StartpageController.php

@@ -22,15 +22,21 @@ class StartpageController extends Controller
         $focusPages = [];
         $theme      = "default";
 
-        foreach ($request->all() as $key => $value) {
-            if ($value === 'on' && $key != 'param_sprueche' && $key != 'param_newtab' && $key !== 'param_maps' && $key !== 'param_autocomplete' && $key !== 'param_lang') {
-                $focusPages[] = str_replace('param_', '', $key);
-            }
-            if ($key === 'param_theme') {
-                $theme = str_replace('param_', '', $key);
+        $optionParams = ['param_sprueche', 'param_newtab', 'param_maps', 'param_autocomplete', 'param_lang', 'param_key'];
+        $option_values = [];
+
+        foreach ($optionParams as $param) {
+            $value = $request->input($param);
+            if ($value) {
+                $option_values[$param] = $value;
             }
         }
 
+        $autocomplete = 'on';
+        if(in_array('autocomplete', array_keys($option_values))) {
+            $autocomplete = $option_values['autocomplete'];
+        }
+
         $lang = LaravelLocalization::getCurrentLocale();
         if ($lang === 'de') {
             $lang = 'all';
@@ -39,20 +45,13 @@ class StartpageController extends Controller
         return view('index')
             ->with('title', trans('titles.index'))
             ->with('homeIcon')
-            ->with('focusPages', $focusPages)
             ->with('browser', (new Agent())->browser())
             ->with('navbarFocus', 'suche')
-            ->with('theme', $theme)
-            ->with('autocomplete', $request->input('param_autocomplete', 'on'))
-            ->with('foki', $this->loadFoki())
             ->with('focus', $request->input('focus', 'web'))
-            ->with('lang', $request->input('param_lang', $lang))
-            ->with('resultCount', $request->input('param_resultCount', '20'))
             ->with('time', $request->input('param_time', '1500'))
-            ->with('sprueche', $request->input('param_sprueche', 'on'))
-            ->with('newtab', $request->input('param_newtab', 'on'))
-            ->with('maps', $maps = $request->input('param_maps', 'off'))
-            ->with('key', $request->input('param_key', ''));
+            ->with('request', $request->input('request', 'GET'))
+            ->with('option_values', $option_values)
+            ->with('autocomplete', $autocomplete);
     }
 
     public function loadPage($subpage)
@@ -123,10 +122,7 @@ class StartpageController extends Controller
 
     public function loadSettings(Request $request)
     {
-        $foki = $this->loadFoki();
-
         return view('settings')
-            ->with('foki', $foki)
             ->with('title', 'Einstellungen')
             ->with('js', ['settings.js'])
             ->with('navbarFocus', 'suche');
@@ -139,7 +135,7 @@ class StartpageController extends Controller
     {
         $link     = "";
         $password = "";
-        if ($request->has('eingabe')) {
+        if ($request->filled('eingabe')) {
             $password = getenv('berlin');
             $password = md5($request->input('eingabe') . " -host:userpage.fu-berlin.de" . $password);
             $link     = "/meta/meta.ger3?eingabe=" . $request->input('eingabe') . " -host:userpage.fu-berlin.de&focus=web&password=" . $password . "&encoding=utf8&lang=all&site=fu-berlin.de&quicktips=off&out=results-with-style";
@@ -149,37 +145,4 @@ class StartpageController extends Controller
             ->with('link', $link)
             ->with('password', $password);
     }
-
-    private function loadFoki()
-    {
-        $sumaFile = "";
-        if (App::isLocale('en')) {
-            $sumaFile = config_path() . "/sumas.xml";
-        } else {
-            $sumaFile = config_path() . "/sumas.xml";
-        }
-
-        $xml   = simplexml_load_file($sumaFile);
-        $sumas = $xml->xpath("suma");
-
-        $foki = [];
-        foreach ($sumas as $suma) {
-            if ((!isset($suma['disabled']) || $suma['disabled'] === "") && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
-                if (isset($suma['type'])) {
-                    $f = explode(",", $suma['type']->__toString());
-                    foreach ($f as $tmp) {
-                        $displayName                             = $suma['displayName']->__toString();
-                        $url                                     = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
-                        $foki[$tmp][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
-                    }
-                } else {
-                    $displayName                                 = $suma['displayName']->__toString();
-                    $url                                         = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
-                    $foki["andere"][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
-                }
-            }
-        }
-
-        return $foki;
-    }
 }

app/Http/Controllers/ZitatController.php

@@ -9,7 +9,7 @@ class ZitatController extends Controller
     public function zitatSuche(Request $request)
     {
         $validResults = [];
-        if ($request->has("q")) {
+        if ($request->filled("q")) {
             # The user searched for something
             $fileName    = storage_path() . "/app/public/zitate.txt";
             $fileContent = file_get_contents($fileName);

app/Jobs/Searcher.php

@@ -73,13 +73,14 @@ class Searcher implements ShouldQueue
                 $mission = $mission[1];
                 $poptime = microtime(true) - $time;
 
-                // The mission is a String which can be divided to retrieve two informations:
+                // The mission is a String which can be divided to retrieve three informations:
                 // 1. The Hash Value where the result should be stored
                 // 2. The Url to Retrieve
-                // These two informations are divided by a ";" in the mission string
+                // 3. The maximum time to take
+                // These three informations are divided by a ";" in the mission string
                 $mission = explode(";", $mission);
-                $hashValue = $mission[0];
-                $url = base64_decode($mission[1]);
+                $hashValue = $mission[0]; // The hash value for redis to store the results under
+                $url = base64_decode($mission[1]); // The url to fetch
                 $timeout = $mission[2]; // Timeout from the MetaGer process in ms
                 $medianFetchTime = $this->getFetchTime();   // The median Fetch time of the search engine in ms
                 Redis::hset('search.' . $hashValue, $this->name, "connected");

app/Models/Quicktips/Quicktip.php

+<?php
+
+namespace App\Models\Quicktips;
+
+class Quicktip
+{
+    public $type;
+    public $title;
+    public $link;
+    public $gefVonTitle;
+    public $gefVonLink;
+    public $descr;
+    public $details;
+
+    # Erstellt ein neues Ergebnis
+    public function __construct($type, $title, $link, $gefVonTitle, $gefVonLink, $descr, $details)
+    {
+        $this->type = $type;
+        $this->title = $title;
+        $this->link = $link;
+        $this->gefVonTitle = $gefVonTitle;
+        $this->gefVonLink = $gefVonLink;
+        $this->descr = $descr;
+        $this->details = $details;
+    }
+}

app/Models/Quicktips/Quicktip_detail.php

+<?php
+
+namespace App\Models\Quicktips;
+
+class Quicktip_detail
+{
+    public $title;
+    public $link;
+    public $descr;
+
+    public function __construct($title, $link, $descr)
+    {
+        $this->title = $title;
+        $this->link = $link;
+        $this->descr = $descr;
+    }
+}

app/Models/Quicktips/Quicktips.php

+<?php
+
+namespace App\Models\Quicktips;
+
+use App\Jobs\Searcher;
+use Cache;
+use Illuminate\Foundation\Bus\DispatchesJobs;
+use Illuminate\Support\Facades\Redis;
+use Log;
+
+class Quicktips
+{
+    use DispatchesJobs;
+
+    const QUICKTIP_URL = "https://quicktips.metager3.de/quicktips.xml";
+    const QUICKTIP_NAME = "quicktips";
+    const CACHE_DURATION = 60;
+
+    private $hash;
+
+    public function __construct($search, $locale, $max_time)
+    {
+        $this->startSearch($search, $locale, $max_time);
+    }
+
+    public function startSearch($search, $locale, $max_time)
+    {
+        $url = self::QUICKTIP_URL . "?search=" . $search . "&locale=" . $locale;
+
+        $hash = md5($url);
+
+        # TODO anders weitergeben
+        $this->hash = $hash;
+
+        # TODO cache wieder einbauen (eventuell)
+        if ( /*!Cache::has($hash)*/true) {
+            Redis::hset("search." . $hash, self::QUICKTIP_NAME, "waiting");
+
+            // Queue this search
+            $mission = $hash . ";" . base64_encode($url) . ";" . $max_time;
+            Redis::rpush(self::QUICKTIP_NAME . ".queue", $mission);
+
+            // Check the current status of Searchers for QUICKTIP_NAME
+            $needSearcher = false;
+            $searcherData = Redis::hgetall(self::QUICKTIP_NAME . ".stats");
+
+            // Create additional Searchers for QUICKTIP_NAME if necessary
+            if (sizeof($searcherData) === 0) {
+                $needSearcher = true;
+            } else {
+                $median = 0;
+                foreach ($searcherData as $pid => $data) {
+                    $data = explode(";", $data);
+                    $median += floatval($data[1]);
+                }
+                $median /= sizeof($searcherData);
+                if ($median < .1) {
+                    $needSearcher = true;
+                }
+            }
+            if ($needSearcher && Redis::get(self::QUICKTIP_NAME) !== "locked") {
+                Redis::set(self::QUICKTIP_NAME, "locked");
+                $this->dispatch(new Searcher(self::QUICKTIP_NAME));
+            }
+        }
+    }
+
+    public function loadResults()
+    {
+        $resultsRaw = $this->retrieveResults($this->hash);
+        $results = $this->parseResults($resultsRaw);
+        return $results;
+    }
+
+    public function retrieveResults($hash)
+    {
+        $body = "";
+        #if (Cache::has($hash)) {
+        $body = Cache::get($hash);
+        #} elseif (Redis::hexists('search.' . $hash, self::QUICKTIP_NAME)) {
+        $body = Redis::hget('search.' . $hash, self::QUICKTIP_NAME);
+        Redis::hdel('search.' . $hash, self::QUICKTIP_NAME);
+        Cache::put($hash, $body, self::CACHE_DURATION);
+        #}
+        if ($body !== "") {
+            return $body;
+        } else {
+            return false;
+        }
+    }
+
+    public function parseResults($quicktips_raw)
+    {
+        $quicktips_raw = preg_replace("/\r\n/si", "", $quicktips_raw);
+        try {
+            $content = simplexml_load_string($quicktips_raw);
+            if (!$content) {
+                return;
+            }
+
+            $content->registerXPathNamespace('main', 'http://www.w3.org/2005/Atom');
+
+            $quicktips = [];
+
+            $quicktips_xpath = $content->xpath('main:entry');
+            foreach ($quicktips_xpath as $quicktip_xml) {
+                // Type
+                $quicktip_xml->registerXPathNamespace('mg', 'http://metager.de/opensearch/quicktips/');
+                $types = $quicktip_xml->xpath('mg:type');
+                if (sizeof($types) > 0) {
+                    $type = $types[0]->__toString();
+                } else {
+                    $type = "";
+                }
+
+                // Title
+                $title = $quicktip_xml->title->__toString();
+
+                // Link
+                $link = $quicktip_xml->link['href']->__toString();
+
+                // gefVon
+                $quicktip_xml->registerXPathNamespace('mg', 'http://metager.de/opensearch/quicktips/');
+                $gefVonTitles = $quicktip_xml->xpath('mg:gefVonTitle');
+                if (sizeof($gefVonTitles) > 0) {
+                    $gefVonTitle = $gefVonTitles[0]->__toString();
+                } else {
+                    $gefVonTitle = "";
+                }
+                $gefVonLinks = $quicktip_xml->xpath('mg:gefVonLink');
+                if (sizeof($gefVonLinks) > 0) {
+                    $gefVonLink = $gefVonLinks[0]->__toString();
+                } else {
+                    $gefVonLink = "";
+                }
+
+                // Description
+                $descr = $quicktip_xml->content->__toString();
+
+                // Details
+                $details = [];
+                $details_xpath = $quicktip_xml->xpath('mg:details');
+                if (sizeof($details_xpath) > 0) {
+                    foreach ($details_xpath[0] as $detail_xml) {
+                        $details_title = $detail_xml->title->__toString();
+                        $details_link = $detail_xml->url->__toString();
+                        $details_descr = $detail_xml->text->__toString();
+                        $details[] = new \App\Models\Quicktips\Quicktip_detail(
+                            $details_title,
+                            $details_link,
+                            $details_descr
+                        );
+                    }
+                }
+                $quicktips[] = new \App\Models\Quicktips\Quicktip(
+                    $type,
+                    $title,
+                    $link,
+                    $gefVonTitle,
+                    $gefVonLink,
+                    $descr,
+                    $details
+                );
+            }
+            return $quicktips;
+        } catch (\Exception $e) {
+            Log::error("A problem occurred parsing quicktips");
+            return [];
+        }
+    }
+}

app/Models/Result.php

@@ -13,7 +13,8 @@ class Result
     public $anzeigeLink; # Der tatsächlich angezeigte Link (rein optisch)
     public $descr; # Die eventuell gekürzte Beschreibung des Suchergebnisses
     public $longDescr; # Die ungekürzte Beschreibung des Suchergebnisses
-    public $gefVon; # Das bei Suchergebnissen angezeigte von ... mitsamt Verlinkung
+    public $gefVon; # Die Suchmaschine von der dieses Ergebnis stammt
+    public $gefVonLink;
     public $sourceRank; # Das Ranking für dieses Suchergebnis von der Seite, die es geliefert hat (implizit durch Ergebnisreihenfolge: 20 - Position in Ergebnisliste)
     public $partnershop; # Ist das Ergebnis von einem Partnershop? (bool)
     public $image; # Ein Vorschaubild für das Suchergebnis (als URL)
@@ -22,14 +23,13 @@ class Result
     public $engineBoost = 1; # Der Boost für den Provider des Suchergebnisses
     public $valid = true; # Ob das Ergebnis noch gültig ist (bool)
     public $host; # Der aus dem Link gelesene Host des Suchergebnisses
-    public $strippedHost; # Der Host      in Form "foo.bar.de"
-    public $strippedDomain; # Die Domain    in Form "bar.de"
-    public $strippedLink; # Der Link      in Form "foo.bar.de/test"
+    public $strippedHost; # Der Host in Form "foo.bar.de"
+    public $strippedDomain; # Die Domain in Form "bar.de"
+    public $strippedLink; # Der Link in Form "foo.bar.de/test"
     public $rank; # Das Ranking für das Ergebnis
 
     # Erstellt ein neues Ergebnis
-    #public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $sourceRank, $partnershop = false, $image = "", $price = 0, $additionalInformation = [])
-    public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $sourceRank, $additionalInformation = [])
+    public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
     {
         $provider = simplexml_load_string($provider);
         $this->titel = strip_tags(trim($titel));
@@ -44,6 +44,7 @@ class Result
 
         }
         $this->gefVon = trim($gefVon);
+        $this->gefVonLink = trim($gefVonLink);
         $this->proxyLink = $this->generateProxyLink($this->link);
         $this->sourceRank = $sourceRank;
         if ($this->sourceRank <= 0 || $this->sourceRank > 20) {

app/Models/Searchengine.php

@@ -23,7 +23,6 @@ abstract class Searchengine
     public $cached   = false;
 
     public $ip; # Die IP aus der metager
-    public $gefVon; # Der HTML-Code für die Verlinkung des Suchanbieters
     public $uses; # Die Anzahl der Nutzungen dieser Suchmaschine
     public $homepage; # Die Homepage dieser Suchmaschine
     public $name; # Der Name dieser Suchmaschine
@@ -71,7 +70,6 @@ abstract class Searchengine
 
         $this->useragent = $metager->getUserAgent();
         $this->ip        = $metager->getIp();
-        $this->gefVon    = "<a href=\"" . $this->homepage . "\" target=\"_blank\" rel=\"noopener\">" . $this->displayName . "</a>";
         $this->startTime = microtime();
 
         # Suchstring generieren
@@ -219,7 +217,6 @@ abstract class Searchengine
     {
         $number = Redis::hget('search.' . $this->hash, $this->name);
         if ($number === null) {
-            die("test");
             return null;
         } else {
             return pfsockopen($this->getHost() . ":" . $this->port . "/$number", $this->port, $errstr, $errno, 1);

app/Models/parserSkripte/Allesklar.php

@@ -43,7 +43,7 @@ class Allesklar extends Searchengine
                         $link,
                         $link,
                         $descr,
-                        $this->gefVon,
+                        $this->displayName,$this->homepage,
                         $this->counter
                     );
                 } catch (\Exception $e) {

app/Models/parserSkripte/BASE.php

@@ -49,7 +49,7 @@ class BASE extends Searchengine