Merge branch 'master' into '57-die-settingsseite-ubersetzen'

Updates from Master



See merge request !73

app/Http/Controllers/MetaGerSearch.php

@@ -15,6 +15,7 @@ class MetaGerSearch extends Controller
 {
     public function search(Request $request, MetaGer $metager)
     {
+        #die($request->header('User-Agent'));
         $time = microtime();
         # Mit gelieferte Formulardaten parsen und abspeichern:
         $metager->parseFormData($request);
@@ -43,10 +44,13 @@ class MetaGerSearch extends Controller
 
         # Zunächst den Spruch
         $spruecheFile = storage_path() . "/app/public/sprueche.txt";
-        if( file_exists($spruecheFile) )
+        if( file_exists($spruecheFile) && $_GET['sprueche'])
         {
             $sprueche = file($spruecheFile);
             $spruch = $sprueche[array_rand($sprueche)];
+        }else
+        {
+            $spruch = "";
         }
 
         # Die manuellen Quicktips:

app/Http/Controllers/Pictureproxy.php

+<?php
+
+namespace App\Http\Controllers;
+
+use App\Http\Controllers\Controller;
+use Illuminate\Http\Request;
+#use App\MetaGer\Forwarder;
+#use App\MetaGer\Results;
+#use App\MetaGer\Search;
+use App;
+use App\MetaGer;
+use Response;
+
+
+class Pictureproxy extends Controller
+{
+    function get(Request $request) {
+        if( $request->has('url') )
+        {
+            $file = file_get_contents($request->input('url'));
+            $responseCode = explode(" ", $http_response_header[0])[1];
+            $contentType = "";
+            foreach($http_response_header as $header)
+            {
+                if( strpos($header, "Content-Type:") === 0)
+                {
+                    $tmp = explode(": ", $header);
+                    $contentType = $tmp[1];
+                }
+            }
+            $response = Response::make($file, $responseCode);
+            $response->header('Content-Type', $contentType);
+            return $response;
+        }
+    } 
+
+}
\ No newline at end of file

app/Http/Controllers/StartpageController.php

@@ -4,6 +4,7 @@ namespace App\Http\Controllers;
 
 use App\Http\Controllers\Controller;
 use Illuminate\Http\Request;
+use Response;
 
 class StartpageController extends Controller
 {
@@ -64,4 +65,35 @@ class StartpageController extends Controller
         \App::setLocale($locale);
         return loadPage($subpage);
     }
+
+    public function loadPlugin(Request $request, $locale = "de")
+    {
+        $requests = $request->all();
+        $params = [];
+        foreach($requests as $key => $value)
+        {
+            if( strpos($key, "param_") === 0 )
+            {
+                $key = substr($key, strpos($key, "param_") + 6 );
+            }
+            $params[$key] = $value;
+        }
+
+        if(!isset($params['focus']))
+            $params['focus'] = 'web';
+        if(!isset($params['encoding']))
+            $params['encoding'] = 'utf8';
+        if(!isset($params['lang']))
+            $params['lang'] = 'all';
+        $params["eingabe"] = "";
+
+
+        $link = action('MetaGerSearch@search', $params);
+
+        $response = Response::make(
+            view('plugin')->with('link', $link), "200");
+        $response->header('Content-Type', "application/xml");
+        return $response;
+        return $link;
+    }
 }
\ No newline at end of file

app/Http/routes.php

@@ -97,7 +97,9 @@
 
         
         Route::get('meta/meta.ger3', 'MetaGerSearch@search');
+        Route::get('meta/picture', 'Pictureproxy@get');
 
         Route::get('qt', 'MetaGerSearch@quicktips');
         Route::get('tips', 'MetaGerSearch@tips');
+        Route::get('opensearch.xml', 'StartpageController@loadPlugin');
     });

app/MetaGer.php

@@ -6,6 +6,8 @@ use Jenssegers\Agent\Agent;
 use App;
 use Storage;
 use Log;
+use Config;
+use Redis;
 use App\lib\TextLanguageDetect\TextLanguageDetect;
 use App\lib\TextLanguageDetect\LanguageDetect\TextLanguageDetectException;
 use Illuminate\Pagination\LengthAwarePaginator;
@@ -49,7 +51,7 @@ class MetaGer
 
 	function __construct()
 	{
-        $this->time = microtime();   
+        $this->starttime = microtime(true);   
         define('CRLF', "\r\n");
         define('BUFFER_LENGTH', 8192);
         if( file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt") )
@@ -86,12 +88,21 @@ class MetaGer
             $viewResults[] = get_object_vars($result);
         }
 
+        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
+        $this->createLogs();
+
         if( $this->fokus === "bilder" )
         {
             switch ($this->out) 
             {
                 case 'results':
-                    return '';
+                    return view('metager3bilderresults')
+                        ->with('results', $viewResults)
+                        ->with('eingabe', $this->eingabe)
+                        ->with('mobile', $this->mobile)
+                        ->with('warnings', $this->warnings)
+                        ->with('errors', $this->errors)
+                        ->with('metager', $this);
                 default:
                     return view('metager3bilder')
                         ->with('results', $viewResults)
@@ -113,9 +124,18 @@ class MetaGer
                     ->with('errors', $this->errors)
                     ->with('metager', $this);
                 break;
-            default:
+            case 'results-with-style':
                 return view('metager3')
                     ->with('results', $viewResults)
+                    ->with('eingabe', $this->eingabe)
+                    ->with('mobile', $this->mobile)
+                    ->with('warnings', $this->warnings)
+                    ->with('errors', $this->errors)
+                    ->with('metager', $this)
+                    ->with('suspendheader', "yes");
+                break;
+            default:
+                return view('metager3')
                     ->with('eingabe', $this->eingabe)
                     ->with('mobile', $this->mobile)
                     ->with('warnings', $this->warnings)
@@ -125,6 +145,36 @@ class MetaGer
         }
 	}
 
+    private function createLogs()
+    {
+        $redis = Redis::connection('redisLogs');
+        if( $redis )
+        {
+            $logEntry = "";
+            $logEntry .= "[" . date(DATE_RFC822, mktime(date("H"),date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
+            $logEntry .= " From=" . $this->ip;
+            $logEntry .= " pid=" . getmypid();
+            $anonId= md5("MySeCrEtSeEdFoRmd5"
+            .$this->request->header('Accept')
+            .$this->request->header('Accept-Charset')
+            .$this->request->header('Accept-Encoding')
+            .$this->request->header('HTTP_LANGUAGE')
+            .$this->request->header('User-Agent')
+            .$this->request->header('Keep-Alive')
+            .$this->request->header('X-Forwarded-For'));
+            $logEntry .= " anonId=$anonId";
+            $logEntry .= " ref=" . $this->request->header('Referer');
+            $useragent = $this->request->header('User-Agent');
+            $useragent = str_replace("(", " ", $useragent);
+            $useragent = str_replace(")", " ", $useragent);
+            $useragent = str_replace(" ", "", $useragent);
+            $logEntry .= " ua=" . $useragent;
+            $logEntry .= " iter= mm= time=" . round((microtime(true)-$this->starttime), 2) . " serv=" . $this->fokus . " which= hits= stringSearch= QuickTips= SSS= check=";
+            $logEntry .= " search=" . $this->eingabe;
+            $redis->rpush('logs.search', $logEntry);
+        }
+    }
+
     public function removeInvalids ()
     {
         $results = [];
@@ -211,6 +261,19 @@ class MetaGer
         }
 
         $this->results = $paginatedSearchResults;
+
+        if( isset($this->password) )
+        {
+            # Wir bieten einen bezahlten API-Zugriff an, bei dem dementsprechend die Werbung ausgeblendet wurde:
+            # Aktuell ist es nur die Uni-Mainz. Deshalb überprüfen wir auch nur diese.
+            $password = getenv('mainz');
+            $eingabe = $this->eingabe;
+            $password = md5($eingabe . $password);
+            if( $this->password === $password )
+            {
+                $this->ads = [];
+            }
+        }
 	}
 
 	public function createSearchEngines (Request $request)
@@ -218,12 +281,11 @@ class MetaGer
 
         #die(SocketRocket::get("tls", "dominik-pfennig.de", "", 443));
 
-
 		# Überprüfe, welche Sumas eingeschaltet sind
         $xml = simplexml_load_file($this->sumaFile);
         $enabledSearchengines = [];
         $overtureEnabled = FALSE;
-        
+        $countSumas = 0;
         if($this->fokus === "angepasst")
         {
             $sumas = $xml->xpath("suma");
@@ -239,12 +301,12 @@ class MetaGer
             foreach($sumas as $suma)
             {
                 if($request->has($suma["service"]) 
-                	|| ( $this->fokus !== "bilder" 
-                		&& ($suma["name"]->__toString() === "qualigo" 
-                			|| $suma["name"]->__toString() === "similar_product_ads" 
-                			|| ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
-                			)
-                		)
+                	#|| ( $this->fokus !== "bilder" 
+                	#	&& ($suma["name"]->__toString() === "qualigo" 
+                	#		|| $suma["name"]->__toString() === "similar_product_ads" 
+                	#		|| ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
+                	#		)
+                	#	)
                     #|| 1 === 1  #Todo: entfernen
                 	){
 
@@ -254,7 +316,8 @@ class MetaGer
                         {
                             $overtureEnabled = TRUE;
                         }
-
+                        if( $suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds" )
+                            $countSumas += 1;
                         $enabledSearchengines[] = $suma;
                     }
                 }
@@ -277,13 +340,15 @@ class MetaGer
                         {
                             $overtureEnabled = TRUE;
                         }
+                        if( $suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds" )
+                            $countSumas += 1;
                         $enabledSearchengines[] = $suma;
                     }
                 }
             }
         }
 
-        if( ( $this->fokus !== "bilder" && sizeof($enabledSearchengines) <= 3 ) || ( $this->fokus === "bilder" && sizeof($enabledSearchengines) === 0) )
+        if( $countSumas <= 0 )
         {
             $this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
         }
@@ -313,19 +378,48 @@ class MetaGer
                 $this->sockets[$tmp->name] = $tmp->fp;
             }
 		}
-
         # Nun passiert ein elementarer Schritt.
         # Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
         # aber natürlich nicht ewig.
         # Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
         # Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
-        usleep(500000);
+
         # Jetzt lesen wir alles aus, was da ist und verwerfen den Rest:
-        foreach($engines as $engine)
+        $enginesToLoad = count($engines);
+        $loadedEngines = 0;
+        $time = 0;
+        while( true )
         {
-            $engine->retrieveResults();
+            # Abbruchbedingung
+            if($time < 500)
+            {
+                if($loadedEngines >= $enginesToLoad)
+                    break;
+            }elseif( $time >= 500 && $time < $this->time)
+            {
+                if( ($loadedEngines / ($enginesToLoad * 1.0)) >= 0.8 )
+                    break;
+            }else
+            {
+                break;
+            }
+            foreach($engines as $engine)
+            {
+                if(!$engine->loaded)
+                {
+                    $success = $engine->retrieveResults();
+                    if($engine->loaded)
+                        $loadedEngines += 1;
+                }
+            }
+            usleep(50000);
+            $time += 50;
+        }
+        foreach( $engines as $engine )
+        {
+            if( !$engine->loaded )
+                $engine->shutdown();
         }
-
 
         $this->engines = $engines;
 	}
@@ -372,13 +466,8 @@ class MetaGer
         $this->q = $this->eingabe;
 
         # IP:
-        if( isset($_SERVER['HTTP_FROM']) )
-        {
-            $this->ip = $_SERVER['HTTP_FROM'];
-        }else
-        {
-            $this->ip = "127.0.0.1";
-        }
+        $this->ip = $request->ip();
+
         # Language:
         if( isset($_SERVER['HTTP_LANGUAGE']) )
         {
@@ -390,7 +479,8 @@ class MetaGer
         # Category
         $this->category = $request->input('category', '');
         # Request Times:
-        $this->time = $request->input('time', 1);
+        $this->time = $request->input('time', 1000);
+       
         # Page
         $this->page = $request->input('page', 1);
         # Lang
@@ -401,20 +491,25 @@ class MetaGer
         }
         $this->agent = new Agent();
         $this->mobile = $this->agent->isMobile();
+
         #Sprüche
-        $this->sprueche = $request->input('sprueche', 'on');
+        $this->sprueche = $request->input('sprueche', 'off');
+        if($this->sprueche === "off" )
+            $this->sprueche = true;
+        else
+            $this->sprueche = false;
         # Ergebnisse pro Seite:
         $this->resultCount = $request->input('resultCount', '20');
 
         # Manchmal müssen wir Parameter anpassen um den Sucheinstellungen gerecht zu werden:
         if( $request->has('dart') )
         {
-        	$this->time = 10;
+        	$this->time = 10000;
         	$this->warnings[] = "Hinweis: Sie haben Dart-Europe aktiviert. Die Suche kann deshalb länger dauern und die maximale Suchzeit wurde auf 10 Sekunden hochgesetzt.";
         }
-        if( $this->time < 0 || $this->time > 20 )
+        if( $this->time <= 500 || $this->time > 20000 )
         {
-        	$this->time = 1;
+        	$this->time = 1000;
         }
         if( $request->has('minism') && ( $request->has('fportal') || $request->has('harvest') ) )
         {
@@ -431,7 +526,7 @@ class MetaGer
         }
         if( $request->has('ebay') )
         {
-        	$this->time = 2;
+        	$this->time = 2000;
         	$this->warnings[] = "Hinweis: Sie haben Ebay aktiviert. Die Suche kann deshalb länger dauern und die maximale Suchzeit wurde auf 2 Sekunden hochgesetzt.";
         }
         if( App::isLocale("en") )
@@ -460,6 +555,13 @@ class MetaGer
         {
             $this->tab = "_blank";
         }
+        if( $request->has('password') )
+            $this->password = $request->input('password');
+        if( $request->has('quicktips') )
+            $this->quicktips = false;
+        else
+            $this->quicktips = true;
+
         $this->out = $request->input('out', "html");
         if($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style")
             $this->out = "html";
@@ -475,6 +577,11 @@ class MetaGer
 			$this->q = $match[1] . $match[3];
 			$this->warnings[] = "Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: \"" . $this->site . "\" angezeigt.";
 		}
+        if( $request->has('site') )
+        {
+            $this->site = $request->input('site');
+            $this->warnings[] = "Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: \"" . $this->site . "\" angezeigt.";
+        }
 		# Wenn die Suchanfrage um das Schlüsselwort "-host:*" ergänzt ist, sollen bestimmte Hosts nicht eingeblendet werden
 		# Wir prüfen, ob das hier der Fall ist:
 		while(preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match))
@@ -676,7 +783,7 @@ class MetaGer
     public function generateSiteSearchLink($host)
     {
         $host = urlencode($host);
-        $requestData = $this->request->except('page');
+        $requestData = $this->request->except(['page','out']);
         $requestData['eingabe'] .= " site:$host";
         $requestData['focus'] = "web";
         $link = action('MetaGerSearch@search', $requestData);
@@ -686,7 +793,7 @@ class MetaGer
     public function generateRemovedHostLink ($host)
     {
         $host = urlencode($host);
-        $requestData = $this->request->except('page');
+        $requestData = $this->request->except(['page','out']);
         $requestData['eingabe'] .= " -host:$host";
         $link = action('MetaGerSearch@search', $requestData);
         return $link;
@@ -695,7 +802,7 @@ class MetaGer
     public function generateRemovedDomainLink ($domain)
     {
         $domain = urlencode($domain);
-        $requestData = $this->request->except('page');
+        $requestData = $this->request->except(['page','out']);
         $requestData['eingabe'] .= " -domain:$domain";
         $link = action('MetaGerSearch@search', $requestData);
         return $link;
@@ -716,4 +823,15 @@ class MetaGer
         else
             return null;
     }
+    public function getImageProxyLink($link)
+    {
+        $requestData = [];
+        $requestData["url"] = $link;
+        $link = action('Pictureproxy@get', $requestData);
+        return $link;
+    }
+    public function showQuicktips ()
+    {
+        return $this->quicktips;
+    }
 }
\ No newline at end of file

app/Models/Result.php

@@ -105,18 +105,23 @@ class Result
 		foreach(explode(" ", trim($tmpEingabe)) as $el)
 		{
 			$el = preg_quote($el, "/");
-			if(preg_match("/\b$el\b/si", $tmpTitle))
+			if(strlen($tmpTitle) > 0)
 			{
-				$tmpRank += .7 * .6 * $maxRank;
-			}elseif (strpos($tmpTitle, $el) !== false) {
-				$tmpRank += .3 * .6 * $maxRank;
+				if(preg_match("/\b$el\b/si", $tmpTitle))
+				{
+					$tmpRank += .7 * .6 * $maxRank;
+				}elseif (strpos($tmpTitle, $el) !== false) {
+					$tmpRank += .3 * .6 * $maxRank;
+				}
 			}
-
-			if(preg_match("/\b$el\b/si", $tmpDescription))
+			if( strlen($tmpDescription) > 0 )
 			{
-				$tmpRank += .7 * .4 * $maxRank;
-			}elseif (strpos($tmpDescription, $el) !== false) {
-				$tmpRank += .3 * .4 * $maxRank;
+				if(preg_match("/\b$el\b/si", $tmpDescription))
+				{
+					$tmpRank += .7 * .4 * $maxRank;
+				}elseif (strpos($tmpDescription, $el) !== false) {
+					$tmpRank += .3 * .4 * $maxRank;
+				}
 			}
 		}
 		$tmpRank /= sizeof(explode(" ", trim($tmpEingabe))) * 10;

app/Models/Searchengine.php

@@ -19,6 +19,7 @@ abstract class Searchengine
 	public $ads = [];
 	public $write_time = 0;
 	public $connection_time = 0;
+	public $loaded = false;
 
 	function __construct(\SimpleXMLElement $engine, MetaGer $metager)
 	{
@@ -231,6 +232,7 @@ abstract class Searchengine
 		// end of headers
 		if(sizeof($headers) > 1){
 			$bodySize = 0;
+			stream_set_blocking($this->fp, 1);
 			if( isset($headers["Transfer-Encoding"]) && $headers["Transfer-Encoding"] === "chunked" )
 			{
 				$body = $this->readChunked();
@@ -245,6 +247,10 @@ abstract class Searchengine
 			{
 				die("Konnte nicht herausfinden, wie ich die Serverantwort von: " . $this->name . " auslesen soll. Header war: " . print_r($headers));
 			}
+			$this->loaded = true;
+		}else
+		{
+			return;
 		}
 
 		Redis::del($this->host . "." . $this->socketNumber);
@@ -253,7 +259,6 @@ abstract class Searchengine
 		{
 			$body = $this->gunzip($body);
 		}
-
 		#print_r($headers);
 		#print($body);
 		#print("\r\n". $bodySize);
@@ -265,6 +270,12 @@ abstract class Searchengine
 		#exit;
 	}
 
+	public function shutdown()
+	{
+		fclose($this->fp);
+		Redis::del($this->host . "." . $this->socketNumber);
+	}
+
 	private function readBody($length)
 	{
 		$theData = '';

app/Models/parserSkripte/Ecoshopper.php

@@ -43,6 +43,7 @@ class Ecoshopper extends Searchengine
 				$anzeigeLink = $link;
 			}
 			$descr = $result->xpath('//doc/str[@name="artikelBeschreibung"]')[0]->__toString();
+			$image = $result->xpath('//doc/str[@name="artikelImageurl"]')[0]->__toString();
 			$this->counter++;
 			$this->results[] = new \App\Models\Result(
 				$this->engine,
@@ -51,7 +52,9 @@ class Ecoshopper extends Searchengine
 				$anzeigeLink,
 				$descr,
 				$this->gefVon,
-				$this->counter
+				$this->counter,
+				false,
+				$image
 			);
 		}
 	}

app/Models/parserSkripte/Mg_produkt2.php

@@ -35,6 +35,7 @@ class Mg_produkt2 extends Searchengine
 			parse_str($anzeigeLink['query'], $query);
 			$anzeigeLink = $query['diurl'];
 			$descr = $result->xpath('/doc/arr[@name="artikelBeschreibung"]')[0]->{"str"}->__toString();
+			$image = $result->xpath('/doc/arr[@name="artikelImageurl"]')[0]->{"str"}->__toString();
 			$this->counter++;
 			$this->results[] = new \App\Models\Result(
 				$this->engine,
@@ -43,7 +44,9 @@ class Mg_produkt2 extends Searchengine
 				$anzeigeLink,
 				$descr,
 				$this->gefVon,
-				$this->counter
+				$this->counter,
+				false,
+				$image
 			);
 		}
 	}

app/Models/parserSkripte/Zeitde.php

@@ -20,6 +20,8 @@ class Zeitde extends Searchengine
 			return;
 		foreach( $results->{"matches"} as $result )
 		{
+			if( !isset($result->{"title"}) || !isset($result->{"href"}) || !isset($result->{"snippet"}))
+				continue;
 			$title = $result->{"title"};
 			$link = $result->{"href"};
 			$anzeigeLink = $link;

config/database.php

@@ -114,6 +114,12 @@ return [
             'port' => env('REDIS_PORT', 6379),
             'database' => 0,
         ],
+        'redisLogs' => [
+            'host' => env('REDIS_LOGS_HOST', 'localhost'),
+            'password' => env('REDIS_LOGS_PASSWORD', null),
+            'port' => env('REDIS_MAIN_PORT', 6379),
+            'database' => 1,
+        ],
 
     ],