Merge branch '42-bugs' of http://gitlab.metager3.de:1234/dominik/MetaGer into 42-bugs

app/Models/Result.php.orig

+<?php
+
+namespace App\Models;
+
+
+
+class Result
+{
+	
+	function __construct ( \SimpleXMLElement $provider, $titel, $link, $anzeigeLink , $descr, $gefVon, $sourceRank, $partnershop = false )
+	{
+		$this->titel = strip_tags(trim($titel));
+		$this->link = trim($link);
+		$this->anzeigeLink = trim($anzeigeLink);
+		$this->descr = strip_tags(trim($descr));
+		$this->descr = preg_replace("/\n+/si", " ", $this->descr);
+		$this->gefVon = trim($gefVon);
+		$this->proxyLink = $this->generateProxyLink($this->link);
+		$this->sourceRank = $sourceRank;
+		if($this->sourceRank <= 0 || $this->sourceRank > 20)
+			$this->sourceRank = 20;
+		$this->sourceRank = 20 - $this->sourceRank;
+		if(isset($provider["engineBoost"]))
+		{
+			$this->engineBoost = $provider["engineBoost"];
+		}else
+		{
+			$this->engineBoost = 1;
+		}
+
+		$this->valid = true;
+		$this->host = @parse_url($link, PHP_URL_HOST);
+		$this->strippedHost = $this->getStrippedHost($this->anzeigeLink);
+		$this->strippedDomain = $this->getStrippedDomain($this->strippedHost);
+		$this->strippedLink = $this->getStrippedLink($this->anzeigeLink);
+		$this->rank = 0;
+		$this->partnershop = $partnershop;
+
+		#die($this->anzeigeLink . "\r\n" . $this->strippedHost);
+	}
+
+	public function rank (\App\MetaGer $metager)
+	{
+
+		$rank = 0;
+		$rank += ($this->sourceRank * 0.02);
+
+		#URL-Boost
+		$link = $this->anzeigeLink;
+		if(strpos($link, "http") !== 0)
+		{
+			$link = "http://" . $link;
+		}
+		$link = @parse_url($link, PHP_URL_HOST) . @parse_url($link, PHP_URL_PATH);
+		$tmpLi = $link;
+		$tmpEingabe = $metager->getQ();
+		$count = 0;
+		$tmpLink = "";
+
+		$regex = [
+			"/\s+/si",
+			"/http:/si",
+			"/https:/si",
+			"/www\./si",
+			"/\//si",
+			"/\./si",
+			"/-/si"
+			];
+		foreach($regex as $reg)
+		{
+			$link = preg_replace($regex, "", $link);
+			$tmpEingabe = preg_replace($regex, "", $tmpEingabe);
+		}
+		#die($tmpLi . "<br>" . $link . "<br>" . $tmpEingabe . "<br><br>");
+		foreach(str_split($tmpEingabe) as $char)
+		{
+			if(strpos(strtolower($tmpLink), strtolower($char)) >= 0)
+			{
+				$count++;
+				$tmpLink = str_replace(urlencode($char), "", $tmpLink);
+			}
+			if(strlen($this->descr) > 80 && strlen($link) > 0)
+			{
+				$rank += $count /((strlen($link)) * 60);
+			}
+		}
+
+		# Boost für Vorkommen der Suchwörter:
+		$maxRank = 0.1;
+		$tmpTitle = $this->titel;
+		$tmpDescription = $this->descr;
+		$isWithin = false;
+		$tmpRank = 0;
+		$tmpEingabe = $metager->getQ();
+		$tmpEingabe = preg_replace("/\b\w{1,3}\b/si", "", $tmpEingabe);
+		$tmpEingabe = preg_replace("/\s+/si", " ", $tmpEingabe);
+		#die($tmpEingabe);
+		foreach(explode(" ", trim($tmpEingabe)) as $el)
+		{
+			$el = preg_quote($el, "/");
+			if(preg_match("/\b$el\b/si", $tmpTitle))
+			{
+				$tmpRank += .7 * .6 * $maxRank;
+			}elseif (strpos($tmpTitle, $el) !== false) {
+				$tmpRank += .3 * .6 * $maxRank;
+			}
+
+			if(preg_match("/\b$el\b/si", $tmpDescription))
+			{
+				$tmpRank += .7 * .4 * $maxRank;
+			}elseif (strpos($tmpDescription, $el) !== false) {
+				$tmpRank += .3 * .4 * $maxRank;
+			}
+		}
+		$tmpRank /= sizeof(explode(" ", trim($tmpEingabe))) * 10;
+		$rank += $tmpRank;
+
+		if($this->engineBoost > 0)
+		{
+			$rank *= floatval($this->engineBoost);
+		}
+
+		$this->rank = $rank;
+	}
+
+	public function getRank ()
+	{
+		return $this->rank;
+	}
+
+	public function isValid (\App\MetaGer $metager)
+	{
+		# Zunächst die persönlich ( über URL-Parameter ) definierten Blacklists:
+		if(in_array($this->strippedHost, $metager->getUserHostBlacklist())
+			|| in_array($this->strippedDomain, $metager->getUserDomainBlacklist()))
+			return false;
+		
+		# Jetzt unsere URL und Domain Blacklist
+		if($this->strippedHost !== "" && (in_array($this->strippedHost, $metager->getDomainBlacklist()) || in_array($this->strippedLink, $metager->getUrlBlacklist())))
+		{
+			return false;
+		}
+
+		$text = $this->titel . " " . $this->descr;
+
+		if($metager->getLang() !== "all")
+		{
+			$result = $metager->getLanguageDetect()->detect($text, 1);
+			$lang = "";
+			foreach($result as $key => $value)
+			{
+				$lang = $key;
+			}
+
+			if($lang !== "" && $lang !== $metager->getLang())
+				return false;
+		}
+
+		# Wir wenden die Stoppwortsuche an und schmeißen entsprechende Ergebnisse raus:
+		foreach($metager->getStopWords() as $stopWord)
+		{
+			if(stripos($text, $stopWord) !== false)
+			{
+				return false;
+			}
+		}
+
+		# Abschließend noch 2 Überprüfungen. Einmal den Host filter, der Sicherstellt, dass von jedem Host maximal 3 Links angezeigt werden
+		# und dann noch den Dublettefilter, der sicher stellt, dass wir nach Möglichkeit keinen Link doppelt in der Ergebnisliste haben
+		# Diese Überprüfung führen wir unter bestimmten Bedingungen nicht durch:
+		if($metager->getSite() === "" &&
+			strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
+			strpos($this->strippedHost, "twitter.com") === false &&
+			strpos($this->strippedHost, "www.ladenpreis.net") === false &&
+			strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
+			strpos($this->strippedHost, "www.onenewspage.com") === false)
+		{
+			$count = $metager->getHostCount($this->strippedHost);
+			if($count >= 3)
+			{
+				return false;
+			}
+		}
+
+		# Unabhängig davon unser Dublettenfilter:
+		if($metager->addLink($this->strippedLink))
+		{
+			$metager->addHostCount($this->strippedHost);
+			return true;
+		}else
+		{
+			return false;
+		}
+	}
+
+<<<<<<< HEAD
+	private function getStrippedHost ($link)
+=======
+	private function getStrippedHost (String $link)
+>>>>>>> e060ccae7fd0ede5daca4f3bfa267bd1418fde7a
+	{
+		if(strpos($link, "http") !== 0)
+			$link = "http://" . $link;
+		$link = @parse_url($link, PHP_URL_HOST);
+		$link = preg_replace("/^www\./si", "", $link);
+		return $link;
+	}
+<<<<<<< HEAD
+	private function getStrippedLink ($link)
+=======
+	private function getStrippedLink (String $link)
+>>>>>>> e060ccae7fd0ede5daca4f3bfa267bd1418fde7a
+	{
+		if(strpos($link, "http") !== 0)
+			$link = "http://" . $link;
+		$host = $this->strippedHost;
+		$path = @parse_url($link , PHP_URL_PATH);
+		return $host . $path;
+	}
+
+<<<<<<< HEAD
+	private function getStrippedDomain ($link)
+=======
+	private function getStrippedDomain (String $link)
+>>>>>>> e060ccae7fd0ede5daca4f3bfa267bd1418fde7a
+	{
+		if(preg_match("/([^\.]*\.[^\.]*)$/si", $link, $match))
+		{
+			return $match[1];
+		}else
+		{
+			return $link;
+		}		
+	}
+
+<<<<<<< HEAD
+	private function generateProxyLink ($link)
+=======
+	private function generateProxyLink (String $link)
+>>>>>>> e060ccae7fd0ede5daca4f3bfa267bd1418fde7a
+	{
+		if(!$link)
+			return "";
+		$tmp = $link;
+		$tmp = preg_replace("/\r?\n$/s", "", $tmp);
+		$tmp = preg_replace("#^([\w+.-]+)://#s", "$1/", $tmp);
+		return "https://proxy.suma-ev.de/cgi-bin/nph-proxy.cgi/en/I0/" . $tmp;
+		
+	}
+}
\ No newline at end of file

app/Models/Searchengine.php.orig

+<?php
+
+namespace App\Models;
+use App\MetaGer;
+use Log;
+use Redis;
+
+abstract class Searchengine
+{
+
+	protected $ch; 	# Curl Handle zum erhalten der Ergebnisse
+	public $fp;
+	protected $getString = "";
+	protected $engine;
+    protected $counter = 0;
+    protected $socketNumber = null;
+    public $enabled = true;
+	public $results = [];
+	public $ads = [];
+	public $write_time = 0;
+	public $connection_time = 0;
+
+	function __construct(\SimpleXMLElement $engine, MetaGer $metager)
+	{
+		foreach($engine->attributes() as $key => $value){
+			$this->$key = $value->__toString();
+		}
+		$this->engine = $engine;
+
+		# Wir registrieren die Benutzung dieser Suchmaschine
+		$this->uses = intval(Redis::hget($this->name, "uses")) + 1;
+		Redis::hset($this->name, "uses", $this->uses);
+
+		# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
+        if(isset($this->disabled) && strtotime($this->disabled) <= time() )
+        {
+        	# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
+        	$this->enable($metager->getSumaFile(), "Die Suchmaschine " . $this->name . " wurde wieder eingeschaltet.");
+        }elseif (isset($this->disabled) && strtotime($this->disabled) > time()) 
+        {
+        	$this->enabled = false;
+        	return;
+        }
+
+		# User-Agent definieren:
+		if( isset($_SERVER['HTTP_USER_AGENT']))
+		{
+			$this->useragent = $_SERVER['HTTP_USER_AGENT'];
+		}else
+		{
+			$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
+		}
+		$this->ip = $metager->getIp();
+		$this->gefVon = "<a href=\"" . $this->homepage . "\" target=\"_blank\">" . $this->displayName . "</a>";
+		$this->startTime = microtime();
+		
+		$this->getString = $this->generateGetString($metager->getQ(), $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
+		$counter = 0;
+		# Wir benötigen einen verfügbaren Socket, über den wir kommunizieren können:
+		$time = microtime(true);
+		$this->fp = $this->getFreeSocket();
+		
+		$this->setStatistic("connection_time", ((microtime(true)-$time) / 1000000));
+		if(!$this->fp)
+		{
+			$this->disable($metager->getSumaFile(), "Die Suchmaschine " . $this->name . " wurde für 1h deaktiviert, weil keine Verbindung aufgebaut werden konnte");
+		}else
+		{
+			$time = microtime(true);
+			$this->writeRequest();
+			$this->setStatistic("write_time", ((microtime(true)-$time) / 1000000));
+		}
+
+	}
+
+<<<<<<< HEAD
+	public abstract function loadResults($result);
+=======
+	public abstract function loadResults(String $result);
+>>>>>>> e060ccae7fd0ede5daca4f3bfa267bd1418fde7a
+
+	private function writeRequest ()
+	{
+		$out = "GET " . $this->getString . " HTTP/1.1\r\n";
+		$out .= "Host: " . $this->host . "\r\n";
+		$out .= "User-Agent: " . $this->useragent . "\r\n";
+		$out .= "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n";
+		$out .= "Accept-Language: de,en-US;q=0.7,en;q=0.3\r\n";
+		$out .= "Accept-Encoding: gzip, deflate, br\r\n";
+		$out .= "Connection: keep-alive\r\n\r\n";
+
+		# Anfrage senden:
+		$sent = 0; $string = $out; $time = microtime(true);
+		while(true)
+		{	
+			try{
+				$tmp = fwrite($this->fp, $string);
+			}catch(\ErrorException $e)
+			{
+				# Irgendwas ist mit unserem Socket passiert. Wir brauchen einen neuen:
+				fclose($this->fp);
+				Redis::del($this->name . "." . $this->socketNumber);
+				$this->fp = $this->getFreeSocket();
+				$sent = 0;
+				$string = $out;
+				continue;
+			}
+			if($tmp){
+				$sent += $tmp;
+				$string = substr($string, $tmp);
+			}else
+				abort(500, "Fehler beim schreiben.");
+
+			if(((microtime(true) - $time) / 1000000) >= 500)
+			{
+				abort(500, "Konnte die Request Daten nicht an: " . $this->name . " senden");
+			}
+
+			if($sent >= strlen($out))
+				break;
+		}
+	}
+
+	public function rank (\App\MetaGer $metager)
+	{
+		foreach($this->results as $result)
+		{
+			$result->rank($metager);
+		}
+	}
+
+	private function getFreeSocket()
+	{
+		# Je nach Auslastung des Servers ( gleichzeitige Abfragen ), kann es sein, dass wir mehrere Sockets benötigen um die Abfragen ohne Wartezeit beantworten zu können.
+		# pfsockopen öffnet dabei einen persistenten Socket, der also auch zwischen den verschiedenen php Prozessen geteilt werden kann. 
+		# Wenn der Hostname mit einem bereits erstellten Socket übereinstimmt, wird die Verbindung also aufgegriffen und fortgeführt.
+		# Allerdings dürfen wir diesen nur verwenden, wenn er nicht bereits von einem anderen Prozess zur Kommunikation verwendet wird.
+		# Wenn dem so ist, probieren wir den nächsten Socket zu verwenden.
+		# Dies festzustellen ist komplizierter, als man sich das vorstellt. Folgendes System sollte funktionieren:
+		# 1. Stelle fest, ob dieser Socket neu erstellt wurde, oder ob ein existierender geöffnet wurde.
+		$counter = 0; $fp = null;
+		do
+		{
+			
+			if( intval(Redis::exists($this->host . ".$counter")) === 0 )              
+			{
+				Redis::set($this->host . ".$counter", 1);
+				Redis::expire($this->host . ".$counter", 5);
+				$this->socketNumber = $counter;
+
+				try
+				{
+					$fp = pfsockopen($this->getHost() . ":" . $this->port . "/$counter", $this->port, $errstr, $errno, 1);
+				}catch(\ErrorException $e)
+				{
+					break;
+				}
+				# Wir gucken, ob der Lesepuffer leer ist:
+				stream_set_blocking($fp, 0);
+				if(fgets($fp, BUFFER_LENGTH) !== false)
+				{
+					Log::error("Der Lesepuffer von: " . $this->name . " war nach dem Erstellen nicht leer. Musste den Socket neu starten.");
+					fclose($fp);
+					$fp = pfsockopen($this->getHost() . ":" . $this->port . "/$counter", $this->port, $errstr, $errno, 1);
+				}
+				header($this->name . ": " . $counter . "_" . $this->getHost());
+				break;
+			}
+			$counter++;
+		}while(true);
+
+		return $fp;
+	}
+
+<<<<<<< HEAD
+	private function setStatistic($key, float $val)
+=======
+	private function setStatistic(String $key, float $val)
+>>>>>>> e060ccae7fd0ede5daca4f3bfa267bd1418fde7a
+	{
+
+		$oldVal = floatval(Redis::hget($this->name, $key)) * $this->uses;
+		$newVal = ($oldVal + max($val, 0)) / $this->uses;
+		Redis::hset($this->name, $key, $newVal);
+		$this->$key = $newVal;
+	}
+
+<<<<<<< HEAD
+	public function disable($sumaFile, $message)
+=======
+	public function disable(string $sumaFile, string $message)
+>>>>>>> e060ccae7fd0ede5daca4f3bfa267bd1418fde7a
+	{
+		Log::info($message);
+		$xml = simplexml_load_file($sumaFile);
+		$xml->xpath("//sumas/suma[@name='" . $this->name . "']")['0']['disabled'] = date(DATE_RFC822, mktime(date("H")+1,date("i"), date("s"), date("m"), date("d"), date("Y")));
+		$xml->saveXML($sumaFile);
+	}
+
+<<<<<<< HEAD
+	public function enable($sumaFile, $message)
+=======
+	public function enable(string $sumaFile, string $message)
+>>>>>>> e060ccae7fd0ede5daca4f3bfa267bd1418fde7a
+	{
+		Log::info($message);
+		$xml = simplexml_load_file($sumaFile);
+		unset($xml->xpath("//sumas/suma[@name='" . $this->name . "']")['0']['disabled']);
+		$xml->saveXML($sumaFile);
+	}
+
+	public function closeFp()
+	{
+		fclose($this->fp);
+	}
+
+	public function retrieveResults()
+	{
+		$time = microtime(true);
+		$headers = '';
+		$body = '';
+		$length = 0;
+		if(!$this->fp)
+		{
+			return;
+		}
+		// get headers FIRST
+		$c = 0;
+		stream_set_blocking($this->fp, 0);
+		do
+		{
+    		// use fgets() not fread(), fgets stops reading at first newline
+   			// or buffer which ever one is reached first
+    		$data = fgets($this->fp, BUFFER_LENGTH);
+    		// a sincle CRLF indicates end of headers
+    		if ($data === false || $data == CRLF || feof($this->fp) || ((microtime()-$time)/1000000) > 100 ) {
+        		// break BEFORE OUTPUT
+        		break;
+    		}
+    		if( sizeof(($tmp = explode(": ", $data))) === 2 )
+    			$headers[trim($tmp[0])] = trim($tmp[1]);
+    		$c++;
+		}
+		while (true);
+		// end of headers
+		if(sizeof($headers) > 1){
+			$bodySize = 0;
+			if( isset($headers["Transfer-Encoding"]) && $headers["Transfer-Encoding"] === "chunked" )
+			{
+				$body = $this->readChunked();
+				
+			}elseif( isset($headers['Content-Length']) )
+			{
+				$length = trim($headers['Content-Length']);
+				if(is_numeric($length) && $length >= 1)
+					$body = $this->readBody($length);
+				$bodySize = strlen($body);
+			}else
+			{
+				die("Konnte nicht herausfinden, wie ich die Serverantwort von: " . $this->name . " auslesen soll. Header war: " . print_r($headers));
+			}
+		}
+
+		Redis::del($this->host . "." . $this->socketNumber);
+		$this->setStatistic("read_time", ((microtime(true)-$time) / 1000000));
+		if( isset($headers["Content-Encoding"]) && $headers['Content-Encoding'] === "gzip")
+		{
+			$body = $this->gunzip($body);
+		}
+
+		#print_r($headers);
+		#print($body);
+		#print("\r\n". $bodySize);
+		#exit;
+		#die(print_r($headers));
+		// $body and $headers should contain your stream data
+		$this->loadResults($body);
+		#print(print_r($headers, TRUE) . $body);
+		#exit;
+	}
+
+	private function readBody(int $length)
+	{
+		$theData = '';
+        $done = false;
+        stream_set_blocking($this->fp, 0);
+        $startTime = time();
+        $lastTime = $startTime;
+        while (!feof($this->fp) && !$done && (($startTime + 1) > time()) && $length !== 0)
+        {
+            usleep(100);
+            $theNewData = fgets($this->fp, BUFFER_LENGTH);
+            $theData .= $theNewData;
+            $length -= strlen($theNewData);
+            $done = (trim($theNewData) === '0');
+
+        }
+        return $theData;
+	}
+
+	private function readChunked()
+	{
+		$body = '';
+		// read from chunked stream
+		// loop though the stream
+		do
+		{
+	    	// NOTE: for chunked encoding to work properly make sure
+	    	// there is NOTHING (besides newlines) before the first hexlength
+
+	    	// get the line which has the length of this chunk (use fgets here)
+	    	$line = fgets($this->fp, BUFFER_LENGTH);
+
+	    	// if it's only a newline this normally means it's read
+	    	// the total amount of data requested minus the newline
+	    	// continue to next loop to make sure we're done
+	    	if ($line == CRLF) {
+	        	continue;
+	    	}
+
+	    	// the length of the block is sent in hex decode it then loop through
+	    	// that much data get the length
+	    	// NOTE: hexdec() ignores all non hexadecimal chars it finds
+	    	$length = hexdec($line);
+
+	    	if (!is_int($length)) {
+	        	trigger_error('Most likely not chunked encoding', E_USER_ERROR);
+	    	}
+
+		    // zero is sent when at the end of the chunks
+		    // or the end of the stream or error
+		    if ($line === false || $length < 1 || feof($this->fp)) {
+		    	if($length <= 0)
+		            	fgets($this->fp, BUFFER_LENGTH);
+		        // break out of the streams loop
+		        break;
+		    }
+
+		    // loop though the chunk
+		    do
+		    {
+		        // read $length amount of data
+		        // (use fread here)
+		        $data = fread($this->fp, $length);
+