Merge remote-tracking branch 'origin/master' into 66-es-werden-noch-keinerleid-logs-geschrieben

.gitlab-ci.yml

@@ -2,8 +2,7 @@ update(144.76.113.134):
   tags:
   - 144.76.113.134
   only:
-    - tags
-    - triggers
+    - master
   before_script:
     # Abhängigkeiten überprüfen
     - which composer
@@ -22,6 +21,8 @@ update(144.76.113.134):
     - scp metager@metager3.de:~/.env .
     - scp metager@metager3.de:~/sumas.xml config/
     - scp metager@metager3.de:~/sumasEn.xml config/
+    - scp metager@metager3.de:~/blacklistUrl.txt config/
+    - scp metager@metager3.de:~/blacklistDomains.txt config/
     - chmod 777 config/sumas.xml config/sumasEn.xml
     - cd ~/
     - rm -rf MetaGer
@@ -31,8 +32,7 @@ update(metager3.de):
   tags:
   - metager3
   only:
-    - tags
-    - triggers
+    - master
   before_script:
     # Abhängigkeiten überprüfen
     - which composer
@@ -51,6 +51,8 @@ update(metager3.de):
     - scp metager@metager3.de:~/.env .
     - scp metager@metager3.de:~/sumas.xml config/
     - scp metager@metager3.de:~/sumasEn.xml config/
+    - scp metager@metager3.de:~/blacklistUrl.txt config/
+    - scp metager@metager3.de:~/blacklistDomains.txt config/
     - chmod 777 config/sumas.xml config/sumasEn.xml
     - cd ~/
     - rm -rf MetaGer

app/Http/Controllers/MetaGerSearch.php

@@ -15,6 +15,7 @@ class MetaGerSearch extends Controller
 {
     public function search(Request $request, MetaGer $metager)
     {
+        #die($request->header('User-Agent'));
         $time = microtime();
         # Mit gelieferte Formulardaten parsen und abspeichern:
         $metager->parseFormData($request);

app/MetaGer.php

@@ -491,6 +491,7 @@ class MetaGer
         }
         $this->agent = new Agent();
         $this->mobile = $this->agent->isMobile();
+
         #Sprüche
         $this->sprueche = $request->input('sprueche', 'off');
         if($this->sprueche === "off" )

app/Models/Results.php

-<?php
-
-namespace App\MetaGer;
-use Illuminate\Http\Request;
-use File;
-class Results
-{
-	private $fokiNames = [];
-	private $fokus;
-	public $results = [];
-
-	function __construct ($engines)
-	{
-		$this->results = $this->loadResults($engines);
-	}
-
-
-
-	private function get($getStrings){
-		# Nimmt ein array aus getStrings entgegen und liefert ein Array aus Antworten zurück:
-		# Zunächst alle Curl Abfragen initialisieren, aber noch nicht ausführen:
-		#return $getStrings;
-		#$getStrings = array($getStrings[0]);
-		$ch = [];
-		foreach($getStrings as $getString)
-		{
-			$tmp = curl_init($getString);
-			curl_setopt($tmp, CURLOPT_RETURNTRANSFER, true);
-			curl_setopt($tmp, CURLOPT_FOLLOWLOCATION, true);
-			curl_setopt($tmp, CURLOPT_CONNECTTIMEOUT , TIME); 
-			$ch[] = $tmp;
-		}
-
-		# Nun initialisieren wir Multicurl:
-		$mh = curl_multi_init();
-		foreach($ch as $handle)
-		{
-			curl_multi_add_handle($mh, $handle);
-		}
-
-		# Nun führen wir die Get-Requests aus und warten auf alle Ergebnisse:
-		$running = null;
-		do
-		{
-			curL_multi_exec($mh, $running);
-		}while($running);
-
-		# Wir haben alle Ergebnisse und schließen die Handles
-		foreach($ch as $handle)
-		{
-			curl_multi_remove_handle($mh, $handle);
-		}
-		# Und auch den Multicurl-Handle:
-		curl_multi_close($mh);
-
-		$results = [];
-		foreach($ch as $handle)
-		{
-			$results[] = curl_multi_getcontent($handle);
-		}
-
-		return $results;
-	}
-
-}
\ No newline at end of file

app/Models/Searchengine.php

@@ -272,7 +272,8 @@ abstract class Searchengine
 
 	public function shutdown()
 	{
-		fclose($this->fp);
+		if( $this->fp )
+			fclose($this->fp);
 		Redis::del($this->host . "." . $this->socketNumber);
 	}
 

app/Models/parserSkripte/Minisucher.php

@@ -14,7 +14,11 @@ class Minisucher extends Searchengine
 
 	public function loadResults ($content)
 	{
-		$content = simplexml_load_string($content);
+		try {
+			$content = simplexml_load_string($content);
+		} catch (\Exception $e) {
+			return;
+		}
 		if(!$content)
 		{
 			return;

app/Models/parserSkripte/Witch.php

@@ -15,15 +15,14 @@ class Witch extends Searchengine
     public function loadResults ($result)
     {
         $result = html_entity_decode(trim(utf8_encode($result)));
-
+        
         $results = explode("\n", $result);
-
         array_shift($results);
         foreach($results as $res)
         {
             
             $res = explode(";", $res);
-            if(sizeof($res) !== 4)
+            if(sizeof($res) !== 4 || $res[3] === "'Kein Ergebnis'")
             {
                 continue;
             }

app/Models/parserSkripte/Zeitde.php

@@ -20,6 +20,8 @@ class Zeitde extends Searchengine
 			return;
 		foreach( $results->{"matches"} as $result )
 		{
+			if( !isset($result->{"title"}) || !isset($result->{"href"}) || !isset($result->{"snippet"}))
+				continue;
 			$title = $result->{"title"};
 			$link = $result->{"href"};
 			$anzeigeLink = $link;

public/index.php

@@ -11,9 +11,19 @@
  # nicht einmal wir selbst noch Zugriff auf die Daten haben:
 if( !isset($_SERVER['HTTP_X_FORWARDED_FOR'] ))
 {
-	$_SERVER['REMOTE_ADDR'] = substr($_SERVER['REMOTE_ADDR'], 0, strrpos($_SERVER['REMOTE_ADDR'], ".")) . ".0";
-	#$_SERVER['HTTP_USER_AGENT'] = substr($_SERVER['HTTP_USER_AGENT'], 0, 23);
+	$_SERVER['REMOTE_ADDR'] = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $_SERVER['REMOTE_ADDR']);
+}else
+{
+	$_SERVER['HTTP_X_FORWARDED_FOR'] = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $_SERVER['HTTP_X_FORWARDED_FOR']);
+}
+$agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);
+
+for($i = 0; $i < count($agentPieces); $i++)
+{
+	$agentPieces[$i] = preg_replace("/(\d+\.\d+)/s", "0.0", $agentPieces[$i]);
+	$agentPieces[$i] = preg_replace("/([^\/]*)\/\w+/s", "$1/0.0", $agentPieces[$i]);
 }
+$_SERVER['HTTP_USER_AGENT'] = implode(" ", $agentPieces);
 
 /*
 |--------------------------------------------------------------------------