Searchengine.php 6.99 KB
Newer Older
1 2
<?php

3
namespace App\Models;
4
use App\MetaGer;
5 6
use Log;
use Redis;
7 8
use App\Jobs\Search;
use Illuminate\Foundation\Bus\DispatchesJobs;
9
use Cache;
10

11 12

abstract class Searchengine 
13
{
14
	use DispatchesJobs;
15

16
	protected $ch; 	# Curl Handle zum erhalten der Ergebnisse
17
	public $fp;
18 19 20 21
	protected $getString = "";
	protected $engine;
    protected $counter = 0;
    protected $socketNumber = null;
22
    public $enabled = true;
23
	public $results = [];
24 25 26
	public $ads = [];
	public $write_time = 0;
	public $connection_time = 0;
27
	public $loaded = false;
28
	public $cached = false;
29

30
	function __construct(\SimpleXMLElement $engine, MetaGer $metager)
31 32 33 34
	{
		foreach($engine->attributes() as $key => $value){
			$this->$key = $value->__toString();
		}
35 36
		if( !isset($this->homepage) )
			$this->homepage = "https://metager.de";
37 38
		$this->engine = $engine;

39 40 41
		if( !isset($this->cacheDuration) )
			$this->cacheDuration = 60;

42 43 44
		# Wir registrieren die Benutzung dieser Suchmaschine
		$this->uses = intval(Redis::hget($this->name, "uses")) + 1;
		Redis::hset($this->name, "uses", $this->uses);
45

46 47 48 49 50 51 52 53 54 55 56
		# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
        if(isset($this->disabled) && strtotime($this->disabled) <= time() )
        {
        	# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
        	$this->enable($metager->getSumaFile(), "Die Suchmaschine " . $this->name . " wurde wieder eingeschaltet.");
        }elseif (isset($this->disabled) && strtotime($this->disabled) > time()) 
        {
        	$this->enabled = false;
        	return;
        }

57
		# User-Agent definieren:
58
		$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
59
		$this->ip = $metager->getIp();
60 61
		$this->gefVon = "<a href=\"" . $this->homepage . "\" target=\"_blank\">" . $this->displayName . "</a>";
		$this->startTime = microtime();
62
		
63 64 65
		$q = "";
		if( isset($this->hasSiteSearch) && $this->hasSiteSearch === "1")
		{
66 67 68 69
			if(strlen($metager->getSite()) === 0)
				$q = $metager->getQ();
			else
				$q = $metager->getQ() . " site:" . $metager->getSite();
70 71 72 73 74
		}else
		{
			$q = $metager->getQ();
		}
		$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
75 76 77 78 79
		$this->hash = md5($this->host . $this->getString . $this->port . $this->name);
		$this->resultHash = $metager->getHashCode();
		if( Cache::has($this->hash) )
		{
			$this->cached = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
80
			$this->retrieveResults();
81 82 83 84 85 86 87 88 89
		}else
		{
			# Die Anfragen an die Suchmaschinen werden nun von der Laravel-Queue bearbeitet:
			# Hinweis: solange in der .env der QUEUE_DRIVER auf "sync" gestellt ist, werden die Abfragen
			# nacheinander abgeschickt.
			# Sollen diese Parallel verarbeitet werden, muss ein anderer QUEUE_DRIVER verwendet werden.
			# siehe auch: https://laravel.com/docs/5.2/queues
			$this->dispatch(new Search($this->resultHash, $this->host, $this->port, $this->name, $this->getString, $this->useragent, $metager->getSumaFile()));
		}
90 91
	}

92
	public abstract function loadResults($result);
93

94
	
95

96 97 98 99 100 101 102
	public function rank (\App\MetaGer $metager)
	{
		foreach($this->results as $result)
		{
			$result->rank($metager);
		}
	}
103

104
	
105

106
	private function setStatistic($key, $val)
107
	{
108

109 110 111 112
		$oldVal = floatval(Redis::hget($this->name, $key)) * $this->uses;
		$newVal = ($oldVal + max($val, 0)) / $this->uses;
		Redis::hset($this->name, $key, $newVal);
		$this->$key = $newVal;
113 114
	}

115
	
116

117
	public function enable($sumaFile, $message)
118 119 120 121
	{
		Log::info($message);
		$xml = simplexml_load_file($sumaFile);
		unset($xml->xpath("//sumas/suma[@name='" . $this->name . "']")['0']['disabled']);
122 123 124 125 126 127 128 129
		$xml->saveXML($sumaFile);
	}

	public function closeFp()
	{
		fclose($this->fp);
	}

130
	public function getSocket()
131
	{
132 133
		$number = Redis::hget('search.' . $this->hash, $this->name);
		if( $number === null )
134
		{
135 136
			die("test");
			return null;
137 138
		}else
		{
139
			return pfsockopen($this->getHost() . ":" . $this->port . "/$number", $this->port, $errstr, $errno, 1);
140
		}
141
	}
142

143 144
	public function retrieveResults()
	{
Dominik Hebeler's avatar
Dominik Hebeler committed
145 146
		if( $this->loaded )
			return true;
147 148 149 150 151 152 153 154 155 156 157
		$body = "";
		if( $this->cacheDuration > 0 && Cache::has($this->hash) )
		{
			$body = Cache::get($this->hash);
		}elseif ( Redis::hexists('search.' . $this->resultHash, $this->name) ) {
			$body = Redis::hget('search.' . $this->resultHash, $this->name);
			if( $this->cacheDuration > 0 )
				Cache::put($this->hash, $body, $this->cacheDuration);
		}

		if( $body !== "" )
158
		{
159 160 161 162
			$this->loadResults($body);
			$this->loaded = true;
			Redis::hdel('search.' . $this->hash, $this->name);
			return true;
163 164 165 166
		}else
		{
			return false;
		}		
167 168
	}

169 170 171 172 173
	public function shutdown()
	{
		Redis::del($this->host . "." . $this->socketNumber);
	}

174 175 176 177 178 179 180 181 182 183 184 185 186
	protected function getHost()
	{
		$return = "";
		if( $this->port === "443" )
		{
			$return .= "tls://";
		}else
		{
			$return .= "tcp://";
		}
		$return .= $this->host;
		return $return;
	}
187

188 189 190 191 192
	public function getCurlInfo()
	{
		return curl_getinfo($this->ch);
	}

193 194 195 196 197
	public function getCurlErrors()
	{
		return curl_errno($this->ch);
	}

198 199 200
	public function addCurlHandle ($mh)
	{
		curl_multi_add_handle($mh, $this->ch);
201 202
	}

203 204 205 206 207
	public function removeCurlHandle ($mh)
	{
		curl_multi_remove_handle($mh, $this->ch);
	}

208
	private function generateGetString($query, $url, $language, $category)
209 210
	{
		$getString = "";
211

212
		# Skript:
213 214 215 216
		if(strlen($this->skript) > 0)
			$getString .= $this->skript;
		else
			$getString .= "/";
217
		# FormData:
218 219
		if(strlen($this->formData) > 0)
			$getString .= "?" . $this->formData;
220 221 222 223 224 225 226 227

		# Wir müssen noch einige Platzhalter in dem GET-String ersetzen:
		if( strpos($getString, "<<USERAGENT>>") ){
			$getString = str_replace("<<USERAGENT>>", $this->urlEncode($this->useragent), $getString);
		}

		if( strpos($getString, "<<QUERY>>") )
		{
228
			$getString = str_replace("<<QUERY>>", $this->urlEncode($query), $getString);
229 230 231 232
		}

		if( strpos($getString, "<<IP>>") )
		{
233
			$getString = str_replace("<<IP>>", $this->urlEncode($this->ip), $getString);
234 235 236 237
		}

		if( strpos($getString, "<<LANGUAGE>>") )
		{
238
			$getString = str_replace("<<LANGUAGE>>", $this->urlEncode($language), $getString);
239 240 241 242
		}

		if( strpos($getString, "<<CATEGORY>>") )
		{
243
			$getString = str_replace("<<CATEGORY>>", $this->urlEncode($category), $getString);
244
		}
245 246 247

		if( strpos($getString, "<<AFFILDATA>>") )
		{
248
			$getString = str_replace("<<AFFILDATA>>", $this->getOvertureAffilData($url), $getString);
249
		}
250 251 252
		return $getString;
	}

253
	protected function urlEncode($string)
254 255 256 257 258 259 260 261 262 263
	{
		if(isset($this->inputEncoding))
		{
			return urlencode(mb_convert_encoding($string, $this->inputEncoding));
		}else
		{
			return urlencode($string);
		}
	}

264
	private function getOvertureAffilData($url)
265
	{
266
	    $affil_data = 'ip=' . $this->ip;
267 268 269
	    $affil_data .= '&ua=' . $this->useragent;  
	    if ( isset($_SERVER['HTTP_X_FORWARDED_FOR']) ) {
	       $affil_data .= '&xfip=' . $_SERVER['HTTP_X_FORWARDED_FOR'];
270
	    }
271
	    $affilDataValue = $this->urlEncode($affil_data);
272
		# Wir benötigen die ServeUrl:
273
		$serveUrl = $this->urlEncode($url);
274

275 276
		return "&affilData=" . $affilDataValue . "&serveUrl=" . $serveUrl;
	}
277 278 279 280 281

	public function isEnabled ()
	{
		return $this->enabled;
	}
282
}