Searchengine.php 7.04 KB
Newer Older
1
2
<?php

3
namespace App\Models;
4
use App\MetaGer;
5
6
use Log;
use Redis;
7
8
use App\Jobs\Search;
use Illuminate\Foundation\Bus\DispatchesJobs;
9
use Cache;
10

11
12

abstract class Searchengine 
13
{
14
	use DispatchesJobs;
15

16
	protected $ch; 	# Curl Handle zum erhalten der Ergebnisse
17
	public $fp;
18
19
20
21
	protected $getString = "";
	protected $engine;
    protected $counter = 0;
    protected $socketNumber = null;
22
    public $enabled = true;
23
	public $results = [];
24
25
26
	public $ads = [];
	public $write_time = 0;
	public $connection_time = 0;
27
	public $loaded = false;
28
	public $cached = false;
29

30
	function __construct(\SimpleXMLElement $engine, MetaGer $metager)
31
32
33
34
	{
		foreach($engine->attributes() as $key => $value){
			$this->$key = $value->__toString();
		}
35
36
		if( !isset($this->homepage) )
			$this->homepage = "https://metager.de";
37
38
		$this->engine = $engine;

39
40
41
		if( !isset($this->cacheDuration) )
			$this->cacheDuration = 60;

42
43
44
		# Wir registrieren die Benutzung dieser Suchmaschine
		$this->uses = intval(Redis::hget($this->name, "uses")) + 1;
		Redis::hset($this->name, "uses", $this->uses);
45

46
47
48
49
50
51
52
53
54
55
56
		# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
        if(isset($this->disabled) && strtotime($this->disabled) <= time() )
        {
        	# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
        	$this->enable($metager->getSumaFile(), "Die Suchmaschine " . $this->name . " wurde wieder eingeschaltet.");
        }elseif (isset($this->disabled) && strtotime($this->disabled) > time()) 
        {
        	$this->enabled = false;
        	return;
        }

57
58
59
60
61
62
63
64
		# User-Agent definieren:
		if( isset($_SERVER['HTTP_USER_AGENT']))
		{
			$this->useragent = $_SERVER['HTTP_USER_AGENT'];
		}else
		{
			$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
		}
65
		$this->ip = $metager->getIp();
66
67
		$this->gefVon = "<a href=\"" . $this->homepage . "\" target=\"_blank\">" . $this->displayName . "</a>";
		$this->startTime = microtime();
68
		
69
70
71
		$q = "";
		if( isset($this->hasSiteSearch) && $this->hasSiteSearch === "1")
		{
72
73
74
75
			if(strlen($metager->getSite()) === 0)
				$q = $metager->getQ();
			else
				$q = $metager->getQ() . " site:" . $metager->getSite();
76
77
78
79
80
		}else
		{
			$q = $metager->getQ();
		}
		$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
81
82
83
84
85
86
87
88
89
90
91
92
93
94
		$this->hash = md5($this->host . $this->getString . $this->port . $this->name);
		$this->resultHash = $metager->getHashCode();
		if( Cache::has($this->hash) )
		{
			$this->cached = true;
		}else
		{
			# Die Anfragen an die Suchmaschinen werden nun von der Laravel-Queue bearbeitet:
			# Hinweis: solange in der .env der QUEUE_DRIVER auf "sync" gestellt ist, werden die Abfragen
			# nacheinander abgeschickt.
			# Sollen diese Parallel verarbeitet werden, muss ein anderer QUEUE_DRIVER verwendet werden.
			# siehe auch: https://laravel.com/docs/5.2/queues
			$this->dispatch(new Search($this->resultHash, $this->host, $this->port, $this->name, $this->getString, $this->useragent, $metager->getSumaFile()));
		}
95
96
	}

97
	public abstract function loadResults($result);
98

99
	
100

101
102
103
104
105
106
107
	public function rank (\App\MetaGer $metager)
	{
		foreach($this->results as $result)
		{
			$result->rank($metager);
		}
	}
108

109
	
110

111
	private function setStatistic($key, $val)
112
	{
113

114
115
116
117
		$oldVal = floatval(Redis::hget($this->name, $key)) * $this->uses;
		$newVal = ($oldVal + max($val, 0)) / $this->uses;
		Redis::hset($this->name, $key, $newVal);
		$this->$key = $newVal;
118
119
	}

120
	
121

122
	public function enable($sumaFile, $message)
123
124
125
126
	{
		Log::info($message);
		$xml = simplexml_load_file($sumaFile);
		unset($xml->xpath("//sumas/suma[@name='" . $this->name . "']")['0']['disabled']);
127
128
129
130
131
132
133
134
		$xml->saveXML($sumaFile);
	}

	public function closeFp()
	{
		fclose($this->fp);
	}

135
	public function getSocket()
136
	{
137
138
		$number = Redis::hget('search.' . $this->hash, $this->name);
		if( $number === null )
139
		{
140
141
			die("test");
			return null;
142
143
		}else
		{
144
			return pfsockopen($this->getHost() . ":" . $this->port . "/$number", $this->port, $errstr, $errno, 1);
145
		}
146
	}
147

148
149
	public function retrieveResults()
	{
150
151
152
153
154
155
156
157
158
159
160
		$body = "";
		if( $this->cacheDuration > 0 && Cache::has($this->hash) )
		{
			$body = Cache::get($this->hash);
		}elseif ( Redis::hexists('search.' . $this->resultHash, $this->name) ) {
			$body = Redis::hget('search.' . $this->resultHash, $this->name);
			if( $this->cacheDuration > 0 )
				Cache::put($this->hash, $body, $this->cacheDuration);
		}

		if( $body !== "" )
161
		{
162
163
164
165
			$this->loadResults($body);
			$this->loaded = true;
			Redis::hdel('search.' . $this->hash, $this->name);
			return true;
166
167
168
169
		}else
		{
			return false;
		}		
170
171
	}

172
173
174
175
176
	public function shutdown()
	{
		Redis::del($this->host . "." . $this->socketNumber);
	}

177
178
179
180
181
182
183
184
185
186
187
188
189
	protected function getHost()
	{
		$return = "";
		if( $this->port === "443" )
		{
			$return .= "tls://";
		}else
		{
			$return .= "tcp://";
		}
		$return .= $this->host;
		return $return;
	}
190

191
192
193
194
195
	public function getCurlInfo()
	{
		return curl_getinfo($this->ch);
	}

196
197
198
199
200
	public function getCurlErrors()
	{
		return curl_errno($this->ch);
	}

201
202
203
	public function addCurlHandle ($mh)
	{
		curl_multi_add_handle($mh, $this->ch);
204
205
	}

206
207
208
209
210
	public function removeCurlHandle ($mh)
	{
		curl_multi_remove_handle($mh, $this->ch);
	}

211
	private function generateGetString($query, $url, $language, $category)
212
213
	{
		$getString = "";
214

215
		# Skript:
216
217
218
219
		if(strlen($this->skript) > 0)
			$getString .= $this->skript;
		else
			$getString .= "/";
220
		# FormData:
221
222
		if(strlen($this->formData) > 0)
			$getString .= "?" . $this->formData;
223
224
225
226
227
228
229
230

		# Wir müssen noch einige Platzhalter in dem GET-String ersetzen:
		if( strpos($getString, "<<USERAGENT>>") ){
			$getString = str_replace("<<USERAGENT>>", $this->urlEncode($this->useragent), $getString);
		}

		if( strpos($getString, "<<QUERY>>") )
		{
231
			$getString = str_replace("<<QUERY>>", $this->urlEncode($query), $getString);
232
233
234
235
		}

		if( strpos($getString, "<<IP>>") )
		{
236
			$getString = str_replace("<<IP>>", $this->urlEncode($this->ip), $getString);
237
238
239
240
		}

		if( strpos($getString, "<<LANGUAGE>>") )
		{
241
			$getString = str_replace("<<LANGUAGE>>", $this->urlEncode($language), $getString);
242
243
244
245
		}

		if( strpos($getString, "<<CATEGORY>>") )
		{
246
			$getString = str_replace("<<CATEGORY>>", $this->urlEncode($category), $getString);
247
		}
248
249
250

		if( strpos($getString, "<<AFFILDATA>>") )
		{
251
			$getString = str_replace("<<AFFILDATA>>", $this->getOvertureAffilData($url), $getString);
252
		}
253
254
255
		return $getString;
	}

256
	protected function urlEncode($string)
257
258
259
260
261
262
263
264
265
266
	{
		if(isset($this->inputEncoding))
		{
			return urlencode(mb_convert_encoding($string, $this->inputEncoding));
		}else
		{
			return urlencode($string);
		}
	}

267
	private function getOvertureAffilData($url)
268
	{
269
	    $affil_data = 'ip=' . $this->ip;
270
271
272
	    $affil_data .= '&ua=' . $this->useragent;  
	    if ( isset($_SERVER['HTTP_X_FORWARDED_FOR']) ) {
	       $affil_data .= '&xfip=' . $_SERVER['HTTP_X_FORWARDED_FOR'];
273
	    }
274
	    $affilDataValue = $this->urlEncode($affil_data);
275
		# Wir benötigen die ServeUrl:
276
		$serveUrl = $this->urlEncode($url);
277

278
279
		return "&affilData=" . $affilDataValue . "&serveUrl=" . $serveUrl;
	}
280
281
282
283
284

	public function isEnabled ()
	{
		return $this->enabled;
	}
285
}