Searchengine.php 6.43 KB
Newer Older
1
2
<?php

3
namespace App\Models;
4
use App\MetaGer;
5
6
use Log;
use Redis;
7
8
use App\Jobs\Search;
use Illuminate\Foundation\Bus\DispatchesJobs;
9

10
11

abstract class Searchengine 
12
{
13
	use DispatchesJobs;
14

15
	protected $ch; 	# Curl Handle zum erhalten der Ergebnisse
16
	public $fp;
17
18
19
20
	protected $getString = "";
	protected $engine;
    protected $counter = 0;
    protected $socketNumber = null;
21
    public $enabled = true;
22
	public $results = [];
23
24
25
	public $ads = [];
	public $write_time = 0;
	public $connection_time = 0;
26
	public $loaded = false;
27

28
	function __construct(\SimpleXMLElement $engine, MetaGer $metager)
29
30
31
32
	{
		foreach($engine->attributes() as $key => $value){
			$this->$key = $value->__toString();
		}
33
34
		if( !isset($this->homepage) )
			$this->homepage = "https://metager.de";
35
36
37
38
39
		$this->engine = $engine;

		# Wir registrieren die Benutzung dieser Suchmaschine
		$this->uses = intval(Redis::hget($this->name, "uses")) + 1;
		Redis::hset($this->name, "uses", $this->uses);
40

41
42
43
44
45
46
47
48
49
50
51
		# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
        if(isset($this->disabled) && strtotime($this->disabled) <= time() )
        {
        	# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
        	$this->enable($metager->getSumaFile(), "Die Suchmaschine " . $this->name . " wurde wieder eingeschaltet.");
        }elseif (isset($this->disabled) && strtotime($this->disabled) > time()) 
        {
        	$this->enabled = false;
        	return;
        }

52
53
54
55
56
57
58
59
		# User-Agent definieren:
		if( isset($_SERVER['HTTP_USER_AGENT']))
		{
			$this->useragent = $_SERVER['HTTP_USER_AGENT'];
		}else
		{
			$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
		}
60
		$this->ip = $metager->getIp();
61
62
		$this->gefVon = "<a href=\"" . $this->homepage . "\" target=\"_blank\">" . $this->displayName . "</a>";
		$this->startTime = microtime();
63
		
64
65
66
67
68
69
70
71
72
		$q = "";
		if( isset($this->hasSiteSearch) && $this->hasSiteSearch === "1")
		{
			$q = $metager->getQ() . " site:" . $metager->getSite();
		}else
		{
			$q = $metager->getQ();
		}
		$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
73
74
75
76
77
78
79
80
		$this->hash = $metager->getHashCode();

		# Die Anfragen an die Suchmaschinen werden nun von der Laravel-Queue bearbeitet:
		# Hinweis: solange in der .env der QUEUE_DRIVER auf "sync" gestellt ist, werden die Abfragen
		# nacheinander abgeschickt.
		# Sollen diese Parallel verarbeitet werden, muss ein anderer QUEUE_DRIVER verwendet werden.
		# siehe auch: https://laravel.com/docs/5.2/queues
		$this->dispatch(new Search($this->hash, $this->host, $this->port, $this->name, $this->getString, $this->useragent, $metager->getSumaFile()));
81
82
	}

83
	public abstract function loadResults($result);
84

85
	
86

87
88
89
90
91
92
93
	public function rank (\App\MetaGer $metager)
	{
		foreach($this->results as $result)
		{
			$result->rank($metager);
		}
	}
94

95
	
96

97
	private function setStatistic($key, $val)
98
	{
99

100
101
102
103
		$oldVal = floatval(Redis::hget($this->name, $key)) * $this->uses;
		$newVal = ($oldVal + max($val, 0)) / $this->uses;
		Redis::hset($this->name, $key, $newVal);
		$this->$key = $newVal;
104
105
	}

106
	
107

108
	public function enable($sumaFile, $message)
109
110
111
112
	{
		Log::info($message);
		$xml = simplexml_load_file($sumaFile);
		unset($xml->xpath("//sumas/suma[@name='" . $this->name . "']")['0']['disabled']);
113
114
115
116
117
118
119
120
		$xml->saveXML($sumaFile);
	}

	public function closeFp()
	{
		fclose($this->fp);
	}

121
	public function getSocket()
122
	{
123
124
		$number = Redis::hget('search.' . $this->hash, $this->name);
		if( $number === null )
125
		{
126
127
			die("test");
			return null;
128
129
		}else
		{
130
			return pfsockopen($this->getHost() . ":" . $this->port . "/$number", $this->port, $errstr, $errno, 1);
131
		}
132
	}
133

134
135
136
	public function retrieveResults()
	{
		if( Redis::hexists('search.' . $this->hash, $this->name))
137
		{
138
139
140
141
142
			$body = Redis::hget('search.' . $this->hash, $this->name);
			$this->loadResults($body);
			$this->loaded = true;
			Redis::hdel('search.' . $this->hash, $this->name);
			return true;
143
		}
144
145
		return false;
		
146
147
	}

148
149
150
151
152
	public function shutdown()
	{
		Redis::del($this->host . "." . $this->socketNumber);
	}

153
154
155
156
157
158
159
160
161
162
163
164
165
	protected function getHost()
	{
		$return = "";
		if( $this->port === "443" )
		{
			$return .= "tls://";
		}else
		{
			$return .= "tcp://";
		}
		$return .= $this->host;
		return $return;
	}
166

167
168
169
170
171
	public function getCurlInfo()
	{
		return curl_getinfo($this->ch);
	}

172
173
174
175
176
	public function getCurlErrors()
	{
		return curl_errno($this->ch);
	}

177
178
179
	public function addCurlHandle ($mh)
	{
		curl_multi_add_handle($mh, $this->ch);
180
181
	}

182
183
184
185
186
	public function removeCurlHandle ($mh)
	{
		curl_multi_remove_handle($mh, $this->ch);
	}

187
	private function generateGetString($query, $url, $language, $category)
188
189
	{
		$getString = "";
190

191
		# Skript:
192
193
194
195
		if(strlen($this->skript) > 0)
			$getString .= $this->skript;
		else
			$getString .= "/";
196
		# FormData:
197
198
		if(strlen($this->formData) > 0)
			$getString .= "?" . $this->formData;
199
200
201
202
203
204
205
206

		# Wir müssen noch einige Platzhalter in dem GET-String ersetzen:
		if( strpos($getString, "<<USERAGENT>>") ){
			$getString = str_replace("<<USERAGENT>>", $this->urlEncode($this->useragent), $getString);
		}

		if( strpos($getString, "<<QUERY>>") )
		{
207
			$getString = str_replace("<<QUERY>>", $this->urlEncode($query), $getString);
208
209
210
211
		}

		if( strpos($getString, "<<IP>>") )
		{
212
			$getString = str_replace("<<IP>>", $this->urlEncode($this->ip), $getString);
213
214
215
216
		}

		if( strpos($getString, "<<LANGUAGE>>") )
		{
217
			$getString = str_replace("<<LANGUAGE>>", $this->urlEncode($language), $getString);
218
219
220
221
		}

		if( strpos($getString, "<<CATEGORY>>") )
		{
222
			$getString = str_replace("<<CATEGORY>>", $this->urlEncode($category), $getString);
223
		}
224
225
226

		if( strpos($getString, "<<AFFILDATA>>") )
		{
227
			$getString = str_replace("<<AFFILDATA>>", $this->getOvertureAffilData($url), $getString);
228
		}
229
230
231
		return $getString;
	}

232
	protected function urlEncode($string)
233
234
235
236
237
238
239
240
241
242
	{
		if(isset($this->inputEncoding))
		{
			return urlencode(mb_convert_encoding($string, $this->inputEncoding));
		}else
		{
			return urlencode($string);
		}
	}

243
	private function getOvertureAffilData($url)
244
	{
245
	    $affil_data = 'ip=' . $this->ip;
246
247
248
	    $affil_data .= '&ua=' . $this->useragent;  
	    if ( isset($_SERVER['HTTP_X_FORWARDED_FOR']) ) {
	       $affil_data .= '&xfip=' . $_SERVER['HTTP_X_FORWARDED_FOR'];
249
	    }
250
	    $affilDataValue = $this->urlEncode($affil_data);
251
		# Wir benötigen die ServeUrl:
252
		$serveUrl = $this->urlEncode($url);
253

254
255
		return "&affilData=" . $affilDataValue . "&serveUrl=" . $serveUrl;
	}
256
257
258
259
260

	public function isEnabled ()
	{
		return $this->enabled;
	}
261
}