Skip to content

GitLab

Commit 755aeb1d authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files
Options

Socketbasis zum Abrufen der Ergebnisse eingefügt

parent 70afddc4
Hide whitespace changes
Inline Side-by-side
Showing with 1165 additions and 165 deletions
app/Http/Controllers/MetaGerSearch.php
View file @ 755aeb1d
......@@ -24,7 +24,9 @@ class MetaGerSearch extends Controller
$metager->checkSpecialSearches($request);
}
# Alle Suchmaschinen erstellen
$metager->createSearchEngines($request);
# Ergebnisse der Suchmaschinen kombinieren:
$metager->combineResults();
# Die Ausgabe erstellen:
......
app/MetaGer.php
View file @ 755aeb1d
......@@ -3,6 +3,7 @@ namespace App;
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
use App\Models\SocketRocket;
use App;
use Storage;
use Log;
......@@ -41,6 +42,8 @@ class MetaGer
function __construct()
{
define('CRLF', "\r\n");
define('BUFFER_LENGTH', 8192);
if( file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt") )
{
# Blacklists einlesen:
......@@ -79,8 +82,9 @@ class MetaGer
public function createSearchEngines (Request $request)
{
# Curl-Multihandle um die Ergebnisse abzurufen:
$mh = curl_multi_init();
#die(SocketRocket::get("tls", "dominik-pfennig.de", "", 443));
# Überprüfe, welche Sumas eingeschaltet sind
$xml = simplexml_load_file($this->sumaFile);
......@@ -93,12 +97,13 @@ class MetaGer
foreach($sumas as $suma)
{
if($request->has($suma["service"])
|| ( $this->fokus !== "bilder"
&& ($suma["name"]->__toString() === "qualigo"
|| $suma["name"]->__toString() === "similar_product_ads"
|| ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
)
)
#|| ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# )
# )
#|| 1 === 1 #Todo: entfernen
){
if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
......@@ -116,12 +121,12 @@ class MetaGer
foreach($sumas as $suma){
$types = explode(",",$suma["type"]);
if(in_array($this->fokus, $types)
|| ( $this->fokus !== "bilder"
&& ($suma["name"]->__toString() === "qualigo"
|| $suma["name"]->__toString() === "similar_product_ads"
|| ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
)
)
#|| ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# )
# )
){
if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
{
......@@ -142,37 +147,22 @@ class MetaGer
$engines = [];
foreach($enabledSearchengines as $engine){
$path = "App\Models\parserSkripte\\" . ucfirst($engine["name"]->__toString());
$tmp = new $path($engine, $mh, $this);
$path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
$tmp = new $path($engine, $this);
if($tmp)
{
$engines[] = $tmp;
}
}
# Nun führen wir die Get-Requests aus und warten auf alle Ergebnisse:
$running = null;
do
{
curL_multi_exec($mh, $running);
}while($running);
# Und beenden noch alle Handles
foreach($engines as $engine)
{
$engine->removeCurlHandle($mh);
$engine->loadResults();
}
# Und auch den Multicurl-Handle:
curl_multi_close($mh);
$string = ["Curl-Timings:"];
foreach($engines as $engine)
{
$string[] = $engine->getCurlInfo();
$engine->retrieveResults();
# $engine->closeFp();
}
Log::debug($string);
$this->engines = $engines;
}
......@@ -396,4 +386,9 @@ class MetaGer
{
return $this->category;
}
public function getSumaFile ()
{
return $this->sumaFile;
}
}
\ No newline at end of file
app/Models/Result.php
View file @ 755aeb1d
......@@ -7,10 +7,11 @@ class Result
function __construct ( $titel, $link, $anzeigeLink , $descr, $gefVon )
{
$this->titel = trim($titel);
$this->titel = strip_tags(trim($titel));
$this->link = trim($link);
$this->anzeigeLink = trim($anzeigeLink);
$this->descr = trim($descr);
$this->descr = strip_tags(trim($descr));
$this->descr = preg_replace("/\n+/si", " ", $this->descr);
$this->gefVon = trim($gefVon);
}
}
\ No newline at end of file
app/Models/Search.php
View file @ 755aeb1d
......@@ -6,6 +6,7 @@ use App\MetaGer\Searchengine;
class Search
{
public static function loadSearchEngines(Request $request)
{
......
app/Models/Searchengine.php
View file @ 755aeb1d
......@@ -2,18 +2,28 @@
namespace App\Models;
use App\MetaGer;
use Log;
use Redis;
abstract class Searchengine
{
protected $ch; # Curl Handle zum erhalten der Ergebnisse
protected $fp;
protected $getString = "";
protected $engine;
protected $counter = 0;
protected $enabled;
protected $socketNumber = null;
public $results = [];
function __construct(\SimpleXMLElement $engine, $mh, MetaGer $metager)
function __construct(\SimpleXMLElement $engine, MetaGer $metager)
{
foreach($engine->attributes() as $key => $value){
$this->$key = $value->__toString();
}
# User-Agent definieren:
if( isset($_SERVER['HTTP_USER_AGENT']))
{
......@@ -23,32 +33,308 @@ abstract class Searchengine
$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
}
$this->ip = $metager->getIp();
$this->ch = curl_init($this->generateGetString($metager->getEingabe(), $metager->getUrl(), $metager->getLanguage(), $metager->getCategory()) );
curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($this->ch, CURLOPT_USERAGENT, $this->useragent); // set browser/user agent
curl_setopt($this->ch, CURLOPT_FOLLOWLOCATION, 1); // automatically follow Location: headers (ie redirects)
curl_setopt($this->ch, CURLOPT_AUTOREFERER, 1); // auto set the referer in the event of a redirect
curl_setopt($this->ch, CURLOPT_MAXREDIRS, 5); // make sure we dont get stuck in a loop
curl_setopt($this->ch, CURLOPT_CONNECTTIMEOUT , $metager->getTime());
curl_setopt($this->ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 );
curl_setopt($this->ch, CURLOPT_TIMEOUT, 1); // 10s timeout time for cURL connection
if($this->port ==="443")
{
curl_setopt($this->ch, CURLOPT_SSL_VERIFYPEER, true); // allow https verification if true
curl_setopt($this->ch, CURLOPT_SSL_VERIFYHOST, 2); // check common name and verify with host name
curl_setopt($this->ch, CURLOPT_SSLVERSION,3); // verify ssl version 2 or 3
}
$this->addCurlHandle($mh);
$this->gefVon = "<a href=\"" . $this->homepage . "\" target=\"_blank\">" . $this->displayName . "</a>";
$this->startTime = microtime();
$this->getString = $this->generateGetString($metager->getEingabe(), $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
$counter = 0;
# Wir benötigen einen verfügbaren Socket, über den wir kommunizieren können:
$this->fp = $this->getFreeSocket();
do
{
#try
#{
/*}catch(\Exception $e)
{
# Connection Timeout: Wir schalten die Suchmaschine aus:
$this->disable($metager->getSumaFile(), "Die Suchmaschine " . $this->name . " wurde deaktiviert, weil keine Verbindung aufgebaut werden konnte");
break;
}*/
if(!$this->fp)
{
// Mache etwas
Log::error("Konnte keine Verbindung zur Suchmaschine: " . $this->name . " aufbauen.");
break;
}else
{
$out = "GET " . $this->getString . " HTTP/1.1\r\n";
$out .= "Host: " . $this->host . "\r\n";
$out .= "User-Agent: " . $this->useragent . "\r\n";
$out .= "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n";
$out .= "Accept-Language: de,en-US;q=0.7,en;q=0.3\r\n";
$out .= "Accept-Encoding: gzip, deflate, br\r\n";
$out .= "Connection: keep-alive\r\n\r\n";
// Daten senden:
#try
#{
if(fwrite($this->fp, $out))
{
break;
}else
{
abort(500, "Fehler beim schreiben.");
}
#}catch(\Exception $e)
#{
# fclose($this->fp);
# if($counter >= 5)
# {
#
# Log::error("Konnte auch nach 6 Versuchen keine schreibbare Verbindung zum Server \"" . $this->host . "\" aufbauen.");
# break;
# }
#}
}
}while(true);
}
public abstract function loadResults(String $result);
private function getFreeSocket()
{
# Je nach Auslastung des Servers ( gleichzeitige Abfragen ), kann es sein, dass wir mehrere Sockets benötigen um die Abfragen ohne Wartezeit beantworten zu können.
# pfsockopen öffnet dabei einen persistenten Socket, der also auch zwischen den verschiedenen php Prozessen geteilt werden kann.
# Wenn der Hostname mit einem bereits erstellten Socket übereinstimmt, wird die Verbindung also aufgegriffen und fortgeführt.
# Allerdings dürfen wir diesen nur verwenden, wenn er nicht bereits von einem anderen Prozess zur Kommunikation verwendet wird.
# Wenn dem so ist, probieren wir den nächsten Socket zu verwenden.
# Dies festzustellen ist komplizierter, als man sich das vorstellt. Folgendes System sollte funktionieren:
# 1. Stelle fest, ob dieser Socket neu erstellt wurde, oder ob ein existierender geöffnet wurde.
$counter = 0;
do
{
if(intval(Redis::getBit($this->name, $counter)) === 0)
{
#die($counter . "Socket.");
Redis::setBit($this->name, $counter, 1);
$this->socketNumber = $counter;
try
{
$fp = pfsockopen($this->getHost() . ":" . $this->port . "/$counter", $this->port, $errstr, $errno, 1);
return $fp;
}catch(\ErrorException $e)
{
return null;
}
}
$counter++;
}while($counter < 20);
# Wenn wir hier hin kommen, läuft etwas falsch! Wir kappen alle bestehenden Verbindungen:
$connectionError = false;
for($i = 0; $i < 20; $i++)
{
if(!$connectionError)
{
try
{
$fp = pfsockopen($this->getHost() . ":" . $this->port . "/$counter", $this->port, $errstr, $errno, 1);
fclose($fp);
}catch(\ErrorException $e)
{
$connectionError = true;
}
}
Redis::setBit($this->name, $i, 0);
}
abort(500, "Konnte keinen freien Socket bekommen für: " . $this->name);
}
public function disable(string $sumaFile, string $message)
{
Log::info($message);
$xml = simplexml_load_file($sumaFile);
$xml->xpath("//sumas/suma[@name='" . $this->name . "']")['0']['disabled'] = "1";
$xml->saveXML($sumaFile);
}
public function closeFp()
{
fclose($this->fp);
}
public function retrieveResults()
{
$headers = '';
$body = '';
$length = 0;
if(!$this->fp)
{
return;
}
// We need a waiting Loop to wait till the server is ready
// get headers FIRST
stream_set_blocking($this->fp, 1);
do
{
// use fgets() not fread(), fgets stops reading at first newline
// or buffer which ever one is reached first
$data = fgets($this->fp, BUFFER_LENGTH);
if($data === false)
{
usleep(10000);
continue;
}
// a sincle CRLF indicates end of headers
if ($data === false || $data == CRLF || feof($this->fp)) {
// break BEFORE OUTPUT
break;
}
if( sizeof(($tmp = explode(": ", $data))) === 2 )
$headers[trim($tmp[0])] = trim($tmp[1]);
}
while (true);
// end of headers
if( isset($headers["Transfer-Encoding"]) && $headers["Transfer-Encoding"] === "chunked" )
{
$body = $this->readChunked();
}elseif( isset($headers['Content-Length']) )
{
$length = trim($headers['Content-Length']);
if(is_numeric($length) && $length >= 1)
$body = fread($this->fp, $headers['Content-Length']);
}
Redis::setBit($this->name, $this->socketNumber, 0 );
if( isset($headers["Content-Encoding"]) && $headers['Content-Encoding'] === "gzip")
{
$body = $this->gunzip($body);
}
#die(print_r($headers));
// $body and $headers should contain your stream data
$this->loadResults($body);
#print(print_r($headers, TRUE) . $body);
#exit;
}
public abstract function loadResults();
private function readChunked()
{
$body = '';
// read from chunked stream
// loop though the stream
do
{
// NOTE: for chunked encoding to work properly make sure
// there is NOTHING (besides newlines) before the first hexlength
// get the line which has the length of this chunk (use fgets here)
$line = fgets($this->fp, BUFFER_LENGTH);
// if it's only a newline this normally means it's read
// the total amount of data requested minus the newline
// continue to next loop to make sure we're done
if ($line == CRLF) {
continue;
}
// the length of the block is sent in hex decode it then loop through
// that much data get the length
// NOTE: hexdec() ignores all non hexadecimal chars it finds
$length = hexdec($line);
if (!is_int($length)) {
trigger_error('Most likely not chunked encoding', E_USER_ERROR);
}
// zero is sent when at the end of the chunks
// or the end of the stream or error
if ($line === false || $length < 1 || feof($this->fp)) {
if($length <= 0)
fgets($this->fp, BUFFER_LENGTH);
// break out of the streams loop
break;
}
// loop though the chunk
do
{
// read $length amount of data
// (use fread here)
$data = fread($this->fp, $length);
// remove the amount received from the total length on the next loop
// it'll attempt to read that much less data
$length -= strlen($data);
// PRINT out directly
#print $data;
#flush();
// you could also save it directly to a file here
// store in string for later use
$body .= $data;
// zero or less or end of connection break
if ($length <= 0 || feof($this->fp))
{
// break out of the chunk loop
if($length <= 0)
fgets($this->fp, BUFFER_LENGTH);
break;
}
}
while (true);
// end of chunk loop
}
while (true);
// end of stream loop
return $body;
}
private function gunzip($zipped) {
$offset = 0;
if (substr($zipped,0,2) == "\x1f\x8b")
$offset = 2;
if (substr($zipped,$offset,1) == "\x08")
{
try
{
return gzinflate(substr($zipped, $offset + 8));
} catch (\Exception $e)
{
abort(500, "Fehler beim unzip des Ergebnisses von folgendem Anbieter: " . $this->name);
}
}
return "Unknown Format";
}
protected function getHost()
{
$return = "";
if( $this->port === "443" )
{
$return .= "tls://";
}else
{
$return .= "tcp://";
}
$return .= $this->host;
return $return;
}
public function getCurlInfo()
{
return curl_getinfo($this->ch);
}
public function getCurlErrors()
{
return curl_errno($this->ch);
}
public function addCurlHandle ($mh)
{
curl_multi_add_handle($mh, $this->ch);
......@@ -62,20 +348,15 @@ abstract class Searchengine
private function generateGetString($query, $url, $language, $category)
{
$getString = "";
# Protokoll:
if($this->port === "443"){
$getString .= "https://";
}else{
$getString .= "http://";
}
# Host:
$getString .= $this->host;
# Port:
$getString .= ":" . $this->port;
# Skript:
$getString .= $this->skript;
if(strlen($this->skript) > 0)
$getString .= $this->skript;
else
$getString .= "/";
# FormData:
$getString .= "?" . $this->formData;
if(strlen($this->formData) > 0)
$getString .= "?" . $this->formData;
# Wir müssen noch einige Platzhalter in dem GET-String ersetzen:
if( strpos($getString, "<<USERAGENT>>") ){
......@@ -133,4 +414,9 @@ abstract class Searchengine
return "&affilData=" . $affilDataValue . "&serveUrl=" . $serveUrl;
}
public function isEnabled ()
{
return $this->enabled;
}
}
\ No newline at end of file
app/Models/parserSkripte/Allesklar.php 0 → 100644
View file @ 755aeb1d
<?php
namespace App\Models\parserSkripte;
use App\Models\Searchengine;
use Symfony\Component\DomCrawler\Crawler;
class Allesklar extends Searchengine
{
protected $tds = "";
function __construct (\SimpleXMLElement $engine, $mh, \App\MetaGer $metager)
{
parent::__construct($engine, $mh, $metager);
}
public function loadResults (String $result)
{
$crawler = new Crawler($result);
$crawler = $crawler
->filter('table[width=585]')
->reduce(function(Crawler $node, $i) {
if($i < 5)
{
return false;
}
});
$this->counter = 0;
$crawler->filter('table')->each(function (Crawler $node, $i)
{
try {
$this->string = "";
$titleTag = $node->filter('tr > td > a')->first();
$title = trim($titleTag->filter('a')->text());
$link = $titleTag->filter('a')->attr('href');
if($i === 0)
{
$descr = trim($node->filter('tr > td.bodytext')->eq(3)->text());
}else
{
$descr = trim($node->filter('tr > td.bodytext')->eq(2)->text());
}
$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$link,
$descr,
$this->gefVon,
$this->counter
);
} catch (\InvalidArgumentException $e)
{
}
});
}
}
\ No newline at end of file
app/Models/parserSkripte/BASE.php 0 → 100644
View file @ 755aeb1d
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;