Commit 755aeb1d authored by Dominik Hebeler's avatar Dominik Hebeler

Socketbasis zum Abrufen der Ergebnisse eingefügt

parent 70afddc4
......@@ -24,7 +24,9 @@ class MetaGerSearch extends Controller
$metager->checkSpecialSearches($request);
}
# Alle Suchmaschinen erstellen
$metager->createSearchEngines($request);
# Ergebnisse der Suchmaschinen kombinieren:
$metager->combineResults();
# Die Ausgabe erstellen:
......
......@@ -3,6 +3,7 @@ namespace App;
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
use App\Models\SocketRocket;
use App;
use Storage;
use Log;
......@@ -41,6 +42,8 @@ class MetaGer
function __construct()
{
define('CRLF', "\r\n");
define('BUFFER_LENGTH', 8192);
if( file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt") )
{
# Blacklists einlesen:
......@@ -79,8 +82,9 @@ class MetaGer
public function createSearchEngines (Request $request)
{
# Curl-Multihandle um die Ergebnisse abzurufen:
$mh = curl_multi_init();
#die(SocketRocket::get("tls", "dominik-pfennig.de", "", 443));
# Überprüfe, welche Sumas eingeschaltet sind
$xml = simplexml_load_file($this->sumaFile);
......@@ -93,12 +97,13 @@ class MetaGer
foreach($sumas as $suma)
{
if($request->has($suma["service"])
|| ( $this->fokus !== "bilder"
&& ($suma["name"]->__toString() === "qualigo"
|| $suma["name"]->__toString() === "similar_product_ads"
|| ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
)
)
#|| ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# )
# )
#|| 1 === 1 #Todo: entfernen
){
if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
......@@ -116,12 +121,12 @@ class MetaGer
foreach($sumas as $suma){
$types = explode(",",$suma["type"]);
if(in_array($this->fokus, $types)
|| ( $this->fokus !== "bilder"
&& ($suma["name"]->__toString() === "qualigo"
|| $suma["name"]->__toString() === "similar_product_ads"
|| ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
)
)
#|| ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# )
# )
){
if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
{
......@@ -142,37 +147,22 @@ class MetaGer
$engines = [];
foreach($enabledSearchengines as $engine){
$path = "App\Models\parserSkripte\\" . ucfirst($engine["name"]->__toString());
$tmp = new $path($engine, $mh, $this);
$path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
$tmp = new $path($engine, $this);
if($tmp)
{
$engines[] = $tmp;
}
}
# Nun führen wir die Get-Requests aus und warten auf alle Ergebnisse:
$running = null;
do
{
curL_multi_exec($mh, $running);
}while($running);
# Und beenden noch alle Handles
foreach($engines as $engine)
{
$engine->removeCurlHandle($mh);
$engine->loadResults();
}
# Und auch den Multicurl-Handle:
curl_multi_close($mh);
$string = ["Curl-Timings:"];
foreach($engines as $engine)
{
$string[] = $engine->getCurlInfo();
$engine->retrieveResults();
# $engine->closeFp();
}
Log::debug($string);
$this->engines = $engines;
}
......@@ -396,4 +386,9 @@ class MetaGer
{
return $this->category;
}
public function getSumaFile ()
{
return $this->sumaFile;
}
}
\ No newline at end of file
......@@ -7,10 +7,11 @@ class Result
function __construct ( $titel, $link, $anzeigeLink , $descr, $gefVon )
{
$this->titel = trim($titel);
$this->titel = strip_tags(trim($titel));
$this->link = trim($link);
$this->anzeigeLink = trim($anzeigeLink);
$this->descr = trim($descr);
$this->descr = strip_tags(trim($descr));
$this->descr = preg_replace("/\n+/si", " ", $this->descr);
$this->gefVon = trim($gefVon);
}
}
\ No newline at end of file
......@@ -6,6 +6,7 @@ use App\MetaGer\Searchengine;
class Search
{
public static function loadSearchEngines(Request $request)
{
......
This diff is collapsed.
<?php
namespace App\Models\parserSkripte;
use App\Models\Searchengine;
use Symfony\Component\DomCrawler\Crawler;
class Allesklar extends Searchengine
{
protected $tds = "";
function __construct (\SimpleXMLElement $engine, $mh, \App\MetaGer $metager)
{
parent::__construct($engine, $mh, $metager);
}
public function loadResults (String $result)
{
$crawler = new Crawler($result);
$crawler = $crawler
->filter('table[width=585]')
->reduce(function(Crawler $node, $i) {
if($i < 5)
{
return false;
}
});
$this->counter = 0;
$crawler->filter('table')->each(function (Crawler $node, $i)
{
try {
$this->string = "";
$titleTag = $node->filter('tr > td > a')->first();
$title = trim($titleTag->filter('a')->text());
$link = $titleTag->filter('a')->attr('href');
if($i === 0)
{
$descr = trim($node->filter('tr > td.bodytext')->eq(3)->text());
}else
{
$descr = trim($node->filter('tr > td.bodytext')->eq(2)->text());
}
$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$link,
$descr,
$this->gefVon,
$this->counter
);
} catch (\InvalidArgumentException $e)
{
}
});
}
}
\ No newline at end of file
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class BASE extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine,\App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults (String $result)
{
$title = "";
$link = "";
$anzeigeLink = $link;
$descr = "";
#die($result);
/*$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);*/
}
}
\ No newline at end of file
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Symfony\Component\DomCrawler\Crawler;
class Bing extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine, $mh, \App\MetaGer $metager)
{
parent::__construct($engine, $mh, $metager);
}
public function loadResults (String $result)
{
$crawler = new Crawler($result);
$crawler->filter('ol#b_results > li.b_algo')->each(function (Crawler $node, $i)
{
$title = $node->filter('li h2 > a')->text();
$link = $node->filter('li h2 > a')->attr('href');
$anzeigeLink = $link;
$descr = $node->filter('li div > p')->text();
#die($result);
$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
} );
}
}
\ No newline at end of file
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class Dmoznebel extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults (String $result)
{
$title = "";
$link = "";
$anzeigeLink = $link;
$descr = "";
die($result);
$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
}
}
\ No newline at end of file
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class Exalead extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults (String $result)
{
$title = "";
$link = "";
$anzeigeLink = $link;
$descr = "";
#die($result);
/*$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
); */
}
}
\ No newline at end of file
......@@ -7,19 +7,19 @@ class Fastbot extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine, $mh, \App\MetaGer $metager)
function __construct (\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $mh, $metager);
parent::__construct($engine, $metager);
if ( strpos($this->urlEncode($metager->getEingabe()), "%") !== FALSE )
{
$this->removeCurlHandle($mh);
return FALSE;
return null;
}
}
public function loadResults ()
public function loadResults (String $result)
{
$result = utf8_encode(curl_multi_getcontent($this->ch));
$result = utf8_encode($result);
$counter = 0;
foreach( explode("\n", $result) as $line )
{
$line = trim($line);
......@@ -29,12 +29,14 @@ class Fastbot extends Searchengine
$link = $result[1];
$link = substr($link, strpos($link, "href=\"") + 6);
$link = substr($link, 0, strpos($link, "\""));
$counter++;
$this->results[] = new \App\Models\Result(
trim(strip_tags($result[1])),
$link,
$result[3],
$result[2],
"<a href=\"http://www.fastbot.de\">fastbot</a>"
$this->gefVon,
$counter
);
}
......
<?php
namespace App\Models\parserSkripte;
use App\Models\Searchengine;
class Minisucher extends Searchengine
{
function __construct (\SimpleXMLElement $engine,\App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults (String $content)
{
$content = simplexml_load_string($content);
if(!$content)
{
return;
}
$results = $content->xpath('//response/result/doc');
$string = "";
$counter = 0;
foreach($results as $result)
{
$counter++;
$result = simplexml_load_string($result->saveXML());
$title = $result->xpath('//doc/arr[@name="title"]/str')[0]->__toString();
$link = $result->xpath('//doc/str[@name="url"]')[0]->__toString();
$anzeigeLink = $link;
$descr = "";
$descriptions = $content->xpath("//response/lst[@name='highlighting']/lst[@name='$link']/arr[@name='content']/str");
foreach($descriptions as $description)
{
$descr .= $description->__toString();
}
$descr = strip_tags($descr);
$provider = $result->xpath('//doc/str[@name="subcollection"]')[0]->__toString();
$this->results[] = new \App\Models\Result(
$title,
$link,
$link,
$descr,
$this->gefVon,
$counter
);
}
}
}
\ No newline at end of file
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Symfony\Component\DomCrawler\Crawler;
class Mnogosearch extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine, $mh, \App\MetaGer $metager)
{
parent::__construct($engine, $mh, $metager);
}
public function loadResults ()
{
$result = curl_multi_getcontent($this->ch);
$counter = 0;
$crawler = new Crawler($result);
$crawler->filter('table[width=600]')
->reduce(function (Crawler $node, $i)
{
if(strpos($node->text(), "Result pages:") !== FALSE)
{
return false;
}
})
->each(function(Crawler $node, $i)
{
$title = $node->filter('table > tr > td ')->eq(1)->filter('td > div')->text();
$title = preg_replace("/\s+/si", " ", $title);
$link = $node->filter('table > tr > td ')->eq(1)->filter('td > div > a')->attr('href');
$anzeigeLink = $link;
$descr = $node->filter('table > tr > td ')->eq(1)->filter('td > div')->eq(1)->text();
$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
});
}
}
\ No newline at end of file
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class Nebel extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine,\App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults (String $result)
{
$results = trim($result);
foreach( explode("\n", $results) as $result )
{
$res = explode("|", $result);
if(sizeof($res) < 3)
continue;
$title = $res[2];
$link = $res[0];
$anzeigeLink = $link;
$descr = $res[1];
$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
}
}
}
\ No newline at end of file
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class Onenewspage extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults (String $result)
{
$results = trim($result);
foreach( explode("\n", $results) as $result )
{
$res = explode("|", $result);
$title = $res[0];
$link = $res[2];
$anzeigeLink = $link;
$descr = $res[1];
$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
}
}
}
\ No newline at end of file
......@@ -8,26 +8,28 @@ class Onenewspagegermany extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine, $mh, $query, $time, $ip, $url)
function __construct (\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $mh, $query, $time, $ip, $url);
parent::__construct($engine, $metager);
}
public function loadResults ()
public function loadResults (String $result)
{
$result = curl_multi_getcontent($this->ch);
$counter = 0;
foreach( explode("\n", $result) as $line )
{
$line = trim($line);
if( strlen($line) > 0 ){
# Hier bekommen wir jedes einzelne Ergebnis
$result = explode("|", $line);
$counter++;
$this->results[] = new Result(
trim(strip_tags($result[0])),
$result[2],
$result[2],
$result[1],
"<a href=\"http://www.newsdeutschland.com/videos.htm\">newsdeutschland.com</a>"
$this->gefVon,
$counter
);
}
......
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class Onenewspagevideo extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults (String $result)
{
$results = trim($result);
foreach( explode("\n", $results) as $result )
{
$res = explode("|", $result);
$title = $res[0];
$link = $res[2];
$anzeigeLink = $link;
$descr = $res[1];
$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
}
}
}
\ No newline at end of file
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
class Overture extends Searchengine
{
public $results = [];
function __construct (\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults (String $result)
{
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if(!$content)
{
return;
}
$results = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
foreach($results as $result)