Commit 70afddc4 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '32-konstruktoren-fur-das-resultsobjekt-und-das-laden-der-suchmaschinen' into 'master'

Bis zu dem Punkt, an dem die Suchmaschinen geladen sind, ist jegliche Logik übernommen

Closes #32

See merge request !27
parents 797c234a 992bf2a6
...@@ -18,8 +18,11 @@ class MetaGerSearch extends Controller ...@@ -18,8 +18,11 @@ class MetaGerSearch extends Controller
{ {
# Mit gelieferte Formulardaten parsen und abspeichern: # Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request); $metager->parseFormData($request);
# Nach Spezialsuchen überprüfen: if($metager->getFokus() !== "bilder" )
$metager->checkSpecialSearches($request); {
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
}
# Alle Suchmaschinen erstellen # Alle Suchmaschinen erstellen
$metager->createSearchEngines($request); $metager->createSearchEngines($request);
# Ergebnisse der Suchmaschinen kombinieren: # Ergebnisse der Suchmaschinen kombinieren:
......
...@@ -4,6 +4,8 @@ namespace App; ...@@ -4,6 +4,8 @@ namespace App;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Jenssegers\Agent\Agent; use Jenssegers\Agent\Agent;
use App; use App;
use Storage;
use Log;
class MetaGer class MetaGer
{ {
...@@ -23,6 +25,7 @@ class MetaGer ...@@ -23,6 +25,7 @@ class MetaGer
protected $engines = []; protected $engines = [];
protected $results = []; protected $results = [];
protected $warnings = []; protected $warnings = [];
protected $errors = [];
# Daten über die Abfrage # Daten über die Abfrage
protected $ip; protected $ip;
protected $language; protected $language;
...@@ -32,10 +35,23 @@ class MetaGer ...@@ -32,10 +35,23 @@ class MetaGer
protected $mobile; protected $mobile;
protected $resultCount; protected $resultCount;
protected $sprueche; protected $sprueche;
protected $domainsBlacklisted = [];
protected $urlsBlacklisted = [];
protected $url;
function __construct() function __construct()
{ {
#$this->eingabe = Input::get('eingabe'); if( file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt") )
{
# Blacklists einlesen:
$tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
$this->domainsBlacklisted = explode("\n", $tmp);
$tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
$this->urlsBlacklisted = explode("\n", $tmp);
}else
{
Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
}
} }
public function createView() public function createView()
...@@ -49,7 +65,8 @@ class MetaGer ...@@ -49,7 +65,8 @@ class MetaGer
return view('metager3') return view('metager3')
->with('results', $viewResults) ->with('results', $viewResults)
->with('eingabe', $this->eingabe) ->with('eingabe', $this->eingabe)
->with('warnings', $this->warnings); ->with('warnings', $this->warnings)
->with('errors', $this->errors);
} }
public function combineResults () public function combineResults ()
...@@ -76,18 +93,22 @@ class MetaGer ...@@ -76,18 +93,22 @@ class MetaGer
foreach($sumas as $suma) foreach($sumas as $suma)
{ {
if($request->has($suma["service"]) if($request->has($suma["service"])
# || ( FOKUS !== "bilder" || ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo" && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads" || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" ) || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# ) )
# ) )
){ ){
if($suma["name"]->__toString() === "overture")
{ if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
$overtureEnabled = TRUE; {
} if($suma["name"]->__toString() === "overture")
$enabledSearchengines[] = $suma; {
$overtureEnabled = TRUE;
}
$enabledSearchengines[] = $suma;
}
} }
} }
}else{ }else{
...@@ -95,26 +116,38 @@ class MetaGer ...@@ -95,26 +116,38 @@ class MetaGer
foreach($sumas as $suma){ foreach($sumas as $suma){
$types = explode(",",$suma["type"]); $types = explode(",",$suma["type"]);
if(in_array($this->fokus, $types) if(in_array($this->fokus, $types)
# || ( FOKUS !== "bilder" || ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo" && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads" || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" ) || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# ) )
# ) )
){ ){
if($suma["name"]->__toString() === "overture") if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
{ {
$overtureEnabled = TRUE; if($suma["name"]->__toString() === "overture")
} {
$enabledSearchengines[] = $suma; $overtureEnabled = TRUE;
}
$enabledSearchengines[] = $suma;
}
} }
} }
} }
if( ( $this->fokus !== "bilder" && sizeof($enabledSearchengines) <= 3 ) || ( $this->fokus === "bilder" && sizeof($enabledSearchengines) === 0) )
{
$this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
}
$engines = []; $engines = [];
foreach($enabledSearchengines as $engine){ foreach($enabledSearchengines as $engine){
$path = "App\Models\parserSkripte\\" . ucfirst($engine["name"]->__toString()); $path = "App\Models\parserSkripte\\" . ucfirst($engine["name"]->__toString());
$engines[] = new $path($engine, $mh, $this->q, $this->time); $tmp = new $path($engine, $mh, $this);
if($tmp)
{
$engines[] = $tmp;
}
} }
# Nun führen wir die Get-Requests aus und warten auf alle Ergebnisse: # Nun führen wir die Get-Requests aus und warten auf alle Ergebnisse:
...@@ -133,6 +166,12 @@ class MetaGer ...@@ -133,6 +166,12 @@ class MetaGer
# Und auch den Multicurl-Handle: # Und auch den Multicurl-Handle:
curl_multi_close($mh); curl_multi_close($mh);
$string = ["Curl-Timings:"];
foreach($engines as $engine)
{
$string[] = $engine->getCurlInfo();
}
Log::debug($string);
$this->engines = $engines; $this->engines = $engines;
} }
...@@ -149,6 +188,7 @@ class MetaGer ...@@ -149,6 +188,7 @@ class MetaGer
} }
$request->replace($input); $request->replace($input);
} }
$this->url = $request->url();
# Zunächst überprüfen wir die eingegebenen Einstellungen: # Zunächst überprüfen wir die eingegebenen Einstellungen:
# FOKUS # FOKUS
$this->fokus = trans('fokiNames.' $this->fokus = trans('fokiNames.'
...@@ -322,4 +362,38 @@ class MetaGer ...@@ -322,4 +362,38 @@ class MetaGer
$this->warnings[] = "Sie führen eine Phrasensuche durch: \"" . $match[1] . "\""; $this->warnings[] = "Sie führen eine Phrasensuche durch: \"" . $match[1] . "\"";
} }
} }
public function getFokus ()
{
return $this->fokus;
}
public function getIp ()
{
return $this->ip;
}
public function getEingabe ()
{
return $this->eingabe;
}
public function getUrl ()
{
return $this->url;
}
public function getTime ()
{
return $this->time;
}
public function getLanguage ()
{
return $this->language;
}
public function getCategory ()
{
return $this->category;
}
} }
\ No newline at end of file
<?php <?php
namespace App\Models; namespace App\Models;
use App\MetaGer;
abstract class Searchengine abstract class Searchengine
{ {
protected $ch; # Curl Handle zum erhalten der Ergebnisse protected $ch; # Curl Handle zum erhalten der Ergebnisse
public $results = [];
function __construct(\SimpleXMLElement $engine, $mh, $query, $time) function __construct(\SimpleXMLElement $engine, $mh, MetaGer $metager)
{ {
foreach($engine->attributes() as $key => $value){ foreach($engine->attributes() as $key => $value){
$this->$key = $value->__toString(); $this->$key = $value->__toString();
...@@ -20,17 +22,33 @@ abstract class Searchengine ...@@ -20,17 +22,33 @@ abstract class Searchengine
{ {
$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1"; $this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
} }
$this->ip = $metager->getIp();
$this->ch = curl_init($this->generateGetString($query)); $this->ch = curl_init($this->generateGetString($metager->getEingabe(), $metager->getUrl(), $metager->getLanguage(), $metager->getCategory()) );
curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($this->ch, CURLOPT_FOLLOWLOCATION, true); curl_setopt($this->ch, CURLOPT_USERAGENT, $this->useragent); // set browser/user agent
curl_setopt($this->ch, CURLOPT_CONNECTTIMEOUT , $time); curl_setopt($this->ch, CURLOPT_FOLLOWLOCATION, 1); // automatically follow Location: headers (ie redirects)
curl_setopt($this->ch, CURLOPT_AUTOREFERER, 1); // auto set the referer in the event of a redirect
curl_setopt($this->ch, CURLOPT_MAXREDIRS, 5); // make sure we dont get stuck in a loop
curl_setopt($this->ch, CURLOPT_CONNECTTIMEOUT , $metager->getTime());
curl_setopt($this->ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 );
curl_setopt($this->ch, CURLOPT_TIMEOUT, 1); // 10s timeout time for cURL connection
if($this->port ==="443")
{
curl_setopt($this->ch, CURLOPT_SSL_VERIFYPEER, true); // allow https verification if true
curl_setopt($this->ch, CURLOPT_SSL_VERIFYHOST, 2); // check common name and verify with host name
curl_setopt($this->ch, CURLOPT_SSLVERSION,3); // verify ssl version 2 or 3
}
$this->addCurlHandle($mh); $this->addCurlHandle($mh);
} }
public abstract function loadResults(); public abstract function loadResults();
public function getCurlInfo()
{
return curl_getinfo($this->ch);
}
public function addCurlHandle ($mh) public function addCurlHandle ($mh)
{ {
curl_multi_add_handle($mh, $this->ch); curl_multi_add_handle($mh, $this->ch);
...@@ -41,7 +59,7 @@ abstract class Searchengine ...@@ -41,7 +59,7 @@ abstract class Searchengine
curl_multi_remove_handle($mh, $this->ch); curl_multi_remove_handle($mh, $this->ch);
} }
private function generateGetString($query) private function generateGetString($query, $url, $language, $category)
{ {
$getString = ""; $getString = "";
# Protokoll: # Protokoll:
...@@ -71,27 +89,27 @@ abstract class Searchengine ...@@ -71,27 +89,27 @@ abstract class Searchengine
if( strpos($getString, "<<IP>>") ) if( strpos($getString, "<<IP>>") )
{ {
$getString = str_replace("<<IP>>", $this->urlEncode(IP), $getString); $getString = str_replace("<<IP>>", $this->urlEncode($this->ip), $getString);
} }
if( strpos($getString, "<<LANGUAGE>>") ) if( strpos($getString, "<<LANGUAGE>>") )
{ {
$getString = str_replace("<<LANGUAGE>>", $this->urlEncode(LANGUAGE), $getString); $getString = str_replace("<<LANGUAGE>>", $this->urlEncode($language), $getString);
} }
if( strpos($getString, "<<CATEGORY>>") ) if( strpos($getString, "<<CATEGORY>>") )
{ {
$getString = str_replace("<<CATEGORY>>", $this->urlEncode(CATEGORY), $getString); $getString = str_replace("<<CATEGORY>>", $this->urlEncode($category), $getString);
} }
if( strpos($getString, "<<AFFILDATA>>") ) if( strpos($getString, "<<AFFILDATA>>") )
{ {
$getString = str_replace("<<AFFILDATA>>", $this->getOvertureAffilData(), $getString); $getString = str_replace("<<AFFILDATA>>", $this->getOvertureAffilData($url), $getString);
} }
return $getString; return $getString;
} }
private function urlEncode($string) protected function urlEncode($string)
{ {
if(isset($this->inputEncoding)) if(isset($this->inputEncoding))
{ {
...@@ -102,16 +120,16 @@ abstract class Searchengine ...@@ -102,16 +120,16 @@ abstract class Searchengine
} }
} }
private function getOvertureAffilData() private function getOvertureAffilData($url)
{ {
$affil_data = 'ip=' . IP; $affil_data = 'ip=' . $this->ip;
$affil_data .= '&ua=' . $this->useragent; $affil_data .= '&ua=' . $this->useragent;
if ( isset($_SERVER['HTTP_X_FORWARDED_FOR']) ) { if ( isset($_SERVER['HTTP_X_FORWARDED_FOR']) ) {
$affil_data .= '&xfip=' . $_SERVER['HTTP_X_FORWARDED_FOR']; $affil_data .= '&xfip=' . $_SERVER['HTTP_X_FORWARDED_FOR'];
} }
$affilDataValue = $this->urlEncode($affil_data); $affilDataValue = $this->urlEncode($affil_data);
# Wir benötigen die ServeUrl: # Wir benötigen die ServeUrl:
$serveUrl = $this->urlEncode(Request::url());# $serveUrl = $this->urlEncode($url);
return "&affilData=" . $affilDataValue . "&serveUrl=" . $serveUrl; return "&affilData=" . $affilDataValue . "&serveUrl=" . $serveUrl;
} }
......
...@@ -7,9 +7,14 @@ class Fastbot extends Searchengine ...@@ -7,9 +7,14 @@ class Fastbot extends Searchengine
{ {
public $results = []; public $results = [];
function __construct (\SimpleXMLElement $engine, $mh, $query, $time) function __construct (\SimpleXMLElement $engine, $mh, \App\MetaGer $metager)
{ {
parent::__construct($engine, $mh, $query, $time); parent::__construct($engine, $mh, $metager);
if ( strpos($this->urlEncode($metager->getEingabe()), "%") !== FALSE )
{
$this->removeCurlHandle($mh);
return FALSE;
}
} }
public function loadResults () public function loadResults ()
......
...@@ -8,9 +8,9 @@ class Onenewspagegermany extends Searchengine ...@@ -8,9 +8,9 @@ class Onenewspagegermany extends Searchengine
{ {
public $results = []; public $results = [];
function __construct (\SimpleXMLElement $engine, $mh, $query, $time) function __construct (\SimpleXMLElement $engine, $mh, $query, $time, $ip, $url)
{ {
parent::__construct($engine, $mh, $query, $time); parent::__construct($engine, $mh, $query, $time, $ip, $url);
} }
public function loadResults () public function loadResults ()
......
<?php
namespace App\Models\parserSkripte;
use App\Models\Searchengine;
class OvertureAds extends Searchengine
{
function __construct (\SimpleXMLElement $engine, $mh, \App\MetaGer $metager)
{
parent::__construct($engine, $mh, $metager);
}
public function loadResults ()
{
$result = utf8_encode(curl_multi_getcontent($this->ch));
#die($result);
}
}
\ No newline at end of file
<?php
namespace App\Models\parserSkripte;
use App\Models\Searchengine;
class Qualigo extends Searchengine
{
function __construct (\SimpleXMLElement $engine, $mh, \App\MetaGer $metager)
{
parent::__construct($engine, $mh, $metager);
}
public function loadResults ()
{
$result = utf8_encode(curl_multi_getcontent($this->ch));
}
}
\ No newline at end of file
<?php
namespace App\Models\parserSkripte;
use App\Models\Searchengine;
class Similar_product_ads extends Searchengine
{
function __construct (\SimpleXMLElement $engine, $mh, \App\MetaGer $metager)
{
parent::__construct($engine, $mh, $metager);
$tmp = $metager->getEingabe();
$tmp = preg_replace("/\W/si", "", $tmp);
if(strlen($tmp) < 3)
{
$this->removeCurlHandle($mh);
}
}
public function loadResults ()
{
$result = utf8_encode(curl_multi_getcontent($this->ch));
}
}
\ No newline at end of file
*.xml *.xml
\ No newline at end of file *.txt
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment