Commit c5624d37 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Einen großen Teil der Suche eingebaut

parent 0b026dbd
...@@ -14,8 +14,9 @@ use App\MetaGer; ...@@ -14,8 +14,9 @@ use App\MetaGer;
class MetaGerSearch extends Controller class MetaGerSearch extends Controller
{ {
public function test(Request $request, MetaGer $metager) public function search(Request $request, MetaGer $metager)
{ {
$time = microtime();
# Mit gelieferte Formulardaten parsen und abspeichern: # Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request); $metager->parseFormData($request);
if($metager->getFokus() !== "bilder" ) if($metager->getFokus() !== "bilder" )
...@@ -24,28 +25,22 @@ class MetaGerSearch extends Controller ...@@ -24,28 +25,22 @@ class MetaGerSearch extends Controller
$metager->checkSpecialSearches($request); $metager->checkSpecialSearches($request);
} }
# Alle Suchmaschinen erstellen # Alle Suchmaschinen erstellen
$metager->createSearchEngines($request); $metager->createSearchEngines($request);
# Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll();
# Ergebnisse der Suchmaschinen kombinieren: # Ergebnisse der Suchmaschinen kombinieren:
$metager->combineResults(); $metager->combineResults();
$metager->removeInvalids();
# Die Ausgabe erstellen: # Die Ausgabe erstellen:
return $metager->createView(); return $metager->createView();
} }
public function search(Request $request) public function quicktips(Request $request)
{ {
$searchengines = Search::loadSearchEngines($request);
$results = new Results($searchengines);
return print_r( $viewResults, TRUE);
} }
} }
\ No newline at end of file
...@@ -56,8 +56,6 @@ ...@@ -56,8 +56,6 @@
Route::post('kontakt', 'MailController@contactMail'); Route::post('kontakt', 'MailController@contactMail');
Route::get('meta/meta.ger3', 'MetaGerSearch@search');
Route::get('spende', function() Route::get('spende', function()
{ {
return view('spende') return view('spende')
...@@ -80,5 +78,7 @@ ...@@ -80,5 +78,7 @@
->with('css', 'help.css'); ->with('css', 'help.css');
}); });
Route::get('meta/meta.ger3', 'MetaGerSearch@test'); Route::get('meta/meta.ger3', 'MetaGerSearch@search');
Route::get('qt', 'MetaGerSearch@quicktips');
}); });
...@@ -3,10 +3,14 @@ namespace App; ...@@ -3,10 +3,14 @@ namespace App;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Jenssegers\Agent\Agent; use Jenssegers\Agent\Agent;
use App\Models\SocketRocket;
use App; use App;
use Storage; use Storage;
use Log; use Log;
use App\lib\TextLanguageDetect\TextLanguageDetect;
use App\lib\TextLanguageDetect\LanguageDetect\TextLanguageDetectException;
use Illuminate\Pagination\LengthAwarePaginator;
use Illuminate\Support\Collection;
#use \Illuminate\Pagination\Paginator;
class MetaGer class MetaGer
{ {
...@@ -25,8 +29,10 @@ class MetaGer ...@@ -25,8 +29,10 @@ class MetaGer
protected $stopWords = []; protected $stopWords = [];
protected $engines = []; protected $engines = [];
protected $results = []; protected $results = [];
protected $ads = [];
protected $warnings = []; protected $warnings = [];
protected $errors = []; protected $errors = [];
protected $addedHosts = [];
# Daten über die Abfrage # Daten über die Abfrage
protected $ip; protected $ip;
protected $language; protected $language;
...@@ -39,9 +45,11 @@ class MetaGer ...@@ -39,9 +45,11 @@ class MetaGer
protected $domainsBlacklisted = []; protected $domainsBlacklisted = [];
protected $urlsBlacklisted = []; protected $urlsBlacklisted = [];
protected $url; protected $url;
protected $languageDetect;
function __construct() function __construct()
{ {
$this->time = microtime();
define('CRLF', "\r\n"); define('CRLF', "\r\n");
define('BUFFER_LENGTH', 8192); define('BUFFER_LENGTH', 8192);
if( file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt") ) if( file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt") )
...@@ -55,16 +63,46 @@ class MetaGer ...@@ -55,16 +63,46 @@ class MetaGer
{ {
Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden"); Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
} }
$this->languageDetect = new TextLanguageDetect();
$this->languageDetect->setNameMode("2");
} }
public function rankAll ()
{
foreach( $this->engines as $engine )
{
$engine->rank($this);
}
}
public function createView() public function createView()
{ {
$viewResults = []; $viewResults = [];
# Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View: # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
foreach($this->results as $result) foreach($this->results as $result)
{ {
$viewResults[] = get_object_vars($result); $viewResults[] = get_object_vars($result);
} }
switch ($this->out) {
case 'html':
return view('metager3')
->with('results', $viewResults)
->with('eingabe', $this->eingabe)
->with('mobile', $this->mobile)
->with('warnings', $this->warnings)
->with('errors', $this->errors)
->with('metager', $this);
# code...
break;
default:
# code...
break;
}
return view('metager3') return view('metager3')
->with('results', $viewResults) ->with('results', $viewResults)
->with('eingabe', $this->eingabe) ->with('eingabe', $this->eingabe)
...@@ -72,12 +110,90 @@ class MetaGer ...@@ -72,12 +110,90 @@ class MetaGer
->with('errors', $this->errors); ->with('errors', $this->errors);
} }
public function removeInvalids ()
{
$results = [];
foreach($this->results as $result)
{
if($result->isValid($this))
$results[] = $result;
}
#$this->results = $results;
}
public function combineResults () public function combineResults ()
{ {
foreach($this->engines as $engine) foreach($this->engines as $engine)
{ {
$this->results = array_merge($this->results, $engine->results); foreach($engine->results as $result)
{
if($result->valid)
$this->results[] = $result;
}
foreach($engine->ads as $ad)
{
$this->ads[] = $ad;
}
} }
uasort($this->results, function($a, $b){
if($a->getRank() == $b->getRank())
return 0;
return ($a->getRank() < $b->getRank()) ? 1 : -1;
});
# Validate Results
$newResults = [];
foreach($this->results as $result)
{
if($result->isValid($this))
$newResults[] = $result;
}
$this->results = $newResults;
$counter = 0;
$firstRank = 0;
foreach($this->results as $result)
{
if($counter === 0)
$firstRank = $result->rank;
$counter++;
$result->number = $counter;
$confidence = 0;
if($firstRank > 0)
$confidence = $result->rank/$firstRank;
else
$confidence = 0;
if($confidence > 0.65)
$result->color = "#FF4000";
elseif($confidence > 0.4)
$result->color = "#FF0080";
elseif($confidence > 0.2)
$result->color = "#C000C0";
else
$result->color = "#000000";
}
//Get current page form url e.g. &page=6
$currentPage = LengthAwarePaginator::resolveCurrentPage();
$offset= $currentPage-1;
//Create a new Laravel collection from the array data
$collection = new Collection($this->results);
//Define how many items we want to be visible in each page
$perPage = $this->resultCount;
//Slice the collection to get the items to display in current page
$currentPageSearchResults = $collection->slice($offset * $perPage, $perPage)->all();
//Create our paginator and pass it to the view
$paginatedSearchResults= new LengthAwarePaginator($currentPageSearchResults, count($collection), $perPage);
$paginatedSearchResults->setPath('/meta/meta.ger3');
foreach($this->request->all() as $key => $value)
{
$paginatedSearchResults->addQuery($key, $value);
}
$this->results = $paginatedSearchResults;
} }
public function createSearchEngines (Request $request) public function createSearchEngines (Request $request)
...@@ -94,15 +210,24 @@ class MetaGer ...@@ -94,15 +210,24 @@ class MetaGer
if($this->fokus === "angepasst") if($this->fokus === "angepasst")
{ {
$sumas = $xml->xpath("suma"); $sumas = $xml->xpath("suma");
/**$maxSumas = 30;
$count = 0;
foreach($sumas as $suma)
{
if($maxSumas === $count)
break;
$enabledSearchengines[] = $suma;
$count++;
}**/
foreach($sumas as $suma) foreach($sumas as $suma)
{ {
if($request->has($suma["service"]) if($request->has($suma["service"])
#|| ( $this->fokus !== "bilder" || ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo" && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads" || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" ) || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# ) )
# ) )
#|| 1 === 1 #Todo: entfernen #|| 1 === 1 #Todo: entfernen
){ ){
...@@ -122,12 +247,12 @@ class MetaGer ...@@ -122,12 +247,12 @@ class MetaGer
foreach($sumas as $suma){ foreach($sumas as $suma){
$types = explode(",",$suma["type"]); $types = explode(",",$suma["type"]);
if(in_array($this->fokus, $types) if(in_array($this->fokus, $types)
#|| ( $this->fokus !== "bilder" || ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo" && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads" || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" ) || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# ) )
# ) )
){ ){
if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
{ {
...@@ -140,7 +265,7 @@ class MetaGer ...@@ -140,7 +265,7 @@ class MetaGer
} }
} }
} }
if( ( $this->fokus !== "bilder" && sizeof($enabledSearchengines) <= 3 ) || ( $this->fokus === "bilder" && sizeof($enabledSearchengines) === 0) ) if( ( $this->fokus !== "bilder" && sizeof($enabledSearchengines) <= 3 ) || ( $this->fokus === "bilder" && sizeof($enabledSearchengines) === 0) )
{ {
$this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt."; $this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
...@@ -149,21 +274,45 @@ class MetaGer ...@@ -149,21 +274,45 @@ class MetaGer
$engines = []; $engines = [];
foreach($enabledSearchengines as $engine){ foreach($enabledSearchengines as $engine){
if(strlen($this->site) > 0 && (!isset($engine["hasSiteSearch"]) || $engine["hasSiteSearch"]->__toString() !== "1"))
{
continue;
}
# Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
$path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString()); $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
$time = microtime();
$tmp = new $path($engine, $this); $tmp = new $path($engine, $this);
if($tmp->enabled && isset($this->debug))
{
$this->warnings[] = $tmp->service . " Connection_Time: " . $tmp->connection_time . " Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime()-$time)/1000);
}else
{
$this->warnings[] = "Suchmaschine " . $tmp->name . " nicht erreichbar" . " Insgesamt:" . ((microtime()-$time)/1000);
}
if($tmp->isEnabled()) if($tmp->isEnabled())
{ {
$engines[] = $tmp; $engines[] = $tmp;
$this->sockets[$tmp->name] = $tmp->fp;
} }
} }
# Nun passiert ein elementarer Schritt.
# Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
# aber natürlich nicht ewig.
# Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
# Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
usleep(500000);
# Jetzt lesen wir alles aus, was da ist und verwerfen den Rest:
foreach($engines as $engine) foreach($engines as $engine)
{ {
$engine->retrieveResults(); $engine->retrieveResults();
# $engine->closeFp();
} }
$this->engines = $engines; $this->engines = $engines;
} }
...@@ -232,7 +381,7 @@ class MetaGer ...@@ -232,7 +381,7 @@ class MetaGer
$this->page = $request->input('page', 1); $this->page = $request->input('page', 1);
# Lang # Lang
$this->lang = $request->input('lang', 'all'); $this->lang = $request->input('lang', 'all');
if ( $this->lang !== "de" || $this->lang !== "en" || $this->lang !== "all" ) if ( $this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all" )
{ {
$this->lang = "all"; $this->lang = "all";
} }
...@@ -284,6 +433,23 @@ class MetaGer ...@@ -284,6 +433,23 @@ class MetaGer
$this->time = 5000; $this->time = 5000;
$this->cache = "cache"; $this->cache = "cache";
} }
if( $request->has('tab'))
{
if($request->input('tab') === "1")
{
$this->tab = "_blank";
}else
{
$this->tab = "_self";
}
}else
{
$this->tab = "_blank";
}
$this->out = $request->input('out', "html");
if($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style")
$this->out = "html";
$this->request = $request;
} }
public function checkSpecialSearches (Request $request) public function checkSpecialSearches (Request $request)
...@@ -369,6 +535,14 @@ class MetaGer ...@@ -369,6 +535,14 @@ class MetaGer
return $this->eingabe; return $this->eingabe;
} }
public function getQ ()
{
if(strlen($this->site) > 0)
return $this->q . " site:" . $this->site;
else
return $this->q;
}
public function getUrl () public function getUrl ()
{ {
return $this->url; return $this->url;
...@@ -383,6 +557,16 @@ class MetaGer ...@@ -383,6 +557,16 @@ class MetaGer
return $this->language; return $this->language;
} }
public function getLang ()
{
return $this->lang;
}
public function getSprueche ()
{
return $this->sprueche;
}
public function getCategory () public function getCategory ()
{ {
return $this->category; return $this->category;
...@@ -392,4 +576,124 @@ class MetaGer ...@@ -392,4 +576,124 @@ class MetaGer
{ {
return $this->sumaFile; return $this->sumaFile;
} }
public function getUserHostBlacklist ()
{
return $this->hostBlacklist;
}
public function getUserDomainBlacklist ()
{
return $this->domainBlacklist;
}
public function getDomainBlacklist ()
{
return $this->domainsBlacklisted;
}
public function getUrlBlacklist ()
{
return $this->urlsBlacklisted;
}
public function getLanguageDetect ()
{
return $this->languageDetect;
}
public function getStopWords ()
{
return $this->stopWords;
}
public function getHostCount(String $host)
{
if(isset($this->addedHosts[$host]))
{
return $this->addedHosts[$host];
}else
{
return 0;
}
}
public function addHostCount(String $host)
{
$hash = md5($host);
if(isset($this->addedHosts[$hash]))
{
$this->addedHosts[$hash] += 1;
}else
{
$this->addedHosts[$hash] = 1;
}
}
public function getSite()
{
return $this->site;
}
public function addLink(String $link)
{
$hash = md5($link);
if(isset($this->addedLinks[$hash]))
{
return false;
}else
{
$this->addedLinks[$hash] = 1;
return true;
}
}
public function generateSearchLink(String $fokus)
{
$link = action('MetaGerSearch@search', $this->request->all());
$link = preg_replace("/focus=[^&]*/si", "focus=$fokus", $link);
return $link;
}
public function generateQuicktipLink()
{
$link = action('MetaGerSearch@quicktips');
return $link;
}
public function generateSiteSearchLink(String $host)
{
$host = urlencode($host);
$link = action('MetaGerSearch@search', $this->request->all());
$link = preg_replace("/eingabe=([^&]*)/si", "eingabe=$1 site:$host", $link);
return $link;
}
public function generateRemovedHostLink (String $host)
{
$host = urlencode($host);
$link = action('MetaGerSearch@search', $this->request->all());
$link = preg_replace("/eingabe=([^&]*)/si", "eingabe=$1 -host:$host", $link);
return $link;
}
public function generateRemovedDomainLink (String $domain)
{
$domain = urlencode($domain);