Skip to content

GitLab

MetaGer
MetaGer
Commit c5624d37 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files
Options

Einen großen Teil der Suche eingebaut

parent 0b026dbd
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 4179 additions and 277 deletions
app/Http/Controllers/MetaGerSearch.php
View file @ c5624d37
......@@ -14,8 +14,9 @@ use App\MetaGer;
class MetaGerSearch extends Controller
{
public function test(Request $request, MetaGer $metager)
public function search(Request $request, MetaGer $metager)
{
$time = microtime();
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
if($metager->getFokus() !== "bilder" )
......@@ -24,28 +25,22 @@ class MetaGerSearch extends Controller
$metager->checkSpecialSearches($request);
}
# Alle Suchmaschinen erstellen
$metager->createSearchEngines($request);
# Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll();
# Ergebnisse der Suchmaschinen kombinieren:
$metager->combineResults();
$metager->removeInvalids();
# Die Ausgabe erstellen:
return $metager->createView();
}
public function search(Request $request)
public function quicktips(Request $request)
{
$searchengines = Search::loadSearchEngines($request);
$results = new Results($searchengines);
return print_r( $viewResults, TRUE);
}
}
\ No newline at end of file
app/Http/routes.php
View file @ c5624d37
......@@ -56,8 +56,6 @@
Route::post('kontakt', 'MailController@contactMail');
Route::get('meta/meta.ger3', 'MetaGerSearch@search');
Route::get('spende', function()
{
return view('spende')
......@@ -80,5 +78,7 @@
->with('css', 'help.css');
});
Route::get('meta/meta.ger3', 'MetaGerSearch@test');
Route::get('meta/meta.ger3', 'MetaGerSearch@search');
Route::get('qt', 'MetaGerSearch@quicktips');
});
app/MetaGer.php
View file @ c5624d37
......@@ -3,10 +3,14 @@ namespace App;
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
use App\Models\SocketRocket;
use App;
use Storage;
use Log;
use App\lib\TextLanguageDetect\TextLanguageDetect;
use App\lib\TextLanguageDetect\LanguageDetect\TextLanguageDetectException;
use Illuminate\Pagination\LengthAwarePaginator;
use Illuminate\Support\Collection;
#use \Illuminate\Pagination\Paginator;
class MetaGer
{
......@@ -25,8 +29,10 @@ class MetaGer
protected $stopWords = [];
protected $engines = [];
protected $results = [];
protected $ads = [];
protected $warnings = [];
protected $errors = [];
protected $addedHosts = [];
# Daten über die Abfrage
protected $ip;
protected $language;
......@@ -39,9 +45,11 @@ class MetaGer
protected $domainsBlacklisted = [];
protected $urlsBlacklisted = [];
protected $url;
protected $languageDetect;
function __construct()
{
$this->time = microtime();
define('CRLF', "\r\n");
define('BUFFER_LENGTH', 8192);
if( file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt") )
......@@ -55,16 +63,46 @@ class MetaGer
{
Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
}
$this->languageDetect = new TextLanguageDetect();
$this->languageDetect->setNameMode("2");
}
public function rankAll ()
{
foreach( $this->engines as $engine )
{
$engine->rank($this);
}
}
public function createView()
{
$viewResults = [];
# Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
foreach($this->results as $result)
{
$viewResults[] = get_object_vars($result);
}
switch ($this->out) {
case 'html':
return view('metager3')
->with('results', $viewResults)
->with('eingabe', $this->eingabe)
->with('mobile', $this->mobile)
->with('warnings', $this->warnings)
->with('errors', $this->errors)
->with('metager', $this);
# code...
break;
default:
# code...
break;
}
return view('metager3')
->with('results', $viewResults)
->with('eingabe', $this->eingabe)
......@@ -72,12 +110,90 @@ class MetaGer
->with('errors', $this->errors);
}
public function removeInvalids ()
{
$results = [];
foreach($this->results as $result)
{
if($result->isValid($this))
$results[] = $result;
}
#$this->results = $results;
}
public function combineResults ()
{
foreach($this->engines as $engine)
{
$this->results = array_merge($this->results, $engine->results);
foreach($engine->results as $result)
{
if($result->valid)
$this->results[] = $result;
}
foreach($engine->ads as $ad)
{
$this->ads[] = $ad;
}
}
uasort($this->results, function($a, $b){
if($a->getRank() == $b->getRank())
return 0;
return ($a->getRank() < $b->getRank()) ? 1 : -1;
});
# Validate Results
$newResults = [];
foreach($this->results as $result)
{
if($result->isValid($this))
$newResults[] = $result;
}
$this->results = $newResults;
$counter = 0;
$firstRank = 0;
foreach($this->results as $result)
{
if($counter === 0)
$firstRank = $result->rank;
$counter++;
$result->number = $counter;
$confidence = 0;
if($firstRank > 0)
$confidence = $result->rank/$firstRank;
else
$confidence = 0;
if($confidence > 0.65)
$result->color = "#FF4000";
elseif($confidence > 0.4)
$result->color = "#FF0080";
elseif($confidence > 0.2)
$result->color = "#C000C0";
else
$result->color = "#000000";
}
//Get current page form url e.g. &page=6
$currentPage = LengthAwarePaginator::resolveCurrentPage();
$offset= $currentPage-1;
//Create a new Laravel collection from the array data
$collection = new Collection($this->results);
//Define how many items we want to be visible in each page
$perPage = $this->resultCount;
//Slice the collection to get the items to display in current page
$currentPageSearchResults = $collection->slice($offset * $perPage, $perPage)->all();
//Create our paginator and pass it to the view
$paginatedSearchResults= new LengthAwarePaginator($currentPageSearchResults, count($collection), $perPage);
$paginatedSearchResults->setPath('/meta/meta.ger3');
foreach($this->request->all() as $key => $value)
{
$paginatedSearchResults->addQuery($key, $value);
}
$this->results = $paginatedSearchResults;
}
public function createSearchEngines (Request $request)
......@@ -94,15 +210,24 @@ class MetaGer
if($this->fokus === "angepasst")
{
$sumas = $xml->xpath("suma");
/**$maxSumas = 30;
$count = 0;
foreach($sumas as $suma)
{
if($maxSumas === $count)
break;
$enabledSearchengines[] = $suma;
$count++;
}**/
foreach($sumas as $suma)
{
if($request->has($suma["service"])
#|| ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# )
# )
|| ( $this->fokus !== "bilder"
&& ($suma["name"]->__toString() === "qualigo"
|| $suma["name"]->__toString() === "similar_product_ads"
|| ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
)
)
#|| 1 === 1 #Todo: entfernen
){
......@@ -122,12 +247,12 @@ class MetaGer
foreach($sumas as $suma){
$types = explode(",",$suma["type"]);
if(in_array($this->fokus, $types)
#|| ( $this->fokus !== "bilder"
# && ($suma["name"]->__toString() === "qualigo"
# || $suma["name"]->__toString() === "similar_product_ads"
# || ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
# )
# )
|| ( $this->fokus !== "bilder"
&& ($suma["name"]->__toString() === "qualigo"
|| $suma["name"]->__toString() === "similar_product_ads"
|| ( !$overtureEnabled && $suma["name"]->__toString() === "overtureAds" )
)
)
){
if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
{
......@@ -140,7 +265,7 @@ class MetaGer
}
}
}
if( ( $this->fokus !== "bilder" && sizeof($enabledSearchengines) <= 3 ) || ( $this->fokus === "bilder" && sizeof($enabledSearchengines) === 0) )
{
$this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
......@@ -149,21 +274,45 @@ class MetaGer
$engines = [];
foreach($enabledSearchengines as $engine){
if(strlen($this->site) > 0 && (!isset($engine["hasSiteSearch"]) || $engine["hasSiteSearch"]->__toString() !== "1"))
{
continue;
}
# Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
$path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
$time = microtime();
$tmp = new $path($engine, $this);
if($tmp->enabled && isset($this->debug))
{
$this->warnings[] = $tmp->service . " Connection_Time: " . $tmp->connection_time . " Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime()-$time)/1000);
}else
{
$this->warnings[] = "Suchmaschine " . $tmp->name . " nicht erreichbar" . " Insgesamt:" . ((microtime()-$time)/1000);
}
if($tmp->isEnabled())
{
$engines[] = $tmp;
$this->sockets[$tmp->name] = $tmp->fp;
}
}
# Nun passiert ein elementarer Schritt.
# Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
# aber natürlich nicht ewig.
# Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
# Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
usleep(500000);
# Jetzt lesen wir alles aus, was da ist und verwerfen den Rest:
foreach($engines as $engine)
{
$engine->retrieveResults();
# $engine->closeFp();
}
$this->engines = $engines;
}
......@@ -232,7 +381,7 @@ class MetaGer
$this->page = $request->input('page', 1);
# Lang
$this->lang = $request->input('lang', 'all');
if ( $this->lang !== "de" || $this->lang !== "en" || $this->lang !== "all" )
if ( $this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all" )
{
$this->lang = "all";
}
......@@ -284,6 +433,23 @@ class MetaGer
$this->time = 5000;
$this->cache = "cache";
}
if( $request->has('tab'))
{
if($request->input('tab') === "1")
{
$this->tab = "_blank";
}else
{
$this->tab = "_self";
}
}else
{
$this->tab = "_blank";
}
$this->out = $request->input('out', "html");
if($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style")
$this->out = "html";
$this->request = $request;
}
public function checkSpecialSearches (Request $request)
......@@ -369,6 +535,14 @@ class MetaGer
return $this->eingabe;
}
public function getQ ()
{
if(strlen($this->site) > 0)
return $this->q . " site:" . $this->site;
else
return $this->q;
}
public function getUrl ()
{
return $this->url;
......@@ -383,6 +557,16 @@ class MetaGer
return $this->language;
}
public function getLang ()
{
return $this->lang;
}
public function getSprueche ()
{
return $this->sprueche;
}
public function getCategory ()
{
return $this->category;
......@@ -392,4 +576,124 @@ class MetaGer
{
return $this->sumaFile;
}
public function getUserHostBlacklist ()
{
return $this->hostBlacklist;
}
public function getUserDomainBlacklist ()
{
return $this->domainBlacklist;
}
public function getDomainBlacklist ()
{
return $this->domainsBlacklisted;
}
public function getUrlBlacklist ()
{
return $this->urlsBlacklisted;
}
public function getLanguageDetect ()
{
return $this->languageDetect;
}
public function getStopWords ()
{
return $this->stopWords;
}
public function getHostCount(String $host)
{
if(isset($this->addedHosts[$host]))
{
return $this->addedHosts[$host];
}else
{
return 0;
}
}
public function addHostCount(String $host)
{
$hash = md5($host);
if(isset($this->addedHosts[$hash]))
{
$this->addedHosts[$hash] += 1;
}else
{
$this->addedHosts[$hash] = 1;
}
}
public function getSite()
{
return $this->site;
}
public function addLink(String $link)
{
$hash = md5($link);
if(isset($this->addedLinks[$hash]))
{
return false;
}else
{
$this->addedLinks[$hash] = 1;
return true;
}
}
public function generateSearchLink(String $fokus)
{
$link = action('MetaGerSearch@search', $this->request->all());
$link = preg_replace("/focus=[^&]*/si", "focus=$fokus", $link);
return $link;
}
public function generateQuicktipLink()
{
$link = action('MetaGerSearch@quicktips');
return $link;
}
public function generateSiteSearchLink(String $host)
{
$host = urlencode($host);
$link = action('MetaGerSearch@search', $this->request->all());
$link = preg_replace("/eingabe=([^&]*)/si", "eingabe=$1 site:$host", $link);
return $link;
}
public function generateRemovedHostLink (String $host)
{
$host = urlencode($host);
$link = action('MetaGerSearch@search', $this->request->all());
$link = preg_replace("/eingabe=([^&]*)/si", "eingabe=$1 -host:$host", $link);
return $link;
}
public function generateRemovedDomainLink (String $domain)
{
$domain = urlencode($domain);
$link = action('MetaGerSearch@search', $this->request->all());
$link = preg_replace("/eingabe=([^&]*)/si", "eingabe=$1 -domain:$domain", $link);
return $link;
}
public function getTab ()
{
return $this->tab;
}
public function getResults ()
{
return $this->results;
}
public function popAd()
{
if(count($this->ads) > 0)
return get_object_vars(array_shift($this->ads));
else
return null;
}
}
\ No newline at end of file
app/Models/Result.php
View file @ c5624d37
......@@ -2,10 +2,12 @@
namespace App\Models;
class Result
{
function __construct ( $titel, $link, $anzeigeLink , $descr, $gefVon )
function __construct ( \SimpleXMLElement $provider, $titel, $link, $anzeigeLink , $descr, $gefVon, $sourceRank, $partnershop = false )
{
$this->titel = strip_tags(trim($titel));
$this->link = trim($link);
......@@ -13,5 +15,220 @@ class Result
$this->descr = strip_tags(trim($descr));
$this->descr = preg_replace("/\n+/si", " ", $this->descr);
$this->gefVon = trim($gefVon);
$this->proxyLink = $this->generateProxyLink($this->link);
$this->sourceRank = $sourceRank;
if($this->sourceRank <= 0 || $this->sourceRank > 20)
$this->sourceRank = 20;
$this->sourceRank = 20 - $this->sourceRank;
if(isset($provider["engineBoost"]))
{
$this->engineBoost = $provider["engineBoost"];
}else
{
$this->engineBoost = 1;
}
$this->valid = true;
$this->host = @parse_url($link, PHP_URL_HOST);
$this->strippedHost = $this->getStrippedHost($this->anzeigeLink);
$this->strippedDomain = $this->getStrippedDomain($this->strippedHost);
$this->strippedLink = $this->getStrippedLink($this->anzeigeLink);
$this->rank = 0;
$this->partnershop = $partnershop;
#die($this->anzeigeLink . "\r\n" . $this->strippedHost);
}
public function rank (\App\MetaGer $metager)
{
$rank = 0;
$rank += ($this->sourceRank * 0.02);
#URL-Boost
$link = $this->anzeigeLink;
if(strpos($link, "http") !== 0)
{
$link = "http://" . $link;
}
$link = @parse_url($link, PHP_URL_HOST) . @parse_url($link, PHP_URL_PATH);
$tmpLi = $link;
$tmpEingabe = $metager->getQ();
$count = 0;
$tmpLink = "";
$regex = [
"/\s+/si",
"/http:/si",
"/https:/si",
"/www\./si",
"/\//si",
"/\./si",
"/-/si"
];
foreach($regex as $reg)
{
$link = preg_replace($regex, "", $link);
$tmpEingabe = preg_replace($regex, "", $tmpEingabe);
}
#die($tmpLi . "<br>" . $link . "<br>" . $tmpEingabe . "<br><br>");
foreach(str_split($tmpEingabe) as $char)
{
if(strpos(strtolower($tmpLink), strtolower($char)) >= 0)
{
$count++;
$tmpLink = str_replace(urlencode($char), "", $tmpLink);
}
if(strlen($this->descr) > 80 && strlen($link) > 0)
{
$rank += $count /((strlen($link)) * 60);
}
}
# Boost für Vorkommen der Suchwörter:
$maxRank = 0.1;
$tmpTitle = $this->titel;
$tmpDescription = $this->descr;
$isWithin = false;
$tmpRank = 0;
$tmpEingabe = $metager->getQ();
$tmpEingabe = preg_replace("/\b\w{1,3}\b/si", "", $tmpEingabe);
$tmpEingabe = preg_replace("/\s+/si", " ", $tmpEingabe);
#die($tmpEingabe);
foreach(explode(" ", trim($tmpEingabe)) as $el)
{
$el = preg_quote($el, "/");
if(preg_match("/\b$el\b/si", $tmpTitle))
{