Skip to content

GitLab

MetaGer
MetaGer
Commit 893b774e authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files
Options

Datenschutzerklärung erweitert und einige Formatierungen durchgeführt

parent 7d20edca
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2788 additions and 2893 deletions
app/MetaGer.php
View file @ 893b774e
This diff is collapsed.
app/Models/Result.php
View file @ 893b774e
......@@ -2,256 +2,244 @@
namespace App\Models;
class Result
{
function __construct ( \SimpleXMLElement $provider, $titel, $link, $anzeigeLink , $descr, $gefVon, $sourceRank, $partnershop = false, $image = "", $price = 0 )
{
$this->titel = strip_tags(trim($titel));
$this->link = trim($link);
$this->anzeigeLink = trim($anzeigeLink);
$this->descr = strip_tags(trim($descr), '<p>');
$this->descr = preg_replace("/\n+/si", " ", $this->descr);
if( strlen($this->descr) > 250 )
{
$this->descr = wordwrap($this->descr, 250);
$this->descr = substr($this->descr, 0, strpos($this->descr, "\n"));
}
$this->gefVon = trim($gefVon);
$this->proxyLink = $this->generateProxyLink($this->link);
$this->sourceRank = $sourceRank;
if($this->sourceRank <= 0 || $this->sourceRank > 20)
$this->sourceRank = 20;
$this->sourceRank = 20 - $this->sourceRank;
if(isset($provider["engineBoost"]))
{
$this->engineBoost = floatval($provider["engineBoost"]->__toString());
}else
{
$this->engineBoost = 1;
}
$this->valid = true;
$this->host = @parse_url($link, PHP_URL_HOST);
$this->strippedHost = $this->getStrippedHost($this->anzeigeLink);
$this->strippedDomain = $this->getStrippedDomain($this->strippedHost);
$this->strippedLink = $this->getStrippedLink($this->anzeigeLink);
$this->rank = 0;
$this->partnershop = $partnershop;
$this->image = $image;
#die($this->anzeigeLink . "\r\n" . $this->strippedHost);
}
public function rank (\App\MetaGer $metager)
{
$rank = 0;
$rank += ($this->sourceRank * 0.02);
#URL-Boost
$link = $this->anzeigeLink;
if(strpos($link, "http") !== 0)
{
$link = "http://" . $link;
}
$link = @parse_url($link, PHP_URL_HOST) . @parse_url($link, PHP_URL_PATH);
$tmpLi = $link;
$tmpEingabe = $metager->getQ();
$count = 0;
$tmpLink = "";
$regex = [
"/\s+/si",
"/http:/si",
"/https:/si",
"/www\./si",
"/\//si",
"/\./si",
"/-/si"
];
foreach($regex as $reg)
{
$link = preg_replace($regex, "", $link);
$tmpEingabe = preg_replace($regex, "", $tmpEingabe);
}
#die($tmpLi . "<br>" . $link . "<br>" . $tmpEingabe . "<br><br>");
foreach(str_split($tmpEingabe) as $char)
{
if( !$char || !$tmpEingabe || strlen($tmpEingabe) === 0 || strlen($char) === 0 )
continue;
if(strpos(strtolower($tmpLink), strtolower($char)) >= 0)
{
$count++;
$tmpLink = str_replace(urlencode($char), "", $tmpLink);
}
if(strlen($this->descr) > 80 && strlen($link) > 0)
{
#$rank += $count /((strlen($link)) * 60);
}
}
# Boost für Vorkommen der Suchwörter:
$maxRank = 0.1;
$tmpTitle = $this->titel;
$tmpDescription = $this->descr;
$isWithin = false;
$tmpRank = 0;
$tmpEingabe = $metager->getQ();
$tmpEingabe = preg_replace("/\b\w{1,3}\b/si", "", $tmpEingabe);
$tmpEingabe = preg_replace("/\s+/si", " ", $tmpEingabe);
#die($tmpEingabe);
foreach(explode(" ", trim($tmpEingabe)) as $el)
{
if( strlen($tmpTitle) === 0 || strlen($el) === 0 || strlen($tmpDescription) === 0 )
continue;
$el = preg_quote($el, "/");
if(strlen($tmpTitle) > 0)
{
if(preg_match("/\b$el\b/si", $tmpTitle))
{
$tmpRank += .7 * .6 * $maxRank;
}elseif (strpos($tmpTitle, $el) !== false) {
$tmpRank += .3 * .6 * $maxRank;
}
}
if( strlen($tmpDescription) > 0 )
{
if(preg_match("/\b$el\b/si", $tmpDescription))
{
$tmpRank += .7 * .4 * $maxRank;
}elseif (strpos($tmpDescription, $el) !== false) {
$tmpRank += .3 * .4 * $maxRank;
}
}
}
$tmpRank /= sizeof(explode(" ", trim($tmpEingabe))) * 10;
$rank += $tmpRank;
if($this->engineBoost > 0)
{
$rank *= floatval($this->engineBoost);
}
$this->rank = $rank;
}
public function getRank ()
{
return $this->rank;
}
public function isValid (\App\MetaGer $metager)
{
# Zunächst die persönlich ( über URL-Parameter ) definierten Blacklists:
if(in_array($this->strippedHost, $metager->getUserHostBlacklist())
|| in_array($this->strippedDomain, $metager->getUserDomainBlacklist()))
return false;
# Jetzt unsere URL und Domain Blacklist
if($this->strippedHost !== "" && (in_array($this->strippedHost, $metager->getDomainBlacklist()) || in_array($this->strippedLink, $metager->getUrlBlacklist())))
{
return false;
}
# Nun der Eventuelle Sprachfilter
if( $metager->getLang() !== "all" )
{
$text = $this->titel . " " . $this->descr;
$path = app_path() . "/Models/lang.pl";
$lang = exec("echo '$text' | $path");
if( $metager->getLang() !== $lang )
return false;
}
# Wir wenden die Stoppwortsuche an und schmeißen entsprechende Ergebnisse raus:
foreach($metager->getStopWords() as $stopWord)
{
$text = $this->titel . " " . $this->descr;
if(stripos($text, $stopWord) !== false)
{
return false;
}
}
# Die Strinsuche:
$text = strtolower($this->titel) . " " . strtolower($this->descr);
foreach($metager->getPhrases() as $phrase)
{
if(strpos($text, $phrase) === FALSE)
return false;
}
# Abschließend noch 2 Überprüfungen. Einmal den Host filter, der Sicherstellt, dass von jedem Host maximal 3 Links angezeigt werden
# und dann noch den Dublettefilter, der sicher stellt, dass wir nach Möglichkeit keinen Link doppelt in der Ergebnisliste haben
# Diese Überprüfung führen wir unter bestimmten Bedingungen nicht durch:
if($metager->getSite() === "" &&
strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
strpos($this->strippedHost, "twitter.com") === false &&
strpos($this->strippedHost, "www.ladenpreis.net") === false &&
strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
strpos($this->strippedHost, "www.onenewspage.com") === false)
{
$count = $metager->getHostCount($this->strippedHost);
if($count >= 3)
{
return false;
}
}
# Unabhängig davon unser Dublettenfilter:
if($metager->addLink($this->strippedLink))
{
$metager->addHostCount($this->strippedHost);
return true;
}else
{
return false;
}
}
private function getStrippedHost ($link)
{
if(strpos($link, "http") !== 0)
$link = "http://" . $link;
$link = @parse_url($link, PHP_URL_HOST);
$link = preg_replace("/^www\./si", "", $link);
return $link;
}
private function getStrippedLink ($link)
{
if(strpos($link, "http") !== 0)
$link = "http://" . $link;
$host = $this->strippedHost;
$path = @parse_url($link , PHP_URL_PATH);
return $host . $path;
}
private function getStrippedDomain ($link)
{
if(preg_match("/([^\.]*\.[^\.]*)$/si", $link, $match))
{
return $match[1];
}else
{
return $link;
}
}
private function generateProxyLink ($link)
{
if(!$link)
return "";
$tmp = $link;
$tmp = preg_replace("/\r?\n$/s", "", $tmp);
$tmp = preg_replace("#^([\w+.-]+)://#s", "$1/", $tmp);
return "https://proxy.suma-ev.de/cgi-bin/nph-proxy.cgi/en/I0/" . $tmp;
}
public function __construct(\SimpleXMLElement $provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $sourceRank, $partnershop = false, $image = "", $price = 0)
{
$this->titel = strip_tags(trim($titel));
$this->link = trim($link);
$this->anzeigeLink = trim($anzeigeLink);
$this->descr = strip_tags(trim($descr), '<p>');
$this->descr = preg_replace("/\n+/si", " ", $this->descr);
if (strlen($this->descr) > 250) {
$this->descr = wordwrap($this->descr, 250);
$this->descr = substr($this->descr, 0, strpos($this->descr, "\n"));
}
$this->gefVon = trim($gefVon);
$this->proxyLink = $this->generateProxyLink($this->link);
$this->sourceRank = $sourceRank;
if ($this->sourceRank <= 0 || $this->sourceRank > 20) {
$this->sourceRank = 20;
}
$this->sourceRank = 20 - $this->sourceRank;
if (isset($provider["engineBoost"])) {
$this->engineBoost = floatval($provider["engineBoost"]->__toString());
} else {
$this->engineBoost = 1;
}
$this->valid = true;
$this->host = @parse_url($link, PHP_URL_HOST);
$this->strippedHost = $this->getStrippedHost($this->anzeigeLink);
$this->strippedDomain = $this->getStrippedDomain($this->strippedHost);
$this->strippedLink = $this->getStrippedLink($this->anzeigeLink);
$this->rank = 0;
$this->partnershop = $partnershop;
$this->image = $image;
#die($this->anzeigeLink . "\r\n" . $this->strippedHost);
}
public function rank(\App\MetaGer $metager)
{
$rank = 0;
$rank += ($this->sourceRank * 0.02);
#URL-Boost
$link = $this->anzeigeLink;
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
}
$link = @parse_url($link, PHP_URL_HOST) . @parse_url($link, PHP_URL_PATH);
$tmpLi = $link;
$tmpEingabe = $metager->getQ();
$count = 0;
$tmpLink = "";
$regex = [
"/\s+/si",
"/http:/si",
"/https:/si",
"/www\./si",
"/\//si",
"/\./si",
"/-/si",
];
foreach ($regex as $reg) {
$link = preg_replace($regex, "", $link);
$tmpEingabe = preg_replace($regex, "", $tmpEingabe);
}
#die($tmpLi . "<br>" . $link . "<br>" . $tmpEingabe . "<br><br>");
foreach (str_split($tmpEingabe) as $char) {
if (!$char || !$tmpEingabe || strlen($tmpEingabe) === 0 || strlen($char) === 0) {
continue;
}
if (strpos(strtolower($tmpLink), strtolower($char)) >= 0) {
$count++;
$tmpLink = str_replace(urlencode($char), "", $tmpLink);
}
if (strlen($this->descr) > 80 && strlen($link) > 0) {
#$rank += $count /((strlen($link)) * 60);
}
}
# Boost für Vorkommen der Suchwörter:
$maxRank = 0.1;
$tmpTitle = $this->titel;
$tmpDescription = $this->descr;
$isWithin = false;
$tmpRank = 0;
$tmpEingabe = $metager->getQ();
$tmpEingabe = preg_replace("/\b\w{1,3}\b/si", "", $tmpEingabe);
$tmpEingabe = preg_replace("/\s+/si", " ", $tmpEingabe);
#die($tmpEingabe);
foreach (explode(" ", trim($tmpEingabe)) as $el) {
if (strlen($tmpTitle) === 0 || strlen($el) === 0 || strlen($tmpDescription) === 0) {
continue;
}
$el = preg_quote($el, "/");
if (strlen($tmpTitle) > 0) {
if (preg_match("/\b$el\b/si", $tmpTitle)) {
$tmpRank += .7 * .6 * $maxRank;
} elseif (strpos($tmpTitle, $el) !== false) {
$tmpRank += .3 * .6 * $maxRank;
}
}
if (strlen($tmpDescription) > 0) {
if (preg_match("/\b$el\b/si", $tmpDescription)) {
$tmpRank += .7 * .4 * $maxRank;
} elseif (strpos($tmpDescription, $el) !== false) {
$tmpRank += .3 * .4 * $maxRank;
}
}
}
$tmpRank /= sizeof(explode(" ", trim($tmpEingabe))) * 10;
$rank += $tmpRank;
if ($this->engineBoost > 0) {
$rank *= floatval($this->engineBoost);
}
$this->rank = $rank;
}
public function getRank()
{
return $this->rank;
}
public function isValid(\App\MetaGer $metager)
{
# Zunächst die persönlich ( über URL-Parameter ) definierten Blacklists:
if (in_array($this->strippedHost, $metager->getUserHostBlacklist())
|| in_array($this->strippedDomain, $metager->getUserDomainBlacklist())) {
return false;
}
# Jetzt unsere URL und Domain Blacklist
if ($this->strippedHost !== "" && (in_array($this->strippedHost, $metager->getDomainBlacklist()) || in_array($this->strippedLink, $metager->getUrlBlacklist()))) {
return false;
}
# Nun der Eventuelle Sprachfilter
if ($metager->getLang() !== "all") {
$text = $this->titel . " " . $this->descr;
$path = app_path() . "/Models/lang.pl";
$lang = exec("echo '$text' | $path");
if ($metager->getLang() !== $lang) {
return false;
}
}
# Wir wenden die Stoppwortsuche an und schmeißen entsprechende Ergebnisse raus:
foreach ($metager->getStopWords() as $stopWord) {
$text = $this->titel . " " . $this->descr;
if (stripos($text, $stopWord) !== false) {
return false;
}
}
# Die Strinsuche:
$text = strtolower($this->titel) . " " . strtolower($this->descr);
foreach ($metager->getPhrases() as $phrase) {
if (strpos($text, $phrase) === false) {
return false;
}
}
# Abschließend noch 2 Überprüfungen. Einmal den Host filter, der Sicherstellt, dass von jedem Host maximal 3 Links angezeigt werden
# und dann noch den Dublettefilter, der sicher stellt, dass wir nach Möglichkeit keinen Link doppelt in der Ergebnisliste haben
# Diese Überprüfung führen wir unter bestimmten Bedingungen nicht durch:
if ($metager->getSite() === "" &&
strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
strpos($this->strippedHost, "twitter.com") === false &&
strpos($this->strippedHost, "www.ladenpreis.net") === false &&
strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
strpos($this->strippedHost, "www.onenewspage.com") === false) {
$count = $metager->getHostCount($this->strippedHost);
if ($count >= 3) {
return false;
}
}
# Unabhängig davon unser Dublettenfilter:
if ($metager->addLink($this->strippedLink)) {
$metager->addHostCount($this->strippedHost);
return true;
} else {
return false;
}
}
private function getStrippedHost($link)
{
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
}
$link = @parse_url($link, PHP_URL_HOST);
$link = preg_replace("/^www\./si", "", $link);
return $link;
}
private function getStrippedLink($link)
{
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
}
$host = $this->strippedHost;
$path = @parse_url($link, PHP_URL_PATH);
return $host . $path;
}
private function getStrippedDomain($link)
{
if (preg_match("/([^\.]*\.[^\.]*)$/si", $link, $match)) {
return $match[1];
} else {
return $link;
}
}
private function generateProxyLink($link)
{
if (!$link) {
return "";
}
$tmp = $link;
$tmp = preg_replace("/\r?\n$/s", "", $tmp);
$tmp = preg_replace("#^([\w+.-]+)://#s", "$1/", $tmp);
return "https://proxy.suma-ev.de/cgi-bin/nph-proxy.cgi/en/I0/" . $tmp;
}
}
app/Models/Searchengine.php
View file @ 893b774e
<?php
namespace App\Models;
use App\Jobs\Search;
use App\MetaGer;
use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Log;
use Redis;
use App\Jobs\Search;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Cache;
abstract class Searchengine
abstract class Searchengine
{
use DispatchesJobs;
use DispatchesJobs;
protected $ch; # Curl Handle zum erhalten der Ergebnisse
public $fp;
protected $getString = "";
protected $engine;
protected $counter = 0;
protected $ch; # Curl Handle zum erhalten der Ergebnisse
public $fp;
protected $getString = "";
protected $engine;
protected $counter = 0;
protected $socketNumber = null;
public $enabled = true;
public $results = [];
public $ads = [];
public $write_time = 0;
public $connection_time = 0;
public $loaded = false;
public $cached = false;
function __construct(\SimpleXMLElement $engine, MetaGer $metager)
{
foreach($engine->attributes() as $key => $value){
$this->$key = $value->__toString();
}
if( !isset($this->homepage) )
$this->homepage = "https://metager.de";
$this->engine = $engine;
if( !isset($this->cacheDuration) )
$this->cacheDuration = 60;
# Wir registrieren die Benutzung dieser Suchmaschine
$this->uses = intval(Redis::hget($this->name, "uses")) + 1;
Redis::hset($this->name, "uses", $this->uses);
# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
if(isset($this->disabled) && strtotime($this->disabled) <= time() )
{
# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
$this->enable($metager->getSumaFile(), "Die Suchmaschine " . $this->name . " wurde wieder eingeschaltet.");
}elseif (isset($this->disabled) && strtotime($this->disabled) > time())
{
$this->enabled = false;
return;
public $enabled = true;
public $results = [];
public $ads = [];
public $write_time = 0;
public $connection_time = 0;
public $loaded = false;
public $cached = false;
public function __construct(\SimpleXMLElement $engine, MetaGer $metager)
{
foreach ($engine->attributes() as $key => $value) {
$this->$key = $value->__toString();
}
if (!isset($this->homepage)) {
$this->homepage = "https://metager.de";
}
$this->engine = $engine;
if (!isset($this->cacheDuration)) {
$this->cacheDuration = 60;
}
# Wir registrieren die Benutzung dieser Suchmaschine
$this->uses = intval(Redis::hget($this->name, "uses")) + 1;
Redis::hset($this->name, "uses", $this->uses);
# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
if (isset($this->disabled) && strtotime($this->disabled) <= time()) {
# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
$this->enable($metager->getSumaFile(), "Die Suchmaschine " . $this->name . " wurde wieder eingeschaltet.");
} elseif (isset($this->disabled) && strtotime($this->disabled) > time()) {
$this->enabled = false;
return;
}
# User-Agent definieren:
$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
$this->ip = $metager->getIp();
$this->gefVon = "<a href=\"" . $this->homepage . "\" target=\"_blank\">" . $this->displayName . "</a>";
$this->startTime = microtime();
$q = "";
if (isset($this->hasSiteSearch) && $this->hasSiteSearch === "1") {
if (strlen($metager->getSite()) === 0) {
$q = $metager->getQ();
} else {
$q = $metager->getQ() . " site:" . $metager->getSite();
}
} else {