Commit 98f9147a authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch '1119-implement-admitad' into 'development'

Resolve "Implement Admitad"

Closes #1119

See merge request !1847
parents e36255ab b3e53cb6
......@@ -93,6 +93,21 @@ class MetaGerSearch extends Controller
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults($timings);
$admitad = [];
$adgoal = [];
if(!$metager->isApiAuthorized() && !$metager->isDummy()){
$newAdmitad = new \App\Models\Admitad($metager);
if(!empty($newAdmitad->hash)){
$admitad[] = $newAdmitad;
}
$newAdgoal = new \App\Models\Adgoal($metager);
if(!empty($newAdgoal->hash)){
$adgoal[] = $newAdgoal;
}
}
$metager->parseAffiliates($admitad);
$metager->parseAffiliates($adgoal);
$finished = true;
foreach ($metager->getEngines() as $engine) {
......@@ -106,10 +121,8 @@ class MetaGerSearch extends Controller
"metager" => [
"apiAuthorized" => $metager->isApiAuthorized(),
],
"adgoal" => [
"loaded" => $metager->isAdgoalLoaded(),
"adgoalHash" => $metager->getAdgoalHash(),
],
"admitad" => $admitad,
"adgoal" => $adgoal,
"engines" => $metager->getEngines(),
], 60 * 60);
} catch (\Exception $e) {
......@@ -190,14 +203,14 @@ class MetaGerSearch extends Controller
$engines = $cached["engines"];
$adgoal = $cached["adgoal"];
$admitad = $cached["admitad"];
$mg = $cached["metager"];
$metager = new MetaGer(substr($hash, strpos($hash, "loader_") + 7));
$metager->setApiAuthorized($mg["apiAuthorized"]);
$metager->setAdgoalLoaded($adgoal["loaded"]);
$metager->setAdgoalHash($adgoal["adgoalHash"]);
$metager->parseFormData($request, false);
$metager->setJsEnabled(true);
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
$metager->restoreEngines($engines);
......@@ -210,6 +223,20 @@ class MetaGerSearch extends Controller
$metager->rankAll();
$metager->prepareResults();
if(!$metager->isApiAuthorized() && !$metager->isDummy()){
$newAdmitad = new \App\Models\Admitad($metager);
if(!empty($newAdmitad->hash)){
$admitad[] = $newAdmitad;
}
$newAdgoal = new \App\Models\Adgoal($metager);
if(!empty($newAdgoal->hash)){
$adgoal[] = $newAdgoal;
}
}
$admitadFinished = $metager->parseAffiliates($admitad);
$adgoalFinished = $metager->parseAffiliates($adgoal);
$result = [
'finished' => true,
'newResults' => [],
......@@ -256,7 +283,7 @@ class MetaGerSearch extends Controller
}
}
if (!$metager->isAdgoalLoaded()) {
if (!$adgoalFinished || !$admitadFinished) {
$finished = false;
}
......@@ -273,10 +300,8 @@ class MetaGerSearch extends Controller
"metager" => [
"apiAuthorized" => $metager->isApiAuthorized(),
],
"adgoal" => [
"loaded" => $metager->isAdgoalLoaded(),
"adgoalHash" => $metager->getAdgoalHash(),
],
"admitad" => $admitad,
"adgoal" => $adgoal,
"engines" => $metager->getEngines(),
], 1 * 60);
......
......@@ -24,7 +24,6 @@ class Stresstest extends MetaGerSearch
# deactivates adgoal
$metager->setDummy(true);
$metager->setAdgoalHash(true);
parent::search($request, $metager, $timing);
}
......
......@@ -48,6 +48,7 @@ class MetaGer
protected $availableFoki = [];
protected $startCount = 0;
protected $canCache = false;
protected $javascript = false;
# Daten über die Abfrage$
protected $ip;
protected $useragent;
......@@ -307,33 +308,12 @@ class MetaGer
if (empty($this->adgoalLoaded)) {
$this->adgoalLoaded = false;
}
if (!$this->apiAuthorized && !$this->adgoalLoaded && !$this->dummy) {
if (empty($this->adgoalHash)) {
if (!empty($this->jskey)) {
$js = Redis::connection('cache')->lpop("js" . $this->jskey);
if ($js !== null && boolval($js)) {
$this->javascript = true;
}
}
$this->adgoalHash = \App\Models\Adgoal::startAdgoal($this->results);
if (!empty($timings)) {
$timings["prepareResults"]["started adgoal"] = microtime(true) - $timings["starttime"];
}
}
if (!$this->javascript) {
$this->adgoalLoaded = \App\Models\Adgoal::parseAdgoal($this->results, $this->adgoalHash, true);
if (!empty($timings)) {
$timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
}
} else {
$this->adgoalLoaded = \App\Models\Adgoal::parseAdgoal($this->results, $this->adgoalHash, false);
if (!empty($timings)) {
$timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
}
if (!empty($this->jskey)) {
$js = Redis::connection('cache')->lpop("js" . $this->jskey);
if ($js !== null && boolval($js)) {
$this->javascript = true;
}
} else {
$this->adgoalLoaded = true;
}
# Human Verification
......@@ -435,7 +415,26 @@ class MetaGer
}
}
/**
* @param \App\Models\Admitad[] $admitads
* @param Boolean $wait Wait for Results?
* @return Boolean whether or not all Admitad Objects are finished
*/
public function parseAffiliates(&$affiliates){
$wait = false;
$finished = true;
if(!$this->javascript){
$wait = true;
}
foreach ($affiliates as $affiliate) {
$affiliate->fetchAffiliates($wait);
$affiliate->parseAffiliates($this->results);
if(!$affiliate->finished){
$finished = false;
}
}
return $finished;
}
public function humanVerification(&$results)
{
......@@ -1718,6 +1717,9 @@ class MetaGer
$this->results = $results;
}
/**
* @return \App\Models\Result[]
*/
public function getResults()
{
return $this->results;
......@@ -1891,26 +1893,6 @@ class MetaGer
return $this->engines;
}
public function setAdgoalHash($hash)
{
$this->adgoalHash = $hash;
}
public function getAdgoalHash()
{
return $this->adgoalHash;
}
public function isAdgoalLoaded()
{
return $this->adgoalLoaded;
}
public function setAdgoalLoaded($adgoalLoaded)
{
$this->adgoalLoaded = $adgoalLoaded;
}
public function isApiAuthorized()
{
return $this->apiAuthorized;
......@@ -1931,6 +1913,17 @@ class MetaGer
return $this->headerPrinted;
}
public function isDummy(){
return $this->dummy;
}
public function jsEnabled() {
return $this->javascript;
}
public function setJsEnabled(bool $bool){
$this->javascript = $bool;
}
/**
* Used by JS result loader to restore MetaGer Object of previous request
*/
......
......@@ -22,29 +22,47 @@ class Adgoal
"lk","sh","kn","lc","pm","vc","sd","sr","za","kr","sz","sy","tj","tw","tz","th","tg","to","tt","td","cz","tn",
"tm","tc","tv","tr","us","ug","ua","xx","hu","uy","uz","vu","va","ve","ae","vn","wf","cx","by","eh","ww","zr","cf","cy",];
public static function startAdgoal(&$results)
public $hash;
public $finished = false; // Is true when the Request was sent to and read from Admitad App
private $affiliates = null;
private $startTime;
/**
* Creates a new Adgoal object which will start a request for affiliate links
* based on a result List from MetaGer.
* It will parse the Links of the results and query any affiliate shops.
*
* @param \App\MetaGer $metager
*/
public function __construct(&$metager)
{
$this->startTime = microtime(true);
$publicKey = getenv('adgoal_public');
$privateKey = getenv('adgoal_private');
if ($publicKey === false) {
return true;
}
$results = $metager->getResults();
$linkList = "";
foreach ($results as $result) {
if (!$result->new) {
continue;
}
$link = $result->link;
$link = $result->originalLink;
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
}
$linkList .= $link . ",";
}
if(empty($linkList)){
return;
}
$linkList = rtrim($linkList, ",");
# Hashwert
$hash = md5($linkList . $privateKey);
$this->hash = md5($linkList . $privateKey);
$link = "https://xf.gdprvalidate.de/v4/check";
......@@ -73,14 +91,14 @@ class Adgoal
"key" => $publicKey,
"panel" => "ZMkW9eSKJS",
"member" => "338b9Bnm",
"signature" => $hash,
"signature" => $this->hash,
"links" => $linkList,
"country" => $country,
];
// Submit fetch job to worker
$mission = [
"resulthash" => $hash,
"resulthash" => $this->hash,
"url" => $link,
"useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
"username" => null,
......@@ -97,76 +115,86 @@ class Adgoal
];
$mission = json_encode($mission);
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
return $hash;
}
public static function parseAdgoal(&$results, $hash, $waitForResult)
{
# Wait for result
$startTime = microtime(true);
$answer = null;
# Hash is true if Adgoal request wasn't started in the first place
if ($hash === true) {
return true;
public function fetchAffiliates($wait = false){
if($this->affiliates !== null){
return;
}
if ($waitForResult) {
$answer = null;
$startTime = microtime(true);
if($wait){
while (microtime(true) - $startTime < 5) {
$answer = Cache::get($hash);
$answer = Cache::get($this->hash);
if ($answer === null) {
usleep(50 * 1000);
} else {
break;
}
}
} else {
$answer = Cache::get($hash);
}else{
$answer = Cache::get($this->hash);
}
if ($answer === null) {
return false;
$answer = json_decode($answer, true);
// If the fetcher had an Error
if($answer === "no-result"){
$this->affiliates = [];
return;
}
if(empty($answer) || !is_array($answer)){
return;
}
try {
$answer = json_decode($answer, true);
foreach ($answer as $partnershop) {
$targetUrl = $partnershop["url"];
$this->affiliates = $answer;
}
$tld = $partnershop["tld"];
$targetHost = parse_url($targetUrl, PHP_URL_HOST);
/**
* Converts all Affiliate Links.
*
* @param \App\Models\Result[] $results
*/
public function parseAffiliates(&$results)
{
if($this->finished || $this->affiliates === null){
return;
}
foreach ($this->affiliates as $partnershop) {
$targetUrl = $partnershop["url"];
/*
Adgoal sometimes returns affiliate Links for every URL
That's why we check if the corresponding TLD matches the orginial URL
*/
if($targetHost !== false && stripos($targetHost, $tld) === false){
continue;
}
$tld = $partnershop["tld"];
$targetHost = parse_url($targetUrl, PHP_URL_HOST);
foreach ($results as $result) {
if ($result->link === $targetUrl && !$result->partnershop) {
# Ein Advertiser gefunden
if ($result->image !== "") {
$result->logo = $partnershop["logo"];
} else {
$result->image = $partnershop["logo"];
}
# Den Link hinzufügen:
$result->link = $partnershop["click_url"];
$result->partnershop = true;
$result->changed = true;
/*
Adgoal sometimes returns affiliate Links for every URL
That's why we check if the corresponding TLD matches the orginial URL
*/
if($targetHost !== false && stripos($targetHost, $tld) === false){
continue;
}
$preference = config("metager.metager.affiliate_preference", "adgoal");
foreach ($results as $result) {
if ($result->originalLink === $targetUrl && (config("metager.metager.affiliate_preference", "adgoal") === "adgoal" || !$result->partnershop)) {
# Ein Advertiser gefunden
if ($result->image !== "" && !$result->partnershop) {
$result->logo = $partnershop["logo"];
} else {
$result->image = $partnershop["logo"];
}
# Den Link hinzufügen:
$result->link = $partnershop["click_url"];
$result->partnershop = true;
$result->changed = true;
}
}
} catch (\ErrorException $e) {
Log::error($e->getMessage());
} finally {
$requestTime = microtime(true) - $startTime;
\App\PrometheusExporter::Duration($requestTime, "adgoal");
}
return true;
$requestTime = microtime(true) - $this->startTime;
\App\PrometheusExporter::Duration($requestTime, "adgoal");
$this->finished = true;
}
}
<?php
namespace App\Models;
use Cache;
use Log;
use LaravelLocalization;
use Illuminate\Support\Facades\Redis;
class Admitad
{
const VALID_LANGS = [
"de",
"en"
];
public $hash;
public $finished = false; // Is true when the Request was sent to and read from Admitad App
private $affiliates = null;
/**
* Creates a new Admitad object which will start a request for affiliate links
* based on a result List from MetaGer.
* It will parse the Links of the results and query any affiliate shops.
*
* @param \App\MetaGer $metager
*/
public function __construct(&$metager)
{
$results = $metager->getResults();
// Generate a list of URLs
$resultLinks = [];
foreach($results as $result){
if ($result->new) {
$resultLinks[] = $result->originalLink;
}
}
if(empty($resultLinks)){
return;
}
$lang = LaravelLocalization::getCurrentLocale();
if(!in_array($lang, self::VALID_LANGS)){
$lang = "de";
}
$requestData = [
"lang" => $lang,
"urls" => $resultLinks,
];
$requestData = json_encode($requestData);
$this->hash = md5($requestData);
$url = "https://direct.metager.de/check";
$token = env("ADMITAD_TOKEN", "");
// Submit fetch job to worker
$mission = [
"resulthash" => $this->hash,
"url" => $url,
"useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
"username" => null,
"password" => null,
"headers" => [
"Authorization" => "Bearer $token"
],
"cacheDuration" => 60,
"name" => "Admitad",
"curlopts" => [
CURLOPT_POST => true,
CURLOPT_POSTFIELDS => $requestData
]
];
$mission = json_encode($mission);
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
}
/**
* Fetches the Admitad Response from Redis
* @param Boolean $wait Whether or not to wait for a response
*/
public function fetchAffiliates($wait = false) {
if($this->affiliates !== null){
return;
}
$answer = null;
$startTime = microtime(true);
if($wait){
while (microtime(true) - $startTime < 5) {
$answer = Cache::get($this->hash);
if ($answer === null) {
usleep(50 * 1000);
} else {
break;
}
}
}else{
$answer = Cache::get($this->hash);
}
$answer = json_decode($answer, true);
// If the fetcher had an Error
if($answer === "no-result"){
$this->affiliates = [];
return;
}
if(empty($answer) || !isset($answer["error"]) || $answer["error"] || !is_array($answer["result"])){
return;
}
$this->affiliates = $answer["result"];
}
/**
* Converts all Affiliate Links.
*
* @param \App\Models\Result[] $results
*/
public function parseAffiliates(&$results){
if($this->finished || $this->affiliates === null){
return;
}
foreach($this->affiliates as $linkResult){
$originalUrl = $linkResult["originalUrl"];
$redirUrl = $linkResult["redirUrl"];
$image = $linkResult["image"];
if(empty($redirUrl)){
// No Partnershop
continue;
}
foreach ($results as $result) {
if ($result->originalLink === $originalUrl && (config("metager.metager.affiliate_preference", "adgoal") === "admitad" || !$result->partnershop)) {
# Ein Advertiser gefunden
if ($result->image !== "" && !$result->partnershop) {
$result->logo = $image;
} else {
$result->image = $image;
}
# Den Link hinzufügen:
$result->link = $redirUrl;
$result->partnershop = true;
$result->changed = true;
}
}
}
$this->finished = true;
}
}
......@@ -9,6 +9,7 @@ class Result
{
public $provider; # Die Engine von der das Suchergebnis kommt
public $titel; # Der Groß Angezeigte Name für das Suchergebnis
public $originalLink;
public $link; # Der Link auf die Ergebnisseite
public $anzeigeLink; # Der tatsächlich angezeigte Link (rein optisch)
public $descr; # Die eventuell gekürzte Beschreibung des Suchergebnisses
......@@ -41,6 +42,7 @@ class Result
$this->provider = $provider;
$this->titel = $this->sanitizeText(strip_tags(trim($titel)));
$this->link = trim($link);
$this->originalLink = trim($link);
$this->anzeigeLink = trim($anzeigeLink);
$this->anzeigeLink = preg_replace("/(http[s]{0,1}:\/\/){0,1}(www\.){0,1}/si", "", $this->anzeigeLink);
$this->descr = $this->sanitizeText(strip_tags(trim($descr), '<p>'));
......
......@@ -6,4 +6,5 @@ return [
"w3m\/",
],
"fail2ban_enabled" => true,
"affiliate_preference" => "adgoal",
];
......@@ -3,6 +3,6 @@
return [
'heading' => 'MetaGer-Partnershop',
'paragraph.1' => 'Wenn in der MetaGer-Ergebnisliste Links erscheinen, bei denen das Wort "Partnershop" steht, dann sind das zunächst normale Ergebnisse einer MetaGer-Suche. Sie werden im Ranking nicht bevorzugt oder höher eingestuft, sondern wie alle anderen Suchergebnisse auch behandelt.',
'paragraph.2' => 'Hinter diesem Link steht dann eine Firma, von der wir dann, wenn der Link angeklickt wird, einen geringen Betrag zur Finanzierung von MetaGer erhalten. Beim Ergebnis wird das zugehörige Firmenlogo angezeigt. Auch daran erkennen Sie immer unsere Partnershops. Beim Anklicken (und nur dann) eines solchen Ergebnisses werden auch Nutzerdaten an die Klickadresse weitergegeben. Zu diesen Nutzerdaten gehören auch die Suchbegriffe, die Sie auf MetaGer eingegeben haben, in Verbindung mit Ihrer IP-Adresse. Wenn Sie dies vermeiden wollen, dann können Sie be