From b3e53cb61fa04a9abc08b8ed170d7b08810be912 Mon Sep 17 00:00:00 2001
From: Dominik Hebeler <dominik@suma-ev.de>
Date: Thu, 25 Feb 2021 14:31:44 +0100
Subject: [PATCH] added preference setting and loading adgoal we reloaded
results
---
app/Http/Controllers/MetaGerSearch.php | 29 ++---
app/Http/Controllers/Stresstest.php | 1 -
app/MetaGer.php | 61 ++---------
app/Models/Adgoal.php | 142 +++++++++++++++----------
app/Models/Admitad.php | 13 ++-
config/metager/metager.php | 1 +
6 files changed, 120 insertions(+), 127 deletions(-)
diff --git a/app/Http/Controllers/MetaGerSearch.php b/app/Http/Controllers/MetaGerSearch.php
index df9268fd2..aa4b076a8 100644
--- a/app/Http/Controllers/MetaGerSearch.php
+++ b/app/Http/Controllers/MetaGerSearch.php
@@ -94,14 +94,20 @@ class MetaGerSearch extends Controller
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults($timings);
$admitad = [];
+ $adgoal = [];
if(!$metager->isApiAuthorized() && !$metager->isDummy()){
$newAdmitad = new \App\Models\Admitad($metager);
if(!empty($newAdmitad->hash)){
$admitad[] = $newAdmitad;
}
+ $newAdgoal = new \App\Models\Adgoal($metager);
+ if(!empty($newAdgoal->hash)){
+ $adgoal[] = $newAdgoal;
+ }
}
- $metager->parseAdmitad($admitad);
+ $metager->parseAffiliates($admitad);
+ $metager->parseAffiliates($adgoal);
$finished = true;
foreach ($metager->getEngines() as $engine) {
@@ -116,10 +122,7 @@ class MetaGerSearch extends Controller
"apiAuthorized" => $metager->isApiAuthorized(),
],
"admitad" => $admitad,
- "adgoal" => [
- "loaded" => $metager->isAdgoalLoaded(),
- "adgoalHash" => $metager->getAdgoalHash(),
- ],
+ "adgoal" => $adgoal,
"engines" => $metager->getEngines(),
], 60 * 60);
} catch (\Exception $e) {
@@ -205,8 +208,6 @@ class MetaGerSearch extends Controller
$metager = new MetaGer(substr($hash, strpos($hash, "loader_") + 7));
$metager->setApiAuthorized($mg["apiAuthorized"]);
- $metager->setAdgoalLoaded($adgoal["loaded"]);
- $metager->setAdgoalHash($adgoal["adgoalHash"]);
$metager->parseFormData($request, false);
$metager->setJsEnabled(true);
@@ -227,9 +228,14 @@ class MetaGerSearch extends Controller
if(!empty($newAdmitad->hash)){
$admitad[] = $newAdmitad;
}
+ $newAdgoal = new \App\Models\Adgoal($metager);
+ if(!empty($newAdgoal->hash)){
+ $adgoal[] = $newAdgoal;
+ }
}
- $admitadFinished = $metager->parseAdmitad($admitad);
+ $admitadFinished = $metager->parseAffiliates($admitad);
+ $adgoalFinished = $metager->parseAffiliates($adgoal);
$result = [
'finished' => true,
@@ -277,7 +283,7 @@ class MetaGerSearch extends Controller
}
}
- if (!$metager->isAdgoalLoaded() || !$admitadFinished) {
+ if (!$adgoalFinished || !$admitadFinished) {
$finished = false;
}
@@ -295,10 +301,7 @@ class MetaGerSearch extends Controller
"apiAuthorized" => $metager->isApiAuthorized(),
],
"admitad" => $admitad,
- "adgoal" => [
- "loaded" => $metager->isAdgoalLoaded(),
- "adgoalHash" => $metager->getAdgoalHash(),
- ],
+ "adgoal" => $adgoal,
"engines" => $metager->getEngines(),
], 1 * 60);
diff --git a/app/Http/Controllers/Stresstest.php b/app/Http/Controllers/Stresstest.php
index a4114a8c6..402c4abba 100644
--- a/app/Http/Controllers/Stresstest.php
+++ b/app/Http/Controllers/Stresstest.php
@@ -24,7 +24,6 @@ class Stresstest extends MetaGerSearch
# deactivates adgoal
$metager->setDummy(true);
- $metager->setAdgoalHash(true);
parent::search($request, $metager, $timing);
}
diff --git a/app/MetaGer.php b/app/MetaGer.php
index b05f35f5c..f68f66081 100644
--- a/app/MetaGer.php
+++ b/app/MetaGer.php
@@ -308,33 +308,12 @@ class MetaGer
if (empty($this->adgoalLoaded)) {
$this->adgoalLoaded = false;
}
- if (!$this->apiAuthorized && !$this->adgoalLoaded && !$this->dummy) {
- if (empty($this->adgoalHash)) {
- if (!empty($this->jskey)) {
- $js = Redis::connection('cache')->lpop("js" . $this->jskey);
- if ($js !== null && boolval($js)) {
- $this->javascript = true;
- }
- }
- $this->adgoalHash = \App\Models\Adgoal::startAdgoal($this->results);
- if (!empty($timings)) {
- $timings["prepareResults"]["started adgoal"] = microtime(true) - $timings["starttime"];
- }
- }
-
- if (!$this->javascript) {
- $this->adgoalLoaded = \App\Models\Adgoal::parseAdgoal($this->results, $this->adgoalHash, true);
- if (!empty($timings)) {
- $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
- }
- } else {
- $this->adgoalLoaded = \App\Models\Adgoal::parseAdgoal($this->results, $this->adgoalHash, false);
- if (!empty($timings)) {
- $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
- }
+
+ if (!empty($this->jskey)) {
+ $js = Redis::connection('cache')->lpop("js" . $this->jskey);
+ if ($js !== null && boolval($js)) {
+ $this->javascript = true;
}
- } else {
- $this->adgoalLoaded = true;
}
# Human Verification
@@ -441,16 +420,16 @@ class MetaGer
* @param Boolean $wait Wait for Results?
* @return Boolean whether or not all Admitad Objects are finished
*/
- public function parseAdmitad(&$admitads){
+ public function parseAffiliates(&$affiliates){
$wait = false;
$finished = true;
if(!$this->javascript){
$wait = true;
}
- foreach ($admitads as $admitad) {
- $admitad->fetchAffiliates($wait);
- $admitad->parseAffiliates($this->results);
- if(!$admitad->finished){
+ foreach ($affiliates as $affiliate) {
+ $affiliate->fetchAffiliates($wait);
+ $affiliate->parseAffiliates($this->results);
+ if(!$affiliate->finished){
$finished = false;
}
}
@@ -1914,26 +1893,6 @@ class MetaGer
return $this->engines;
}
- public function setAdgoalHash($hash)
- {
- $this->adgoalHash = $hash;
- }
-
- public function getAdgoalHash()
- {
- return $this->adgoalHash;
- }
-
- public function isAdgoalLoaded()
- {
- return $this->adgoalLoaded;
- }
-
- public function setAdgoalLoaded($adgoalLoaded)
- {
- $this->adgoalLoaded = $adgoalLoaded;
- }
-
public function isApiAuthorized()
{
return $this->apiAuthorized;
diff --git a/app/Models/Adgoal.php b/app/Models/Adgoal.php
index 38ecc00bd..dcd1ff07e 100644
--- a/app/Models/Adgoal.php
+++ b/app/Models/Adgoal.php
@@ -22,29 +22,47 @@ class Adgoal
"lk","sh","kn","lc","pm","vc","sd","sr","za","kr","sz","sy","tj","tw","tz","th","tg","to","tt","td","cz","tn",
"tm","tc","tv","tr","us","ug","ua","xx","hu","uy","uz","vu","va","ve","ae","vn","wf","cx","by","eh","ww","zr","cf","cy",];
- public static function startAdgoal(&$results)
+
+ public $hash;
+ public $finished = false; // Is true when the Request was sent to and read from Admitad App
+ private $affiliates = null;
+ private $startTime;
+
+ /**
+ * Creates a new Adgoal object which will start a request for affiliate links
+ * based on a result List from MetaGer.
+ * It will parse the Links of the results and query any affiliate shops.
+ *
+ * @param \App\MetaGer $metager
+ */
+ public function __construct(&$metager)
{
+ $this->startTime = microtime(true);
$publicKey = getenv('adgoal_public');
$privateKey = getenv('adgoal_private');
if ($publicKey === false) {
return true;
}
+ $results = $metager->getResults();
$linkList = "";
foreach ($results as $result) {
if (!$result->new) {
continue;
}
- $link = $result->link;
+ $link = $result->originalLink;
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
}
$linkList .= $link . ",";
}
+ if(empty($linkList)){
+ return;
+ }
$linkList = rtrim($linkList, ",");
# Hashwert
- $hash = md5($linkList . $privateKey);
+ $this->hash = md5($linkList . $privateKey);
$link = "https://xf.gdprvalidate.de/v4/check";
@@ -73,14 +91,14 @@ class Adgoal
"key" => $publicKey,
"panel" => "ZMkW9eSKJS",
"member" => "338b9Bnm",
- "signature" => $hash,
+ "signature" => $this->hash,
"links" => $linkList,
"country" => $country,
];
// Submit fetch job to worker
$mission = [
- "resulthash" => $hash,
+ "resulthash" => $this->hash,
"url" => $link,
"useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
"username" => null,
@@ -97,76 +115,86 @@ class Adgoal
];
$mission = json_encode($mission);
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
-
- return $hash;
}
-
- public static function parseAdgoal(&$results, $hash, $waitForResult)
- {
- # Wait for result
- $startTime = microtime(true);
- $answer = null;
-
- # Hash is true if Adgoal request wasn't started in the first place
- if ($hash === true) {
- return true;
+
+ public function fetchAffiliates($wait = false){
+ if($this->affiliates !== null){
+ return;
}
-
- if ($waitForResult) {
+
+ $answer = null;
+ $startTime = microtime(true);
+ if($wait){
while (microtime(true) - $startTime < 5) {
- $answer = Cache::get($hash);
+ $answer = Cache::get($this->hash);
if ($answer === null) {
usleep(50 * 1000);
} else {
break;
}
}
- } else {
- $answer = Cache::get($hash);
+ }else{
+ $answer = Cache::get($this->hash);
}
- if ($answer === null) {
- return false;
+ $answer = json_decode($answer, true);
+
+ // If the fetcher had an Error
+ if($answer === "no-result"){
+ $this->affiliates = [];
+ return;
+ }
+
+ if(empty($answer) || !is_array($answer)){
+ return;
}
-
- try {
- $answer = json_decode($answer, true);
- foreach ($answer as $partnershop) {
- $targetUrl = $partnershop["url"];
+ $this->affiliates = $answer;
+ }
- $tld = $partnershop["tld"];
- $targetHost = parse_url($targetUrl, PHP_URL_HOST);
+ /**
+ * Converts all Affiliate Links.
+ *
+ * @param \App\Models\Result[] $results
+ */
+ public function parseAffiliates(&$results)
+ {
+ if($this->finished || $this->affiliates === null){
+ return;
+ }
+
+ foreach ($this->affiliates as $partnershop) {
+ $targetUrl = $partnershop["url"];
- /*
- Adgoal sometimes returns affiliate Links for every URL
- That's why we check if the corresponding TLD matches the orginial URL
- */
- if($targetHost !== false && stripos($targetHost, $tld) === false){
- continue;
- }
+ $tld = $partnershop["tld"];
+ $targetHost = parse_url($targetUrl, PHP_URL_HOST);
- foreach ($results as $result) {
- if ($result->link === $targetUrl && !$result->partnershop) {
- # Ein Advertiser gefunden
- if ($result->image !== "") {
- $result->logo = $partnershop["logo"];
- } else {
- $result->image = $partnershop["logo"];
- }
-
- # Den Link hinzufügen:
- $result->link = $partnershop["click_url"];
- $result->partnershop = true;
- $result->changed = true;
+ /*
+ Adgoal sometimes returns affiliate Links for every URL
+ That's why we check if the corresponding TLD matches the orginial URL
+ */
+ if($targetHost !== false && stripos($targetHost, $tld) === false){
+ continue;
+ }
+ $preference = config("metager.metager.affiliate_preference", "adgoal");
+ foreach ($results as $result) {
+ if ($result->originalLink === $targetUrl && (config("metager.metager.affiliate_preference", "adgoal") === "adgoal" || !$result->partnershop)) {
+ # Ein Advertiser gefunden
+ if ($result->image !== "" && !$result->partnershop) {
+ $result->logo = $partnershop["logo"];
+ } else {
+ $result->image = $partnershop["logo"];
}
+
+ # Den Link hinzufügen:
+ $result->link = $partnershop["click_url"];
+ $result->partnershop = true;
+ $result->changed = true;
}
}
- } catch (\ErrorException $e) {
- Log::error($e->getMessage());
- } finally {
- $requestTime = microtime(true) - $startTime;
- \App\PrometheusExporter::Duration($requestTime, "adgoal");
}
- return true;
+
+ $requestTime = microtime(true) - $this->startTime;
+ \App\PrometheusExporter::Duration($requestTime, "adgoal");
+ $this->finished = true;
}
}
diff --git a/app/Models/Admitad.php b/app/Models/Admitad.php
index 8bd2d3e9f..67546117e 100644
--- a/app/Models/Admitad.php
+++ b/app/Models/Admitad.php
@@ -16,7 +16,7 @@ class Admitad
public $hash;
public $finished = false; // Is true when the Request was sent to and read from Admitad App
- private $affiliates = [];
+ private $affiliates = null;
/**
* Creates a new Admitad object which will start a request for affiliate links
@@ -81,7 +81,7 @@ class Admitad
* @param Boolean $wait Whether or not to wait for a response
*/
public function fetchAffiliates($wait = false) {
- if($this->finished){
+ if($this->affiliates !== null){
return;
}
@@ -103,7 +103,7 @@ class Admitad
// If the fetcher had an Error
if($answer === "no-result"){
- $this->finished = true;
+ $this->affiliates = [];
return;
}
@@ -112,7 +112,6 @@ class Admitad
}
$this->affiliates = $answer["result"];
- $this->finished = true;
}
/**
@@ -121,6 +120,9 @@ class Admitad
* @param \App\Models\Result[] $results
*/
public function parseAffiliates(&$results){
+ if($this->finished || $this->affiliates === null){
+ return;
+ }
foreach($this->affiliates as $linkResult){
$originalUrl = $linkResult["originalUrl"];
$redirUrl = $linkResult["redirUrl"];
@@ -132,7 +134,7 @@ class Admitad
}
foreach ($results as $result) {
- if ($result->originalLink === $originalUrl) {
+ if ($result->originalLink === $originalUrl && (config("metager.metager.affiliate_preference", "adgoal") === "admitad" || !$result->partnershop)) {
# Ein Advertiser gefunden
if ($result->image !== "" && !$result->partnershop) {
$result->logo = $image;
@@ -147,5 +149,6 @@ class Admitad
}
}
}
+ $this->finished = true;
}
}
diff --git a/config/metager/metager.php b/config/metager/metager.php
index fda3d2ee5..583e6a9a7 100644
--- a/config/metager/metager.php
+++ b/config/metager/metager.php
@@ -6,4 +6,5 @@ return [
"w3m\/",
],
"fail2ban_enabled" => true,
+ "affiliate_preference" => "adgoal",
];
--
GitLab