Commit b3e53cb6 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

added preference setting and loading adgoal we reloaded results

parent 4dddf205
......@@ -94,14 +94,20 @@ class MetaGerSearch extends Controller
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults($timings);
$admitad = [];
$adgoal = [];
if(!$metager->isApiAuthorized() && !$metager->isDummy()){
$newAdmitad = new \App\Models\Admitad($metager);
if(!empty($newAdmitad->hash)){
$admitad[] = $newAdmitad;
}
$newAdgoal = new \App\Models\Adgoal($metager);
if(!empty($newAdgoal->hash)){
$adgoal[] = $newAdgoal;
}
}
$metager->parseAdmitad($admitad);
$metager->parseAffiliates($admitad);
$metager->parseAffiliates($adgoal);
$finished = true;
foreach ($metager->getEngines() as $engine) {
......@@ -116,10 +122,7 @@ class MetaGerSearch extends Controller
"apiAuthorized" => $metager->isApiAuthorized(),
],
"admitad" => $admitad,
"adgoal" => [
"loaded" => $metager->isAdgoalLoaded(),
"adgoalHash" => $metager->getAdgoalHash(),
],
"adgoal" => $adgoal,
"engines" => $metager->getEngines(),
], 60 * 60);
} catch (\Exception $e) {
......@@ -205,8 +208,6 @@ class MetaGerSearch extends Controller
$metager = new MetaGer(substr($hash, strpos($hash, "loader_") + 7));
$metager->setApiAuthorized($mg["apiAuthorized"]);
$metager->setAdgoalLoaded($adgoal["loaded"]);
$metager->setAdgoalHash($adgoal["adgoalHash"]);
$metager->parseFormData($request, false);
$metager->setJsEnabled(true);
......@@ -227,9 +228,14 @@ class MetaGerSearch extends Controller
if(!empty($newAdmitad->hash)){
$admitad[] = $newAdmitad;
}
$newAdgoal = new \App\Models\Adgoal($metager);
if(!empty($newAdgoal->hash)){
$adgoal[] = $newAdgoal;
}
}
$admitadFinished = $metager->parseAdmitad($admitad);
$admitadFinished = $metager->parseAffiliates($admitad);
$adgoalFinished = $metager->parseAffiliates($adgoal);
$result = [
'finished' => true,
......@@ -277,7 +283,7 @@ class MetaGerSearch extends Controller
}
}
if (!$metager->isAdgoalLoaded() || !$admitadFinished) {
if (!$adgoalFinished || !$admitadFinished) {
$finished = false;
}
......@@ -295,10 +301,7 @@ class MetaGerSearch extends Controller
"apiAuthorized" => $metager->isApiAuthorized(),
],
"admitad" => $admitad,
"adgoal" => [
"loaded" => $metager->isAdgoalLoaded(),
"adgoalHash" => $metager->getAdgoalHash(),
],
"adgoal" => $adgoal,
"engines" => $metager->getEngines(),
], 1 * 60);
......
......@@ -24,7 +24,6 @@ class Stresstest extends MetaGerSearch
# deactivates adgoal
$metager->setDummy(true);
$metager->setAdgoalHash(true);
parent::search($request, $metager, $timing);
}
......
......@@ -308,33 +308,12 @@ class MetaGer
if (empty($this->adgoalLoaded)) {
$this->adgoalLoaded = false;
}
if (!$this->apiAuthorized && !$this->adgoalLoaded && !$this->dummy) {
if (empty($this->adgoalHash)) {
if (!empty($this->jskey)) {
$js = Redis::connection('cache')->lpop("js" . $this->jskey);
if ($js !== null && boolval($js)) {
$this->javascript = true;
}
}
$this->adgoalHash = \App\Models\Adgoal::startAdgoal($this->results);
if (!empty($timings)) {
$timings["prepareResults"]["started adgoal"] = microtime(true) - $timings["starttime"];
}
}
if (!$this->javascript) {
$this->adgoalLoaded = \App\Models\Adgoal::parseAdgoal($this->results, $this->adgoalHash, true);
if (!empty($timings)) {
$timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
}
} else {
$this->adgoalLoaded = \App\Models\Adgoal::parseAdgoal($this->results, $this->adgoalHash, false);
if (!empty($timings)) {
$timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
}
if (!empty($this->jskey)) {
$js = Redis::connection('cache')->lpop("js" . $this->jskey);
if ($js !== null && boolval($js)) {
$this->javascript = true;
}
} else {
$this->adgoalLoaded = true;
}
# Human Verification
......@@ -441,16 +420,16 @@ class MetaGer
* @param Boolean $wait Wait for Results?
* @return Boolean whether or not all Admitad Objects are finished
*/
public function parseAdmitad(&$admitads){
public function parseAffiliates(&$affiliates){
$wait = false;
$finished = true;
if(!$this->javascript){
$wait = true;
}
foreach ($admitads as $admitad) {
$admitad->fetchAffiliates($wait);
$admitad->parseAffiliates($this->results);
if(!$admitad->finished){
foreach ($affiliates as $affiliate) {
$affiliate->fetchAffiliates($wait);
$affiliate->parseAffiliates($this->results);
if(!$affiliate->finished){
$finished = false;
}
}
......@@ -1914,26 +1893,6 @@ class MetaGer
return $this->engines;
}
public function setAdgoalHash($hash)
{
$this->adgoalHash = $hash;
}
public function getAdgoalHash()
{
return $this->adgoalHash;
}
public function isAdgoalLoaded()
{
return $this->adgoalLoaded;
}
public function setAdgoalLoaded($adgoalLoaded)
{
$this->adgoalLoaded = $adgoalLoaded;
}
public function isApiAuthorized()
{
return $this->apiAuthorized;
......
......@@ -22,29 +22,47 @@ class Adgoal
"lk","sh","kn","lc","pm","vc","sd","sr","za","kr","sz","sy","tj","tw","tz","th","tg","to","tt","td","cz","tn",
"tm","tc","tv","tr","us","ug","ua","xx","hu","uy","uz","vu","va","ve","ae","vn","wf","cx","by","eh","ww","zr","cf","cy",];
public static function startAdgoal(&$results)
public $hash;
public $finished = false; // Is true when the Request was sent to and read from Admitad App
private $affiliates = null;
private $startTime;
/**
* Creates a new Adgoal object which will start a request for affiliate links
* based on a result List from MetaGer.
* It will parse the Links of the results and query any affiliate shops.
*
* @param \App\MetaGer $metager
*/
public function __construct(&$metager)
{
$this->startTime = microtime(true);
$publicKey = getenv('adgoal_public');
$privateKey = getenv('adgoal_private');
if ($publicKey === false) {
return true;
}
$results = $metager->getResults();
$linkList = "";
foreach ($results as $result) {
if (!$result->new) {
continue;
}
$link = $result->link;
$link = $result->originalLink;
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
}
$linkList .= $link . ",";
}
if(empty($linkList)){
return;
}
$linkList = rtrim($linkList, ",");
# Hashwert
$hash = md5($linkList . $privateKey);
$this->hash = md5($linkList . $privateKey);
$link = "https://xf.gdprvalidate.de/v4/check";
......@@ -73,14 +91,14 @@ class Adgoal
"key" => $publicKey,
"panel" => "ZMkW9eSKJS",
"member" => "338b9Bnm",
"signature" => $hash,
"signature" => $this->hash,
"links" => $linkList,
"country" => $country,
];
// Submit fetch job to worker
$mission = [
"resulthash" => $hash,
"resulthash" => $this->hash,
"url" => $link,
"useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
"username" => null,
......@@ -97,76 +115,86 @@ class Adgoal
];
$mission = json_encode($mission);
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
return $hash;
}
public static function parseAdgoal(&$results, $hash, $waitForResult)
{
# Wait for result
$startTime = microtime(true);
$answer = null;
# Hash is true if Adgoal request wasn't started in the first place
if ($hash === true) {
return true;
public function fetchAffiliates($wait = false){
if($this->affiliates !== null){
return;
}
if ($waitForResult) {
$answer = null;
$startTime = microtime(true);
if($wait){
while (microtime(true) - $startTime < 5) {
$answer = Cache::get($hash);
$answer = Cache::get($this->hash);
if ($answer === null) {
usleep(50 * 1000);
} else {
break;
}
}
} else {
$answer = Cache::get($hash);
}else{
$answer = Cache::get($this->hash);
}
if ($answer === null) {
return false;
$answer = json_decode($answer, true);
// If the fetcher had an Error
if($answer === "no-result"){
$this->affiliates = [];
return;
}
if(empty($answer) || !is_array($answer)){
return;
}
try {
$answer = json_decode($answer, true);
foreach ($answer as $partnershop) {
$targetUrl = $partnershop["url"];
$this->affiliates = $answer;
}
$tld = $partnershop["tld"];
$targetHost = parse_url($targetUrl, PHP_URL_HOST);
/**
* Converts all Affiliate Links.
*
* @param \App\Models\Result[] $results
*/
public function parseAffiliates(&$results)
{
if($this->finished || $this->affiliates === null){
return;
}
foreach ($this->affiliates as $partnershop) {
$targetUrl = $partnershop["url"];
/*
Adgoal sometimes returns affiliate Links for every URL
That's why we check if the corresponding TLD matches the orginial URL
*/
if($targetHost !== false && stripos($targetHost, $tld) === false){
continue;
}
$tld = $partnershop["tld"];
$targetHost = parse_url($targetUrl, PHP_URL_HOST);
foreach ($results as $result) {
if ($result->link === $targetUrl && !$result->partnershop) {
# Ein Advertiser gefunden
if ($result->image !== "") {
$result->logo = $partnershop["logo"];
} else {
$result->image = $partnershop["logo"];
}
# Den Link hinzufügen:
$result->link = $partnershop["click_url"];
$result->partnershop = true;
$result->changed = true;
/*
Adgoal sometimes returns affiliate Links for every URL
That's why we check if the corresponding TLD matches the orginial URL
*/
if($targetHost !== false && stripos($targetHost, $tld) === false){
continue;
}
$preference = config("metager.metager.affiliate_preference", "adgoal");
foreach ($results as $result) {
if ($result->originalLink === $targetUrl && (config("metager.metager.affiliate_preference", "adgoal") === "adgoal" || !$result->partnershop)) {
# Ein Advertiser gefunden
if ($result->image !== "" && !$result->partnershop) {
$result->logo = $partnershop["logo"];
} else {
$result->image = $partnershop["logo"];
}
# Den Link hinzufügen:
$result->link = $partnershop["click_url"];
$result->partnershop = true;
$result->changed = true;
}
}
} catch (\ErrorException $e) {
Log::error($e->getMessage());
} finally {
$requestTime = microtime(true) - $startTime;
\App\PrometheusExporter::Duration($requestTime, "adgoal");
}
return true;
$requestTime = microtime(true) - $this->startTime;
\App\PrometheusExporter::Duration($requestTime, "adgoal");
$this->finished = true;
}
}
......@@ -16,7 +16,7 @@ class Admitad
public $hash;
public $finished = false; // Is true when the Request was sent to and read from Admitad App
private $affiliates = [];
private $affiliates = null;
/**
* Creates a new Admitad object which will start a request for affiliate links
......@@ -81,7 +81,7 @@ class Admitad
* @param Boolean $wait Whether or not to wait for a response
*/
public function fetchAffiliates($wait = false) {
if($this->finished){
if($this->affiliates !== null){
return;
}
......@@ -103,7 +103,7 @@ class Admitad
// If the fetcher had an Error
if($answer === "no-result"){
$this->finished = true;
$this->affiliates = [];
return;
}
......@@ -112,7 +112,6 @@ class Admitad
}
$this->affiliates = $answer["result"];
$this->finished = true;
}
/**
......@@ -121,6 +120,9 @@ class Admitad
* @param \App\Models\Result[] $results
*/
public function parseAffiliates(&$results){
if($this->finished || $this->affiliates === null){
return;
}
foreach($this->affiliates as $linkResult){
$originalUrl = $linkResult["originalUrl"];
$redirUrl = $linkResult["redirUrl"];
......@@ -132,7 +134,7 @@ class Admitad
}
foreach ($results as $result) {
if ($result->originalLink === $originalUrl) {
if ($result->originalLink === $originalUrl && (config("metager.metager.affiliate_preference", "adgoal") === "admitad" || !$result->partnershop)) {
# Ein Advertiser gefunden
if ($result->image !== "" && !$result->partnershop) {
$result->logo = $image;
......@@ -147,5 +149,6 @@ class Admitad
}
}
}
$this->finished = true;
}
}
......@@ -6,4 +6,5 @@ return [
"w3m\/",
],
"fail2ban_enabled" => true,
"affiliate_preference" => "adgoal",
];
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment