Commit 000f9a68 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

loading adgoal async when javascript is enabled

parent 711365ba
......@@ -111,7 +111,16 @@ class MetaGerSearch extends Controller
}
try {
Cache::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 60 * 60);
Cache::put("loader_" . $metager->getSearchUid(), [
"metager" => [
"apiAuthorized" => $metager->isApiAuthorized(),
],
"adgoal" => [
"loaded" => $metager->isAdgoalLoaded(),
"adgoalHash" => $metager->getAdgoalHash(),
],
"engines" => $metager->getEngines(),
], 60 * 60);
} catch (\Exception $e) {
Log::error($e->getMessage());
}
......@@ -190,12 +199,19 @@ class MetaGerSearch extends Controller
}
}
$engines = Cache::get($hash);
if ($engines === null) {
$cached = Cache::get($hash);
if ($cached === null) {
return response()->json(['finished' => true]);
}
$engines = $cached["engines"];
$adgoal = $cached["adgoal"];
$mg = $cached["metager"];
$metager = new MetaGer(substr($hash, strpos($hash, "loader_") + 7));
$metager->setApiAuthorized($mg["apiAuthorized"]);
$metager->setAdgoalLoaded($adgoal["loaded"]);
$metager->setAdgoalHash($adgoal["adgoalHash"]);
$metager->parseFormData($request);
# Nach Spezialsuchen überprüfen:
......@@ -203,27 +219,37 @@ class MetaGerSearch extends Controller
$metager->restoreEngines($engines);
$metager->retrieveResults();
$metager->rankAll();
$metager->prepareResults();
$result = [
'finished' => true,
'newResults' => [],
'changedResults' => [],
];
$result["nextSearchLink"] = $metager->nextSearchLink();
$newResults = 0;
foreach ($metager->getResults() as $index => $resultTmp) {
if ($resultTmp->new) {
if ($resultTmp->new || $resultTmp->adgoalChanged) {
if ($metager->getFokus() !== "bilder") {
$view = View::make('layouts.result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
if (!$resultTmp->new && $resultTmp->adgoalChanged) {
$result['changedResults'][$index] = $html;
} else {
$result['newResults'][$index] = $html;
}
$result["imagesearch"] = false;
} else {
$view = View::make('layouts.image_result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
if (!$resultTmp->new && $resultTmp->adgoalChanged) {
$result['changedResults'][$index] = $html;
} else {
$result['newResults'][$index] = $html;
}
$result["imagesearch"] = true;
}
$newResults++;
......@@ -239,6 +265,9 @@ class MetaGerSearch extends Controller
$engine->markNew();
}
}
if (!$metager->isAdgoalLoaded()) {
$finished = false;
}
$result["finished"] = $finished;
......@@ -248,7 +277,16 @@ class MetaGerSearch extends Controller
$counter->incBy($newResults);
}
// Update new Engines
Cache::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 1 * 60);
Cache::put("loader_" . $metager->getSearchUid(), [
"metager" => [
"apiAuthorized" => $metager->isApiAuthorized(),
],
"adgoal" => [
"loaded" => $metager->isAdgoalLoaded(),
"adgoalHash" => $metager->getAdgoalHash(),
],
"engines" => $metager->getEngines(),
], 1 * 60);
return response()->json($result);
}
......
......@@ -300,13 +300,25 @@ class MetaGer
$this->ads = $newResults;
#Adgoal Implementation
if (!$this->apiAuthorized) {
$this->results = $this->parseAdgoal($this->results);
if (empty($this->adgoalLoaded)) {
$this->adgoalLoaded = false;
}
if (!$this->apiAuthorized && !$this->adgoalLoaded) {
if (empty($this->adgoalHash)) {
$this->adgoalHash = $this->startAdgoal($this->results);
}
if (!$this->javascript) {
$this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, true);
} else {
$this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, false);
}
} else {
$this->adgoalLoaded = true;
}
# Human Verification
$this->results = $this->humanVerification($this->results);
$this->ads = $this->humanVerification($this->ads);
$this->humanVerification($this->results);
$this->humanVerification($this->ads);
$counter = 0;
$firstRank = 0;
......@@ -343,54 +355,98 @@ class MetaGer
}
foreach ($engine->results as $result) {
if ($result->valid) {
$this->results[] = $result;
$this->results[] = clone $result;
}
}
foreach ($engine->ads as $ad) {
$this->ads[] = $ad;
$this->ads[] = clone $ad;
}
}
}
public function parseAdgoal($results)
public function startAdgoal(&$results)
{
$time = microtime(true);
$publicKey = getenv('adgoal_public');
$privateKey = getenv('adgoal_private');
if ($publicKey === false) {
return $results;
}
$tldList = "";
foreach ($results as $result) {
if (!$result->new) {
continue;
}
$link = $result->link;
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
}
$tldList .= parse_url($link, PHP_URL_HOST) . ",";
$result->tld = parse_url($link, PHP_URL_HOST);
}
$tldList = rtrim($tldList, ",");
# Hashwert
$hash = md5("meta" . $publicKey . $tldList . "GER");
Redis::del($hash); # TODO remove
# Query
$query = $this->q;
$link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);
// Submit fetch job to worker
$mission = [
"resulthash" => $hash,
"url" => $link,
"useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
"username" => null,
"password" => null,
"headers" => null,
"cacheDuration" => 60,
];
$mission = json_encode($mission);
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
return $hash;
}
public function parseAdgoal(&$results, $hash, $waitForResult)
{
# Wait for result
$startTime = microtime(true);
$answer = null;
if ($waitForResult) {
while (microtime(true) - $startTime < 5) {
$answer = Redis::get($hash);
if ($answer === null) {
usleep(50 * 1000);
} else {
break;
}
}
} else {
$answer = Redis::get($hash);
}
if ($answer === null) {
return false;
}
try {
$answer = json_decode($answer);
$publicKey = getenv('adgoal_public');
$privateKey = getenv('adgoal_private');
# Nun müssen wir nur noch die Links für die Advertiser ändern:
foreach ($results as $result) {
if (!$result->new) {
continue;
}
$link = $result->link;
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
}
$tldList .= parse_url($link, PHP_URL_HOST) . ",";
$result->tld = parse_url($link, PHP_URL_HOST);
}
$tldList = rtrim($tldList, ",");
# Hashwert
$hash = md5("meta" . $publicKey . $tldList . "GER");
# Query
$query = $this->q;
$link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);
$answer = json_decode(file_get_contents($link));
# Nun müssen wir nur noch die Links für die Advertiser ändern:
foreach ($answer as $el) {
$hoster = $el[0];
$hash = $el[1];
foreach ($results as $result) {
if ($result->new && $hoster === $result->tld && !$result->partnershop) {
if ($hoster === $result->tld && !$result->partnershop) {
# Hier ist ein Advertiser:
# Das Logo hinzufügen:
if ($result->image !== "") {
......@@ -400,27 +456,29 @@ class MetaGer
}
# Den Link hinzufügen:
$publicKey = $publicKey;
$targetUrl = $result->link;
# Query
$query = $this->q;
$gateHash = md5($targetUrl . $privateKey);
$newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
$result->link = $newLink;
$result->partnershop = true;
$result->adgoalChanged = true;
}
}
}
} catch (\ErrorException $e) {
return $results;
Log::error($e->getMessage());
} finally {
$requestTime = microtime(true) - $time;
$requestTime = microtime(true) - $startTime;
\App\PrometheusExporter::Duration($requestTime, "adgoal");
}
return true;
return $results;
}
public function humanVerification($results)
public function humanVerification(&$results)
{
# Let's check if we need to implement a redirect for human verification
if ($this->verificationCount > 10) {
......@@ -434,9 +492,6 @@ class MetaGer
$result->link = $url;
$result->proxyLink = $proxyUrl;
}
return $results;
} else {
return $results;
}
}
......@@ -1833,6 +1888,36 @@ class MetaGer
return $this->engines;
}
public function setAdgoalHash($hash)
{
$this->adgoalHash = $hash;
}
public function getAdgoalHash()
{
return $this->adgoalHash;
}
public function isAdgoalLoaded()
{
return $this->adgoalLoaded;
}
public function setAdgoalLoaded($adgoalLoaded)
{
$this->adgoalLoaded = $adgoalLoaded;
}
public function isApiAuthorized()
{
return $this->apiAuthorized;
}
public function setApiAuthorized($authorized)
{
$this->apiAuthorized = $authorized;
}
public function isFramed()
{
return $this->framed;
......@@ -1849,5 +1934,8 @@ class MetaGer
public function restoreEngines($engines)
{
$this->engines = $engines;
foreach ($this->engines as $engine) {
$engine->setCached(true);
}
}
}
......@@ -31,6 +31,7 @@ class Result
public $strippedLinkAnzeige; # Der Link in Form "foo.bar.de/test"
public $rank; # Das Ranking für das Ergebnis
public $new = true;
public $adgoalChanged = false;
# Erstellt ein neues Ergebnis
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
......
......@@ -208,10 +208,12 @@ abstract class Searchengine
}
if ($body !== null) {
try {
Cache::put($this->hash, $body, $this->cacheDuration * 60);
} catch (\Exception $e) {
Log::error($e->getMessage());
if (!$this->cached) {
try {
Cache::put($this->hash, $body, $this->cacheDuration * 60);
} catch (\Exception $e) {
Log::error($e->getMessage());
}
}
$this->loadResults($body);
$this->getNext($metager, $body);
......@@ -288,4 +290,9 @@ abstract class Searchengine
{
$this->new = $new;
}
public function setCached($cached)
{
$this->cached = $cached;
}
}
......@@ -116,6 +116,17 @@ function loadMoreResults() {
}
}
}
if (typeof data.changedResults != "undefined") {
for (var key in data.changedResults) {
var value = data.changedResults[key];
// If there are more results than the given index we will prepend otherwise we will append the result
if (!data.imagesearch) {
$($(".result:not(.ad)")[key]).replaceWith($(value));
} else {
$($(".image-container > .image")[key]).replaceWith($(value));
}
}
}
currentlyLoading = false;
});
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment