Commit f85b4408 authored by Karl Hasselbring's avatar Karl Hasselbring
Browse files

Quicktips werden jetzt über Worker als XML geladen, das xml-parsen...

Quicktips werden jetzt über Worker als XML geladen, das xml-parsen funktioniert allerdings noch nicht
parent 160261c1
...@@ -11,6 +11,7 @@ class MetaGerSearch extends Controller ...@@ -11,6 +11,7 @@ class MetaGerSearch extends Controller
{ {
public function search(Request $request, MetaGer $metager) public function search(Request $request, MetaGer $metager)
{ {
$focus = $request->input("focus", "web"); $focus = $request->input("focus", "web");
if ($focus === "maps") { if ($focus === "maps") {
...@@ -31,9 +32,16 @@ class MetaGerSearch extends Controller ...@@ -31,9 +32,16 @@ class MetaGerSearch extends Controller
# Nach Spezialsuchen überprüfen: # Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request); $metager->checkSpecialSearches($request);
# Alle Suchmaschinen erstellen # Die Quicktips als Job erstellen
$quicktips = $metager->createQuicktips();
# Suche für alle zu verwendenden Suchmaschinen als Job erstellen,
# auf Ergebnisse warten und die Ergebnisse laden
$metager->createSearchEngines($request); $metager->createSearchEngines($request);
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
# Alle Ergebnisse vor der Zusammenführung ranken: # Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll(); $metager->rankAll();
...@@ -41,7 +49,7 @@ class MetaGerSearch extends Controller ...@@ -41,7 +49,7 @@ class MetaGerSearch extends Controller
$metager->prepareResults(); $metager->prepareResults();
# Die Ausgabe erstellen: # Die Ausgabe erstellen:
return $metager->createView(); return $metager->createView($quicktipResults);
} }
public function botProtection($redirect) public function botProtection($redirect)
......
...@@ -68,13 +68,14 @@ class Searcher implements ShouldQueue ...@@ -68,13 +68,14 @@ class Searcher implements ShouldQueue
$mission = $mission[1]; $mission = $mission[1];
$poptime = microtime(true) - $time; $poptime = microtime(true) - $time;
// The mission is a String which can be divided to retrieve two informations: // The mission is a String which can be divided to retrieve three informations:
// 1. The Hash Value where the result should be stored // 1. The Hash Value where the result should be stored
// 2. The Url to Retrieve // 2. The Url to Retrieve
// These two informations are divided by a ";" in the mission string // 3. The maximum time to take
// These three informations are divided by a ";" in the mission string
$mission = explode(";", $mission); $mission = explode(";", $mission);
$hashValue = $mission[0]; $hashValue = $mission[0]; // The hash value for redis to store the results under
$url = base64_decode($mission[1]); $url = base64_decode($mission[1]); // The url to fetch
$timeout = $mission[2]; // Timeout from the MetaGer process in ms $timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms $medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected"); Redis::hset('search.' . $hashValue, $this->name, "connected");
......
...@@ -88,7 +88,7 @@ class MetaGer ...@@ -88,7 +88,7 @@ class MetaGer
} }
# Erstellt aus den gesammelten Ergebnissen den View # Erstellt aus den gesammelten Ergebnissen den View
public function createView() public function createView($quicktipResults = NULL)
{ {
$viewResults = []; $viewResults = [];
# Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View: # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
...@@ -184,7 +184,8 @@ class MetaGer ...@@ -184,7 +184,8 @@ class MetaGer
->with('errors', $this->errors) ->with('errors', $this->errors)
->with('apiAuthorized', $this->apiAuthorized) ->with('apiAuthorized', $this->apiAuthorized)
->with('metager', $this) ->with('metager', $this)
->with('browser', (new Agent())->browser()); ->with('browser', (new Agent())->browser())
->with('quicktips', $quicktipResults);
break; break;
} }
} }
...@@ -468,6 +469,12 @@ class MetaGer ...@@ -468,6 +469,12 @@ class MetaGer
} }
} }
public function createQuicktips() {
# Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
$quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
return $quicktips;
}
/* /*
* Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
*/ */
...@@ -570,7 +577,6 @@ class MetaGer ...@@ -570,7 +577,6 @@ class MetaGer
foreach ($engines as $engine) { foreach ($engines as $engine) {
$engine->startSearch($this); $engine->startSearch($this);
} }
$quicktips = new \App\Models\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
/* Wir warten auf die Antwort der Suchmaschinen /* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet. * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
......
<?php <?php
namespace App\Models; namespace App\Models\Quicktips;
class Quicktip class Quicktip
{ {
private $type; public $type;
private $title; public $title;
private $link; public $link;
private $descr; public $descr;
private $details; public $details;
# Erstellt ein neues Ergebnis # Erstellt ein neues Ergebnis
public function __construct($type, $title, $link, $descr, $details) public function __construct($type, $title, $link, $descr, $details)
...@@ -20,17 +20,3 @@ class Quicktip ...@@ -20,17 +20,3 @@ class Quicktip
$this->details = $details; $this->details = $details;
} }
} }
class Quicktip_Detail
{
private $title;
private $link;
private $descr;
public function __construct($title, $link, $descr)
{
$this->title = $title;
$this->link = $link;
$this->descr = $descr;
}
}
<?php
namespace App\Models\Quicktips;
class Quicktip_detail
{
public $title;
public $link;
public $descr;
public function __construct($title, $link, $descr)
{
$this->title = $title;
$this->link = $link;
$this->descr = $descr;
}
}
<?php <?php
namespace App\Models; namespace App\Models\Quicktips;
use App\Jobs\Searcher;
use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Log;
class Quicktips class Quicktips
{ {
use DispatchesJobs; use DispatchesJobs;
const QUICKTIP_URL = "https://quicktips.metager3.de/quicktips.xml"; const QUICKTIP_URL = "https://quicktips.metager3.de/quicktips.xml";
const QUICKTIP_NAME = "quicktips"; const QUICKTIP_NAME = "quicktips";
const CACHE_DURATION = 60;
public function __construct($search, $locale, $max_time, $resultHash) private $hash;
public function __construct($search, $locale, $max_time)
{ {
$this->startSearch($search, $locale, $max_time, $resultHash); $this->startSearch($search, $locale, $max_time);
} }
public function startSearch($search, $locale, $max_time, $resultHash) public function startSearch($search, $locale, $max_time)
{ {
$full_url = QUICKTIP_URL . "?search=" . $search . "&locale=" . $locale; $url = self::QUICKTIP_URL . "?search=" . $search . "&locale=" . $locale;
$hash = md5($full_url); $hash = md5($url);
if (Cache::has($hash)) { # TODO anders weitergeben
$cached = true; $this->hash = $hash;
$this->retrieveResults();
} else { # TODO cache wieder einbauen (eventuell)
// ??? if ( /*!Cache::has($hash)*/true) {
Redis::hset("search." . $resultHash, QUICKTIP_NAME, "waiting"); Redis::hset("search." . $hash, self::QUICKTIP_NAME, "waiting");
// Queue this search // Queue this search
$mission = $resultHash . ";" . $url . ";" . $max_time; $mission = $hash . ";" . base64_encode($url) . ";" . $max_time;
Redis::rpush(QUICKTIP_NAME . ".queue", $mission); Redis::rpush(self::QUICKTIP_NAME . ".queue", $mission);
// Check the current status of Searchers for QUICKTIP_NAME // Check the current status of Searchers for QUICKTIP_NAME
$needSearcher = false; $needSearcher = false;
$searcherData = Redis::hgetall(QUICKTIP_NAME . ".stats"); $searcherData = Redis::hgetall(self::QUICKTIP_NAME . ".stats");
// Create additional Searchers for QUICKTIP_NAME if necessary // Create additional Searchers for QUICKTIP_NAME if necessary
if (sizeof($searcherData) === 0) { if (sizeof($searcherData) === 0) {
...@@ -49,31 +58,39 @@ class Quicktips ...@@ -49,31 +58,39 @@ class Quicktips
$needSearcher = true; $needSearcher = true;
} }
} }
if ($needSearcher && Redis::get(QUICKTIP_NAME) !== "locked") { if ($needSearcher && Redis::get(self::QUICKTIP_NAME) !== "locked") {
Redis::set(QUICKTIP_NAME, "locked"); Redis::set(self::QUICKTIP_NAME, "locked");
$this->dispatch(new Searcher(QUICKTIP_NAME)); $this->dispatch(new Searcher(self::QUICKTIP_NAME));
} }
} }
} }
public function retrieveResults($hash, $resultHash) public function loadResults()
{
$resultsRaw = $this->retrieveResults($this->hash);
$results = $this->parseResults($resultsRaw);
return $results;
}
public function retrieveResults($hash)
{ {
$body = ""; $body = "";
if (Cache::has($hash)) { #if (Cache::has($hash)) {
$body = Cache::get($hash); $body = Cache::get($hash);
} elseif (Redis::hexists('search.' . $resultHash, QUICKTIP_NAME)) { #} elseif (Redis::hexists('search.' . $hash, self::QUICKTIP_NAME)) {
$body = Redis::hget('search.' . $resultHash, QUICKTIP_NAME); $body = Redis::hget('search.' . $hash, self::QUICKTIP_NAME);
Redis::hdel('search.' . $resultHash, QUICKTIP_NAME); Redis::hdel('search.' . $hash, self::QUICKTIP_NAME);
Cache::put($hash, $body, $cacheDuration); Cache::put($hash, $body, self::CACHE_DURATION);
} #}
if ($body !== "") { if ($body !== "") {
return $this->loadResults($body); return $body;
} else { } else {
return false; return false;
} }
} }
public function loadResults($quicktips_raw) { public function parseResults($quicktips_raw)
{
$quicktips_raw = preg_replace("/\r\n/si", "", $quicktips_raw); $quicktips_raw = preg_replace("/\r\n/si", "", $quicktips_raw);
try { try {
$content = simplexml_load_string($quicktips_raw); $content = simplexml_load_string($quicktips_raw);
...@@ -81,26 +98,47 @@ class Quicktips ...@@ -81,26 +98,47 @@ class Quicktips
return; return;
} }
$content->registerXPathNamespace('main', 'http://www.w3.org/2005/Atom');
$content->registerXPathNamespace('opensearch', 'http://a9.com/-/spec/opensearch/1.1/');
$content->registerXPathNamespace('relevance', 'http://a9.com/-/opensearch/extensions/relevance/1.0/');
$content->registerXPathNamespace('mg', 'http://metager.de/opensearch/quicktips/');
$quicktips = []; $quicktips = [];
#die(var_dump($content->xpath('//main:entry/mg:type')));
$quicktips_xpath = $content->xpath('feed/entry'); $quicktips_xpath = $content->xpath('main:entry');
foreach ($quicktips_xpath as $quicktip_xml) { foreach ($quicktips_xpath as $quicktip_xml) {
$type = $quicktip_xml->{"mg:type"}->__toString(); $content->registerXPathNamespace('mg', 'http://metager.de/opensearch/quicktips/');
$types = $quicktip_xml->xpath('mg:type');
if (sizeof($types) > 0) {
$type = $types[0]->__toString();
}
$title = $quicktip_xml->{"title"}->__toString(); $title = $quicktip_xml->{"title"}->__toString();
$link_xml = $quicktip_xml->link['href'];
if ($link_xml->count() > 0) {
$link = $link_xml->toString();
} else {
$link = "";
}
$link = $quicktip_xml->{"link"}->__toString(); $link = $quicktip_xml->{"link"}->__toString();
$descr = $quicktip_xml->{"content"}->__toString(); $descr = $quicktip_xml->{"content"}->__toString();
$details = []; $details = [];
foreach ($quicktip_xml->{"mg:details"}->{"entry"} as $detail_xml) { $details_xpath = $quicktip_xml->xpath('mg:details');
$details_title = $detail_xml->{"title"}->__toString(); if (sizeof($details_xpath) > 0) {
$details_link = $detail_xml->{"url"}->__toString(); foreach ($details_xpath as $detail_xml) {
$details_descr = $detail_xml->{"text"}->__toString(); $details_title = $detail_xml->{"title"}->__toString();
$details[] = new \App\Models\Detail( $details_link = $detail_xml->{"url"}->__toString();
$details_title, $details_descr = $detail_xml->{"text"}->__toString();
$details_link, $details[] = new \App\Models\Quicktips\Quicktip_detail(
$details_descr $details_title,
); $details_link,
$details_descr
);
}
} }
$quicktips[] = new \App\Models\Quicktip( $quicktips[] = new \App\Models\Quicktips\Quicktip(
$type, $type,
$title, $title,
$link, $link,
...@@ -108,8 +146,10 @@ class Quicktips ...@@ -108,8 +146,10 @@ class Quicktips
$details $details
); );
} }
die(var_dump($quicktips));
return $quicktips; return $quicktips;
} catch (\Exception $e) { } catch (\Exception $e) {
die($e);
Log::error("A problem occurred parsing quicktips"); Log::error("A problem occurred parsing quicktips");
return []; return [];
} }
......
...@@ -219,7 +219,6 @@ abstract class Searchengine ...@@ -219,7 +219,6 @@ abstract class Searchengine
{ {
$number = Redis::hget('search.' . $this->hash, $this->name); $number = Redis::hget('search.' . $this->hash, $this->name);
if ($number === null) { if ($number === null) {
die("test");
return null; return null;
} else { } else {
return pfsockopen($this->getHost() . ":" . $this->port . "/$number", $this->port, $errstr, $errno, 1); return pfsockopen($this->getHost() . ":" . $this->port . "/$number", $this->port, $errstr, $errno, 1);
......
...@@ -49,7 +49,9 @@ ...@@ -49,7 +49,9 @@
@endforeach @endforeach
</div> </div>
@if( $metager->showQuicktips() ) @if( $metager->showQuicktips() )
<div id="quicktips"></div> <div id="quicktips">
@include('quicktips', ['quicktips', $quicktips])
</div>
@endif @endif
</div> </div>
</div> </div>
{{ die(var_dump($quicktip)) }}
@if (sizeof($quicktip->details) > 0) @if (sizeof($quicktip->details) > 0)
<details> <details>
<summary class="quicktip-summary"> <summary class="quicktip-summary">
...@@ -9,7 +10,6 @@ ...@@ -9,7 +10,6 @@
@endif @endif
</h1> </h1>
<p>{{ $quicktip->descr }}</p> <p>{{ $quicktip->descr }}</p>
@if
</summary> </summary>
@foreach ($quicktip->details as $detail) @foreach ($quicktip->details as $detail)
<div class="quicktip-detail"> <div class="quicktip-detail">
...@@ -33,6 +33,5 @@ ...@@ -33,6 +33,5 @@
{{ $quicktip->title }} {{ $quicktip->title }}
@endif @endif
</h1> </h1>
@if
</summary> </summary>
@endif @endif
\ No newline at end of file
@foreach($quicktips as $quicktip) @foreach($quicktips as $quicktip)
<div class="quicktip" type="{{ $quicktip->type }}"> <div class="quicktip" type="{{ $quicktip->type }}">
@include('layouts.quicktip', ['quicktip' => $quicktip]) @include('parts.quicktip', ['quicktip' => $quicktip])
</div> </div>
@endforeach @endforeach
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment