Commit 21e4b6bc authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch '795-scopia-einbauen' into 'development'

Resolve "Scopia einbauen"

Closes #795

See merge request !1305
parents 6b6a1566 7854aa46
......@@ -10,12 +10,12 @@ class FokiLoader
{
$sumaFile = "";
if (App::isLocale('en')) {
$sumaFile = config_path() . "/sumas.xml";
$sumaFile = config_path() . "/sumasEn.xml";
} else {
$sumaFile = config_path() . "/sumas.xml";
}
$xml = simplexml_load_file($sumaFile);
$xml = simplexml_load_file($sumaFile);
$sumas = $xml->xpath("suma");
$foki = [];
......@@ -24,13 +24,13 @@ class FokiLoader
if (isset($suma['type'])) {
$f = explode(",", $suma['type']->__toString());
foreach ($f as $tmp) {
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$foki[$tmp][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
}
} else {
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$foki["andere"][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
}
}
......
......@@ -5,8 +5,8 @@ namespace App\Http\Controllers;
use App;
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
use Response;
use LaravelLocalization;
use Response;
class StartpageController extends Controller
{
......@@ -20,7 +20,7 @@ class StartpageController extends Controller
public function loadStartPage(Request $request)
{
$focusPages = [];
$theme = "default";
$theme = "default";
$optionParams = ['param_sprueche', 'param_newtab', 'param_maps', 'param_autocomplete', 'param_lang', 'param_key'];
$option_values = [];
......@@ -33,12 +33,12 @@ class StartpageController extends Controller
}
$autocomplete = 'on';
if(in_array('autocomplete', array_keys($option_values))) {
if (in_array('autocomplete', array_keys($option_values))) {
$autocomplete = $option_values['autocomplete'];
}
$lang = LaravelLocalization::getCurrentLocale();
if ($lang === 'de') {
if ($lang === 'de' || $lang === "en") {
$lang = 'all';
}
......@@ -78,9 +78,9 @@ class StartpageController extends Controller
public function loadPlugin($params, $locale = "de")
{
$params = unserialize(base64_decode($params));
$params = unserialize(base64_decode($params));
$requests = $params;
$params = [];
$params = [];
foreach ($requests as $key => $value) {
if (strpos($key, "param_") === 0) {
$key = substr($key, strpos($key, "param_") + 6);
......@@ -135,12 +135,12 @@ class StartpageController extends Controller
public function berlin(Request $request)
{
$link = "";
$link = "";
$password = "";
if ($request->filled('eingabe')) {
$password = getenv('berlin');
$password = md5($request->input('eingabe') . " -host:userpage.fu-berlin.de" . $password);
$link = "/meta/meta.ger3?eingabe=" . $request->input('eingabe') . " -host:userpage.fu-berlin.de&focus=web&password=" . $password . "&encoding=utf8&lang=all&site=fu-berlin.de&quicktips=off&out=results-with-style";
$link = "/meta/meta.ger3?eingabe=" . $request->input('eingabe') . " -host:userpage.fu-berlin.de&focus=web&password=" . $password . "&encoding=utf8&lang=all&site=fu-berlin.de&quicktips=off&out=results-with-style";
}
return view('berlin')
->with('title', 'Testseite für die FU-Berlin')
......
This diff is collapsed.
......@@ -34,6 +34,7 @@ class MetaGer
protected $warnings = [];
protected $errors = [];
protected $addedHosts = [];
protected $availableFoki = [];
protected $startCount = 0;
protected $canCache = false;
# Daten über die Abfrage$
......@@ -297,10 +298,6 @@ class MetaGer
}
if (LaravelLocalization::getCurrentLocale() === "en") {
$this->ads = [];
}
if ($this->validated) {
$this->ads = [];
$this->maps = false;
......@@ -525,6 +522,20 @@ class MetaGer
$overtureEnabled = false;
$sumaCount = 0;
/*
* Erstellt eine Liste mit Foki, die verfügbar sind
*/
$this->availableFoki = [];
foreach ($sumas as $suma) {
$foki = explode(",", trim($suma["type"]));
foreach ($foki as $fokus) {
if (!empty($fokus)) {
$this->availableFoki[$fokus] = "available";
}
}
}
$isCustomSearch = $this->startsWith($this->fokus, 'focus_');
# Im Falle einer Custom-Suche ohne mindestens einer selektierter Suchmaschine wird eine Web-Suche durchgeführt
......@@ -646,7 +657,7 @@ class MetaGer
public function sumaIsSelected($suma, $request, $custom)
{
if ($custom) {
if ($request->filled("engine_" . $suma["name"])) {
if ($request->filled("engine_" . strtolower($suma["name"]))) {
return true;
}
} else {
......@@ -950,7 +961,7 @@ class MetaGer
$this->fokus = $request->input('focus', 'web');
# Suma-File
if (App::isLocale("en")) {
$this->sumaFile = config_path() . "/sumas.xml";
$this->sumaFile = config_path() . "/sumasEn.xml";
} else {
$this->sumaFile = config_path() . "/sumas.xml";
}
......@@ -1561,6 +1572,12 @@ class MetaGer
return $this->lang;
}
public function getAvailableFoki()
{
return $this->availableFoki;
}
public function getSprueche()
{
return $this->sprueche;
......
......@@ -15,12 +15,12 @@ abstract class Searchengine
public $ch; # Curl Handle zum erhalten der Ergebnisse
public $getString = ""; # Der String für die Get-Anfrage
public $engine; # Die ursprüngliche Engine XML
public $enabled = true; # true, wenn die Suchmaschine nicht explizit disabled ist
public $results = []; # Die geladenen Ergebnisse
public $ads = []; # Die geladenen Werbungen
public $enabled = true; # true, wenn die Suchmaschine nicht explizit disabled ist
public $results = []; # Die geladenen Ergebnisse
public $ads = []; # Die geladenen Werbungen
public $products = []; # Die geladenen Produkte
public $loaded = false; # wahr, sobald die Ergebnisse geladen wurden
public $cached = false;
public $loaded = false; # wahr, sobald die Ergebnisse geladen wurden
public $cached = false;
public $ip; # Die IP aus der metager
public $uses; # Die Anzahl der Nutzungen dieser Suchmaschine
......@@ -31,10 +31,13 @@ abstract class Searchengine
public $startTime; # Die Zeit der Erstellung dieser Suchmaschine
public $hash; # Der Hash-Wert dieser Suchmaschine
private $user; # Username für HTTP-Auth (falls angegeben)
private $password; # Passwort für HTTP-Auth (falls angegeben)
public $fp; # Wird für Artefakte benötigt
public $socketNumber = null; # Wird für Artefakte benötigt
public $counter = 0; # Wird eventuell für Artefakte benötigt
public $write_time = 0; # Wird eventuell für Artefakte benötigt
public $socketNumber = null; # Wird für Artefakte benötigt
public $counter = 0; # Wird eventuell für Artefakte benötigt
public $write_time = 0; # Wird eventuell für Artefakte benötigt
public $connection_time = 0; # Wird eventuell für Artefakte benötigt
public function __construct(\SimpleXMLElement $engine, MetaGer $metager)
......@@ -69,7 +72,7 @@ abstract class Searchengine
}
$this->useragent = $metager->getUserAgent();
$this->ip = $metager->getIp();
$this->ip = $metager->getIp();
$this->startTime = microtime();
# Suchstring generieren
......@@ -84,10 +87,10 @@ abstract class Searchengine
} else {
$q = $metager->getQ();
}
$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
$this->hash = md5($this->host . $this->getString . $this->port . $this->name);
$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
$this->hash = md5($this->host . $this->getString . $this->port . $this->name);
$this->resultHash = $metager->getHashCode();
$this->canCache = $metager->canCache();
$this->canCache = $metager->canCache();
if (!isset($this->additionalHeaders)) {$this->additionalHeaders = "";}
}
......@@ -114,13 +117,13 @@ abstract class Searchengine
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if($this->port === "443"){
if ($this->port === "443") {
$url = "https://";
}else{
} else {
$url = "http://";
}
$url .= $this->host;
if($this->port !== 80 && $this->port !== 443){
if ($this->port !== 80 && $this->port !== 443) {
$url .= ":" . $this->port;
}
$url .= $this->getString;
......@@ -132,27 +135,27 @@ abstract class Searchengine
Redis::rpush($this->name . ".queue", $mission);
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher = false;
$searcherData = Redis::hgetall($this->name . ".stats");
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if(sizeof($searcherData) === 0){
if (sizeof($searcherData) === 0) {
$needSearcher = true;
}else{
} else {
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
......@@ -160,18 +163,18 @@ abstract class Searchengine
// Let's hardcode a minimum of 100ms between every search job.
// First calculate the median of all Times
$median = 0;
foreach($searcherData as $pid => $data){
foreach ($searcherData as $pid => $data) {
$data = explode(";", $data);
$median += floatval($data[1]);
}
$median /= sizeof($searcherData);
if($median < .1){
if ($median < .1) {
$needSearcher = true;
}
}
if($needSearcher && Redis::get($this->name) !== "locked"){
if ($needSearcher && Redis::get($this->name) !== "locked") {
Redis::set($this->name, "locked");
$this->dispatch(new Searcher($this->name));
$this->dispatch(new Searcher($this->name, $this->user, $this->password));
}
}
}
......@@ -203,7 +206,8 @@ abstract class Searchengine
$this->enabled = true;
}
public function setResultHash($hash){
public function setResultHash($hash)
{
$this->resultHash = $hash;
}
......
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class Fastbot extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
if (strpos($this->urlEncode($metager->getEingabe()), "%") !== false) {
$this->enabled = false;
return null;
}
}
public function loadResults($result)
{
$result = utf8_encode($result);
$counter = 0;
foreach (explode("\n", $result) as $line) {
$line = trim($line);
if (strlen($line) > 0) {
# Hier bekommen wir jedes einzelne Ergebnis
$result = explode("|:|", $line);
if(count($result) < 4) continue;
$link = $result[1];
$link = substr($link, strpos($link, "href=\"") + 6);
$link = substr($link, 0, strpos($link, "\""));
$counter++;
$this->gefVon = "<a href=\"" . $this->homepage . "\" target=\"_blank\" rel=\"noopener\">" . $this->displayName . " " . trans('results.redirect') . "</a>";
$this->results[] = new \App\Models\Result(
$this->engine,
trim(strip_tags($result[1])),
$link,
$result[3],
$result[2],
$this->displayName,$this->homepage,
$counter
);
}
}
}
}
......@@ -25,10 +25,10 @@ class Overture extends Searchengine
$results = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
foreach ($results as $result) {
$title = $result["title"];
$link = $result->{"ClickUrl"}->__toString();
$title = $result["title"];
$link = $result->{"ClickUrl"}->__toString();
$anzeigeLink = $result["siteHost"];
$descr = $result["description"];
$descr = $result["description"];
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
......@@ -36,7 +36,8 @@ class Overture extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->displayName,
$this->homepage,
$this->counter
);
}
......@@ -44,10 +45,10 @@ class Overture extends Searchengine
# Nun noch die Werbeergebnisse:
$ads = $content->xpath('//Results/ResultSet[@id="searchResults"]/Listing');
foreach ($ads as $ad) {
$title = $ad["title"];
$link = $ad->{"ClickUrl"}->__toString();
$title = $ad["title"];
$link = $ad->{"ClickUrl"}->__toString();
$anzeigeLink = $ad["siteHost"];
$descr = $ad["description"];
$descr = $ad["description"];
$this->counter++;
$this->ads[] = new \App\Models\Result(
$this->engine,
......@@ -55,7 +56,7 @@ class Overture extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->displayName, $this->homepage,
$this->counter
);
}
......@@ -87,9 +88,9 @@ class Overture extends Searchengine
// Yahoo liefert, wenn es keine weiteren Ergebnisse hat immer wieder die gleichen Ergebnisse
// Wir müssen also überprüfen, ob wir am Ende der Ergebnisse sind
$resultCount = $content->xpath('//Results/ResultSet[@id="inktomi"]/MetaData/TotalHits');
$results = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
$results = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
if (isset($resultCount[0]) && sizeof($results) > 0) {
$resultCount = intval($resultCount[0]->__toString());
$resultCount = intval($resultCount[0]->__toString());
$lastResultOnPage = intval($results[sizeof($results) - 1]["rank"]);
if ($resultCount <= $lastResultOnPage) {
return;
......@@ -111,9 +112,9 @@ class Overture extends Searchengine
}
# Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
$next = new Overture(simplexml_load_string($this->engine), $metager);
$next = new Overture(simplexml_load_string($this->engine), $metager);
$next->getString = preg_replace("/&Keywords=.*?&/si", "&", $next->getString) . "&" . $nextArgs;
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
}
}
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
class Scopia extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults($result)
{
$result = html_entity_decode($result);
$result = preg_replace("/<description>(.*?)<\/description>/si", "<description><![CDATA[ $1 ]]></description>", $result);
$result = preg_replace("/<title>(.*?)<\/title>/si", "<title><![CDATA[ $1 ]]></title>", $result);
$result = preg_replace("/<url>(.*?)<\/url>/si", "<url><![CDATA[ $1 ]]></url>", $result);
try {
$content = simplexml_load_string($result);
if (!$content) {
return;
}
$results = $content->xpath('//results/result');
foreach ($results as $result) {
$title = $result->title->__toString();
$link = $result->url->__toString();
$anzeigeLink = $link;
$descr = $result->description->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->displayName,
$this->homepage,
$this->counter
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name:");
Log::error($e->getMessage());
return;
}
}
public function getNext(\App\MetaGer $metager, $result)
{
$result = html_entity_decode($result);
$result = str_replace("&", "&amp;", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name:");
Log::error($e->getMessage());
return;
}
if (!$content) {
return;
}
$more = $content->xpath('//results/more')[0]->__toString() === "1" ? true : false;
if ($more) {
$results = $content->xpath('//results/result');
$number = $results[sizeof($results) - 1]->number->__toString();
# Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
$next = new Scopia(simplexml_load_string($this->engine), $metager);
$next->getString = preg_replace("/\\?s=.*?&/si", "?s=" . $number, $next->getString);
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
}
}
}
@if( array_has($metager->getAvailableFoki(), "web"))
<div id="web" @if($metager->getFokus() === "web")class="active"@endif>
<a href="@if($metager->getFokus() === "web")#@else{!!$metager->generateSearchLink('web')!!}@endif" target="_self" tabindex="2">@lang('index.foki.web')</a>
</div>
@endif
@if( array_has($metager->getAvailableFoki(), "nachrichten"))
<div id="nachrichten" @if($metager->getFokus() === "nachrichten")class="active"@endif>
<a href="@if($metager->getFokus() === "nachrichten")#@else{!!$metager->generateSearchLink('nachrichten')!!}@endif" target="_self" tabindex="3">@lang('index.foki.nachrichten')</a>
</div>
@endif
@if( array_has($metager->getAvailableFoki(), "wissenschaft"))
<div id="wissenschaft" @if($metager->getFokus() === "wissenschaft")class="active"@endif>
<a href="@if($metager->getFokus() === "wissenschaft")#@else{!!$metager->generateSearchLink('wissenschaft')!!}@endif" target="_self" tabindex="4">@lang('index.foki.wissenschaft')</a>
</div>
@endif
@if( array_has($metager->getAvailableFoki(), "produktsuche"))
<div id="produkte" @if($metager->getFokus() === "produktsuche")class="active"@endif>
<a href="@if($metager->getFokus() === "produktsuche")#@else{!!$metager->generateSearchLink('produktsuche')!!}@endif" target="_self" tabindex="5">@lang('index.foki.produkte')</a>
</div>
@endif
<div id="maps">
<a href="https://maps.metager.de/map/{{ $metager->getQ() }}/9.7380161,52.37119740000003,12" target="_self" tabindex="6">@lang('index.foki.maps')</a>
</div>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment