Commit 2c301cd2 authored by Karl Hasselbring's avatar Karl Hasselbring
Browse files

Merge branch 'development' into 483-style-und-funktionsfehler-nach-merge-beheben

parents a365bf95 3b7bf022
......@@ -4,40 +4,82 @@ namespace App\Http\Controllers;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use Response;
class AdminInterface extends Controller
{
public function index(Request $request)
{
$time = $request->input('time', 60);
// Let's get the stats for this server.
// First we need to check, which Fetcher could be available by parsing the sumas.xml
$names = $this->getSearchEngineNames();
# Zunächst einmal die Redis-Verbindung:
$redis = Redis::connection('redisLogs');
# Dann lesen wir alle Server aus:
$member = $redis->smembers('logs.worker');
$today = strtotime(date(DATE_RFC822, mktime(0, 0, 0, date("m"), date("d"), date("Y"))));
$beginningTime = strtotime(date(DATE_RFC822, mktime(date("H"), date("i") - $time, date("s"), date("m"), date("d"), date("Y")))) - $today;
# Jetzt besorgen wir uns die Daten für jeden Server:
$data = [];
foreach ($member as $mem) {
$tmp = $redis->hgetall('logs.worker.' . $mem);
ksort($tmp, SORT_NUMERIC);
$tmp2 = [];
foreach ($tmp as $el => $value) {
if ($el >= $beginningTime) {
$data[$mem][$el] = $value;
// Now we gonna check which stats we can find
$stati = array();
foreach($names as $name){
$stats = Redis::hgetall($name . ".stats");
if(sizeof($stats) > 0){
$fetcherStatus = Redis::get($name);
$stati[$name]["status"] = $fetcherStatus;
foreach($stats as $pid => $value){
if(strstr($value, ";")){
$value = explode(";", $value);
$connection = json_decode(base64_decode($value[0]), true);
foreach($connection as $key => $val){
if(strstr($key, "_time"))
$stati[$name]["fetcher"][$pid]["connection"][$key] = $val;
}
$stati[$name]["fetcher"][$pid]["poptime"] = $value[1];
}
}
}
}
// So now we can generate Median Times for every Fetcher
$fetcherCount = 0;
foreach($stati as $engineName => $engineStats){
$connection = array();
$poptime = 0;
$fetcherCount += sizeof($engineStats["fetcher"]);
foreach($engineStats["fetcher"] as $pid => $stats){
foreach($stats["connection"] as $key => $value){
if(!isset($connection[$key])){
$connection[$key] = $value;
}else{
$connection[$key] += $value;
}
}
$poptime += floatval($stats["poptime"]);
}
foreach($connection as $key => $value){
$connection[$key] /= sizeof($engineStats["fetcher"]);
}
$poptime /= sizeof($engineStats["fetcher"]);
$stati[$engineName]["median-connection"] = $connection;
$stati[$engineName]["median-poptime"] = $poptime;
}
#$data = [ 5 => "majm", 2 => "mngsn", 7 => "akljsd"];
#arsort($data);
return view('admin.admin')
->with('data', $data)
->with('title', "Admin-Interface-MetaGer")
->with('time', $time);
->with('title', 'Fetcher Status')
->with('stati', $stati)
->with('fetcherCount', $fetcherCount);
$stati = json_encode($stati);
$response = Response::make($stati, 200);
$response->header("Content-Type", "application/json");
return $response;
}
private function getSearchEngineNames(){
$url = config_path() . "/sumas.xml";
$xml = simplexml_load_file($url);
$sumas = $xml->xpath("suma");
$names = array();
foreach($sumas as $suma){
$names[] = $suma["name"]->__toString();
}
return $names;
}
public function count()
......
......@@ -11,6 +11,7 @@ class ImageController extends Controller
{
public function generateImage(Request $request)
{
/*
#Piwik Code
PiwikTracker::$URL = 'http://piwik.metager3.de';
$piwikTracker = new PiwikTracker($idSite = 1);
......@@ -23,7 +24,7 @@ class ImageController extends Controller
// Sendet Tracker request per http
$piwikTracker->doTrackPageView($site);
*/
$path = public_path() . '/img/1px.png';
$fileType = File::type($path);
$response = Response::make(File::get($path), 200);
......
......@@ -11,6 +11,14 @@ class MetaGerSearch extends Controller
{
public function search(Request $request, MetaGer $metager)
{
$focus = $request->input("focus", "web");
if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
$metager->parseFormData($request);
if ($metager->doBotProtection($request->input('bot', ""))) {
return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), url("/noaccess", ['redirect' => base64_encode(url()->full())])));
}
return $metager->createView();
}
#die($request->header('User-Agent'));
$time = microtime();
# Mit gelieferte Formulardaten parsen und abspeichern:
......@@ -161,10 +169,15 @@ class MetaGerSearch extends Controller
}
}
/*
# Wikipedia Quicktip
$url = "https://" . APP::getLocale() . ".wikipedia.org/w/api.php?action=opensearch&search=" . urlencode($q) . "&limit=10&namespace=0&format=json&redirects=resolve";
$decodedResponse = json_decode($this->get($url), true);
try{
$content = $this->get($url);
}catch(\ErrorException $e){
$content = "";
}
$decodedResponse = json_decode($content, true);
if (isset($decodedResponse[1][0]) && isset($decodedResponse[2][0]) && isset($decodedResponse[3][0])) {
$quicktip = [];
$firstSummary = $decodedResponse[2][0];
......@@ -190,7 +203,7 @@ class MetaGerSearch extends Controller
$quicktips[] = $quicktip;
}
$mquicktips = array_merge($mquicktips, $quicktips);
*/
if (APP::getLocale() === "de") {
# Dict.cc Quicktip
if (count(explode(' ', $q)) < 3) {
......@@ -253,6 +266,13 @@ class MetaGerSearch extends Controller
public function get($url)
{
return file_get_contents($url);
$ctx = stream_context_create(array('http'=>array('timeout' => 2,)));
return file_get_contents($url, false, $ctx);
}
private function startsWith($haystack, $needle)
{
$length = strlen($needle);
return (substr($haystack, 0, $length) === $needle);
}
}
......@@ -45,14 +45,13 @@ class StartpageController extends Controller
->with('theme', $theme)
->with('autocomplete', $request->input('param_autocomplete', 'on'))
->with('foki', $this->loadFoki())
->with('focus', $request->input('focus', 'web'))
->with('lang', $request->input('param_lang', 'all'))
->with('resultCount', $request->input('param_resultCount', '20'))
->with('time', $request->input('param_time', '1000'))
->with('sprueche', $request->input('param_sprueche', 'off'))
->with('time', $request->input('param_time', '1500'))
->with('sprueche', $request->input('param_sprueche', 'on'))
->with('newtab', $request->input('param_newtab', 'on'))
->with('maps', $maps = $request->input('param_maps', 'on'));
->with('maps', $maps = $request->input('param_maps', 'off'));
}
public function loadPage($subpage)
......
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Redis;
class Search implements ShouldQueue
{
use InteractsWithQueue, Queueable, SerializesModels;
protected $hash, $host, $port, $name, $getString, $useragent, $fp, $additionalHeaders;
protected $buffer_length = 8192;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($hash, $host, $port, $name, $getString, $useragent, $additionalHeaders)
{
$this->hash = $hash;
$this->host = $host;
$this->port = $port;
$this->name = $name;
$this->getString = $getString;
$this->useragent = $useragent;
$this->additionalHeaders = $additionalHeaders;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$this->fp = $this->getFreeSocket();
if ($this->fp) {
if ($this->writeRequest()) {
$this->readAnswer();
}
}
}
private function readAnswer()
{
$time = microtime(true);
$headers = '';
$body = '';
$length = 0;
if (!$this->fp) {
return;
}
// get headers FIRST
$c = 0;
stream_set_blocking($this->fp, 1);
do {
// use fgets() not fread(), fgets stops reading at first newline
// or buffer which ever one is reached first
$data = fgets($this->fp, 8192);
// a sincle CRLF indicates end of headers
if ($data === false || $data == "\r\n" || feof($this->fp)) {
// break BEFORE OUTPUT
break;
}
if (sizeof(($tmp = explode(": ", $data))) === 2) {
$headers[strtolower(trim($tmp[0]))] = trim($tmp[1]);
}
$c++;
} while (true);
// end of headers
if (sizeof($headers) > 1) {
$bodySize = 0;
if (isset($headers["transfer-encoding"]) && $headers["transfer-encoding"] === "chunked") {
$body = $this->readChunked();
} elseif (isset($headers['content-length'])) {
$length = trim($headers['content-length']);
if (is_numeric($length) && $length >= 1) {
$body = $this->readBody($length);
}
$bodySize = strlen($body);
} elseif (isset($headers["connection"]) && strtolower($headers["connection"]) === "close") {
$body = $this->readUntilClose();
}else {
exit;
}
} else {
return;
}
Redis::del($this->host . "." . $this->socketNumber);
if (isset($headers["content-encoding"]) && $headers['content-encoding'] === "gzip") {
$body = $this->gunzip($body);
}
Redis::hset('search.' . $this->hash, $this->name, $body);
Redis::expire('search.' . $this->hash, 5);
}
private function readUntilClose()
{
$data = '';
stream_set_blocking($this->fp, 1);
while (!feof($this->fp)) {
$data .= fgets($this->fp, 8192);
}
# Bei dieser Funktion unterstützt der Host kein Keep-Alive:
# Wir beenden die Verbindung:
fclose($this->fp);
Redis::del($this->host . "." . $this->socketNumber);
return $data;
}
private function readBody($length)
{
$theData = '';
$done = false;
stream_set_blocking($this->fp, 0);
$startTime = time();
$lastTime = $startTime;
while (!feof($this->fp) && !$done && (($startTime + 1) > time()) && $length !== 0) {
usleep(100);
$theNewData = fgets($this->fp, 8192);
$theData .= $theNewData;
$length -= strlen($theNewData);
$done = (trim($theNewData) === '0');
}
return $theData;
}
private function readChunked()
{
$body = '';
// read from chunked stream
// loop though the stream
do {
// NOTE: for chunked encoding to work properly make sure
// there is NOTHING (besides newlines) before the first hexlength
// get the line which has the length of this chunk (use fgets here)
$line = fgets($this->fp, 8192);
// if it's only a newline this normally means it's read
// the total amount of data requested minus the newline
// continue to next loop to make sure we're done
if ($line == "\r\n") {
continue;
}
// the length of the block is sent in hex decode it then loop through
// that much data get the length
// NOTE: hexdec() ignores all non hexadecimal chars it finds
$length = hexdec($line);
if (!is_int($length)) {
trigger_error('Most likely not chunked encoding', E_USER_ERROR);
}
// zero is sent when at the end of the chunks
// or the end of the stream or error
if ($line === false || $length < 1 || feof($this->fp)) {
if ($length <= 0) {
fgets($this->fp, 8192);
}
// break out of the streams loop
break;
}
// loop though the chunk
do {
// read $length amount of data
// (use fread here)
$data = fread($this->fp, $length);
// remove the amount received from the total length on the next loop
// it'll attempt to read that much less data
$length -= strlen($data);
// PRINT out directly
// you could also save it directly to a file here
// store in string for later use
$body .= $data;
// zero or less or end of connection break
if ($length <= 0 || feof($this->fp)) {
// break out of the chunk loop
if ($length <= 0) {
fgets($this->fp, 8192);
}
break;
}
} while (true);
// end of chunk loop
} while (true);
// end of stream loop
return $body;
}
private function gunzip($zipped)
{
$offset = 0;
if (substr($zipped, 0, 2) == "\x1f\x8b") {
$offset = 2;
}
if (substr($zipped, $offset, 1) == "\x08") {
try
{
return gzinflate(substr($zipped, $offset + 8));
} catch (\Exception $e) {
abort(500, "Fehler beim unzip des Ergebnisses von folgendem Anbieter: " . $this->name);
}
}
return "Unknown Format";
}
private function writeRequest()
{
$out = "GET " . $this->getString . " HTTP/1.1\r\n";
$out .= "Host: " . $this->host . "\r\n";
$out .= "User-Agent: " . $this->useragent . "\r\n";
$out .= "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n";
$out .= "Accept-Language: de,en-US;q=0.7,en;q=0.3\r\n";
$out .= "Accept-Encoding: gzip, deflate, br\r\n";
$out .= str_replace("$#!#$", "\r\n", $this->additionalHeaders);
$out .= "Connection: keep-alive\r\n\r\n";
# Anfrage senden:
$sent = 0;
$string = $out;
$time = microtime(true);
while (true) {
$timeElapsed = microtime(true) - $time;
if ($timeElapsed > 1.0) {
# Irgendwas ist mit unserem Socket passiert. Wir brauchen einen neuen:
if ($this->fp && is_resource($this->fp)) {
fclose($this->fp);
}
Redis::del($this->name . "." . $this->socketNumber);
$this->fp = $this->getFreeSocket();
$sent = 0;
$string = $out;
continue;
}
try {
$tmp = fwrite($this->fp, $string);
} catch (\ErrorException $e) {
# Irgendwas ist mit unserem Socket passiert. Wir brauchen einen neuen:
if ($this->fp && is_resource($this->fp)) {
fclose($this->fp);
}
Redis::del($this->name . "." . $this->socketNumber);
$this->fp = $this->getFreeSocket();
$sent = 0;
$string = $out;
continue;
}
if ($tmp) {
$sent += $tmp;
$string = substr($string, $tmp);
}
if ($sent >= strlen($out)) {
break;
}
}
if ($sent === strlen($out)) {
return true;
}
return false;
}
public function getFreeSocket()
{
# Je nach Auslastung des Servers ( gleichzeitige Abfragen ), kann es sein, dass wir mehrere Sockets benötigen um die Abfragen ohne Wartezeit beantworten zu können.
# pfsockopen öffnet dabei einen persistenten Socket, der also auch zwischen den verschiedenen php Prozessen geteilt werden kann.
# Wenn der Hostname mit einem bereits erstellten Socket übereinstimmt, wird die Verbindung also aufgegriffen und fortgeführt.
# Allerdings dürfen wir diesen nur verwenden, wenn er nicht bereits von einem anderen Prozess zur Kommunikation verwendet wird.
# Wenn dem so ist, probieren wir den nächsten Socket zu verwenden.
# Dies festzustellen ist komplizierter, als man sich das vorstellt. Folgendes System sollte funktionieren:
# 1. Stelle fest, ob dieser Socket neu erstellt wurde, oder ob ein existierender geöffnet wurde.
$counter = 0;
$fp = null;
$time = microtime(true);
do {
if (intval(Redis::exists($this->host . ".$counter")) === 0) {
Redis::set($this->host . ".$counter", 1);
Redis::expire($this->host . ".$counter", 5);
$this->socketNumber = $counter;
try
{
$fp = pfsockopen($this->getHost(), $this->port, $errstr, $errno, 1);
} catch (\ErrorException $e) {
break;
}
# Wir gucken, ob der Lesepuffer leer ist:
stream_set_blocking($fp, 0);
$string = fgets($fp, 8192);
if ($string !== false || feof($fp)) {
if ($this->fp && is_resource($this->fp)) {
fclose($fp);
}
$this->socketNumber = null;
Redis::del($this->host . ".$counter");
continue;
}
break;
}
$counter++;
} while (true);
return $fp;
}
public function getHost()
{
$return = "";
if ($this->port === "443") {
$return .= "tls://";
} else {
$return .= "tcp://";
}
$return .= $this->host;
return $return;
}
}
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Log;
class Searcher implements ShouldQueue
{
use InteractsWithQueue, Queueable, SerializesModels;
protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo;
protected $MAX_REQUESTS = 100;
protected $importantEngines = array("Fastbot", "overture", "overtureAds");
protected $recheck;
/**
* Create a new job instance.
* This is our new Worker/Searcher Class
* It will take it's name from the sumas.xml as constructor argument
* Each Searcher is dedicated to one remote server from our supported Searchengines
* It will listen to a queue in the Redis Database within the handle() method and
* answer requests to this specific search engine.
* The curl handle will be left initialized and opened so that we can make full use of
* keep-alive requests.
* @return void
*/
public function __construct($name)
{
$this->name = $name;
$this->pid = getmypid();
$this->recheck = false;
// Submit this worker to the Redis System
Redis::expire($this->name, 5);
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
// This Searches is freshly called so we need to initialize the curl handle $ch
$this->ch = $this->initCurlHandle();
$this->counter = 0; // Counts the number of answered jobs
$time = microtime(true);
while(true){
// Update the expire
Redis::expire($this->name, 5);
Redis::expire($this->name . ".stats", 5);
// One Searcher can handle a ton of requests to the same server
// Each search to the server of this Searcher will be submitted to a queue
// stored in redis which has the same name as this searchengine appended by a ".queue"