Skip to content

GitLab

Commit 27111d7e authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge remote-tracking branch 'origin/master' into 134-links-in-sitesearch-mit-klickbarer-url

parents d2eee28d ee9311fd
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 27111d7e
......@@ -17,22 +17,59 @@ update(144.76.113.134):
- git clone "$path" MetaGer_neu
- cd MetaGer_neu
- composer install
- scp metager@metager3.de:~/.env .
- scp metager@metager3.de:~/sumas.xml config/
- scp metager@metager3.de:~/sumasEn.xml config/
- scp metager@metager3.de:~/blacklistUrl.txt config/
- scp metager@metager3.de:~/blacklistDomains.txt config/
- scp -P 63824 metager@metager3.de:~/.env .
- scp -P 63824 metager@metager3.de:~/sumas.xml config/
- scp -P 63824 metager@metager3.de:~/sumasEn.xml config/
- scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- php ~/MetaGer/artisan down
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- rm -rf MetaGer
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan queue:restart
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
update(metager2):
tags:
- metager2
only:
- master
before_script:
# Abhängigkeiten überprüfen
- which composer
- which git
- which php7.0
- which sqlite3
script:
# Pfad zum neu geklonten Repo
- path=`pwd`
- cd ~/
- if [ -d MetaGer_neu ]; then rm -rf MetaGer_neu;fi
- git clone "$path" MetaGer_neu
- cd MetaGer_neu
- composer install
- scp -P 63824 metager@metager3.de:~/.env .
- scp -P 63824 metager@metager3.de:~/sumas.xml config/
- scp -P 63824 metager@metager3.de:~/sumasEn.xml config/
- scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
update(metager3.de):
......@@ -54,20 +91,20 @@ update(metager3.de):
- git clone "$path" MetaGer_neu
- cd MetaGer_neu
- composer install
- scp metager@metager3.de:~/.env .
- scp metager@metager3.de:~/sumas.xml config/
- scp metager@metager3.de:~/sumasEn.xml config/
- scp metager@metager3.de:~/blacklistUrl.txt config/
- scp metager@metager3.de:~/blacklistDomains.txt config/
- scp -P 63824 metager@metager3.de:~/.env .
- scp -P 63824 metager@metager3.de:~/sumas.xml config/
- scp -P 63824 metager@metager3.de:~/sumasEn.xml config/
- scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- php ~/MetaGer/artisan down
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- rm -rf MetaGer
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan queue:restart
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
\ No newline at end of file
app/Http/Controllers/AdminInterface.php
View file @ 27111d7e
......@@ -57,9 +57,11 @@ class AdminInterface extends Controller
$yesterday = 0;
$rekordTag = 0;
$rekordTagDate = "";
$size = 0;
$count = 0;
for($i = 1; $i <= 28; $i ++ )
{
$logDate = "/var/log/metager/archive/" . "mg3.log.$i";
$logDate = "/var/log/metager/archive/mg3.log.$i";
if( file_exists($logDate) )
{
$sameTime = exec("grep -n '" . date('H') . ":" . date('i') . ":' $logDate | tail -1 | cut -f1 -d':'");
......@@ -72,69 +74,22 @@ class AdminInterface extends Controller
}
$oldLogs[$i]['sameTime'] = $sameTime;
$oldLogs[$i]['insgesamt'] = $insgesamt;
}
}
$median = [];
# Median für 7 Tage:
$size = 0;
$count = 0;
for($i = 1; $i <= 7; $i++)
{
if( isset($oldLogs[$i]) )
{
$count += $oldLogs[$i]['insgesamt'];
# Nun noch den median:
$count += $insgesamt;
$size++;
if($size > 0)
$oldLogs[$i]['median'] = ($count/$size);
}
}
$median[7] = ($count/$size);
# Median für 14 Tage:
$size = 0;
$count = 0;
for($i = 1; $i <= 14; $i++)
{
if( isset($oldLogs[$i]) )
{
$count += $oldLogs[$i]['insgesamt'];
$size++;
}
}
$median[14] = ($count/$size);
# Median für 21 Tage:
$size = 0;
$count = 0;
for($i = 1; $i <= 21; $i++)
{
if( isset($oldLogs[$i]) )
{
$count += $oldLogs[$i]['insgesamt'];
$size++;
}
}
$median[21] = ($count/$size);
# Median für 28 Tage:
$size = 0;
$count = 0;
for($i = 1; $i <= 28; $i++)
{
if( isset($oldLogs[$i]) )
{
$count += $oldLogs[$i]['insgesamt'];
$size++;
}
}
$median[28] = ($count/$size);
return view('admin.count')
->with('title', 'Suchanfragen - MetaGer')
->with('today', number_format(floatval(sizeof($logToday)), 0, ",", "."))
->with('oldLogs', $oldLogs)
->with('rekordCount', number_format(floatval($rekordTag), 0, ",", "."))
->with('rekordTagSameTime', number_format(floatval($rekordTagSameTime), 0, ",", "."))
->with('rekordDate', $rekordTagDate)
->with('median', $median);
->with('rekordDate', $rekordTagDate);
}
public function check ()
{
......
app/Http/Controllers/MailController.php
View file @ 27111d7e
......@@ -32,12 +32,12 @@ class MailController extends Controller
}else{
# Wir versenden die Mail des Benutzers an uns:
$message = $request->input('message');
if( Mail::send(['text' => 'kontakt.mail'], ['messageText'=>$message], function($message) use($replyTo){
$subject = "[Ticket " . date("Y") . date("d") . date("m") . date("H") . date("i") . date("s") . "] MetaGer - Kontaktanfrage";
if( Mail::send(['text' => 'kontakt.mail'], ['messageText'=>$message], function($message) use($replyTo, $subject){
$message->to("office@suma-ev.de", $name = null);
$message->from($replyTo, $name = null);
$message->replyTo($replyTo, $name = null);
$message->subject("MetaGer - Kontaktanfrage");
$message->subject($subject);
}) ){
# Mail erfolgreich gesendet
$messageType = "success";
......@@ -76,11 +76,11 @@ class MailController extends Controller
$messageToUser = "Sie haben eins der folgenden Felder nicht ausgefüllt: IBAN, BIC, Nachricht. Bitte korrigieren Sie Ihre Eingabe und versuchen es erneut.\n";
$messageType = "error";
}else{
$message = "\r\n Name:" . $request->input('Name', 'Keine Angabe');
$message .= "\r\n Telefon:" . $request->input('Telefon', 'Keine Angabe');
$message .= "\r\n Kontonummer:" . $request->input('Kontonummer');
$message .= "\r\n Bankleitzahl:" . $request->input('Bankleitzahl');
$message .= "\r\n Nachricht:" . $request->input('Nachricht');
$message = "\r\nName: " . $request->input('Name', 'Keine Angabe');
$message .= "\r\nTelefon: " . $request->input('Telefon', 'Keine Angabe');
$message .= "\r\nKontonummer: " . $request->input('Kontonummer');
$message .= "\r\nBankleitzahl: " . $request->input('Bankleitzahl');
$message .= "\r\nNachricht: " . $request->input('Nachricht');
$replyTo = $request->input('email', 'anonymous-user@metager.de');
if (!filter_var($replyTo, FILTER_VALIDATE_EMAIL)) {
......@@ -107,7 +107,7 @@ class MailController extends Controller
}
return view('spende')
return view('spende.spende')
->with('title', 'Kontakt')
->with('css', 'donation.css')
->with($messageType,$messageToUser);
......
app/Http/Controllers/MetaGerSearch.php
View file @ 27111d7e
......@@ -32,8 +32,7 @@ class MetaGerSearch extends Controller
# Ergebnisse der Suchmaschinen kombinieren:
$metager->combineResults();
$metager->removeInvalids();
# Die Ausgabe erstellen:
return $metager->createView();
}
......
app/Http/Controllers/Pictureproxy.php
View file @ 27111d7e
......@@ -17,19 +17,24 @@ class Pictureproxy extends Controller
function get(Request $request) {
if( $request->has('url') )
{
$file = file_get_contents($request->input('url'));
$responseCode = explode(" ", $http_response_header[0])[1];
$contentType = "";
foreach($http_response_header as $header)
{
if( strpos($header, "Content-Type:") === 0)
try{
$file = file_get_contents($request->input('url'));
$responseCode = explode(" ", $http_response_header[0])[1];
$contentType = "";
foreach($http_response_header as $header)
{
$tmp = explode(": ", $header);
$contentType = $tmp[1];
if( strpos($header, "Content-Type:") === 0)
{
$tmp = explode(": ", $header);
$contentType = $tmp[1];
}
}
$response = Response::make($file, $responseCode);
$response->header('Content-Type', $contentType);
}catch(\ErrorException $e)
{
$response = Response::make("", 404);
}
$response = Response::make($file, $responseCode);
$response->header('Content-Type', $contentType);
return $response;
}
}
......
app/Http/Controllers/StartpageController.php
View file @ 27111d7e
......@@ -110,7 +110,7 @@ class StartpageController extends Controller
{
$sumaFile = "";
if(App::isLocale('en'))
$sumaFile = config_path() . "/sumasEn.xml";
$sumaFile = config_path() . "/sumas.xml";
else
$sumaFile = config_path() . "/sumas.xml";
......
app/Http/routes.php
View file @ 27111d7e
......@@ -72,14 +72,34 @@
});
Route::get('spende', function()
{
return view('spende')
return view('spende.spende')
->with('title', trans('titles.spende'))
->with('css', 'donation.css')
->with('navbarFocus', 'foerdern');
});
Route::get('beitritt', function ()
{
return view('spende.beitritt')
->with('title', trans('titles.beitritt'))
->with('css', 'beitritt.css')
->with('navbarFocus', 'foerdern');
});
Route::get('bform1.htm', function()
{
return redirect('beitritt');
});
Route::get('spendenaufruf', function()
{
return view('spende.spendenaufruf')
->with('title', 'Spendenaufruf - MetaGer')
->with('navbarFocus', 'foerdern');
});
Route::post('spende', 'MailController@donation');
Route::get('datenschutz', function()
{
return view('datenschutz')
......@@ -128,4 +148,8 @@
Route::get('qt', 'MetaGerSearch@quicktips');
Route::get('tips', 'MetaGerSearch@tips');
Route::get('opensearch.xml', 'StartpageController@loadPlugin');
Route::get('owi', function()
{
return redirect('https://metager.de/klassik/en/owi/');
});
});
app/Jobs/Search.php
View file @ 27111d7e
......@@ -83,7 +83,7 @@ class Search extends Job implements ShouldQueue
break;
}
if( sizeof(($tmp = explode(": ", $data))) === 2 )
$headers[trim($tmp[0])] = trim($tmp[1]);
$headers[strtolower(trim($tmp[0]))] = trim($tmp[1]);
$c++;
}
while (true);
......@@ -91,13 +91,13 @@ class Search extends Job implements ShouldQueue
// end of headers
if(sizeof($headers) > 1){
$bodySize = 0;
if( isset($headers["Transfer-Encoding"]) && $headers["Transfer-Encoding"] === "chunked" )
if( isset($headers["transfer-encoding"]) && $headers["transfer-encoding"] === "chunked" )
{
$body = $this->readChunked();
}elseif( isset($headers['Content-Length']) )
}elseif( isset($headers['content-length']) )
{
$length = trim($headers['Content-Length']);
$length = trim($headers['content-length']);
if(is_numeric($length) && $length >= 1)
$body = $this->readBody($length);
$bodySize = strlen($body);
......@@ -111,7 +111,7 @@ class Search extends Job implements ShouldQueue
}
Redis::del($this->host . "." . $this->socketNumber);
if( isset($headers["Content-Encoding"]) && $headers['Content-Encoding'] === "gzip")
if( isset($headers["content-encoding"]) && $headers['content-encoding'] === "gzip")
{
$body = $this->gunzip($body);
}
......
app/MetaGer.php
View file @ 27111d7e
......@@ -29,6 +29,7 @@ class MetaGer
protected $hostBlacklist = [];
protected $domainBlacklist = [];
protected $stopWords = [];
protected $phrases = [];
protected $engines = [];
protected $results = [];
protected $ads = [];
......@@ -152,7 +153,7 @@ class MetaGer
private function createLogs()
{
$redis = Redis::connection('redisLogs');
if( $redis )
try
{
$logEntry = "";
$logEntry .= "[" . date(DATE_RFC822, mktime(date("H"),date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
......@@ -177,6 +178,9 @@ class MetaGer
$logEntry .= " iter= mm= time=" . round((microtime(true)-$this->starttime), 2) . " serv=" . $this->fokus . " which= hits= stringSearch= QuickTips= SSS= check=";
$logEntry .= " search=" . $this->eingabe;
$redis->rpush('logs.search', $logEntry);
}catch( \Exception $e)
{
return;
}
}
......@@ -267,6 +271,7 @@ class MetaGer
$this->results = $paginatedSearchResults;
$this->validated = false;
if( isset($this->password) )
{
# Wir bieten einen bezahlten API-Zugriff an, bei dem dementsprechend die Werbung ausgeblendet wurde:
......@@ -277,6 +282,7 @@ class MetaGer
if( $this->password === $password )
{
$this->ads = [];
$this->validated = true;
}
}
}
......@@ -388,7 +394,7 @@ class MetaGer
}
if( $enginesWithSite === 0 )
{
$this->errors[] = "Sie wollten eine Sitesearch auf " . $this->site . " durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=\"" . $this->generateSearchLink("web", false) . "\">hier</a> die Sitesearch im Web-FoKus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.";
$this->errors[] = "Sie wollten eine Sitesearch auf " . $this->site . " durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=\"" . $this->generateSearchLink("web", false) . "\">hier</a> die Sitesearch im Web-Fokus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.";
$siteSearchFailed = true;
}else
{
......@@ -505,7 +511,7 @@ class MetaGer
# SUMA-FILE
if(App::isLocale("en")){
$this->sumaFile = config_path() . "/sumasEn.xml";
$this->sumaFile = config_path() . "/sumas.xml";
}else{
$this->sumaFile = config_path() . "/sumas.xml";
}
......@@ -693,9 +699,19 @@ class MetaGer
}
# Meldung über eine Phrasensuche
if(preg_match("/\"(.+)\"/si", $this->q, $match)){
$this->warnings[] = "Sie führen eine Phrasensuche durch: \"" . $match[1] . "\"";
$p = "";
$tmp = $this->q;
while(preg_match("/(.*)\"(.+)\"(.*)/si", $tmp, $match)){
$tmp = $match[1] . $match[3];
$this->phrases[] = strtolower($match[2]);
}
foreach($this->phrases as $phrase)
{
$p .= "\"$phrase\", ";
}
$p = rtrim($p, ", ");
if(sizeof($this->phrases) > 0)
$this->warnings[] = "Sie führen eine Phrasensuche durch: $p";
}
public function getFokus ()
......@@ -747,6 +763,11 @@ class MetaGer
return $this->category;
}
public function getPhrases ()
{
return $this->phrases;
}
public function getSumaFile ()
{
return $this->sumaFile;
......@@ -824,6 +845,8 @@ class MetaGer
$requestData['focus'] = $fokus;
if($results)
$requestData['out'] = "results";
else
$requestData['out'] = "";
$link = action('MetaGerSearch@search', $requestData);
return $link;
}
......
app/Models/Forwarder.php deleted 100644 → 0
View file @ d2eee28d
<?php
namespace App\MetaGer;
use App\lib\Redis;
class Forwarder
{
public static function getFreeServer()
{
# Variablen Deklarationen:
$host = $_SERVER["SERVER_NAME"]; # Dieser Server ist der Host für den Redis-Server
$redis; # In diesem Objekt erhalten wir Zugriff auf die Redis-Datenbank (Redis)
$cfg; # In diesem Objekt erhalten wir Zugriff auf unsere Konfiguration (Config::Simple)
$servers; # Beinhaltet später eine Liste, aller verbundenen Server, an die wir die Anfrage schicken können
$server;
$protocol; # Beinhaltet später den Server ( und sein Protokoll ), der die Suchanfrage tatsächlich beantworten soll
$result;
# Load Config File;
$cfg = parse_ini_file(config_path() . '/metager.ini', TRUE);
# Connect to our Redis Server
try {
$redis = new Redis($host, $cfg['redis']['port']);
} catch(\Exception $e){
$redis = new Redis($cfg['redis']['server'], $cfg['redis']['port']);
}
$response = $redis->cmd('auth', $cfg['redis']['password'] )->get();
if($response !== "OK"){
die("Couldn't authenticate to Redis Server");
}
# Fill $servers Array from Redis:
$serversArray = self::indexedToAssociative($redis->cmd('hgetall', 'servers')->get());
foreach($serversArray as $key => $value ){
$data = explode("\t", $value);
$age = time() - $data[0]; # Erste Stelle ist die Zeit in Sekunden, an der sich der Server das letzte Mal angemeldet hat.
if($age > 3){
# Der Server hat sich seit 3 Sekunden nicht mehr gemeldet. Er kommt schon einmal nicht in Frage und wird aus der Datenbank geworfen:
$redis->cmd("hdel", "servers", $key)->set();
}elseif()
}
return time();
# Select a matching Server:
}
/**
* Converts an indexed Array to an Associative one
* Every Element on an even index is the Ḱey to the Element on the next uneven index
*
* @param indexedArray $array
* @return associativeArray $result
*/
private static function indexedToAssociative($array){
$result = array();
for($i = 0; $i < sizeof($array); $i++){
if($i %2 === 0){
$server = $array[$i];
}else{
$result[$server] = $array[$i];
$server = "";
}
}
return $result;
}
}
\ No newline at end of file
app/Models/Result.php
View file @ 27111d7e
......@@ -157,30 +157,37 @@ class Result
return false;
}
$text = $this->titel . " " . $this->descr;
if($metager->getLang() !== "all")
# Nun der Eventuelle Sprachfilter
if( $metager->getLang() !== "all" )
{
$result = $metager->getLanguageDetect()->detect($text, 1);
$lang = "";
foreach($result as $key => $value)
{
$lang = $key;
}
$text = $this->titel . " " . $this->descr;
$path = app_path() . "/Models/lang.pl";
$lang = exec("echo '$text' | $path");
if($lang !== "" && $lang !== $metager->getLang())
if( $metager->getLang() !== $lang )
return false;
}
# Wir wenden die Stoppwortsuche an und schmeißen entsprechende Ergebnisse raus:
foreach($metager->getStopWords() as $stopWord)
{
$text = $this->titel . " " . $this->descr;
if(stripos($text, $stopWord) !== false)
{
return false;
}
}
# Die Strinsuche:
$text = strtolower($this->titel) . " " . strtolower($this->descr);
foreach($metager->getPhrases() as $phrase)
{
if(strpos($text, $phrase) === FALSE)
return false;
}
# Abschließend noch 2 Überprüfungen. Einmal den Host filter, der Sicherstellt, dass von jedem Host maximal 3 Links angezeigt werden
# und dann noch den Dublettefilter, der sicher stellt, dass wir nach Möglichkeit keinen Link doppelt in der Ergebnisliste haben
# Diese Überprüfung führen wir unter bestimmten Bedingungen nicht durch:
......@@ -247,4 +254,4 @@ class Result
return "https://proxy.suma-ev.de/cgi-bin/nph-proxy.cgi/en/I0/" . $tmp;
}
}
\ No newline at end of file
}
app/Models/Search.php deleted 100644 → 0
View file @ d2eee28d
<?php
namespace App\MetaGer;
use Illuminate\Http\Request;
use App\MetaGer\Searchengine;
class Search
{
public static function loadSearchEngines(Request $request)
{
# Überprüfe, welche Sumas eingeschaltet sind
$xml = simplexml_load_file(SUMA_FILE);
$enabledSearchengines = [];
$overtureEnabled = FALSE;
if(FOKUS === "angepasst")
{
$sumas = $xml->xpath("suma");
foreach($sumas as $suma)
{
if($request->has($suma["service"])