Commit 653286c1 authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch 'development' into phil-1010-add-quotes-and-did-you-know-to-metager-patch-52193

parents 6051ac95 28ca0eef
......@@ -33,6 +33,8 @@ stages:
- integrationtest
- cleanup
.auto-deploy:
image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.6"
build:
services:
......@@ -75,6 +77,10 @@ review:
variables:
HELM_UPGRADE_VALUES_FILE: .gitlab/review-apps-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment
environment:
name: review/$CI_COMMIT_REF_NAME
on_stop: stop_review
auto_stop_in: 2 days
rules:
- if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
when: never
......@@ -87,6 +93,11 @@ review:
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
stop_review:
variables:
GIT_STRATEGY: none
environment:
name: review/$CI_COMMIT_REF_NAME
action: stop
rules:
- if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
when: never
......@@ -163,4 +174,4 @@ integrationtest:
- sed -i "s#^APP_URL=.*#APP_URL=$URL#g" .env
- sed -i "s#^BRANCH_NAME=.*#BRANCH_NAME=$CI_COMMIT_REF_NAME#g" .env
- sed -i "s#^COMMIT_NAME=.*#COMMIT_NAME=$CI_COMMIT_REF_SLUG#g" .env
- php artisan dusk
\ No newline at end of file
- php artisan dusk
postgresql:
enabled: false
service:
externalPort: 80
internalPort: 80
......@@ -5,13 +7,26 @@ hpa:
enabled: true
minReplicas: 1
maxReplicas: 5
resources:
limits:
resourcesPhpfpm:
requests:
cpu: 500m
memory: 500M
limits:
resourcesNginx:
requests:
cpu: 100m
memory: 100M
limits:
resourcesRedis:
requests:
cpu: 100m
memory: 1Gi
limits:
resourcesFetcher:
requests:
cpu: 500m
memory: 1Gi
memory: 100M
limits:
podDisruptionBudget:
enabled: true
minAvailable:
......
postgresql:
enabled: false
service:
externalPort: 80
internalPort: 80
......@@ -5,13 +7,6 @@ hpa:
enabled: true
minReplicas: 3
maxReplicas: 100
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
podDisruptionBudget:
enabled: true
minAvailable:
......@@ -21,6 +16,29 @@ podAnnotations:
prometheus.io/path: /metrics
prometheus.io/port: "80"
deploymentApiVersion: apps/v1
resources:
requests:
limits:
resourcesPhpfpm:
requests:
cpu: 500m
memory: 500M
limits:
resourcesNginx:
requests:
cpu: 100m
memory: 100M
limits:
resourcesRedis:
requests:
cpu: 100m
memory: 1Gi
limits:
resourcesFetcher:
requests:
cpu: 500m
memory: 100M
limits:
ingress:
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
......
---
postgresql:
enabled: false
hpa:
enabled: false
resources:
requests:
limits:
ingress:
annotations:
kubernetes.io/tls-acme: "false"
......@@ -21,3 +25,15 @@ service:
externalPort: 80
internalPort: 80
deploymentApiVersion: apps/v1
resourcesPhpfpm:
requests:
limits:
resourcesNginx:
requests:
limits:
resourcesRedis:
requests:
limits:
resourcesFetcher:
requests:
limits:
\ No newline at end of file
......@@ -31,7 +31,6 @@ class AdminInterface extends Controller
if (strstr($key, "_time")) {
$stati[$name]["fetcher"][$pid]["connection"][$key] = $val;
}
}
$stati[$name]["fetcher"][$pid]["poptime"] = $value[1];
}
......@@ -157,7 +156,7 @@ class AdminInterface extends Controller
$now->minute = 0;
$now->second = 0;
while($now->lessThanOrEqualTo(Carbon::now())){
while ($now->lessThanOrEqualTo(Carbon::now())) {
$sameTime += empty($stats->time->{$now->format('H:i')}->{$interface}) ? 0 : $stats->time->{$now->format('H:i')}->{$interface};
$now->addMinutes(5);
}
......@@ -178,19 +177,38 @@ class AdminInterface extends Controller
if ($size > 0) {
$oldLogs[$key]['median'] = number_format(floatval(round($count / $size)), 0, ",", ".");
}
}
$sameTimes = [];
$sum = 0;
foreach ($oldLogs as $index => $oldLog) {
if ($index % 7 === 0) {
$sameTime = $oldLog["sameTime"];
$sameTime = str_replace(".", "", $sameTime);
$sameTime = \intval($sameTime);
$sameTimes[] = ($logToday - $sameTime);
$sum += ($logToday - $sameTime);
}
}
$averageIncrease = 0;
if (sizeof($sameTimes) > 0) {
$averageIncrease = $sum / sizeof($sameTimes);
}
if ($request->input('out', 'web') === "web") {
return view('admin.count')
->with('title', 'Suchanfragen - MetaGer')
->with('today', number_format(floatval($logToday), 0, ",", "."))
->with('averageIncrease', $averageIncrease)
->with('oldLogs', $oldLogs)
->with('minCount', $minCount)
->with('rekordCount', number_format(floatval($rekordTag), 0, ",", "."))
->with('rekordTagSameTime', number_format(floatval($rekordTagSameTime), 0, ",", "."))
->with('rekordDate', $rekordTagDate)
->with('days', $days);
->with('days', $days)
->with('css', [mix('/css/count/style.css')])
->with('darkcss', [mix('/css/count/dark.css')]);
} else {
$result = "";
foreach ($oldLogs as $key => $value) {
......@@ -203,9 +221,7 @@ class AdminInterface extends Controller
return response($result, 200)
->header('Content-Type', 'text/csv')
->header('Content-Disposition', 'attachment; filename="count.csv"');
}
}
public function countGraphToday()
......@@ -235,7 +251,6 @@ class AdminInterface extends Controller
return response()
->view('admin.countGraphToday', ["data" => $result], 200)
->header('Content-Type', "image/svg+xml");
}
public function engineStats()
......
......@@ -93,6 +93,12 @@ class MetaGerSearch extends Controller
$metager->startSearch($timings);
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
if (!empty($timings)) {
$timings["Loaded Quicktip Results"] = microtime(true) - $time;
}
$metager->waitForMainResults();
if (!empty($timings)) {
$timings["waitForMainResults"] = microtime(true) - $time;
......@@ -245,11 +251,11 @@ class MetaGerSearch extends Controller
$newResults = 0;
foreach ($metager->getResults() as $index => $resultTmp) {
if ($resultTmp->new || $resultTmp->adgoalChanged) {
if ($resultTmp->new || $resultTmp->changed) {
if ($metager->getFokus() !== "bilder") {
$view = View::make('layouts.result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
if (!$resultTmp->new && $resultTmp->adgoalChanged) {
if (!$resultTmp->new && $resultTmp->changed) {
$result['changedResults'][$index] = $html;
} else {
$result['newResults'][$index] = $html;
......@@ -258,7 +264,7 @@ class MetaGerSearch extends Controller
} else {
$view = View::make('layouts.image_result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
if (!$resultTmp->new && $resultTmp->adgoalChanged) {
if (!$resultTmp->new && $resultTmp->changed) {
$result['changedResults'][$index] = $html;
} else {
$result['newResults'][$index] = $html;
......
......@@ -56,7 +56,7 @@ class SettingsController extends Controller
$cookies = Cookie::get();
$settingActive = false;
foreach ($cookies as $key => $value) {
if (\starts_with($key, [$fokus . "_engine_", $fokus . "_setting_"]) || strpos($key, $fokus . '_blpage') === 0) {
if (\starts_with($key, [$fokus . "_engine_", $fokus . "_setting_"]) || strpos($key, $fokus . '_blpage') === 0 || $key === 'dark_mode') {
$settingActive = true;
}
}
......@@ -209,23 +209,32 @@ class SettingsController extends Controller
$fokus = $request->input('fokus', '');
$url = $request->input('url', '');
// Currently only the setting for quotes is supported
$quotes = $request->input('zitate', '');
if (empty($fokus) || empty($quotes)) {
abort(404);
if(!empty($quotes)){
if($quotes === "off"){
$path = \Request::path();
$cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
Cookie::queue($fokus . "_setting_zitate", "off", 0, $cookiePath, null, false, false);
}elseif($quotes === "on") {
$path = \Request::path();
$cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
Cookie::queue($fokus . "_setting_zitate", "", 0, $cookiePath, null, false, false);
}
}
if($quotes === "off"){
$path = \Request::path();
$cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
Cookie::queue($fokus . "_setting_zitate", "off", 0, $cookiePath, null, false, false);
}elseif($quotes === "on") {
$path = \Request::path();
$cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
Cookie::queue($fokus . "_setting_zitate", "", 0, $cookiePath, null, false, false);
}else{
abort(404);
$darkmode = $request->input('dm');
if(!empty($darkmode)){
if($darkmode === "off"){
Cookie::queue('dark_mode', '1', 0, '/', null, false, false);
}elseif($darkmode === "on") {
Cookie::queue('dark_mode', '2', 0, '/', null, false, false);
}elseif($darkmode === "system"){
Cookie::queue('dark_mode', '', 0, '/', null, false, false);
}
}
return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), route('settings', ["fokus" => $fokus, "url" => $url])));
}
......@@ -244,6 +253,10 @@ class SettingsController extends Controller
$cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
Cookie::queue($key, "", 0, $cookiePath, null, false, false);
}
if($key === 'dark_mode'){
Cookie::queue($key, "", 0, '/', null, false, false);
}
}
$this->clearBlacklist($request);
......@@ -266,8 +279,11 @@ class SettingsController extends Controller
$key = $request->input('key', '');
$path = \Request::path();
$cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
Cookie::queue($key, "", 0, $cookiePath, null, false, false);
if($key === 'dark_mode'){
Cookie::queue($key, "", 0, '/', null, false, false);
}else{
Cookie::queue($key, "", 0, $cookiePath, null, false, false);
}
return redirect($request->input('url', 'https://metager.de'));
}
......@@ -278,7 +294,11 @@ class SettingsController extends Controller
$cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
foreach (Cookie::get() as $key => $value) {
Cookie::queue($key, "", 0, $cookiePath, null, false, false);
if($key === 'dark_mode'){
Cookie::queue($key, "", 0, '/', null, false, false);
}else{
Cookie::queue($key, "", 0, $cookiePath, null, false, false);
}
}
return redirect($request->input('url', 'https://metager.de'));
}
......@@ -384,6 +404,9 @@ class SettingsController extends Controller
if($blpage){
continue;
}
if($key === 'dark_mode'){
Cookie::queue($key, $value, 0, '/', null, false, false);
}
foreach($sumaFile['filter']['parameter-filter'] as $suma => $filter){
if($key === $suma && $value === $filter){
Cookie::queue($key, $value, 0, $cookiePath, null, false, false);
......
......@@ -3,6 +3,7 @@
namespace App\Http\Controllers;
use App;
use Cookie;
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
use LaravelLocalization;
......@@ -52,7 +53,8 @@ class StartpageController extends Controller
->with('request', $request->input('request', 'GET'))
->with('option_values', $option_values)
->with('autocomplete', $autocomplete)
->with('pluginmodal', $request->input('plugin-modal', 'off'));
->with('pluginmodal', $request->input('plugin-modal', 'off'))
->with('darkcss', [mix('css/themes/startpage-only-dark.css')]);
}
public function loadPage($subpage)
......@@ -87,8 +89,9 @@ class StartpageController extends Controller
}
$response = Response::make(
view('plugin')
->with('link', $link)
, "200");
->with('link', $link),
"200"
);
$response->header('Content-Type', "application/opensearchdescription+xml");
return $response;
}
......
......@@ -95,15 +95,13 @@ class ConvertCountFile implements ShouldQueue
fclose($fh);
}
if (!$error) {
$oldUmask = umask(0);
// Write the result to a File
if (!file_exists($this->files["countPath"])) {
mkdir($this->files["countPath"], 0777, true);
}
file_put_contents($this->files["countFile"], json_encode($result, JSON_PRETTY_PRINT));
umask($oldUmask);
$oldUmask = umask(0);
// Write the result to a File
if (!file_exists($this->files["countPath"])) {
mkdir($this->files["countPath"], 0777, true);
}
file_put_contents($this->files["countFile"], json_encode($result, JSON_PRETTY_PRINT));
umask($oldUmask);
Redis::del(md5($this->files["countFile"]));
}
......
......@@ -297,6 +297,11 @@ class MetaGer
if (!empty($timings)) {
$timings["prepareResults"]["validated results"] = microtime(true) - $timings["starttime"];
}
$this->duplicationCheck();
if (!empty($timings)) {
$timings["prepareResults"]["duplications checked"] = microtime(true) - $timings["starttime"];
}
# Validate Advertisements
$newResults = [];
foreach ($this->ads as $ad) {
......@@ -399,6 +404,41 @@ class MetaGer
}
}
public function duplicationCheck()
{
$arr = [];
for ($i = 0; $i < count($this->results); $i++) {
$link = $this->results[$i]->link;
if (strpos($link, "http://") === 0) {
$link = substr($link, 7);
}
if (strpos($link, "https://") === 0) {
$link = substr($link, 8);
}
if (strpos($link, "www.") === 0) {
$link = substr($link, 4);
}
$link = trim($link, "/");
$hash = md5($link);
if (isset($arr[$link])) {
$arr[$link]->gefVon[] = $this->results[$i]->gefVon[0];
$arr[$link]->gefVonLink[] = $this->results[$i]->gefVonLink[0];
array_splice($this->results, $i, 1);
$i--;
if ($arr[$link]->new === true || $this->results[$i]->new === true) {
$arr[$link]->changed = true;
}
} else {
$arr[$link] = &$this->results[$i];
}
}
}
public function startAdgoal(&$results)
{
$publicKey = getenv('adgoal_public');
......@@ -505,7 +545,7 @@ class MetaGer
$newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
$result->link = $newLink;
$result->partnershop = true;
$result->adgoalChanged = true;
$result->changed = true;
}
}
}
......@@ -939,7 +979,7 @@ class MetaGer
$mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
foreach ($mainEngines as $mainEngine) {
foreach ($engines as $engine) {
if ($engine->name === $mainEngine && !$engine->loaded) {
if ($engine->name === $mainEngine) {
$enginesToWaitFor[] = $engine->hash;
}
}
......@@ -950,6 +990,17 @@ class MetaGer
foreach ($engines as $engine) {
$enginesToWaitFor[] = $engine->hash;
}
} else {
$newEnginesToWaitFor = [];
// Don't wait for engines that are already loaded in Cache
foreach ($enginesToWaitFor as $engineToWaitFor) {
foreach ($engines as $engine) {
if ($engine->hash === $engineToWaitFor && !$engine->loaded) {
$newEnginesToWaitFor[] = $engineToWaitFor;
}
}
}
$enginesToWaitFor = $newEnginesToWaitFor;
}
$timeStart = microtime(true);
......@@ -1048,7 +1099,7 @@ class MetaGer
$this->eingabe = trim($request->input('eingabe', ''));
$this->q = $this->eingabe;
if ($request->filled("mgv")) {
if ($request->filled("mgv") || $request->input("out", "") === "results-with-style") {
$this->framed = true;
} else {
$this->framed = false;
......@@ -1213,10 +1264,10 @@ class MetaGer
}
}
public function createQuicktips()
public function createQuicktips()
{
# Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
$quicktips = new \App\Models\Quicktips\Quicktips($this->q, LaravelLocalization::getCurrentLocale(), $this->getTime());
$quicktips = new \App\Models\Quicktips\Quicktips($this->q, LaravelLocalization::getCurrentLocale(), $this->getTime(), $this->sprueche);
return $quicktips;
}
......
......@@ -3,53 +3,59 @@
namespace App\Models\Quicktips;
use Cache;
use LaravelLocalization;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Log;
class Quicktips
{
use DispatchesJobs;
private $quicktipUrl = "/1.1/quicktips.xml";
private $results = [];
const QUICKTIP_NAME = "quicktips";
const CACHE_DURATION = 60 * 60;