Merge branch 'development' into phil-1010-add-quotes-and-did-you-know-to-metager-patch-52193

.gitlab-ci.yml

@@ -33,6 +33,8 @@ stages:
   - integrationtest
   - cleanup
 
+.auto-deploy:
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.6"
 
 build:
   services:
@@ -75,6 +77,10 @@ review:
   variables:
     HELM_UPGRADE_VALUES_FILE: .gitlab/review-apps-values.yaml
     ROLLOUT_RESOURCE_TYPE: deployment
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    on_stop: stop_review
+    auto_stop_in: 2 days
   rules:
     - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
       when: never
@@ -87,6 +93,11 @@ review:
     - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
 
 stop_review:
+  variables:
+    GIT_STRATEGY: none
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    action: stop
   rules:
     - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
       when: never
@@ -163,4 +174,4 @@ integrationtest:
     - sed -i "s#^APP_URL=.*#APP_URL=$URL#g" .env
     - sed -i "s#^BRANCH_NAME=.*#BRANCH_NAME=$CI_COMMIT_REF_NAME#g" .env
     - sed -i "s#^COMMIT_NAME=.*#COMMIT_NAME=$CI_COMMIT_REF_SLUG#g" .env
-    - php artisan dusk
\ No newline at end of file
+    - php artisan dusk

.gitlab/development-values.yaml

+postgresql:
+  enabled: false
 service:
   externalPort: 80
   internalPort: 80
@@ -5,13 +7,26 @@ hpa:
   enabled: true
   minReplicas: 1
   maxReplicas: 5
-resources:
-  limits:
+resourcesPhpfpm: 
+  requests:
     cpu: 500m
+    memory: 500M
+  limits:
+resourcesNginx: 
+  requests:
+    cpu: 100m
+    memory: 100M
+  limits:
+resourcesRedis: 
+  requests:
+    cpu: 100m
     memory: 1Gi
+  limits:
+resourcesFetcher: 
   requests:
     cpu: 500m
-    memory: 1Gi
+    memory: 100M
+  limits:
 podDisruptionBudget:
   enabled: true
   minAvailable:

.gitlab/production-values.yaml

+postgresql:
+  enabled: false
 service:
   externalPort: 80
   internalPort: 80
@@ -5,13 +7,6 @@ hpa:
   enabled: true
   minReplicas: 3
   maxReplicas: 100
-resources:
-  limits:
-    cpu: 500m
-    memory: 1Gi
-  requests:
-    cpu: 500m
-    memory: 1Gi
 podDisruptionBudget:
   enabled: true
   minAvailable:
@@ -21,6 +16,29 @@ podAnnotations:
   prometheus.io/path: /metrics
   prometheus.io/port: "80"
 deploymentApiVersion: apps/v1
+resources:
+  requests:
+  limits:
+resourcesPhpfpm: 
+  requests:
+    cpu: 500m
+    memory: 500M
+  limits:
+resourcesNginx: 
+  requests:
+    cpu: 100m
+    memory: 100M
+  limits:
+resourcesRedis: 
+  requests:
+    cpu: 100m
+    memory: 1Gi
+  limits:
+resourcesFetcher: 
+  requests:
+    cpu: 500m
+    memory: 100M
+  limits:
 ingress:
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod

.gitlab/review-apps-values.yaml

---- 
+postgresql:
+    enabled: false
 hpa: 
   enabled: false
+resources:
+  requests:
+  limits:
 ingress: 
   annotations: 
     kubernetes.io/tls-acme: "false"
@@ -21,3 +25,15 @@ service:
   externalPort: 80
   internalPort: 80
 deploymentApiVersion: apps/v1
+resourcesPhpfpm: 
+  requests:
+  limits:
+resourcesNginx: 
+  requests:
+  limits:
+resourcesRedis: 
+  requests:
+  limits:
+resourcesFetcher: 
+  requests:
+  limits:
\ No newline at end of file

app/Http/Controllers/AdminInterface.php

@@ -31,7 +31,6 @@ class AdminInterface extends Controller
                             if (strstr($key, "_time")) {
                                 $stati[$name]["fetcher"][$pid]["connection"][$key] = $val;
                             }
-
                         }
                         $stati[$name]["fetcher"][$pid]["poptime"] = $value[1];
                     }
@@ -157,7 +156,7 @@ class AdminInterface extends Controller
             $now->minute = 0;
             $now->second = 0;
 
-            while($now->lessThanOrEqualTo(Carbon::now())){
+            while ($now->lessThanOrEqualTo(Carbon::now())) {
                 $sameTime += empty($stats->time->{$now->format('H:i')}->{$interface}) ? 0 : $stats->time->{$now->format('H:i')}->{$interface};
                 $now->addMinutes(5);
             }
@@ -178,19 +177,38 @@ class AdminInterface extends Controller
             if ($size > 0) {
                 $oldLogs[$key]['median'] = number_format(floatval(round($count / $size)), 0, ",", ".");
             }
+        }
 
+        $sameTimes = [];
+        $sum = 0;
+        foreach ($oldLogs as $index => $oldLog) {
+            if ($index % 7 === 0) {
+                $sameTime = $oldLog["sameTime"];
+                $sameTime = str_replace(".", "", $sameTime);
+                $sameTime = \intval($sameTime);
+                $sameTimes[] = ($logToday - $sameTime);
+                $sum += ($logToday - $sameTime);
+            }
+        }
+        
+        $averageIncrease = 0;
+        if (sizeof($sameTimes) > 0) {
+            $averageIncrease = $sum / sizeof($sameTimes);
         }
 
         if ($request->input('out', 'web') === "web") {
             return view('admin.count')
                 ->with('title', 'Suchanfragen - MetaGer')
                 ->with('today', number_format(floatval($logToday), 0, ",", "."))
+                ->with('averageIncrease', $averageIncrease)
                 ->with('oldLogs', $oldLogs)
                 ->with('minCount', $minCount)
                 ->with('rekordCount', number_format(floatval($rekordTag), 0, ",", "."))
                 ->with('rekordTagSameTime', number_format(floatval($rekordTagSameTime), 0, ",", "."))
                 ->with('rekordDate', $rekordTagDate)
-                ->with('days', $days);
+                ->with('days', $days)
+                ->with('css', [mix('/css/count/style.css')])
+                ->with('darkcss', [mix('/css/count/dark.css')]);
         } else {
             $result = "";
             foreach ($oldLogs as $key => $value) {
@@ -203,9 +221,7 @@ class AdminInterface extends Controller
             return response($result, 200)
                 ->header('Content-Type', 'text/csv')
                 ->header('Content-Disposition', 'attachment; filename="count.csv"');
-
         }
-
     }
 
     public function countGraphToday()
@@ -235,7 +251,6 @@ class AdminInterface extends Controller
         return response()
             ->view('admin.countGraphToday', ["data" => $result], 200)
             ->header('Content-Type', "image/svg+xml");
-
     }
 
     public function engineStats()

app/Http/Controllers/MetaGerSearch.php

@@ -93,6 +93,12 @@ class MetaGerSearch extends Controller
 
         $metager->startSearch($timings);
 
+        # Versuchen die Ergebnisse der Quicktips zu laden
+        $quicktipResults = $quicktips->loadResults();
+        if (!empty($timings)) {
+            $timings["Loaded Quicktip Results"] = microtime(true) - $time;
+        }
+
         $metager->waitForMainResults();
         if (!empty($timings)) {
             $timings["waitForMainResults"] = microtime(true) - $time;
@@ -245,11 +251,11 @@ class MetaGerSearch extends Controller
 
         $newResults = 0;
         foreach ($metager->getResults() as $index => $resultTmp) {
-            if ($resultTmp->new || $resultTmp->adgoalChanged) {
+            if ($resultTmp->new || $resultTmp->changed) {
                 if ($metager->getFokus() !== "bilder") {
                     $view = View::make('layouts.result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
                     $html = $view->render();
-                    if (!$resultTmp->new && $resultTmp->adgoalChanged) {
+                    if (!$resultTmp->new && $resultTmp->changed) {
                         $result['changedResults'][$index] = $html;
                     } else {
                         $result['newResults'][$index] = $html;
@@ -258,7 +264,7 @@ class MetaGerSearch extends Controller
                 } else {
                     $view = View::make('layouts.image_result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
                     $html = $view->render();
-                    if (!$resultTmp->new && $resultTmp->adgoalChanged) {
+                    if (!$resultTmp->new && $resultTmp->changed) {
                         $result['changedResults'][$index] = $html;
                     } else {
                         $result['newResults'][$index] = $html;

app/Http/Controllers/SettingsController.php

@@ -56,7 +56,7 @@ class SettingsController extends Controller
         $cookies = Cookie::get();
         $settingActive = false;
         foreach ($cookies as $key => $value) {
-            if (\starts_with($key, [$fokus . "_engine_", $fokus . "_setting_"]) || strpos($key, $fokus . '_blpage') === 0) {
+            if (\starts_with($key, [$fokus . "_engine_", $fokus . "_setting_"]) || strpos($key, $fokus . '_blpage') === 0 || $key === 'dark_mode') {
                 $settingActive = true;
             }
         }
@@ -209,23 +209,32 @@ class SettingsController extends Controller
         $fokus = $request->input('fokus', '');
         $url = $request->input('url', '');
         // Currently only the setting for quotes is supported
+
         $quotes = $request->input('zitate', '');
-        if (empty($fokus) || empty($quotes)) {
-            abort(404);
+        if(!empty($quotes)){
+            if($quotes === "off"){
+                $path = \Request::path();
+                $cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
+                Cookie::queue($fokus . "_setting_zitate", "off", 0, $cookiePath, null, false, false);
+            }elseif($quotes === "on") {
+                $path = \Request::path();
+                $cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
+                Cookie::queue($fokus . "_setting_zitate", "", 0, $cookiePath, null, false, false);
+            }
         }
 
-        if($quotes === "off"){
-            $path = \Request::path();
-            $cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
-            Cookie::queue($fokus . "_setting_zitate", "off", 0, $cookiePath, null, false, false);
-        }elseif($quotes === "on") {
-            $path = \Request::path();
-            $cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
-            Cookie::queue($fokus . "_setting_zitate", "", 0, $cookiePath, null, false, false);
-        }else{
-            abort(404);
+        $darkmode = $request->input('dm');
+        if(!empty($darkmode)){
+            if($darkmode === "off"){
+                Cookie::queue('dark_mode', '1', 0, '/', null, false, false);
+            }elseif($darkmode === "on") {
+                Cookie::queue('dark_mode', '2', 0, '/', null, false, false);
+            }elseif($darkmode === "system"){
+                Cookie::queue('dark_mode', '', 0, '/', null, false, false);
+            }
         }
 
+
         return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), route('settings', ["fokus" => $fokus, "url" => $url])));
     }
 
@@ -244,6 +253,10 @@ class SettingsController extends Controller
                 $cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
                 Cookie::queue($key, "", 0, $cookiePath, null, false, false);
             }
+            if($key === 'dark_mode'){
+                Cookie::queue($key, "", 0, '/', null, false, false);
+            }
+
         }
         $this->clearBlacklist($request);
 
@@ -266,8 +279,11 @@ class SettingsController extends Controller
         $key = $request->input('key', '');
         $path = \Request::path();
         $cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
-        Cookie::queue($key, "", 0, $cookiePath, null, false, false);
-
+        if($key === 'dark_mode'){
+            Cookie::queue($key, "", 0, '/', null, false, false);
+        }else{
+            Cookie::queue($key, "", 0, $cookiePath, null, false, false);
+        }
         return redirect($request->input('url', 'https://metager.de'));
 
     }
@@ -278,7 +294,11 @@ class SettingsController extends Controller
         $cookiePath = "/" . substr($path, 0, strpos($path, "meta/") + 5);
 
         foreach (Cookie::get() as $key => $value) {
-            Cookie::queue($key, "", 0, $cookiePath, null, false, false);
+            if($key === 'dark_mode'){
+                Cookie::queue($key, "", 0, '/', null, false, false);    
+            }else{
+                Cookie::queue($key, "", 0, $cookiePath, null, false, false);
+            }
         }
         return redirect($request->input('url', 'https://metager.de'));
     }
@@ -384,6 +404,9 @@ class SettingsController extends Controller
             if($blpage){
                 continue;
             }
+            if($key === 'dark_mode'){
+                Cookie::queue($key, $value, 0, '/', null, false, false);
+            }
             foreach($sumaFile['filter']['parameter-filter'] as $suma => $filter){
                 if($key === $suma && $value === $filter){
                     Cookie::queue($key, $value, 0, $cookiePath, null, false, false);

app/Http/Controllers/StartpageController.php

@@ -3,6 +3,7 @@
 namespace App\Http\Controllers;
 
 use App;
+use Cookie;
 use Illuminate\Http\Request;
 use Jenssegers\Agent\Agent;
 use LaravelLocalization;
@@ -52,7 +53,8 @@ class StartpageController extends Controller
             ->with('request', $request->input('request', 'GET'))
             ->with('option_values', $option_values)
             ->with('autocomplete', $autocomplete)
-            ->with('pluginmodal', $request->input('plugin-modal', 'off'));
+            ->with('pluginmodal', $request->input('plugin-modal', 'off'))
+            ->with('darkcss', [mix('css/themes/startpage-only-dark.css')]);
     }
 
     public function loadPage($subpage)
@@ -87,8 +89,9 @@ class StartpageController extends Controller
         }
         $response = Response::make(
             view('plugin')
-                ->with('link', $link)
-            , "200");
+                ->with('link', $link),
+            "200"
+        );
         $response->header('Content-Type', "application/opensearchdescription+xml");
         return $response;
     }

app/Jobs/ConvertCountFile.php

@@ -95,15 +95,13 @@ class ConvertCountFile implements ShouldQueue
                 fclose($fh);
             }
 
-            if (!$error) {
-                $oldUmask = umask(0);
-                // Write the result to a File
-                if (!file_exists($this->files["countPath"])) {
-                    mkdir($this->files["countPath"], 0777, true);
-                }
-                file_put_contents($this->files["countFile"], json_encode($result, JSON_PRETTY_PRINT));
-                umask($oldUmask);
+            $oldUmask = umask(0);
+            // Write the result to a File
+            if (!file_exists($this->files["countPath"])) {
+                mkdir($this->files["countPath"], 0777, true);
             }
+            file_put_contents($this->files["countFile"], json_encode($result, JSON_PRETTY_PRINT));
+            umask($oldUmask);
 
             Redis::del(md5($this->files["countFile"]));
         }

app/MetaGer.php

@@ -297,6 +297,11 @@ class MetaGer
         if (!empty($timings)) {
             $timings["prepareResults"]["validated results"] = microtime(true) - $timings["starttime"];
         }
+        
+        $this->duplicationCheck();
+        if (!empty($timings)) {
+            $timings["prepareResults"]["duplications checked"] = microtime(true) - $timings["starttime"];
+        }
         # Validate Advertisements
         $newResults = [];
         foreach ($this->ads as $ad) {
@@ -399,6 +404,41 @@ class MetaGer
         }
     }
 
+    public function duplicationCheck()
+    {
+        $arr = [];
+        for ($i = 0; $i < count($this->results); $i++) {
+            $link = $this->results[$i]->link;
+
+            if (strpos($link, "http://") === 0) {
+                $link = substr($link, 7);
+            }
+    
+            if (strpos($link, "https://") === 0) {
+                $link = substr($link, 8);
+            }
+    
+            if (strpos($link, "www.") === 0) {
+                $link = substr($link, 4);
+            }
+    
+            $link = trim($link, "/");
+            $hash = md5($link);
+
+            if (isset($arr[$link])) {
+                $arr[$link]->gefVon[] = $this->results[$i]->gefVon[0];
+                $arr[$link]->gefVonLink[] = $this->results[$i]->gefVonLink[0];
+                array_splice($this->results, $i, 1);
+                $i--;
+                if ($arr[$link]->new === true || $this->results[$i]->new === true) {
+                    $arr[$link]->changed = true;
+                }
+            } else {
+                $arr[$link] = &$this->results[$i];
+            }
+        }
+    }
+
     public function startAdgoal(&$results)
     {
         $publicKey = getenv('adgoal_public');
@@ -505,7 +545,7 @@ class MetaGer
                         $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
                         $result->link = $newLink;
                         $result->partnershop = true;
-                        $result->adgoalChanged = true;
+                        $result->changed = true;
                     }
                 }
             }
@@ -939,7 +979,7 @@ class MetaGer
         $mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
         foreach ($mainEngines as $mainEngine) {
             foreach ($engines as $engine) {
-                if ($engine->name === $mainEngine && !$engine->loaded) {
+                if ($engine->name === $mainEngine) {
                     $enginesToWaitFor[] = $engine->hash;
                 }
             }
@@ -950,6 +990,17 @@ class MetaGer
             foreach ($engines as $engine) {
                 $enginesToWaitFor[] = $engine->hash;
             }
+        } else {
+            $newEnginesToWaitFor = [];
+            // Don't wait for engines that are already loaded in Cache
+            foreach ($enginesToWaitFor as $engineToWaitFor) {
+                foreach ($engines as $engine) {
+                    if ($engine->hash === $engineToWaitFor && !$engine->loaded) {
+                        $newEnginesToWaitFor[] = $engineToWaitFor;
+                    }
+                }
+            }
+            $enginesToWaitFor = $newEnginesToWaitFor;
         }
 
         $timeStart = microtime(true);
@@ -1048,7 +1099,7 @@ class MetaGer
         $this->eingabe = trim($request->input('eingabe', ''));
         $this->q = $this->eingabe;
 
-        if ($request->filled("mgv")) {
+        if ($request->filled("mgv") || $request->input("out", "") === "results-with-style") {
             $this->framed = true;
         } else {
             $this->framed = false;
@@ -1213,10 +1264,10 @@ class MetaGer
         }
     }
 
-     public function createQuicktips()
+    public function createQuicktips()
     {
         # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
-        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, LaravelLocalization::getCurrentLocale(), $this->getTime());
+        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, LaravelLocalization::getCurrentLocale(), $this->getTime(), $this->sprueche);
         return $quicktips;
     }
 

app/Models/Quicktips/Quicktips.php

@@ -3,53 +3,59 @@
 namespace App\Models\Quicktips;
 
 use Cache;
-use LaravelLocalization;
+use Illuminate\Foundation\Bus\DispatchesJobs;
+use Illuminate\Support\Facades\Redis;
 use Log;
 
 class Quicktips
 {
+    use DispatchesJobs;
 
     private $quicktipUrl = "/1.1/quicktips.xml";
-    private $results = [];
     const QUICKTIP_NAME = "quicktips";
-    const CACHE_DURATION = 60 * 60;
+    const CACHE_DURATION = 60;
 
     private $hash;
+    private $startTime;
+    private $quotes;
 
-    public function __construct($search, $quotes)
+    public function __construct($search, $locale, $max_time, $quotes = "on")
     {
-        $locale = LaravelLocalization::getCurrentLocale();
         if (env("APP_ENV") === "production") {
             $this->quicktipUrl = "https://quicktips.metager.de" . $this->quicktipUrl;
         } else {
             $this->quicktipUrl = "https://dev.quicktips.metager.de" . $this->quicktipUrl;
         }
-        $this->startSearch($search, $quotes, $locale);
+        $this->quotes = $quotes;
+        $this->startTime = microtime(true);
+        $this->startSearch($search, $locale, $max_time);
     }
 
-    public function startSearch($search, $quotes, $locale)
+    public function startSearch($search, $locale, $max_time)
     {
-        $url = $this->quicktipUrl . "?search=" . $this->normalize_search($search) . "&locale=" . $locale . "&quotes=" . $quotes;
+        $url = $this->quicktipUrl . "?search=" . $this->normalize_search($search) . "&locale=" . $locale  . "&quotes=" . $this->quotes;
         $this->hash = md5($url);
 
-        $results = null;
+        if (!Cache::has($this->hash)) {
+            if (!Redis::exists($this->hash)) {
 
-        try {
-            if (!Cache::has($this->hash)) {
-                $results = file_get_contents($url);
-                Cache::put($this->hash, $results, Quicktips::CACHE_DURATION);
-            } else {
-                $results = Cache::get($this->hash);
-            }
-        } catch (\Exception $e) {
-            Log::error($e->getMessage());
-        }
+                // Queue this search
+                $mission = [
+                    "resulthash" => $this->hash,
+                    "url" => $url,
+                    "useragent" => "",
+                    "username" => null,
+                    "password" => null,
+                    "headers" => [],
+                    "cacheDuration" => self::CACHE_DURATION,
+                    "name" => "Quicktips",
+                ];
 
-        if ($results === null) {
-            $results = file_get_contents($url);
-        }
+                $mission = json_encode($mission);
 
-        $this->results = $this->loadResults($results);
+                Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
+            }
+        }
     }