Merge branch '1328-optimize-statistics' into 'development'

Resolve "Optimize Statistics"

Closes #1328

See merge request !2241

metager/app/Models/Matomo.php → metager/app/Http/Controllers/StatisticsController.php

 <?php
 
-namespace App\Models;
+namespace App\Http\Controllers;
 
+use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Redis;
-use Request;
 
-class Matomo
+
+class StatisticsController extends Controller
 {
-    static function PAGE_VISIT()
+    public function pageLoad(Request $request)
     {
-        if (!config("metager.matomo.enabled") || config("metager.matomo.url") === null || request()->is("health-check/*"))
+        if (!config("metager.matomo.enabled") || config("metager.matomo.url") === null)
             return;
 
         $params = [
@@ -18,18 +19,16 @@ class Matomo
             "rand" => md5(microtime(true)),
             "rec" => "1",
             "send_image" => "0",
-            "cip" => request()->ip(),
+            "cip" => $request->ip(),
+            "_id" => substr(md5($request->ip() . now()->format("Y-m-d")), 0, 16)
         ];
-        // Page URL
-        $url = request()->getPathInfo();
-        if (stripos($url, "/img") === 0 || stripos($url, "/meta/meta.ger3") === 0 || stripos($url, "/meta/loadMore") === 0 || preg_match("/\.(gif|png|jpg|jpeg|css)$/", $url) || preg_match("/csp-report$/", $url))
-            return;
-        $url = request()->schemeAndHttpHost() . preg_replace("/^\/[a-z]{2}-[A-Z]{2}/", "", $url);
-        $params["url"] = $url;
-        // Referer
-        $params["urlref"] = request()->headers->get("referer");
+        $http_params = $request->all();
+
         // Useragent
-        $params["ua"] = request()->userAgent();
+        $params["ua"] = $request->userAgent();
+        // Accept-Language
+        $params["lang"] = $request->header("Accept-Language");
+        $params = array_merge($http_params, $params);   // Merge arrays keeping our serverside defined options if key is set multiple times
 
         $url = config("metager.matomo.url") . "/matomo.php?" . http_build_query($params);
 
@@ -48,4 +47,4 @@ class Matomo
 
         Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
     }
-}
+}
\ No newline at end of file

metager/app/Http/Middleware/Statistics.php

-<?php
-
-namespace App\Http\Middleware;
-
-use App\Models\Matomo;
-use Closure;
-use Illuminate\Http\Request;
-use Symfony\Component\HttpFoundation\Response;
-
-class Statistics
-{
-    /**
-     * Handle an incoming request.
-     *
-     * @param  \Closure(\Illuminate\Http\Request): (\Symfony\Component\HttpFoundation\Response)  $next
-     */
-    public function handle(Request $request, Closure $next): Response
-    {
-        Matomo::PAGE_VISIT();
-        return $next($request);
-    }
-}

metager/bootstrap/app.php

@@ -24,7 +24,6 @@ return Application::configure(basePath: dirname(__DIR__))
             \App\Http\Middleware\TrimStrings::class,
             \Illuminate\Foundation\Http\Middleware\ConvertEmptyStringsToNull::class,
             TrustProxies::class,
-            Statistics::class,
         ]);
         $middleware->trustProxies(at: [
             '10.0.0.0/8',

metager/resources/js/statistics.js

+/**
+ * Class to gather and report anonymous statistics 
+ * to our self hosted matomo instance.
+ */
+class Statistics {
+    #load_complete = false;
+
+    constructor() {
+        let performance = window.performance.getEntriesByType('navigation')[0];
+        if (performance.loadEventEnd != 0) {
+            this.#init();
+        } else {
+            window.addEventListener("load", e => {
+                let readyStateCheckInterval = setInterval(() => {
+                    performance = window.performance.getEntriesByType('navigation')[0];
+                    if (performance.loadEventEnd == 0) return;
+                    console.log("load end");
+                    clearInterval(readyStateCheckInterval);
+                    this.#init();
+                }, 100);
+            });
+        }
+    }
+
+    #init() {
+        this.pageLoad();
+        document.querySelectorAll("a").forEach(anchor => {
+            anchor.addEventListener("click", e => this.pageLeave(e.target.href).bind(this));
+        });
+    }
+
+    pageLeave(target) {
+        let params = {};
+        params.url = target;
+        params.link = target;
+        try {
+            let url = new URL(target);
+            if (url.host == document.location.host) return;
+            this.pageLoad(params);
+            navigator.sendBeacon("/stats/pl", new URLSearchParams(params));
+        } catch (error) { }
+    }
+
+    pageLoad(overwrite_params = {}) {
+        if (this.#load_complete) return;
+        this.#load_complete = true;
+
+        let params = {};
+
+        // Page performance
+        try {
+            let performance = window.performance.getEntriesByType('navigation')[0];
+            params.pf_net = performance.connectEnd - performance.connectStart;
+            params.pf_srv = performance.responseStart - performance.requestStart;
+            params.pf_tfr = performance.responseEnd - performance.responseStart;
+            params.pf_dm1 = performance.domInteractive - performance.responseEnd;
+            params.pf_dm2 = performance.domContentLoadedEventEnd - performance.domContentLoadedEventStart;
+            params.pf_onl = performance.loadEventEnd - performance.loadEventStart;
+        } catch (error) { }
+
+        try {
+            params.res = window.screen.width + "x" + window.screen.height;
+        } catch (error) { }
+
+        // Page URL
+        try {
+            params.url = document.location.href;
+        } catch (error) { }
+
+        // Page Title
+        try {
+            params.action_name = document.title
+        } catch (error) { }
+
+        // Referrer
+        try {
+            params.urlref = document.referrer;
+        } catch (error) { }
+
+        // Cookies available 
+        try {
+            params.cookie = navigator.cookieEnabled ? "1" : "0";
+        } catch (error) { }
+
+        params = { ...params, ...overwrite_params };
+
+        navigator.sendBeacon("/stats/pl", new URLSearchParams(params));
+    }
+}
+export const statistics = new Statistics();
\ No newline at end of file

metager/resources/js/utility.js

+import { statistics } from "./statistics";
+
 document.addEventListener("DOMContentLoaded", (event) => {
   document
     .querySelectorAll(".js-only")

metager/routes/web.php

@@ -13,6 +13,7 @@ use App\Http\Controllers\Prometheus;
 use App\Http\Controllers\SearchEngineList;
 use App\Http\Controllers\SitesearchController;
 use App\Http\Controllers\StartpageController;
+use App\Http\Controllers\StatisticsController;
 use App\Http\Controllers\SuggestionController;
 use App\Http\Controllers\TTSController;
 use App\Http\Controllers\ZitatController;
@@ -344,7 +345,7 @@ Route::withoutMiddleware([\Illuminate\Foundation\Http\Middleware\ValidateCsrfTok
         return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), '/'));
     });
 
-    Route::match (['get', 'post'], 'meta/meta.ger3', [MetaGerSearch::class, 'search'])->middleware('httpcache', 'externalimagesearch', 'spam', 'browserverification', 'humanverification', 'useragentmaster')->name("resultpage");
+    Route::match(['get', 'post'], 'meta/meta.ger3', [MetaGerSearch::class, 'search'])->middleware('httpcache', 'externalimagesearch', 'spam', 'browserverification', 'humanverification', 'useragentmaster')->name("resultpage");
 
     Route::get('meta/loadMore', [MetaGerSearch::class, 'loadMore']);
 
@@ -449,4 +450,8 @@ Route::withoutMiddleware([\Illuminate\Foundation\Http\Middleware\ValidateCsrfTok
         Route::get('liveness-scheduler', [HealthcheckController::class, 'livenessScheduler']);
         Route::get('liveness-worker', [HealthcheckController::class, 'livenessWorker']);
     });
+
+    Route::group(['prefix' => 'stats'], function () {
+        Route::post('pl', [StatisticsController::class, 'pageLoad']);
+    });
 });
\ No newline at end of file