Merge branch '1327-include-statistics' into 'development'

Resolve "Include Statistics"

Closes #1327

See merge request !2238

metager/app/Console/Commands/RequestFetcher.php

@@ -156,7 +156,7 @@ class RequestFetcher extends Command
                     Log::error($error);
                 }
 
-                if ($responseCode !== 200 && $responseCode !== 201) {
+                if ($responseCode < 200 || $responseCode > 299) {
                     Log::debug($resulthash);
                     Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
                     Log::debug(\curl_multi_getcontent($info["handle"]));

metager/app/Http/Middleware/Statistics.php

+<?php
+
+namespace App\Http\Middleware;
+
+use App\Models\Matomo;
+use Closure;
+use Illuminate\Http\Request;
+use Symfony\Component\HttpFoundation\Response;
+
+class Statistics
+{
+    /**
+     * Handle an incoming request.
+     *
+     * @param  \Closure(\Illuminate\Http\Request): (\Symfony\Component\HttpFoundation\Response)  $next
+     */
+    public function handle(Request $request, Closure $next): Response
+    {
+        Matomo::PAGE_VISIT();
+        return $next($request);
+    }
+}

metager/app/Models/Matomo.php

+<?php
+
+namespace App\Models;
+
+use Illuminate\Support\Facades\Redis;
+use Request;
+
+class Matomo
+{
+    static function PAGE_VISIT()
+    {
+        if (!config("metager.matomo.enabled") || config("metager.matomo.url") === null || request()->is("health-check/*"))
+            return;
+        if (request()->is("meta/meta.ger3") && request()->filled("mgv"))
+            return;
+        $params = [
+            "idsite" => config("metager.matomo.site_id"),
+            "token_auth" => config("metager.matomo.token_auth"),
+            "rand" => md5(microtime(true)),
+            "rec" => "1",
+            "send_image" => "0",
+            "cip" => request()->ip(),
+        ];
+        // Page URL
+        $url = request()->getPathInfo();
+        if (stripos($url, "/img") === 0 || stripos($url, "/meta/meta.ger3") === 0 || stripos($url, "/meta/loadMore") === 0 || preg_match("/\.css$/", $url) || preg_match("/csp-report$/", $url))
+            return;
+        $url = request()->schemeAndHttpHost() . preg_replace("/^\/[a-z]{2}-[A-Z]{2}/", "", $url);
+        $params["url"] = $url;
+        // Referer
+        $params["urlref"] = request()->headers->get("referer");
+        // Useragent
+        $params["ua"] = request()->userAgent();
+
+        $url = config("metager.matomo.url") . "/matomo.php?" . http_build_query($params);
+
+        // Submit fetch job to worker
+        $hash = hash("sha256", \json_encode($params));
+        $mission = [
+            "resulthash" => $hash,
+            "url" => $url,
+            "useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
+            "username" => null,
+            "password" => null,
+            "cacheDuration" => 0,
+            "name" => "Matomo"
+        ];
+        $mission = json_encode($mission);
+
+        Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
+    }
+}
\ No newline at end of file

metager/app/Models/parserSkripte/Overture.php

@@ -43,7 +43,6 @@ class Overture extends Searchengine
 
         // Apply default get parameters
         $this->configuration->addQueryParameters([
-            "Partner" => "tripledoubleu_xml_de_searchbox_metager",
             "on" => "6",
             "in" => "20",
             "adEnableActionExt" => "1",

metager/app/Models/parserSkripte/OvertureAds.php

@@ -22,7 +22,6 @@ class OvertureAds extends Searchengine
 
         // Apply default get parameters
         $this->configuration->addQueryParameters([
-            "Partner" => "tripledoubleu_xml_de_searchbox_metager",
             "on" => "6",
             "in" => "20",
             "adEnableActionExt" => "1",

metager/bootstrap/app.php

@@ -3,6 +3,7 @@
 use App\Http\Middleware\AllowLocalOnly;
 use App\Http\Middleware\ExternalImagesearch;
 use App\Http\Middleware\HttpCache;
+use App\Http\Middleware\Statistics;
 use Illuminate\Foundation\Application;
 use Illuminate\Foundation\Configuration\Exceptions;
 use Illuminate\Foundation\Configuration\Middleware;
@@ -22,6 +23,7 @@ return Application::configure(basePath: dirname(__DIR__))
             \Illuminate\Foundation\Http\Middleware\ValidatePostSize::class,
             \App\Http\Middleware\TrimStrings::class,
             \Illuminate\Foundation\Http\Middleware\ConvertEmptyStringsToNull::class,
+            Statistics::class,
             TrustProxies::class,
         ]);
         $middleware->trustProxies(at: [

metager/config/metager/matomo.php

+<?php
+
+return [
+    "enabled" => env("MATOMO_ENABLED", false),
+    "url" => env("MATOMO_URL", null),
+    "site_id" => env("MATOMO_SITE_ID", 0),
+    "token_auth" => env("MATOMO_TOKEN_AUTH", "")
+];
\ No newline at end of file

metager/lang/de/privacy.php

@@ -10,6 +10,10 @@ return [
         'title' => 'Grundsätze',
         'description' => 'Wir haben uns als gemeinnütziger Verein dem freien Wissenszugang verschrieben. Da wir wissen, dass freie Recherche nicht mit Massenüberwachung vereinbar ist, nehmen wir auch Datenschutz sehr ernst. Wir verarbeiten schon immer nur die Daten, die zum Betrieb unserer Dienste unbedingt nötig sind. Datenschutz ist bei uns immer der Standard. Profiling – also die automatische Erstellung von Nutzerprofilen – betreiben wir nicht.',
     ],
+    'stats' => [
+        'title' => 'Anonyme Statistiken',
+        'description' => 'Wir arbeiten stets daran unsere Dienste zu verbessern. Um dazu in der Lage zu sein, müssen wir wissen, welche unserer Funktionen verwendet werden. Aus diesem Grund erfassen wir vollkommen anonyme Daten über die Häufigkeit von Seitenaufrufen und die Verwendung einzelner Funktionen unserer Webseiten. Ebenfalls erfassen wir anonymisierte Daten über die Verteilung von Browserarten und Versionen. Diese Statistiken basieren nicht auf einzelnen Nutzerprofilen und werden ohne Cookies oder ähnlichem erstellt. Sie beinhalten keine personenbezogenen Daten.'
+    ],
     'contexts' => [
         'title' => 'Anfallende Daten nach Kontext',
         'metager' => [

metager/lang/en/privacy.php

@@ -11,6 +11,10 @@ return [
         'title' => 'Principles',
         'description' => 'As a non-profit association, we are committed to free access to knowledge. Since we know that free research is not compatible with mass surveillance, we also take data protection very seriously. We have always only processed the data that is absolutely necessary for the operation of our services. Data protection is always our standard. We do not operate profiling – i.e. the automatic creation of user profiles.'
     ],
+    'stats' => [
+        'title' => 'Anonymous statistics',
+        'description' => 'We are always working to improve our services. To be able to do this, we need to know which of our functions are being used. For this reason, we collect completely anonymous data on the frequency of page views and the use of individual functions on our websites. We also collect anonymized data on the distribution of browser types and versions. These statistics are not based on individual user profiles and are created without cookies or similar technologies. They do not contain any personal data.'
+    ],
     'contexts' => [
         'title' => 'Incoming data by context',
         'metager' => [

metager/resources/views/privacy.blade.php

 @extends('layouts.subPages', ['page' => 'privacy'])
 
-@section('title', trans('titles.datenschutz') )
+@section('title', trans('titles.datenschutz'))
 
 @section('navbarFocus.datenschutz', 'class="active"')
 
@@ -12,12 +12,18 @@
     </div class="section">
     <div class="section">
         <h1>@lang('privacy.responsible_party.title')</h1>
-        <div>@lang('privacy.responsible_party.description', ['link_impress' => route('impress'), 'link_contact' => route('contact')])</div>
+        <div>
+            @lang('privacy.responsible_party.description', ['link_impress' => route('impress'), 'link_contact' => route('contact')])
+        </div>
     </div>
     <div class="section">
         <h1>@lang('privacy.principles.title')</h1>
         <div>@lang('privacy.principles.description')</div>
     </div>
+    <div class="section">
+        <h1>@lang('privacy.stats.title')</h1>
+        <div>@lang('privacy.stats.description')</div>
+    </div>
     <div class="section">
         <h1>@lang('privacy.contexts.title')</h1>
         <ol>
@@ -28,18 +34,23 @@
                     <ol class="datum-list">
                         <li><a href="#ip-address">@lang('privacy.data.ip'):</a> @lang('privacy.data.unused')</li>
                         <li><a href="#user-agent">@lang('privacy.data.useragent'):</a> @lang('privacy.data.unused')</li>
-                        <li><a href="#search-request">@lang('privacy.data.query'):</a>  @lang('privacy.contexts.metager.query')</li>
-                        <li><a href="#preferences">@lang('privacy.data.preferences'):</a> @lang('privacy.contexts.metager.preferences')</li>
+                        <li><a href="#search-request">@lang('privacy.data.query'):</a>
+                            @lang('privacy.contexts.metager.query')</li>
+                        <li><a href="#preferences">@lang('privacy.data.preferences'):</a>
+                            @lang('privacy.contexts.metager.preferences')</li>
                     </ol>
                     <h3>@lang('privacy.contexts.metager.additionally')</h3>
                     <ol class="datum-list">
                         <li>
-                            <div><a href="#ip-address">@lang('privacy.data.ip')</a>, <a href="#user-agent">@lang('privacy.data.useragent')</a>:</div>
+                            <div><a href="#ip-address">@lang('privacy.data.ip')</a>, <a
+                                    href="#user-agent">@lang('privacy.data.useragent')</a>:</div>
                             <div>@lang('privacy.contexts.metager.botprotection')</div>
                         </li>
                         <li>
-                            <div><a href="https://privacy.microsoft.com/privacystatement">Microsoft Clarity & Yahoo</a></div>
-                            <div>@lang('privacy.contexts.metager.clarity')</div></li>
+                            <div><a href="https://privacy.microsoft.com/privacystatement">Microsoft Clarity & Yahoo</a>
+                            </div>
+                            <div>@lang('privacy.contexts.metager.clarity')</div>
+                        </li>
                     </ol>
                 </article>
                 <article class="kontext">
@@ -48,8 +59,10 @@
                     <ol class="datum-list">
                         <li><a href="#ip-address">@lang('privacy.data.ip'):</a> @lang('privacy.data.unused')</li>
                         <li><a href="#user-agent">@lang('privacy.data.useragent'):</a> @lang('privacy.data.unused')</li>
-                        <li><a href="#contact-data">@lang('privacy.data.contact'):</a> @lang('privacy.contexts.contact.contact')</li>
-                        <li><a href="#message">@lang('privacy.data.message'):</a> @lang('privacy.contexts.contact.contact')</li>
+                        <li><a href="#contact-data">@lang('privacy.data.contact'):</a>
+                            @lang('privacy.contexts.contact.contact')</li>
+                        <li><a href="#message">@lang('privacy.data.message'):</a>
+                            @lang('privacy.contexts.contact.contact')</li>
                     </ol>
                 </article>
                 <article class="kontext">
@@ -58,9 +71,12 @@
                     <ol class="datum-list">
                         <li><a href="#ip-address">@lang('privacy.data.ip'):</a> @lang('privacy.data.unused')</li>
                         <li><a href="#user-agent">@lang('privacy.data.useragent'):</a> @lang('privacy.data.unused')</li>
-                        <li><a href="#contact-data">@lang('privacy.data.contact'):</a> @lang('privacy.contexts.donate.contact')</li>
-                        <li><a href="#payment-data">@lang('privacy.data.payment'):</a> @lang('privacy.contexts.donate.payment')</li>
-                        <li><a href="#message">@lang('privacy.data.message') (@lang('privacy.data.optional')):</a> @lang('privacy.contexts.donate.message')</li>
+                        <li><a href="#contact-data">@lang('privacy.data.contact'):</a>
+                            @lang('privacy.contexts.donate.contact')</li>
+                        <li><a href="#payment-data">@lang('privacy.data.payment'):</a>
+                            @lang('privacy.contexts.donate.payment')</li>
+                        <li><a href="#message">@lang('privacy.data.message') (@lang('privacy.data.optional')):</a>
+                            @lang('privacy.contexts.donate.message')</li>
                     </ol>
                 </article>
                 <article class="kontext">
@@ -68,8 +84,10 @@
                     <ol class="datum-list">
                         <li><a href="#ip-address">@lang('privacy.data.ip'):</a> @lang('privacy.data.unused')</li>
                         <li><a href="#user-agent">@lang('privacy.data.useragent'):</a> @lang('privacy.data.unused')</li>
-                        <li><a href="#contact-data">@lang('privacy.data.contact') (@lang('privacy.data.optional')):</a> @lang('privacy.contexts.key.contact')</li>
-                        <li><a href="#payment-data">@lang('privacy.data.payment'):</a> @lang('privacy.contexts.key.payment')</li>
+                        <li><a href="#contact-data">@lang('privacy.data.contact') (@lang('privacy.data.optional')):</a>
+                            @lang('privacy.contexts.key.contact')</li>
+                        <li><a href="#payment-data">@lang('privacy.data.payment'):</a>
+                            @lang('privacy.contexts.key.payment')</li>
                     </ol>
                 </article>
                 <article class="kontext">
@@ -88,7 +106,8 @@
                     <h1>@lang('privacy.contexts.newsletter.title')</h1>
                     <div>@lang('privacy.contexts.newsletter.description')</div>
                     <ol class="datum-list">
-                        <li><a href="#contact-data">@lang('privacy.data.contact'):</a> @lang('privacy.contexts.newsletter.contact')</li>
+                        <li><a href="#contact-data">@lang('privacy.data.contact'):</a>
+                            @lang('privacy.contexts.newsletter.contact')</li>
                     </ol>
                 </article>
                 <article class="kontext">
@@ -218,7 +237,7 @@
         <ol>
             <li><b>@lang('privacy.rights.information.title'):</b></li>
             <article class="kontext">
-            @lang('privacy.rights.information.description')
+                @lang('privacy.rights.information.description')
             </article>
             <li><b>@lang('privacy.rights.correction.title'):</b></li>
             <article class="kontext">
@@ -257,6 +276,6 @@
     <div class="section">
         <h1>@lang('privacy.changes.title')</h1>
         <div>@lang('privacy.changes.description')</div>
-        <div>@lang('privacy.changes.date', ['date' => '2024-06-11'])</div>
+        <div>@lang('privacy.changes.date', ['date' => '2024-07-24'])</div>
     </div>
-@endsection
\ No newline at end of file
+    @endsection
\ No newline at end of file