From 39bc0e0dc41f5c9ada38852cbcc747922a257fab Mon Sep 17 00:00:00 2001
From: Dominik Hebeler <dominik@suma-ev.de>
Date: Fri, 12 Jul 2024 14:59:24 +0200
Subject: [PATCH] add some basic statistics
---
.../app/Console/Commands/RequestFetcher.php | 2 +-
metager/app/Http/Middleware/Statistics.php | 22 ++++++++
metager/app/Models/Matomo.php | 52 ++++++++++++++++++
metager/app/Models/parserSkripte/Overture.php | 1 -
.../app/Models/parserSkripte/OvertureAds.php | 1 -
metager/bootstrap/app.php | 2 +
metager/config/metager/matomo.php | 8 +++
metager/lang/de/privacy.php | 4 ++
metager/lang/en/privacy.php | 4 ++
metager/resources/views/privacy.blade.php | 55 +++++++++++++------
10 files changed, 130 insertions(+), 21 deletions(-)
create mode 100644 metager/app/Http/Middleware/Statistics.php
create mode 100644 metager/app/Models/Matomo.php
create mode 100644 metager/config/metager/matomo.php
diff --git a/metager/app/Console/Commands/RequestFetcher.php b/metager/app/Console/Commands/RequestFetcher.php
index aa6666650..380b9d1ab 100644
--- a/metager/app/Console/Commands/RequestFetcher.php
+++ b/metager/app/Console/Commands/RequestFetcher.php
@@ -156,7 +156,7 @@ class RequestFetcher extends Command
Log::error($error);
}
- if ($responseCode !== 200 && $responseCode !== 201) {
+ if ($responseCode < 200 || $responseCode > 299) {
Log::debug($resulthash);
Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
Log::debug(\curl_multi_getcontent($info["handle"]));
diff --git a/metager/app/Http/Middleware/Statistics.php b/metager/app/Http/Middleware/Statistics.php
new file mode 100644
index 000000000..5671ad62b
--- /dev/null
+++ b/metager/app/Http/Middleware/Statistics.php
@@ -0,0 +1,22 @@
+<?php
+
+namespace App\Http\Middleware;
+
+use App\Models\Matomo;
+use Closure;
+use Illuminate\Http\Request;
+use Symfony\Component\HttpFoundation\Response;
+
+class Statistics
+{
+ /**
+ * Handle an incoming request.
+ *
+ * @param \Closure(\Illuminate\Http\Request): (\Symfony\Component\HttpFoundation\Response) $next
+ */
+ public function handle(Request $request, Closure $next): Response
+ {
+ Matomo::PAGE_VISIT();
+ return $next($request);
+ }
+}
diff --git a/metager/app/Models/Matomo.php b/metager/app/Models/Matomo.php
new file mode 100644
index 000000000..ade30b5c5
--- /dev/null
+++ b/metager/app/Models/Matomo.php
@@ -0,0 +1,52 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Support\Facades\Redis;
+use Request;
+
+class Matomo
+{
+ static function PAGE_VISIT()
+ {
+ if (!config("metager.matomo.enabled") || config("metager.matomo.url") === null || request()->is("health-check/*"))
+ return;
+ if (request()->is("meta/meta.ger3") && request()->filled("mgv"))
+ return;
+ $params = [
+ "idsite" => config("metager.matomo.site_id"),
+ "token_auth" => config("metager.matomo.token_auth"),
+ "rand" => md5(microtime(true)),
+ "rec" => "1",
+ "send_image" => "0",
+ "cip" => request()->ip(),
+ ];
+ // Page URL
+ $url = request()->getPathInfo();
+ if (stripos($url, "/img") === 0 || stripos($url, "/meta/meta.ger3") === 0 || stripos($url, "/meta/loadMore") === 0 || preg_match("/\.css$/", $url) || preg_match("/csp-report$/", $url))
+ return;
+ $url = request()->schemeAndHttpHost() . preg_replace("/^\/[a-z]{2}-[A-Z]{2}/", "", $url);
+ $params["url"] = $url;
+ // Referer
+ $params["urlref"] = request()->headers->get("referer");
+ // Useragent
+ $params["ua"] = request()->userAgent();
+
+ $url = config("metager.matomo.url") . "/matomo.php?" . http_build_query($params);
+
+ // Submit fetch job to worker
+ $hash = hash("sha256", \json_encode($params));
+ $mission = [
+ "resulthash" => $hash,
+ "url" => $url,
+ "useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
+ "username" => null,
+ "password" => null,
+ "cacheDuration" => 0,
+ "name" => "Matomo"
+ ];
+ $mission = json_encode($mission);
+
+ Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
+ }
+}
\ No newline at end of file
diff --git a/metager/app/Models/parserSkripte/Overture.php b/metager/app/Models/parserSkripte/Overture.php
index bb4971a1d..61b878d88 100644
--- a/metager/app/Models/parserSkripte/Overture.php
+++ b/metager/app/Models/parserSkripte/Overture.php
@@ -43,7 +43,6 @@ class Overture extends Searchengine
// Apply default get parameters
$this->configuration->addQueryParameters([
- "Partner" => "tripledoubleu_xml_de_searchbox_metager",
"on" => "6",
"in" => "20",
"adEnableActionExt" => "1",
diff --git a/metager/app/Models/parserSkripte/OvertureAds.php b/metager/app/Models/parserSkripte/OvertureAds.php
index cc01bed37..30c1b680e 100644
--- a/metager/app/Models/parserSkripte/OvertureAds.php
+++ b/metager/app/Models/parserSkripte/OvertureAds.php
@@ -22,7 +22,6 @@ class OvertureAds extends Searchengine
// Apply default get parameters
$this->configuration->addQueryParameters([
- "Partner" => "tripledoubleu_xml_de_searchbox_metager",
"on" => "6",
"in" => "20",
"adEnableActionExt" => "1",
diff --git a/metager/bootstrap/app.php b/metager/bootstrap/app.php
index e232c31fd..9d1ad9b8c 100644
--- a/metager/bootstrap/app.php
+++ b/metager/bootstrap/app.php
@@ -3,6 +3,7 @@
use App\Http\Middleware\AllowLocalOnly;
use App\Http\Middleware\ExternalImagesearch;
use App\Http\Middleware\HttpCache;
+use App\Http\Middleware\Statistics;
use Illuminate\Foundation\Application;
use Illuminate\Foundation\Configuration\Exceptions;
use Illuminate\Foundation\Configuration\Middleware;
@@ -22,6 +23,7 @@ return Application::configure(basePath: dirname(__DIR__))
\Illuminate\Foundation\Http\Middleware\ValidatePostSize::class,
\App\Http\Middleware\TrimStrings::class,
\Illuminate\Foundation\Http\Middleware\ConvertEmptyStringsToNull::class,
+ Statistics::class,
TrustProxies::class,
]);
$middleware->trustProxies(at: [
diff --git a/metager/config/metager/matomo.php b/metager/config/metager/matomo.php
new file mode 100644
index 000000000..f52c473e5
--- /dev/null
+++ b/metager/config/metager/matomo.php
@@ -0,0 +1,8 @@
+<?php
+
+return [
+ "enabled" => env("MATOMO_ENABLED", false),
+ "url" => env("MATOMO_URL", null),
+ "site_id" => env("MATOMO_SITE_ID", 0),
+ "token_auth" => env("MATOMO_TOKEN_AUTH", "")
+];
\ No newline at end of file
diff --git a/metager/lang/de/privacy.php b/metager/lang/de/privacy.php
index 8a1b5045b..ee88fa83d 100644
--- a/metager/lang/de/privacy.php
+++ b/metager/lang/de/privacy.php
@@ -10,6 +10,10 @@ return [
'title' => 'Grundsätze',
'description' => 'Wir haben uns als gemeinnütziger Verein dem freien Wissenszugang verschrieben. Da wir wissen, dass freie Recherche nicht mit Massenüberwachung vereinbar ist, nehmen wir auch Datenschutz sehr ernst. Wir verarbeiten schon immer nur die Daten, die zum Betrieb unserer Dienste unbedingt nötig sind. Datenschutz ist bei uns immer der Standard. Profiling – also die automatische Erstellung von Nutzerprofilen – betreiben wir nicht.',
],
+ 'stats' => [
+ 'title' => 'Anonyme Statistiken',
+ 'description' => 'Wir arbeiten stets daran unsere Dienste zu verbessern. Um dazu in der Lage zu sein, müssen wir wissen, welche unserer Funktionen verwendet werden. Aus diesem Grund erfassen wir vollkommen anonyme Daten über die Häufigkeit von Seitenaufrufen und die Verwendung einzelner Funktionen unserer Webseiten. Ebenfalls erfassen wir anonymisierte Daten über die Verteilung von Browserarten und Versionen. Diese Statistiken basieren nicht auf einzelnen Nutzerprofilen und werden ohne Cookies oder ähnlichem erstellt. Sie beinhalten keine personenbezogenen Daten.'
+ ],
'contexts' => [
'title' => 'Anfallende Daten nach Kontext',
'metager' => [
diff --git a/metager/lang/en/privacy.php b/metager/lang/en/privacy.php
index 1f81ba0b2..519bc6f6d 100644
--- a/metager/lang/en/privacy.php
+++ b/metager/lang/en/privacy.php
@@ -11,6 +11,10 @@ return [
'title' => 'Principles',
'description' => 'As a non-profit association, we are committed to free access to knowledge. Since we know that free research is not compatible with mass surveillance, we also take data protection very seriously. We have always only processed the data that is absolutely necessary for the operation of our services. Data protection is always our standard. We do not operate profiling – i.e. the automatic creation of user profiles.'
],
+ 'stats' => [
+ 'title' => 'Anonymous statistics',
+ 'description' => 'We are always working to improve our services. To be able to do this, we need to know which of our functions are being used. For this reason, we collect completely anonymous data on the frequency of page views and the use of individual functions on our websites. We also collect anonymized data on the distribution of browser types and versions. These statistics are not based on individual user profiles and are created without cookies or similar technologies. They do not contain any personal data.'
+ ],
'contexts' => [
'title' => 'Incoming data by context',
'metager' => [
diff --git a/metager/resources/views/privacy.blade.php b/metager/resources/views/privacy.blade.php
index 98d586f8f..fc25680b7 100644
--- a/metager/resources/views/privacy.blade.php
+++ b/metager/resources/views/privacy.blade.php
@@ -1,6 +1,6 @@
@extends('layouts.subPages', ['page' => 'privacy'])
-@section('title', trans('titles.datenschutz') )
+@section('title', trans('titles.datenschutz'))
@section('navbarFocus.datenschutz', 'class="active"')
@@ -12,12 +12,18 @@
</div class="section">
<div class="section">
<h1>@lang('privacy.responsible_party.title')</h1>
- <div>@lang('privacy.responsible_party.description', ['link_impress' => route('impress'), 'link_contact' => route('contact')])</div>
+ <div>
+ @lang('privacy.responsible_party.description', ['link_impress' => route('impress'), 'link_contact' => route('contact')])
+ </div>
</div>
<div class="section">
<h1>@lang('privacy.principles.title')</h1>
<div>@lang('privacy.principles.description')</div>
</div>
+ <div class="section">
+ <h1>@lang('privacy.stats.title')</h1>
+ <div>@lang('privacy.stats.description')</div>
+ </div>
<div class="section">
<h1>@lang('privacy.contexts.title')</h1>
<ol>
@@ -28,18 +34,23 @@
<ol class="datum-list">
<li><a href="#ip-address">@lang('privacy.data.ip'):</a> @lang('privacy.data.unused')</li>
<li><a href="#user-agent">@lang('privacy.data.useragent'):</a> @lang('privacy.data.unused')</li>
- <li><a href="#search-request">@lang('privacy.data.query'):</a> @lang('privacy.contexts.metager.query')</li>
- <li><a href="#preferences">@lang('privacy.data.preferences'):</a> @lang('privacy.contexts.metager.preferences')</li>
+ <li><a href="#search-request">@lang('privacy.data.query'):</a>
+ @lang('privacy.contexts.metager.query')</li>
+ <li><a href="#preferences">@lang('privacy.data.preferences'):</a>
+ @lang('privacy.contexts.metager.preferences')</li>
</ol>
<h3>@lang('privacy.contexts.metager.additionally')</h3>
<ol class="datum-list">
<li>
- <div><a href="#ip-address">@lang('privacy.data.ip')</a>, <a href="#user-agent">@lang('privacy.data.useragent')</a>:</div>
+ <div><a href="#ip-address">@lang('privacy.data.ip')</a>, <a
+ href="#user-agent">@lang('privacy.data.useragent')</a>:</div>
<div>@lang('privacy.contexts.metager.botprotection')</div>
</li>
<li>
- <div><a href="https://privacy.microsoft.com/privacystatement">Microsoft Clarity & Yahoo</a></div>
- <div>@lang('privacy.contexts.metager.clarity')</div></li>
+ <div><a href="https://privacy.microsoft.com/privacystatement">Microsoft Clarity & Yahoo</a>
+ </div>
+ <div>@lang('privacy.contexts.metager.clarity')</div>
+ </li>
</ol>
</article>
<article class="kontext">
@@ -48,8 +59,10 @@
<ol class="datum-list">
<li><a href="#ip-address">@lang('privacy.data.ip'):</a> @lang('privacy.data.unused')</li>
<li><a href="#user-agent">@lang('privacy.data.useragent'):</a> @lang('privacy.data.unused')</li>
- <li><a href="#contact-data">@lang('privacy.data.contact'):</a> @lang('privacy.contexts.contact.contact')</li>
- <li><a href="#message">@lang('privacy.data.message'):</a> @lang('privacy.contexts.contact.contact')</li>
+ <li><a href="#contact-data">@lang('privacy.data.contact'):</a>
+ @lang('privacy.contexts.contact.contact')</li>
+ <li><a href="#message">@lang('privacy.data.message'):</a>
+ @lang('privacy.contexts.contact.contact')</li>
</ol>
</article>
<article class="kontext">
@@ -58,9 +71,12 @@
<ol class="datum-list">
<li><a href="#ip-address">@lang('privacy.data.ip'):</a> @lang('privacy.data.unused')</li>
<li><a href="#user-agent">@lang('privacy.data.useragent'):</a> @lang('privacy.data.unused')</li>
- <li><a href="#contact-data">@lang('privacy.data.contact'):</a> @lang('privacy.contexts.donate.contact')</li>
- <li><a href="#payment-data">@lang('privacy.data.payment'):</a> @lang('privacy.contexts.donate.payment')</li>
- <li><a href="#message">@lang('privacy.data.message') (@lang('privacy.data.optional')):</a> @lang('privacy.contexts.donate.message')</li>
+ <li><a href="#contact-data">@lang('privacy.data.contact'):</a>
+ @lang('privacy.contexts.donate.contact')</li>
+ <li><a href="#payment-data">@lang('privacy.data.payment'):</a>
+ @lang('privacy.contexts.donate.payment')</li>
+ <li><a href="#message">@lang('privacy.data.message') (@lang('privacy.data.optional')):</a>
+ @lang('privacy.contexts.donate.message')</li>
</ol>
</article>
<article class="kontext">
@@ -68,8 +84,10 @@
<ol class="datum-list">
<li><a href="#ip-address">@lang('privacy.data.ip'):</a> @lang('privacy.data.unused')</li>
<li><a href="#user-agent">@lang('privacy.data.useragent'):</a> @lang('privacy.data.unused')</li>
- <li><a href="#contact-data">@lang('privacy.data.contact') (@lang('privacy.data.optional')):</a> @lang('privacy.contexts.key.contact')</li>
- <li><a href="#payment-data">@lang('privacy.data.payment'):</a> @lang('privacy.contexts.key.payment')</li>
+ <li><a href="#contact-data">@lang('privacy.data.contact') (@lang('privacy.data.optional')):</a>
+ @lang('privacy.contexts.key.contact')</li>
+ <li><a href="#payment-data">@lang('privacy.data.payment'):</a>
+ @lang('privacy.contexts.key.payment')</li>
</ol>
</article>
<article class="kontext">
@@ -88,7 +106,8 @@
<h1>@lang('privacy.contexts.newsletter.title')</h1>
<div>@lang('privacy.contexts.newsletter.description')</div>
<ol class="datum-list">
- <li><a href="#contact-data">@lang('privacy.data.contact'):</a> @lang('privacy.contexts.newsletter.contact')</li>
+ <li><a href="#contact-data">@lang('privacy.data.contact'):</a>
+ @lang('privacy.contexts.newsletter.contact')</li>
</ol>
</article>
<article class="kontext">
@@ -218,7 +237,7 @@
<ol>
<li><b>@lang('privacy.rights.information.title'):</b></li>
<article class="kontext">
- @lang('privacy.rights.information.description')
+ @lang('privacy.rights.information.description')
</article>
<li><b>@lang('privacy.rights.correction.title'):</b></li>
<article class="kontext">
@@ -257,6 +276,6 @@
<div class="section">
<h1>@lang('privacy.changes.title')</h1>
<div>@lang('privacy.changes.description')</div>
- <div>@lang('privacy.changes.date', ['date' => '2024-06-11'])</div>
+ <div>@lang('privacy.changes.date', ['date' => '2024-07-24'])</div>
</div>
-@endsection
\ No newline at end of file
+ @endsection
\ No newline at end of file
--
GitLab