Merge branch '928-fix-admin-interface' into 'development'

Resolve "Fix Admin Interface"

Closes #928

See merge request !1500

.env.example

@@ -4,6 +4,13 @@ APP_LOG_LEVEL=debug
 APP_KEY=
 APP_URL=http://localhost
 
+DB_CONNECTION=mysql
+DB_HOST=mgdb
+DB_PORT=3306
+DB_DATABASE=metager
+DB_USERNAME=metager
+DB_PASSWORD="metager"
+
 REDIS_RESULT_CONNECTION=default
 REDIS_RESULT_CACHE_DURATION=60
 

Dockerfile

@@ -34,7 +34,6 @@ EXPOSE 80
 COPY config/nginx.conf /etc/nginx/nginx.conf
 COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf
 COPY --chown=root:nginx . /html
-RUN chmod -R g+w storage bootstrap/cache
 
 CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
     chmod -R g+w storage/logs/metager bootstrap/cache && \

app/Console/Commands/WaitDB.php

+<?php
+
+namespace App\Console\Commands;
+
+use DB;
+use Illuminate\Console\Command;
+
+class WaitDB extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'wait:db';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Waits until default DB connection is available. ';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        $starttime = microtime(true);
+
+        while (microtime(true) - $starttime < 60) {
+            try {
+                $connection = DB::connection('mysql')->getPdo();
+                $this->line("Connection to database successfull");
+                return 0;
+            } catch (\Exception $e) {
+                $this->error($e->getMessage());
+                sleep(1);
+            }
+        }
+        return 1;
+    }
+}

app/Http/Controllers/AdminInterface.php

@@ -87,8 +87,48 @@ class AdminInterface extends Controller
         return $names;
     }
 
+    private function convertLogs()
+    {
+        $oldLogsPath = \storage_path("logs/metager/old/");
+        $dir = new \DirectoryIterator($oldLogsPath);
+        foreach ($dir as $fileinfo) {
+            if ($fileinfo->isDot()) {
+                continue;
+            }
+            $filename = $oldLogsPath . "/" . $fileinfo->getFilename();
+            $daysAgo = substr($fileinfo->getFilename(), strrpos($fileinfo->getFilename(), ".") + 1);
+            $dateOfFile = Carbon::now()->subDays($daysAgo);
+            $outputFile = \storage_path("logs/metager/" . $dateOfFile->format("Y/m/d") . ".log");
+            if (!file_exists(dirname($outputFile))) {
+                \mkdir(dirname($outputFile), 0777, true);
+            }
+            $fhw = fopen($outputFile, "w");
+            $fhr = fopen($filename, "r");
+            try {
+                $first = true;
+                while (($line = fgets($fhr)) != false) {
+                    $date = trim(substr($line, 0, strpos($line, "]")), "[");
+                    $date = trim(substr($date, strrpos($date, " ")));
+                    $rest = trim(substr($line, strpos($line, "]") + 1));
+                    $outputString = "";
+                    if (!$first) {
+                        $outputString .= PHP_EOL;
+                    }
+                    $outputString .= $date . " " . $rest;
+                    $first = false;
+                    fwrite($fhw, $outputString);
+                }
+            } finally {
+                fclose($fhw);
+                fclose($fhr);
+            }
+        }
+    }
+
     public function count(Request $request)
     {
+        #$this->convertLogs();
+        #return;
         $days = intval($request->input('days', 28));
         $interface = $request->input('interface', 'all');
         if (!is_int($days) || $days <= 0) {
@@ -98,6 +138,7 @@ class AdminInterface extends Controller
 
         $oldLogs = [];
         $rekordTag = 0;
+        $rekordTagSameTime = 0;
         $minCount = 0;
         $rekordTagDate = "";
         $size = 0;
@@ -249,14 +290,14 @@ class AdminInterface extends Controller
             $days = $maxDate->diffInDays(Carbon::now());
         }
 
-        $logToday = "/var/log/metager/mg3.log";
+        $logToday = \App\MetaGer::getMGLogFile();
 
-        $archivePath = "/var/log/metager/archive/";
+        $archivePath = storage_path("logs/metager/");
 
         $today = [
             'logFile' => $logToday,
-            'countPath' => storage_path('logs/count/'),
-            'countFile' => storage_path('logs/count/' . getmypid()),
+            'countPath' => storage_path('logs/metager/count/'),
+            'countFile' => storage_path('logs/metager/count/' . getmypid()),
         ];
         if (\file_exists($today["countFile"])) {
             unlink($today["countFile"]);
@@ -271,10 +312,10 @@ class AdminInterface extends Controller
         $requestedLogs = [];
         for ($i = 1; $i <= $days; $i++) {
             $date = Carbon::now()->subDays($i);
-            $countPath = storage_path('logs/count/' . $date->year . "/" . $date->month . "/");
+            $countPath = storage_path('logs/metager/count/' . $date->format("Y/m") . "/");
             $countFile = $countPath . $date->day . ".json";
             $neededLogs[$i] = [
-                'logFile' => $archivePath . "mg3.log.$i",
+                'logFile' => $archivePath . $date->format("Y/m/d") . ".log",
                 'countPath' => $countPath,
                 'countFile' => $countFile,
             ];

app/Jobs/ConvertCountFile.php

@@ -39,6 +39,7 @@ class ConvertCountFile implements ShouldQueue
         ];
         $fh = false;
         $fullRound = false;
+        $error = false;
         try {
             $fh = fopen($this->files["logFile"], "r");
             $currentLogTime = Carbon::now()->hour(0)->minute(0)->second(0)->addMinutes(5);
@@ -47,7 +48,7 @@ class ConvertCountFile implements ShouldQueue
                 $logTime = [];
                 $interface = "";
                 // i.e. [Wed Apr 17 00:00:01] ref=https://metager.de/ time=0.51 serv=web interface=de
-                if (preg_match('/\[[a-zA-z]{3}\s[a-zA-Z]{3}\s\d{2}\s(\d{2}:\d{2}:\d{2}).*?\sinterface=(\S+)/', $line, $matches)) {
+                if (preg_match('/(\d{2}:\d{2}:\d{2}).*?\sinterface=(\S+)/', $line, $matches)) {
                     // Create Date Object
                     $logTime = explode(":", $matches[1]);
                     $interface = $matches[2];
@@ -84,18 +85,22 @@ class ConvertCountFile implements ShouldQueue
             if (empty($result["time"][$currentLogTime->format('H:i')])) {
                 $result["time"][$currentLogTime->format('H:i')] = $result["insgesamt"];
             }
+        } catch (\ErrorException $e) {
+            $error = true;
         } finally {
             if ($fh !== false) {
                 fclose($fh);
             }
 
-            $oldUmask = umask(0);
-            // Write the result to a File
-            if (!file_exists($this->files["countPath"])) {
-                mkdir($this->files["countPath"], 0777, true);
+            if (!$error) {
+                $oldUmask = umask(0);
+                // Write the result to a File
+                if (!file_exists($this->files["countPath"])) {
+                    mkdir($this->files["countPath"], 0777, true);
+                }
+                file_put_contents($this->files["countFile"], json_encode($result, JSON_PRETTY_PRINT));
+                umask($oldUmask);
             }
-            file_put_contents($this->files["countFile"], json_encode($result, JSON_PRETTY_PRINT));
-            umask($oldUmask);
 
             Redis::del(md5($this->files["countFile"]));
         }

app/MetaGer.php

@@ -1338,11 +1338,11 @@ class MetaGer
 
     public static function getMGLogFile()
     {
-        $logpath = storage_path("logs/metager/");
+        $logpath = storage_path("logs/metager/" . date("Y") . "/" . date("m") . "/");
         if (!file_exists($logpath)) {
             mkdir($logpath, 0777, true);
         }
-        $logpath .= (new \DateTime())->format('Y-m-d') . ".log";
+        $logpath .= date("d") . ".log";
         return $logpath;
     }
 
@@ -1351,19 +1351,7 @@ class MetaGer
         if ($this->shouldLog) {
             try {
                 $logEntry = "";
-                $logEntry .= "[" . date("D M d H:i:s") . "]";
-                /*
-                Someone might wonder now why we are saving the IP-Adress to the log file here:
-                It's because we were targets of heavy Bot attacks which created so many Search-Request to our Servers that
-                not only our servers but the ones from some of our search engines too collapsed.
-                At that point we could'nt prevent the Bot from accessing our server because we would need it's IP-Adress to do so.
-
-                That's why we need to save the IP-Adress to our Log-Files temporarily. The logrotate process that shifts our Log-Files will then
-                automatically remove the IP-Adresses from the Log-File after a few hours.
-                This method gives us just enough time to prevent malicious Software from bringing our servers down and at the same time having not a single
-                IP-Adress older than one day stored on our servers. (Except the ones who got banned in that short period of course) ;-)
-                 */
-                $logEntry .= " ip=" . $this->request->ip();
+                $logEntry .= date("H:s:i");
                 $logEntry .= " ref=" . $this->request->header('Referer');
                 $logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus;
                 $logEntry .= " interface=" . LaravelLocalization::getCurrentLocale();

database/seeds/UsersSeeder.php

@@ -11,6 +11,13 @@ class UsersSeeder extends Seeder
      */
     public function run()
     {
-
+        DB::table('users')->truncate();
+        DB::table('users')->insert(
+            [
+                'email' => 'admin',
+                'name' => 'admin',
+                'password' => Hash::make('admin'),
+            ]
+        );
     }
 }

docker-compose.yml

@@ -12,17 +12,29 @@ services:
     working_dir: /usr/src/app
     command: bash -c "npm install && npm run watch"
   dependencies:
-    image: alpine:latest
+    depends_on:
+      - "mgdb"
+    image: php:7.3-cli
     volumes:
       - .:/data
     working_dir: /data
     command: /data/init.sh
+  mgdb:
+    restart: unless-stopped
+    image: mariadb:latest
+    command: --default-authentication-plugin=mysql_native_password
+    environment:
+      - MYSQL_RANDOM_ROOT_PASSWORD=yes
+      - MYSQL_USER=metager
+      - MYSQL_PASSWORD=metager
+      - MYSQL_DATABASE=metager
   web:
     depends_on:
+      - "mgdb"
       - "dependencies"
       - "phpdeps"
       - "assets"
-    restart: on-failure
+    restart: unless-stopped
     build: .
     working_dir: /html
     volumes:

init.sh

 #!/bin/sh
 
+# This commands will help initialize data for docker-compose setup
+# Its supposed to run in a php docker image
+docker-php-ext-install pdo pdo_mysql
+
 if [ ! -f "/data/.env" ]; then
     cp /data/.env.example /data/.env
 fi
@@ -7,6 +11,11 @@ fi
 if [ -f "/data/database/useragents.sqlite" ]; then
     rm /data/database/useragents.sqlite
 fi
-cp /data/database/useragents.sqlite.example /data/database/useragents.sqlite
 
-chmod -R go+w storage bootstrap/cache
+touch /data/database/useragents.sqlite
\ No newline at end of file
+
+chmod -R go+w storage bootstrap/cache
+
+php artisan wait:db
+php artisan migrate
+php artisan db:seed
\ No newline at end of file