Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
open-source
MetaGer
Commits
799030b3
Commit
799030b3
authored
Apr 26, 2019
by
Dominik Hebeler
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
count is now faster and can parse all logs
parent
ac0a48a0
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
190 additions
and
60 deletions
+190
-60
app/Http/Controllers/AdminInterface.php
app/Http/Controllers/AdminInterface.php
+88
-59
app/Jobs/ConvertCountFile.php
app/Jobs/ConvertCountFile.php
+101
-0
routes/web.php
routes/web.php
+1
-1
No files found.
app/Http/Controllers/AdminInterface.php
View file @
799030b3
...
...
@@ -2,11 +2,11 @@
namespace
App\Http\Controllers
;
use
App\Jobs\ConvertCountFile
;
use
Carbon\Carbon
;
use
Illuminate\Http\Request
;
use
Illuminate\Support\Facades\Redis
;
use
Response
;
use
\
SplFileObject
;
class
AdminInterface
extends
Controller
{
...
...
@@ -95,46 +95,113 @@ class AdminInterface extends Controller
$days
=
28
;
}
$maxDate
=
Carbon
::
createFromFormat
(
'd.m.Y'
,
"28.06.2016"
);
$selectedDate
=
Carbon
::
now
()
->
subDays
(
$days
);
if
(
$maxDate
>
$selectedDate
)
{
$days
=
$maxDate
->
diffInDays
(
Carbon
::
now
());
}
$logToday
=
"/var/log/metager/mg3.log"
;
if
(
file_exists
(
$logToday
))
{
if
(
$interface
===
"all"
)
{
$logToday
=
exec
(
"cat
$logToday
| wc -l"
);
$archivePath
=
"/var/log/metager/archive/"
;
$today
=
[
'logFile'
=>
$logToday
,
'countPath'
=>
storage_path
(
'logs/count/'
),
'countFile'
=>
storage_path
(
'logs/count/'
.
getmypid
()),
];
if
(
\
file_exists
(
$today
[
"countFile"
]))
{
unlink
(
$today
[
"countFile"
]);
}
$neededLogs
=
[
0
=>
$today
,
];
$logsToRequest
=
[
0
=>
$today
,
];
$requestedLogs
=
[];
for
(
$i
=
1
;
$i
<=
$days
;
$i
++
)
{
$date
=
Carbon
::
now
()
->
subDays
(
$i
);
$countPath
=
storage_path
(
'logs/count/'
.
$date
->
year
.
"/"
.
$date
->
month
.
"/"
);
$countFile
=
$countPath
.
$date
->
day
.
".json"
;
$neededLogs
[
$i
]
=
[
'logFile'
=>
$archivePath
.
"mg3.log.
$i
"
,
'countPath'
=>
$countPath
,
'countFile'
=>
$countFile
,
];
if
(
!
file_exists
(
$neededLogs
[
$i
][
'countFile'
]))
{
$logsToRequest
[
$i
]
=
$neededLogs
[
$i
];
}
}
if
(
sizeof
(
$logsToRequest
)
>
100
)
{
set_time_limit
(
600
);
}
// Create the Jobs for count file creation
while
(
sizeof
(
$logsToRequest
)
>
0
||
sizeof
(
$requestedLogs
)
>
0
)
{
if
(
sizeof
(
$requestedLogs
)
<
20
&&
sizeof
(
$logsToRequest
)
>
0
)
{
$newJob
=
array_shift
(
$logsToRequest
);
$newJob
[
"startedAt"
]
=
time
();
$requestedLogs
[]
=
$newJob
;
Redis
::
set
(
md5
(
$newJob
[
"countFile"
]),
"running"
);
Redis
::
expire
(
md5
(
$newJob
[
"countFile"
]),
15
);
ConvertCountFile
::
dispatch
(
$newJob
);
}
else
{
$logToday
=
exec
(
"cat
$logToday
| grep interface="
.
$interface
.
" | wc -l"
);
usleep
(
50000
);
}
}
else
{
return
redirect
(
''
);
// Remove all finished Jobs
$removedOne
=
false
;
do
{
$removedOne
=
false
;
foreach
(
$requestedLogs
as
$index
=>
$requestedLog
)
{
if
(
!
Redis
::
exists
(
md5
(
$requestedLog
[
"countFile"
])))
{
unset
(
$requestedLogs
[
$index
]);
$removedOne
=
true
;
break
;
}
}
}
while
(
$removedOne
===
true
);
}
$oldLogs
=
[];
$rekordTag
=
0
;
$rekordTagDate
=
""
;
$size
=
0
;
$count
=
0
;
for
(
$i
=
1
;
$i
<=
$days
;
$i
++
)
{
$logDate
=
"/var/log/metager/archive/mg3.log.
$i
"
;
if
(
file_exists
(
$logDate
))
{
$now
=
Carbon
::
now
();
$sameTimeLine
=
$this
->
findLineForTime
(
$logDate
,
Carbon
::
now
());
if
(
$interface
===
"all"
)
{
$sameTime
=
exec
(
"head -n "
.
$sameTimeLine
.
" "
.
$logDate
.
" | wc -l"
);
$insgesamt
=
exec
(
"wc -l
$logDate
| cut -f1 -d' '"
);
}
else
{
$sameTime
=
exec
(
"head -n "
.
$sameTimeLine
.
" "
.
$logDate
.
" | grep interface="
.
$interface
.
" | wc -l"
);
$insgesamt
=
exec
(
"cat
$logDate
| grep interface="
.
$interface
.
" | wc -l"
);
$now
=
Carbon
::
now
()
->
subMinutes
(
Carbon
::
now
()
->
minute
%
5
)
->
format
(
'H:i'
);
if
(
$now
===
"00:00"
)
{
$now
=
"00:05"
;
}
foreach
(
$neededLogs
as
$key
=>
$value
)
{
$countFile
=
$value
[
"countFile"
];
if
(
file_exists
(
$countFile
))
{
$stats
=
json_decode
(
file_get_contents
(
$countFile
));
if
(
$key
===
0
)
{
// Log for today
$logToday
=
empty
(
$stats
->
insgesamt
->
{
$interface
})
?
0
:
$stats
->
insgesamt
->
{
$interface
};
if
(
\
file_exists
(
$today
[
"countFile"
]))
{
unlink
(
$today
[
"countFile"
]);
}
continue
;
}
$insgesamt
=
empty
(
$stats
->
insgesamt
->
{
$interface
})
?
0
:
$stats
->
insgesamt
->
{
$interface
};
$sameTime
=
empty
(
$stats
->
time
->
{
$now
}
->
{
$interface
})
?
0
:
$stats
->
time
->
{
$now
}
->
{
$interface
};
if
(
$insgesamt
>
$rekordTag
)
{
$rekordTag
=
$insgesamt
;
$rekordTagSameTime
=
$sameTime
;
$rekordTagDate
=
date
(
"d.m.Y"
,
mktime
(
date
(
"H"
),
date
(
"i"
),
date
(
"s"
),
date
(
"m"
),
date
(
"d"
)
-
$i
,
date
(
"Y"
)));
}
$oldLogs
[
$
i
][
'sameTime'
]
=
number_format
(
floatval
(
$sameTime
),
0
,
","
,
"."
);
$oldLogs
[
$
i
][
'insgesamt'
]
=
number_format
(
floatval
(
$insgesamt
),
0
,
","
,
"."
);
$oldLogs
[
$
key
][
'sameTime'
]
=
number_format
(
floatval
(
$sameTime
),
0
,
","
,
"."
);
$oldLogs
[
$
key
][
'insgesamt'
]
=
number_format
(
floatval
(
$insgesamt
),
0
,
","
,
"."
);
# Nun noch den median:
$count
+=
$insgesamt
;
$size
++
;
if
(
$size
>
0
)
{
$oldLogs
[
$
i
][
'median'
]
=
number_format
(
floatval
(
round
(
$count
/
$size
)),
0
,
","
,
"."
);
$oldLogs
[
$
key
][
'median'
]
=
number_format
(
floatval
(
round
(
$count
/
$size
)),
0
,
","
,
"."
);
}
}
...
...
@@ -166,44 +233,6 @@ class AdminInterface extends Controller
}
private
function
findLineForTime
(
$file
,
$time
)
{
$file
=
new
SplFileObject
(
$file
);
$file
->
seek
(
PHP_INT_MAX
);
$numberOfRows
=
$file
->
key
();
$minLine
=
1
;
$maxLine
=
$numberOfRows
;
$current
=
round
(
$maxLine
/
2
);
$finished
=
false
;
$counter
=
0
;
while
(
!
$finished
)
{
$counter
++
;
if
(
$counter
==
1000
)
{
return
$maxLine
;
}
$current
=
$minLine
+
round
((
$maxLine
-
$minLine
)
/
2
);
$file
->
seek
(
$current
);
$line
=
$file
->
fgets
();
# Extract time from line
$line
=
substr
(
$line
,
1
);
$line
=
substr
(
$line
,
0
,
strpos
(
$line
,
"]"
));
$lineTime
=
Carbon
::
createFromFormat
(
'D M d H:i:s'
,
$line
)
->
day
(
date
(
'd'
))
->
month
(
date
(
'm'
))
->
year
(
date
(
'Y'
));
if
((
$maxLine
-
$minLine
)
<=
1
)
{
return
$maxLine
;
}
else
if
(
$time
<
$lineTime
)
{
$maxLine
=
$current
;
continue
;
}
else
if
(
$time
>
$lineTime
)
{
$minLine
=
$current
;
continue
;
}
}
}
public
function
check
()
{
$q
=
""
;
...
...
app/Jobs/ConvertCountFile.php
0 → 100644
View file @
799030b3
<?php
namespace
App\Jobs
;
use
Carbon
;
use
Illuminate\Bus\Queueable
;
use
Illuminate\Contracts\Queue\ShouldQueue
;
use
Illuminate\Foundation\Bus\Dispatchable
;
use
Illuminate\Queue\InteractsWithQueue
;
use
Illuminate\Queue\SerializesModels
;
use
Illuminate\Support\Facades\Redis
;
class
ConvertCountFile
implements
ShouldQueue
{
use
Dispatchable
,
InteractsWithQueue
,
Queueable
,
SerializesModels
;
private
$files
;
/**
* Create a new job instance.
*
* @return void
*/
public
function
__construct
(
$files
)
{
$this
->
files
=
$files
;
}
/**
* Execute the job.
*
* @return void
*/
public
function
handle
()
{
$result
=
[];
$fh
=
false
;
$fullRound
=
false
;
try
{
$fh
=
fopen
(
$this
->
files
[
"logFile"
],
"r"
);
$currentLogTime
=
Carbon
::
now
()
->
hour
(
0
)
->
minute
(
0
)
->
second
(
0
)
->
addMinutes
(
5
);
while
(
$fh
!==
false
&&
(
$line
=
fgets
(
$fh
))
!==
false
)
{
$logTime
=
[];
$interface
=
""
;
// i.e. [Wed Apr 17 00:00:01] ref=https://metager.de/ time=0.51 serv=web interface=de
if
(
preg_match
(
'/\[[a-zA-z]{3}\s[a-zA-Z]{3}\s\d{2}\s(\d{2}:\d{2}:\d{2}).*?\sinterface=(\S+)/'
,
$line
,
$matches
))
{
// Create Date Object
$logTime
=
explode
(
":"
,
$matches
[
1
]);
$interface
=
$matches
[
2
];
}
else
if
(
preg_match
(
'/\[[a-zA-z]{3},\s\d{2}\s[a-zA-Z]{3}\s\d{2}\s(\d{2}:\d{2}:\d{2})/'
,
$line
,
$matches
))
{
$logTime
=
explode
(
":"
,
$matches
[
1
]);
}
else
if
(
preg_match
(
'/\[\d{2}\/\d{2}\/\d{4}:(\d{2}:\d{2}:\d{2})/'
,
$line
,
$matches
))
{
$logTime
=
explode
(
":"
,
$matches
[
1
]);
}
else
{
continue
;
}
while
(
!
$fullRound
&&
((
intval
(
$logTime
[
0
])
*
60
)
+
intval
(
$logTime
[
1
]))
>
((
$currentLogTime
->
hour
*
60
)
+
$currentLogTime
->
minute
))
{
if
(
empty
(
$result
[
"insgesamt"
]))
{
continue
2
;
}
$result
[
"time"
][
$currentLogTime
->
format
(
'H:i'
)]
=
$result
[
"insgesamt"
];
$currentLogTime
->
addMinutes
(
5
);
if
(
$currentLogTime
->
hour
===
0
&&
$currentLogTime
->
minute
===
0
)
{
$fullRound
=
true
;
}
}
// Update the total statistics
if
(
empty
(
$result
[
"insgesamt"
][
"all"
]))
{
$result
[
"insgesamt"
][
"all"
]
=
1
;
}
else
{
$result
[
"insgesamt"
][
"all"
]
++
;
}
if
(
!
empty
(
$interface
))
{
if
(
empty
(
$result
[
"insgesamt"
][
$interface
]))
{
$result
[
"insgesamt"
][
$interface
]
=
1
;
}
else
{
$result
[
"insgesamt"
][
$interface
]
++
;
}
}
}
if
(
empty
(
$result
[
"time"
][
$currentLogTime
->
format
(
'H:i'
)]))
{
$result
[
"time"
][
$currentLogTime
->
format
(
'H:i'
)]
=
$result
[
"insgesamt"
];
}
}
finally
{
if
(
$fh
!==
false
)
{
fclose
(
$fh
);
}
$oldUmask
=
umask
(
0
);
// Write the result to a File
if
(
!
file_exists
(
$this
->
files
[
"countPath"
]))
{
mkdir
(
$this
->
files
[
"countPath"
],
0777
,
true
);
}
file_put_contents
(
$this
->
files
[
"countFile"
],
json_encode
(
$result
,
JSON_PRETTY_PRINT
));
umask
(
$oldUmask
);
Redis
::
del
(
md5
(
$this
->
files
[
"countFile"
]));
}
}
}
routes/web.php
View file @
799030b3
...
...
@@ -167,7 +167,7 @@ Route::group(
->
with
(
'request'
,
$this
->
input
(
'request'
,
'GET'
));
});
Route
::
group
([
'middleware'
=>
[
'referer.check'
],
'prefix'
=>
'admin'
],
function
()
{
Route
::
group
([
/*
'middleware' => ['referer.check'],
*/
'prefix'
=>
'admin'
],
function
()
{
Route
::
get
(
'/'
,
'AdminInterface@index'
);
Route
::
match
([
'get'
,
'post'
],
'count'
,
'AdminInterface@count'
);
Route
::
get
(
'check'
,
'AdminInterface@check'
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment