Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
open-source
MetaGer
Commits
29aafe9f
Commit
29aafe9f
authored
Feb 19, 2019
by
Dominik Hebeler
Browse files
Added the option to auto disable search engines
parent
54e2b9d5
Changes
6
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
app/Console/Commands/LogRotate.php
deleted
100644 → 0
View file @
54e2b9d5
<?php
namespace
App\Console\Commands
;
use
Illuminate\Console\Command
;
use
Illuminate\Support\Facades\Redis
;
class
LogRotate
extends
Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected
$signature
=
'log:rotate'
;
/**
* The console command description.
*
* @var string
*/
protected
$description
=
'Dieses Kommando bezieht alle Log-Einträge aus dem Redis System und exportiert diese in entsprechende Dateien.'
;
/**
* Create a new command instance.
*
* @return void
*/
public
function
__construct
()
{
parent
::
__construct
();
}
/**
* Execute the console command.
*
* @return mixed
*/
public
function
handle
()
{
# Wir extrahieren die Logs mit den Daten der einzelnen Suchmaschinen:
# In den Daten ist festgehalten, wie oft eine jede Suchmaschine abgefragt wurde
# und wie häufig diese geantwortet hat:
$this
->
extractSearchEngineLogs
();
}
private
function
extractSearchEngineLogs
()
{
$redis
=
Redis
::
connection
(
'redisLogs'
);
# Hier legen wir das Ergebnis ab:
# [
# 'fastbot' => [
# ''
# ]
# ]
$searchEngineLogs
=
[
'recent'
=>
[],
'overall'
=>
[]];
try
{
# Wir benötigen zunächst die Namen aller Suchmaschinen:
$sumasFile
=
config_path
()
.
"/sumas.xml"
;
$xml
=
simplexml_load_file
(
$sumasFile
);
$xml
=
$xml
->
xpath
(
"suma"
);
$sumaNames
=
[];
foreach
(
$xml
as
$suma
)
{
$searchEngineLogs
[
'recent'
][
$suma
[
"name"
]
->
__toString
()]
=
[
"requests"
=>
0
,
"answered"
=>
0
];
}
#Auslesen/hinzufügen der Daten:
foreach
(
$searchEngineLogs
[
'recent'
]
as
$name
=>
$values
)
{
$searchEngineLogs
[
'recent'
][
$name
][
"requests"
]
=
$redis
->
getset
(
'logs.engines.requests.'
.
$name
,
0
);
$searchEngineLogs
[
'recent'
][
$name
][
"answered"
]
=
$redis
->
getset
(
'logs.engines.answered.'
.
$name
,
0
);
}
$filePath
=
"/var/log/metager/"
;
# Wir haben nun die aktuellen daten und müssen diese noch mit den aktuellen Kombinieren:
# Hierbei behalten wir die Daten, seid dem letzten Rotate so wie sie sind und verknüpfen diese mit einer gesamt Statistik.
if
(
file_exists
(
$filePath
.
"engine.log"
)
&&
is_readable
(
$filePath
.
"engine.log"
))
{
$oldData
=
file_get_contents
(
$filePath
.
"engine.log"
);
# JSON Decode
$oldData
=
json_decode
(
$oldData
,
true
);
if
(
isset
(
$oldData
[
'overall'
]))
{
$searchEngineLogs
[
'overall'
]
=
$oldData
[
'overall'
];
}
}
# Jetzt fügen wir zu den Gesamtdaten die zuletzt erfassten hinzu:
foreach
(
$searchEngineLogs
[
'recent'
]
as
$name
=>
$values
)
{
if
(
isset
(
$searchEngineLogs
[
'overall'
][
$name
][
"requests"
]))
{
$searchEngineLogs
[
'overall'
][
$name
][
"requests"
]
+=
$values
[
"requests"
];
}
else
{
$searchEngineLogs
[
'overall'
][
$name
][
"requests"
]
=
$values
[
"requests"
];
}
if
(
isset
(
$searchEngineLogs
[
'overall'
][
$name
][
"answered"
]))
{
$searchEngineLogs
[
'overall'
][
$name
][
"answered"
]
+=
$values
[
"answered"
];
}
else
{
$searchEngineLogs
[
'overall'
][
$name
][
"answered"
]
=
$values
[
"answered"
];
}
}
# Ins JSON Format umwandeln:
$searchEngineLogs
=
json_encode
(
$searchEngineLogs
,
JSON_PRETTY_PRINT
);
# Und in die Datei sichern
if
(((
file_exists
(
$filePath
.
"engine.log"
)
&&
is_writable
(
$filePath
.
"engine.log"
))
||
(
!
file_exists
(
$filePath
.
"engine.log"
)
&&
is_writable
(
$filePath
)))
&&
file_put_contents
(
$filePath
.
"engine.log"
,
$searchEngineLogs
)
!==
false
)
{
$this
->
info
(
"Logs der Suchmaschinen erfolgreich exportiert."
);
return
;
}
else
{
# Der Schreibvorgang war nicht erfolgreich. Wir packen die Daten zurück
foreach
(
json_decode
(
$searchEngineLogs
,
true
)
as
$name
=>
$values
)
{
$redis
->
incrby
(
'logs.engines.requests.'
.
$name
,
$values
[
"requests"
]);
$redis
->
incrby
(
'logs.engines.answered.'
.
$name
,
$values
[
"answered"
]);
}
$this
->
info
(
"Konnte die Datei
\"
$filePath
"
.
"engine.log
\"
nicht erstellen. Keine Rechte."
);
return
;
}
}
catch
(
\
ErrorException
$e
)
{
return
;
}
}
}
app/Console/Commands/MonthlyRequestsGather.php
0 → 100644
View file @
29aafe9f
<?php
namespace
App\Console\Commands
;
use
DB
;
use
Illuminate\Console\Command
;
use
Illuminate\Support\Facades\Redis
;
class
MonthlyRequestsGather
extends
Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected
$signature
=
'requests:gather'
;
/**
* The console command description.
*
* @var string
*/
protected
$description
=
'Sends the gathered monthly requests from the redis to the central Database'
;
private
$values
=
[];
/**
* Create a new command instance.
*
* @return void
*/
public
function
__construct
()
{
parent
::
__construct
();
}
/**
* Execute the console command.
*
* @return mixed
*/
public
function
handle
()
{
# Read in the suma Files
$sumaFile
=
config_path
(
'sumas.json'
);
$sumaEnFile
=
config_path
(
'sumasEn.json'
);
if
(
file_exists
(
$sumaFile
)
&&
file_exists
(
$sumaEnFile
))
{
$sumas
=
json_decode
(
file_get_contents
(
$sumaFile
));
$sumasEn
=
json_decode
(
file_get_contents
(
$sumaEnFile
));
$this
->
gatherLogs
(
$sumas
);
$this
->
gatherLogs
(
$sumasEn
);
foreach
(
$this
->
values
as
$name
=>
$value
)
{
$entry
=
DB
::
table
(
'monthlyrequests'
)
->
where
([
'name'
=>
$name
])
->
first
();
$newCount
=
$value
;
if
(
$entry
===
null
)
{
DB
::
table
(
'monthlyrequests'
)
->
insert
([
'name'
=>
$name
,
'count'
=>
$newCount
]);
}
else
{
$newCount
=
$value
+
$entry
->
count
;
DB
::
table
(
'monthlyrequests'
)
->
where
([
'name'
=>
$name
])
->
update
([
'count'
=>
$newCount
]);
}
}
$this
->
disableOverusedEngines
(
$sumaFile
,
$sumas
);
$this
->
disableOverusedEngines
(
$sumaEnFile
,
$sumasEn
);
}
}
private
function
disableOverusedEngines
(
$sumasFile
,
$sumas
)
{
$currentValues
=
DB
::
table
(
'monthlyrequests'
)
->
get
();
foreach
(
$sumas
->
sumas
as
$sumaName
=>
$suma
)
{
if
(
!
empty
(
$suma
->
disabled
)
&&
$suma
->
disabled
===
true
)
{
continue
;
}
if
(
empty
(
$suma
->
{
"monthly-requests"
}))
{
continue
;
}
$currentValue
=
0
;
foreach
(
$currentValues
as
$value
)
{
if
(
$value
->
name
===
$sumaName
)
{
$currentValue
=
$value
->
count
;
}
}
$monthlyRequests
=
$suma
->
{
"monthly-requests"
};
if
(
!
empty
(
$suma
->
{
"auto-disabled"
})
&&
$suma
->
{
"auto-disabled"
}
===
true
)
{
# Maybe this engine should be enabled again
if
(
$currentValue
<
$monthlyRequests
)
{
unset
(
$sumas
->
sumas
->
{
$sumaName
}
->
{
"auto-disabled"
});
file_put_contents
(
$sumasFile
,
json_encode
(
$sumas
,
JSON_PRETTY_PRINT
));
}
}
else
if
(
$currentValue
>=
$monthlyRequests
)
{
$sumas
->
sumas
->
{
$sumaName
}
->
{
"auto-disabled"
}
=
true
;
file_put_contents
(
$sumasFile
,
json_encode
(
$sumas
,
JSON_PRETTY_PRINT
));
}
}
}
private
function
gatherLogs
(
$sumas
)
{
foreach
(
$sumas
->
sumas
as
$sumaName
=>
$suma
)
{
if
(
!
empty
(
$suma
->
{
"monthly-requests"
}))
{
$monthlyLimit
=
$suma
->
{
"monthly-requests"
};
$currentValue
=
Redis
::
get
(
'monthlyRequests:'
.
$sumaName
);
Redis
::
del
(
'monthlyRequests:'
.
$sumaName
);
if
(
empty
(
$currentValue
))
{
$currentValue
=
0
;
}
if
(
empty
(
$this
->
values
[
$sumaName
])
&&
$currentValue
>
0
)
{
$this
->
values
[
$sumaName
]
=
intval
(
$currentValue
);
}
}
}
}
}
app/Console/Kernel.php
View file @
29aafe9f
...
...
@@ -25,10 +25,14 @@ class Kernel extends ConsoleKernel
*/
protected
function
schedule
(
Schedule
$schedule
)
{
$schedule
->
command
(
'log:rotate'
)
->
everyTenMinutes
();
$schedule
->
command
(
'requests:gather'
)
->
everyFifteenMinutes
();
$schedule
->
call
(
function
()
{
DB
::
table
(
'monthlyrequests'
)
->
truncate
();
})
->
monthlyOn
(
1
,
'00:00'
);
// Delete all of the old humanverification entries
$schedule
->
call
(
function
()
{
$schedule
->
call
(
function
()
{
DB
::
delete
(
'DELETE FROM humanverification WHERE updated_at < (now() - interval 72 hour) AND whitelist = 0 ORDER BY updated_at DESC'
);
DB
::
delete
(
'DELETE FROM humanverification WHERE updated_at < (now() - interval 2 week) AND whitelist = 1 ORDER BY updated_at DESC'
);
})
->
everyThirtyMinutes
();
...
...
@@ -42,6 +46,6 @@ class Kernel extends ConsoleKernel
protected
function
commands
()
{
require
base_path
(
'routes/console.php'
);
$this
->
load
(
__DIR__
.
'/Commands'
);
$this
->
load
(
__DIR__
.
'/Commands'
);
}
}
app/MetaGer.php
View file @
29aafe9f
...
...
@@ -542,7 +542,8 @@ class MetaGer
foreach
(
$this
->
sumaFile
->
foki
->
{
$this
->
fokus
}
->
sumas
as
$suma
)
{
# Check if this engine is disabled and can't be used
$disabled
=
empty
(
$this
->
sumaFile
->
sumas
->
{
$suma
}
->
disabled
)
?
false
:
$this
->
sumaFile
->
sumas
->
{
$suma
}
->
disabled
;
if
(
$disabled
)
{
$autoDisabled
=
empty
(
$this
->
sumaFile
->
sumas
->
{
$suma
}
->
{
"auto-disabled"
})
?
false
:
$this
->
sumaFile
->
sumas
->
{
$suma
}
->
{
"auto-disabled"
};
if
(
$disabled
||
$autoDisabled
)
{
continue
;
}
...
...
@@ -710,6 +711,11 @@ class MetaGer
foreach
(
$this
->
sumaFile
->
foki
->
{
$this
->
fokus
}
->
sumas
as
$suma
)
{
if
(
$this
->
sumaFile
->
sumas
->
{
$suma
}
->
{
"filter-opt-in"
})
{
if
(
!
empty
(
$filter
->
sumas
->
{
$suma
}))
{
# If the searchengine is disabled this filter shouldn't be available
if
((
!
empty
(
$this
->
sumaFile
->
sumas
->
{
$suma
}
->
disabled
)
&&
$this
->
sumaFile
->
sumas
->
{
$suma
}
->
disabled
===
true
)
||
(
!
empty
(
$this
->
sumaFile
->
sumas
->
{
$suma
}
->
{
"auto-disabled"
})
&&
$this
->
sumaFile
->
sumas
->
{
$suma
}
->
{
"auto-disabled"
}
===
true
))
{
continue
;
}
$availableFilter
[
$filterName
]
=
$filter
;
}
}
...
...
@@ -1343,16 +1349,7 @@ class MetaGer
# 2 Arten von Logs in einem wird die Anzahl der Abfragen an eine Suchmaschine gespeichert und in der anderen
# die Anzahl, wie häufig diese Ergebnisse geliefert hat.
$enginesToLoad
=
$this
->
enginesToLoad
;
$redis
->
pipeline
(
function
(
$pipe
)
use
(
$enginesToLoad
,
$logEntry
)
{
$pipe
->
rpush
(
'logs.search'
,
$logEntry
);
foreach
(
$this
->
enginesToLoad
as
$name
=>
$answered
)
{
$pipe
->
incr
(
'logs.engines.requests.'
.
$name
);
if
(
$answered
)
{
$pipe
->
incr
(
'logs.engines.answered.'
.
$name
);
}
}
});
$redis
->
rpush
(
'logs.search'
,
$logEntry
);
}
catch
(
\
Exception
$e
)
{
return
;
}
...
...
app/Models/Searchengine.php
View file @
29aafe9f
...
...
@@ -134,6 +134,12 @@ abstract class Searchengine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis
::
rpush
(
$this
->
name
.
".queue"
,
$mission
);
// The request is not cached and will be submitted to the searchengine
// We need to check if the number of requests to this engine are limited
if
(
!
empty
(
$this
->
engine
->
{
"monthly-requests"
}))
{
Redis
::
incr
(
"monthlyRequests:"
.
$this
->
name
);
}
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
...
...
database/migrations/2019_02_19_094337_monthlyrequests.php
0 → 100644
View file @
29aafe9f
<?php
use
Illuminate\Database\Migrations\Migration
;
use
Illuminate\Database\Schema\Blueprint
;
use
Illuminate\Support\Facades\Schema
;
class
Monthlyrequests
extends
Migration
{
/**
* Run the migrations.
*
* @return void
*/
public
function
up
()
{
Schema
::
create
(
'monthlyrequests'
,
function
(
Blueprint
$table
)
{
$table
->
string
(
'name'
)
->
unique
();
$table
->
integer
(
'count'
);
$table
->
timestamp
(
'updated_at'
);
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public
function
down
()
{
Schema
::
drop
(
'sponsorenlinks'
);
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment