Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
MetaGer
Project overview
Project overview
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
42
Issues
42
List
Boards
Labels
Milestones
Merge Requests
2
Merge Requests
2
Packages
Packages
Container Registry
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Commits
Issue Boards
Open sidebar
open-source
MetaGer
Commits
9ebf4465
Commit
9ebf4465
authored
May 08, 2017
by
Dominik Hebeler
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Weitere Änderungen am Searcher
parent
23d10c3e
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
58 additions
and
27 deletions
+58
-27
app/Jobs/Searcher.php
app/Jobs/Searcher.php
+23
-15
app/MetaGer.php
app/MetaGer.php
+1
-0
app/Models/Searchengine.php
app/Models/Searchengine.php
+31
-9
config/database.php
config/database.php
+3
-3
No files found.
app/Jobs/Searcher.php
View file @
9ebf4465
...
...
@@ -12,9 +12,9 @@ use Log;
class
Searcher
implements
ShouldQueue
{
use
InteractsWithQueue
, Queueable, SerializesModels
, DispatchesJobs
;
use
InteractsWithQueue
, Queueable, SerializesModels
;
protected
$name
,
$ch
;
protected
$name
,
$ch
,
$pid
,
$counter
,
$lastTime
;
protected
$MAX_REQUESTS
=
500
;
/**
...
...
@@ -31,6 +31,7 @@ class Searcher implements ShouldQueue
public
function
__construct
(
$name
)
{
$this
->
name
=
$name
;
$this
->
pid
=
getmypid
();
// Submit this worker to the Redis System
Redis
::
set
(
$this
->
name
,
"running"
);
Redis
::
expire
(
$this
->
name
,
5
);
...
...
@@ -46,7 +47,7 @@ class Searcher implements ShouldQueue
// This Searches is freshly called so we need to initialize the curl handle $ch
$this
->
ch
=
$this
->
initCurlHandle
();
$this
->
counter
=
0
;
// Counts the number of answered jobs
$
lastJob
=
microtime
(
true
);
$
time
=
microtime
(
true
);
while
(
true
){
// Update the expire
Redis
::
expire
(
$this
->
name
,
5
);
...
...
@@ -59,15 +60,12 @@ class Searcher implements ShouldQueue
// The mission can be empty when blpop hit the timeout
if
(
empty
(
$mission
)){
// In that case it should be safe to simply exit this job
if
(((
microtime
(
true
)
-
$lastJob
)
)
>
300
)
break
;
else
continue
;
continue
;
}
else
{
$mission
=
$mission
[
1
];
$this
->
counter
++
;
$lastJob
=
microtime
(
true
);
$this
->
counter
++
;
#
$poptime
=
microtime
(
true
)
-
$time
;
$time
=
microtime
(
true
);
}
// The mission is a String which can be divided to retrieve two informations:
...
...
@@ -81,18 +79,16 @@ class Searcher implements ShouldQueue
$result
=
$this
->
retrieveUrl
(
$url
);
$this
->
storeResult
(
$result
,
$hashValue
);
$this
->
storeResult
(
$result
,
$
poptime
,
$
hashValue
);
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if
(
getenv
(
"QUEUE_DRIVER"
)
===
"sync"
||
$this
->
counter
>
$this
->
MAX_REQUESTS
){
if
(
getenv
(
"QUEUE_DRIVER"
)
===
"sync"
)
Redis
::
del
(
$this
->
name
);
break
;
}
}
// When we reach this point, time has come for this Searcher to retire
// We should close our curl handle before we do so
curl_close
(
$this
->
ch
);
$this
->
exit
();
}
private
function
retrieveUrl
(
$url
){
...
...
@@ -104,8 +100,20 @@ class Searcher implements ShouldQueue
return
$result
;
}
private
function
storeResult
(
$result
,
$hashValue
){
private
function
storeResult
(
$result
,
$
poptime
,
$
hashValue
){
Redis
::
hset
(
'search.'
.
$hashValue
,
$this
->
name
,
$result
);
$connectionInfo
=
base64_encode
(
json_encode
(
curl_getinfo
(
$this
->
ch
),
true
));
Redis
::
hset
(
$this
->
name
.
".stats"
,
$this
->
pid
,
$connectionInfo
.
";"
.
$poptime
);
$this
->
lastTime
=
microtime
(
true
);
}
private
function
exit
(){
Redis
::
hdel
(
$this
->
name
.
".stats"
,
$this
->
pid
);
if
(
sizeof
(
Redis
::
hgetall
(
$this
->
name
.
".stats"
))
===
0
){
Redis
::
del
(
$this
->
name
);
}
// We should close our curl handle before we do so
curl_close
(
$this
->
ch
);
}
private
function
initCurlHandle
(){
...
...
app/MetaGer.php
View file @
9ebf4465
...
...
@@ -509,6 +509,7 @@ class MetaGer
$sumaCount
+=
1
;
}
$enabledSearchengines
[]
=
$suma
;
break
;
}
}
...
...
app/Models/Searchengine.php
View file @
9ebf4465
...
...
@@ -129,15 +129,37 @@ abstract class Searchengine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis
::
rpush
(
$this
->
name
.
".queue"
,
$mission
);
// If there is no Searcher process for this engine running at this time, we start one
if
(
Redis
::
get
(
$this
->
name
)
===
NULL
){
Log
::
info
(
"Starting Searcher"
);
/* Die Anfragen an die Suchmaschinen werden nun von der Laravel-Queue bearbeitet:
* Hinweis: solange in der .env der QUEUE_DRIVER auf "sync" gestellt ist, werden die Abfragen
* nacheinander abgeschickt.
* Sollen diese Parallel verarbeitet werden, muss ein anderer QUEUE_DRIVER verwendet werden.
* siehe auch: https://laravel.com/docs/5.2/queues
*/
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher
=
false
;
$searcherData
=
Redis
::
hgetall
(
$this
->
name
.
".stats"
);
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if
(
sizeof
(
$searcherData
)
===
0
){
$needSearcher
=
true
;
}
else
{
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
// searches or if it needs help.
// Let's hardcode a minimum of 100ms between every search job.
die
(
var_dump
(
$searcherData
));
}
if
(
$needSearcher
){
$this
->
dispatch
(
new
Searcher
(
$this
->
name
));
}
}
...
...
config/database.php
View file @
9ebf4465
...
...
@@ -111,19 +111,19 @@ return [
'default'
=>
[
'host'
=>
env
(
'REDIS_HOST'
,
'localhost'
),
#
'password' => env('REDIS_PASSWORD', null),
'password'
=>
env
(
'REDIS_PASSWORD'
,
null
),
'port'
=>
env
(
'REDIS_PORT'
,
6379
),
'database'
=>
0
,
],
'redisLogs'
=>
[
'host'
=>
env
(
'REDIS_LOGS_HOST'
,
'localhost'
),
#
'password' => env('REDIS_LOGS_PASSWORD', env('REDIS_PASSWORD', null)),
'password'
=>
env
(
'REDIS_LOGS_PASSWORD'
,
env
(
'REDIS_PASSWORD'
,
null
)),
'port'
=>
env
(
'REDIS_LOGS_PORT'
,
6379
),
'database'
=>
1
,
],
'redisCache'
=>
[
'host'
=>
env
(
'REDIS_CACHE_HOST'
,
'localhost'
),
#
'password' => env('REDIS_CACHE_PASSWORD', env('REDIS_PASSWORD', null)),
'password'
=>
env
(
'REDIS_CACHE_PASSWORD'
,
env
(
'REDIS_PASSWORD'
,
null
)),
'port'
=>
env
(
'REDIS_CACHE_PORT'
,
6379
),
'database'
=>
2
,
],
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment