Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
open-source
MetaGer
Commits
fff0bc3e
Commit
fff0bc3e
authored
Dec 05, 2019
by
Dominik Hebeler
Browse files
caching should work now
parent
9a5ccb35
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
app/Console/Commands/RequestCacher.php
0 → 100644
View file @
fff0bc3e
<?php
namespace
App\Console\Commands
;
use
Cache
;
use
Illuminate\Console\Command
;
use
Illuminate\Support\Facades\Redis
;
class
RequestCacher
extends
Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected
$signature
=
'requests:cacher'
;
const
CACHER_QUEUE
=
'cacher.queue'
;
protected
$shouldRun
=
true
;
/**
* The console command description.
*
* @var string
*/
protected
$description
=
'Listens to a buffer of fetched search results and writes them to the filesystem cache.'
;
/**
* Create a new command instance.
*
* @return void
*/
public
function
__construct
()
{
parent
::
__construct
();
}
/**
* Execute the console command.
*
* @return mixed
*/
public
function
handle
()
{
pcntl_async_signals
(
true
);
pcntl_signal
(
SIGINT
,
[
$this
,
"sig_handler"
]);
pcntl_signal
(
SIGTERM
,
[
$this
,
"sig_handler"
]);
pcntl_signal
(
SIGHUP
,
[
$this
,
"sig_handler"
]);
while
(
$this
->
shouldRun
)
{
$cacheItem
=
Redis
::
blpop
(
self
::
CACHER_QUEUE
,
1
);
if
(
!
empty
(
$cacheItem
))
{
$cacheItem
=
json_decode
(
$cacheItem
[
1
],
true
);
if
(
empty
(
$cacheItem
[
"body"
]))
{
$cacheItem
[
"body"
]
=
"no-result"
;
}
Cache
::
put
(
$cacheItem
[
"hash"
],
$cacheItem
[
"body"
],
now
()
->
addMinutes
(
$cacheItem
[
"cacheDuration"
]));
}
}
}
public
function
sig_handler
(
$sig
)
{
$this
->
shouldRun
=
false
;
echo
(
"Terminating Cacher Process
\n
"
);
}
}
app/Console/Commands/
WorkerSpawn
er.php
→
app/Console/Commands/
RequestFetch
er.php
View file @
fff0bc3e
...
...
@@ -2,11 +2,12 @@
namespace
App\Console\Commands
;
use
Artisan
;
use
Illuminate\Console\Command
;
use
Illuminate\Support\Facades\Redis
;
use
Log
;
class
WorkerSpawn
er
extends
Command
class
RequestFetch
er
extends
Command
{
/**
* The name and signature of the console command.
...
...
@@ -20,7 +21,7 @@ class WorkerSpawner extends Command
*
* @var string
*/
protected
$description
=
'This command
makes sure that enough worker processes are spaw
ne
d
'
;
protected
$description
=
'This command
s fetches requests to the installed search engi
ne
s
'
;
protected
$shouldRun
=
true
;
protected
$multicurl
=
null
;
...
...
@@ -49,10 +50,16 @@ class WorkerSpawner extends Command
*/
public
function
handle
()
{
pcntl_async_signals
(
true
);
pcntl_signal
(
SIGINT
,
[
$this
,
"sig_handler"
]);
pcntl_signal
(
SIGTERM
,
[
$this
,
"sig_handler"
]);
pcntl_signal
(
SIGHUP
,
[
$this
,
"sig_handler"
]);
$pid
=
\
pcntl_fork
();
if
(
$pid
===
0
)
{
Artisan
::
call
(
'requests:cacher'
);
exit
;
}
else
{
pcntl_async_signals
(
true
);
pcntl_signal
(
SIGINT
,
[
$this
,
"sig_handler"
]);
pcntl_signal
(
SIGTERM
,
[
$this
,
"sig_handler"
]);
pcntl_signal
(
SIGHUP
,
[
$this
,
"sig_handler"
]);
}
try
{
$blocking
=
false
;
...
...
@@ -62,7 +69,7 @@ class WorkerSpawner extends Command
if
(
!
$blocking
)
{
$currentJob
=
Redis
::
lpop
(
\
App\MetaGer
::
FETCHQUEUE_KEY
);
}
else
{
$currentJob
=
Redis
::
blpop
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
1
0
);
$currentJob
=
Redis
::
blpop
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
1
);
if
(
!
empty
(
$currentJob
))
{
$currentJob
=
$currentJob
[
1
];
}
...
...
@@ -97,9 +104,15 @@ class WorkerSpawner extends Command
$body
=
\
curl_multi_getcontent
(
$info
[
"handle"
]);
}
Redis
::
pipeline
(
function
(
$pipe
)
use
(
$resulthash
,
$body
)
{
Redis
::
pipeline
(
function
(
$pipe
)
use
(
$resulthash
,
$body
,
$cacheDuration
)
{
$pipe
->
set
(
$resulthash
,
$body
);
$pipe
->
expire
(
$resulthash
,
60
);
$cacherItem
=
[
'cacheDuration'
=>
$cacheDuration
,
'hash'
=>
$resulthash
,
'body'
=>
$body
,
];
$pipe
->
rpush
(
\
App\Console\Commands\RequestCacher
::
CACHER_QUEUE
,
json_encode
(
$cacherItem
));
});
#Cache::put($resulthash, $body, now()->addMinutes($cacheDuration));
\
curl_multi_remove_handle
(
$this
->
multicurl
,
$info
[
"handle"
]);
...
...
@@ -111,6 +124,8 @@ class WorkerSpawner extends Command
}
finally
{
curl_multi_close
(
$this
->
multicurl
);
}
\
pcntl_waitpid
(
$pid
,
$status
,
WNOHANG
);
}
private
function
getCurlHandle
(
$job
)
...
...
app/Models/Searchengine.php
View file @
fff0bc3e
...
...
@@ -102,7 +102,7 @@ abstract class Searchengine
{
if
(
$this
->
canCache
&&
Cache
::
has
(
$this
->
hash
))
{
$this
->
cached
=
true
;
$this
->
retrieveResults
(
$metager
);
$this
->
retrieveResults
(
$metager
,
true
);
}
else
{
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
...
...
@@ -170,20 +170,16 @@ abstract class Searchengine
return
true
;
}
$body
=
Redis
::
get
(
$this
->
hash
);
/*if (Cache::has($this->hash)) {
$body = Cache::get($this->hash);
}*/
/*
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
$body = Cache::get($this->hash);
} elseif ($redis->hexists($metager->getRedisEngineResult() . $this->name, "response")) {
$body = $redis->hget($metager->getRedisEngineResult() . $this->name, "response");
if ($this->canCache && $this->cacheDuration > 0) {
Cache::put($this->hash, $body, $this->cacheDuration);
$body
=
null
;
if
(
$this
->
cached
)
{
$body
=
Cache
::
get
(
$this
->
hash
);
if
(
$body
===
"no-result"
)
{
$body
=
""
;
}
}
else
{
$body
=
Redis
::
get
(
$this
->
hash
);
}
}*/
if
(
$body
!==
null
)
{
$this
->
loadResults
(
$body
);
$this
->
getNext
(
$metager
,
$body
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment