Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
open-source
MetaGer
Commits
f7762e75
Commit
f7762e75
authored
May 19, 2020
by
Dominik Hebeler
Browse files
Request fetcher will now accept all pending jobs on each cycle
parent
f5579908
Changes
1
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
app/Console/Commands/RequestFetcher.php
View file @
f7762e75
...
...
@@ -82,33 +82,39 @@ class RequestFetcher extends Command
$blocking
=
false
;
while
(
$this
->
shouldRun
)
{
$status
=
curl_multi_exec
(
$this
->
multicurl
,
$active
);
$currentJob
=
null
;
$currentJob
s
=
[]
;
if
(
!
$blocking
)
{
$currentJob
=
Redis
::
lpop
(
\
App\MetaGer
::
FETCHQUEUE_KEY
);
$elements
=
Redis
::
pipeline
(
function
(
$redis
){
$redis
->
lrange
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
0
,
-
1
);
$redis
->
del
(
\
App\MetaGer
::
FETCHQUEUE_KEY
);
});
$currentJobs
=
$elements
[
0
];
}
else
{
$currentJob
=
Redis
::
blpop
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
1
);
if
(
!
empty
(
$currentJob
))
{
$currentJob
=
$currentJob
[
1
];
$currentJob
s
[]
=
$currentJob
[
1
];
}
}
if
(
!
empty
(
$currentJob
))
{
$currentJob
=
json_decode
(
$currentJob
,
true
);
$ch
=
$this
->
getCurlHandle
(
$currentJob
);
if
(
curl_multi_add_handle
(
$this
->
multicurl
,
$ch
)
!==
0
)
{
$this
->
shouldRun
=
false
;
Log
::
error
(
"Couldn't add Handle to multicurl"
);
break
;
}
$this
->
fetchedDocuments
++
;
if
(
$this
->
fetchedDocuments
>
$this
->
maxFetchedDocuments
)
{
Log
::
info
(
"Reinitializing Multicurl after "
.
$this
->
fetchedDocuments
.
" requests."
);
$this
->
oldMultiCurl
=
$this
->
multicurl
;
$this
->
multicurl
=
curl_multi_init
();
$this
->
fetchedDocuments
=
0
;
if
(
sizeof
(
$currentJobs
)
>
0
)
{
foreach
(
$currentJobs
as
$currentJob
){
$currentJob
=
json_decode
(
$currentJob
,
true
);
$ch
=
$this
->
getCurlHandle
(
$currentJob
);
if
(
curl_multi_add_handle
(
$this
->
multicurl
,
$ch
)
!==
0
)
{
$this
->
shouldRun
=
false
;
Log
::
error
(
"Couldn't add Handle to multicurl"
);
break
;
}
$this
->
fetchedDocuments
++
;
if
(
$this
->
fetchedDocuments
>
$this
->
maxFetchedDocuments
)
{
Log
::
info
(
"Reinitializing Multicurl after "
.
$this
->
fetchedDocuments
.
" requests."
);
$this
->
oldMultiCurl
=
$this
->
multicurl
;
$this
->
multicurl
=
curl_multi_init
();
$this
->
fetchedDocuments
=
0
;
}
$blocking
=
false
;
$active
=
true
;
}
$blocking
=
false
;
$active
=
true
;
}
$answerRead
=
$this
->
readMultiCurl
(
$this
->
multicurl
);
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment