Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
open-source
MetaGer
Commits
d52875a7
Commit
d52875a7
authored
Feb 24, 2020
by
Dominik Hebeler
Browse files
Merge branch 'development' into '943-optimize-deployment'
Development See merge request
!1540
parents
d0496140
301b9974
Changes
6
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
app/Console/Commands/RequestFetcher.php
View file @
d52875a7
...
...
@@ -63,14 +63,13 @@ class RequestFetcher extends Command
try
{
$blocking
=
false
;
$redis
=
Redis
::
connection
(
"cache"
);
while
(
$this
->
shouldRun
)
{
$status
=
curl_multi_exec
(
$this
->
multicurl
,
$active
);
$currentJob
=
null
;
if
(
!
$blocking
)
{
$currentJob
=
$r
edis
->
lpop
(
\
App\MetaGer
::
FETCHQUEUE_KEY
);
$currentJob
=
R
edis
::
lpop
(
\
App\MetaGer
::
FETCHQUEUE_KEY
);
}
else
{
$currentJob
=
$r
edis
->
blpop
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
1
);
$currentJob
=
R
edis
::
blpop
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
1
);
if
(
!
empty
(
$currentJob
))
{
$currentJob
=
$currentJob
[
1
];
}
...
...
@@ -105,8 +104,11 @@ class RequestFetcher extends Command
$body
=
\
curl_multi_getcontent
(
$info
[
"handle"
]);
}
Cache
::
put
(
$resulthash
,
$body
,
$cacheDurationMinutes
*
60
);
Redis
::
pipeline
(
function
(
$pipe
)
use
(
$resulthash
,
$body
,
$cacheDurationMinutes
)
{
$pipe
->
set
(
$resulthash
,
$body
);
$pipe
->
expire
(
$resulthash
,
60
);
Cache
::
put
(
$resulthash
,
$body
,
$cacheDurationMinutes
*
60
);
});
\
curl_multi_remove_handle
(
$this
->
multicurl
,
$info
[
"handle"
]);
}
if
(
!
$active
&&
!
$answerRead
)
{
...
...
@@ -131,11 +133,11 @@ class RequestFetcher extends Command
CURLOPT_RETURNTRANSFER
=>
1
,
CURLOPT_USERAGENT
=>
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1"
,
CURLOPT_FOLLOWLOCATION
=>
true
,
CURLOPT_CONNECTTIMEOUT
=>
10
,
CURLOPT_CONNECTTIMEOUT
=>
2
,
CURLOPT_MAXCONNECTS
=>
500
,
CURLOPT_LOW_SPEED_LIMIT
=>
500
,
CURLOPT_LOW_SPEED_TIME
=>
5
,
CURLOPT_TIMEOUT
=>
10
,
CURLOPT_LOW_SPEED_LIMIT
=>
500
00
,
CURLOPT_LOW_SPEED_TIME
=>
2
,
CURLOPT_TIMEOUT
=>
3
,
));
if
(
!
empty
(
$this
->
proxyhost
)
&&
!
empty
(
$this
->
proxyport
)
&&
!
empty
(
$this
->
proxyuser
)
&&
!
empty
(
$this
->
proxypassword
))
{
...
...
app/MetaGer.php
View file @
d52875a7
...
...
@@ -6,6 +6,7 @@ use App;
use
Cache
;
use
Carbon
;
use
Illuminate\Http\Request
;
use
Illuminate\Support\Facades\Redis
;
use
Jenssegers\Agent\Agent
;
use
LaravelLocalization
;
use
Log
;
...
...
@@ -853,7 +854,7 @@ class MetaGer
while
(
sizeof
(
$enginesToWaitFor
)
>
0
||
(
$forceTimeout
!==
null
&&
(
microtime
(
true
)
-
$timeStart
)
<
$forceTimeout
))
{
foreach
(
$enginesToWaitFor
as
$index
=>
$engine
)
{
if
(
Cache
::
get
(
$engine
->
hash
)
!==
null
)
{
if
(
Redis
::
get
(
$engine
->
hash
)
!==
null
)
{
$answered
[]
=
$engine
;
unset
(
$enginesToWaitFor
[
$index
]);
break
;
...
...
app/Models/Quicktips/Quicktips.php
View file @
d52875a7
...
...
@@ -46,7 +46,7 @@ class Quicktips
$mission
=
json_encode
(
$mission
);
Redis
::
connection
(
'cache'
)
->
rpush
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
$mission
);
Redis
::
rpush
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
$mission
);
}
}
...
...
app/Models/Searchengine.php
View file @
d52875a7
...
...
@@ -3,7 +3,6 @@
namespace
App\Models
;
use
App\MetaGer
;
use
Cache
;
use
Illuminate\Support\Facades\Redis
;
abstract
class
Searchengine
...
...
@@ -148,7 +147,7 @@ abstract class Searchengine
// Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis
::
connection
(
'cache'
)
->
rpush
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
$mission
);
Redis
::
rpush
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
$mission
);
if
(
!
empty
(
$timings
))
{
$timings
[
"startSearch"
][
$this
->
name
][
"pushed job"
]
=
microtime
(
true
)
-
$timings
[
"starttime"
];
}
...
...
@@ -194,7 +193,7 @@ abstract class Searchengine
$body
=
""
;
}
}
else
{
$body
=
Cache
::
get
(
$this
->
hash
);
$body
=
Redis
::
get
(
$this
->
hash
);
}
if
(
$body
!==
null
)
{
...
...
app/Models/parserSkripte/Qualigo.php
View file @
d52875a7
...
...
@@ -3,6 +3,7 @@
namespace
App\Models\parserSkripte
;
use
App\Models\XmlSearchengine
;
use
Log
;
class
Qualigo
extends
XmlSearchengine
{
...
...
@@ -14,6 +15,10 @@ class Qualigo extends XmlSearchengine
protected
function
loadXmlResults
(
$resultsXml
)
{
if
(
empty
(
$resultsXml
)
||
!
$resultsXml
){
Log
::
error
(
"A problem occurred parsing results from
$this->name
:"
);
return
;
}
try
{
$results
=
$resultsXml
->
xpath
(
'//RL/RANK'
);
foreach
(
$results
as
$result
)
{
...
...
routes/web.php
View file @
d52875a7
...
...
@@ -21,9 +21,7 @@ Route::group(
/** ADD ALL LOCALIZED ROUTES INSIDE THIS GROUP **/
Route
::
get
(
'/'
,
'StartpageController@loadStartPage'
);
Route
::
get
(
'sand'
,
function
()
{
abort
(
500
,
"test"
);
});
Route
::
get
(
'asso'
,
function
()
{
return
view
(
'assoziator.asso'
)
->
with
(
'title'
,
trans
(
'titles.asso'
))
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment