Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
open-source
MetaGer
Commits
12a13ff5
Commit
12a13ff5
authored
Feb 11, 2019
by
Dominik Hebeler
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Query Filter are working switched to sumas.json
parent
81c74a85
Changes
10
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
173 additions
and
494 deletions
+173
-494
app/Jobs/Searcher.php
app/Jobs/Searcher.php
+6
-2
app/MetaGer.php
app/MetaGer.php
+82
-190
app/Models/Result.php
app/Models/Result.php
+2
-19
app/Models/Searchengine.php
app/Models/Searchengine.php
+45
-173
app/Models/parserSkripte/Bing.php
app/Models/parserSkripte/Bing.php
+0
-46
app/Models/parserSkripte/BingBilder.php
app/Models/parserSkripte/BingBilder.php
+6
-9
app/Models/parserSkripte/Bing_bilder.php
app/Models/parserSkripte/Bing_bilder.php
+0
-41
app/Models/parserSkripte/Overture.php
app/Models/parserSkripte/Overture.php
+22
-7
app/Models/parserSkripte/Scopia.php
app/Models/parserSkripte/Scopia.php
+8
-7
resources/lang/de/metaGer.php
resources/lang/de/metaGer.php
+2
-0
No files found.
app/Jobs/Searcher.php
View file @
12a13ff5
...
...
@@ -161,6 +161,7 @@ class Searcher implements ShouldQueue
// Set this URL to the Curl handle
curl_setopt
(
$this
->
ch
,
CURLOPT_URL
,
$url
);
$result
=
curl_exec
(
$this
->
ch
);
$this
->
connectionInfo
=
curl_getinfo
(
$this
->
ch
);
return
$result
;
}
...
...
@@ -203,10 +204,13 @@ class Searcher implements ShouldQueue
}
if
(
$this
->
headers
!==
null
)
{
$headers
=
[];
foreach
(
$this
->
headers
as
$key
=>
$value
)
{
$headers
[]
=
$key
.
":"
.
$value
;
}
# Headers are in the Form:
# <key>:<value>;<key>:<value>
$headerArray
=
explode
(
";"
,
$this
->
headers
);
curl_setopt
(
$ch
,
CURLOPT_HTTPHEADER
,
$headerArray
);
curl_setopt
(
$ch
,
CURLOPT_HTTPHEADER
,
$headers
);
}
return
$ch
;
...
...
app/MetaGer.php
View file @
12a13ff5
...
...
@@ -17,12 +17,11 @@ class MetaGer
protected
$fokus
;
protected
$eingabe
;
protected
$q
;
protected
$category
;
protected
$time
;
protected
$page
;
protected
$lang
;
protected
$cache
=
""
;
protected
$site
;
protected
$time
=
2000
;
protected
$hostBlacklist
=
[];
protected
$domainBlacklist
=
[];
private
$urlBlacklist
=
[];
...
...
@@ -49,7 +48,6 @@ class MetaGer
protected
$mobile
;
protected
$resultCount
;
protected
$sprueche
;
protected
$maps
;
protected
$newtab
;
protected
$domainsBlacklisted
=
[];
protected
$urlsBlacklisted
=
[];
...
...
@@ -309,7 +307,6 @@ class MetaGer
if
(
$this
->
validated
)
{
$this
->
ads
=
[];
$this
->
maps
=
false
;
}
if
(
count
(
$this
->
results
)
<=
0
)
{
...
...
@@ -531,99 +528,48 @@ class MetaGer
return
;
}
$xml
=
simplexml_load_file
(
$this
->
sumaFile
);
$sumas
=
$xml
->
xpath
(
"suma"
);
$enabledSearchengines
=
[];
$overtureEnabled
=
false
;
$sumaCount
=
0
;
/*
* Erstellt eine Liste mit Foki, die verfügbar sind
*/
$this
->
availableFoki
=
[];
foreach
(
$sumas
as
$suma
)
{
$foki
=
explode
(
","
,
trim
(
$suma
[
"type"
]));
foreach
(
$foki
as
$fokus
)
{
if
(
!
empty
(
$fokus
))
{
$this
->
availableFoki
[
$fokus
]
=
"available"
;
}
}
# Check if selected focus is valid
if
(
empty
(
$this
->
sumaFile
->
foki
->
{
$this
->
fokus
}))
{
$this
->
fokus
=
"web"
;
}
$isCustomSearch
=
$this
->
startsWith
(
$this
->
fokus
,
'focus_'
);
# Im Falle einer Custom-Suche ohne mindestens einer selektierter Suchmaschine wird eine Web-Suche durchgeführt
if
(
$isCustomSearch
&&
!
$this
->
atLeastOneSearchengineSelected
(
$request
))
{
$isCustomSearch
=
false
;
$this
->
fokus
=
'web'
;
}
/* Erstellt die Liste der eingestellten Sumas
* Der einzige Unterschied bei angepasstem Suchfokus ist,
* dass nicht nach den Typen einer Suma,
* sondern den im Request mitgegebenen Typen entschieden wird.
* Ansonsten wird genau das selbe geprüft und gemacht:
* Handelt es sich um spezielle Suchmaschinen die immer an sein müssen
* Wenn es Overture ist vermerken dass Overture an ist
* Suma Zähler erhöhen
* Zu Liste hinzufügen
*/
foreach
(
$sumas
as
$suma
)
{
if
((
$this
->
sumaIsSelected
(
$suma
,
$request
,
$isCustomSearch
)
||
(
!
$this
->
isBildersuche
()
&&
$this
->
sumaIsAdsuche
(
$suma
,
$overtureEnabled
)))
&&
(
!
$this
->
sumaIsDisabled
(
$suma
)))
{
if
(
$this
->
sumaIsOverture
(
$suma
))
{
$overtureEnabled
=
true
;
}
if
(
$this
->
sumaIsNotAdsuche
(
$suma
))
{
$sumaCount
+=
1
;
}
$enabledSearchengines
[]
=
$suma
;
foreach
(
$this
->
sumaFile
->
foki
->
{
$this
->
fokus
}
->
sumas
as
$suma
)
{
# Check if this engine is disabled and can't be used
$disabled
=
empty
(
$suma
->
disabled
)
?
false
:
$suma
->
disabled
;
if
(
$disabled
)
{
continue
;
}
}
# Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
$subcollections
=
[];
$tmp
=
[];
// Es gibt den Schalter "minism=on" Dieser soll bewirken, dass alle Minisucher angeschaltet werden.
// Wenn also "minism=on" ist, dann durchsuchen wir statt den tatsächlich angeschalteten Suchmaschinen,
// alle Suchmaschinen nach "minismCollection"
if
(
$request
->
input
(
"minism"
,
"off"
)
===
"on"
)
{
// Wir laden alle Minisucher
foreach
(
$sumas
as
$engine
)
{
if
(
isset
(
$engine
[
"minismCollection"
]))
{
$subcollections
[]
=
$engine
[
"minismCollection"
]
->
__toString
();
}
}
# Nur noch alle eventuell angeschalteten Minisucher deaktivieren
foreach
(
$enabledSearchengines
as
$index
=>
$engine
)
{
if
(
!
isset
(
$engine
[
"minismCollection"
]))
{
$tmp
[]
=
$engine
;
# Check if this engine can use eventually defined query-filter
$valid
=
true
;
foreach
(
$this
->
queryFilter
as
$queryFilter
=>
$filter
)
{
if
(
empty
(
$this
->
sumaFile
->
filter
->
$queryFilter
->
sumas
->
$suma
))
{
$valid
=
false
;
break
;
}
}
}
else
{
// Wir schalten eine Teilmenge, oder aber gar keine an
foreach
(
$enabledSearchengines
as
$engine
)
{
if
(
isset
(
$engine
[
'minismCollection'
]))
{
$subcollections
[]
=
$engine
[
'minismCollection'
]
->
__toString
();
}
else
{
$tmp
[]
=
$engine
;
}
# If it can we add it
if
(
$valid
)
{
$enabledSearchengines
[
$suma
]
=
$this
->
sumaFile
->
sumas
->
{
$suma
};
}
}
$enabledSearchengines
=
$tmp
;
if
(
sizeof
(
$subcollections
)
>
0
)
{
$enabledSearchengines
[]
=
$this
->
loadMiniSucher
(
$xml
,
$subcollections
);
}
if
(
$sumaCount
<=
0
)
{
$this
->
errors
[]
=
trans
(
'metaGer.settings.noneSelected'
);
if
(
sizeof
(
$enabledSearchengines
)
===
0
)
{
$filter
=
""
;
foreach
(
$this
->
queryFilter
as
$queryFilter
=>
$filterPhrase
)
{
$filter
.
=
trans
(
$this
->
sumaFile
->
filter
->
{
$queryFilter
}
->
name
)
.
","
;
}
$filter
=
rtrim
(
$filter
,
","
);
$error
=
trans
(
'metaGer.engines.noSpecialSearch'
,
[
'fokus'
=>
trans
(
$this
->
sumaFile
->
foki
->
{
$this
->
fokus
}
->
{
"display-name"
}),
'filter'
=>
$filter
]);
$this
->
errors
[]
=
$error
;
}
$engines
=
[];
# Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob überhaupt eine der Suchmaschinen eine Sitesearch unterstützt
$siteSearchFailed
=
$this
->
checkCanNotSitesearch
(
$enabledSearchengines
);
$typeslist
=
[];
$counter
=
0
;
...
...
@@ -635,9 +581,8 @@ class MetaGer
$engine
->
setResultHash
(
$this
->
getHashCode
());
}
}
else
{
$engines
=
$this
->
actuallyCreateSearchEngines
(
$enabledSearchengines
,
$siteSearchFailed
);
$engines
=
$this
->
actuallyCreateSearchEngines
(
$enabledSearchengines
);
}
# Wir starten alle Suchen
foreach
(
$engines
as
$engine
)
{
$engine
->
startSearch
(
$this
);
...
...
@@ -684,50 +629,34 @@ class MetaGer
return
false
;
}
public
function
actuallyCreateSearchEngines
(
$enabledSearchengines
,
$siteSearchFailed
)
public
function
actuallyCreateSearchEngines
(
$enabledSearchengines
)
{
$engines
=
[];
foreach
(
$enabledSearchengines
as
$engine
)
{
foreach
(
$enabledSearchengines
as
$engineName
=>
$engine
)
{
# Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
if
(
!
$siteSearchFailed
&&
strlen
(
$this
->
site
)
>
0
&&
(
!
isset
(
$engine
[
'hasSiteSearch'
])
||
$engine
[
'hasSiteSearch'
]
->
__toString
()
===
"0"
))
{
continue
;
}
if
(
!
isset
(
$engine
[
"package"
]))
{
if
(
!
isset
(
$engine
->
{
"parser-class"
}))
{
die
(
var_dump
(
$engine
));
}
# Setze Pfad zu Parser
$path
=
"App\Models\parserSkripte
\\
"
.
ucfirst
(
$engine
[
"package"
]
->
__toString
())
;
$path
=
"App
\
\
Models
\
\
parserSkripte
\\
"
.
$engine
->
{
"parser-class"
}
;
# Prüfe ob Parser vorhanden
if
(
!
file_exists
(
app_path
()
.
"/Models/parserSkripte/"
.
ucfirst
(
$engine
[
"package"
]
->
__toString
())
.
".php"
))
{
Log
::
error
(
"Konnte "
.
$engine
[
"
name"
]
.
" nicht abfragen, da kein Parser existiert"
);
$this
->
errors
[]
=
trans
(
'metaGer.engines.noParser'
,
[
'engine'
=>
$engine
[
"
name"
]
]);
if
(
!
file_exists
(
app_path
()
.
"/Models/parserSkripte/"
.
$engine
->
{
"parser-class"
}
.
".php"
))
{
Log
::
error
(
"Konnte "
.
$engine
->
{
"display-
name"
}
.
" nicht abfragen, da kein Parser existiert"
);
$this
->
errors
[]
=
trans
(
'metaGer.engines.noParser'
,
[
'engine'
=>
$engine
->
{
"display-
name"
}
]);
continue
;
}
# Es wird versucht die Suchengine zu erstellen
$time
=
microtime
();
try
{
$tmp
=
new
$path
(
$engine
,
$this
);
$tmp
=
new
$path
(
$engineName
,
$engine
,
$this
);
}
catch
(
\
ErrorException
$e
)
{
Log
::
error
(
"Konnte "
.
$engine
[
"
name"
]
.
" nicht abfragen. "
.
var_dump
(
$e
));
Log
::
error
(
"Konnte "
.
$engine
->
{
"display-
name"
}
.
" nicht abfragen. "
.
var_dump
(
$e
));
continue
;
}
# Ausgabe bei Debug-Modus
if
(
$tmp
->
enabled
&&
isset
(
$this
->
debug
))
{
$this
->
warnings
[]
=
$tmp
->
service
.
" Connection_Time: "
.
$tmp
->
connection_time
.
" Write_Time: "
.
$tmp
->
write_time
.
" Insgesamt:"
.
((
microtime
()
-
$time
)
/
1000
);
}
# Wenn die neu erstellte Engine eingeschaltet ist, wird sie der Liste hinzugefügt
if
(
$tmp
->
isEnabled
())
{
$engines
[]
=
$tmp
;
}
$engines
[]
=
$tmp
;
}
return
$engines
;
}
...
...
@@ -790,16 +719,6 @@ class MetaGer
return
$engines
;
}
public
function
loadMiniSucher
(
$xml
,
$subcollections
)
{
$minisucherEngine
=
$xml
->
xpath
(
'suma[@name="minism"]'
)[
0
];
$minisucherEngine
[
"subcollections"
]
=
implode
(
", "
,
$subcollections
);
$subcollectionsString
=
urlencode
(
"("
.
implode
(
" OR "
,
$subcollections
)
.
")"
);
$minisucherEngine
[
"formData"
]
=
str_replace
(
"<<SUBCOLLECTIONS>>"
,
$subcollectionsString
,
$minisucherEngine
[
"formData"
]);
$minisucherEngine
[
"formData"
]
=
str_replace
(
"<<COUNT>>"
,
sizeof
(
$subcollections
)
*
10
,
$minisucherEngine
[
"formData"
]);
return
$minisucherEngine
;
}
# Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
public
function
adjustFocus
(
$sumas
,
$enabledSearchengines
)
{
...
...
@@ -864,26 +783,6 @@ class MetaGer
}
}
public
function
checkCanNotSitesearch
(
$enabledSearchengines
)
{
if
(
strlen
(
$this
->
site
)
>
0
)
{
$enginesWithSite
=
0
;
foreach
(
$enabledSearchengines
as
$engine
)
{
if
(
isset
(
$engine
[
'hasSiteSearch'
])
&&
$engine
[
'hasSiteSearch'
]
->
__toString
()
===
"1"
)
{
$enginesWithSite
++
;
}
}
if
(
$enginesWithSite
===
0
)
{
$this
->
errors
[]
=
trans
(
'metaGer.sitesearch.failed'
,
[
'site'
=>
$this
->
site
,
'searchLink'
=>
$this
->
generateSearchLink
(
"web"
,
false
)]);
return
true
;
}
else
{
$this
->
warnings
[]
=
trans
(
'metaGer.sitesearch.success'
,
[
'site'
=>
$this
->
site
]);
return
false
;
}
}
return
false
;
}
public
function
waitForResults
(
$enginesToLoad
,
$overtureEnabled
,
$canBreak
)
{
...
...
@@ -977,12 +876,14 @@ class MetaGer
$this
->
fokus
=
$request
->
input
(
'focus'
,
'web'
);
# Suma-File
if
(
App
::
isLocale
(
"en"
))
{
$this
->
sumaFile
=
config_path
()
.
"/sumasEn.
xml
"
;
$this
->
sumaFile
=
config_path
()
.
"/sumasEn.
json
"
;
}
else
{
$this
->
sumaFile
=
config_path
()
.
"/sumas.
xml
"
;
$this
->
sumaFile
=
config_path
()
.
"/sumas.
json
"
;
}
if
(
!
file_exists
(
$this
->
sumaFile
))
{
die
(
trans
(
'metaGer.formdata.cantLoad'
));
}
else
{
$this
->
sumaFile
=
json_decode
(
file_get_contents
(
$this
->
sumaFile
));
}
# Sucheingabe
$this
->
eingabe
=
trim
(
$request
->
input
(
'eingabe'
,
''
));
...
...
@@ -1001,10 +902,7 @@ class MetaGer
}
else
{
$this
->
language
=
""
;
}
# Category
$this
->
category
=
$request
->
input
(
'category'
,
''
);
# Request Times
$this
->
time
=
$request
->
input
(
'time'
,
1500
);
# Page
$this
->
page
=
1
;
# Lang
...
...
@@ -1022,12 +920,7 @@ class MetaGer
}
else
{
$this
->
sprueche
=
false
;
}
$this
->
maps
=
$request
->
input
(
'maps'
,
'off'
);
if
(
$this
->
maps
===
"on"
)
{
$this
->
maps
=
true
;
}
else
{
$this
->
maps
=
false
;
}
$this
->
newtab
=
$request
->
input
(
'newtab'
,
'on'
);
if
(
$this
->
newtab
===
"on"
)
{
$this
->
newtab
=
"_blank"
;
...
...
@@ -1038,14 +931,7 @@ class MetaGer
$this
->
theme
=
preg_replace
(
"/[^[:alnum:][:space:]]/u"
,
''
,
$request
->
input
(
'theme'
,
'default'
));
# Ergebnisse pro Seite:
$this
->
resultCount
=
$request
->
input
(
'resultCount'
,
'20'
);
# Manchmal müssen wir Parameter anpassen um den Sucheinstellungen gerecht zu werden:
if
(
$request
->
filled
(
'dart'
))
{
$this
->
time
=
10000
;
$this
->
warnings
[]
=
trans
(
'metaGer.formdata.dartEurope'
);
}
if
(
$this
->
time
<=
500
||
$this
->
time
>
20000
)
{
$this
->
time
=
1000
;
}
if
(
$request
->
filled
(
'minism'
)
&&
(
$request
->
filled
(
'fportal'
)
||
$request
->
filled
(
'harvest'
)))
{
$input
=
$request
->
all
();
$newInput
=
[];
...
...
@@ -1075,6 +961,7 @@ class MetaGer
$this
->
quicktips
=
true
;
}
$this
->
queryFilter
=
[];
$this
->
verificationId
=
$request
->
input
(
'verification_id'
,
null
);
$this
->
verificationCount
=
intval
(
$request
->
input
(
'verification_count'
,
'0'
));
$this
->
apiKey
=
$request
->
input
(
'key'
,
''
);
...
...
@@ -1117,7 +1004,29 @@ class MetaGer
public
function
checkSpecialSearches
(
Request
$request
)
{
$this
->
searchCheckPhrase
();
$this
->
searchCheckSitesearch
(
$request
);
# Check for query-filter (i.e. Sitesearch, etc.):
foreach
(
$this
->
sumaFile
->
filter
as
$filterName
=>
$filter
)
{
if
(
$filter
->
type
!==
"query-filter"
)
{
continue
;
}
if
(
!
empty
(
$filter
->
{
"optional-parameter"
})
&&
$request
->
filled
(
$filter
->
{
"optional-parameter"
}))
{
$this
->
queryFilter
[
$filterName
]
=
$request
->
input
(
$filter
->
{
"optional-parameter"
});
}
else
if
(
preg_match_all
(
"/"
.
$filter
->
regex
.
"/si"
,
$this
->
q
,
$matches
)
>
0
)
{
switch
(
$filter
->
match
)
{
case
"last"
:
$this
->
queryFilter
[
$filterName
]
=
$matches
[
$filter
->
save
][
sizeof
(
$matches
[
$filter
->
save
])
-
1
];
$toDelete
=
preg_quote
(
$matches
[
$filter
->
delete
][
sizeof
(
$matches
[
$filter
->
delete
])
-
1
],
"/"
);
$this
->
q
=
preg_replace
(
'/('
.
$toDelete
.
'(?!.*'
.
$toDelete
.
'))/si'
,
''
,
$this
->
q
);
break
;
default
:
# First occurence
$this
->
queryFilter
[
$filterName
]
=
$matches
[
$filter
->
save
][
0
];
$toDelete
=
preg_quote
(
$matches
[
$filter
->
delete
][
0
],
"/"
);
$this
->
q
=
preg_replace
(
'/'
.
$toDelete
.
'/si'
,
''
,
$this
->
q
,
1
);
}
}
}
$this
->
searchCheckHostBlacklist
(
$request
);
$this
->
searchCheckDomainBlacklist
(
$request
);
$this
->
searchCheckUrlBlacklist
();
...
...
@@ -1143,19 +1052,6 @@ class MetaGer
}
}
private
function
searchCheckSitesearch
(
$request
)
{
// matches '[... ]site:test.de[ ...]'
while
(
preg_match
(
"/(^|.*?\s)site:(\S+)(\s.*|$)/si"
,
$this
->
q
,
$match
))
{
$this
->
site
=
$match
[
2
];
$this
->
q
=
$match
[
1
]
.
$match
[
3
];
}
# Overwrite Setting if it's submitted via Parameter
if
(
$request
->
has
(
'site'
))
{
$this
->
site
=
$request
->
input
(
'site'
);
}
}
private
function
searchCheckHostBlacklist
(
$request
)
{
// matches '[... ]-site:test.de[ ...]'
...
...
@@ -1573,10 +1469,6 @@ class MetaGer
{
return
$this
->
url
;
}
public
function
getTime
()
{
return
$this
->
time
;
}
public
function
getLanguage
()
{
...
...
@@ -1598,16 +1490,6 @@ class MetaGer
return
$this
->
sprueche
;
}
public
function
getMaps
()
{
return
$this
->
maps
;
}
public
function
getCategory
()
{
return
$this
->
category
;
}
public
function
getPhrases
()
{
return
$this
->
phrases
;
...
...
@@ -1622,6 +1504,16 @@ class MetaGer
return
$this
->
sumaFile
;
}
public
function
getQueryFilter
()
{
return
$this
->
queryFilter
;
}
public
function
getTime
()
{
return
$this
->
time
;
}
public
function
getUserHostBlacklist
()
{
return
$this
->
hostBlacklist
;
...
...
app/Models/Result.php
View file @
12a13ff5
...
...
@@ -31,7 +31,6 @@ class Result
# Erstellt ein neues Ergebnis
public
function
__construct
(
$provider
,
$titel
,
$link
,
$anzeigeLink
,
$descr
,
$gefVon
,
$gefVonLink
,
$sourceRank
,
$additionalInformation
=
[])
{
$provider
=
simplexml_load_string
(
$provider
);
$this
->
titel
=
strip_tags
(
trim
(
$titel
));
$this
->
link
=
trim
(
$link
);
$this
->
anzeigeLink
=
trim
(
$anzeigeLink
);
...
...
@@ -52,8 +51,8 @@ class Result
$this
->
sourceRank
=
20
;
}
$this
->
sourceRank
=
20
-
$this
->
sourceRank
;
if
(
isset
(
$provider
[
"engine
B
oost"
]
))
{
$this
->
engineBoost
=
floatval
(
$provider
[
"engine
B
oost"
]
->
__toString
()
);
if
(
isset
(
$provider
->
{
"engine
-b
oost"
}
))
{
$this
->
engineBoost
=
floatval
(
$provider
->
{
"engine
-b
oost"
}
);
}
else
{
$this
->
engineBoost
=
1
;
}
...
...
@@ -275,22 +274,6 @@ class Result
}
}
*/
/* Der Host-Filter der sicherstellt,
* dass von jedem Host maximal 3 Links angezeigt werden.
* Diese Überprüfung führen wir unter bestimmten Bedingungen nicht durch.
*/
if
((
$metager
->
getSite
()
===
""
||
$metager
->
getSite
()
===
null
)
&&
strpos
(
$this
->
strippedHost
,
"ncbi.nlm.nih.gov"
)
===
false
&&
strpos
(
$this
->
strippedHost
,
"twitter.com"
)
===
false
&&
strpos
(
$this
->
strippedHost
,
"www.ladenpreis.net"
)
===
false
&&
strpos
(
$this
->
strippedHost
,
"ncbi.nlm.nih.gov"
)
===
false
&&
strpos
(
$this
->
strippedHost
,
"www.onenewspage.com"
)
===
false
&&
$this
->
gefVon
!==
"Shopzilla"
)
{
$count
=
$metager
->
getHostCount
(
$this
->
strippedHost
);
if
(
$count
>=
3
)
{
return
false
;
}
}
/* Der Dublettenfilter, der sicher stellt,
* dass wir nach Möglichkeit keinen Link doppelt in der Ergebnisliste haben.
...
...
app/Models/Searchengine.php
View file @
12a13ff5
...
...
@@ -12,10 +12,8 @@ abstract class Searchengine
{
use
DispatchesJobs
;
public
$ch
;
# Curl Handle zum erhalten der Ergebnisse
public
$getString
=
""
;
# Der String für die Get-Anfrage
public
$engine
;
# Die ursprüngliche Engine XML
public
$enabled
=
true
;
# true, wenn die Suchmaschine nicht explizit disabled ist
public
$results
=
[];
# Die geladenen Ergebnisse
public
$ads
=
[];
# Die geladenen Werbungen
public
$products
=
[];
# Die geladenen Produkte
...
...
@@ -31,7 +29,7 @@ abstract class Searchengine
public
$startTime
;
# Die Zeit der Erstellung dieser Suchmaschine
public
$hash
;
# Der Hash-Wert dieser Suchmaschine
private
$user
;
# Username für HTTP-Auth (falls angegeben)
private
$user
name
;
# Username für HTTP-Auth (falls angegeben)
private
$password
;
# Passwort für HTTP-Auth (falls angegeben)
private
$headers
;
# Headers to add
...
...
@@ -42,58 +40,40 @@ abstract class Searchengine
public
$write_time
=
0
;
# Wird eventuell für Artefakte benötigt
public
$connection_time
=
0
;
# Wird eventuell für Artefakte benötigt
public
function
__construct
(
\
SimpleXMLElement
$engine
,
MetaGer
$metager
)
public
function
__construct
(
$name
,
\
stdClass
$engine
,
MetaGer
$metager
)
{
# Versucht möglichst viele attribute aus dem engine XML zu laden
foreach
(
$engine
->
attributes
()
as
$key
=>
$value
)
{
$this
->
$key
=
$value
->
__toString
();
}
# Standardhomepage metager.de
if
(
!
isset
(
$this
->
homepage
))
{
$this
->
homepage
=
"https://metager.de"
;
}
# Speichert die XML der Engine
$this
->
engine
=
$engine
->
asXML
();
$this
->
engine
=
$engine
;
$this
->
name
=
$name
;
# Cache Standarddauer 60
if
(
!
isset
(
$this
->
cacheDuration
))
{
$this
->
cacheDuration
=
60
;
}
$this
->
enabled
=
true
;
# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
if
(
isset
(
$this
->
disabled
)
&&
strtotime
(
$this
->
disabled
)
<=
time
())
{
# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
$this
->
enable
(
$metager
->
getSumaFile
(),
"Die Suchmaschine "
.
$this
->
name
.
" wurde wieder eingeschaltet."
);
}
elseif
(
isset
(
$this
->
disabled
)
&&
strtotime
(
$this
->
disabled
)
>
time
())
{
$this
->
enabled
=
false
;
return
;
}
$this
->
useragent
=
$metager
->
getUserAgent
();
$this
->
ip
=
$metager
->
getIp
();
$this
->
startTime
=
microtime
();
# check for http Auth
if
(
!
empty
(
$this
->
engine
->
{
"http-auth-credentials"
}
->
username
)
&&
!
empty
(
$this
->
engine
->
{
"http-auth-credentials"
}
->
password
))
{
$this
->
username
=
$this
->
engine
->
{
"http-auth-credentials"
}
->
username
;
$this
->
password
=
$this
->
engine
->
{
"http-auth-credentials"
}
->
password
;
}
# Suchstring generieren
$q
=
""
;
if
(
isset
(
$this
->
hasSiteSearch
)
&&
$this
->
hasSiteSearch
===
"1"
)
{
if
(
strlen
(
$metager
->
getSite
())
===
0
)
{
$q
=
$metager
->
getQ
();
}
else
{
$q
=
$metager
->
getQ
()
.
" site:"
.
$metager
->
getSite
();
}
$this
->
headers
=
$this
->
engine
->
{
"request-header"
};
}
else
{
$q
=
$metager
->
getQ
();
# Suchstring generieren
$q
=
$metager
->
getQ
();
$filters
=
$metager
->
getSumaFile
()
->
filter
;
foreach
(
$metager
->
getQueryFilter
()
as
$queryFilter
=>
$filter
)
{
$filterOptions
=
$filters
->
$queryFilter
;
$filterOptionsEngine
=
$filterOptions
->
sumas
->
{
$this
->
name
};
$query
=
$filterOptionsEngine
->
prefix
.
$filter
.
$filterOptionsEngine
->
suffix
;
$q
=
$query
.
" "
.
$q
;
}
$this
->
getString
=
$this
->
generateGetString
(
$q
,
$metager
->
getUrl
(),
$metager
->
getLanguage
(),
$metager
->
getCategory
());
$this
->
hash
=
md5
(
$this
->
host
.
$this
->
getString
.
$this
->
port
.
$this
->
name
);
$this
->
getString
=
$this
->
generateGetString
(
$q
);
$this
->
hash
=
md5
(
$this
->
engine
->
host
.
$this
->
getString
.
$this
->
engine
->
port
.
$this
->
name
);
$this
->
resultHash
=
$metager
->
getHashCode
();
$this
->
canCache
=
$metager
->
canCache
();
if
(
!
isset
(
$this
->
additionalHeaders
))
{
$this
->
additionalHeaders
=
""
;}
}
abstract
public
function
loadResults
(
$result
);
...
...
@@ -107,6 +87,7 @@ abstract class Searchengine
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
public
function
startSearch
(
\
App\MetaGer
$metager
)
{
if
(
$this
->
canCache
&&
Cache
::
has
(
$this
->
hash
))
{
$this
->
cached
=
true
;
$this
->
retrieveResults
(
$metager
);
...
...
@@ -118,18 +99,20 @@ abstract class Searchengine
// <ResultHash>;<URL to fetch>
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url
=
""
;
if
(
$this
->
port
===
"
443
"
)
{
if
(
$this
->
engine
->
port
===
443
)
{
$url
=
"https://"
;
}
else
{
$url
=
"http://"
;
}
$url
.
=
$this
->
host
;
if
(
$this
->
port
!==
80
&&
$this
->
port
!==
443
)
{