Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
open-source
MetaGer
Commits
4f27633e
Commit
4f27633e
authored
May 26, 2016
by
Dominik Hebeler
Browse files
Websuche funktioniert. Unter umständen aber noch nicht fehlerfrei
parent
d404667a
Changes
5
Hide whitespace changes
Inline
Side-by-side
app/MetaGer.php
View file @
4f27633e
...
...
@@ -112,6 +112,7 @@ class MetaGer
{
$overtureEnabled
=
TRUE
;
}
$enabledSearchengines
[]
=
$suma
;
}
}
...
...
@@ -152,7 +153,7 @@ class MetaGer
$tmp
=
new
$path
(
$engine
,
$this
);
if
(
$tmp
)
if
(
$tmp
->
isEnabled
()
)
{
$engines
[]
=
$tmp
;
}
...
...
app/Models/Searchengine.php
View file @
4f27633e
...
...
@@ -13,8 +13,8 @@ abstract class Searchengine
protected
$getString
=
""
;
protected
$engine
;
protected
$counter
=
0
;
protected
$enabled
;
protected
$socketNumber
=
null
;
protected
$enabled
=
true
;
public
$results
=
[];
function
__construct
(
\
SimpleXMLElement
$engine
,
MetaGer
$metager
)
...
...
@@ -24,6 +24,17 @@ abstract class Searchengine
$this
->
$key
=
$value
->
__toString
();
}
# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
if
(
isset
(
$this
->
disabled
)
&&
strtotime
(
$this
->
disabled
)
<=
time
()
)
{
# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
$this
->
enable
(
$metager
->
getSumaFile
(),
"Die Suchmaschine "
.
$this
->
name
.
" wurde wieder eingeschaltet."
);
}
elseif
(
isset
(
$this
->
disabled
)
&&
strtotime
(
$this
->
disabled
)
>
time
())
{
$this
->
enabled
=
false
;
return
;
}
# User-Agent definieren:
if
(
isset
(
$_SERVER
[
'HTTP_USER_AGENT'
]))
{
...
...
@@ -56,7 +67,7 @@ abstract class Searchengine
if
(
!
$this
->
fp
)
{
// Mache etwas
Log
::
error
(
"Konnte keine Verbindung zur
Suchmaschine
:
"
.
$this
->
name
.
" aufbau
en.
"
);
$this
->
disable
(
$metager
->
getSumaFile
(),
"Die
Suchmaschine "
.
$this
->
name
.
"
wurde für 1h deaktiviert, weil keine Verbindung
auf
ge
bau
t werden konnte
"
);
break
;
}
else
{
...
...
@@ -148,14 +159,22 @@ abstract class Searchengine
}
abort
(
500
,
"Konnte keinen freien Socket bekommen für: "
.
$this
->
name
)
;
return
null
;
}
public
function
disable
(
string
$sumaFile
,
string
$message
)
{
Log
::
info
(
$message
);
$xml
=
simplexml_load_file
(
$sumaFile
);
$xml
->
xpath
(
"//sumas/suma[@name='"
.
$this
->
name
.
"']"
)[
'0'
][
'disabled'
]
=
"1"
;
$xml
->
xpath
(
"//sumas/suma[@name='"
.
$this
->
name
.
"']"
)[
'0'
][
'disabled'
]
=
date
(
DATE_RFC822
,
mktime
(
date
(
"H"
)
+
1
,
date
(
"i"
),
date
(
"s"
),
date
(
"m"
),
date
(
"d"
),
date
(
"Y"
)));
$xml
->
saveXML
(
$sumaFile
);
}
public
function
enable
(
string
$sumaFile
,
string
$message
)
{
Log
::
info
(
$message
);
$xml
=
simplexml_load_file
(
$sumaFile
);
unset
(
$xml
->
xpath
(
"//sumas/suma[@name='"
.
$this
->
name
.
"']"
)[
'0'
][
'disabled'
]);
$xml
->
saveXML
(
$sumaFile
);
}
...
...
@@ -182,11 +201,6 @@ abstract class Searchengine
// use fgets() not fread(), fgets stops reading at first newline
// or buffer which ever one is reached first
$data
=
fgets
(
$this
->
fp
,
BUFFER_LENGTH
);
if
(
$data
===
false
)
{
usleep
(
10000
);
continue
;
}
// a sincle CRLF indicates end of headers
if
(
$data
===
false
||
$data
==
CRLF
||
feof
(
$this
->
fp
))
{
// break BEFORE OUTPUT
...
...
@@ -197,15 +211,20 @@ abstract class Searchengine
}
while
(
true
);
// end of headers
$bodySize
=
0
;
if
(
isset
(
$headers
[
"Transfer-Encoding"
])
&&
$headers
[
"Transfer-Encoding"
]
===
"chunked"
)
{
$body
=
$this
->
readChunked
();
}
elseif
(
isset
(
$headers
[
'Content-Length'
])
)
{
$length
=
trim
(
$headers
[
'Content-Length'
]);
if
(
is_numeric
(
$length
)
&&
$length
>=
1
)
$body
=
fread
(
$this
->
fp
,
$headers
[
'Content-Length'
]);
$body
=
$this
->
readBody
(
$length
);
$bodySize
=
strlen
(
$body
);
}
else
{
abort
(
500
,
"Konnte nicht herausfinden, wie ich die Serverantwort von: "
.
$this
->
name
.
" auslesen soll. Header war: "
.
print_r
(
$headers
));
}
Redis
::
setBit
(
$this
->
name
,
$this
->
socketNumber
,
0
);
...
...
@@ -213,6 +232,10 @@ abstract class Searchengine
{
$body
=
$this
->
gunzip
(
$body
);
}
#print_r($headers);
#print($body);
#print("\r\n". $bodySize);
#exit;
#die(print_r($headers));
// $body and $headers should contain your stream data
$this
->
loadResults
(
$body
);
...
...
@@ -220,6 +243,25 @@ abstract class Searchengine
#exit;
}
private
function
readBody
(
int
$length
)
{
$theData
=
''
;
$done
=
false
;
stream_set_blocking
(
$this
->
fp
,
0
);
$startTime
=
time
();
$lastTime
=
$startTime
;
while
(
!
feof
(
$this
->
fp
)
&&
!
$done
&&
((
$startTime
+
1
)
>
time
())
&&
$length
!==
0
)
{
usleep
(
100
);
$theNewData
=
fgets
(
$this
->
fp
,
BUFFER_LENGTH
);
$theData
.
=
$theNewData
;
$length
-=
strlen
(
$theNewData
);
$done
=
(
trim
(
$theNewData
)
===
'0'
);
}
return
$theData
;
}
private
function
readChunked
()
{
$body
=
''
;
...
...
app/Models/parserSkripte/Onenewspagevideo.php
View file @
4f27633e
...
...
@@ -15,11 +15,13 @@ class Onenewspagevideo extends Searchengine
public
function
loadResults
(
String
$result
)
{
$results
=
trim
(
$result
);
foreach
(
explode
(
"
\n
"
,
$results
)
as
$result
)
{
$res
=
explode
(
"|"
,
$result
);
if
(
sizeof
(
$res
)
<
3
)
{
continue
;
}
$title
=
$res
[
0
];
$link
=
$res
[
2
];
$anzeigeLink
=
$link
;
...
...
app/Models/parserSkripte/Yacy.php
0 → 100644
View file @
4f27633e
<?php
namespace
app\Models\parserSkripte
;
use
App\Models\Searchengine
;
class
Yacy
extends
Searchengine
{
public
$results
=
[];
function
__construct
(
\
SimpleXMLElement
$engine
,
\
App\MetaGer
$metager
)
{
parent
::
__construct
(
$engine
,
$metager
);
}
public
function
loadResults
(
String
$result
)
{
# die($result);
try
{
$content
=
simplexml_load_string
(
$result
);
}
catch
(
\
Exception
$e
)
{
abort
(
500
,
"
$result
is not a valid xml string"
);
}
$results
=
$content
->
xpath
(
"//rss/channel/item"
);
foreach
(
$results
as
$res
)
{
$title
=
$res
->
{
"title"
};
$link
=
$res
->
{
"link"
};
$anzeigeLink
=
$link
;
$descr
=
$res
->
{
"description"
};
$this
->
counter
++
;
$this
->
results
[]
=
new
\
App\Models\Result
(
$title
,
$link
,
$anzeigeLink
,
$descr
,
$this
->
gefVon
,
$this
->
counter
);
}
}
}
\ No newline at end of file
config/app.php
View file @
4f27633e
...
...
@@ -52,7 +52,7 @@ return [
|
*/
'timezone'
=>
'
UTC
'
,
'timezone'
=>
'
Europe/Berlin
'
,
/*
|--------------------------------------------------------------------------
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment