Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
MetaGer
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
54
Issues
54
List
Boards
Labels
Service Desk
Milestones
Merge Requests
6
Merge Requests
6
Operations
Operations
Incidents
Environments
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
open-source
MetaGer
Commits
4f27633e
Commit
4f27633e
authored
May 26, 2016
by
Dominik Hebeler
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Websuche funktioniert. Unter umständen aber noch nicht fehlerfrei
parent
d404667a
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
105 additions
and
15 deletions
+105
-15
app/MetaGer.php
app/MetaGer.php
+2
-1
app/Models/Searchengine.php
app/Models/Searchengine.php
+53
-11
app/Models/parserSkripte/Onenewspagevideo.php
app/Models/parserSkripte/Onenewspagevideo.php
+4
-2
app/Models/parserSkripte/Yacy.php
app/Models/parserSkripte/Yacy.php
+45
-0
config/app.php
config/app.php
+1
-1
No files found.
app/MetaGer.php
View file @
4f27633e
...
...
@@ -112,6 +112,7 @@ class MetaGer
{
$overtureEnabled
=
TRUE
;
}
$enabledSearchengines
[]
=
$suma
;
}
}
...
...
@@ -152,7 +153,7 @@ class MetaGer
$tmp
=
new
$path
(
$engine
,
$this
);
if
(
$tmp
)
if
(
$tmp
->
isEnabled
()
)
{
$engines
[]
=
$tmp
;
}
...
...
app/Models/Searchengine.php
View file @
4f27633e
...
...
@@ -13,8 +13,8 @@ abstract class Searchengine
protected
$getString
=
""
;
protected
$engine
;
protected
$counter
=
0
;
protected
$enabled
;
protected
$socketNumber
=
null
;
protected
$enabled
=
true
;
public
$results
=
[];
function
__construct
(
\
SimpleXMLElement
$engine
,
MetaGer
$metager
)
...
...
@@ -24,6 +24,17 @@ abstract class Searchengine
$this
->
$key
=
$value
->
__toString
();
}
# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
if
(
isset
(
$this
->
disabled
)
&&
strtotime
(
$this
->
disabled
)
<=
time
()
)
{
# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
$this
->
enable
(
$metager
->
getSumaFile
(),
"Die Suchmaschine "
.
$this
->
name
.
" wurde wieder eingeschaltet."
);
}
elseif
(
isset
(
$this
->
disabled
)
&&
strtotime
(
$this
->
disabled
)
>
time
())
{
$this
->
enabled
=
false
;
return
;
}
# User-Agent definieren:
if
(
isset
(
$_SERVER
[
'HTTP_USER_AGENT'
]))
{
...
...
@@ -56,7 +67,7 @@ abstract class Searchengine
if
(
!
$this
->
fp
)
{
// Mache etwas
Log
::
error
(
"Konnte keine Verbindung zur Suchmaschine: "
.
$this
->
name
.
" aufbauen.
"
);
$this
->
disable
(
$metager
->
getSumaFile
(),
"Die Suchmaschine "
.
$this
->
name
.
" wurde für 1h deaktiviert, weil keine Verbindung aufgebaut werden konnte
"
);
break
;
}
else
{
...
...
@@ -148,14 +159,22 @@ abstract class Searchengine
}
abort
(
500
,
"Konnte keinen freien Socket bekommen für: "
.
$this
->
name
)
;
return
null
;
}
public
function
disable
(
string
$sumaFile
,
string
$message
)
{
Log
::
info
(
$message
);
$xml
=
simplexml_load_file
(
$sumaFile
);
$xml
->
xpath
(
"//sumas/suma[@name='"
.
$this
->
name
.
"']"
)[
'0'
][
'disabled'
]
=
"1"
;
$xml
->
xpath
(
"//sumas/suma[@name='"
.
$this
->
name
.
"']"
)[
'0'
][
'disabled'
]
=
date
(
DATE_RFC822
,
mktime
(
date
(
"H"
)
+
1
,
date
(
"i"
),
date
(
"s"
),
date
(
"m"
),
date
(
"d"
),
date
(
"Y"
)));
$xml
->
saveXML
(
$sumaFile
);
}
public
function
enable
(
string
$sumaFile
,
string
$message
)
{
Log
::
info
(
$message
);
$xml
=
simplexml_load_file
(
$sumaFile
);
unset
(
$xml
->
xpath
(
"//sumas/suma[@name='"
.
$this
->
name
.
"']"
)[
'0'
][
'disabled'
]);
$xml
->
saveXML
(
$sumaFile
);
}
...
...
@@ -182,11 +201,6 @@ abstract class Searchengine
// use fgets() not fread(), fgets stops reading at first newline
// or buffer which ever one is reached first
$data
=
fgets
(
$this
->
fp
,
BUFFER_LENGTH
);
if
(
$data
===
false
)
{
usleep
(
10000
);
continue
;
}
// a sincle CRLF indicates end of headers
if
(
$data
===
false
||
$data
==
CRLF
||
feof
(
$this
->
fp
))
{
// break BEFORE OUTPUT
...
...
@@ -197,15 +211,20 @@ abstract class Searchengine
}
while
(
true
);
// end of headers
$bodySize
=
0
;
if
(
isset
(
$headers
[
"Transfer-Encoding"
])
&&
$headers
[
"Transfer-Encoding"
]
===
"chunked"
)
{
$body
=
$this
->
readChunked
();
}
elseif
(
isset
(
$headers
[
'Content-Length'
])
)
{
$length
=
trim
(
$headers
[
'Content-Length'
]);
if
(
is_numeric
(
$length
)
&&
$length
>=
1
)
$body
=
fread
(
$this
->
fp
,
$headers
[
'Content-Length'
]);
$body
=
$this
->
readBody
(
$length
);
$bodySize
=
strlen
(
$body
);
}
else
{
abort
(
500
,
"Konnte nicht herausfinden, wie ich die Serverantwort von: "
.
$this
->
name
.
" auslesen soll. Header war: "
.
print_r
(
$headers
));
}
Redis
::
setBit
(
$this
->
name
,
$this
->
socketNumber
,
0
);
...
...
@@ -213,6 +232,10 @@ abstract class Searchengine
{
$body
=
$this
->
gunzip
(
$body
);
}
#print_r($headers);
#print($body);
#print("\r\n". $bodySize);
#exit;
#die(print_r($headers));
// $body and $headers should contain your stream data
$this
->
loadResults
(
$body
);
...
...
@@ -220,6 +243,25 @@ abstract class Searchengine
#exit;
}
private
function
readBody
(
int
$length
)
{
$theData
=
''
;
$done
=
false
;
stream_set_blocking
(
$this
->
fp
,
0
);
$startTime
=
time
();
$lastTime
=
$startTime
;
while
(
!
feof
(
$this
->
fp
)
&&
!
$done
&&
((
$startTime
+
1
)
>
time
())
&&
$length
!==
0
)
{
usleep
(
100
);
$theNewData
=
fgets
(
$this
->
fp
,
BUFFER_LENGTH
);
$theData
.
=
$theNewData
;
$length
-=
strlen
(
$theNewData
);
$done
=
(
trim
(
$theNewData
)
===
'0'
);
}
return
$theData
;
}
private
function
readChunked
()
{
$body
=
''
;
...
...
app/Models/parserSkripte/Onenewspagevideo.php
View file @
4f27633e
...
...
@@ -15,11 +15,13 @@ class Onenewspagevideo extends Searchengine
public
function
loadResults
(
String
$result
)
{
$results
=
trim
(
$result
);
foreach
(
explode
(
"
\n
"
,
$results
)
as
$result
)
{
$res
=
explode
(
"|"
,
$result
);
if
(
sizeof
(
$res
)
<
3
)
{
continue
;
}
$title
=
$res
[
0
];
$link
=
$res
[
2
];
$anzeigeLink
=
$link
;
...
...
app/Models/parserSkripte/Yacy.php
0 → 100644
View file @
4f27633e
<?php
namespace
app\Models\parserSkripte
;
use
App\Models\Searchengine
;
class
Yacy
extends
Searchengine
{
public
$results
=
[];
function
__construct
(
\
SimpleXMLElement
$engine
,
\
App\MetaGer
$metager
)
{
parent
::
__construct
(
$engine
,
$metager
);
}
public
function
loadResults
(
String
$result
)
{
# die($result);
try
{
$content
=
simplexml_load_string
(
$result
);
}
catch
(
\
Exception
$e
)
{
abort
(
500
,
"
$result
is not a valid xml string"
);
}
$results
=
$content
->
xpath
(
"//rss/channel/item"
);
foreach
(
$results
as
$res
)
{
$title
=
$res
->
{
"title"
};
$link
=
$res
->
{
"link"
};
$anzeigeLink
=
$link
;
$descr
=
$res
->
{
"description"
};
$this
->
counter
++
;
$this
->
results
[]
=
new
\
App\Models\Result
(
$title
,
$link
,
$anzeigeLink
,
$descr
,
$this
->
gefVon
,
$this
->
counter
);
}
}
}
\ No newline at end of file
config/app.php
View file @
4f27633e
...
...
@@ -52,7 +52,7 @@ return [
|
*/
'timezone'
=>
'
UTC
'
,
'timezone'
=>
'
Europe/Berlin
'
,
/*
|--------------------------------------------------------------------------
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment