Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • sebastian/MetaGer
  • cpietsch/MetaGer
  • kossow/MetaGer
  • jens/MetaGer
  • open-source/MetaGer
5 results
Show changes
Commits on Source (28)
Hide whitespace changes
Inline Side-by-side
Showing
with 607 additions and 362 deletions
app/Http/Controllers/FokiLoader.php
View file @ 0b99ab10
......@@ -10,12 +10,12 @@ class FokiLoader
{
$sumaFile = "";
if (App::isLocale('en')) {
$sumaFile = config_path() . "/sumas.xml";
$sumaFile = config_path() . "/sumasEn.xml";
} else {
$sumaFile = config_path() . "/sumas.xml";
}
$xml = simplexml_load_file($sumaFile);
$xml = simplexml_load_file($sumaFile);
$sumas = $xml->xpath("suma");
$foki = [];
......@@ -24,13 +24,13 @@ class FokiLoader
if (isset($suma['type'])) {
$f = explode(",", $suma['type']->__toString());
foreach ($f as $tmp) {
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$foki[$tmp][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
}
} else {
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$foki["andere"][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
}
}
......
app/Http/Controllers/HumanVerification.php
View file @ 0b99ab10
......@@ -2,66 +2,120 @@
namespace App\Http\Controllers;
use Captcha;
use Carbon;
use DB;
use Illuminate\Hashing\BcryptHasher as Hasher;
use Illuminate\Http\Request;
use Validator;
use Input;
use DB;
use Carbon;
class HumanVerification extends Controller
{
public static function captcha(Request $request, $id, $url){
if($request->getMethod() == 'POST'){
$rules = ['captcha' => 'required|captcha'];
$validator = Validator::make($request->all(), $rules);
if($validator->fails()){
return view('captcha')->with('title', 'Bestätigung notwendig')->with('id', $id)->with('url', base64_decode($url))->with('errorMessage', 'Bitte Captcha eingeben:');
}else{
# If we can unlock the Account of this user we will redirect him to the result page
$id = $request->input('id');
$url = $request->input('url');
public static function captcha(Request $request, Hasher $hasher, $id, $url = null)
{
if ($url != null) {
$url = base64_decode(str_replace("<<SLASH>>", "/", $url));
} else {
$url = $request->input('url');
}
$user = DB::table('humanverification')->where('id', $id)->first();
if($user !== null && $user->locked === 1){
DB::table('humanverification')->where('id', $id)->update(['locked' => false]);
if ($request->getMethod() == 'POST') {
$user = DB::table('humanverification')->where('uid', $id)->first();
$lockedKey = $user->lockedKey;
$key = $request->input('captcha');
$key = strtolower($key);
if (!$hasher->check($key, $lockedKey)) {
$captcha = Captcha::create("default", true);
DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
->with('id', $id)
->with('url', $url)
->with('image', $captcha["img"])
->with('errorMessage', 'Fehler: Falsches Captcha eingegeben!');
} else {
# If we can unlock the Account of this user we will redirect him to the result page
if ($user !== null && $user->locked === 1) {
# The Captcha was correct. We can remove the key from the user
DB::table('humanverification')->where('uid', $id)->update(['locked' => false, 'lockedKey' => "", 'whitelist' => 1]);
return redirect($url);
}else{
} else {
return redirect('/');
}
}
}
return view('captcha')->with('title', 'Bestätigung notwendig')->with('id', $id)->with('url', base64_decode($url));
$captcha = Captcha::create("default", true);
DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
->with('id', $id)
->with('url', $url)
->with('image', $captcha["img"]);
}
public static function remove(Request $request){
if(!$request->has('mm')){
public static function remove(Request $request)
{
if (!$request->has('mm')) {
abort(404, "Keine Katze gefunden.");
}
$id = md5($request->ip());
if(HumanVerification::checkId($request, $request->input('mm'))){
# Remove the entry from the database
DB::table('humanverification')->where('id', $id)->where('updated_at', '<', Carbon::NOW()->subSeconds(2) )->delete();
if (HumanVerification::checkId($request, $request->input('mm'))) {
HumanVerification::removeUser($request, $request->input('mm'));
}
return response(hex2bin('89504e470d0a1a0a0000000d494844520000000100000001010300000025db56ca00000003504c5445000000a77a3dda0000000174524e530040e6d8660000000a4944415408d76360000000020001e221bc330000000049454e44ae426082'), 200)
->header('Content-Type', 'image/png');
}
public static function removeGet(Request $request, $mm, $password, $url){
$url = base64_decode($url);
public static function removeGet(Request $request, $mm, $password, $url)
{
$url = base64_decode(str_replace("<<SLASH>>", "/", $url));
# If the user is correct and the password is we will delete any entry in the database
$requiredPass = md5($mm . Carbon::NOW()->day . $url . env("PROXY_PASSWORD"));
if(HumanVerification::checkId($request, $mm) && $requiredPass === $password){
# Remove the entry from the database
DB::table('humanverification')->where('id', $mm)->where('updated_at', '<', Carbon::NOW()->subSeconds(2) )->delete();
if (HumanVerification::checkId($request, $mm) && $requiredPass === $password) {
HumanVerification::removeUser($request, $mm);
}
return redirect($url);
}
private static function checkId($request, $id){
if(md5($request->ip()) === $id){
private static function removeUser($request, $uid)
{
$id = hash("sha512", $request->ip());
$sum = DB::table('humanverification')->where('id', $id)->where('whitelist', false)->sum('unusedResultPages');
$user = DB::table('humanverification')->where('uid', $uid)->first();
if ($user === null) {
return;
}
# Check if we have to whitelist the user or if we can simply delete the data
if ($user->unusedResultPages < $sum && $user->whitelist === 0) {
# Whitelist
DB::table('humanverification')->where('uid', $uid)->update(['whitelist' => true, 'whitelistCounter' => 0]);
$user->whitelist = 1;
$user->whitelistCounter = 0;
}
if ($user->whitelist === 1) {
if (
DB::table('humanverification')->where('uid', $uid)->update(['unusedResultPages' => 0])
=== 1
) {
DB::table('usedurls')->where('uid', $uid)->delete();
}
} else {
DB::table('humanverification')->where('uid', $uid)->where('updated_at', '<', Carbon::NOW()->subSeconds(2))->delete();
}
}
private static function checkId($request, $id)
{
if (hash("sha512", $request->ip() . $_SERVER["AGENT"]) === $id) {
return true;
}else{
} else {
return false;
}
}
......
app/Http/Controllers/StartpageController.php
View file @ 0b99ab10
......@@ -5,8 +5,8 @@ namespace App\Http\Controllers;
use App;
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
use Response;
use LaravelLocalization;
use Response;
class StartpageController extends Controller
{
......@@ -20,7 +20,7 @@ class StartpageController extends Controller
public function loadStartPage(Request $request)
{
$focusPages = [];
$theme = "default";
$theme = "default";
$optionParams = ['param_sprueche', 'param_newtab', 'param_maps', 'param_autocomplete', 'param_lang', 'param_key'];
$option_values = [];
......@@ -33,12 +33,12 @@ class StartpageController extends Controller
}
$autocomplete = 'on';
if(in_array('autocomplete', array_keys($option_values))) {
if (in_array('autocomplete', array_keys($option_values))) {
$autocomplete = $option_values['autocomplete'];
}
$lang = LaravelLocalization::getCurrentLocale();
if ($lang === 'de') {
if ($lang === 'de' || $lang === "en") {
$lang = 'all';
}
......@@ -78,9 +78,9 @@ class StartpageController extends Controller
public function loadPlugin($params, $locale = "de")
{
$params = unserialize(base64_decode($params));
$params = unserialize(base64_decode($params));
$requests = $params;
$params = [];
$params = [];
foreach ($requests as $key => $value) {
if (strpos($key, "param_") === 0) {
$key = substr($key, strpos($key, "param_") + 6);
......@@ -135,12 +135,12 @@ class StartpageController extends Controller
public function berlin(Request $request)
{
$link = "";
$link = "";
$password = "";
if ($request->filled('eingabe')) {
$password = getenv('berlin');
$password = md5($request->input('eingabe') . " -host:userpage.fu-berlin.de" . $password);
$link = "/meta/meta.ger3?eingabe=" . $request->input('eingabe') . " -host:userpage.fu-berlin.de&focus=web&password=" . $password . "&encoding=utf8&lang=all&site=fu-berlin.de&quicktips=off&out=results-with-style";
$link = "/meta/meta.ger3?eingabe=" . $request->input('eingabe') . " -host:userpage.fu-berlin.de&focus=web&password=" . $password . "&encoding=utf8&lang=all&site=fu-berlin.de&quicktips=off&out=results-with-style";
}
return view('berlin')
->with('title', 'Testseite für die FU-Berlin')
......
app/Http/Kernel.php
View file @ 0b99ab10
......@@ -36,11 +36,6 @@ class Kernel extends HttpKernel
],
'session' => [
\App\Http\Middleware\EncryptCookies::class,
\Illuminate\Cookie\Middleware\AddQueuedCookiesToResponse::class,
\Illuminate\Session\Middleware\StartSession::class,
\Illuminate\View\Middleware\ShareErrorsFromSession::class,
\App\Http\Middleware\VerifyCsrfToken::class,
\Illuminate\Routing\Middleware\SubstituteBindings::class,
],
];
......@@ -53,12 +48,12 @@ class Kernel extends HttpKernel
* @var array
*/
protected $routeMiddleware = [
'auth' => \Illuminate\Auth\Middleware\Authenticate::class,
'auth.basic' => \Illuminate\Auth\Middleware\AuthenticateWithBasicAuth::class,
'bindings' => \Illuminate\Routing\Middleware\SubstituteBindings::class,
'can' => \Illuminate\Auth\Middleware\Authorize::class,
'guest' => \App\Http\Middleware\RedirectIfAuthenticated::class,
'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
'auth' => \Illuminate\Auth\Middleware\Authenticate::class,
'auth.basic' => \Illuminate\Auth\Middleware\AuthenticateWithBasicAuth::class,
'bindings' => \Illuminate\Routing\Middleware\SubstituteBindings::class,
'can' => \Illuminate\Auth\Middleware\Authorize::class,
'guest' => \App\Http\Middleware\RedirectIfAuthenticated::class,
'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
'referer.check' => \App\Http\Middleware\RefererCheck::class,
'humanverification' => \App\Http\Middleware\HumanVerification::class,
];
......
app/Http/Middleware/HumanVerification.php
View file @ 0b99ab10
......@@ -2,9 +2,12 @@
namespace App\Http\Middleware;
use Captcha;
use Carbon;
use Closure;
use DB;
use Carbon;
use Illuminate\Http\Response;
use URL;
class HumanVerification
{
......@@ -17,52 +20,113 @@ class HumanVerification
*/
public function handle($request, Closure $next)
{
$id = md5($request->ip());
/**
* If the user sends a Password or a key
* We will not verificate the user.
* If someone that uses a bot finds this out we
* might have to change it at some point.
*/
if($request->filled('password') || $request->filled('key') || $request->filled('appversion') || !env('BOT_PROTECTION', false)){
return $next($request);
}
try {
$id = hash("sha512", $request->ip());
$uid = hash("sha512", $request->ip() . $_SERVER["AGENT"]);
unset($_SERVER["AGENT"]);
/**
* If the user sends a Password or a key
* We will not verificate the user.
* If someone that uses a bot finds this out we
* might have to change it at some point.
*/
if ($request->filled('password') || $request->filled('key') || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
return $next($request);
}
// The specific user
$user = DB::table('humanverification')->where('uid', $uid)->first();
$createdAt = Carbon::now();
$unusedResultPages = 1;
$locked = false;
# If this user doesn't have an entry we will create one
if ($user === null) {
DB::table('humanverification')->insert(
[
'uid' => $uid,
'id' => $id,
'unusedResultPages' => 0,
'whitelist' => false,
'whitelistCounter' => 0,
'locked' => false,
"lockedKey" => "",
'updated_at' => Carbon::now(),
]
);
# Insert the URL the user tries to reach
$url = url()->full();
DB::table('usedurls')->insert(['uid' => $uid, 'id' => $id, 'eingabe' => $request->input('eingabe', '')]);
$user = DB::table('humanverification')->where('uid', $uid)->first();
}
# Lock out everyone in a Bot network
# Find out how many requests this IP has made
$sum = DB::table('humanverification')->where('id', $id)->where('whitelist', false)->sum('unusedResultPages');
# A lot of automated requests are from websites that redirect users to our result page.
# We will detect those requests and put a captcha
$referer = URL::previous();
# Just the URL-Parameter
$refererLock = false;
if (stripos($referer, "?") !== false) {
$referer = substr($referer, stripos($referer, "?") + 1);
$referer = urldecode($referer);
if (preg_match("/http[s]{0,1}:\/\/metager\.de\/meta\/meta.ger3\?.*?eingabe=([\w\d]+\.){1,2}[\w\d]+/si", $referer) === 1) {
$refererLock = true;
}
}
// Defines if this is the only user using that IP Adress
$alone = DB::table('humanverification')->where('id', $id)->count() === 1;
if ((!$alone && $sum >= 50 && $user->whitelist !== 1) || $refererLock) {
DB::table('humanverification')->where('uid', $uid)->update(['locked' => true]);
$user->locked = 1;
}
# If the user is locked we will force a Captcha validation
if ($user->locked === 1) {
$captcha = Captcha::create("default", true);
DB::table('humanverification')->where('uid', $uid)->update(['lockedKey' => $captcha["key"]]);
return
new Response(
view('humanverification.captcha')
->with('title', "Bestätigung erforderlich")
->with('id', $uid)
->with('url', url()->full())
->with('image', $captcha["img"])
);
}
$user = DB::table('humanverification')->where('id', $id)->first();
$createdAt = now();
$unusedResultPages = 1;
$locked = false;
# If this user doesn't have an entry we will create one
if($user === null){
DB::table('humanverification')->insert(
['id' => $id, 'unusedResultPages' => 1, 'locked' => false, 'updated_at' => now()]
);
# Insert the URL the user tries to reach
DB::table('usedurls')->insert(['user_id' => $id, 'url' => url()->full()]);
$user = DB::table('humanverification')->where('id', $id)->first();
}else if($user->locked !== 1){
$unusedResultPages = intval($user->unusedResultPages);
$unusedResultPages++;
# We have different security gates:
# 50, 75, 85, >=90 => Captcha validated Result Pages
# If the user shows activity on our result page the counter will be deleted
# Maybe I'll add a ban if the user reaches 100
if($unusedResultPages === 50 || $unusedResultPages === 75 || $unusedResultPages === 85 || $unusedResultPages >= 90){
$locked = true;
$locked = false;
if ($alone || $user->whitelist === 1) {
# This IP doesn't need verification yet
# The user currently isn't locked
# We have different security gates:
# 50, 75, 85, >=90 => Captcha validated Result Pages
# If the user shows activity on our result page the counter will be deleted
# Maybe I'll add a ban if the user reaches 100
if ($unusedResultPages === 50 || $unusedResultPages === 75 || $unusedResultPages === 85 || $unusedResultPages >= 90) {
$locked = true;
}
}
DB::table('humanverification')->where('id', $id)->update(['unusedResultPages' => $unusedResultPages, 'locked' => $locked, 'updated_at' => $createdAt]);
DB::table('humanverification')->where('uid', $uid)->update(['unusedResultPages' => $unusedResultPages, 'locked' => $locked]);
# Insert the URL the user tries to reach
DB::table('usedurls')->insert(['user_id' => $id, 'url' => url()->full()]);
}
$request->request->add(['verification_id' => $id, 'verification_count' => $unusedResultPages]);
DB::table('usedurls')->insert(['uid' => $uid, 'id' => $id, 'eingabe' => $request->input('eingabe', '')]);
# If the user is locked we will force a Captcha validation
if($user->locked === 1){
return redirect('meta/verification/' . $id . '/' . urlencode(base64_encode(url()->full())));
} catch (\Illuminate\Database\QueryException $e) {
// Failure in contacting metager3.de
}
$request->request->add(['verification_id' => $uid, 'verification_count' => $unusedResultPages]);
return $next($request);
}
}
app/Jobs/Searcher.php
View file @ 0b99ab10
......@@ -3,18 +3,16 @@
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Redis;
use Log;
class Searcher implements ShouldQueue
{
use InteractsWithQueue, Queueable, SerializesModels;
protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo;
protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo, $user, $password;
# Each Searcher will shutdown after a specified time(s) or number of requests
protected $MAX_REQUESTS = 100;
# This value should always be below the retry_after value in config/queue.php
......@@ -34,12 +32,14 @@ class Searcher implements ShouldQueue
* keep-alive requests.
* @return void
*/
public function __construct($name)
public function __construct($name, $user = null, $password = null)
{
$this->name = $name;
$this->pid = getmypid();
$this->recheck = false;
$this->startTime = microtime(true);
$this->user = $user;
$this->password = $password;
// Submit this worker to the Redis System
Redis::expire($this->name, 5);
}
......@@ -53,95 +53,101 @@ class Searcher implements ShouldQueue
{
// This Searches is freshly called so we need to initialize the curl handle $ch
$this->ch = $this->initCurlHandle();
$this->counter = 0; // Counts the number of answered jobs
$time = microtime(true);
while(true){
// Update the expire
Redis::expire($this->name, 5);
Redis::expire($this->name . ".stats", 5);
// One Searcher can handle a ton of requests to the same server
// Each search to the server of this Searcher will be submitted to a queue
// stored in redis which has the same name as this searchengine appended by a ".queue"
// We will perform a blocking pop on this queue so the queue can remain empty for a while
// without killing this searcher directly.
$mission = Redis::blpop($this->name . ".queue", 4);
$this->counter++;
$this->updateStats(microtime(true) - $time);
$this->switchToRunning();
// The mission can be empty when blpop hit the timeout
if(!empty($mission)){
$mission = $mission[1];
$poptime = microtime(true) - $time;
// The mission is a String which can be divided to retrieve three informations:
// 1. The Hash Value where the result should be stored
// 2. The Url to Retrieve
// 3. The maximum time to take
// These three informations are divided by a ";" in the mission string
$mission = explode(";", $mission);
$hashValue = $mission[0]; // The hash value for redis to store the results under
$url = base64_decode($mission[1]); // The url to fetch
$timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected");
$result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue);
// Reset the time of the last Job so we can calculate
// the time we have spend waiting for a new job
// We submit that calculation to the Redis systemin the method
$time = microtime(true);
try {
$this->counter = 0; // Counts the number of answered jobs
$time = microtime(true);
while (true) {
// Update the expire
Redis::expire($this->name, 5);
Redis::expire($this->name . ".stats", 5);
// One Searcher can handle a ton of requests to the same server
// Each search to the server of this Searcher will be submitted to a queue
// stored in redis which has the same name as this searchengine appended by a ".queue"
// We will perform a blocking pop on this queue so the queue can remain empty for a while
// without killing this searcher directly.
$mission = Redis::blpop($this->name . ".queue", 4);
$this->counter++;
$this->updateStats(microtime(true) - $time);
$this->switchToRunning();
// The mission can be empty when blpop hit the timeout
if (!empty($mission)) {
$mission = $mission[1];
$poptime = microtime(true) - $time;
// The mission is a String which can be divided to retrieve three informations:
// 1. The Hash Value where the result should be stored
// 2. The Url to Retrieve
// 3. The maximum time to take
// These three informations are divided by a ";" in the mission string
$mission = explode(";", $mission);
$hashValue = $mission[0]; // The hash value for redis to store the results under
$url = base64_decode($mission[1]); // The url to fetch
$timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected");
$result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue);
// Reset the time of the last Job so we can calculate
// the time we have spend waiting for a new job
// We submit that calculation to the Redis systemin the method
$time = microtime(true);
}
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if (getenv("QUEUE_DRIVER") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true) - $this->startTime) > $this->MAX_TIME) {
break;
}
}
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if(getenv("QUEUE_DRIVER") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true)-$this->startTime) > $this->MAX_TIME){
break;
}
} finally {
// When we reach this point, time has come for this Searcher to retire
$this->shutdown();
}
// When we reach this point, time has come for this Searcher to retire
$this->shutdown();
}
private function switchToRunning(){
private function switchToRunning()
{
/**
* When a Searcher is initially started the redis value for $this->name is set to "locked"
* which effectively will prevent new Searchers of this type to be started. (Value is checked by the MetaGer process which starts the Searchers)
* This is done so the MetaGer processes won't start hundreds of Searchers parallely when under high work load.
* It will force that Searchers can only be started one after the other.
* When a new Searcher has served a minimum of three requests we have enough data to decide whether we need even more Searchers.
* To do so we will then set the redis value for $this->name to "running".
* There is a case where we don't want new Searchers to be started even if we would need to do so to serve every Request:
* When a search engine needs more time to produce search results than the timeout of the MetaGer process, we won't even bother of spawning
* more and more Searchers because they would just block free worker processes from serving the important engines which will give results in time.
**/
if($this->counter === 3 || getenv("QUEUE_DRIVER") === "sync"){
* When a Searcher is initially started the redis value for $this->name is set to "locked"
* which effectively will prevent new Searchers of this type to be started. (Value is checked by the MetaGer process which starts the Searchers)
* This is done so the MetaGer processes won't start hundreds of Searchers parallely when under high work load.
* It will force that Searchers can only be started one after the other.
* When a new Searcher has served a minimum of three requests we have enough data to decide whether we need even more Searchers.
* To do so we will then set the redis value for $this->name to "running".
* There is a case where we don't want new Searchers to be started even if we would need to do so to serve every Request:
* When a search engine needs more time to produce search results than the timeout of the MetaGer process, we won't even bother of spawning
* more and more Searchers because they would just block free worker processes from serving the important engines which will give results in time.
**/
if ($this->counter === 3 || getenv("QUEUE_DRIVER") === "sync") {
# If the MetaGer process waits longer for the results than this Fetcher will probably need to fetch
# Or if this engine is in the array of important engines which we will always try to serve
Redis::set($this->name, "running");
$this->recheck = false;
}
}
private function updateStats($poptime){
if($this->connectionInfo !== NULL){
private function updateStats($poptime)
{
if ($this->connectionInfo !== null) {
$connectionInfo = base64_encode(json_encode($this->connectionInfo));
Redis::hset($this->name . ".stats", $this->pid, $connectionInfo . ";" . $poptime);
}
}
private function getFetchTime(){
private function getFetchTime()
{
$vals = Redis::hgetall($this->name . ".stats");
if(sizeof($vals) === 0){
if (sizeof($vals) === 0) {
return 0;
}else{
} else {
$totalTime = 0;
foreach ($vals as $pid => $value) {
$time = floatval(json_decode(base64_decode(explode(";", $value)[0]), true)["total_time"]);
$time *= 1000; // Transform from seconds to milliseconds
$time *= 1000; // Transform from seconds to milliseconds
$totalTime += $time;
}
$totalTime /= sizeof($vals);
......@@ -149,7 +155,8 @@ class Searcher implements ShouldQueue
}
}
private function retrieveUrl($url){
private function retrieveUrl($url)
{
// Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch);
......@@ -157,36 +164,43 @@ class Searcher implements ShouldQueue
return $result;
}
private function storeResult($result, $poptime, $hashValue){
private function storeResult($result, $poptime, $hashValue)
{
Redis::hset('search.' . $hashValue, $this->name, $result);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
Redis::expire('search.' . $hashValue, 60);
$this->lastTime = microtime(true);
}
private function shutdown(){
private function shutdown()
{
Redis::hdel($this->name . ".stats", $this->pid);
if(sizeof(Redis::hgetall($this->name . ".stats")) === 0){
if (sizeof(Redis::hgetall($this->name . ".stats")) === 0) {
Redis::del($this->name);
}
// We should close our curl handle before we do so
curl_close($this->ch);
}
private function initCurlHandle(){
private function initCurlHandle()
{
$ch = curl_init();
curl_setopt_array($ch, array(
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
CURLOPT_FOLLOWLOCATION => TRUE,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 500,
CURLOPT_LOW_SPEED_TIME => 5,
CURLOPT_TIMEOUT => 10
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 500,
CURLOPT_LOW_SPEED_TIME => 5,
CURLOPT_TIMEOUT => 10,
));
if ($this->user !== null && $this->password !== null) {
curl_setopt($ch, CURLOPT_USERPWD, $this->user . ":" . $this->password);
}
return $ch;
}
}
app/MetaGer.php
View file @ 0b99ab10
......@@ -3,12 +3,12 @@ namespace App;
use App;
use Cache;
use Carbon;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use Jenssegers\Agent\Agent;
use LaravelLocalization;
use Log;
use Carbon;
use Predis\Connection\ConnectionException;
class MetaGer
......@@ -34,6 +34,7 @@ class MetaGer
protected $warnings = [];
protected $errors = [];
protected $addedHosts = [];
protected $availableFoki = [];
protected $startCount = 0;
protected $canCache = false;
# Daten über die Abfrage$
......@@ -258,6 +259,7 @@ class MetaGer
# Human Verification
$this->results = $this->humanVerification($this->results);
$this->ads = $this->humanVerification($this->ads);
$counter = 0;
$firstRank = 0;
......@@ -296,10 +298,6 @@ class MetaGer
}
if (LaravelLocalization::getCurrentLocale() === "en") {
$this->ads = [];
}
if ($this->validated) {
$this->ads = [];
$this->maps = false;
......@@ -460,8 +458,11 @@ class MetaGer
$link = $result->link;
$day = Carbon::now()->day;
$pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
$url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(base64_encode($link))]);
$url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
$proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
$proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
$result->link = $url;
$result->proxyLink = $proxyUrl;
}
return $results;
}else{
......@@ -521,6 +522,20 @@ class MetaGer
$overtureEnabled = false;
$sumaCount = 0;
/*
* Erstellt eine Liste mit Foki, die verfügbar sind
*/
$this->availableFoki = [];
foreach ($sumas as $suma) {
$foki = explode(",", trim($suma["type"]));
foreach ($foki as $fokus) {
if (!empty($fokus)) {
$this->availableFoki[$fokus] = "available";
}
}
}
$isCustomSearch = $this->startsWith($this->fokus, 'focus_');
# Im Falle einer Custom-Suche ohne mindestens einer selektierter Suchmaschine wird eine Web-Suche durchgeführt
......@@ -642,7 +657,7 @@ class MetaGer
public function sumaIsSelected($suma, $request, $custom)
{
if ($custom) {
if ($request->filled("engine_" . $suma["name"])) {
if ($request->filled("engine_" . strtolower($suma["name"]))) {
return true;
}
} else {
......@@ -930,7 +945,7 @@ class MetaGer
public function parseFormData(Request $request)
{
$this->request = $request;
# Sichert, dass der request in UTF-8 formatiert ist
if ($request->input('encoding', 'utf8') !== "utf8") {
# In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
......@@ -946,7 +961,7 @@ class MetaGer
$this->fokus = $request->input('focus', 'web');
# Suma-File
if (App::isLocale("en")) {
$this->sumaFile = config_path() . "/sumas.xml";
$this->sumaFile = config_path() . "/sumasEn.xml";
} else {
$this->sumaFile = config_path() . "/sumas.xml";
}
......@@ -955,7 +970,7 @@ class MetaGer
}
# Sucheingabe
$this->eingabe = trim($request->input('eingabe', ''));
$this->q = $this->eingabe;
$this->q = $this->eingabe;
# IP
$this->ip = $request->ip();
# Unser erster Schritt wird sein, IP-Adresse und USER-Agent zu anonymisieren, damit
......@@ -1047,6 +1062,8 @@ class MetaGer
$this->verificationId = $request->input('verification_id', null);
$this->verificationCount = intval($request->input('verification_count', '0'));
$this->apiKey = $request->input('key', '');
// Remove Inputs that are not used
$this->request = $request->replace($request->except(['verification_id', 'uid', 'verification_count']));
$this->validated = false;
if (isset($this->password)) {
......@@ -1083,19 +1100,37 @@ class MetaGer
public function checkSpecialSearches(Request $request)
{
$this->searchCheckPhrase();
$this->searchCheckSitesearch($request);
$this->searchCheckHostBlacklist($request);
$this->searchCheckDomainBlacklist($request);
$this->searchCheckUrlBlacklist();
$this->searchCheckPhrase();
$this->searchCheckStopwords($request);
$this->searchCheckNoSearch();
}
private function searchCheckPhrase()
{
$p = "";
$tmp = $this->q;
// matches '[... ]"test satz"[ ...]'
while (preg_match("/(^|.*?\s)\"(\S+)\"(\s.*|$)/si", $tmp, $match)) {
$tmp = $match[1] . $match[3];
$this->phrases[] = $match[2];
}
foreach ($this->phrases as $phrase) {
$p .= "\"$phrase\", ";
}
$p = rtrim($p, ", ");
if (sizeof($this->phrases) > 0) {
$this->warnings[] = trans('metaGer.formdata.phrase', ['phrase' => $p]);
}
}
private function searchCheckSitesearch($request)
{
// matches '[... ]site:test.de[ ...]'
while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
while (preg_match("/(^|.*?\s)site:(\S+)(\s.*|$)/si", $this->q, $match)) {
$this->site = $match[2];
$this->q = $match[1] . $match[3];
}
......@@ -1108,7 +1143,7 @@ class MetaGer
private function searchCheckHostBlacklist($request)
{
// matches '[... ]-site:test.de[ ...]'
while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) {
while (preg_match("/(^|.*?\s)-site:([^\*\s]\S*)(\s.*|$)/si", $this->q, $match)) {
$this->hostBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
......@@ -1143,7 +1178,7 @@ class MetaGer
private function searchCheckDomainBlacklist($request)
{
// matches '[... ]-site:*.test.de[ ...]'
while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
while (preg_match("/(^|.*?\s)-site:\*\.(\S+)(\s.*|$)/si", $this->q, $match)) {
$this->domainBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
......@@ -1177,7 +1212,7 @@ class MetaGer
private function searchCheckUrlBlacklist()
{
// matches '[... ]-site:*.test.de[ ...]'
while (preg_match("/(^|.+\s)-url:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
while (preg_match("/(^|.*?\s)-url:(\S+)(\s.*|$)/si", $this->q, $match)) {
$this->urlBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
......@@ -1194,11 +1229,18 @@ class MetaGer
private function searchCheckStopwords($request)
{
$oldQ = $this->q;
// matches '[... ]-test[ ...]'
while (preg_match("/(^|[^\s]+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->stopWords[] = $match[2];
$this->q = $match[1] . $match[3];
$words = preg_split("/\s+/si",$this->q);
$newQ = "";
foreach($words as $word){
if(strpos($word, "-") === 0 && strlen($word) > 1){
$this->stopWords[] = substr($word, 1);
}else{
$newQ .= " " . $word;
}
}
$this->q = trim($newQ);
# Overwrite Setting if submitted via Parameter
if ($request->has('stop')) {
$this->stopWords = [];
......@@ -1212,6 +1254,7 @@ class MetaGer
} else {
$this->stopWords[] = $stop;
}
$this->q = $oldQ;
}
// print the stopwords as a user warning
if (sizeof($this->stopWords) > 0) {
......@@ -1224,24 +1267,6 @@ class MetaGer
}
}
private function searchCheckPhrase()
{
$p = "";
$tmp = $this->q;
// matches '[... ]"test satz"[ ...]'
while (preg_match("/(^|.+\s)\"(.+)\"(?:\s(.+)|($))/si", $tmp, $match)) {
$tmp = $match[1] . $match[3];
$this->phrases[] = $match[2];
}
foreach ($this->phrases as $phrase) {
$p .= "\"$phrase\", ";
}
$p = rtrim($p, ", ");
if (sizeof($this->phrases) > 0) {
$this->warnings[] = trans('metaGer.formdata.phrase', ['phrase' => $p]);
}
}
private function searchCheckNoSearch()
{
if ($this->q === "") {
......@@ -1480,6 +1505,14 @@ class MetaGer
# Einfache Getter
public function getVerificationId() {
return $this->verificationId;
}
public function getVerificationCount() {
return $this->verificationCount;
}
public function getSite()
{
return $this->site;
......@@ -1539,6 +1572,12 @@ class MetaGer
return $this->lang;
}
public function getAvailableFoki()
{
return $this->availableFoki;
}
public function getSprueche()
{
return $this->sprueche;
......
app/Models/Result.php
View file @ 0b99ab10
......@@ -35,11 +35,12 @@ class Result
$this->titel = strip_tags(trim($titel));
$this->link = trim($link);
$this->anzeigeLink = trim($anzeigeLink);
$this->anzeigeLink = preg_replace("/(http[s]{0,1}:\/\/){0,1}(www\.){0,1}/si", "", $this->anzeigeLink);
$this->descr = strip_tags(trim($descr), '<p>');
$this->descr = preg_replace("/\n+/si", " ", $this->descr);
$this->longDescr = $this->descr;
if (strlen($this->descr) > 250) {
$this->descr = wordwrap($this->descr, 250);
if (strlen($this->descr) > 150) {
$this->descr = wordwrap($this->descr, 150);
$this->descr = substr($this->descr, 0, strpos($this->descr, "\n"));
}
......
app/Models/Searchengine.php
View file @ 0b99ab10
......@@ -15,12 +15,12 @@ abstract class Searchengine
public $ch; # Curl Handle zum erhalten der Ergebnisse
public $getString = ""; # Der String für die Get-Anfrage
public $engine; # Die ursprüngliche Engine XML
public $enabled = true; # true, wenn die Suchmaschine nicht explizit disabled ist
public $results = []; # Die geladenen Ergebnisse
public $ads = []; # Die geladenen Werbungen
public $enabled = true; # true, wenn die Suchmaschine nicht explizit disabled ist
public $results = []; # Die geladenen Ergebnisse
public $ads = []; # Die geladenen Werbungen
public $products = []; # Die geladenen Produkte
public $loaded = false; # wahr, sobald die Ergebnisse geladen wurden
public $cached = false;
public $loaded = false; # wahr, sobald die Ergebnisse geladen wurden
public $cached = false;
public $ip; # Die IP aus der metager
public $uses; # Die Anzahl der Nutzungen dieser Suchmaschine
......@@ -31,10 +31,13 @@ abstract class Searchengine
public $startTime; # Die Zeit der Erstellung dieser Suchmaschine
public $hash; # Der Hash-Wert dieser Suchmaschine
private $user; # Username für HTTP-Auth (falls angegeben)
private $password; # Passwort für HTTP-Auth (falls angegeben)
public $fp; # Wird für Artefakte benötigt
public $socketNumber = null; # Wird für Artefakte benötigt
public $counter = 0; # Wird eventuell für Artefakte benötigt
public $write_time = 0; # Wird eventuell für Artefakte benötigt
public $socketNumber = null; # Wird für Artefakte benötigt
public $counter = 0; # Wird eventuell für Artefakte benötigt
public $write_time = 0; # Wird eventuell für Artefakte benötigt
public $connection_time = 0; # Wird eventuell für Artefakte benötigt
public function __construct(\SimpleXMLElement $engine, MetaGer $metager)
......@@ -69,7 +72,7 @@ abstract class Searchengine
}
$this->useragent = $metager->getUserAgent();
$this->ip = $metager->getIp();
$this->ip = $metager->getIp();
$this->startTime = microtime();
# Suchstring generieren
......@@ -84,10 +87,10 @@ abstract class Searchengine
} else {
$q = $metager->getQ();
}
$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
$this->hash = md5($this->host . $this->getString . $this->port . $this->name);
$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
$this->hash = md5($this->host . $this->getString . $this->port . $this->name);
$this->resultHash = $metager->getHashCode();
$this->canCache = $metager->canCache();
$this->canCache = $metager->canCache();
if (!isset($this->additionalHeaders)) {$this->additionalHeaders = "";}
}
......@@ -114,13 +117,13 @@ abstract class Searchengine
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if($this->port === "443"){
if ($this->port === "443") {
$url = "https://";
}else{
} else {
$url = "http://";
}
$url .= $this->host;
if($this->port !== 80 && $this->port !== 443){
if ($this->port !== 80 && $this->port !== 443) {
$url .= ":" . $this->port;
}
$url .= $this->getString;
......@@ -132,27 +135,27 @@ abstract class Searchengine
Redis::rpush($this->name . ".queue", $mission);
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher = false;
$searcherData = Redis::hgetall($this->name . ".stats");
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if(sizeof($searcherData) === 0){
if (sizeof($searcherData) === 0) {
$needSearcher = true;
}else{
} else {
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
......@@ -160,18 +163,18 @@ abstract class Searchengine
// Let's hardcode a minimum of 100ms between every search job.
// First calculate the median of all Times
$median = 0;
foreach($searcherData as $pid => $data){
foreach ($searcherData as $pid => $data) {
$data = explode(";", $data);
$median += floatval($data[1]);
}
$median /= sizeof($searcherData);
if($median < .1){
if ($median < .1) {
$needSearcher = true;
}
}
if($needSearcher && Redis::get($this->name) !== "locked"){
if ($needSearcher && Redis::get($this->name) !== "locked") {
Redis::set($this->name, "locked");
$this->dispatch(new Searcher($this->name));
$this->dispatch(new Searcher($this->name, $this->user, $this->password));
}
}
}
......@@ -203,7 +206,8 @@ abstract class Searchengine
$this->enabled = true;
}
public function setResultHash($hash){
public function setResultHash($hash)
{
$this->resultHash = $hash;
}
......
app/Models/parserSkripte/Fastbot.php deleted 100644 → 0
View file @ 23cda137
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class Fastbot extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
if (strpos($this->urlEncode($metager->getEingabe()), "%") !== false) {
$this->enabled = false;
return null;
}
}
public function loadResults($result)
{
$result = utf8_encode($result);
$counter = 0;
foreach (explode("\n", $result) as $line) {
$line = trim($line);
if (strlen($line) > 0) {
# Hier bekommen wir jedes einzelne Ergebnis
$result = explode("|:|", $line);
if(count($result) < 4) continue;
$link = $result[1];
$link = substr($link, strpos($link, "href=\"") + 6);
$link = substr($link, 0, strpos($link, "\""));
$counter++;
$this->gefVon = "<a href=\"" . $this->homepage . "\" target=\"_blank\" rel=\"noopener\">" . $this->displayName . " " . trans('results.redirect') . "</a>";
$this->results[] = new \App\Models\Result(
$this->engine,
trim(strip_tags($result[1])),
$link,
$result[3],
$result[2],
$this->displayName,$this->homepage,
$counter
);
}
}
}
}
app/Models/parserSkripte/Overture.php
View file @ 0b99ab10
......@@ -25,10 +25,10 @@ class Overture extends Searchengine
$results = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
foreach ($results as $result) {
$title = $result["title"];
$link = $result->{"ClickUrl"}->__toString();
$title = $result["title"];
$link = $result->{"ClickUrl"}->__toString();
$anzeigeLink = $result["siteHost"];
$descr = $result["description"];
$descr = $result["description"];
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
......@@ -36,7 +36,8 @@ class Overture extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->displayName,
$this->homepage,
$this->counter
);
}
......@@ -44,10 +45,10 @@ class Overture extends Searchengine
# Nun noch die Werbeergebnisse:
$ads = $content->xpath('//Results/ResultSet[@id="searchResults"]/Listing');
foreach ($ads as $ad) {
$title = $ad["title"];
$link = $ad->{"ClickUrl"}->__toString();
$title = $ad["title"];
$link = $ad->{"ClickUrl"}->__toString();
$anzeigeLink = $ad["siteHost"];
$descr = $ad["description"];
$descr = $ad["description"];
$this->counter++;
$this->ads[] = new \App\Models\Result(
$this->engine,
......@@ -55,7 +56,7 @@ class Overture extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->displayName, $this->homepage,
$this->counter
);
}
......@@ -87,9 +88,9 @@ class Overture extends Searchengine
// Yahoo liefert, wenn es keine weiteren Ergebnisse hat immer wieder die gleichen Ergebnisse
// Wir müssen also überprüfen, ob wir am Ende der Ergebnisse sind
$resultCount = $content->xpath('//Results/ResultSet[@id="inktomi"]/MetaData/TotalHits');
$results = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
$results = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
if (isset($resultCount[0]) && sizeof($results) > 0) {
$resultCount = intval($resultCount[0]->__toString());
$resultCount = intval($resultCount[0]->__toString());
$lastResultOnPage = intval($results[sizeof($results) - 1]["rank"]);
if ($resultCount <= $lastResultOnPage) {
return;
......@@ -111,9 +112,9 @@ class Overture extends Searchengine
}
# Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
$next = new Overture(simplexml_load_string($this->engine), $metager);
$next = new Overture(simplexml_load_string($this->engine), $metager);
$next->getString = preg_replace("/&Keywords=.*?&/si", "&", $next->getString) . "&" . $nextArgs;
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
}
}
app/Models/parserSkripte/Scopia.php 0 → 100644
View file @ 0b99ab10
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
class Scopia extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults($result)
{
$result = html_entity_decode($result);
$result = preg_replace("/<description>(.*?)<\/description>/si", "<description><![CDATA[ $1 ]]></description>", $result);
$result = preg_replace("/<title>(.*?)<\/title>/si", "<title><![CDATA[ $1 ]]></title>", $result);
$result = preg_replace("/<url>(.*?)<\/url>/si", "<url><![CDATA[ $1 ]]></url>", $result);
try {
$content = simplexml_load_string($result);
if (!$content) {
return;
}
$results = $content->xpath('//results/result');
foreach ($results as $result) {
$title = $result->title->__toString();
$link = $result->url->__toString();
$anzeigeLink = $link;
$descr = $result->description->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->displayName,
$this->homepage,
$this->counter
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name:");
Log::error($e->getMessage());
return;
}
}
public function getNext(\App\MetaGer $metager, $result)
{
$result = html_entity_decode($result);
$result = str_replace("&", "&amp;", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name:");
Log::error($e->getMessage());
return;
}
if (!$content) {
return;
}
$more = $content->xpath('//results/more')[0]->__toString() === "1" ? true : false;
if ($more) {
$results = $content->xpath('//results/result');
$number = $results[sizeof($results) - 1]->number->__toString();
# Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
$next = new Scopia(simplexml_load_string($this->engine), $metager);
$next->getString = preg_replace("/\\?s=.*?&/si", "?s=" . $number, $next->getString);
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
}
}
}
database/migrations/2018_04_26_114745_create_humanverification_table.php
View file @ 0b99ab10
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class CreateHumanverificationTable extends Migration
{
......@@ -14,9 +14,13 @@ class CreateHumanverificationTable extends Migration
public function up()
{
Schema::create('humanverification', function (Blueprint $table) {
$table->string('id')->unique();
$table->string('uid')->unique();
$table->string('id');
$table->integer('unusedResultPages');
$table->boolean('whitelist');
$table->integer('whitelistCounter');
$table->boolean('locked');
$table->string('lockedKey');
$table->timestamp('updated_at');
});
}
......
database/migrations/2018_05_03_101301_usedurls.php
View file @ 0b99ab10
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class Usedurls extends Migration
{
......@@ -14,11 +14,12 @@ class Usedurls extends Migration
public function up()
{
Schema::create('usedurls', function (Blueprint $table) {
$table->increments('id')->unique();
$table->string('user_id');
$table->text('url');
$table->increments('number')->unique();
$table->string('uid');
$table->string('id');
$table->text('eingabe');
$table->timestamp('created_at');
$table->foreign('user_id')->references('id')->on('humanverification')->onDelete('cascade');
$table->foreign('uid')->references('uid')->on('humanverification')->onDelete('cascade');
});
}
......
database/migrations/2018_08_30_102121_sponsoren_links.php 0 → 100644
View file @ 0b99ab10
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class SponsorenLinks extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('sponsorenlinks', function (Blueprint $table) {
$table->increments('id')->unique();
$table->string('linktext');
$table->string('link');
$table->string('langcode');
$table->timestamp('updated_at');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('sponsorenlinks');
}
}
public/index.php
View file @ 0b99ab10
......@@ -13,6 +13,8 @@ if (isset($_SERVER["HTTP_FORWARDED"]) && isset($_SERVER["HTTP_X_FORWARDED_FOR"])
unset($_SERVER["HTTP_FORWARDED"]);
}
$_SERVER["AGENT"] = $_SERVER["HTTP_USER_AGENT"];
if (isset($_SERVER['HTTP_USER_AGENT'])) {
$agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);
......
public/robots.txt
View file @ 0b99ab10
......@@ -2,7 +2,7 @@ User-agent: *
Disallow: /klassik/meta/
Disallow: /en/klassik/meta/
Disallow: /meta/
Disallow: /en/meta/
Disallow: /*/meta/
Disallow: /qt
Disallow: /en/qt
Disallow: /domains/
......
resources/assets/js/lib/imagesloaded.js deleted 100644 → 0
View file @ 23cda137
/*!
* imagesLoaded PACKAGED v3.1.8
* JavaScript is all like "You images are done yet or what?"
* MIT License
*/
(function(){function e(){}function t(e,t){for(var n=e.length;n--;)if(e[n].listener===t)return n;return-1}function n(e){return function(){return this[e].apply(this,arguments)}}var i=e.prototype,r=this,o=r.EventEmitter;i.getListeners=function(e){var t,n,i=this._getEvents();if("object"==typeof e){t={};for(n in i)i.hasOwnProperty(n)&&e.test(n)&&(t[n]=i[n])}else t=i[e]||(i[e]=[]);return t},i.flattenListeners=function(e){var t,n=[];for(t=0;e.length>t;t+=1)n.push(e[t].listener);return n},i.getListenersAsObject=function(e){var t,n=this.getListeners(e);return n instanceof Array&&(t={},t[e]=n),t||n},i.addListener=function(e,n){var i,r=this.getListenersAsObject(e),o="object"==typeof n;for(i in r)r.hasOwnProperty(i)&&-1===t(r[i],n)&&r[i].push(o?n:{listener:n,once:!1});return this},i.on=n("addListener"),i.addOnceListener=function(e,t){return this.addListener(e,{listener:t,once:!0})},i.once=n("addOnceListener"),i.defineEvent=function(e){return this.getListeners(e),this},i.defineEvents=function(e){for(var t=0;e.length>t;t+=1)this.defineEvent(e[t]);return this},i.removeListener=function(e,n){var i,r,o=this.getListenersAsObject(e);for(r in o)o.hasOwnProperty(r)&&(i=t(o[r],n),-1!==i&&o[r].splice(i,1));return this},i.off=n("removeListener"),i.addListeners=function(e,t){return this.manipulateListeners(!1,e,t)},i.removeListeners=function(e,t){return this.manipulateListeners(!0,e,t)},i.manipulateListeners=function(e,t,n){var i,r,o=e?this.removeListener:this.addListener,s=e?this.removeListeners:this.addListeners;if("object"!=typeof t||t instanceof RegExp)for(i=n.length;i--;)o.call(this,t,n[i]);else for(i in t)t.hasOwnProperty(i)&&(r=t[i])&&("function"==typeof r?o.call(this,i,r):s.call(this,i,r));return this},i.removeEvent=function(e){var t,n=typeof e,i=this._getEvents();if("string"===n)delete i[e];else if("object"===n)for(t in i)i.hasOwnProperty(t)&&e.test(t)&&delete i[t];else delete this._events;return this},i.removeAllListeners=n("removeEvent"),i.emitEvent=function(e,t){var n,i,r,o,s=this.getListenersAsObject(e);for(r in s)if(s.hasOwnProperty(r))for(i=s[r].length;i--;)n=s[r][i],n.once===!0&&this.removeListener(e,n.listener),o=n.listener.apply(this,t||[]),o===this._getOnceReturnValue()&&this.removeListener(e,n.listener);return this},i.trigger=n("emitEvent"),i.emit=function(e){var t=Array.prototype.slice.call(arguments,1);return this.emitEvent(e,t)},i.setOnceReturnValue=function(e){return this._onceReturnValue=e,this},i._getOnceReturnValue=function(){return this.hasOwnProperty("_onceReturnValue")?this._onceReturnValue:!0},i._getEvents=function(){return this._events||(this._events={})},e.noConflict=function(){return r.EventEmitter=o,e},"function"==typeof define&&define.amd?define("eventEmitter/EventEmitter",[],function(){return e}):"object"==typeof module&&module.exports?module.exports=e:this.EventEmitter=e}).call(this),function(e){function t(t){var n=e.event;return n.target=n.target||n.srcElement||t,n}var n=document.documentElement,i=function(){};n.addEventListener?i=function(e,t,n){e.addEventListener(t,n,!1)}:n.attachEvent&&(i=function(e,n,i){e[n+i]=i.handleEvent?function(){var n=t(e);i.handleEvent.call(i,n)}:function(){var n=t(e);i.call(e,n)},e.attachEvent("on"+n,e[n+i])});var r=function(){};n.removeEventListener?r=function(e,t,n){e.removeEventListener(t,n,!1)}:n.detachEvent&&(r=function(e,t,n){e.detachEvent("on"+t,e[t+n]);try{delete e[t+n]}catch(i){e[t+n]=void 0}});var o={bind:i,unbind:r};"function"==typeof define&&define.amd?define("eventie/eventie",o):e.eventie=o}(this),function(e,t){"function"==typeof define&&define.amd?define(["eventEmitter/EventEmitter","eventie/eventie"],function(n,i){return t(e,n,i)}):"object"==typeof exports?module.exports=t(e,require("wolfy87-eventemitter"),require("eventie")):e.imagesLoaded=t(e,e.EventEmitter,e.eventie)}(window,function(e,t,n){function i(e,t){for(var n in t)e[n]=t[n];return e}function r(e){return"[object Array]"===d.call(e)}function o(e){var t=[];if(r(e))t=e;else if("number"==typeof e.length)for(var n=0,i=e.length;i>n;n++)t.push(e[n]);else t.push(e);return t}function s(e,t,n){if(!(this instanceof s))return new s(e,t);"string"==typeof e&&(e=document.querySelectorAll(e)),this.elements=o(e),this.options=i({},this.options),"function"==typeof t?n=t:i(this.options,t),n&&this.on("always",n),this.getImages(),a&&(this.jqDeferred=new a.Deferred);var r=this;setTimeout(function(){r.check()})}function f(e){this.img=e}function c(e){this.src=e,v[e]=this}var a=e.jQuery,u=e.console,h=u!==void 0,d=Object.prototype.toString;s.prototype=new t,s.prototype.options={},s.prototype.getImages=function(){this.images=[];for(var e=0,t=this.elements.length;t>e;e++){var n=this.elements[e];"IMG"===n.nodeName&&this.addImage(n);var i=n.nodeType;if(i&&(1===i||9===i||11===i))for(var r=n.querySelectorAll("img"),o=0,s=r.length;s>o;o++){var f=r[o];this.addImage(f)}}},s.prototype.addImage=function(e){var t=new f(e);this.images.push(t)},s.prototype.check=function(){function e(e,r){return t.options.debug&&h&&u.log("confirm",e,r),t.progress(e),n++,n===i&&t.complete(),!0}var t=this,n=0,i=this.images.length;if(this.hasAnyBroken=!1,!i)return this.complete(),void 0;for(var r=0;i>r;r++){var o=this.images[r];o.on("confirm",e),o.check()}},s.prototype.progress=function(e){this.hasAnyBroken=this.hasAnyBroken||!e.isLoaded;var t=this;setTimeout(function(){t.emit("progress",t,e),t.jqDeferred&&t.jqDeferred.notify&&t.jqDeferred.notify(t,e)})},s.prototype.complete=function(){var e=this.hasAnyBroken?"fail":"done";this.isComplete=!0;var t=this;setTimeout(function(){if(t.emit(e,t),t.emit("always",t),t.jqDeferred){var n=t.hasAnyBroken?"reject":"resolve";t.jqDeferred[n](t)}})},a&&(a.fn.imagesLoaded=function(e,t){var n=new s(this,e,t);return n.jqDeferred.promise(a(this))}),f.prototype=new t,f.prototype.check=function(){var e=v[this.img.src]||new c(this.img.src);if(e.isConfirmed)return this.confirm(e.isLoaded,"cached was confirmed"),void 0;if(this.img.complete&&void 0!==this.img.naturalWidth)return this.confirm(0!==this.img.naturalWidth,"naturalWidth"),void 0;var t=this;e.on("confirm",function(e,n){return t.confirm(e.isLoaded,n),!0}),e.check()},f.prototype.confirm=function(e,t){this.isLoaded=e,this.emit("confirm",this,t)};var v={};return c.prototype=new t,c.prototype.check=function(){if(!this.isChecked){var e=new Image;n.bind(e,"load",this),n.bind(e,"error",this),e.src=this.src,this.isChecked=!0}},c.prototype.handleEvent=function(e){var t="on"+e.type;this[t]&&this[t](e)},c.prototype.onload=function(e){this.confirm(!0,"onload"),this.unbindProxyEvents(e)},c.prototype.onerror=function(e){this.confirm(!1,"onerror"),this.unbindProxyEvents(e)},c.prototype.confirm=function(e,t){this.isConfirmed=!0,this.isLoaded=e,this.emit("confirm",this,t)},c.prototype.unbindProxyEvents=function(e){n.unbind(e.target,"load",this),n.unbind(e.target,"error",this)},s});
\ No newline at end of file
resources/assets/js/lib/openpgp.min.js deleted 100644 → 0
View file @ 23cda137
source diff could not be displayed: it is too large. Options to address this: view the blob.
resources/assets/js/searchbar.js
View file @ 0b99ab10
$(function () {
$(function() {
loadLocalStorage();
setActionListenersSearchbar();
updateLangLabelCode();
});
function setActionListenersSearchbar() {
$('#input-key').change(saveKey);
$('#input-lang').change(saveLang);
$("#input-key").change(saveKey);
$("#input-lang").change(saveLang);
}
function saveKey() {
var key = $('#input-key').val();
localStorage.setItem('key', key);
var key = $("#input-key").val();
localStorage.setItem("key", key);
}
function loadKey() {
var key = localStorage.getItem('key');
var key = localStorage.getItem("key");
if (key != null) {
$('#input-key').val(key);
$("#input-key").val(key);
}
}
function saveLang() {
var lang = $('#input-lang').val();
localStorage.setItem('lang', lang);
var lang = $("#input-lang").val();
if (lang != "all") localStorage.setItem("lang", lang);
else localStorage.removeItem("lang");
updateLangLabelCode(lang);
}
function loadLang() {
var lang = localStorage.getItem('lang');
var lang = localStorage.getItem("lang");
if (lang != null) {
$('#input-lang').val(lang);
$("#input-lang").val(lang);
}
}
......@@ -46,12 +47,12 @@ function loadLocalStorage() {
}
function setSettings() {
var acceptedParams = ['autocomplete', 'key', 'lang', 'newtab', 'sprueche'];
var acceptedParams = ["autocomplete", "key", "lang", "newtab", "sprueche"];
for (var key in localStorage) {
var value = localStorage.getItem(key);
var accepted = false;
for (var i in acceptedParams) {
if (key === 'param_' + acceptedParams[i]) {
if (key === "param_" + acceptedParams[i]) {
accepted = true;
}
}
......@@ -61,20 +62,26 @@ function setSettings() {
var existing = $('.search-hidden input[name="' + key + '"]');
if (existing.length === 0) {
// if none exist, create a new one
$('.search-hidden').append('<input type="hidden" name="' + key + '" value="' + value + '">');
$(".search-hidden").append(
'<input type="hidden" name="' + key + '" value="' + value + '">'
);
}
}
}
// Change the request method to the given parameter
var requestMethod = localStorage.getItem('request');
if (requestMethod !== null && (requestMethod === 'GET' || requestMethod === 'POST')) {
$('#searchForm').attr('method', requestMethod);
var requestMethod = localStorage.getItem("request");
if (
requestMethod !== null &&
(requestMethod === "GET" || requestMethod === "POST")
) {
$("#searchForm").attr("method", requestMethod);
}
}
function updateLangLabelCode(langcode = null) {
if (!langcode) {
var langcode = localStorage.getItem('lang');
var langcode = localStorage.getItem("lang");
}
$('#lang-label-code').html(langcode);
}
\ No newline at end of file
if (langcode == "all") langcode = "";
$("#lang-label-code").html(langcode);
}