Compare revisions

Karl Hasselbring · Phil Höfer · Karl Hasselbring · Phil Höfer · Karl Hasselbring · Karl Hasselbring
--- a/app/Http/Controllers/FokiLoader.php
+++ b/app/Http/Controllers/FokiLoader.php
@@ -10,12 +10,12 @@ class FokiLoader
    {
        $sumaFile = "";
        if (App::isLocale('en')) {
-            $sumaFile = config_path() . "/sumas.xml";
+            $sumaFile = config_path() . "/sumasEn.xml";
        } else {
            $sumaFile = config_path() . "/sumas.xml";
        }
-        $xml   = simplexml_load_file($sumaFile);
+        $xml = simplexml_load_file($sumaFile);
        $sumas = $xml->xpath("suma");
        $foki = [];
@@ -24,13 +24,13 @@ class FokiLoader
                if (isset($suma['type'])) {
                    $f = explode(",", $suma['type']->__toString());
                    foreach ($f as $tmp) {
-                        $displayName                             = $suma['displayName']->__toString();
+                        $displayName = $suma['displayName']->__toString();
-                        $url                                     = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
+                        $url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
                        $foki[$tmp][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
                    }
                } else {
-                    $displayName                                 = $suma['displayName']->__toString();
+                    $displayName = $suma['displayName']->__toString();
-                    $url                                         = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
+                    $url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
                    $foki["andere"][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
                }
            }

--- a/app/Http/Controllers/HumanVerification.php
+++ b/app/Http/Controllers/HumanVerification.php
@@ -2,66 +2,120 @@
 namespace App\Http\Controllers;
+use Captcha;
+use Carbon;
+use DB;
+use Illuminate\Hashing\BcryptHasher as Hasher;
 use Illuminate\Http\Request;
-use Validator;
 use Input;
-use DB;
-use Carbon;
 class HumanVerification extends Controller
 {
-    public static function captcha(Request $request, $id, $url){
+    public static function captcha(Request $request, Hasher $hasher, $id, $url = null)
-        if($request->getMethod() == 'POST'){
+    {
-            $rules = ['captcha' => 'required|captcha'];
+        if ($url != null) {
-            $validator = Validator::make($request->all(), $rules);
+            $url = base64_decode(str_replace("<<SLASH>>", "/", $url));
-            if($validator->fails()){
+        } else {
-                return view('captcha')->with('title', 'Bestätigung notwendig')->with('id', $id)->with('url', base64_decode($url))->with('errorMessage', 'Bitte Captcha eingeben:');
+            $url = $request->input('url');
-            }else{
+        }
-                # If we can unlock the Account of this user we will redirect him to the result page
-                $id = $request->input('id');
-                $url = $request->input('url');
-                $user = DB::table('humanverification')->where('id', $id)->first();
+        if ($request->getMethod() == 'POST') {
-                if($user !== null && $user->locked === 1){
+            $user = DB::table('humanverification')->where('uid', $id)->first();
-                    DB::table('humanverification')->where('id', $id)->update(['locked' => false]);
+            $lockedKey = $user->lockedKey;
+            $key = $request->input('captcha');
+            $key = strtolower($key);
+            if (!$hasher->check($key, $lockedKey)) {
+                $captcha = Captcha::create("default", true);
+                DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
+                return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
+                    ->with('id', $id)
+                    ->with('url', $url)
+                    ->with('image', $captcha["img"])
+                    ->with('errorMessage', 'Fehler: Falsches Captcha eingegeben!');
+            } else {
+                # If we can unlock the Account of this user we will redirect him to the result page
+                if ($user !== null && $user->locked === 1) {
+                    # The Captcha was correct. We can remove the key from the user
+                    DB::table('humanverification')->where('uid', $id)->update(['locked' => false, 'lockedKey' => "", 'whitelist' => 1]);
                    return redirect($url);
-                }else{
+                } else {
                    return redirect('/');
                }
            }
        }
-        return view('captcha')->with('title', 'Bestätigung notwendig')->with('id', $id)->with('url', base64_decode($url));
+        $captcha = Captcha::create("default", true);
+        DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
+        return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
+            ->with('id', $id)
+            ->with('url', $url)
+            ->with('image', $captcha["img"]);
    }
-    public static function remove(Request $request){
+    public static function remove(Request $request)
-        if(!$request->has('mm')){
+    {
+        if (!$request->has('mm')) {
            abort(404, "Keine Katze gefunden.");
        }
-        $id = md5($request->ip());
-        if(HumanVerification::checkId($request, $request->input('mm'))){
+        if (HumanVerification::checkId($request, $request->input('mm'))) {
-            # Remove the entry from the database
+            HumanVerification::removeUser($request, $request->input('mm'));
-            DB::table('humanverification')->where('id', $id)->where('updated_at', '<', Carbon::NOW()->subSeconds(2) )->delete();
        }
        return response(hex2bin('89504e470d0a1a0a0000000d494844520000000100000001010300000025db56ca00000003504c5445000000a77a3dda0000000174524e530040e6d8660000000a4944415408d76360000000020001e221bc330000000049454e44ae426082'), 200)
            ->header('Content-Type', 'image/png');
    }
-    public static function removeGet(Request $request, $mm, $password, $url){
+    public static function removeGet(Request $request, $mm, $password, $url)
-        $url = base64_decode($url);
+    {
+        $url = base64_decode(str_replace("<<SLASH>>", "/", $url));
        # If the user is correct and the password is we will delete any entry in the database
        $requiredPass = md5($mm . Carbon::NOW()->day . $url . env("PROXY_PASSWORD"));
-        if(HumanVerification::checkId($request, $mm) && $requiredPass === $password){
+        if (HumanVerification::checkId($request, $mm) && $requiredPass === $password) {
-            # Remove the entry from the database
+            HumanVerification::removeUser($request, $mm);
-            DB::table('humanverification')->where('id', $mm)->where('updated_at', '<', Carbon::NOW()->subSeconds(2) )->delete();
        }
        return redirect($url);
    }
-    private static function checkId($request, $id){
+    private static function removeUser($request, $uid)
-        if(md5($request->ip()) === $id){
+    {
+        $id = hash("sha512", $request->ip());
+        $sum = DB::table('humanverification')->where('id', $id)->where('whitelist', false)->sum('unusedResultPages');
+        $user = DB::table('humanverification')->where('uid', $uid)->first();
+        if ($user === null) {
+            return;
+        }
+        # Check if we have to whitelist the user or if we can simply delete the data
+        if ($user->unusedResultPages < $sum && $user->whitelist === 0) {
+            # Whitelist
+            DB::table('humanverification')->where('uid', $uid)->update(['whitelist' => true, 'whitelistCounter' => 0]);
+            $user->whitelist = 1;
+            $user->whitelistCounter = 0;
+        }
+        if ($user->whitelist === 1) {
+            if (
+                DB::table('humanverification')->where('uid', $uid)->update(['unusedResultPages' => 0])
+                === 1
+            ) {
+                DB::table('usedurls')->where('uid', $uid)->delete();
+            }
+        } else {
+            DB::table('humanverification')->where('uid', $uid)->where('updated_at', '<', Carbon::NOW()->subSeconds(2))->delete();
+        }
+    }
+    private static function checkId($request, $id)
+    {
+        if (hash("sha512", $request->ip() . $_SERVER["AGENT"]) === $id) {
            return true;
-        }else{
+        } else {
            return false;
        }
    }

--- a/app/Http/Controllers/StartpageController.php
+++ b/app/Http/Controllers/StartpageController.php
@@ -5,8 +5,8 @@ namespace App\Http\Controllers;
 use App;
 use Illuminate\Http\Request;
 use Jenssegers\Agent\Agent;
-use Response;
 use LaravelLocalization;
+use Response;
 class StartpageController extends Controller
 {
@@ -20,7 +20,7 @@ class StartpageController extends Controller
    public function loadStartPage(Request $request)
    {
        $focusPages = [];
-        $theme      = "default";
+        $theme = "default";
        $optionParams = ['param_sprueche', 'param_newtab', 'param_maps', 'param_autocomplete', 'param_lang', 'param_key'];
        $option_values = [];
@@ -33,12 +33,12 @@ class StartpageController extends Controller
        }
        $autocomplete = 'on';
-        if(in_array('autocomplete', array_keys($option_values))) {
+        if (in_array('autocomplete', array_keys($option_values))) {
            $autocomplete = $option_values['autocomplete'];
        }
        $lang = LaravelLocalization::getCurrentLocale();
-        if ($lang === 'de') {
+        if ($lang === 'de' || $lang === "en") {
            $lang = 'all';
        }
@@ -78,9 +78,9 @@ class StartpageController extends Controller
    public function loadPlugin($params, $locale = "de")
    {
-        $params   = unserialize(base64_decode($params));
+        $params = unserialize(base64_decode($params));
        $requests = $params;
-        $params   = [];
+        $params = [];
        foreach ($requests as $key => $value) {
            if (strpos($key, "param_") === 0) {
                $key = substr($key, strpos($key, "param_") + 6);
@@ -135,12 +135,12 @@ class StartpageController extends Controller
    public function berlin(Request $request)
    {
-        $link     = "";
+        $link = "";
        $password = "";
        if ($request->filled('eingabe')) {
            $password = getenv('berlin');
            $password = md5($request->input('eingabe') . " -host:userpage.fu-berlin.de" . $password);
-            $link     = "/meta/meta.ger3?eingabe=" . $request->input('eingabe') . " -host:userpage.fu-berlin.de&focus=web&password=" . $password . "&encoding=utf8&lang=all&site=fu-berlin.de&quicktips=off&out=results-with-style";
+            $link = "/meta/meta.ger3?eingabe=" . $request->input('eingabe') . " -host:userpage.fu-berlin.de&focus=web&password=" . $password . "&encoding=utf8&lang=all&site=fu-berlin.de&quicktips=off&out=results-with-style";
        }
        return view('berlin')
            ->with('title', 'Testseite für die FU-Berlin')

--- a/app/Http/Kernel.php
+++ b/app/Http/Kernel.php
@@ -36,11 +36,6 @@ class Kernel extends HttpKernel
        ],
        'session' => [
-            \App\Http\Middleware\EncryptCookies::class,
-            \Illuminate\Cookie\Middleware\AddQueuedCookiesToResponse::class,
-            \Illuminate\Session\Middleware\StartSession::class,
-            \Illuminate\View\Middleware\ShareErrorsFromSession::class,
-            \App\Http\Middleware\VerifyCsrfToken::class,
            \Illuminate\Routing\Middleware\SubstituteBindings::class,
        ],
    ];
@@ -53,12 +48,12 @@ class Kernel extends HttpKernel
     * @var array
     */
    protected $routeMiddleware = [
-        'auth'          => \Illuminate\Auth\Middleware\Authenticate::class,
+        'auth' => \Illuminate\Auth\Middleware\Authenticate::class,
-        'auth.basic'    => \Illuminate\Auth\Middleware\AuthenticateWithBasicAuth::class,
+        'auth.basic' => \Illuminate\Auth\Middleware\AuthenticateWithBasicAuth::class,
-        'bindings'      => \Illuminate\Routing\Middleware\SubstituteBindings::class,
+        'bindings' => \Illuminate\Routing\Middleware\SubstituteBindings::class,
-        'can'           => \Illuminate\Auth\Middleware\Authorize::class,
+        'can' => \Illuminate\Auth\Middleware\Authorize::class,
-        'guest'         => \App\Http\Middleware\RedirectIfAuthenticated::class,
+        'guest' => \App\Http\Middleware\RedirectIfAuthenticated::class,
-        'throttle'      => \Illuminate\Routing\Middleware\ThrottleRequests::class,
+        'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
        'referer.check' => \App\Http\Middleware\RefererCheck::class,
        'humanverification' => \App\Http\Middleware\HumanVerification::class,
    ];

--- a/app/Http/Middleware/HumanVerification.php
+++ b/app/Http/Middleware/HumanVerification.php
@@ -2,9 +2,12 @@
 namespace App\Http\Middleware;
+use Captcha;
+use Carbon;
 use Closure;
 use DB;
-use Carbon;
+use Illuminate\Http\Response;
+use URL;
 class HumanVerification
 {
@@ -17,52 +20,113 @@ class HumanVerification
     */
    public function handle($request, Closure $next)
    {
-        $id = md5($request->ip());
+        try {
+            $id = hash("sha512", $request->ip());
-        /**
+            $uid = hash("sha512", $request->ip() . $_SERVER["AGENT"]);
-         * If the user sends a Password or a key
+            unset($_SERVER["AGENT"]);
-         * We will not verificate the user.
-         * If someone that uses a bot finds this out we 
+            /**
-         * might have to change it at some point.
+             * If the user sends a Password or a key
-         */
+             * We will not verificate the user.
-        if($request->filled('password') || $request->filled('key') || $request->filled('appversion') || !env('BOT_PROTECTION', false)){
+             * If someone that uses a bot finds this out we
-            return $next($request);
+             * might have to change it at some point.
-        }
+             */
+            if ($request->filled('password') || $request->filled('key') || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
+                return $next($request);
+            }
+            // The specific user
+            $user = DB::table('humanverification')->where('uid', $uid)->first();
+            $createdAt = Carbon::now();
+            $unusedResultPages = 1;
+            $locked = false;
+            # If this user doesn't have an entry we will create one
+            if ($user === null) {
+                DB::table('humanverification')->insert(
+                    [
+                        'uid' => $uid,
+                        'id' => $id,
+                        'unusedResultPages' => 0,
+                        'whitelist' => false,
+                        'whitelistCounter' => 0,
+                        'locked' => false,
+                        "lockedKey" => "",
+                        'updated_at' => Carbon::now(),
+                    ]
+                );
+                # Insert the URL the user tries to reach
+                $url = url()->full();
+                DB::table('usedurls')->insert(['uid' => $uid, 'id' => $id, 'eingabe' => $request->input('eingabe', '')]);
+                $user = DB::table('humanverification')->where('uid', $uid)->first();
+            }
+            # Lock out everyone in a Bot network
+            # Find out how many requests this IP has made
+            $sum = DB::table('humanverification')->where('id', $id)->where('whitelist', false)->sum('unusedResultPages');
+            # A lot of automated requests are from websites that redirect users to our result page.
+            # We will detect those requests and put a captcha
+            $referer = URL::previous();
+            # Just the URL-Parameter
+            $refererLock = false;
+            if (stripos($referer, "?") !== false) {
+                $referer = substr($referer, stripos($referer, "?") + 1);
+                $referer = urldecode($referer);
+                if (preg_match("/http[s]{0,1}:\/\/metager\.de\/meta\/meta.ger3\?.*?eingabe=([\w\d]+\.){1,2}[\w\d]+/si", $referer) === 1) {
+                    $refererLock = true;
+                }
+            }
+            // Defines if this is the only user using that IP Adress
+            $alone = DB::table('humanverification')->where('id', $id)->count() === 1;
+            if ((!$alone && $sum >= 50 && $user->whitelist !== 1) || $refererLock) {
+                DB::table('humanverification')->where('uid', $uid)->update(['locked' => true]);
+                $user->locked = 1;
+            }
+            # If the user is locked we will force a Captcha validation
+            if ($user->locked === 1) {
+                $captcha = Captcha::create("default", true);
+                DB::table('humanverification')->where('uid', $uid)->update(['lockedKey' => $captcha["key"]]);
+                return
+                new Response(
+                    view('humanverification.captcha')
+                        ->with('title', "Bestätigung erforderlich")
+                        ->with('id', $uid)
+                        ->with('url', url()->full())
+                        ->with('image', $captcha["img"])
+                );
+            }
-        $user = DB::table('humanverification')->where('id', $id)->first();
-        $createdAt = now();
-        $unusedResultPages = 1;
-        $locked = false;
-        # If this user doesn't have an entry we will create one
-        if($user === null){
-            DB::table('humanverification')->insert(
-                ['id' => $id, 'unusedResultPages' => 1, 'locked' => false, 'updated_at' => now()]
-            );
-            # Insert the URL the user tries to reach
-            DB::table('usedurls')->insert(['user_id' => $id, 'url' => url()->full()]);
-            $user = DB::table('humanverification')->where('id', $id)->first();
-        }else if($user->locked !== 1){
            $unusedResultPages = intval($user->unusedResultPages);
            $unusedResultPages++;
-            # We have different security gates:
+            $locked = false;
-            #   50, 75, 85, >=90 => Captcha validated Result Pages
-            # If the user shows activity on our result page the counter will be deleted
+            if ($alone || $user->whitelist === 1) {
-            # Maybe I'll add a ban if the user reaches 100
+                # This IP doesn't need verification yet
-            if($unusedResultPages === 50 || $unusedResultPages === 75 || $unusedResultPages === 85 || $unusedResultPages >= 90){
+                # The user currently isn't locked
-                $locked = true;
+                # We have different security gates:
+                #   50, 75, 85, >=90 => Captcha validated Result Pages
+                # If the user shows activity on our result page the counter will be deleted
+                # Maybe I'll add a ban if the user reaches 100
+                if ($unusedResultPages === 50 || $unusedResultPages === 75 || $unusedResultPages === 85 || $unusedResultPages >= 90) {
+                    $locked = true;
+                }
            }
-            DB::table('humanverification')->where('id', $id)->update(['unusedResultPages' => $unusedResultPages, 'locked' => $locked,  'updated_at' => $createdAt]);
+            DB::table('humanverification')->where('uid', $uid)->update(['unusedResultPages' => $unusedResultPages, 'locked' => $locked]);
            # Insert the URL the user tries to reach
-            DB::table('usedurls')->insert(['user_id' => $id, 'url' => url()->full()]);
+            DB::table('usedurls')->insert(['uid' => $uid, 'id' => $id, 'eingabe' => $request->input('eingabe', '')]);
-        }
-        $request->request->add(['verification_id' => $id, 'verification_count' => $unusedResultPages]);
-        # If the user is locked we will force a Captcha validation
+        } catch (\Illuminate\Database\QueryException $e) {
-        if($user->locked === 1){
+            // Failure in contacting metager3.de
-            return redirect('meta/verification/' . $id . '/' . urlencode(base64_encode(url()->full())));
        }
+        $request->request->add(['verification_id' => $uid, 'verification_count' => $unusedResultPages]);
        return $next($request);
    }
 }
--- a/app/Jobs/Searcher.php
+++ b/app/Jobs/Searcher.php
@@ -3,18 +3,16 @@
 namespace App\Jobs;
 use Illuminate\Bus\Queueable;
-use Illuminate\Queue\SerializesModels;
-use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Contracts\Queue\ShouldQueue;
-use Illuminate\Foundation\Bus\DispatchesJobs;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Redis;
-use Log;
 class Searcher implements ShouldQueue
 {
    use InteractsWithQueue, Queueable, SerializesModels;
-    protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo;
+    protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo, $user, $password;
    # Each Searcher will shutdown after a specified time(s) or number of requests
    protected $MAX_REQUESTS = 100;
    # This value should always be below the retry_after value in config/queue.php
@@ -34,12 +32,14 @@ class Searcher implements ShouldQueue
     * keep-alive requests.
     * @return void
     */
-    public function __construct($name)
+    public function __construct($name, $user = null, $password = null)
    {
        $this->name = $name;
        $this->pid = getmypid();
        $this->recheck = false;
        $this->startTime = microtime(true);
+        $this->user = $user;
+        $this->password = $password;
        // Submit this worker to the Redis System
        Redis::expire($this->name, 5);
    }
@@ -53,95 +53,101 @@ class Searcher implements ShouldQueue
    {
        // This Searches is freshly called so we need to initialize the curl handle $ch
        $this->ch = $this->initCurlHandle();
-        $this->counter = 0;                 // Counts the number of answered jobs
+        try {
-        $time = microtime(true);
+            $this->counter = 0; // Counts the number of answered jobs
-        while(true){
+            $time = microtime(true);
-            // Update the expire
+            while (true) {
-            Redis::expire($this->name, 5);
+                // Update the expire
-            Redis::expire($this->name . ".stats", 5);
+                Redis::expire($this->name, 5);
-            // One Searcher can handle a ton of requests to the same server
+                Redis::expire($this->name . ".stats", 5);
-            // Each search to the server of this Searcher will be submitted to a queue
+                // One Searcher can handle a ton of requests to the same server
-            // stored in redis which has the same name as this searchengine appended by a ".queue"
+                // Each search to the server of this Searcher will be submitted to a queue
-            // We will perform a blocking pop on this queue so the queue can remain empty for a while 
+                // stored in redis which has the same name as this searchengine appended by a ".queue"
-            // without killing this searcher directly.
+                // We will perform a blocking pop on this queue so the queue can remain empty for a while
-            $mission = Redis::blpop($this->name . ".queue", 4);
+                // without killing this searcher directly.
-            $this->counter++;
+                $mission = Redis::blpop($this->name . ".queue", 4);
-            $this->updateStats(microtime(true) - $time);
+                $this->counter++;
-            $this->switchToRunning();
+                $this->updateStats(microtime(true) - $time);
-            // The mission can be empty when blpop hit the timeout
+                $this->switchToRunning();
-            if(!empty($mission)){
+                // The mission can be empty when blpop hit the timeout
-                $mission = $mission[1];
+                if (!empty($mission)) {
-                $poptime = microtime(true) - $time;
+                    $mission = $mission[1];
+                    $poptime = microtime(true) - $time;
-                // The mission is a String which can be divided to retrieve three informations:
-                // 1. The Hash Value where the result should be stored
+                    // The mission is a String which can be divided to retrieve three informations:
-                // 2. The Url to Retrieve
+                    // 1. The Hash Value where the result should be stored
-                // 3. The maximum time to take
+                    // 2. The Url to Retrieve
-                // These three informations are divided by a ";" in the mission string
+                    // 3. The maximum time to take
-                $mission = explode(";", $mission);
+                    // These three informations are divided by a ";" in the mission string
-                $hashValue = $mission[0]; // The hash value for redis to store the results under
+                    $mission = explode(";", $mission);
-                $url = base64_decode($mission[1]); // The url to fetch
+                    $hashValue = $mission[0]; // The hash value for redis to store the results under
-                $timeout = $mission[2]; // Timeout from the MetaGer process in ms
+                    $url = base64_decode($mission[1]); // The url to fetch
-                $medianFetchTime = $this->getFetchTime();   // The median Fetch time of the search engine in ms
+                    $timeout = $mission[2]; // Timeout from the MetaGer process in ms
-                Redis::hset('search.' . $hashValue, $this->name, "connected");
+                    $medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
+                    Redis::hset('search.' . $hashValue, $this->name, "connected");
-                $result = $this->retrieveUrl($url);
+                    $result = $this->retrieveUrl($url);
-                $this->storeResult($result, $poptime, $hashValue);
+                    $this->storeResult($result, $poptime, $hashValue);
-                // Reset the time of the last Job so we can calculate
-                // the time we have spend waiting for a new job
+                    // Reset the time of the last Job so we can calculate
-                // We submit that calculation to the Redis systemin the method
+                    // the time we have spend waiting for a new job
-                $time = microtime(true);
+                    // We submit that calculation to the Redis systemin the method
+                    $time = microtime(true);
+                }
+                // In sync mode every Searcher may only retrieve one result because it would block
+                // the execution of the remaining code otherwise:
+                if (getenv("QUEUE_DRIVER") === "sync"
+                    || $this->counter > $this->MAX_REQUESTS
+                    || (microtime(true) - $this->startTime) > $this->MAX_TIME) {
+                    break;
+                }
            }
+        } finally {
-            // In sync mode every Searcher may only retrieve one result because it would block
+            // When we reach this point, time has come for this Searcher to retire
-            // the execution of the remaining code otherwise:
+            $this->shutdown();
-            if(getenv("QUEUE_DRIVER") === "sync" 
-                || $this->counter > $this->MAX_REQUESTS 
-                || (microtime(true)-$this->startTime) > $this->MAX_TIME){
-               break;
-            } 
        }
-        // When we reach this point, time has come for this Searcher to retire
-        $this->shutdown();
    }
-    private function switchToRunning(){
+    private function switchToRunning()
+    {
        /**
-        * When a Searcher is initially started the redis value for $this->name is set to "locked"
+         * When a Searcher is initially started the redis value for $this->name is set to "locked"
-        * which effectively will prevent new Searchers of this type to be started. (Value is checked by the MetaGer process which starts the Searchers)
+         * which effectively will prevent new Searchers of this type to be started. (Value is checked by the MetaGer process which starts the Searchers)
-        * This is done so the MetaGer processes won't start hundreds of Searchers parallely when under high work load.
+         * This is done so the MetaGer processes won't start hundreds of Searchers parallely when under high work load.
-        * It will force that Searchers can only be started one after the other.
+         * It will force that Searchers can only be started one after the other.
-        * When a new Searcher has served a minimum of three requests we have enough data to decide whether we need even more Searchers.
+         * When a new Searcher has served a minimum of three requests we have enough data to decide whether we need even more Searchers.
-        * To do so we will then set the redis value for $this->name to "running".
+         * To do so we will then set the redis value for $this->name to "running".
-        * There is a case where we don't want new Searchers to be started even if we would need to do so to serve every Request:
+         * There is a case where we don't want new Searchers to be started even if we would need to do so to serve every Request:
-        *   When a search engine needs more time to produce search results than the timeout of the MetaGer process, we won't even bother of spawning
+         *   When a search engine needs more time to produce search results than the timeout of the MetaGer process, we won't even bother of spawning
-        *   more and more Searchers because they would just block free worker processes from serving the important engines which will give results in time.
+         *   more and more Searchers because they would just block free worker processes from serving the important engines which will give results in time.
-        **/
+         **/
-        if($this->counter === 3 || getenv("QUEUE_DRIVER") === "sync"){
+        if ($this->counter === 3 || getenv("QUEUE_DRIVER") === "sync") {
            # If the MetaGer process waits longer for the results than this Fetcher will probably need to fetch
            # Or if this engine is in the array of important engines which we will always try to serve
            Redis::set($this->name, "running");
            $this->recheck = false;
        }
    }
-    private function updateStats($poptime){
+    private function updateStats($poptime)
-        if($this->connectionInfo !== NULL){
+    {
+        if ($this->connectionInfo !== null) {
            $connectionInfo = base64_encode(json_encode($this->connectionInfo));
            Redis::hset($this->name . ".stats", $this->pid, $connectionInfo . ";" . $poptime);
        }
    }
-    private function getFetchTime(){
+    private function getFetchTime()
+    {
        $vals = Redis::hgetall($this->name . ".stats");
-        if(sizeof($vals) === 0){
+        if (sizeof($vals) === 0) {
            return 0;
-        }else{
+        } else {
            $totalTime = 0;
            foreach ($vals as $pid => $value) {
                $time = floatval(json_decode(base64_decode(explode(";", $value)[0]), true)["total_time"]);
-                $time *= 1000;  // Transform from seconds to milliseconds
+                $time *= 1000; // Transform from seconds to milliseconds
                $totalTime += $time;
            }
            $totalTime /= sizeof($vals);
@@ -149,7 +155,8 @@ class Searcher implements ShouldQueue
        }
    }
-    private function retrieveUrl($url){
+    private function retrieveUrl($url)
+    {
        // Set this URL to the Curl handle
        curl_setopt($this->ch, CURLOPT_URL, $url);
        $result = curl_exec($this->ch);
@@ -157,36 +164,43 @@ class Searcher implements ShouldQueue
        return $result;
    }
-    private function storeResult($result, $poptime, $hashValue){
+    private function storeResult($result, $poptime, $hashValue)
+    {
        Redis::hset('search.' . $hashValue, $this->name, $result);
        // After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
        Redis::expire('search.' . $hashValue, 60);
        $this->lastTime = microtime(true);
    }
-    private function shutdown(){
+    private function shutdown()
+    {
        Redis::hdel($this->name . ".stats", $this->pid);
-        if(sizeof(Redis::hgetall($this->name . ".stats")) === 0){
+        if (sizeof(Redis::hgetall($this->name . ".stats")) === 0) {
            Redis::del($this->name);
        }
        // We should close our curl handle before we do so
        curl_close($this->ch);
    }
-    private function initCurlHandle(){
+    private function initCurlHandle()
+    {
        $ch = curl_init();
        curl_setopt_array($ch, array(
-                CURLOPT_RETURNTRANSFER => 1,
+            CURLOPT_RETURNTRANSFER => 1,
-                CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
+            CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
-                CURLOPT_FOLLOWLOCATION => TRUE,
+            CURLOPT_FOLLOWLOCATION => true,
-                CURLOPT_CONNECTTIMEOUT => 10,
+            CURLOPT_CONNECTTIMEOUT => 10,
-                CURLOPT_MAXCONNECTS => 500,
+            CURLOPT_MAXCONNECTS => 500,
-                CURLOPT_LOW_SPEED_LIMIT => 500,
+            CURLOPT_LOW_SPEED_LIMIT => 500,
-                CURLOPT_LOW_SPEED_TIME => 5,
+            CURLOPT_LOW_SPEED_TIME => 5,
-                CURLOPT_TIMEOUT => 10
+            CURLOPT_TIMEOUT => 10,
        ));
+        if ($this->user !== null && $this->password !== null) {
+            curl_setopt($ch, CURLOPT_USERPWD, $this->user . ":" . $this->password);
+        }
        return $ch;
    }
 }
--- a/app/MetaGer.php
+++ b/app/MetaGer.php
@@ -3,12 +3,12 @@ namespace App;
 use App;
 use Cache;
+use Carbon;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Redis;
 use Jenssegers\Agent\Agent;
 use LaravelLocalization;
 use Log;
-use Carbon;
 use Predis\Connection\ConnectionException;
 class MetaGer
@@ -34,6 +34,7 @@ class MetaGer
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
+    protected $availableFoki = [];
    protected $startCount = 0;
    protected $canCache = false;
    # Daten über die Abfrage$
@@ -258,6 +259,7 @@ class MetaGer
        # Human Verification
        $this->results = $this->humanVerification($this->results);
+        $this->ads = $this->humanVerification($this->ads);
        $counter = 0;
        $firstRank = 0;
@@ -296,10 +298,6 @@ class MetaGer
        }
-        if (LaravelLocalization::getCurrentLocale() === "en") {
-            $this->ads = [];
-        }
        if ($this->validated) {
            $this->ads = [];
            $this->maps = false;
@@ -460,8 +458,11 @@ class MetaGer
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
-                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(base64_encode($link))]);
+                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
+                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
+                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
                $result->link = $url;
+                $result->proxyLink = $proxyUrl;
            }
            return $results;
        }else{
@@ -521,6 +522,20 @@ class MetaGer
        $overtureEnabled = false;
        $sumaCount = 0;
+        /*
+        * Erstellt eine Liste mit Foki, die verfügbar sind
+        */
+        $this->availableFoki = [];
+        foreach ($sumas as $suma) {
+            $foki = explode(",", trim($suma["type"]));
+            foreach ($foki as $fokus) {
+                if (!empty($fokus)) {
+                    $this->availableFoki[$fokus] = "available";
+                }
+            }
+        }
        $isCustomSearch = $this->startsWith($this->fokus, 'focus_');
        # Im Falle einer Custom-Suche ohne mindestens einer selektierter Suchmaschine wird eine Web-Suche durchgeführt
@@ -642,7 +657,7 @@ class MetaGer
    public function sumaIsSelected($suma, $request, $custom)
    {
        if ($custom) {
-            if ($request->filled("engine_" . $suma["name"])) {
+            if ($request->filled("engine_" . strtolower($suma["name"]))) {
                return true;
            }
        } else {
@@ -930,7 +945,7 @@ class MetaGer
    public function parseFormData(Request $request)
    {
-        $this->request = $request;
        # Sichert, dass der request in UTF-8 formatiert ist
        if ($request->input('encoding', 'utf8') !== "utf8") {
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
@@ -946,7 +961,7 @@ class MetaGer
        $this->fokus = $request->input('focus', 'web');
        # Suma-File
        if (App::isLocale("en")) {
-            $this->sumaFile = config_path() . "/sumas.xml";
+            $this->sumaFile = config_path() . "/sumasEn.xml";
        } else {
            $this->sumaFile = config_path() . "/sumas.xml";
        }
@@ -955,7 +970,7 @@ class MetaGer
        }
        # Sucheingabe
        $this->eingabe = trim($request->input('eingabe', ''));
-        $this->q       = $this->eingabe;
+        $this->q = $this->eingabe;
        # IP
        $this->ip = $request->ip();
        # Unser erster Schritt wird sein, IP-Adresse und USER-Agent zu anonymisieren, damit
@@ -1047,6 +1062,8 @@ class MetaGer
        $this->verificationId = $request->input('verification_id', null);
        $this->verificationCount = intval($request->input('verification_count', '0'));
        $this->apiKey = $request->input('key', '');
+        // Remove Inputs that are not used
+        $this->request = $request->replace($request->except(['verification_id', 'uid', 'verification_count']));
        $this->validated = false;
        if (isset($this->password)) {
@@ -1083,19 +1100,37 @@ class MetaGer
    public function checkSpecialSearches(Request $request)
    {
+        $this->searchCheckPhrase();
        $this->searchCheckSitesearch($request);
        $this->searchCheckHostBlacklist($request);
        $this->searchCheckDomainBlacklist($request);
        $this->searchCheckUrlBlacklist();
-        $this->searchCheckPhrase();
        $this->searchCheckStopwords($request);
        $this->searchCheckNoSearch();
    }
+    private function searchCheckPhrase()
+    {
+        $p = "";
+        $tmp = $this->q;
+        // matches '[... ]"test satz"[ ...]'
+        while (preg_match("/(^|.*?\s)\"(\S+)\"(\s.*|$)/si", $tmp, $match)) {
+            $tmp = $match[1] . $match[3];
+            $this->phrases[] = $match[2];
+        }
+        foreach ($this->phrases as $phrase) {
+            $p .= "\"$phrase\", ";
+        }
+        $p = rtrim($p, ", ");
+        if (sizeof($this->phrases) > 0) {
+            $this->warnings[] = trans('metaGer.formdata.phrase', ['phrase' => $p]);
+        }
+    }
    private function searchCheckSitesearch($request)
    {
        // matches '[... ]site:test.de[ ...]'
-        while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
+        while (preg_match("/(^|.*?\s)site:(\S+)(\s.*|$)/si", $this->q, $match)) {
            $this->site = $match[2];
            $this->q = $match[1] . $match[3];
        }
@@ -1108,7 +1143,7 @@ class MetaGer
    private function searchCheckHostBlacklist($request)
    {
        // matches '[... ]-site:test.de[ ...]'
-        while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) {
+        while (preg_match("/(^|.*?\s)-site:([^\*\s]\S*)(\s.*|$)/si", $this->q, $match)) {
            $this->hostBlacklist[] = $match[2];
            $this->q = $match[1] . $match[3];
        }
@@ -1143,7 +1178,7 @@ class MetaGer
    private function searchCheckDomainBlacklist($request)
    {
        // matches '[... ]-site:*.test.de[ ...]'
-        while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
+        while (preg_match("/(^|.*?\s)-site:\*\.(\S+)(\s.*|$)/si", $this->q, $match)) {
            $this->domainBlacklist[] = $match[2];
            $this->q = $match[1] . $match[3];
        }
@@ -1177,7 +1212,7 @@ class MetaGer
    private function searchCheckUrlBlacklist()
    {
        // matches '[... ]-site:*.test.de[ ...]'
-        while (preg_match("/(^|.+\s)-url:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
+        while (preg_match("/(^|.*?\s)-url:(\S+)(\s.*|$)/si", $this->q, $match)) {
            $this->urlBlacklist[] = $match[2];
            $this->q = $match[1] . $match[3];
        }
@@ -1194,11 +1229,18 @@ class MetaGer
    private function searchCheckStopwords($request)
    {
+        $oldQ = $this->q;
        // matches '[... ]-test[ ...]'
-        while (preg_match("/(^|[^\s]+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
+        $words = preg_split("/\s+/si",$this->q);
-            $this->stopWords[] = $match[2];
+        $newQ = "";
-            $this->q = $match[1] . $match[3];
+        foreach($words as $word){
+            if(strpos($word, "-") === 0 && strlen($word) > 1){
+                $this->stopWords[] = substr($word, 1);
+            }else{
+                $newQ .= " " . $word;
+            }
        }
+        $this->q = trim($newQ);
        # Overwrite Setting if submitted via Parameter
        if ($request->has('stop')) {
            $this->stopWords = [];
@@ -1212,6 +1254,7 @@ class MetaGer
            } else {
                $this->stopWords[] = $stop;
            }
+            $this->q = $oldQ;
        }
        // print the stopwords as a user warning
        if (sizeof($this->stopWords) > 0) {
@@ -1224,24 +1267,6 @@ class MetaGer
        }
    }
-    private function searchCheckPhrase()
-    {
-        $p = "";
-        $tmp = $this->q;
-        // matches '[... ]"test satz"[ ...]'
-        while (preg_match("/(^|.+\s)\"(.+)\"(?:\s(.+)|($))/si", $tmp, $match)) {
-            $tmp             = $match[1] . $match[3];
-            $this->phrases[] = $match[2];
-        }
-        foreach ($this->phrases as $phrase) {
-            $p .= "\"$phrase\", ";
-        }
-        $p = rtrim($p, ", ");
-        if (sizeof($this->phrases) > 0) {
-            $this->warnings[] = trans('metaGer.formdata.phrase', ['phrase' => $p]);
-        }
-    }
    private function searchCheckNoSearch()
    {
        if ($this->q === "") {
@@ -1480,6 +1505,14 @@ class MetaGer
 # Einfache Getter
+    public function getVerificationId() {
+        return $this->verificationId;
+    }
+    public function getVerificationCount() {
+        return $this->verificationCount;
+    }
    public function getSite()
    {
        return $this->site;
@@ -1539,6 +1572,12 @@ class MetaGer
        return $this->lang;
    }
+    public function getAvailableFoki()
+    {
+        return $this->availableFoki;
+    }
    public function getSprueche()
    {
        return $this->sprueche;

--- a/app/Models/Result.php
+++ b/app/Models/Result.php
@@ -35,11 +35,12 @@ class Result
        $this->titel = strip_tags(trim($titel));
        $this->link = trim($link);
        $this->anzeigeLink = trim($anzeigeLink);
+        $this->anzeigeLink = preg_replace("/(http[s]{0,1}:\/\/){0,1}(www\.){0,1}/si", "", $this->anzeigeLink);
        $this->descr = strip_tags(trim($descr), '<p>');
        $this->descr = preg_replace("/\n+/si", " ", $this->descr);
        $this->longDescr = $this->descr;
-        if (strlen($this->descr) > 250) {
+        if (strlen($this->descr) > 150) {
-            $this->descr = wordwrap($this->descr, 250);
+            $this->descr = wordwrap($this->descr, 150);
            $this->descr = substr($this->descr, 0, strpos($this->descr, "\n"));
        }

--- a/app/Models/Searchengine.php
+++ b/app/Models/Searchengine.php
@@ -15,12 +15,12 @@ abstract class Searchengine
    public $ch; # Curl Handle zum erhalten der Ergebnisse
    public $getString = ""; # Der String für die Get-Anfrage
    public $engine; # Die ursprüngliche Engine XML
-    public $enabled  = true; # true, wenn die Suchmaschine nicht explizit disabled ist
+    public $enabled = true; # true, wenn die Suchmaschine nicht explizit disabled ist
-    public $results  = []; # Die geladenen Ergebnisse
+    public $results = []; # Die geladenen Ergebnisse
-    public $ads      = []; # Die geladenen Werbungen
+    public $ads = []; # Die geladenen Werbungen
    public $products = []; # Die geladenen Produkte
-    public $loaded   = false; # wahr, sobald die Ergebnisse geladen wurden
+    public $loaded = false; # wahr, sobald die Ergebnisse geladen wurden
-    public $cached   = false;
+    public $cached = false;
    public $ip; # Die IP aus der metager
    public $uses; # Die Anzahl der Nutzungen dieser Suchmaschine
@@ -31,10 +31,13 @@ abstract class Searchengine
    public $startTime; # Die Zeit der Erstellung dieser Suchmaschine
    public $hash; # Der Hash-Wert dieser Suchmaschine
+    private $user; # Username für HTTP-Auth (falls angegeben)
+    private $password; # Passwort für HTTP-Auth (falls angegeben)
    public $fp; # Wird für Artefakte benötigt
-    public $socketNumber    = null; # Wird für Artefakte benötigt
+    public $socketNumber = null; # Wird für Artefakte benötigt
-    public $counter         = 0; # Wird eventuell für Artefakte benötigt
+    public $counter = 0; # Wird eventuell für Artefakte benötigt
-    public $write_time      = 0; # Wird eventuell für Artefakte benötigt
+    public $write_time = 0; # Wird eventuell für Artefakte benötigt
    public $connection_time = 0; # Wird eventuell für Artefakte benötigt
    public function __construct(\SimpleXMLElement $engine, MetaGer $metager)
@@ -69,7 +72,7 @@ abstract class Searchengine
        }
        $this->useragent = $metager->getUserAgent();
-        $this->ip        = $metager->getIp();
+        $this->ip = $metager->getIp();
        $this->startTime = microtime();
        # Suchstring generieren
@@ -84,10 +87,10 @@ abstract class Searchengine
        } else {
            $q = $metager->getQ();
        }
-        $this->getString  = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
+        $this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
-        $this->hash       = md5($this->host . $this->getString . $this->port . $this->name);
+        $this->hash = md5($this->host . $this->getString . $this->port . $this->name);
        $this->resultHash = $metager->getHashCode();
-        $this->canCache   = $metager->canCache();
+        $this->canCache = $metager->canCache();
        if (!isset($this->additionalHeaders)) {$this->additionalHeaders = "";}
    }
@@ -114,13 +117,13 @@ abstract class Searchengine
            // With <ResultHash> being the Hash Value where the fetcher will store the result.
            // and <URL to fetch> being the full URL to the searchengine
            $url = "";
-            if($this->port === "443"){
+            if ($this->port === "443") {
                $url = "https://";
-            }else{
+            } else {
                $url = "http://";
            }
            $url .= $this->host;
-            if($this->port !== 80 && $this->port !== 443){
+            if ($this->port !== 80 && $this->port !== 443) {
                $url .= ":" . $this->port;
            }
            $url .= $this->getString;
@@ -132,27 +135,27 @@ abstract class Searchengine
            Redis::rpush($this->name . ".queue", $mission);
            /**
-            * We have Searcher processes running for MetaGer
+             * We have Searcher processes running for MetaGer
-            * Each Searcher is dedicated to one specific Searchengine and fetches it's results.
+             * Each Searcher is dedicated to one specific Searchengine and fetches it's results.
-            * We can have multiple Searchers for each engine, if needed.
+             * We can have multiple Searchers for each engine, if needed.
-            * At this point we need to decide, whether we need to start a new Searcher process or
+             * At this point we need to decide, whether we need to start a new Searcher process or
-            * if we have enough of them running.
+             * if we have enough of them running.
-            * The information for that is provided through the redis system. Each running searcher 
+             * The information for that is provided through the redis system. Each running searcher
-            * gives information how long it has waited to be given the last fetcher job.
+             * gives information how long it has waited to be given the last fetcher job.
-            * The longer this time value is, the less frequent the search engine is used and the less
+             * The longer this time value is, the less frequent the search engine is used and the less
-            * searcher of that type we need.
+             * searcher of that type we need.
-            * But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
+             * But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
-            **/
+             **/
            $needSearcher = false;
            $searcherData = Redis::hgetall($this->name . ".stats");
            // We now have an array of statistical data from the searchers
            // Each searcher has one entry in it.
-            // So if it's empty, then we have currently no searcher running and 
+            // So if it's empty, then we have currently no searcher running and
            // of course need to spawn a new one.
-            if(sizeof($searcherData) === 0){
+            if (sizeof($searcherData) === 0) {
                $needSearcher = true;
-            }else{
+            } else {
                // There we go:
                // There's at least one Fetcher running for this search engine.
                // Now we have to check if the current count is enough to fetch all the
@@ -160,18 +163,18 @@ abstract class Searchengine
                // Let's hardcode a minimum of 100ms between every search job.
                // First calculate the median of all Times
                $median = 0;
-                foreach($searcherData as $pid => $data){
+                foreach ($searcherData as $pid => $data) {
                    $data = explode(";", $data);
                    $median += floatval($data[1]);
                }
                $median /= sizeof($searcherData);
-                if($median < .1){
+                if ($median < .1) {
                    $needSearcher = true;
                }
            }
-            if($needSearcher && Redis::get($this->name) !== "locked"){
+            if ($needSearcher && Redis::get($this->name) !== "locked") {
                Redis::set($this->name, "locked");
-                $this->dispatch(new Searcher($this->name));
+                $this->dispatch(new Searcher($this->name, $this->user, $this->password));
            }
        }
    }
@@ -203,7 +206,8 @@ abstract class Searchengine
        $this->enabled = true;
    }
-    public function setResultHash($hash){
+    public function setResultHash($hash)
+    {
        $this->resultHash = $hash;
    }

--- a/app/Models/parserSkripte/Fastbot.php
+++ b/app/Models/parserSkripte/Fastbot.php
-<?php
-namespace app\Models\parserSkripte;
-use App\Models\Searchengine;
-class Fastbot extends Searchengine
-{
-    public $results = [];
-    public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
-    {
-        parent::__construct($engine, $metager);
-        if (strpos($this->urlEncode($metager->getEingabe()), "%") !== false) {
-            $this->enabled = false;
-            return null;
-        }
-    }
-    public function loadResults($result)
-    {
-        $result  = utf8_encode($result);
-        $counter = 0;
-        foreach (explode("\n", $result) as $line) {
-            $line = trim($line);
-            if (strlen($line) > 0) {
-                # Hier bekommen wir jedes einzelne Ergebnis
-                $result = explode("|:|", $line);
-                if(count($result) < 4) continue;
-                $link   = $result[1];
-                $link   = substr($link, strpos($link, "href=\"") + 6);
-                $link   = substr($link, 0, strpos($link, "\""));
-                $counter++;
-                $this->gefVon    = "<a href=\"" . $this->homepage . "\" target=\"_blank\" rel=\"noopener\">" . $this->displayName . " " . trans('results.redirect') . "</a>";
-                $this->results[] = new \App\Models\Result(
-                    $this->engine,
-                    trim(strip_tags($result[1])),
-                    $link,
-                    $result[3],
-                    $result[2],
-                    $this->displayName,$this->homepage,
-                    $counter
-                );
-            }
-        }
-    }
-}
--- a/app/Models/parserSkripte/Overture.php
+++ b/app/Models/parserSkripte/Overture.php
@@ -25,10 +25,10 @@ class Overture extends Searchengine
            $results = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
            foreach ($results as $result) {
-                $title       = $result["title"];
+                $title = $result["title"];
-                $link        = $result->{"ClickUrl"}->__toString();
+                $link = $result->{"ClickUrl"}->__toString();
                $anzeigeLink = $result["siteHost"];
-                $descr       = $result["description"];
+                $descr = $result["description"];
                $this->counter++;
                $this->results[] = new \App\Models\Result(
                    $this->engine,
@@ -36,7 +36,8 @@ class Overture extends Searchengine
                    $link,
                    $anzeigeLink,
                    $descr,
-                    $this->displayName,$this->homepage,
+                    $this->displayName,
+                    $this->homepage,
                    $this->counter
                );
            }
@@ -44,10 +45,10 @@ class Overture extends Searchengine
            # Nun noch die Werbeergebnisse:
            $ads = $content->xpath('//Results/ResultSet[@id="searchResults"]/Listing');
            foreach ($ads as $ad) {
-                $title       = $ad["title"];
+                $title = $ad["title"];
-                $link        = $ad->{"ClickUrl"}->__toString();
+                $link = $ad->{"ClickUrl"}->__toString();
                $anzeigeLink = $ad["siteHost"];
-                $descr       = $ad["description"];
+                $descr = $ad["description"];
                $this->counter++;
                $this->ads[] = new \App\Models\Result(
                    $this->engine,
@@ -55,7 +56,7 @@ class Overture extends Searchengine
                    $link,
                    $anzeigeLink,
                    $descr,
-                    $this->displayName,$this->homepage,
+                    $this->displayName, $this->homepage,
                    $this->counter
                );
            }
@@ -87,9 +88,9 @@ class Overture extends Searchengine
        // Yahoo liefert, wenn es keine weiteren Ergebnisse hat immer wieder die gleichen Ergebnisse
        // Wir müssen also überprüfen, ob wir am Ende der Ergebnisse sind
        $resultCount = $content->xpath('//Results/ResultSet[@id="inktomi"]/MetaData/TotalHits');
-        $results     = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
+        $results = $content->xpath('//Results/ResultSet[@id="inktomi"]/Listing');
        if (isset($resultCount[0]) && sizeof($results) > 0) {
-            $resultCount      = intval($resultCount[0]->__toString());
+            $resultCount = intval($resultCount[0]->__toString());
            $lastResultOnPage = intval($results[sizeof($results) - 1]["rank"]);
            if ($resultCount <= $lastResultOnPage) {
                return;
@@ -111,9 +112,9 @@ class Overture extends Searchengine
        }
        # Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
-        $next            = new Overture(simplexml_load_string($this->engine), $metager);
+        $next = new Overture(simplexml_load_string($this->engine), $metager);
        $next->getString = preg_replace("/&Keywords=.*?&/si", "&", $next->getString) . "&" . $nextArgs;
-        $next->hash      = md5($next->host . $next->getString . $next->port . $next->name);
+        $next->hash = md5($next->host . $next->getString . $next->port . $next->name);
-        $this->next      = $next;
+        $this->next = $next;
    }
 }
--- a/app/Models/parserSkripte/Scopia.php
+++ b/app/Models/parserSkripte/Scopia.php
+<?php
+namespace app\Models\parserSkripte;
+use App\Models\Searchengine;
+use Log;
+class Scopia extends Searchengine
+{
+    public $results = [];
+    public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
+    {
+        parent::__construct($engine, $metager);
+    }
+    public function loadResults($result)
+    {
+        $result = html_entity_decode($result);
+        $result = preg_replace("/<description>(.*?)<\/description>/si", "<description><![CDATA[ $1 ]]></description>", $result);
+        $result = preg_replace("/<title>(.*?)<\/title>/si", "<title><![CDATA[ $1 ]]></title>", $result);
+        $result = preg_replace("/<url>(.*?)<\/url>/si", "<url><![CDATA[ $1 ]]></url>", $result);
+        try {
+            $content = simplexml_load_string($result);
+            if (!$content) {
+                return;
+            }
+            $results = $content->xpath('//results/result');
+            foreach ($results as $result) {
+                $title = $result->title->__toString();
+                $link = $result->url->__toString();
+                $anzeigeLink = $link;
+                $descr = $result->description->__toString();
+                $this->counter++;
+                $this->results[] = new \App\Models\Result(
+                    $this->engine,
+                    $title,
+                    $link,
+                    $anzeigeLink,
+                    $descr,
+                    $this->displayName,
+                    $this->homepage,
+                    $this->counter
+                );
+            }
+        } catch (\Exception $e) {
+            Log::error("A problem occurred parsing results from $this->name:");
+            Log::error($e->getMessage());
+            return;
+        }
+    }
+    public function getNext(\App\MetaGer $metager, $result)
+    {
+        $result = html_entity_decode($result);
+        $result = str_replace("&", "&amp;", $result);
+        try {
+            $content = simplexml_load_string($result);
+        } catch (\Exception $e) {
+            Log::error("A problem occurred parsing results from $this->name:");
+            Log::error($e->getMessage());
+            return;
+        }
+        if (!$content) {
+            return;
+        }
+        $more = $content->xpath('//results/more')[0]->__toString() === "1" ? true : false;
+        if ($more) {
+            $results = $content->xpath('//results/result');
+            $number = $results[sizeof($results) - 1]->number->__toString();
+            # Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
+            $next = new Scopia(simplexml_load_string($this->engine), $metager);
+            $next->getString = preg_replace("/\\?s=.*?&/si", "?s=" . $number, $next->getString);
+            $next->hash = md5($next->host . $next->getString . $next->port . $next->name);
+            $this->next = $next;
+        }
+    }
+}
--- a/database/migrations/2018_04_26_114745_create_humanverification_table.php
+++ b/database/migrations/2018_04_26_114745_create_humanverification_table.php
 <?php
-use Illuminate\Support\Facades\Schema;
-use Illuminate\Database\Schema\Blueprint;
 use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
 class CreateHumanverificationTable extends Migration
 {
@@ -14,9 +14,13 @@ class CreateHumanverificationTable extends Migration
    public function up()
    {
        Schema::create('humanverification', function (Blueprint $table) {
-            $table->string('id')->unique();
+            $table->string('uid')->unique();
+            $table->string('id');
            $table->integer('unusedResultPages');
+            $table->boolean('whitelist');
+            $table->integer('whitelistCounter');
            $table->boolean('locked');
+            $table->string('lockedKey');
            $table->timestamp('updated_at');
        });
    }

--- a/database/migrations/2018_05_03_101301_usedurls.php
+++ b/database/migrations/2018_05_03_101301_usedurls.php
 <?php
-use Illuminate\Support\Facades\Schema;
-use Illuminate\Database\Schema\Blueprint;
 use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
 class Usedurls extends Migration
 {
@@ -14,11 +14,12 @@ class Usedurls extends Migration
    public function up()
    {
        Schema::create('usedurls', function (Blueprint $table) {
-            $table->increments('id')->unique();
+            $table->increments('number')->unique();
-            $table->string('user_id');
+            $table->string('uid');
-            $table->text('url');
+            $table->string('id');
+            $table->text('eingabe');
            $table->timestamp('created_at');
-            $table->foreign('user_id')->references('id')->on('humanverification')->onDelete('cascade');
+            $table->foreign('uid')->references('uid')->on('humanverification')->onDelete('cascade');
        });
    }

--- a/database/migrations/2018_08_30_102121_sponsoren_links.php
+++ b/database/migrations/2018_08_30_102121_sponsoren_links.php
+<?php
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+class SponsorenLinks extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('sponsorenlinks', function (Blueprint $table) {
+            $table->increments('id')->unique();
+            $table->string('linktext');
+            $table->string('link');
+            $table->string('langcode');
+            $table->timestamp('updated_at');
+        });
+    }
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::drop('sponsorenlinks');
+    }
+}
--- a/public/index.php
+++ b/public/index.php
@@ -13,6 +13,8 @@ if (isset($_SERVER["HTTP_FORWARDED"]) && isset($_SERVER["HTTP_X_FORWARDED_FOR"])
    unset($_SERVER["HTTP_FORWARDED"]);
 }
+$_SERVER["AGENT"] = $_SERVER["HTTP_USER_AGENT"];
 if (isset($_SERVER['HTTP_USER_AGENT'])) {
    $agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);

--- a/public/robots.txt
+++ b/public/robots.txt
@@ -2,7 +2,7 @@ User-agent: *
 Disallow: /klassik/meta/
 Disallow: /en/klassik/meta/
 Disallow: /meta/
-Disallow: /en/meta/
+Disallow: /*/meta/
 Disallow: /qt
 Disallow: /en/qt
 Disallow: /domains/

--- a/resources/assets/js/lib/imagesloaded.js
+++ b/resources/assets/js/lib/imagesloaded.js
-/*!
- * imagesLoaded PACKAGED v3.1.8
- * JavaScript is all like "You images are done yet or what?"
- * MIT License
- */
-(function(){function e(){}function t(e,t){for(var n=e.length;n--;)if(e[n].listener===t)return n;return-1}function n(e){return function(){return this[e].apply(this,arguments)}}var i=e.prototype,r=this,o=r.EventEmitter;i.getListeners=function(e){var t,n,i=this._getEvents();if("object"==typeof e){t={};for(n in i)i.hasOwnProperty(n)&&e.test(n)&&(t[n]=i[n])}else t=i[e]||(i[e]=[]);return t},i.flattenListeners=function(e){var t,n=[];for(t=0;e.length>t;t+=1)n.push(e[t].listener);return n},i.getListenersAsObject=function(e){var t,n=this.getListeners(e);return n instanceof Array&&(t={},t[e]=n),t||n},i.addListener=function(e,n){var i,r=this.getListenersAsObject(e),o="object"==typeof n;for(i in r)r.hasOwnProperty(i)&&-1===t(r[i],n)&&r[i].push(o?n:{listener:n,once:!1});return this},i.on=n("addListener"),i.addOnceListener=function(e,t){return this.addListener(e,{listener:t,once:!0})},i.once=n("addOnceListener"),i.defineEvent=function(e){return this.getListeners(e),this},i.defineEvents=function(e){for(var t=0;e.length>t;t+=1)this.defineEvent(e[t]);return this},i.removeListener=function(e,n){var i,r,o=this.getListenersAsObject(e);for(r in o)o.hasOwnProperty(r)&&(i=t(o[r],n),-1!==i&&o[r].splice(i,1));return this},i.off=n("removeListener"),i.addListeners=function(e,t){return this.manipulateListeners(!1,e,t)},i.removeListeners=function(e,t){return this.manipulateListeners(!0,e,t)},i.manipulateListeners=function(e,t,n){var i,r,o=e?this.removeListener:this.addListener,s=e?this.removeListeners:this.addListeners;if("object"!=typeof t||t instanceof RegExp)for(i=n.length;i--;)o.call(this,t,n[i]);else for(i in t)t.hasOwnProperty(i)&&(r=t[i])&&("function"==typeof r?o.call(this,i,r):s.call(this,i,r));return this},i.removeEvent=function(e){var t,n=typeof e,i=this._getEvents();if("string"===n)delete i[e];else if("object"===n)for(t in i)i.hasOwnProperty(t)&&e.test(t)&&delete i[t];else delete this._events;return this},i.removeAllListeners=n("removeEvent"),i.emitEvent=function(e,t){var n,i,r,o,s=this.getListenersAsObject(e);for(r in s)if(s.hasOwnProperty(r))for(i=s[r].length;i--;)n=s[r][i],n.once===!0&&this.removeListener(e,n.listener),o=n.listener.apply(this,t||[]),o===this._getOnceReturnValue()&&this.removeListener(e,n.listener);return this},i.trigger=n("emitEvent"),i.emit=function(e){var t=Array.prototype.slice.call(arguments,1);return this.emitEvent(e,t)},i.setOnceReturnValue=function(e){return this._onceReturnValue=e,this},i._getOnceReturnValue=function(){return this.hasOwnProperty("_onceReturnValue")?this._onceReturnValue:!0},i._getEvents=function(){return this._events||(this._events={})},e.noConflict=function(){return r.EventEmitter=o,e},"function"==typeof define&&define.amd?define("eventEmitter/EventEmitter",[],function(){return e}):"object"==typeof module&&module.exports?module.exports=e:this.EventEmitter=e}).call(this),function(e){function t(t){var n=e.event;return n.target=n.target||n.srcElement||t,n}var n=document.documentElement,i=function(){};n.addEventListener?i=function(e,t,n){e.addEventListener(t,n,!1)}:n.attachEvent&&(i=function(e,n,i){e[n+i]=i.handleEvent?function(){var n=t(e);i.handleEvent.call(i,n)}:function(){var n=t(e);i.call(e,n)},e.attachEvent("on"+n,e[n+i])});var r=function(){};n.removeEventListener?r=function(e,t,n){e.removeEventListener(t,n,!1)}:n.detachEvent&&(r=function(e,t,n){e.detachEvent("on"+t,e[t+n]);try{delete e[t+n]}catch(i){e[t+n]=void 0}});var o={bind:i,unbind:r};"function"==typeof define&&define.amd?define("eventie/eventie",o):e.eventie=o}(this),function(e,t){"function"==typeof define&&define.amd?define(["eventEmitter/EventEmitter","eventie/eventie"],function(n,i){return t(e,n,i)}):"object"==typeof exports?module.exports=t(e,require("wolfy87-eventemitter"),require("eventie")):e.imagesLoaded=t(e,e.EventEmitter,e.eventie)}(window,function(e,t,n){function i(e,t){for(var n in t)e[n]=t[n];return e}function r(e){return"[object Array]"===d.call(e)}function o(e){var t=[];if(r(e))t=e;else if("number"==typeof e.length)for(var n=0,i=e.length;i>n;n++)t.push(e[n]);else t.push(e);return t}function s(e,t,n){if(!(this instanceof s))return new s(e,t);"string"==typeof e&&(e=document.querySelectorAll(e)),this.elements=o(e),this.options=i({},this.options),"function"==typeof t?n=t:i(this.options,t),n&&this.on("always",n),this.getImages(),a&&(this.jqDeferred=new a.Deferred);var r=this;setTimeout(function(){r.check()})}function f(e){this.img=e}function c(e){this.src=e,v[e]=this}var a=e.jQuery,u=e.console,h=u!==void 0,d=Object.prototype.toString;s.prototype=new t,s.prototype.options={},s.prototype.getImages=function(){this.images=[];for(var e=0,t=this.elements.length;t>e;e++){var n=this.elements[e];"IMG"===n.nodeName&&this.addImage(n);var i=n.nodeType;if(i&&(1===i||9===i||11===i))for(var r=n.querySelectorAll("img"),o=0,s=r.length;s>o;o++){var f=r[o];this.addImage(f)}}},s.prototype.addImage=function(e){var t=new f(e);this.images.push(t)},s.prototype.check=function(){function e(e,r){return t.options.debug&&h&&u.log("confirm",e,r),t.progress(e),n++,n===i&&t.complete(),!0}var t=this,n=0,i=this.images.length;if(this.hasAnyBroken=!1,!i)return this.complete(),void 0;for(var r=0;i>r;r++){var o=this.images[r];o.on("confirm",e),o.check()}},s.prototype.progress=function(e){this.hasAnyBroken=this.hasAnyBroken||!e.isLoaded;var t=this;setTimeout(function(){t.emit("progress",t,e),t.jqDeferred&&t.jqDeferred.notify&&t.jqDeferred.notify(t,e)})},s.prototype.complete=function(){var e=this.hasAnyBroken?"fail":"done";this.isComplete=!0;var t=this;setTimeout(function(){if(t.emit(e,t),t.emit("always",t),t.jqDeferred){var n=t.hasAnyBroken?"reject":"resolve";t.jqDeferred[n](t)}})},a&&(a.fn.imagesLoaded=function(e,t){var n=new s(this,e,t);return n.jqDeferred.promise(a(this))}),f.prototype=new t,f.prototype.check=function(){var e=v[this.img.src]||new c(this.img.src);if(e.isConfirmed)return this.confirm(e.isLoaded,"cached was confirmed"),void 0;if(this.img.complete&&void 0!==this.img.naturalWidth)return this.confirm(0!==this.img.naturalWidth,"naturalWidth"),void 0;var t=this;e.on("confirm",function(e,n){return t.confirm(e.isLoaded,n),!0}),e.check()},f.prototype.confirm=function(e,t){this.isLoaded=e,this.emit("confirm",this,t)};var v={};return c.prototype=new t,c.prototype.check=function(){if(!this.isChecked){var e=new Image;n.bind(e,"load",this),n.bind(e,"error",this),e.src=this.src,this.isChecked=!0}},c.prototype.handleEvent=function(e){var t="on"+e.type;this[t]&&this[t](e)},c.prototype.onload=function(e){this.confirm(!0,"onload"),this.unbindProxyEvents(e)},c.prototype.onerror=function(e){this.confirm(!1,"onerror"),this.unbindProxyEvents(e)},c.prototype.confirm=function(e,t){this.isConfirmed=!0,this.isLoaded=e,this.emit("confirm",this,t)},c.prototype.unbindProxyEvents=function(e){n.unbind(e.target,"load",this),n.unbind(e.target,"error",this)},s});
\ No newline at end of file
--- a/resources/assets/js/lib/openpgp.min.js
+++ b/resources/assets/js/lib/openpgp.min.js
--- a/resources/assets/js/searchbar.js
+++ b/resources/assets/js/searchbar.js
-$(function () {
+$(function() {
  loadLocalStorage();
  setActionListenersSearchbar();
  updateLangLabelCode();
 });
 function setActionListenersSearchbar() {
-  $('#input-key').change(saveKey);
+  $("#input-key").change(saveKey);
-  $('#input-lang').change(saveLang);
+  $("#input-lang").change(saveLang);
 }
 function saveKey() {
-  var key = $('#input-key').val();
+  var key = $("#input-key").val();
-  localStorage.setItem('key', key);
+  localStorage.setItem("key", key);
 }
 function loadKey() {
-  var key = localStorage.getItem('key');
+  var key = localStorage.getItem("key");
  if (key != null) {
-    $('#input-key').val(key);
+    $("#input-key").val(key);
  }
 }
 function saveLang() {
-  var lang = $('#input-lang').val();
+  var lang = $("#input-lang").val();
-  localStorage.setItem('lang', lang);
+  if (lang != "all") localStorage.setItem("lang", lang);
+  else localStorage.removeItem("lang");
  updateLangLabelCode(lang);
 }
 function loadLang() {
-  var lang = localStorage.getItem('lang');
+  var lang = localStorage.getItem("lang");
  if (lang != null) {
-    $('#input-lang').val(lang);
+    $("#input-lang").val(lang);
  }
 }
@@ -46,12 +47,12 @@ function loadLocalStorage() {
 }
 function setSettings() {
-  var acceptedParams = ['autocomplete', 'key', 'lang', 'newtab', 'sprueche'];
+  var acceptedParams = ["autocomplete", "key", "lang", "newtab", "sprueche"];
  for (var key in localStorage) {
    var value = localStorage.getItem(key);
    var accepted = false;
    for (var i in acceptedParams) {
-      if (key === 'param_' + acceptedParams[i]) {
+      if (key === "param_" + acceptedParams[i]) {
        accepted = true;
      }
    }
@@ -61,20 +62,26 @@ function setSettings() {
      var existing = $('.search-hidden input[name="' + key + '"]');
      if (existing.length === 0) {
        // if none exist, create a new one
-        $('.search-hidden').append('<input type="hidden" name="' + key + '" value="' + value + '">');
+        $(".search-hidden").append(
+          '<input type="hidden" name="' + key + '" value="' + value + '">'
+        );
      }
    }
  }
  // Change the request method to the given parameter
-  var requestMethod = localStorage.getItem('request');
+  var requestMethod = localStorage.getItem("request");
-  if (requestMethod !== null && (requestMethod === 'GET' || requestMethod === 'POST')) {
+  if (
-    $('#searchForm').attr('method', requestMethod);
+    requestMethod !== null &&
+    (requestMethod === "GET" || requestMethod === "POST")
+  ) {
+    $("#searchForm").attr("method", requestMethod);
  }
 }
 function updateLangLabelCode(langcode = null) {
  if (!langcode) {
-    var langcode = localStorage.getItem('lang');
+    var langcode = localStorage.getItem("lang");
  }
-  $('#lang-label-code').html(langcode);
+  if (langcode == "all") langcode = "";
-}
+  $("#lang-label-code").html(langcode);
\ No newline at end of file
+}
No results found