...
 
Commits (139)
......@@ -16,3 +16,5 @@ langfiles.zip
**/*.map
/.buildpath
/.project
config/sumas.json
config/sumasEn.json
update(144.76.113.134):
update(MetaGer Preredesign):
tags:
- 144.76.113.134
only:
- master@open-source/MetaGer
- Preredesign@open-source/MetaGer
before_script:
# Abhängigkeiten überprüfen
- which composer
......@@ -13,11 +13,11 @@ update(144.76.113.134):
# Pfad zum neu geklonten Repo
- path=`pwd`
- cd ~/
- if [ -d MetaGer_neu ]; then rm -rf MetaGer_neu;fi
- git clone "$path" MetaGer_neu
- cd MetaGer_neu
- if [ -d .MetaGer_alt ]; then rm -rf .MetaGer_alt;fi
- git clone "$path" .MetaGer_alt
- cd .MetaGer_alt
- composer install
- scp -P 63824 metager@metager3.de:~/.env .
- scp -P 63824 metager@metager3.de:~/.env.preredesign .env
- scp -P 63824 metager@metager3.de:~/sumas.xml config/
- scp -P 63824 metager@metager3.de:~/sumasEn.xml config/
- scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
......@@ -31,89 +31,9 @@ update(144.76.113.134):
- npm install
- php artisan migrate --force
- npm run prod
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- if [ -f ~/MetaGer_alt/artisan ]; then php ~/MetaGer_alt/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- while [ -d ~/MetaGer_alt ]; do rm -rf ~/MetaGer_alt;done
- mv .MetaGer_alt MetaGer_alt
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
update(metager2):
tags:
- metager2
only:
- master@open-source/MetaGer
before_script:
# Abhängigkeiten überprüfen
- which composer
- which git
- which php
- which sqlite3
script:
# Pfad zum neu geklonten Repo
- path=`pwd`
- cd ~/
- if [ -d MetaGer_neu ]; then rm -rf MetaGer_neu;fi
- git clone "$path" MetaGer_neu
- cd MetaGer_neu
- composer install
- scp -P 63824 metager@metager3.de:~/.env .
- scp -P 63824 metager@metager3.de:~/sumas.xml config/
- scp -P 63824 metager@metager3.de:~/sumasEn.xml config/
- scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
- php artisan migrate --force
- npm run prod
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
update(metager3.de):
tags:
- metager3
only:
- development@open-source/MetaGer
before_script:
# Abhängigkeiten überprüfen
- which composer
- which git
- which php
- which sqlite3
script:
# Pfad zum neu geklonten Repo
- path=`pwd`
- cd ~/
- if [ -d MetaGer_neu ]; then rm -rf MetaGer_neu;fi
- git clone "$path" MetaGer_neu
- cd MetaGer_neu
- composer install
- scp -P 63824 metager@metager3.de:~/.env .
- scp -P 63824 metager@metager3.de:~/sumas.xml config/
- scp -P 63824 metager@metager3.de:~/sumasEn.xml config/
- scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
- php artisan migrate --force
- npm run prod
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
\ No newline at end of file
- php ~/MetaGer_alt/artisan up
\ No newline at end of file
......@@ -2,6 +2,7 @@
namespace App\Console;
use DB;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
......@@ -25,6 +26,12 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule)
{
$schedule->command('log:rotate')->everyTenMinutes();
// Delete all of the old humanverification entries
$schedule->call(function() {
DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 72 hour) AND whitelist = 0 ORDER BY updated_at DESC');
DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 2 week) AND whitelist = 1 ORDER BY updated_at DESC');
})->everyThirtyMinutes();
}
/**
......
......@@ -145,18 +145,6 @@ class AdminInterface extends Controller
}
}
public function check()
{
$q = "";
$redis = Redis::connection('redisLogs');
if ($redis) {
$q = $redis->lrange("logs.search", -1, -1)[0];
$q = substr($q, strpos($q, "search=") + 7);
}
return view('admin.check')
->with('title', 'Wer sucht was? - MetaGer')
->with('q', $q);
}
public function engines()
{
......
<?php
namespace App\Http\Controllers;
use Captcha;
use Carbon;
use DB;
use Illuminate\Hashing\BcryptHasher as Hasher;
use Illuminate\Http\Request;
use Input;
class HumanVerification extends Controller
{
public static function captcha(Request $request, Hasher $hasher, $id, $url = null)
{
if ($url != null) {
$url = base64_decode(str_replace("<<SLASH>>", "/", $url));
} else {
$url = $request->input('url');
}
if ($request->getMethod() == 'POST') {
$user = DB::table('humanverification')->where('uid', $id)->first();
$lockedKey = $user->lockedKey;
$key = $request->input('captcha');
$key = strtolower($key);
if (!$hasher->check($key, $lockedKey)) {
$captcha = Captcha::create("default", true);
DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
->with('id', $id)
->with('url', $url)
->with('image', $captcha["img"])
->with('errorMessage', 'Fehler: Falsches Captcha eingegeben!');
} else {
# If we can unlock the Account of this user we will redirect him to the result page
if ($user !== null && $user->locked === 1) {
# The Captcha was correct. We can remove the key from the user
DB::table('humanverification')->where('uid', $id)->update(['locked' => false, 'lockedKey' => "", 'whitelist' => 1]);
return redirect($url);
} else {
return redirect('/');
}
}
}
$captcha = Captcha::create("default", true);
DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
->with('id', $id)
->with('url', $url)
->with('image', $captcha["img"]);
}
public static function remove(Request $request)
{
if (!$request->has('mm')) {
abort(404, "Keine Katze gefunden.");
}
if (HumanVerification::checkId($request, $request->input('mm'))) {
HumanVerification::removeUser($request, $request->input('mm'));
}
return response(hex2bin('89504e470d0a1a0a0000000d494844520000000100000001010300000025db56ca00000003504c5445000000a77a3dda0000000174524e530040e6d8660000000a4944415408d76360000000020001e221bc330000000049454e44ae426082'), 200)
->header('Content-Type', 'image/png');
}
public static function removeGet(Request $request, $mm, $password, $url)
{
$url = base64_decode(str_replace("<<SLASH>>", "/", $url));
# If the user is correct and the password is we will delete any entry in the database
$requiredPass = md5($mm . Carbon::NOW()->day . $url . env("PROXY_PASSWORD"));
if (HumanVerification::checkId($request, $mm) && $requiredPass === $password) {
HumanVerification::removeUser($request, $mm);
}
return redirect($url);
}
private static function removeUser($request, $uid)
{
$id = hash("sha512", $request->ip());
$sum = DB::table('humanverification')->where('id', $id)->where('whitelist', false)->sum('unusedResultPages');
$user = DB::table('humanverification')->where('uid', $uid)->first();
if ($user === null) {
return;
}
# Check if we have to whitelist the user or if we can simply delete the data
if ($user->unusedResultPages < $sum && $user->whitelist === 0) {
# Whitelist
DB::table('humanverification')->where('uid', $uid)->update(['whitelist' => true, 'whitelistCounter' => 0]);
$user->whitelist = 1;
$user->whitelistCounter = 0;
}
if ($user->whitelist === 1) {
DB::table('humanverification')->where('uid', $uid)->update(['unusedResultPages' => 0]);
} else {
DB::table('humanverification')->where('uid', $uid)->where('updated_at', '<', Carbon::NOW()->subSeconds(2))->delete();
}
}
private static function checkId($request, $id)
{
if (hash("sha512", $request->ip() . $_SERVER["AGENT"]) === $id) {
return true;
} else {
return false;
}
}
}
<?php
namespace App\Http\Controllers;
use File;
use Illuminate\Http\Request;
use PiwikTracker;
use Response;
class ImageController extends Controller
{
public function generateImage(Request $request)
{
/*
#Piwik Code
PiwikTracker::$URL = 'http://piwik.metager3.de';
$piwikTracker = new PiwikTracker($idSite = 1);
// Cookies ausschalten
$piwikTracker->disableCookieSupport();
$piwikTracker->deleteCookies();
$site = $request->input('site', '/');
// Sendet Tracker request per http
$piwikTracker->doTrackPageView($site);
*/
$path = public_path() . '/img/1px.png';
$fileType = File::type($path);
$response = Response::make(File::get($path), 200);
$response->header('Content-Type', $fileType);
return $response;
}
}
This diff is collapsed.
......@@ -3,10 +3,12 @@
namespace App\Http\Controllers;
use App;
use DB;
use Log;
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
use Response;
use LaravelLocalization;
use Response;
class StartpageController extends Controller
{
......@@ -20,7 +22,7 @@ class StartpageController extends Controller
public function loadStartPage(Request $request)
{
$focusPages = [];
$theme = "default";
$theme = "default";
foreach ($request->all() as $key => $value) {
if ($value === 'on' && $key != 'param_sprueche' && $key != 'param_newtab' && $key !== 'param_maps' && $key !== 'param_autocomplete' && $key !== 'param_lang') {
......@@ -32,10 +34,25 @@ class StartpageController extends Controller
}
$lang = LaravelLocalization::getCurrentLocale();
if ($lang === 'de') {
if ($lang === 'de' || $lang === "en") {
$lang = 'all';
}
$foki = $this->loadFoki();
$availableFoki = [];
// Find out which Foki are available
foreach ($foki as $fokus => $sumas) {
$availableFoki[$fokus] = "available";
}
$sponsors = [];
try{
$sponsors = DB::table('sponsorenlinks')->where('langcode', 'de')->orderByRaw('LENGTH(linktext)', 'ASC')->get();
}catch(\Illuminate\Database\QueryException $e){
Log::error($e);
}
return view('index')
->with('title', trans('titles.index'))
->with('homeIcon')
......@@ -44,7 +61,9 @@ class StartpageController extends Controller
->with('navbarFocus', 'suche')
->with('theme', $theme)
->with('autocomplete', $request->input('param_autocomplete', 'on'))
->with('foki', $this->loadFoki())
->with('foki', $foki)
->with('sponsors', $sponsors)
->with('availableFoki', $availableFoki)
->with('focus', $request->input('focus', 'web'))
->with('lang', $request->input('param_lang', $lang))
->with('resultCount', $request->input('param_resultCount', '20'))
......@@ -78,9 +97,9 @@ class StartpageController extends Controller
public function loadPlugin($params, $locale = "de")
{
$params = unserialize(base64_decode($params));
$params = unserialize(base64_decode($params));
$requests = $params;
$params = [];
$params = [];
foreach ($requests as $key => $value) {
if (strpos($key, "param_") === 0) {
$key = substr($key, strpos($key, "param_") + 6);
......@@ -137,12 +156,12 @@ class StartpageController extends Controller
public function berlin(Request $request)
{
$link = "";
$link = "";
$password = "";
if ($request->has('eingabe')) {
$password = getenv('berlin');
$password = md5($request->input('eingabe') . " -host:userpage.fu-berlin.de" . $password);
$link = "/meta/meta.ger3?eingabe=" . $request->input('eingabe') . " -host:userpage.fu-berlin.de&focus=web&password=" . $password . "&encoding=utf8&lang=all&site=fu-berlin.de&quicktips=off&out=results-with-style";
$link = "/meta/meta.ger3?eingabe=" . $request->input('eingabe') . " -host:userpage.fu-berlin.de&focus=web&password=" . $password . "&encoding=utf8&lang=all&site=fu-berlin.de&quicktips=off&out=results-with-style";
}
return view('berlin')
->with('title', 'Testseite für die FU-Berlin')
......@@ -154,12 +173,12 @@ class StartpageController extends Controller
{
$sumaFile = "";
if (App::isLocale('en')) {
$sumaFile = config_path() . "/sumas.xml";
$sumaFile = config_path() . "/sumasEn.xml";
} else {
$sumaFile = config_path() . "/sumas.xml";
}
$xml = simplexml_load_file($sumaFile);
$xml = simplexml_load_file($sumaFile);
$sumas = $xml->xpath("suma");
$foki = [];
......@@ -168,13 +187,13 @@ class StartpageController extends Controller
if (isset($suma['type'])) {
$f = explode(",", $suma['type']->__toString());
foreach ($f as $tmp) {
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$foki[$tmp][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
}
} else {
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$displayName = $suma['displayName']->__toString();
$url = isset($suma['homepage']) ? $suma['homepage']->__toString() : "https://metager.de";
$foki["andere"][$suma['name']->__toString()] = ['displayName' => $displayName, 'url' => $url];
}
}
......
......@@ -24,15 +24,13 @@ class Kernel extends HttpKernel
* @var array
*/
protected $middlewareGroups = [
'session' => [
\Illuminate\Routing\Middleware\SubstituteBindings::class,
],
'web' => [
\App\Http\Middleware\EncryptCookies::class,
\Illuminate\Cookie\Middleware\AddQueuedCookiesToResponse::class,
\Illuminate\Session\Middleware\StartSession::class,
\Illuminate\View\Middleware\ShareErrorsFromSession::class,
#\App\Http\Middleware\VerifyCsrfToken::class,
\Illuminate\Routing\Middleware\SubstituteBindings::class,
],
'api' => [
'throttle:60,1',
'bindings',
......@@ -54,5 +52,6 @@ class Kernel extends HttpKernel
'guest' => \App\Http\Middleware\RedirectIfAuthenticated::class,
'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
'referer.check' => \App\Http\Middleware\RefererCheck::class,
'humanverification' => \App\Http\Middleware\HumanVerification::class,
];
}
<?php
namespace App\Http\Middleware;
use Captcha;
use Carbon;
use Closure;
use DB;
use Illuminate\Http\Response;
use URL;
class HumanVerification
{
/**
* Handle an incoming request.
*
* @param \Illuminate\Http\Request $request
* @param \Closure $next
* @return mixed
*/
public function handle($request, Closure $next)
{
// The specific user
$user = null;
$newUser = true;
$update = true;
try {
$id = hash("sha512", $request->ip());
$uid = hash("sha512", $request->ip() . $_SERVER["AGENT"]);
unset($_SERVER["AGENT"]);
/**
* If the user sends a Password or a key
* We will not verificate the user.
* If someone that uses a bot finds this out we
* might have to change it at some point.
*/
if ($request->has('password') || $request->has('key') || $request->has('appversion') || !env('BOT_PROTECTION', false)) {
$update = false;
return $next($request);
}
$users = DB::select('select * from humanverification where id = ?', [$id]);
# Lock out everyone in a Bot network
# Find out how many requests this IP has made
$sum = 0;
foreach ($users as $userTmp) {
if ($uid == $userTmp->uid) {
$user = ['uid' => $userTmp->uid,
'id' => $userTmp->id,
'unusedResultPages' => intval($userTmp->unusedResultPages),
'whitelist' => filter_var($userTmp->whitelist, FILTER_VALIDATE_BOOLEAN),
'whitelistCounter' => $userTmp->whitelistCounter,
'locked' => filter_var($userTmp->locked, FILTER_VALIDATE_BOOLEAN),
"lockedKey" => $userTmp->lockedKey,
'updated_at' => Carbon::now(),
];
$newUser = false;
}
if ($userTmp->whitelist === 0) {
$sum += $userTmp->unusedResultPages;
}
}
# If this user doesn't have an entry we will create one
if ($user === null) {
$user =
[
'uid' => $uid,
'id' => $id,
'unusedResultPages' => 0,
'whitelist' => false,
'whitelistCounter' => 0,
'locked' => false,
"lockedKey" => "",
'updated_at' => Carbon::now(),
];
}
# A lot of automated requests are from websites that redirect users to our result page.
# We will detect those requests and put a captcha
$referer = URL::previous();
# Just the URL-Parameter
$refererLock = false;
if (stripos($referer, "?") !== false) {
$referer = substr($referer, stripos($referer, "?") + 1);
$referer = urldecode($referer);
if (preg_match("/http[s]{0,1}:\/\/metager\.de\/meta\/meta.ger3\?.*?eingabe=([\w\d]+\.){1,2}[\w\d]+/si", $referer) === 1) {
$refererLock = true;
}
}
// Defines if this is the only user using that IP Adress
$alone = true;
foreach ($users as $userTmp) {
if ($userTmp->uid != $uid && !$userTmp->whitelist) {
$alone = false;
}
}
if ((!$alone && $sum >= 50 && !$user["whitelist"]) || $refererLock) {
$user["locked"] = true;
}
# If the user is locked we will force a Captcha validation
if ($user["locked"]) {
$captcha = Captcha::create("default", true);
$user["lockedKey"] = $captcha["key"];
return
new Response(
view('humanverification.captcha')
->with('title', "Bestätigung erforderlich")
->with('id', $uid)
->with('url', url()->full())
->with('image', $captcha["img"])
);
}
$user["unusedResultPages"]++;
if ($alone || $user["whitelist"]) {
# This IP doesn't need verification yet
# The user currently isn't locked
# We have different security gates:
# 50 and then every 25 => Captcha validated Result Pages
# If the user shows activity on our result page the counter will be deleted
if ($user["unusedResultPages"] === 50 || ($user["unusedResultPages"] > 50 && $user["unusedResultPages"] % 25 === 0)) {
$user["locked"] = true;
}
}
} catch (\Illuminate\Database\QueryException $e) {
// Failure in contacting metager3.de
} finally {
if ($update) {
// Update the user in the database
if ($newUser) {
DB::table('humanverification')->insert(
[
'uid' => $user["uid"],
'id' => $user["id"],
'unusedResultPages' => $user['unusedResultPages'],
'whitelist' => $user["whitelist"],
'whitelistCounter' => $user["whitelistCounter"],
'locked' => $user["locked"],
"lockedKey" => $user["lockedKey"],
'updated_at' => $user["updated_at"],
]
);
} else {
DB::table('humanverification')->where('uid', $uid)->update(
[
'uid' => $user["uid"],
'id' => $user["id"],
'unusedResultPages' => $user['unusedResultPages'],
'whitelist' => $user["whitelist"],
'whitelistCounter' => $user["whitelistCounter"],
'locked' => $user["locked"],
"lockedKey" => $user["lockedKey"],
'updated_at' => $user["updated_at"],
]
);
}
}
}
$request->request->add(['verification_id' => $user["uid"], 'verification_count' => $user["unusedResultPages"]]);
return $next($request);
}
}
......@@ -14,7 +14,7 @@ class Searcher implements ShouldQueue
{
use InteractsWithQueue, Queueable, SerializesModels;
protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo;
protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo, $user, $password;
# Each Searcher will shutdown after a specified time(s) or number of requests
protected $MAX_REQUESTS = 100;
# This value should always be below the retry_after value in config/queue.php
......@@ -34,12 +34,14 @@ class Searcher implements ShouldQueue
* keep-alive requests.
* @return void
*/
public function __construct($name)
public function __construct($name, $user = null, $password = null)
{
$this->name = $name;
$this->pid = getmypid();
$this->recheck = false;
$this->startTime = microtime(true);
$this->user = $user;
$this->password = $password;
// Submit this worker to the Redis System
Redis::expire($this->name, 5);
}
......@@ -53,57 +55,61 @@ class Searcher implements ShouldQueue
{
// This Searches is freshly called so we need to initialize the curl handle $ch
$this->ch = $this->initCurlHandle();
$this->counter = 0; // Counts the number of answered jobs
$time = microtime(true);
while(true){
// Update the expire
Redis::expire($this->name, 5);
Redis::expire($this->name . ".stats", 5);
// One Searcher can handle a ton of requests to the same server
// Each search to the server of this Searcher will be submitted to a queue
// stored in redis which has the same name as this searchengine appended by a ".queue"
// We will perform a blocking pop on this queue so the queue can remain empty for a while
// without killing this searcher directly.
$mission = Redis::blpop($this->name . ".queue", 4);
$this->counter++;
$this->updateStats(microtime(true) - $time);
$this->switchToRunning();
// The mission can be empty when blpop hit the timeout
if(!empty($mission)){
$mission = $mission[1];
$poptime = microtime(true) - $time;
// The mission is a String which can be divided to retrieve two informations:
// 1. The Hash Value where the result should be stored
// 2. The Url to Retrieve
// These two informations are divided by a ";" in the mission string
$mission = explode(";", $mission);
$hashValue = $mission[0];
$url = base64_decode($mission[1]);
$timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected");
$result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue);
// Reset the time of the last Job so we can calculate
// the time we have spend waiting for a new job
// We submit that calculation to the Redis systemin the method
$time = microtime(true);
try{
$this->counter = 0; // Counts the number of answered jobs
$time = microtime(true);
while(true){
// Update the expire
Redis::expire($this->name, 5);
Redis::expire($this->name . ".stats", 5);
// One Searcher can handle a ton of requests to the same server
// Each search to the server of this Searcher will be submitted to a queue
// stored in redis which has the same name as this searchengine appended by a ".queue"
// We will perform a blocking pop on this queue so the queue can remain empty for a while
// without killing this searcher directly.
$mission = Redis::blpop($this->name . ".queue", 4);
$this->counter++;
$this->updateStats(microtime(true) - $time);
$this->switchToRunning();
// The mission can be empty when blpop hit the timeout
if(!empty($mission)){
$mission = $mission[1];
$poptime = microtime(true) - $time;
// The mission is a String which can be divided to retrieve two informations:
// 1. The Hash Value where the result should be stored
// 2. The Url to Retrieve
// These two informations are divided by a ";" in the mission string
$mission = explode(";", $mission);
$hashValue = $mission[0];
$url = base64_decode($mission[1]);
$timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected");
$result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue);
// Reset the time of the last Job so we can calculate
// the time we have spend waiting for a new job
// We submit that calculation to the Redis systemin the method
$time = microtime(true);
}
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if(getenv("QUEUE_DRIVER") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true)-$this->startTime) > $this->MAX_TIME){
break;
}
}
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if(getenv("QUEUE_DRIVER") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true)-$this->startTime) > $this->MAX_TIME){
break;
}
}finally{
// When we reach this point, time has come for this Searcher to retire
$this->shutdown();
}
// When we reach this point, time has come for this Searcher to retire
$this->shutdown();
}
private function switchToRunning(){
......@@ -186,6 +192,11 @@ class Searcher implements ShouldQueue
CURLOPT_TIMEOUT => 10
));
if($this->user !== null && $this->password !== null){
curl_setopt($ch, CURLOPT_USERPWD, $this->user . ":" . $this->password);
}
return $ch;
}
}
......@@ -15,11 +15,12 @@ class Spende extends Mailable
*
* @return void
*/
public function __construct($from, $message)
public function __construct($from, $message, $name)
{
$this->subject = "MetaGer - Spende";
$this->reply = $from;
$this->reply = $from;
$this->message = $message;
$this->name = $name;
}
/**
......@@ -29,7 +30,7 @@ class Spende extends Mailable
*/
public function build()
{
return $this->from($this->reply)
return $this->from($this->reply, $this->name)
->subject($this->subject)
->text('kontakt.mail')
->with('messageText', $this->message);
......
This diff is collapsed.
......@@ -57,9 +57,9 @@ class Result
}
$this->valid = true;
$this->host = @parse_url($link, PHP_URL_HOST);
$this->strippedHost = $this->getStrippedHost($this->anzeigeLink);
$this->strippedDomain = $this->getStrippedDomain($this->strippedHost);
$this->strippedLink = $this->getStrippedLink($this->anzeigeLink);
$this->strippedHost = $this->getStrippedHost($this->link);
$this->strippedDomain = $this->getStrippedDomain($this->link);
$this->strippedLink = $this->getStrippedLink($this->link);
$this->rank = 0;
$this->partnershop = isset($additionalInformation["partnershop"]) ? $additionalInformation["partnershop"] : false;
$this->image = isset($additionalInformation["image"]) ? $additionalInformation["image"] : "";
......@@ -269,7 +269,8 @@ class Result
strpos($this->strippedHost, "twitter.com") === false &&
strpos($this->strippedHost, "www.ladenpreis.net") === false &&
strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
strpos($this->strippedHost, "www.onenewspage.com") === false) {
strpos($this->strippedHost, "www.onenewspage.com") === false &&
strpos($this->gefVon, "Shopzilla") === false) {
$count = $metager->getHostCount($this->strippedHost);
if ($count >= 3) {
return false;
......@@ -300,13 +301,14 @@ class Result
/* Entfernt "http://", "www" und Parameter von einem Link
* Dieser wird dabei in die Form:
* "http://www.foo.bar.de/test?ja=1" -> "foo.bar.de/test"
* "http://www.foo.bar.de/test?ja=1" -> "foo.bar.de/test?ja=1"
* gebracht.
*/
public function getStrippedLink($link)
{
$match = $this->getUrlElements($link);
return $match['host'] . $match['path'];
$link = preg_replace("/^http[s]{0,1}:\/\//", "", $link);
$link = preg_replace("/^www\./", "", $link);
return $link;
}
/* Liest aus einem Link die Domain.
......@@ -369,22 +371,31 @@ return "https://proxy.suma-ev.de/mger/nph-proxy.cgi/en/w0/" . $tmp;
*/
public function getUrlElements($url)
{
if (!preg_match("/(?:((?:http)|(?:https))(?::\/\/))?(?:([a-z0-9.\-_~]+):([a-z0-9.\-_~]+)@)?(?:(www)(?:\.))?((?:(?:[a-z0-9.\-_~]+\.)+)?([a-z0-9.\-_~]+\.[a-z0-9.\-_~]+))(?:(?::)(\d+))?((?:(?:\/[a-z0-9.\-_~]+)+)(?:\.[a-z0-9.\-_~]+)?)?(\?[a-z0-9.\-_~]+=[a-z0-9.\-_~]+(?:&[a-z0-9.\-_~]+=[a-z0-9.\-_~]+)*)?(?:(?:#)([a-z0-9.\-_~]+))?/i", $url, $match)) {
return;
$parts = parse_url($url);
$re = [];
$re["schema"] = empty($parts["scheme"]) ? "" : $parts["scheme"];
$re["username"] = empty($parts["user"]) ? "" : $parts["user"];
$re["password"] = empty($parts["pass"]) ? "" : $parts["pass"];
$re["web"] = "";
$re["host"] = $parts["host"];
if (stripos($re["host"], "www.") === 0) {
$re["web"] = "www.";
$re["host"] = substr($re["host"], strpos($re["host"], ".") + 1);
}
$hostParts = explode(".", $re["host"]);
if (sizeof($hostParts) >= 3) {
$re["domain"] = $hostParts[sizeof($hostParts) - 2] . "." . $hostParts[sizeof($hostParts) - 1];
} else {
$re = [];
if (isset($match[1])) {$re['schema'] = $match[1];} else { $re['schema'] = "";};
if (isset($match[2])) {$re['username'] = $match[2];} else { $re['username'] = "";};
if (isset($match[3])) {$re['password'] = $match[3];} else { $re['password'] = "";};
if (isset($match[4])) {$re['web'] = $match[4];} else { $re['web'] = "";};
if (isset($match[5])) {$re['host'] = $match[5];} else { $re['host'] = "";};
if (isset($match[6])) {$re['domain'] = $match[6];} else { $re['domain'] = "";};
if (isset($match[7])) {$re['port'] = $match[7];} else { $re['port'] = "";};
if (isset($match[8])) {$re['path'] = $match[8];} else { $re['path'] = "";};
if (isset($match[9])) {$re['query'] = $match[9];} else { $re['query'] = "";};
if (isset($match[10])) {$re['fragment'] = $match[10];} else { $re['fragment'] = "";};
return $re;
$re["domain"] = $re["host"];
}
$re["port"] = empty($parts["port"]) ? ($re["schema"] === "http" ? 80 : ($re["schema"] === "https" ? 443 : 80)) : $parts["port"];
$re["path"] = empty($parts["path"]) ? "" : $parts["path"];
$re["query"] = empty($parts["query"]) ? "" : $parts["query"];
$re["fragment"] = empty($parts["fragment"]) ? "" : $parts["fragment"];
return $re;
}
# Getter
......
......@@ -32,6 +32,9 @@ abstract class Searchengine
public $startTime; # Die Zeit der Erstellung dieser Suchmaschine
public $hash; # Der Hash-Wert dieser Suchmaschine
private $user; # Username für HTTP-Auth (falls angegeben)
private $password; # Passwort für HTTP-Auth (falls angegeben)
public $fp; # Wird für Artefakte benötigt
public $socketNumber = null; # Wird für Artefakte benötigt
public $counter = 0; # Wird eventuell für Artefakte benötigt
......@@ -171,9 +174,10 @@ abstract class Searchengine
$needSearcher = true;
}
}
if($needSearcher && Redis::get($this->name) !== "locked"){
Redis::set($this->name, "locked");
$this->dispatch(new Searcher($this->name));
$this->dispatch(new Searcher($this->name, $this->user, $this->password));
}
}
}
......
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
class Scopia extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults($result)
{
$result = html_entity_decode($result);
$result = str_replace("&", "&amp;", $result);
try {
$content = simplexml_load_string($result);
if (!$content) {
return;
}
$results = $content->xpath('//results/result');
foreach ($results as $result) {
$title = $result->title->__toString();
$link = $result->url->__toString();
$anzeigeLink = $link;
$descr = $result->description->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name:");
Log::error($e->getMessage());
return;
}
}
public function getNext(\App\MetaGer $metager, $result)
{
$result = html_entity_decode($result);
$result = str_replace("&", "&amp;", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name:");
Log::error($e->getMessage());
return;
}
if (!$content) {
return;
}
$more = $content->xpath('//results/more')[0]->__toString() === "1" ? true : false;
if ($more) {
$results = $content->xpath('//results/result');
$number = $results[sizeof($results) - 1]->number->__toString();
# Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
$next = new Scopia(simplexml_load_string($this->engine), $metager);
$next->getString = preg_replace("/\\?s=.*?&/si", "?s=" . $number, $next->getString);
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
}
}
}
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
class Shopzilla extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults($result)
{
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
if (!$content) {
return;
}
$results = $content->{"Products"};
$count = 0;
foreach ($results->{"Offer"} as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"url"}->__toString();
$anzeigeLink = $result->{"rawUrl"}->__toString();
// Try to extract the display Urls:
if(preg_match("/^http[s]{0,1}:\/\/ad.eanalyzer.de/", $anzeigeLink)){
$parts = parse_url($anzeigeLink);
parse_str($parts['query'], $query);
$anzeigeLink = $query["url"];
}
$descr = $result->{"description"}->__toString();
$image = $result->{"Images"}->{"Image"}[1]->__toString();
$price = $result->{"price"}->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter,
['partnershop' => true,
'price' => $price,
'image' => $image]
);
$count++;
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name:");
Log::error($e->getMessage());
return;
}
}
public function getNext(\App\MetaGer $metager, $result)
{
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
if (!$content) {
return;
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name:");
Log::error($e->getMessage());
return;
}
// Get total Result Count
$totalResults = intval($content->{"Products"}["totalResults"]->__toString());
// Get the current Result Position
$current = 0;
if(strpos($this->getString, "&start=") !== FALSE){
$tmp = substr($this->getString, strpos($this->getString, "&start=")+7);
if(strpos($tmp, "&") !== FALSE){
$tmp = substr($tmp, 0, strpos($tmp, "&"));
}
$current = intval($tmp);
}
if($current >= ($totalResults - 10)){
return;
}
# Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
$next = new Shopzilla(simplexml_load_string($this->engine), $metager);
if(strpos($next->getString, "&start=") !== FALSE){
$next->getString = preg_replace("/&start=\d+/si", "&start=" . ($current+10), $next->getString);
}else{
$next->getString .= "&start=" . ($current+10);
}
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
}
}
......@@ -7,6 +7,8 @@ use Illuminate\Queue\Events\JobProcessing;
use Illuminate\Support\ServiceProvider;
use Queue;
use Illuminate\Support\Facades\Redis;
use Request;
use App;
class AppServiceProvider extends ServiceProvider
{
......@@ -17,6 +19,17 @@ class AppServiceProvider extends ServiceProvider
*/
public function boot()
{
/**
* metager.org is our english Domain
* We will change the Locale to en
*/
$host = Request::header("X_Forwarded_Host", "");
if(stripos($host, "metager.org") !== FALSE){
App::setLocale('en');
}
# Wir loggen im Redis-System für jede Sekunde des Tages, wie viele Worker aktiv am Laufen waren.
# Dies ist notwendig, damit wir mitbekommen können, ab welchem Zeitpunkt wir zu wenig Worker zur Verfügung haben.
Queue::before(function (JobProcessing $event) {
......
......@@ -39,6 +39,8 @@ class RouteServiceProvider extends ServiceProvider
$this->mapWebRoutes();
$this->mapSessionRoutes();
//
}
......@@ -76,4 +78,22 @@ class RouteServiceProvider extends ServiceProvider
require base_path('routes/api.php');
});
}
/**
* Define the "session" routes for the application.
*
* These routes all receive session state, CSRF protection, etc.
*
* @return void
*/
protected function mapSessionRoutes()
{
Route::group([
'middleware' => 'session',
'namespace' => $this->namespace,
], function ($router) {
require base_path('routes/session.php');
});
}
}
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
......@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"content-hash": "987d8a3a450027401e164f760b2226d2",
"content-hash": "2b5d36f7b51c797b1fe02dcbf11f3b06",
"packages": [
{
"name": "dnoegel/php-xdg-base-dir",
......@@ -208,6 +208,34 @@
],
"time": "2017-06-15T17:19:42+00:00"
},
{
"name": "globalcitizen/php-iban",
"version": "v2.6.9",
"source": {
"type": "git",
"url": "https://github.com/globalcitizen/php-iban.git",
"reference": "16b8c416e8a4047b52dca00794f161bfa68afb7a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/globalcitizen/php-iban/zipball/16b8c416e8a4047b52dca00794f161bfa68afb7a",
"reference": "16b8c416e8a4047b52dca00794f161bfa68afb7a",
"shasum": ""
},
"type": "library",
"autoload": {
"files": [
"oophp-iban.php",
"php-iban.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"LGPL-3.0"
],
"description": "php-iban is a library for parsing and validating IBAN (and IIBAN) bank account information.",
"time": "2018-08-14T00:04:02+00:00"
},
{
"name": "guzzlehttp/guzzle",
"version": "6.3.0",
......@@ -389,6 +417,76 @@
],
"time": "2017-03-20T17:10:46+00:00"
},
{
"name": "intervention/image",
"version": "2.4.2",
"source": {
"type": "git",
"url": "https://github.com/Intervention/image.git",
"reference": "e82d274f786e3d4b866a59b173f42e716f0783eb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Intervention/image/zipball/e82d274f786e3d4b866a59b173f42e716f0783eb",
"reference": "e82d274f786e3d4b866a59b173f42e716f0783eb",
"shasum": ""
},
"require": {
"ext-fileinfo": "*",
"guzzlehttp/psr7": "~1.1",
"php": ">=5.4.0"
},
"require-dev": {
"mockery/mockery": "~0.9.2",
"phpunit/phpunit": "^4.8 || ^5.7"
},
"suggest": {
"ext-gd": "to use GD library based image processing.",
"ext-imagick": "to use Imagick based image processing.",
"intervention/imagecache": "Caching extension for the Intervention Image library"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.4-dev"
},
"laravel": {
"providers": [
"Intervention\\Image\\ImageServiceProvider"
],
"aliases": {
"Image": "Intervention\\Image\\Facades\\Image"
}
}
},
"autoload": {
"psr-4": {
"Intervention\\Image\\": "src/Intervention/Image"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Oliver Vogel",
"email": "oliver@olivervogel.com",
"homepage": "http://olivervogel.com/"
}
],
"description": "Image handling and manipulation library with support for Laravel integration",
"homepage": "http://image.intervention.io/",
"keywords": [
"gd",
"image",
"imagick",
"laravel",
"thumbnail",
"watermark"
],
"time": "2018-05-29T14:19:03+00:00"
},
{
"name": "jakub-onderka/php-console-color",
"version": "0.1",
......@@ -980,6 +1078,73 @@
],
"time": "2017-08-08T04:41:28+00:00"
},
{
"name": "mews/captcha",
"version": "2.2.0",
"source": {
"type": "git",
"url": "https://github.com/mewebstudio/captcha.git",
"reference": "c9885e31bb2c30fe185c1af6078a53a3bef5db8f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/mewebstudio/captcha/zipball/c9885e31bb2c30fe185c1af6078a53a3bef5db8f",
"reference": "c9885e31bb2c30fe185c1af6078a53a3bef5db8f",
"shasum": ""
},
"require": {
"ext-gd": "*",
"illuminate/config": "~5.0",
"illuminate/filesystem": "~5.0",
"illuminate/hashing": "~5.0",
"illuminate/support": "~5.0",
"intervention/image": "~2.2",
"php": ">=5.4"
},
"require-dev": {
"mockery/mockery": "0.9.*",
"phpunit/phpunit": "~4.1"
},
"type": "package",
"extra": {
"laravel": {
"providers": [
"Mews\\Captcha\\CaptchaServiceProvider"
],
"aliases": {
"Captcha": "Mews\\Captcha\\Facades\\Captcha"
}
}
},
"autoload": {
"psr-4": {
"Mews\\Captcha\\": "src/"
},
"files": [
"src/helpers.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Muharrem ERİN",
"email": "me@mewebstudio.com",
"homepage": "https://github.com/mewebstudio",
"role": "Developer"
}
],
"description": "Laravel 5 Captcha Package",
"homepage": "https://github.com/mewebstudio/captcha",
"keywords": [
"captcha",
"laravel5 Captcha",
"laravel5 Security"
],
"time": "2018-04-25T13:44:49+00:00"
},