Skip to content

GitLab

MetaGer
MetaGer
Commit f33b2a0d authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files
Options

Merge branch 'development' into carsten-membership-form

parents ac15ec9c 28ca0eef
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 4980 additions and 19218 deletions
.gitignore
View file @ f33b2a0d
......@@ -21,4 +21,7 @@ npm-debug.log
/.project
composer.lock
local.log
\ No newline at end of file
package-lock.json
local.log
browserstack.err
.gitlab-ci.yml
View file @ f33b2a0d
......@@ -33,6 +33,8 @@ stages:
- integrationtest
- cleanup
.auto-deploy:
image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.6"
build:
services:
......@@ -75,6 +77,10 @@ review:
variables:
HELM_UPGRADE_VALUES_FILE: .gitlab/review-apps-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment
environment:
name: review/$CI_COMMIT_REF_NAME
on_stop: stop_review
auto_stop_in: 2 days
rules:
- if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
when: never
......@@ -87,6 +93,11 @@ review:
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
stop_review:
variables:
GIT_STRATEGY: none
environment:
name: review/$CI_COMMIT_REF_NAME
action: stop
rules:
- if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
when: never
......@@ -163,4 +174,4 @@ integrationtest:
- sed -i "s#^APP_URL=.*#APP_URL=$URL#g" .env
- sed -i "s#^BRANCH_NAME=.*#BRANCH_NAME=$CI_COMMIT_REF_NAME#g" .env
- sed -i "s#^COMMIT_NAME=.*#COMMIT_NAME=$CI_COMMIT_REF_SLUG#g" .env
- php artisan dusk
\ No newline at end of file
- php artisan dusk
.gitlab/development-values.yaml
View file @ f33b2a0d
postgresql:
enabled: false
service:
externalPort: 80
internalPort: 80
......@@ -5,13 +7,26 @@ hpa:
enabled: true
minReplicas: 1
maxReplicas: 5
resources:
limits:
resourcesPhpfpm:
requests:
cpu: 500m
memory: 500M
limits:
resourcesNginx:
requests:
cpu: 100m
memory: 100M
limits:
resourcesRedis:
requests:
cpu: 100m
memory: 1Gi
limits:
resourcesFetcher:
requests:
cpu: 500m
memory: 1Gi
memory: 100M
limits:
podDisruptionBudget:
enabled: true
minAvailable:
......@@ -25,7 +40,7 @@ ingress:
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self'; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self' www.paypal.com; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "X-Frame-Options: sameorigin";
more_set_headers "X-Content-Type-Options: nosniff";
more_set_headers "ReferrerPolicy: origin";
......
.gitlab/production-values.yaml
View file @ f33b2a0d
postgresql:
enabled: false
service:
externalPort: 80
internalPort: 80
......@@ -5,13 +7,6 @@ hpa:
enabled: true
minReplicas: 3
maxReplicas: 100
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
podDisruptionBudget:
enabled: true
minAvailable:
......@@ -21,11 +16,34 @@ podAnnotations:
prometheus.io/path: /metrics
prometheus.io/port: "80"
deploymentApiVersion: apps/v1
resources:
requests:
limits:
resourcesPhpfpm:
requests:
cpu: 500m
memory: 500M
limits:
resourcesNginx:
requests:
cpu: 100m
memory: 100M
limits:
resourcesRedis:
requests:
cpu: 100m
memory: 1Gi
limits:
resourcesFetcher:
requests:
cpu: 500m
memory: 100M
limits:
ingress:
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self'; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self' www.paypal.com; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "X-Frame-Options: sameorigin";
more_set_headers "X-Content-Type-Options: nosniff";
more_set_headers "ReferrerPolicy: origin";
......
.gitlab/review-apps-values.yaml
View file @ f33b2a0d
---
postgresql:
enabled: false
hpa:
enabled: false
resources:
requests:
limits:
ingress:
annotations:
kubernetes.io/tls-acme: "false"
......@@ -21,3 +25,15 @@ service:
externalPort: 80
internalPort: 80
deploymentApiVersion: apps/v1
resourcesPhpfpm:
requests:
limits:
resourcesNginx:
requests:
limits:
resourcesRedis:
requests:
limits:
resourcesFetcher:
requests:
limits:
\ No newline at end of file
app/Console/Commands/RequestFetcher.php
View file @ f33b2a0d
......@@ -25,10 +25,9 @@ class RequestFetcher extends Command
protected $shouldRun = true;
protected $multicurl = null;
protected $oldMultiCurl = null;
protected $maxFetchedDocuments = 100000;
protected $fetchedDocuments = 0;
protected $proxyhost, $proxyuser, $proxypassword;
protected $proxyhost;
protected $proxyuser;
protected $proxypassword;
/**
* Create a new command instance.
......@@ -43,7 +42,6 @@ class RequestFetcher extends Command
$this->proxyport = env("PROXY_PORT", "");
$this->proxyuser = env("PROXY_USER", "");
$this->proxypassword = env("PROXY_PASSWORD", "");
}
/**
......@@ -53,7 +51,6 @@ class RequestFetcher extends Command
*/
public function handle()
{
$pidFile = "/tmp/fetcher";
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
......@@ -80,61 +77,16 @@ class RequestFetcher extends Command
}
try {
$blocking = false;
while ($this->shouldRun) {
$status = curl_multi_exec($this->multicurl, $active);
$currentJobs = [];
if (!$blocking) {
$elements = Redis::pipeline(function ($redis) {
$redis->lrange(\App\MetaGer::FETCHQUEUE_KEY, 0, -1);
$redis->del(\App\MetaGer::FETCHQUEUE_KEY);
});
$currentJobs = $elements[0];
} else {
$currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($currentJob)) {
$currentJobs[] = $currentJob[1];
}
}
if (sizeof($currentJobs) > 0) {
foreach ($currentJobs as $currentJob) {
$currentJob = json_decode($currentJob, true);
$ch = $this->getCurlHandle($currentJob);
if (curl_multi_add_handle($this->multicurl, $ch) !== 0) {
$this->shouldRun = false;
Log::error("Couldn't add Handle to multicurl");
break;
}
$this->fetchedDocuments++;
if ($this->fetchedDocuments > $this->maxFetchedDocuments) {
Log::info("Reinitializing Multicurl after " . $this->fetchedDocuments . " requests.");
$this->oldMultiCurl = $this->multicurl;
$this->multicurl = curl_multi_init();
$this->fetchedDocuments = 0;
}
$blocking = false;
$active = true;
}
}
$answerRead = $this->readMultiCurl($this->multicurl);
if ($this->oldMultiCurl != null) {
$this->readMultiCurl($this->oldMultiCurl);
$messagesLeft = -1;
if (curl_multi_info_read($this->oldMultiCurl, $messagesLeft) === false) {
if ($messagesLeft = 0) {
Log::debug("Removing finished multicurl handle");
curl_multi_close($this->oldMultiCurl);
$this->oldMultiCurl = null;
}
}
}
if (!$active && !$answerRead) {
$blocking = true;
} else {
usleep(50 * 1000);
$operationsRunning = true;
curl_multi_exec($this->multicurl, $operationsRunning);
$status = $this->readMultiCurl($this->multicurl);
$answersRead = $status[0];
$messagesLeft = $status[1];
$newJobs = $this->checkNewJobs($operationsRunning, $messagesLeft);
if ($newJobs === 0 && $answersRead === 0) {
usleep(10 * 1000);
}
}
} finally {
......@@ -143,21 +95,59 @@ class RequestFetcher extends Command
}
}
/**
* Checks the Redis queue if any new fetch jobs where submitted
* and adds them to multicurl if there are.
* Will be blocking call to redis if there are no running jobs in multicurl
*/
private function checkNewJobs($operationsRunning, $messagesLeft)
{
$newJobs = [];
if ($operationsRunning === 0 && $messagesLeft === -1) {
$newJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($newJob)) {
$newJobs[] = $newJob[1];
}
} else {
$elements = Redis::pipeline(function ($redis) {
$redis->lrange(\App\MetaGer::FETCHQUEUE_KEY, 0, -1);
$redis->del(\App\MetaGer::FETCHQUEUE_KEY);
});
$newJobs = $elements[0];
}
$addedJobs = 0;
foreach ($newJobs as $newJob) {
$newJob = json_decode($newJob, true);
$ch = $this->getCurlHandle($newJob);
if (curl_multi_add_handle($this->multicurl, $ch) !== 0) {
$this->shouldRun = false;
Log::error("Couldn't add Handle to multicurl");
break;
} else {
$addedJobs++;
}
}
return $addedJobs;
}
private function readMultiCurl($mc)
{
$answerRead = false;
while (($info = curl_multi_info_read($mc)) !== false) {
$messagesLeft = -1;
$answersRead = 0;
while (($info = curl_multi_info_read($mc, $messagesLeft)) !== false) {
try {
$answerRead = true;
$answersRead++;
$infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
$infos = explode(";", $infos);
$resulthash = $infos[0];
$cacheDurationMinutes = intval($infos[1]);
$name = $infos[2];
$responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
$body = "";
$body = "no-result";
$totalTime = curl_getinfo($info["handle"], CURLINFO_TOTAL_TIME);
$totalTime = curl_getinfo($info["handle"], CURLINFO_TOTAL_TIME);
\App\PrometheusExporter::Duration($totalTime, $name);
$error = curl_error($info["handle"]);
......@@ -166,13 +156,14 @@ class RequestFetcher extends Command
}
if ($responseCode !== 200) {
Log::debug($resulthash);
Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
} else {
$body = \curl_multi_getcontent($info["handle"]);
}
Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->lpush($resulthash, $body);
$pipe->expire($resulthash, 60);
});
......@@ -187,7 +178,7 @@ class RequestFetcher extends Command
\curl_multi_remove_handle($mc, $info["handle"]);
}
}
return $answerRead;
return [$answersRead, $messagesLeft];
}
private function getCurlHandle($job)
......@@ -234,7 +225,6 @@ class RequestFetcher extends Command
public function sig_handler($sig)
{
$this->shouldRun = false;
echo ("Terminating Process\n");
echo("Terminating Process\n");
}
}
app/Http/Controllers/AdminInterface.php
View file @ f33b2a0d
......@@ -31,7 +31,6 @@ class AdminInterface extends Controller
if (strstr($key, "_time")) {
$stati[$name]["fetcher"][$pid]["connection"][$key] = $val;
}
}
$stati[$name]["fetcher"][$pid]["poptime"] = $value[1];
}
......@@ -157,7 +156,7 @@ class AdminInterface extends Controller
$now->minute = 0;
$now->second = 0;
while($now->lessThanOrEqualTo(Carbon::now())){
while ($now->lessThanOrEqualTo(Carbon::now())) {
$sameTime += empty($stats->time->{$now->format('H:i')}->{$interface}) ? 0 : $stats->time->{$now->format('H:i')}->{$interface};
$now->addMinutes(5);
}
......@@ -178,19 +177,38 @@ class AdminInterface extends Controller
if ($size > 0) {
$oldLogs[$key]['median'] = number_format(floatval(round($count / $size)), 0, ",", ".");
}
}
$sameTimes = [];
$sum = 0;
foreach ($oldLogs as $index => $oldLog) {
if ($index % 7 === 0) {
$sameTime = $oldLog["sameTime"];
$sameTime = str_replace(".", "", $sameTime);
$sameTime = \intval($sameTime);
$sameTimes[] = ($logToday - $sameTime);
$sum += ($logToday - $sameTime);
}
}
$averageIncrease = 0;
if (sizeof($sameTimes) > 0) {
$averageIncrease = $sum / sizeof($sameTimes);
}
if ($request->input('out', 'web') === "web") {
return view('admin.count')
->with('title', 'Suchanfragen - MetaGer')
->with('today', number_format(floatval($logToday), 0, ",", "."))
->with('averageIncrease', $averageIncrease)
->with('oldLogs', $oldLogs)
->with('minCount', $minCount)
->with('rekordCount', number_format(floatval($rekordTag), 0, ",", "."))
->with('rekordTagSameTime', number_format(floatval($rekordTagSameTime), 0, ",", "."))
->with('rekordDate', $rekordTagDate)
->with('days', $days);
->with('days', $days)
->with('css', [mix('/css/count/style.css')])
->with('darkcss', [mix('/css/count/dark.css')]);
} else {
$result = "";
foreach ($oldLogs as $key => $value) {
......@@ -203,9 +221,7 @@ class AdminInterface extends Controller
return response($result, 200)
->header('Content-Type', 'text/csv')
->header('Content-Disposition', 'attachment; filename="count.csv"');
}
}
public function countGraphToday()
......@@ -235,7 +251,6 @@ class AdminInterface extends Controller
return response()
->view('admin.countGraphToday', ["data" => $result], 200)
->header('Content-Type', "image/svg+xml");
}
public function engineStats()
......
app/Http/Controllers/AdminSpamController.php
View file @ f33b2a0d
......@@ -44,6 +44,9 @@ class AdminSpamController extends Controller
public function jsonQueries()
{
$queries = $this->getQueries();
# JSON encoding will fail if invalid UTF-8 Characters are in this string
# mb_convert_encoding will remove thise invalid characters for us
$queries = mb_convert_encoding($queries, "UTF-8", "UTF-8");
return response()->json($queries);
}
......@@ -77,8 +80,10 @@ class AdminSpamController extends Controller
];
}
# JSON encoding will fail if invalid UTF-8 Characters are in this string
# mb_convert_encoding will remove thise invalid characters for us
$resultData = mb_convert_encoding($resultData, "UTF-8", "UTF-8");
return response()->json($resultData);
}
private function getQueries()
......
app/Http/Controllers/MetaGerSearch.php
View file @ f33b2a0d
......@@ -12,9 +12,11 @@ use View;
class MetaGerSearch extends Controller
{
public function search(Request $request, MetaGer $metager, $timing = false)
{
if ($request->filled("chrome-plugin")) {
return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), "/plugin"));
}
$timings = null;
if ($timing) {
$timings = ['starttime' => microtime(true)];
......@@ -74,12 +76,23 @@ class MetaGerSearch extends Controller
return response($responseContent);
}
$quicktips = $metager->createQuicktips();
if (!empty($timings)) {
$timings["createQuicktips"] = microtime(true) - $time;
}
# Suche für alle zu verwendenden Suchmaschinen als Job erstellen,
# auf Ergebnisse warten und die Ergebnisse laden
$metager->createSearchEngines($request, $timings);
$metager->startSearch($timings);
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
if (!empty($timings)) {
$timings["Loaded Quicktip Results"] = microtime(true) - $time;
}
$metager->waitForMainResults();
if (!empty($timings)) {
$timings["waitForMainResults"] = microtime(true) - $time;
......@@ -97,10 +110,7 @@ class MetaGerSearch extends Controller
}
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults();
if (!empty($timings)) {
$timings["prepareResults"] = microtime(true) - $time;
}
$metager->prepareResults($timings);
$finished = true;
foreach ($metager->getEngines() as $engine) {
......@@ -129,7 +139,7 @@ class MetaGerSearch extends Controller
}
# Die Ausgabe erstellen:
$resultpage = $metager->createView();
$resultpage = $metager->createView($quicktipResults);
if ($spamEntry !== null) {
try {
Cache::put('spam.' . $metager->getFokus() . "." . md5($spamEntry), $resultpage->render(), 604800);
......@@ -156,7 +166,7 @@ class MetaGerSearch extends Controller
// This might speed up page view time for users with slow network
$responseArray = str_split($resultpage->render(), 1024);
foreach ($responseArray as $responsePart) {