Commit 27ad7f8d authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch 'development' into '903-integrating-kelkoo-product-feeds'

Development

See merge request !1541
parents b4125948 5dafde00
README.md
CHANGELOG.md
docker-compose.yml
Dockerfile
Dockerfile
\ No newline at end of file
APP_ENV=local
APP_DEBUG=true
APP_LOG_LEVEL=debug
LOG_CHANNEL=stderr
APP_KEY=
APP_URL=http://localhost
......@@ -21,7 +22,8 @@ CACHE_DRIVER=redis
SESSION_DRIVER=file
QUEUE_CONNECTION=sync
REDIS_HOST=127.0.0.1
REDIS_CACHE_HOST=redis
REDIS_HOST=redis
REDIS_PASSWORD=null
REDIS_PORT=6379
......
variables:
DOCKER_HOST: "tcp://docker-dind.gitlab:2375"
AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS: "--network host"
POSTGRES_ENABLED: "false"
CODE_QUALITY_DISABLED: "true"
CONTAINER_SCANNING_DISABLED: "true"
......
FROM alpine:latest
FROM alpine:3.11.3
RUN apk add --update \
nginx \
......@@ -29,7 +29,11 @@ RUN apk add --update \
WORKDIR /html
RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stdout/g' /etc/php7/php-fpm.conf && \
sed -i 's/;daemonize = yes/daemonize = no/g' /etc/php7/php-fpm.conf && \
sed -i 's/listen = 127.0.0.1:9000/listen = 9000/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
......@@ -56,6 +60,7 @@ RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
COPY config/nginx.conf /etc/nginx/nginx.conf
COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf
RUN sed -i 's/fastcgi_pass phpfpm:9000;/fastcgi_pass localhost:9000;/g' /etc/nginx/conf.d/default.conf
COPY --chown=root:nginx . /html
WORKDIR /html
......@@ -64,7 +69,4 @@ EXPOSE 80
CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
chmod -R g+w storage/logs/metager bootstrap/cache && \
crond -L /dev/stdout && \
nginx && \
php-fpm7 -D && \
redis-server /etc/redis.conf && \
su -s /bin/sh -c 'php artisan requests:fetcher' nginx
php-fpm7
FROM alpine:latest
FROM alpine:3.11.3
RUN apk add --update \
nginx \
......@@ -23,12 +23,17 @@ RUN apk add --update \
php7-gd \
php7-json \
php7-pcntl \
php7-opcache \
php7-fileinfo \
&& rm -rf /var/cache/apk/*
WORKDIR /html
RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stdout/g' /etc/php7/php-fpm.conf && \
sed -i 's/;daemonize = yes/daemonize = no/g' /etc/php7/php-fpm.conf && \
sed -i 's/listen = 127.0.0.1:9000/listen = 9000/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
......@@ -38,6 +43,14 @@ RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
sed -i 's/expose_php = On/expose_php = Off/g' /etc/php7/php.ini && \
# Opcache configuration
sed -i 's/;opcache.enable=1/opcache.enable=1/g' /etc/php7/php.ini && \
sed -i 's/;opcache.memory_consumption=128/opcache.memory_consumption=128/g' /etc/php7/php.ini && \
sed -i 's/;opcache.interned_strings_buffer=8/opcache.interned_strings_buffer=8/g' /etc/php7/php.ini && \
sed -i 's/;opcache.max_accelerated_files=10000/opcache.max_accelerated_files=10000/g' /etc/php7/php.ini && \
sed -i 's/;opcache.max_wasted_percentage=5/opcache.max_wasted_percentage=5/g' /etc/php7/php.ini && \
sed -i 's/;opcache.validate_timestamps=1/opcache.validate_timestamps=1/g' /etc/php7/php.ini && \
sed -i 's/;opcache.revalidate_freq=2/opcache.revalidate_freq=300/g' /etc/php7/php.ini && \
echo "daemonize yes" >> /etc/redis.conf && \
ln -s /dev/null /var/log/nginx/access.log && \
ln -s /dev/stdout /var/log/nginx/error.log && \
......@@ -51,7 +64,4 @@ EXPOSE 80
CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
chmod -R g+w storage/logs/metager bootstrap/cache && \
crond -L /dev/stdout && \
nginx && \
php-fpm7 -D && \
redis-server /etc/redis.conf && \
su -s /bin/sh -c 'php artisan requests:fetcher' nginx
php-fpm7
README.md
CHANGELOG.md
vendor/*
node_modules/*
storage/logs/*
\ No newline at end of file
......@@ -50,20 +50,26 @@ class RequestFetcher extends Command
*/
public function handle()
{
$pidFile = "/tmp/fetcher";
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
touch($pidFile);
if (!file_exists($pidFile)) {
return;
}
try {
$blocking = false;
$redis = Redis::connection("cache");
while ($this->shouldRun) {
$status = curl_multi_exec($this->multicurl, $active);
$currentJob = null;
if (!$blocking) {
$currentJob = $redis->lpop(\App\MetaGer::FETCHQUEUE_KEY);
$currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
} else {
$currentJob = $redis->blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
$currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($currentJob)) {
$currentJob = $currentJob[1];
}
......@@ -98,8 +104,11 @@ class RequestFetcher extends Command
$body = \curl_multi_getcontent($info["handle"]);
}
Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->expire($resulthash, 60);
Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
});
\curl_multi_remove_handle($this->multicurl, $info["handle"]);
}
if (!$active && !$answerRead) {
......@@ -108,9 +117,8 @@ class RequestFetcher extends Command
usleep(50 * 1000);
}
}
} catch (\Exception $e) {
Log::error($e->getMessage());
} finally {
unlink($pidFile);
curl_multi_close($this->multicurl);
}
}
......@@ -125,11 +133,11 @@ class RequestFetcher extends Command
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_CONNECTTIMEOUT => 2,
CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 500,
CURLOPT_LOW_SPEED_TIME => 5,
CURLOPT_TIMEOUT => 10,
CURLOPT_LOW_SPEED_LIMIT => 50000,
CURLOPT_LOW_SPEED_TIME => 2,
CURLOPT_TIMEOUT => 3,
));
if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
......
......@@ -6,6 +6,7 @@ use App;
use Cache;
use Carbon;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use Jenssegers\Agent\Agent;
use LaravelLocalization;
use Log;
......@@ -869,7 +870,7 @@ class MetaGer
while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
foreach ($enginesToWaitFor as $index => $engine) {
if (Cache::get($engine->hash) !== null) {
if (Redis::get($engine->hash) !== null) {
$answered[] = $engine;
unset($enginesToWaitFor[$index]);
break;
......
......@@ -46,7 +46,7 @@ class Quicktips
$mission = json_encode($mission);
Redis::connection('cache')->rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
}
}
......
......@@ -3,7 +3,6 @@
namespace App\Models;
use App\MetaGer;
use Cache;
use Illuminate\Support\Facades\Redis;
abstract class Searchengine
......@@ -148,7 +147,7 @@ abstract class Searchengine
// Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::connection('cache')->rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
if (!empty($timings)) {
$timings["startSearch"][$this->name]["pushed job"] = microtime(true) - $timings["starttime"];
}
......@@ -194,7 +193,7 @@ abstract class Searchengine
$body = "";
}
} else {
$body = Cache::get($this->hash);
$body = Redis::get($this->hash);
}
if ($body !== null) {
......
......@@ -3,6 +3,7 @@
namespace App\Models\parserSkripte;
use App\Models\XmlSearchengine;
use Log;
class Qualigo extends XmlSearchengine
{
......@@ -14,6 +15,10 @@ class Qualigo extends XmlSearchengine
protected function loadXmlResults($resultsXml)
{
if(empty($resultsXml) || !$resultsXml){
Log::error("A problem occurred parsing results from $this->name:");
return;
}
try {
$results = $resultsXml->xpath('//RL/RANK');
foreach ($results as $result) {
......
......@@ -39,7 +39,8 @@ spec:
persistentVolumeClaim:
claimName: mglogs
containers:
- name: {{ .Chart.Name }}
# Main PHP-FPM Container
- name: {{ .Chart.Name }}-phpfpm
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- if .Values.application.secretName }}
......@@ -48,23 +49,42 @@ spec:
name: {{ .Values.application.secretName }}
{{- end }}
env:
{{- if .Values.postgresql.managed }}
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: app-postgres
key: username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: app-postgres
key: password
- name: POSTGRES_HOST
valueFrom:
secretKeyRef:
name: app-postgres
key: privateIP
{{- end }}
- name: DATABASE_URL
value: {{ .Values.application.database_url | quote }}
- name: GITLAB_ENVIRONMENT_NAME
value: {{ .Values.gitlab.envName }}
- name: GITLAB_ENVIRONMENT_URL
value: {{ .Values.gitlab.envURL }}
ports:
- name: "{{ .Values.service.name }}-phpfpm"
containerPort: 9000
livenessProbe:
tcpSocket:
port: "{{ .Values.service.name }}-phpfpm"
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
readinessProbe:
tcpSocket:
port: "{{ .Values.service.name }}-phpfpm"
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
volumeMounts:
- name: mglogs-persistent-storage
mountPath: /html/storage/logs/metager
readOnly: false
resources:
{{ toYaml .Values.resources | indent 12 }}
# Nginx Container
- name: {{ .Chart.Name }}-nginx
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
command: ["nginx"]
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- if .Values.application.secretName }}
envFrom:
- secretRef:
name: {{ .Values.application.secretName }}
{{- end }}
env:
- name: DATABASE_URL
value: {{ .Values.application.database_url | quote }}
- name: GITLAB_ENVIRONMENT_NAME
......@@ -98,10 +118,41 @@ spec:
{{- end }}
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
volumeMounts:
- name: mglogs-persistent-storage
mountPath: /html/storage/logs/metager
readOnly: false
resources:
{{ toYaml .Values.resources | indent 12 }}
# Redis Container
- name: {{ .Chart.Name }}-redis
image: "redis:6.0-rc1-alpine"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: "{{ .Values.service.name }}-redis"
containerPort: 6379
livenessProbe:
tcpSocket:
port: "{{ .Values.service.name }}-redis"
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
readinessProbe:
tcpSocket:
port: "{{ .Values.service.name }}-redis"
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
# Fetcher Container
- name: {{ .Chart.Name }}-fetcher
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
command: ["su"]
args: ["-s", "/bin/sh", "-c", "php artisan requests:fetcher", "nginx"]
imagePullPolicy: {{ .Values.image.pullPolicy }}
livenessProbe:
exec:
command:
- cat
- /tmp/fetcher
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
readinessProbe:
exec:
command:
- cat
- /tmp/fetcher
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
{{- end -}}
......@@ -17,6 +17,7 @@
"laravel/tinker": "^1.0",
"mcamara/laravel-localization": "^1.3",
"mews/captcha": "^2.2",
"monospice/laravel-redis-sentinel-drivers": "^2.6",
"predis/predis": "^1.1",
"symfony/dom-crawler": "^4.1"
},
......@@ -66,4 +67,4 @@
"@php artisan key:generate --ansi"
]
}
}
\ No newline at end of file
}
......@@ -69,7 +69,7 @@ return [
],
'redis' => [
'driver' => 'redis',
'driver' => env('REDIS_CACHE_DRIVER', 'redis'),
'connection' => 'cache',
],
......
......@@ -14,7 +14,7 @@ server {
location ~ \.php$ {
try_files $uri /index.php =404;
fastcgi_split_path_info ^(.+\.php)(/.+)$;
fastcgi_pass 127.0.0.1:9000;
fastcgi_pass phpfpm:9000;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
include fastcgi_params;
......
......@@ -5,6 +5,7 @@ worker_processes 100;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
daemon off;
events {
worker_connections 1024;
......
......@@ -28,20 +28,43 @@ services:
- MYSQL_USER=metager
- MYSQL_PASSWORD=metager
- MYSQL_DATABASE=metager
web:
redis:
restart: on-failure
image: redis:6.0-rc1-alpine
phpfpm:
depends_on:
- "mgdb"
- "dependencies"
- "phpdeps"
- "assets"
- "redis"
restart: on-failure
build:
context: .
dockerfile: Dockerfile.dev
image: metager:latest
working_dir: /html
volumes:
- .:/html
nginx:
depends_on:
- "phpfpm"
restart: on-failure
image: metager:latest
working_dir: /html
command: nginx
volumes:
- .:/html
- ./config/nginx.conf:/etc/nginx/nginx.conf
- ./config/nginx-default.conf:/etc/nginx/conf.d/default.conf
ports:
- "8080:80"
worker:
depends_on:
- "phpfpm"
restart: on-failure
image: metager:latest
working_dir: /html
volumes:
- .:/html
command: "su -s /bin/sh -c 'php artisan requests:fetcher' nginx"
......@@ -21,9 +21,7 @@ Route::group(
/** ADD ALL LOCALIZED ROUTES INSIDE THIS GROUP **/
Route::get('/', 'StartpageController@loadStartPage');
Route::get('sand', function () {
abort(500, "test");
});
Route::get('asso', function () {
return view('assoziator.asso')
->with('title', trans('titles.asso'))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment