Merge branch '943-optimize-deployment' into 'development'

Resolve "Optimize Deployment"

Closes #943

See merge request !1534

.dockerignore

 README.md
 CHANGELOG.md
 docker-compose.yml
-Dockerfile
+Dockerfile
\ No newline at end of file

.env.example

 APP_ENV=local
 APP_DEBUG=true
 APP_LOG_LEVEL=debug
+LOG_CHANNEL=stderr
 APP_KEY=
 APP_URL=http://localhost
 
@@ -21,7 +22,8 @@ CACHE_DRIVER=redis
 SESSION_DRIVER=file
 QUEUE_CONNECTION=sync
 
-REDIS_HOST=127.0.0.1
+REDIS_CACHE_HOST=redis
+REDIS_HOST=redis
 REDIS_PASSWORD=null
 REDIS_PORT=6379
 

Dockerfile

-FROM alpine:latest
+FROM alpine:3.11.3
 
 RUN apk add --update \
     nginx \
@@ -29,7 +29,11 @@ RUN apk add --update \
 
 WORKDIR /html
 
-RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
+RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stdout/g' /etc/php7/php-fpm.conf && \
+    sed -i 's/;daemonize = yes/daemonize = no/g' /etc/php7/php-fpm.conf && \
+    sed -i 's/listen = 127.0.0.1:9000/listen = 9000/g' /etc/php7/php-fpm.d/www.conf && \
+    sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
+    sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
     sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
     sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
     sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
@@ -56,6 +60,7 @@ RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
 
 COPY config/nginx.conf /etc/nginx/nginx.conf
 COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf
+RUN sed -i 's/fastcgi_pass phpfpm:9000;/fastcgi_pass localhost:9000;/g' /etc/nginx/conf.d/default.conf 
 COPY --chown=root:nginx . /html
 
 WORKDIR /html
@@ -64,7 +69,4 @@ EXPOSE 80
 CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
     chmod -R g+w storage/logs/metager bootstrap/cache && \
     crond -L /dev/stdout && \
-    nginx && \
-    php-fpm7 -D && \
-    redis-server /etc/redis.conf && \
-    su -s /bin/sh -c 'php artisan requests:fetcher' nginx
+    php-fpm7

Dockerfile.dev → DockerfileDev

-FROM alpine:latest
+FROM alpine:3.11.3
 
 RUN apk add --update \
     nginx \
@@ -23,12 +23,17 @@ RUN apk add --update \
     php7-gd \
     php7-json \
     php7-pcntl \
+    php7-opcache \
     php7-fileinfo \
     && rm -rf /var/cache/apk/*
 
 WORKDIR /html
 
-RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
+RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stdout/g' /etc/php7/php-fpm.conf && \
+    sed -i 's/;daemonize = yes/daemonize = no/g' /etc/php7/php-fpm.conf && \
+    sed -i 's/listen = 127.0.0.1:9000/listen = 9000/g' /etc/php7/php-fpm.d/www.conf && \
+    sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
+    sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
     sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
     sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
     sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
@@ -38,6 +43,14 @@ RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
     sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
     sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
     sed -i 's/expose_php = On/expose_php = Off/g' /etc/php7/php.ini && \
+    # Opcache configuration
+    sed -i 's/;opcache.enable=1/opcache.enable=1/g' /etc/php7/php.ini && \
+    sed -i 's/;opcache.memory_consumption=128/opcache.memory_consumption=128/g' /etc/php7/php.ini && \
+    sed -i 's/;opcache.interned_strings_buffer=8/opcache.interned_strings_buffer=8/g' /etc/php7/php.ini && \
+    sed -i 's/;opcache.max_accelerated_files=10000/opcache.max_accelerated_files=10000/g' /etc/php7/php.ini && \
+    sed -i 's/;opcache.max_wasted_percentage=5/opcache.max_wasted_percentage=5/g' /etc/php7/php.ini && \
+    sed -i 's/;opcache.validate_timestamps=1/opcache.validate_timestamps=1/g' /etc/php7/php.ini && \
+    sed -i 's/;opcache.revalidate_freq=2/opcache.revalidate_freq=300/g' /etc/php7/php.ini && \
     echo "daemonize yes" >> /etc/redis.conf && \
     ln -s /dev/null /var/log/nginx/access.log && \
     ln -s /dev/stdout /var/log/nginx/error.log && \
@@ -51,7 +64,4 @@ EXPOSE 80
 CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
     chmod -R g+w storage/logs/metager bootstrap/cache && \
     crond -L /dev/stdout && \
-    nginx && \
-    php-fpm7 -D && \
-    redis-server /etc/redis.conf && \
-    su -s /bin/sh -c 'php artisan requests:fetcher' nginx
+    php-fpm7

DockerfileDev.ignore

+README.md
+CHANGELOG.md
+vendor/*
+node_modules/*
+storage/logs/*
\ No newline at end of file

app/Console/Commands/RequestFetcher.php

@@ -50,10 +50,17 @@ class RequestFetcher extends Command
      */
     public function handle()
     {
+        $pidFile = "/tmp/fetcher";
         pcntl_signal(SIGINT, [$this, "sig_handler"]);
         pcntl_signal(SIGTERM, [$this, "sig_handler"]);
         pcntl_signal(SIGHUP, [$this, "sig_handler"]);
 
+        touch($pidFile);
+
+        if (!file_exists($pidFile)) {
+            return;
+        }
+
         try {
             $blocking = false;
             $redis = Redis::connection("cache");
@@ -108,9 +115,8 @@ class RequestFetcher extends Command
                     usleep(50 * 1000);
                 }
             }
-        } catch (\Exception $e) {
-            Log::error($e->getMessage());
         } finally {
+            unlink($pidFile);
             curl_multi_close($this->multicurl);
         }
     }

chart/templates/deployment.yaml

@@ -39,7 +39,8 @@ spec:
         persistentVolumeClaim:
           claimName: mglogs
       containers:
-      - name: {{ .Chart.Name }}
+      # Main PHP-FPM Container
+      - name: {{ .Chart.Name }}-phpfpm
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         {{- if .Values.application.secretName }}
@@ -48,23 +49,42 @@ spec:
             name: {{ .Values.application.secretName }}
         {{- end }}
         env:
-{{- if .Values.postgresql.managed }}
-        - name: POSTGRES_USER
-          valueFrom:
-            secretKeyRef:
-              name: app-postgres
-              key: username
-        - name: POSTGRES_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: app-postgres
-              key: password
-        - name: POSTGRES_HOST
-          valueFrom:
-            secretKeyRef:
-              name: app-postgres
-              key: privateIP
-{{- end }}
+        - name: DATABASE_URL
+          value: {{ .Values.application.database_url | quote }}
+        - name: GITLAB_ENVIRONMENT_NAME
+          value: {{ .Values.gitlab.envName }}
+        - name: GITLAB_ENVIRONMENT_URL
+          value: {{ .Values.gitlab.envURL }}
+        ports:
+        - name: "{{ .Values.service.name }}-phpfpm"
+          containerPort: 9000
+        livenessProbe:
+          tcpSocket:
+            port: "{{ .Values.service.name }}-phpfpm"
+          initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+          timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+        readinessProbe:
+          tcpSocket:
+            port: "{{ .Values.service.name }}-phpfpm"
+          initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+          timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
+        volumeMounts:
+        - name: mglogs-persistent-storage
+          mountPath: /html/storage/logs/metager
+          readOnly: false
+        resources:
+{{ toYaml .Values.resources | indent 12 }}
+      # Nginx Container
+      - name: {{ .Chart.Name }}-nginx
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        command: ["nginx"]
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        {{- if .Values.application.secretName }}
+        envFrom:
+        - secretRef:
+            name: {{ .Values.application.secretName }}
+        {{- end }}
+        env:
         - name: DATABASE_URL
           value: {{ .Values.application.database_url | quote }}
         - name: GITLAB_ENVIRONMENT_NAME
@@ -98,10 +118,41 @@ spec:
 {{- end }}
           initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
           timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
-        volumeMounts:
-        - name: mglogs-persistent-storage
-          mountPath: /html/storage/logs/metager
-          readOnly: false
-        resources:
-{{ toYaml .Values.resources | indent 12 }}
+      # Redis Container
+      - name: {{ .Chart.Name }}-redis
+        image: "redis:6.0-rc1-alpine"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - name: "{{ .Values.service.name }}-redis"
+          containerPort: 6379
+        livenessProbe:
+          tcpSocket:
+            port: "{{ .Values.service.name }}-redis"
+          initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+          timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+        readinessProbe:
+          tcpSocket:
+            port: "{{ .Values.service.name }}-redis"
+          initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+          timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
+      # Fetcher Container
+      - name: {{ .Chart.Name }}-fetcher
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        command: ["su"]
+        args: ["-s", "/bin/sh", "-c", "php artisan requests:fetcher", "nginx"]
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        livenessProbe:
+          exec:
+            command:
+            - cat
+            - /tmp/fetcher
+          initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+          timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+        readinessProbe:
+          exec:
+            command:
+            - cat
+            - /tmp/fetcher
+          initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+          timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
 {{- end -}}

config/nginx-default.conf

@@ -14,7 +14,7 @@ server {
     location ~ \.php$ {
         try_files $uri /index.php =404;
         fastcgi_split_path_info ^(.+\.php)(/.+)$;
-        fastcgi_pass 127.0.0.1:9000;
+        fastcgi_pass phpfpm:9000;
         fastcgi_index index.php;
         fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
         include fastcgi_params;

config/nginx.conf

@@ -5,6 +5,7 @@ worker_processes  100;
 error_log  /var/log/nginx/error.log warn;
 pid        /var/run/nginx.pid;
 
+daemon off;
 
 events {
     worker_connections  1024;

docker-compose.yml

@@ -28,20 +28,43 @@ services:
       - MYSQL_USER=metager
       - MYSQL_PASSWORD=metager
       - MYSQL_DATABASE=metager
-  web:
+  redis:
+    restart: on-failure
+    image: redis:6.0-rc1-alpine
+  phpfpm:
     depends_on:
       - "mgdb"
       - "dependencies"
       - "phpdeps"
       - "assets"
+      - "redis"
     restart: on-failure
     build:
       context: .
       dockerfile: Dockerfile.dev
+    image: metager:latest
+    working_dir: /html
+    volumes:
+      - .:/html
+  nginx:
+    depends_on:
+      - "phpfpm"
+    restart: on-failure
+    image: metager:latest
     working_dir: /html
+    command: nginx
     volumes:
       - .:/html
       - ./config/nginx.conf:/etc/nginx/nginx.conf
       - ./config/nginx-default.conf:/etc/nginx/conf.d/default.conf
     ports: 
     - "8080:80"
+  worker:
+    depends_on:
+      - "phpfpm"
+    restart: on-failure
+    image: metager:latest
+    working_dir: /html
+    volumes:
+      - .:/html
+    command: "su -s /bin/sh -c 'php artisan requests:fetcher' nginx"