Commit 9252b311 authored by Aria Givi's avatar Aria Givi
Browse files

merge development

parents d8fd4693 2e4b5308
......@@ -4,10 +4,22 @@ APP_LOG_LEVEL=debug
APP_KEY=
APP_URL=http://localhost
BOT_PROTECTION=true
DB_CONNECTION=mysql
DB_HOST=mgdb
DB_PORT=3306
DB_DATABASE=metager
DB_USERNAME=metager
DB_PASSWORD="metager"
REDIS_RESULT_CONNECTION=default
REDIS_RESULT_CACHE_DURATION=60
BROADCAST_DRIVER=log
CACHE_DRIVER=file
CACHE_DRIVER=redis
SESSION_DRIVER=file
QUEUE_DRIVER=sync
QUEUE_CONNECTION=sync
REDIS_HOST=127.0.0.1
REDIS_PASSWORD=null
......
update(144.76.113.134):
tags:
- 144.76.113.134
variables:
DOCKER_HOST: "tcp://docker-dind.gitlab:2375"
POSTGRES_ENABLED: "false"
CODE_QUALITY_DISABLED: "true"
CONTAINER_SCANNING_DISABLED: "true"
DAST_DISABLED: "true"
DEPENDENCY_SCANNING_DISABLED: "true"
LICENSE_MANAGEMENT_DISABLED: "true"
PERFORMANCE_DISABLED: "true"
SAST_DISABLED: "true"
TEST_DISABLED: "true"
include:
- template: Jobs/Build.gitlab-ci.yml
- template: Jobs/Deploy.gitlab-ci.yml
stages:
- prepare
- build
- deploy # dummy stage to follow the template guidelines
- review
- dast
- staging
- canary
- development
- production
- incremental rollout 10%
- incremental rollout 25%
- incremental rollout 50%
- incremental rollout 100%
- performance
- cleanup
build:
services:
# Prepares the secret files that we cannot or don't want to share with public
prepare_secrets:
stage: prepare
image: alpine:latest
script:
- cp $ENVFILE .env
- cp $SUMAS config/sumas.json
- cp $SUMASEN config/sumasEn.json
- cp $BLACKLISTURL config/blacklistUrl.txt
- cp $BLACKLISTDOMAINS config/blacklistDomains.txt
- cp $ADBLACKLISTURL config/adBlacklistUrl.txt
- cp $ADBLACKLISTDOMAINS config/adBlacklistDomains.txt
- cp $SPAM config/spam.txt
- cp $USERSSEEDER database/seeds/UsersSeeder.php
- cp database/useragents.sqlite.example database/useragents.sqlite
- sed -i 's/^REDIS_PASSWORD=.*/REDIS_PASSWORD=null/g' .env
artifacts:
paths:
- .env
- config/sumas.json
- config/sumasEn.json
- config/blacklistUrl.txt
- config/blacklistDomains.txt
- config/adBlacklistUrl.txt
- config/adBlacklistDomains.txt
- config/spam.txt
- database/seeds/UsersSeeder.php
- database/useragents.sqlite
only:
- master@open-source/MetaGer
- branches
- tags
prepare_node:
stage: prepare
image: node:10
before_script:
# Abhängigkeiten überprüfen
- which composer
- which git
- which php
- which sqlite3
- npm install
script:
- sh build.sh
variables:
STAGE: production
update(metager2):
tags:
- metager2
- npm run prod
artifacts:
paths:
- public/js/
- public/css/
- public/mix-manifest.json
cache:
# Cache per Branch
key: "node-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
paths:
- node_modules
only:
- master@open-source/MetaGer
before_script:
# Abhängigkeiten überprüfen
- which composer
- which git
- which php
- which sqlite3
- branches
- tags
prepare_composer:
stage: prepare
image: prooph/composer:7.3
script:
- sh build.sh
- composer install
artifacts:
paths:
- vendor
cache:
key: "composer-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
paths:
- vendor
review:
variables:
STAGE: production
update(metager3.de):
tags:
- metager3
only:
- development@open-source/MetaGer
before_script:
# Abhängigkeiten überprüfen
- which composer
- which git
- which php
- which sqlite3
HELM_UPGRADE_VALUES_FILE: .gitlab/review-apps-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment
except:
refs:
- master
- development
variables:
- $REVIEW_DISABLED
stop_review:
except:
refs:
- master
- development
variables:
- $REVIEW_DISABLED
.development: &development_template
extends: .auto-deploy
stage: development
script:
- sh build.sh
- auto-deploy check_kube_domain
- auto-deploy download_chart
- auto-deploy ensure_namespace
- auto-deploy initialize_tiller
- auto-deploy create_secret
- auto-deploy deploy
- auto-deploy delete canary
- auto-deploy delete rollout
- auto-deploy persist_environment_url
variables:
ADDITIONAL_HOSTS: "www.metager3.de"
HELM_UPGRADE_VALUES_FILE: .gitlab/development-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment
environment:
name: development
url: https://metager3.de
artifacts:
paths: [environment_url.txt]
development:
<<: *development_template
only:
refs:
- development
kubernetes: active
except:
variables:
- $STAGING_ENABLED
- $CANARY_ENABLED
- $INCREMENTAL_ROLLOUT_ENABLED
- $INCREMENTAL_ROLLOUT_MODE
production:
variables:
STAGE: development
\ No newline at end of file
ADDITIONAL_HOSTS: "www.metager.de,metager.org,www.metager.org,metager.es,www.metager.es,klassik.metager.org"
HELM_UPGRADE_VALUES_FILE: .gitlab/production-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment
environment:
url: https://metager.de
\ No newline at end of file
service:
externalPort: 80
internalPort: 80
hpa:
minReplicas: 1
maxReplicas: 5
ingress:
annotations:
certmanager.k8s.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/configuration-snippet: |
if ($host = "www.metager3.de") {
return 301 https://metager3.de$request_uri;
}
\ No newline at end of file
service:
externalPort: 80
internalPort: 80
hpa:
minReplicas: 5
maxReplicas: 25
ingress:
annotations:
certmanager.k8s.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
if ($host = "www.metager.de") {
return 301 https://metager.de$request_uri;
}
if ($host = "www.metager.org") {
return 301 https://metager.org$request_uri;
}
if ($host = "www.metager.es") {
return 301 https://metager.es$request_uri;
}
if ($host = "klassik.metager.org") {
return 301 https://metager.de$request_uri;
}
\ No newline at end of file
---
hpa:
enabled: false
ingress:
annotations:
kubernetes.io/tls-acme: "false"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
tls:
enabled: false
service:
commonName: ""
externalPort: 80
internalPort: 80
FROM debian:buster
FROM alpine:latest
RUN apt-get update && apt-get install -y \
composer \
php7.2 \
php-mbstring \
php7.2-xml\
php-zip \
php-gd \
php-sqlite3 \
php-mysql \
php-curl \
redis-server \
sqlite3 \
nodejs \
libpng-dev \
unzip \
npm
RUN npm install gulp -g
RUN apk add --update \
nginx \
tzdata \
ca-certificates \
dcron \
zip \
redis \
php7 \
php7-fpm \
php7-common \
php7-curl \
php7-mbstring \
php7-sqlite3 \
php7-pdo_mysql \
php7-pdo_sqlite \
php7-dom \
php7-simplexml \
php7-tokenizer \
php7-zip \
php7-redis \
php7-gd \
php7-json \
php7-pcntl \
php7-opcache \
php7-fileinfo \
&& rm -rf /var/cache/apk/*
COPY . /app
WORKDIR app
RUN mv config/sumas.xml.example config/sumas.xml && mv .env.example .env
RUN composer install --no-plugins --no-scripts
RUN npm install
RUN npm run dev
WORKDIR /html
RUN php artisan key:generate
RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
sed -i 's/expose_php = On/expose_php = Off/g' /etc/php7/php.ini && \
# Opcache configuration
sed -i 's/;opcache.enable=1/opcache.enable=1/g' /etc/php7/php.ini && \
sed -i 's/;opcache.memory_consumption=128/opcache.memory_consumption=128/g' /etc/php7/php.ini && \
sed -i 's/;opcache.interned_strings_buffer=8/opcache.interned_strings_buffer=8/g' /etc/php7/php.ini && \
sed -i 's/;opcache.max_accelerated_files=10000/opcache.max_accelerated_files=10000/g' /etc/php7/php.ini && \
sed -i 's/;opcache.max_wasted_percentage=5/opcache.max_wasted_percentage=5/g' /etc/php7/php.ini && \
sed -i 's/;opcache.validate_timestamps=1/opcache.validate_timestamps=1/g' /etc/php7/php.ini && \
sed -i 's/;opcache.revalidate_freq=2/opcache.revalidate_freq=300/g' /etc/php7/php.ini && \
echo "daemonize yes" >> /etc/redis.conf && \
ln -s /dev/null /var/log/nginx/access.log && \
ln -s /dev/stdout /var/log/nginx/error.log && \
cp /usr/share/zoneinfo/Europe/Berlin /etc/localtime && \
echo "Europe/Berlin" > /etc/timezone && \
(crontab -l ; echo "* * * * * php /html/artisan schedule:run >> /dev/null 2>&1") | crontab -
CMD redis-server --daemonize yes && php artisan serve --host=0.0.0.0
COPY config/nginx.conf /etc/nginx/nginx.conf
COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf
COPY --chown=root:nginx . /html
EXPOSE 8000
WORKDIR /html
EXPOSE 80
CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
chmod -R g+w storage/logs/metager bootstrap/cache && \
crond -L /dev/stdout && \
nginx && \
php-fpm7 -D && \
redis-server /etc/redis.conf && \
su -s /bin/sh -c 'php artisan requests:fetcher' nginx
FROM alpine:latest
RUN apk add --update \
nginx \
tzdata \
ca-certificates \
dcron \
zip \
redis \
php7 \
php7-fpm \
php7-common \
php7-curl \
php7-mbstring \
php7-sqlite3 \
php7-pdo_mysql \
php7-pdo_sqlite \
php7-dom \
php7-simplexml \
php7-tokenizer \
php7-zip \
php7-redis \
php7-gd \
php7-json \
php7-pcntl \
php7-fileinfo \
&& rm -rf /var/cache/apk/*
WORKDIR /html
RUN sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
sed -i 's/expose_php = On/expose_php = Off/g' /etc/php7/php.ini && \
echo "daemonize yes" >> /etc/redis.conf && \
ln -s /dev/null /var/log/nginx/access.log && \
ln -s /dev/stdout /var/log/nginx/error.log && \
cp /usr/share/zoneinfo/Europe/Berlin /etc/localtime && \
echo "Europe/Berlin" > /etc/timezone && \
(crontab -l ; echo "* * * * * php /html/artisan schedule:run >> /dev/null 2>&1") | crontab -
WORKDIR /html
EXPOSE 80
CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
chmod -R g+w storage/logs/metager bootstrap/cache && \
crond -L /dev/stdout && \
nginx && \
php-fpm7 -D && \
redis-server /etc/redis.conf && \
su -s /bin/sh -c 'php artisan requests:fetcher' nginx
<?php
namespace App\Console\Commands;
use Cache;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
use Log;
class RequestFetcher extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'requests:fetcher';
/**
* The console command description.
*
* @var string
*/
protected $description = 'This commands fetches requests to the installed search engines';
protected $shouldRun = true;
protected $multicurl = null;
protected $proxyhost, $proxyuser, $proxypassword;
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
$this->multicurl = curl_multi_init();
$this->proxyhost = env("PROXY_HOST", "");
$this->proxyport = env("PROXY_PORT", "");
$this->proxyuser = env("PROXY_USER", "");
$this->proxypassword = env("PROXY_PASSWORD", "");
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
try {
$blocking = false;
$redis = Redis::connection("cache");
while ($this->shouldRun) {
$status = curl_multi_exec($this->multicurl, $active);
$currentJob = null;
if (!$blocking) {
$currentJob = $redis->lpop(\App\MetaGer::FETCHQUEUE_KEY);
} else {
$currentJob = $redis->blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($currentJob)) {
$currentJob = $currentJob[1];
}
}
if (!empty($currentJob)) {
$currentJob = json_decode($currentJob, true);
$ch = $this->getCurlHandle($currentJob);
curl_multi_add_handle($this->multicurl, $ch);
$blocking = false;
$active = true;
}
$answerRead = false;
while (($info = curl_multi_info_read($this->multicurl)) !== false) {
$answerRead = true;
$infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
$infos = explode(";", $infos);
$resulthash = $infos[0];
$cacheDurationMinutes = intval($infos[1]);
$responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
$body = "";
$error = curl_error($info["handle"]);
if (!empty($error)) {
Log::error($error);
}
if ($responseCode !== 200) {
Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
} else {
$body = \curl_multi_getcontent($info["handle"]);
}
Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
\curl_multi_remove_handle($this->multicurl, $info["handle"]);
}
if (!$active && !$answerRead) {
$blocking = true;
} else {
usleep(50 * 1000);
}
}
} catch (\Exception $e) {
Log::error($e->getMessage());
} finally {
curl_multi_close($this->multicurl);
}
}
private function getCurlHandle($job)
{
$ch = curl_init();
curl_setopt_array($ch, array(
CURLOPT_URL => $job["url"],
CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 500,
CURLOPT_LOW_SPEED_TIME => 5,
CURLOPT_TIMEOUT => 10,
));
if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
}
if (!empty($job["username"]) && !empty($job["password"])) {
curl_setopt($ch, CURLOPT_USERPWD, $job["username"] . ":" . $job["password"]);
}
if (!empty($job["headers"]) && sizeof($job["headers"]) > 0) {
$headers = [];
foreach ($job["headers"] as $key => $value) {
$headers[] = $key . ":" . $value;
}
# Headers are in the Form:
# <key>:<value>;<key>:<value>
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
}
return $ch;
}
public function sig_handler($sig)
{
$this->shouldRun = false;
echo ("Terminating Process\n");
}