Ziel dieses LAB ist es, die Ausfallsicherheit des Nextcloud Systems zu erhöhen, also eine skalierte und somit resistentere Nextcloud Umgebung zu betreiben, die sowohl ein Loadbalancing nutzt, als auch redundante Backendsysteme (MariaDB Master/Slave und Redis-Server Master/Slave) und ein zentrales NFS berücksichtigt.
Aufbau der Umgebung, basierend auf Ubunu 18.04.x LTS
nginx Loadbalancer: Externe/Öffentlich IP und 192.168.2.10
Nextcloud Backend 1: 192.168.2.11
Nextcloud Backend 2: 192.168.2.12
Master MariaDB und Redis-Slave: 192.168.2.20
Redis-Master und Slave MariaDB: 192.168.2.21
NFS Server für Daten: 192.168.2.30
Hinweis: Ich empfehle dringend den SSH Standardport 22 zu ändern – berücksichtigen Sie das bitte in den exemplarisch dargestellten Firewallsettings. Zudem empfehle ich auch den Einsatz von fail2ban und weitere Härtungsmaßnahmen, die nicht Gegenstand dieses LABs sind.
Installationsreihenfolge:
1. MariaDB Master/Slave
Server: MariaDB Master- und Redis-Master
sudo -s
apt-get install software-properties-common -y apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 add-apt-repository 'deb [arch=amd64] http://mirror.lstn.net/mariadb/repo/10.5/ubuntu bionic main' apt update && apt install mariadb-server -y ufw allow 3306/tcp mysql_secure_installation
Server: MariaDB-Master und -Slave
systemctl enable mariadb
service mariadb stop
cd /etc/mysql/
cp my.cnf my.cnf.bak
vi my.cnf
[client] default-character-set = utf8mb4 port = 3306 socket = /var/run/mysqld/mysqld.sock [mysqld_safe] log_error=/var/log/mysql/mysql_error.log nice = 0 socket = /var/run/mysqld/mysqld.sock [mysqld] server-id = 01 replicate-do-db = nextcloud basedir = /usr #bind-address = 127.0.0.1 bind-address = 192.168.2.20 binlog_format = mixed # binlog_format = ROW bulk_insert_buffer_size = 16M character-set-server = utf8mb4 collation-server = utf8mb4_general_ci concurrent_insert = 2 connect_timeout = 5 datadir = /var/lib/mysql default_storage_engine = InnoDB expire_logs_days = 2 general_log_file = /var/log/mysql/mysql.log general_log = 0 innodb_buffer_pool_size = 1024M innodb_buffer_pool_instances = 1 innodb_flush_log_at_trx_commit = 2 innodb_log_buffer_size = 32M innodb_max_dirty_pages_pct = 90 innodb_file_per_table = 1 innodb_open_files = 400 innodb_io_capacity = 4000 innodb_flush_method = O_DIRECT key_buffer_size = 128M lc_messages_dir = /usr/share/mysql lc_messages = en_US log_bin = /var/log/mysql/master-bin # log_bin = /var/log/mysql/mariadb-bin log_bin_index = /var/log/mysql/master-bin.index # log_bin_index = /var/log/mysql/mariadb-bin.index log_error = /var/log/mysql/mysql_error.log log_slow_verbosity = query_plan log_warnings = 2 long_query_time = 1 max_allowed_packet = 16M max_binlog_size = 100M max_connections = 300 max_heap_table_size = 64M myisam_recover_options = BACKUP myisam_sort_buffer_size = 512M port = 3306 pid-file = /var/run/mysqld/mysqld.pid query_cache_limit = 2M query_cache_size = 64M query_cache_type = 1 query_cache_min_res_unit = 2k read_buffer_size = 2M read_rnd_buffer_size = 1M relay_log = /var/log/mysql/master-relay-bin relay_log_index = /var/log/mysql/master-relay-bin.index skip-external-locking skip-name-resolve slow_query_log_file = /var/log/mysql/mariadb-slow.log slow-query-log = 1 socket = /var/run/mysqld/mysqld.sock sort_buffer_size = 4M table_open_cache = 400 thread_cache_size = 128 tmp_table_size = 64M tmpdir = /tmp transaction_isolation = READ-COMMITTED # unix_socket=OFF user = mysql wait_timeout = 600 [mysqldump] max_allowed_packet = 16M quick quote-names [isamchk] key_buffer = 16M
service mariadb restart mysql -e "CREATE USER 'nextcloud'@'localhost' identified by 'nextcloud'; flush privileges;" mysql -e "CREATE USER 'nextcloud'@'192.168.2.21' identified by 'nextcloud'; flush privileges;" mysql -e "CREATE USER 'nextcloud'@'192.168.2.20' identified by 'nextcloud'; flush privileges;" mysql -e "grant replication slave on *.* to 'nextcloud'@'192.168.2.21';" mysql -e "grant ALL PRIVILEGES on nextcloud.* to nextcloud@'192.168.2.20'; FLUSH privileges;" mysql -e "grant ALL PRIVILEGES on nextcloud.* to nextcloud@'192.168.2.21'; FLUSH privileges;" mysql -e "grant ALL PRIVILEGES on nextcloud.* to nextcloud@'localhost'; FLUSH privileges;" mysql -e "SHOW MASTER STATUS\G;"
Server: MariaDB-Slave
systemctl enable mariadb
service mariad stop
cd /etc/mysql/
cp my.cnf my.cnf.bak
vi my.cnf
[client] default-character-set = utf8mb4 port = 3306 socket = /var/run/mysqld/mysqld.sock [mysqld_safe] log_error=/var/log/mysql/mysql_error.log nice = 0 socket = /var/run/mysqld/mysqld.sock [mysqld] server-id = 02 replicate-do-db = nextcloud basedir = /usr #bind-address = 127.0.0.1 bind-address = 192.168.2.21 binlog_format = mixed # binlog_format = ROW bulk_insert_buffer_size = 16M character-set-server = utf8mb4 collation-server = utf8mb4_general_ci concurrent_insert = 2 connect_timeout = 5 datadir = /var/lib/mysql default_storage_engine = InnoDB expire_logs_days = 2 general_log_file = /var/log/mysql/mysql.log general_log = 0 innodb_buffer_pool_size = 1024M innodb_buffer_pool_instances = 1 innodb_flush_log_at_trx_commit = 2 innodb_log_buffer_size = 32M innodb_max_dirty_pages_pct = 90 innodb_file_per_table = 1 innodb_open_files = 400 innodb_io_capacity = 4000 innodb_flush_method = O_DIRECT key_buffer_size = 128M lc_messages_dir = /usr/share/mysql lc_messages = en_US log_bin = /var/log/mysql/slave-bin # log_bin = /var/log/mysql/mariadb-bin log_bin_index = /var/log/mysql/slave-bin.index # log_bin_index = /var/log/mysql/mariadb-bin.index log_error = /var/log/mysql/mysql_error.log log_slow_verbosity = query_plan log_warnings = 2 long_query_time = 1 max_allowed_packet = 16M max_binlog_size = 100M max_connections = 300 max_heap_table_size = 64M myisam_recover_options = BACKUP myisam_sort_buffer_size = 512M port = 3306 pid-file = /var/run/mysqld/mysqld.pid query_cache_limit = 2M query_cache_size = 64M query_cache_type = 1 query_cache_min_res_unit = 2k read_buffer_size = 2M read_rnd_buffer_size = 1M relay_log = /var/log/mysql/slave-relay-bin relay_log_index = /var/log/mysql/slave-relay-bin.index skip-external-locking skip-name-resolve slow_query_log_file = /var/log/mysql/mariadb-slow.log slow-query-log = 1 socket = /var/run/mysqld/mysqld.sock sort_buffer_size = 4M table_open_cache = 400 thread_cache_size = 128 tmp_table_size = 64M tmpdir = /tmp transaction_isolation = READ-COMMITTED # unix_socket=OFF user = mysql wait_timeout = 600 [mysqldump] max_allowed_packet = 16M quick quote-names [isamchk] key_buffer = 16M
service mariadb restart
Server: MariaDB Master
mysql -uroot -p CREATE DATABASE nextcloud CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; flush tables with read lock; show master status;
WICHTIG: diese mysql-Verbindung offen und bestehen lassen und eine weitere SSH Session starten:
mariadb-dump --single-transaction --routines -uroot -p*SECRET* nextcloud > nextcloud-dump.sql
mysql -e "SHOW MASTER STATUS\G;"
root@DBMASTER:~# mysql -e "SHOW MASTER STATUS\G;"
*************************** 1. row ***************************
File: master-bin.000001
Position: 8233
Binlog_Do_DB:
Binlog_Ignore_DB:
root@DBMASTER:~#
Server: MariaDB Slave
mysql -e "CREATE DATABASE nextcloud CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" mysql -uroot -p*SECRET* nextcloud < nextcloud-dump.sql service mariadb restart
mysql -e "change master 'master01' to master_host='192.168.2.20', master_user='nextcloud', master_password='nextcloud', master_port=3306, master_log_file='master-bin.000001, master_log_pos=8233, master_connect_retry=10, master_use_gtid=slave_pos;"
mysql -e "start slave 'master01';" mysql -e "show slave 'master01' status\G;"
Prüfen der Werte „Slave_IO_Running: Yes undSlave_SQL_Running: Yes„
root@DBSLAVE:~# mysql -e "show slave 'master01' status\G;"
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: 192.168.2.11
Master_User: nextcloud
Master_Port: 3306
Connect_Retry: 10
Master_Log_File: master-bin.000001
Read_Master_Log_Pos: 8233
Relay_Log_File: slave-relay-bin-master01.000002
Relay_Log_Pos: 1160
Relay_Master_Log_File: master-bin.000001
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
Replicate_Do_DB: nextcloud
Replicate_Ignore_DB:
[...]
Prüfen des „synchronen“ Repliaktionsstatus:
Server: MariaDB Master
show variables like 'gtid_binlog_pos'; +-----------------+--------+ | Variable_name | Value | +-----------------+--------+ | gtid_binlog_pos | 0-1-17 | +-----------------+--------+
Server: MariaDB Slave
show variables like 'gtid_slave_pos'; +----------------+--------+ | Variable_name | Value | +----------------+--------+ | gtid_slave_pos | 0-1-17 | +----------------+--------+
Die Werte vom Master sollten identisch mit denen des Slaves sein.
Server: MariaDB-Master und -Slave
root@DBMASTER:~# ufw status verbose Status: active Logging: on (medium) Default: deny (incoming), allow (outgoing), disabled (routed) New profiles: skip To Action From -- ------ ---- 22/tcp ALLOW IN Anywhere 3306 ALLOW IN 192.168.2.11 3306 ALLOW IN 192.168.2.12 22/tcp (v6) ALLOW IN Anywhere (v6)
2. Redis Master/Slave
Server: Redis-Master und -Slave
sudo -s apt install redis-server -y redis-cli ping
PONG
Server: Redis-Master
vi /etc/redis/redis.conf
bind 127.0.0.1 192.168.2.21 requirepass nextcloud
systemctl restart redis-server.service
systemctl enable redis-server.service
Server: Redis-Slave
vi /etc/redis/redis.conf
bind 127.0.0.1 192.168.2.20 slaveof 192.168.2.21 6379 requirepass nextcloud masterauth nextcloud
systemctl restart redis-server.service
systemctl enable redis-server.service
Server: Redis-Master
root@DBSLAVE:~# redis-cli 127.0.0.1:6379> auth nextcloud OK 127.0.0.1:6379> set MasterSlave-TEST 'Carsten Rieger IT-Services - Quelle ist der Master' OK 127.0.0.1:6379> quit root@DBSLAVE:~#
Server: Redis-Slave
root@DBMASTER:~# redis-cli 127.0.0.1:6379> auth nextcloud OK 127.0.0.1:6379> get MasterSlave-TEST "Carsten Rieger IT-Services - Quelle ist der Master" 127.0.0.1:6379> quit root@DBMASTER:~#
Server: Redis-Master und -Slave
root@DBMASTER:~# ufw status verbose Status: active Logging: on (medium) Default: deny (incoming), allow (outgoing), disabled (routed) New profiles: skip To Action From -- ------ ---- 22/tcp ALLOW IN Anywhere 3306 ALLOW IN 192.168.2.11 3306 ALLOW IN 192.168.2.12 6379 ALLOW IN 192.168.2.11 6379 ALLOW IN 192.168.2.12 16379 ALLOW IN 192.168.2.11 16379 ALLOW IN 192.168.2.12 22/tcp (v6) ALLOW IN Anywhere (v6)
3. NFS Server
sudo -s
apt install nfs-kernel-server -y
mkdir -p /daten /nextcloud chown -R www-data:www-data /daten /nextcloud chmod -R 777 /daten /nextcloud
vi /etc/exports
/daten 192.168.2.11(rw,sync,anonuid=33,anongid=33,no_subtree_check) /daten 192.168.2.12(rw,sync,anonuid=33,anongid=33,no_subtree_check) /nextcloud 192.168.2.11(rw,sync,anonuid=33,anongid=33,no_subtree_check) /nextcloud 192.168.2.12(rw,sync,anonuid=33,anongid=33,no_subtree_check)
exportfs -a
systemctl enable nfs-kernel-server
systemctl restart nfs-kernel-server
root@NFS:~# ufw status verbose Status: active Logging: on (medium) Default: deny (incoming), allow (outgoing), disabled (routed) New profiles: skip To Action From -- ------ ---- 22/tcp ALLOW IN Anywhere 2049 ALLOW IN 192.168.2.11 2049 ALLOW IN 192.168.2.12 22/tcp (v6) ALLOW IN Anywhere (v6)
4. Nextcloud-Server
Server: Nextcloud Server 1 und 2
sudo -s apt install nfs-common -y mkdir /daten -p chown -R www-data:www-data /daten
vi /etc/fstab
192.168.2.30:/daten /daten nfs auto,nofail,noatime,nolock,intr,tcp,actimeo=1800 0 0 192.168.2.30:/nextcloud /var/www/nextcloud nfs auto,nofail,noatime,nolock,intr,tcp,actimeo=1800 0 0
mount -a
Server: Nextcloud Server 1
vi /etc/nginx/conf.d/nextcloud-server1.conf
server {
server_name 192.168.2.11;
listen 192.168.2.11:80;
root /var/www/nextcloud/;
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
location ^~ / {
client_max_body_size 10G;
location / {
rewrite ^ /index.php;
}
location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ {
deny all;
}
location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) {
deny all;
}
location ~ ^\/(?:index|ipcheck|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) {
fastcgi_split_path_info ^(.+?\.php)(\/.*|)$;
set $path_info $fastcgi_path_info;
try_files $fastcgi_script_name =404;
include fastcgi_params;
include /etc/nginx/php_optimization.conf;
fastcgi_param REMOTE_ADDR $http_x_real_ip;
}
location ~ ^\/(?:updater|ocs-provider|ocm-provider)(?:$|\/) {
try_files $uri/ =404;
index index.php;
}
location ~ ^\/.+[^\/]\.(?:css|js|woff2?|svg|gif)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
location ~ ^\/.+[^\/]\.(?:png|html|ttf|ico|jpg|jpeg)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
}
}
Server: Nextcloud Server 2
vi /etc/nginx/conf.d/nextcloud-server2.conf
server {
server_name 192.168.2.12;
listen 192.168.2.12:80;
root /var/www/nextcloud/;
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
location ^~ / {
client_max_body_size 10G;
location / {
rewrite ^ /index.php;
}
location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ {
deny all;
}
location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) {
deny all;
}
location ~ ^\/(?:index|ipcheck|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) {
fastcgi_split_path_info ^(.+?\.php)(\/.*|)$;
set $path_info $fastcgi_path_info;
try_files $fastcgi_script_name =404;
include fastcgi_params;
include /etc/nginx/php_optimization.conf;
fastcgi_param REMOTE_ADDR $http_x_real_ip;
}
location ~ ^\/(?:updater|ocs-provider|ocm-provider)(?:$|\/) {
try_files $uri/ =404;
index index.php;
}
location ~ ^\/.+[^\/]\.(?:css|js|woff2?|svg|gif)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
location ~ ^\/.+[^\/]\.(?:png|html|ttf|ico|jpg|jpeg)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
}
}
Server: Nextcloud Server 1 und 2
sudo -u www-data vi /var/www/nextcloud/ipcheck.php
<?php header( 'Content-Type: text/plain' ); echo 'NGINX'; echo 'Host: ' . $_SERVER['HTTP_HOST'] . "\n"; echo 'Remote Address: ' . $_SERVER['REMOTE_ADDR'] . "\n"; echo 'X-Forwarded-For: ' . $_SERVER['HTTP_X_FORWARDED_FOR'] . "\n"; echo 'X-Forwarded-Proto: ' . $_SERVER['HTTP_X_FORWARDED_PROTO'] . "\n"; echo 'Server Address: ' . $_SERVER['SERVER_ADDR'] . "\n"; echo 'Server Port: ' . $_SERVER['SERVER_PORT'] . "\n\n"; ?>
root@NEXTCLOUDSERVER#:~# curl https://ihre.domain.de/ipcheck.php
Test im Browser: curl https://ihre.domain.de/ipcheck.php
root@NEXTCLOUDSERVER#:~# ufw status verbose Status: active Logging: on (medium) Default: deny (incoming), allow (outgoing), disabled (routed) New profiles: skip To Action From -- ------ ---- 22/tcp ALLOW IN Anywhere 80/tcp ALLOW IN 192.168.2.10 22/tcp (v6) ALLOW IN Anywhere (v6)
5. nginx Loadbalancer
sudo -s vi /etc/nginx/conf.d/gateway.conf
upstream Nextcloud-Loadbalancer {
hash $remote_addr;
server 192.168.2.11; # <- Nextcloud Server 1
server 192.168.2.12; # <- Nextcloud Server 2
}
proxy_cache_path /tmp/nginx-cache levels=1:2 keys_zone=nextcloud_cache:10m max_size=10g inactive=60m use_temp_path=off;
server {
server_name ihre.domäne.de 192.168.2.10; # External/Public IP und Loadbalancer-IP
listen 80 default_server;
listen [::]:80 default_server;
location ^~ /.well-known/acme-challenge {
proxy_pass http://127.0.0.1:81;
proxy_set_header Host $host;
}
location / {
return 301 https://$server_name$request_uri;
}
}
server {
server_name ihre.domäne.de 192.168.2.10; # External/Public IP und Loadbalancer-IP
listen 443 ssl http2 default_server;
listen [::]:443 ssl http2 default_server;
root /var/www/nextcloud/;
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
include /etc/nginx/proxy.conf;
include /etc/nginx/header.conf;
client_max_body_size 10240M;
location ^~ / {
proxy_cache nextcloud_cache;
proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
proxy_cache_revalidate on;
proxy_cache_background_update on;
client_max_body_size 10G;
proxy_connect_timeout 3600;
proxy_send_timeout 3600;
proxy_read_timeout 3600;
send_timeout 3600;
proxy_buffering off;
proxy_request_buffering off;
proxy_max_temp_file_size 10240m;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://Nextcloud-Loadbalancer;
proxy_redirect off;
}
location = /.well-known/carddav {
return 301 $scheme://$host/remote.php/dav;
}
location = /.well-known/caldav {
return 301 $scheme://$host/remote.php/dav;
}
location = /.well-known/webfinger {
return 301 $scheme://$host/public.php?service=webfinger;
}
location ~ /(ocm-provider|ocs-provider)/ {
return 301 $scheme://$host/$1/;
}
}
root@LOADBALANCER:~# ufw status verbose Status: active Logging: on (medium) Default: deny (incoming), allow (outgoing), disabled (routed) New profiles: skip To Action From -- ------ ---- 22/tcp ALLOW IN Anywhere 80/tcp ALLOW IN Anywhere 443/tcp ALLOW IN Anywhere 22/tcp (v6) ALLOW IN Anywhere (v6) 80/tcp (v6) ALLOW IN Anywhere (v6) 443/tcp (v6) ALLOW IN Anywhere (v6)
