docker-compose 搭建多主机分布式minio

一.minio介绍

Minio 是个基于 Golang 编写的开源对象存储套件，虽然轻量，却拥有着不错的性能。

官网地址：MinIO | High Performance, Kubernetes Native Object Storage

何为对象存储？我们来看下阿里云 OSS （Object Storage Service）的介绍：

对象存储服务（Object Storage Service，OSS）是一种海量、安全、低成本、高可靠的云存储服务，适合存放任意类型的文件。容量和处理能力弹性扩展，多种存储类型供选择，全面优化存储成本。对于中小型企业，如果不选择存储上云，那么 Minio 是个不错的选择，麻雀虽小，五脏俱全。当然 Minio 除了直接作为对象存储使用，还可以作为云上对象存储服务的网关层，无缝对接到 Amazon S3、MicroSoft Azure。

二.准备工作

1.在所有机器上配置主机名解析

cat >> /etc/hosts << EOF
192.168.188.40 minio1
192.168.188.41 minio2
EOF

2.配置时间同步、关闭防火墙和selinux

3.确保主机都安装了docker 以及docker compose

docker-compose.yml

version: '3.7'

# Settings and configurations that are common for all containers
# minio节点之间默认使用9000来连通，所以容器把9000暴露出来，9001是console端口，每个节点设置两块磁盘
x-minio-common: &minio-common
  image: minio/minio:latest
  command: server --console-address ":9001" http://minio{1...2}/data{1...2}
  expose:
    - "9000"
    - "9001"
# 增加host映射，以便两个节点之间通过域名连通
  extra_hosts:
    minio1: 192.168.188.40
    minio2: 192.168.188.41
  depends_on:
    - nginx
  environment:
    MINIO_ROOT_USER: minio
    MINIO_ROOT_PASSWORD: minio123
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
    interval: 30s
    timeout: 20s
    retries: 5

# starts 3 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
# 数据盘挂载目录，按需修改
services:

  minio1:
    <<: *minio-common
    container_name: minio1
    hostname: minio1
    volumes:
      - /bigdata/minio/data1-1:/data1      
      - /bigdata/minio/data1-2:/data2
    ports:
      - "9000:9000"
      - "9001:9001"

  minio2:
    <<: *minio-common
    container_name: minio2
    hostname: minio2
    volumes:
      - /media/minio/data2-1:/data1
      - /media/minio/data2-2:/data2
    ports:
      - "9000:9000"
      - "9001:9001"
# 两个节点都安装nginx，并且负载到两个节点，nginx暴露端口按需修改19000/19001
  nginx:
    image: nginx:1.19.2-alpine
    container_name: nginx-minio
    hostname: nginx
    volumes:
      - /data/nginx-minio.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "19000:9000"
      - "19001:9001"
    extra_hosts:
      minio1: 192.168.188.40
      minio2: 192.168.188.41

nginx-minio.conf

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;

    # include /etc/nginx/conf.d/*.conf;

    upstream minio {
        server minio1:9000;
        server minio2:9000;
    }

    upstream console {
        ip_hash;
        server minio1:9001;
        server minio2:9001;
    }

    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;

            proxy_pass http://minio;
        }
    }

    server {
        listen       9001;
        listen  [::]:9001;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;

            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;

            proxy_connect_timeout 300;

            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";

            chunked_transfer_encoding off;

            proxy_pass http://console;
        }
    }
}

4.将以上两个文件放在每台机器的同一目录下，去分别启动

#拉镜像
./docker compose  pull

#后台启动容器，三台机器分别对应各自节点 mini01 /minio2 (切勿单台启动多个)

在188-40上执行
./docker compose up -d   minio1  

在188-41上执行
./docker compose up -d   minio2

5.验证，登录 http://192.168.188.40:19001

账号密码 minio / minio123

参考链接：

https://www.jianshu.com/p/4428713e22f4

一.minio介绍#

二.准备工作#

一.minio介绍

二.准备工作