k8s►k8s registry install

k8s registry install

  1. 安装nginx

    1
    2
    3
    4
    5
    6
    7
    8
    9
    root@192:~# apt-get install gcc make libpcre3 libpcre3-dev openssl libssl-dev
    root@192:~# apt-get install libauthen-htpasswd-perl libapache-htpasswd-perl apache2-utils
    root@192:~# useradd -s /usr/sbin/nologin nginx
    root@192:~# wget http://nginx.org/download/nginx-1.10.2.tar.gz
    root@192:~# tar xzf nginx-1.10.2.tar.gz
    root@192:~# cd nginx-1.10.2/
    root@192:~/nginx-1.10.2# ./configure --prefix=/opt/nginx --user=nginx --group=nginx --with-pcre --with-http_stub_status_module --with-http_ssl_module --with-http_addition_module --with-http_realip_module --with-http_flv_module
    root@192:~/nginx-1.10.2# make
    root@192:~/nginx-1.10.2# make install

    可以设置认证,但实际部署时不建议设置账号和密码,请看nginx配置文件

    1
    root@192:~# htpasswd -cb /opt/nginx/conf/.htpasswd admin admin

  2. 创建证书

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    root@192:~# mkdir -p /opt/nginx/ssl
    root@192:~# cd /opt/nginx/ssl/
    root@192:/opt/nginx/ssl# openssl req -newkey rsa:4096 -nodes -sha256 -keyout domain.key -x509 -days 365 -out domain.crt
    Generating a 4096 bit RSA private key
    ........++
    .........................................................................................................................................................................................................................................++
    writing new private key to \'domain.key\'
    \-----
    You are about to be asked to enter information that will be incorporated
    into your certificate request.
    What you are about to enter is what is called a Distinguished Name or a DN.
    There are quite a few fields but you can leave some blank
    For some fields there will be a default value,
    If you enter '.', the field will be left blank.
    \-----
    Country Name (2 letter code) [AU]:CN
    State or Province Name (full name) [Some-State]:BJ
    Locality Name (eg, city) []:bj
    Organization Name (eg, company) [Internet Widgits Pty Ltd]:zhonghui
    Organizational Unit Name (eg, section) []:
    Common Name (e.g. server FQDN or YOUR name) []:d.hub.io
    Email Address []:pengliu@brandwisdom.cn

  3. 拷贝CA证书到docker客户端

    1
    2
    root@192:~# mkdir -p /etc/docker/certs.d/d.hub.io
    root@192:~# cp /opt/nginx/ssl/domain.crt /etc/docker/certs.d/d.hub.io/ca.crt

    注意:将ca.crt 拷贝到所有的k8s node节点上:

    1
    2
    root@192:~# mkdir -p /etc/docker/certs.d/d.hub.io
    root@192:~# cp /opt/nginx/ssl/domain.crt /etc/docker/certs.d/d.hub.io/ca.crt

  4. 启动docker服务

    1
    2
    root@192:~# source /var/run/flannel/subnet.env
    root@192:~# /usr/bin/dockerd --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU} &

  5. 启动docker registry容器

    1
    2
    3
    4
    root@192:~# docker pull index.alauda.cn/library/registry
    root@192:~# docker tag index.alauda.cn/library/registry:latest registry:latest
    root@192:~# mkdir -p /mnt/data/registry
    root@192:~# docker run -d -p 5000:5000 --restart=always --name registry -v /mnt/data/registry/:/var/lib/registry/ registry:latest

  6. 修改nginx配置文件

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    root@192:~# cat /opt/nginx/conf/nginx.conf
    user  nginx nginx;
    worker_processes  auto;

    error_log  logs/error.log;
    pid        logs/nginx.pid;
    worker_rlimit_nofile 51200;

    events {
        use epoll;
        worker_connections  51200;
        multi_accept on;
    }

    http {
        include       mime.types;
        default_type  application/octet-stream;

        log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                          '$status $body_bytes_sent "$http_referer" '
                          '"$http_user_agent" "$http_x_forwarded_for"';
        access_log  logs/access.log  main;

        server_names_hash_bucket_size 128;
        client_header_buffer_size 32k;
        large_client_header_buffers 4 32k;

        sendfile       on;
        tcp_nopush     on;
        tcp_nodelay    on;
        keepalive_timeout  65;

        upstream registry {
          server 192.168.15.114:5000;
        }

        server {
            listen       443;
            server_name  d.hub.io;

            ssl    on;
            ssl_certificate /opt/nginx/ssl/domain.crt;
            ssl_certificate_key /opt/nginx/ssl/domain.key;

            client_max_body_size 0;
            chunked_transfer_encoding on;
            location /v2/ {
              #auth_basic "Registry realm";
              #auth_basic_user_file /opt/nginx/conf/.htpasswd;
              add_header 'Docker-Distribution-Api-Version' 'registry/2.0' always;

              proxy_pass http://registry;
              proxy_set_header Host               $http_host;
              proxy_set_header X-Real-IP          $remote_addr;
              proxy_set_header X-Forwarded-For    $proxy_add_x_forwarded_for;
              proxy_set_header X-Forwarded-Proto  $scheme;
              proxy_read_timeout                  900;
            }

            error_page   500 502 503 504  /50x.html;
            location = /50x.html {
                root   html;
            }
        }
    }

  7. 启动nginx

    1
    2
    root@192:~# /opt/nginx/sbin/nginx -t -c /opt/nginx/conf/nginx.conf
    root@192:~# /opt/nginx/sbin/nginx -c /opt/nginx/conf/nginx.conf &

  8. 将域名加入hosts文件

    1
    root@192:/etc/docker/ssl# echo "192.168.15.114 d.hub.io" | tee -a  /etc/hosts

  9. 验证

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    root@192:~# curl -i -k -v https://d.hub.io/v2/
    *   Trying 192.168.15.114...
    * Connected to d.hub.io (192.168.15.114) port 443 (#0)
    * found 173 certificates in /etc/ssl/certs/ca-certificates.crt
    * found 692 certificates in /etc/ssl/certs
    * ALPN, offering http/1.1
    * SSL connection using TLS1.2 / ECDHE_RSA_AES_128_GCM_SHA256
    *        server certificate verification SKIPPED
    *        server certificate status verification SKIPPED
    *        common name: d.hub.io (matched)
    *        server certificate expiration date OK
    *        server certificate activation date OK
    *        certificate public key: RSA
    *        certificate version: #3
    *        subject: C=CN,ST=BJ,L=bj,O=zhonghui,CN=d.hub.io,EMAIL=pengliu@brandwisdom.cn
    *        start date: Mon, 19 Dec 2016 02:42:31 GMT
    *        expire date: Tue, 19 Dec 2017 02:42:31 GMT
    *        issuer: C=CN,ST=BJ,L=bj,O=zhonghui,CN=d.hub.io,EMAIL=pengliu@brandwisdom.cn
    *        compression: NULL
    * ALPN, server accepted to use http/1.1
    * Server auth using Basic with user \'admin\'
    > GET /v2/ HTTP/1.1
    > Host: d.hub.io
    > Authorization: Basic YWRtaW46YWRtaW4=
    > User-Agent: curl/7.47.0
    > Accept: */*
    >
    < HTTP/1.1 200 OK
    HTTP/1.1 200 OK
    < Server: nginx/1.10.2
    Server: nginx/1.10.2
    < Date: Mon, 19 Dec 2016 02:57:43 GMT
    Date: Mon, 19 Dec 2016 02:57:43 GMT
    < Content-Type: application/json; charset=utf-8
    Content-Type: application/json; charset=utf-8
    < Content-Length: 2
    Content-Length: 2
    < Connection: keep-alive
    Connection: keep-alive
    < Docker-Distribution-Api-Version: registry/2.0
    Docker-Distribution-Api-Version: registry/2.0
    < X-Content-Type-Options: nosniff
    X-Content-Type-Options: nosniff
    < Docker-Distribution-Api-Version: registry/2.0
    Docker-Distribution-Api-Version: registry/2.0
    <
    * Connection #0 to host d.hub.io left intact

  10. 提交下载

    1
    2
    3
    root@192:~# docker tag 192.168.15.114:5000/node-web:latest d.hub.io/node-web:latest
    root@192:~# docker push d.hub.io/node-web:latest
    root@192:~# docker pull d.hub.io/node-web:latest

  11. 安装完docker 私有仓库后需要重启kubernetes minion上的dockerd服务

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    root@192:~# /usr/bin/dockerd --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU} --insecure-registry=d.hub.io &
    root@192:~# docker tag ubuntu:latest 192.168.15.114:5000/ubuntu:latest
    root@192:~# docker images
    REPOSITORY                   TAG                 IMAGE ID            CREATED             SIZE
    192.168.15.114:5000/ubuntu   latest              4ca3a192ff2a        10 days ago         128.2 MB
    ubuntu                       latest              4ca3a192ff2a        10 days ago         128.2 MB
    registry                     latest              c9bd19d022f6        7 weeks ago         33.3 MB
    root@192:~# docker push 192.168.15.114:5000/ubuntu:latest
    0d45be5b95d8: Pushed
    18568efa7ad4: Pushed
    1c53295311c1: Pushed
    dfcc17ddae9e: Pushed
    d29d52f94ad5: Pushed
    latest: digest: sha256:3b64c309deae7ab0f7dbdd42b6b326261ccd6261da5d88396439353162703fb5 size: 1357