Kubernetes集群部署1
1.规划
192.168.100.102------>Master[kube-apiserver、kube-controller-manager、kube-scheduler]
Node[kubelet、kube-proxy]
192.168.100.103------>Node1[kubelet、kube-proxy]
192.168.100.104------>Node2[kubelet、kube-proxy]
注:这里的集群我们使用 https 来部署。
2.配置互信
|
1
2
3
4
5
6
7
|
# vim /etc/hosts
192.168.100.102 Master
192.168.100.103 Node1
192.168.100.104 Node2
# ssh-keygen -t rsa -P ''
# ssh-copy-id -i .ssh/id_rsa.pub root@192.168.100.103
# ssh-copy-id -i .ssh/id_rsa.pub root@192.168.100.104
|
3.安装Ansible
|
1
2
3
4
5
6
|
# yum -y install ansible
# cat /etc/ansible/hosts | grep -v ^# | grep -v ^$
[node]
192.168.100.103
192.168.100.104
# ansible node -m copy -a 'src=/etc/hosts dest=/etc/'
|
4.关闭 SELinux 和 Firewall
|
1
2
3
4
5
6
|
# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
# ansible node -m copy -a 'src=/etc/selinux/config dest=/etc/selinux/'
# systemctl stop firewalld
# systemctl disable firewalld
# ansible node -a 'systemctl stop firewalld'
# ansible node -a 'systemctl disable firewalld'
|
5.安装 docker
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
|
# yum install -y yum-utils device-mapper-persistent-data lvm2
# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# yum list docker-ce --showduplicates | sort -r
# yum -y install docker-ce
# docker --version
Docker version 17.06.2-ce, build cec0b72
# systemctl start docker
# systemctl status docker
# systemctl enable docker
# ansible node -m yum -a "state=present name=yum-utils"
# ansible node -m yum -a "state=present name=device-mapper-persistent-data"
# ansible node -m yum -a "state=present name=lvm2"
# ansible node -m copy -a 'src=/etc/yum.repos.d/docker-ce.repo dest=/etc/yum.repos.d/'
# ansible node -m yum -a "state=present name=docker-ce"
# ansible node -a 'systemctl start docker'
# ansible node -a 'systemctl status docker'
# ansible node -a 'systemctl enable docker'
# ansible node -a 'docker --version'
192.168.100.104 | SUCCESS | rc=0 >>
Docker version 17.06.2-ce, build cec0b72
192.168.100.103 | SUCCESS | rc=0 >>
Docker version 17.06.2-ce, build cec0b72
|
6.安装开源PKI工具箱----CFSSL
|
1
2
3
4
5
6
7
8
9
10
11
|
# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
# chmod u+x cfssl*
# mv cfssl_linux-amd64 /usr/local/bin/cfssl
# mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
# mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
# cfssl version
Version: 1.2.0
Revision: dev
Runtime: go1.6
|
7.安装 Etcd 键值存储系统
|
1
2
3
4
5
6
7
8
9
10
11
12
13
|
# curl -L https://storage.googleapis.com/etcd/v3.2.9/etcd-v3.2.9-linux-amd64.tar.gz -o /root/etcd-v3.2.9-linux-amd64.tar.gz
# tar -zxvf etcd-v3.2.9-linux-amd64.tar.gz
# cp etcd-v3.2.9-linux-amd64/etcd* /usr/bin/
# etcd --version
etcd Version: 3.2.9
Git SHA: f1d7dd8
Go Version: go1.8.4
Go OS
/Arch
: linux
/amd64
# etcdctl --version
etcdctl version: 3.2.9
API version: 2
# ansible node -m copy -a 'src=etcd-v3.2.9-linux-amd64/etcd dest=/usr/bin/ mode=755'
# ansible node -m copy -a 'src=etcd-v3.2.9-linux-amd64/etcdctl dest=/usr/bin/ mode=755'
|
8.安装 Kubernetes 容器集群管理系统
|
1
2
3
4
5
6
7
8
|
# wget https://storage.googleapis.com/kubernetes-release/release/v1.8.2/kubernetes-server-linux-amd64.tar.gz
# tar -zxvf kubernetes-server-linux-amd64.tar.gz
# cd kubernetes/server/bin/
# cp kubectl kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy /usr/bin/
# kube-apiserver --version
Kubernetes v1.8.2
# ansible node -m copy -a 'src=kubelet dest=/usr/bin/ mode=755'
# ansible node -m copy -a 'src=kube-proxy dest=/usr/bin/ mode=755'
|
9.安装 flanneld[为容器提供网络服务]
|
1
2
3
4
5
6
7
8
9
|
# curl -L https://github.com/coreos/flannel/releases/download/v0.9.0/flannel-v0.9.0-linux-amd64.tar.gz -o flannel-v0.9.0-linux-amd64.tar.gz
# mkdir flannel
# tar -zxvf flannel-v0.9.0-linux-amd64.tar.gz -C flannel
# cp flannel/flanneld /usr/bin
# mkdir /usr/libexec/flannel/ && cp flannel/mk-docker-opts.sh /usr/libexec/flannel/
# ansible node -m copy -a 'src=flannel/flanneld dest=/usr/bin/ mode=755'
# ansible node -m copy -a 'src=flannel/mk-docker-opts.sh dest=/usr/libexec/flannel/ mode=755'
# flanneld --version
v0.9.0
|
10.创建 SSL 证书
A. 创建 CA(Certificate Authority)
a. 创建配置文件(注:这里证书签名为10年)
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
|
# mkdir ssl && cd ssl
# cat << EOF > ca_config.json
{
"signing"
: {
"default"
: {
"expiry"
:
"87600h"
},
"profiles"
: {
"kubernetes"
: {
"usages"
: [
"signing"
,
"key encipherment"
,
"server auth"
,
"client auth"
],
"expiry"
:
"87600h"
}
}
}
}
EOF
|
b. 创建 CA 证书签名请求
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
|
# cat << EOF > ca_csr.json
{
"CN"
:
"kubernetes"
,
"key"
: {
"algo"
:
"rsa"
,
"size"
: 2048
},
"names"
: [
{
"C"
:
"CN"
,
"ST"
:
"BeiJing"
,
"L"
:
"BeiJing"
,
"O"
:
"k8s"
,
"OU"
:
"System"
}
]
}
EOF
|
c. 生成 CA 证书和私钥
|
1
2
3
4
5
6
7
|
# cfssl gencert -initca ca_csr.json | cfssljson -bare ca
[INFO] generating a new CA key and certificate from CSR
[INFO] generate received request
[INFO] received CSR
[INFO] generating key: rsa-2048
[INFO] encoded CSR
[INFO] signed certificate with serial number 364190696737289470871577587903292790301152267546
|
B. 创建 Kubernetes 证书
a.创建 Kubernetes 证书签名请求
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
# cat << EOF > kubernetes_csr.json
{
"CN"
:
"kubernetes"
,
"hosts"
: [
"127.0.0.1"
,
"localhost"
,
"10.254.0.1"
,
"192.168.100.102"
,
"192.168.100.103"
,
"192.168.100.104"
,
"kubernetes"
,
"kubernetes.default"
,
"kubernetes.default.svc"
,
"kubernetes.default.svc.cluster"
,
"kubernetes.default.svc.cluster.local"
],
"key"
: {
"algo"
:
"rsa"
,
"size"
: 2048
},
"names"
: [
{
"C"
:
"CN"
,
"ST"
:
"BeiJing"
,
"L"
:
"BeiJing"
,
"O"
:
"k8s"
,
"OU"
:
"System"
}
]
}
EOF
|
b.生成 Kubernetes 证书和私钥
|
1
2
3
4
5
6
|
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca_config.json -profile=kubernetes kubernetes_csr.json | cfssljson -bare kubernetes
[INFO] generate received request
[INFO] received CSR
[INFO] generating key: rsa-2048
[INFO] encoded CSR
[INFO] signed certificate with serial number 562624490776452851974857846236319432028751121504
|
注:出现的 WARNING 是因为 hosts 字段未设置域名，这里我们就是要给kubernetes的IP生成证书，所以可以忽略该警告。
c.查看所生成的证书
|
1
2
|
# ls kubernetes*
kubernetes.csr kubernetes_csr.json kubernetes-key.pem kubernetes.pem
|
C. 创建 Admin 证书
a.创建 Admin 证书签名请求
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# cat << EOF > admin_csr.json
{
"CN"
:
"admin"
,
"hosts"
: [],
"key"
: {
"algo"
:
"rsa"
,
"size"
: 2048
},
"names"
: [
{
"C"
:
"CN"
,
"ST"
:
"BeiJing"
,
"L"
:
"BeiJing"
,
"O"
:
"system:masters"
,
"OU"
:
"System"
}
]
}
EOF
|
注:OU 指定该证书的 Group 为 system:masters，kubelet 使用该证书访问 kube-apiserver 时 ，由于证书被 CA 签名，所以认证通过，同时由于证书用户组为经过预授权的 system:masters，所以被授予访问所有 API 的权限。
b.生成 Admin 证书和私钥
|
1
2
3
4
5
6
|
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca_config.json -profile=kubernetes admin_csr.json | cfssljson -bare admin
[INFO] generate received request
[INFO] received CSR
[INFO] generating key: rsa-2048
[INFO] encoded CSR
[INFO] signed certificate with serial number 98602736507310427106587925783522327459817057634
|
c.查看所生成的证书
|
1
2
|
# ls admin*
admin.csr admin_csr.json admin-key.pem admin.pem
|
D. 创建 Kube-Proxy 证书
a.创建 kube-proxy 证书签名请求
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# cat << EOF > kube-proxy_csr.json
{
"CN"
:
"system:kube-proxy"
,
"hosts"
: [],
"key"
: {
"algo"
:
"rsa"
,
"size"
: 2048
},
"names"
: [
{
"C"
:
"CN"
,
"ST"
:
"BeiJing"
,
"L"
:
"BeiJing"
,
"O"
:
"k8s"
,
"OU"
:
"System"
}
]
}
EOF
|
b.生成 kube-proxy 客户端证书和私钥
|
1
2
3
4
5
6
|
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca_config.json -profile=kubernetes kube-proxy_csr.json | cfssljson -bare kube-proxy
[INFO] generate received request
[INFO] received CSR
[INFO] generating key: rsa-2048
[INFO] encoded CSR
[INFO] signed certificate with serial number 15961203695365328046366272691608837430729281180
|
c.查看所生成的证书
|
1
2
|
# ls kube-proxy*
kube-proxy.csr kube-proxy_csr.json kube-proxy-key.pem kube-proxy.pem
|
E. 创建 etcd 证书
a. 创建 etcd 证书签名请求
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
|
# cat << EOF > etcd_csr.json
{
"CN"
:
"etcd"
,
"hosts"
: [
"127.0.0.1"
,
"localhost"
,
"192.168.100.102"
,
"192.168.100.103"
,
"192.168.100.104"
],
"key"
: {
"algo"
:
"rsa"
,
"size"
: 2048
},
"names"
: [
{
"C"
:
"CN"
,
"L"
:
"BeiJing"
,
"ST"
:
"BeiJing"
,
"O"
:
"k8s"
,
"OU"
:
"System"
}
]
}
EOF
|
b.生成 etcd 客户端证书和私钥
|
1
2
3
4
5
6
|
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca_config.json -profile=kubernetes etcd_csr.json | cfssljson -bare etcd
[INFO] generate received request
[INFO] received CSR
[INFO] generating key: rsa-2048
[INFO] encoded CSR
[INFO] signed certificate with serial number 168388022915225919243296361863710051151902347190
|
c.查看所生成的证书
|
1
2
|
# ls etcd*
etcd.csr etcd_csr.json etcd-key.pem etcd.pem
|
F.查看、验证并分发证书
a.查看所生成的证书
|
1
2
|
# ls *.pem
admin-key.pem admin.pem ca-key.pem ca.pem etcd-key.pem etcd.pem kube-proxy-key.pem kube-proxy.pem kubernetes-key.pem kubernetes.pem
|
b.校验证书
|
1
2
|
# openssl x509 -noout -text -in kubernetes.pem
# cfssl-certinfo -cert kubernetes.pem
|
c.验证证书是否该CA签发
|
1
2
|
# openssl verify -CAfile ca.pem kubernetes.pem
kubernetes: OK
|
d.分发证书至所有 Node
|
1
2
3
4
|
# mkdir -p /etc/kubernetes/ssl
# cp *.pem /etc/kubernetes/ssl
# ansible node -m file -a 'path=/etc/kubernetes/ssl state=directory'
# ansible node -m copy -a 'src=/etc/kubernetes/ssl dest=/etc/kubernetes/'
|
e.配置使系统信任自签名证书
|
1
2
3
4
5
6
7
8
|
# yum -y install ca-certificates
# update-ca-trust force-enable
# cp ca.pem /etc/pki/ca-trust/source/anchors/
# update-ca-trust extract
# ansible node -m yum -a "state=present name=ca-certificates"
# ansible node -a "update-ca-trust force-enable"
# ansible node -m copy -a 'src=ca.pem dest=/etc/pki/ca-trust/source/anchors/'
# ansible node -a "update-ca-trust extract"
|
11.创建 kubeconfig 文件
A. 创建 TLS Bootstrapping Token
Token auth file:Token可以是任意的包涵128 bit的字符串，可使用安全的随机数发生器生成。
kubelet 首次启动时向 kube-apiserver 发送 TLS Bootstrapping 请求，kube-apiserver 验证 kubelet 请求中的 token 是否与它配置的 token 一致，如果一致则自动为 kubelet生成证书和秘钥。
|
1
2
3
4
5
|
# cd /etc/kubernetes/
# export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
# cat > token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,
"system:kubelet-bootstrap"
EOF
|
B. 将token.csv分发至所有 Node 的 /etc/kubernetes/ 目录
|
1
|
# ansible node -m copy -a 'src=token.csv dest=/etc/kubernetes/'
|
C. 创建 kubectl kubeconfig 文件
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
|
# pwd
/root
# export KUBE_APISERVER="https://192.168.100.102:6443"
## 设置集群参数
# kubectl config set-cluster kubernetes \
--certificate-authority=
/etc/kubernetes/ssl/ca
.pem \
--embed-certs=
true
\
--server=${KUBE_APISERVER}
Cluster
"kubernetes"
set
.
## 设置客户端认证参数
# kubectl config set-credentials admin \
--client-certificate=
/etc/kubernetes/ssl/admin
.pem \
--embed-certs=
true
\
--client-key=
/etc/kubernetes/ssl/admin-key
.pem
User
"admin"
set
.
## 设置关联参数
# kubectl config set-context kubernetes \
--cluster=kubernetes \
--user=admin
Context
"kubernetes"
created.
## 设置默认关联
# kubectl config use-context kubernetes
Switched to context
"kubernetes"
.
|
该kubeconfig 文件在如下位置:
|
1
2
|
# ls /root/.kube/config
/root/
.kube
/config
|
D.创建 kubelet kubeconfig 文件
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
|
# cd /etc/kubernetes
### 设置集群参数
# kubectl config set-cluster kubernetes \
--certificate-authority=
/etc/kubernetes/ssl/ca
.pem \
--embed-certs=
true
\
--server=${KUBE_APISERVER} \
--kubeconfig=bootstrap.kubeconfig
Cluster
"kubernetes"
set
.
### 设置客户端认证参数
# kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=bootstrap.kubeconfig
User
"kubelet-bootstrap"
set
.
### 设置关联参数
# kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig
Context
"default"
created.
### 设置默认关联
# kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
Switched to context
"default"
.
|
E.创建 kube-proxy kubeconfig 文件
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
|
## 设置集群参数
# kubectl config set-cluster kubernetes \
--certificate-authority=
/etc/kubernetes/ssl/ca
.pem \
--embed-certs=
true
\
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig
Cluster
"kubernetes"
set
.
## 设置客户端认证参数
# kubectl config set-credentials kube-proxy \
--client-certificate=
/etc/kubernetes/ssl/kube-proxy
.pem \
--client-key=
/etc/kubernetes/ssl/kube-proxy-key
.pem \
--embed-certs=
true
\
--kubeconfig=kube-proxy.kubeconfig
User
"kube-proxy"
set
.
## 设置上下文参数
# kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
Context
"default"
created.
## 设置默认上下文
# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
Switched to context
"default"
.
|
F.分发至所有 node 节点
|
1
2
|
# ansible node -m copy -a 'src=bootstrap.kubeconfig dest=/etc/kubernetes/'
# ansible node -m copy -a 'src=kube-proxy.kubeconfig dest=/etc/kubernetes/'
|
注:对看这篇文章的朋友表示抱歉，写得有点长，我又分篇了。
本文转自 结束的伤感 51CTO博客，原文链接:http://blog.51cto.com/wangzhijian/2044919
