kubernetes集群部署 – master节点部署

2018年7月21日 0条评论 1,600次阅读 0人点赞

获取软件包

[root@k8s-master ~]# wget https://dl.k8s.io/v1.9.2/kubernetes-server-linux-amd64.tar.gz
[root@k8s-master ~]# tar zxvf kubernetes-server-linux-amd64.tar.gz
[root@k8s-master ~]# cd kubernetes/server/bin
[root@k8s-master bin]# cp kube-controller-manager kube-scheduler kube-apiserver kubectl  /opt/kubernetes/bin/
[root@k8s-master bin]# ll /opt/kubernetes/bin/
total 496080
-rwxrwxr-x 1 1000 1000  17817664 Dec 21  2017 etcd
-rwxrwxr-x 1 1000 1000  15234432 Dec 21  2017 etcdctl
-rwxr-xr-x 1 root root 209287758 Jul 21 22:35 kube-apiserver
-rwxr-xr-x 1 root root 136677128 Jul 21 22:35 kube-controller-manager
-rwxr-xr-x 1 root root  67390556 Jul  8 06:24 kubectl
-rwxr-xr-x 1 root root  61566971 Jul 21 22:35 kube-scheduler

生成相关配置文件

创建 TLS Bootstrapping Token (自动生成kubu证书)

# 生成token
[root@k8s-master ~]# export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
[root@k8s-master ~]# cat > token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

# 查看token,随机生成的
[root@k8s-master ~]# cat token.csv 
6a20311b97c62786dcbf684716ec66a3,kubelet-bootstrap,10001,"system:kubelet-bootstrap"

创建 kubelet bootstrapping kubeconfig

# 设置API的入口
[root@k8s-master ~]# export KUBE_APISERVER="https://192.168.1.101:6443"

# 设置集群参数
[root@k8s-master ~]# kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=bootstrap.kubeconfig
Cluster "kubernetes" set.

# 设置客户端认证参数
[root@k8s-master ~]# kubectl config set-credentials kubelet-bootstrap \
  --token=${BOOTSTRAP_TOKEN} \
  --kubeconfig=bootstrap.kubeconfig
User "kubelet-bootstrap" set.
# 验证配置是否成功
[root@k8s-master ~]# echo ${BOOTSTRAP_TOKEN}
6a20311b97c62786dcbf684716ec66a3

[root@k8s-master ~]# tail -5 bootstrap.kubeconfig
users:
- name: kubelet-bootstrap
  user:
    as-user-extra: {}
    token: 6a20311b97c62786dcbf684716ec66a3
users:


# 设置上下文参数,指定访问集群的用户
[root@k8s-master ~]# kubectl config set-context default \
  --cluster=kubernetes \
  --user=kubelet-bootstrap \
  --kubeconfig=bootstrap.kubeconfig
Context "default" created.
# 验证上下文设置
[root@k8s-master ~]# cat bootstrap.kubeconfig
contexts:
- context:
    cluster: kubernetes
    user: kubelet-bootstrap
  name: default

# 设置默认上下文
[root@k8s-master ~]# kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
Switched to context "default".
# 验证默认上下文
current-context: default

创建kube-proxy kubeconfig文件

# 设置集群参数
[root@k8s-master ~]# kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=kube-proxy.kubeconfig
Cluster "kubernetes" set.

# 设置客户端认证参数
[root@k8s-master ~]# kubectl config set-credentials kube-proxy \
  --client-certificate=/opt/kubernetes/ssl/kube-proxy.pem \
  --client-key=/opt/kubernetes/ssl/kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-proxy.kubeconfig
User "kube-proxy" set.

# 设置上下文参数
[root@k8s-master ~]# kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=kube-proxy.kubeconfig
Context "default" created

# 设置默认上下文
[root@k8s-master ~]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
Switched to context "default".

# 主要参数验证
[root@k8s-master ~]# cat kube-proxy.kubeconfig 
    server: https://192.168.1.101:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kube-proxy
  name: default
current-context: default
kind: Config
preferences: {}
users:
- name: kube-proxy
  user:
    as-user-extra: {}

部署apiserver


# 拷贝token文件到kubernetes的配置文件中
[root@k8s-master ~]# cp token.csv  /opt/kubernetes/cfg/

#kube-apiserver部署脚本
[root@k8s-master scripts]# cat apiserver.sh 
#/bin/bash

MASTER_ADDRESS=${1:-"8.8.8.18"}
ETCD_SERVERS=${2:-"https://8.8.8.18:2379"}

cat <<EOF >/opt/kubernetes/cfg/kube-apiserver

KUBE_APISERVER_OPTS="--logtostderr=true \\
--v=4 \\
--etcd-servers=${ETCD_SERVERS} \\
--insecure-bind-address=127.0.0.1 \\
--bind-address=${MASTER_ADDRESS} \\
--insecure-port=8080 \\
--secure-port=6443 \\
--advertise-address=${MASTER_ADDRESS} \\
--allow-privileged=true \\
--service-cluster-ip-range=10.10.10.0/24 \\
--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,NodeRestriction  --authorization-mode=RBAC,Node \\
--kubelet-https=true \\
--enable-bootstrap-token-auth \\
--token-auth-file=/opt/kubernetes/cfg/token.csv \\
--service-node-port-range=30000-50000 \\
--tls-cert-file=/opt/kubernetes/ssl/server.pem  \\
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \\
--client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--etcd-cafile=/opt/kubernetes/ssl/ca.pem \\
--etcd-certfile=/opt/kubernetes/ssl/server.pem \\
--etcd-keyfile=/opt/kubernetes/ssl/server-key.pem"
EOF

cat <<EOF >/usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-apiserver
ExecStart=/opt/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF


systemctl daemon-reload
systemctl enable kube-apiserver
systemctl restart kube-apiserver

#执行脚本启动apiserver
[root@k8s-master scripts]# chmod  +x apiserver.sh 
[root@k8s-master scripts]# ./apiserver.sh 192.168.1.101  https://192.168.1.101:2379,https://192.168.1.102:2379,https://192.168.1.103:2379
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.

#查看apiserver状态
[root@k8s-master scripts]# systemctl  status kube-apiserver
● kube-apiserver.service - Kubernetes API Server
   Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)
   Active: active (running) since Sun 2018-07-22 00:50:04 CST; 1min 29s ago
     Docs: https://github.com/kubernetes/kubernetes
 Main PID: 2818 (kube-apiserver)
   CGroup: /system.slice/kube-apiserver.service
           └─2818 /opt/kubernetes/bin/kube-apiserver --logtostderr=true --v=4 --etcd-servers=https://192.168.1.101:2379,https://192.168.1.102:2379,https://192.168.1.103:2379 --insecur...

Jul 22 00:51:09 k8s-master kube-apiserver[2818]: I0722 00:51:09.501709    2818 wrap.go:42] GET /api/v1/namespaces/kube-public: (1.964844ms) 200 [[kube-apiserver/v1.9.2 (lin....101:35936]
Jul 22 00:51:09 k8s-master kube-apiserver[2818]: I0722 00:51:09.604721    2818 wrap.go:42] GET /api/v1/namespaces/default: (4.731942ms) 200 [[kube-apiserver/v1.9.2 (linux/a....101:35936]
Jul 22 00:51:09 k8s-master kube-apiserver[2818]: I0722 00:51:09.610351    2818 wrap.go:42] GET /api/v1/namespaces/default/services/kubernetes: (4.742735ms) 200 [[kube-apise....101:35936]
Jul 22 00:51:09 k8s-master kube-apiserver[2818]: I0722 00:51:09.617677    2818 wrap.go:42] GET /api/v1/namespaces/default/endpoints/kubernetes: (6.340002ms) 200 [[kube-apis....101:35936]
Jul 22 00:51:19 k8s-master kube-apiserver[2818]: I0722 00:51:19.624091    2818 wrap.go:42] GET /api/v1/namespaces/default: (4.926282ms) 200 [[kube-apiserver/v1.9.2 (linux/a....101:35936]
Jul 22 00:51:19 k8s-master kube-apiserver[2818]: I0722 00:51:19.631174    2818 wrap.go:42] GET /api/v1/namespaces/default/services/kubernetes: (6.018714ms) 200 [[kube-apise....101:35936]
Jul 22 00:51:19 k8s-master kube-apiserver[2818]: I0722 00:51:19.639534    2818 wrap.go:42] GET /api/v1/namespaces/default/endpoints/kubernetes: (7.334255ms) 200 [[kube-apis....101:35936]
Jul 22 00:51:29 k8s-master kube-apiserver[2818]: I0722 00:51:29.648483    2818 wrap.go:42] GET /api/v1/namespaces/default: (6.076445ms) 200 [[kube-apiserver/v1.9.2 (linux/a....101:35936]
Jul 22 00:51:29 k8s-master kube-apiserver[2818]: I0722 00:51:29.657957    2818 wrap.go:42] GET /api/v1/namespaces/default/services/kubernetes: (8.436691ms) 200 [[kube-apise....101:35936]
Jul 22 00:51:29 k8s-master kube-apiserver[2818]: I0722 00:51:29.662673    2818 wrap.go:42] GET /api/v1/namespaces/default/endpoints/kubernetes: (4.164289ms) 200 [[kube-apis....101:35936]
Hint: Some lines were ellipsized, use -l to show in full.

#查看进程
[root@k8s-master scripts]# ps -ef|grep kube-apiserver
root       2818      1  5 00:50 ?        00:00:06 /opt/kubernetes/bin/kube-apiserver --logtostderr=true --v=4 --etcd-servers=https://192.168.1.101:2379,https://192.168.1.102:2379,https://192.168.1.103:2379 --insecure-bind-address=127.0.0.1 --bind-address=192.168.1.101 --insecure-port=8080 --secure-port=6443 --advertise-address=192.168.1.101 --allow-privileged=true --service-cluster-ip-range=10.10.10.0/24 --admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,NodeRestriction --authorization-mode=RBAC,Node --kubelet-https=true --enable-bootstrap-token-auth --token-auth-file=/opt/kubernetes/cfg/token.csv --service-node-port-range=30000-50000 --tls-cert-file=/opt/kubernetes/ssl/server.pem --tls-private-key-file=/opt/kubernetes/ssl/server-key.pem --client-ca-file=/opt/kubernetes/ssl/ca.pem --service-account-key-file=/opt/kubernetes/ssl/ca-key.pem --etcd-cafile=/opt/kubernetes/ssl/ca.pem --etcd-certfile=/opt/kubernetes/ssl/server.pem --etcd-keyfile=/opt/kubernetes/ssl/server-key.pem

#查看监听端口
[root@k8s-master scripts]# netstat -nplt|grep apiserver
tcp        0      0 192.168.1.101:6443      0.0.0.0:*               LISTEN      2818/kube-apiserver 
tcp        0      0 127.0.0.1:8080          0.0.0.0:*               LISTEN      2818/kube-apiserver 

#参数解释
--v=4 \\   #日志级别
--etcd-servers=${ETCD_SERVERS} \\  #etcd的集群地址
--insecure-bind-address=127.0.0.1 \\  #apiserver访问地址,http协议
--bind-address=${MASTER_ADDRESS} \\   #apiserver访问地址,https协议
--insecure-port=8080 \\  #http协议端口
--secure-port=6443 \\    #https协议端口
--advertise-address=${MASTER_ADDRESS} \\  #集群通信地址
--allow-privileged=true \\  #允许的授权,即docker容器的root权限
--service-cluster-ip-range=10.10.10.0/24 \\  #service分配集群的vip的网段
--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,NodeRestriction  \\  #准入模块,权限控制
--authorization-mode=RBAC,Node \\  #基于角色的访问权限控制
--kubelet-https=true \\ #启用https访问
--enable-bootstrap-token-auth \\  #启用token认证,下发到node节点
--token-auth-file=/opt/kubernetes/cfg/token.csv \\ #指定token文件位置
--service-node-port-range=30000-50000 \\  #service端口范围

安装kube-controller-manager

# kube-controller-manager 部署脚本
[root@k8s-master scripts]# cat controller-manager.sh 
#!/bin/bash

MASTER_ADDRESS=${1:-"127.0.0.1"}

cat <<EOF >/opt/kubernetes/cfg/kube-controller-manager

KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=true \\
--v=4 \\
--master=${MASTER_ADDRESS}:8080 \\
--leader-elect=true \\
--address=127.0.0.1 \\
--service-cluster-ip-range=10.10.10.0/24 \\
--cluster-name=kubernetes \\
--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \\
--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem  \\
--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--root-ca-file=/opt/kubernetes/ssl/ca.pem"
EOF

cat <<EOF >/usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-controller-manager
ExecStart=/opt/kubernetes/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl restart kube-controller-manager

#赋予脚本执行权限,并执行脚本
[root@k8s-master scripts]# chmod +x controller-manager.sh 
[root@k8s-master scripts]# ./controller-manager.sh 
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.

#查看服务状态
[root@k8s-master scripts]# systemctl status kube-controller-manager
● kube-controller-manager.service - Kubernetes Controller Manager
   Loaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)
   Active: active (running) since Sun 2018-07-22 01:03:16 CST; 1min 47s ago
     Docs: https://github.com/kubernetes/kubernetes
 Main PID: 2940 (kube-controller)
   CGroup: /system.slice/kube-controller-manager.service
           └─2940 /opt/kubernetes/bin/kube-controller-manager --logtostderr=true --v=4 --master=127.0.0.1:8...

Jul 22 01:04:54 k8s-master kube-controller-manager[2940]: I0722 01:04:54.642009    2940 leaderelection.g...ger
Jul 22 01:04:56 k8s-master kube-controller-manager[2940]: I0722 01:04:56.671728    2940 leaderelection.g...ger
Jul 22 01:04:58 k8s-master kube-controller-manager[2940]: I0722 01:04:58.691827    2940 leaderelection.g...ger
Jul 22 01:04:59 k8s-master kube-controller-manager[2940]: I0722 01:04:59.695249    2940 gc_controller.go...ned
Jul 22 01:04:59 k8s-master kube-controller-manager[2940]: I0722 01:04:59.700401    2940 gc_controller.go...ng.
Jul 22 01:04:59 k8s-master kube-controller-manager[2940]: I0722 01:04:59.740556    2940 cronjob_controll...obs
Jul 22 01:04:59 k8s-master kube-controller-manager[2940]: I0722 01:04:59.748609    2940 cronjob_controll...obs
Jul 22 01:04:59 k8s-master kube-controller-manager[2940]: I0722 01:04:59.748635    2940 cronjob_controll...ups
Jul 22 01:05:00 k8s-master kube-controller-manager[2940]: I0722 01:05:00.711456    2940 leaderelection.g...ger
Jul 22 01:05:02 k8s-master kube-controller-manager[2940]: I0722 01:05:02.737916    2940 leaderelection.g...ger
Hint: Some lines were ellipsized, use -l to show in full.

#查看进程
[root@k8s-master scripts]# ps -ef|grep kube-controller-manager
root       2940      1  1 01:03 ?        00:00:01 /opt/kubernetes/bin/kube-controller-manager --logtostderr=true --v=4 --master=127.0.0.1:8080 --leader-elect=true --address=127.0.0.1 --service-cluster-ip-range=10.10.10.0/24 --cluster-name=kubernetes --cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem --cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem --service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem --root-ca-file=/opt/kubernetes/ssl/ca.pem

#查看端口号
[root@k8s-master scripts]# netstat -nplt|grep kube-controlle 
tcp        0      0 127.0.0.1:10252         0.0.0.0:*               LISTEN      2940/kube-controlle

安装kube-scheduler

#kube-scheduler 部署脚本
[root@k8s-master scripts]# cat scheduler.sh 
#!/bin/bash

MASTER_ADDRESS=${1:-"127.0.0.1"}

cat <<EOF >/opt/kubernetes/cfg/kube-scheduler
KUBE_SCHEDULER_OPTS="--logtostderr=true \\
--v=4 \\
--master=${MASTER_ADDRESS}:8080 \\
--leader-elect"
EOF

cat <<EOF >/usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler
ExecStart=/opt/kubernetes/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload 
systemctl enable kube-scheduler 
systemctl restart kube-scheduler

#赋予脚本执行权限,并执行脚本
[root@k8s-master scripts]# chmod +x scheduler.sh 
[root@k8s-master scripts]# ./scheduler.sh 
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.

#查看服务状态
[root@k8s-master scripts]# systemctl  status kube-scheduler
● kube-scheduler.service - Kubernetes Scheduler
   Loaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)
   Active: active (running) since Sun 2018-07-22 01:08:37 CST; 30s ago
     Docs: https://github.com/kubernetes/kubernetes
 Main PID: 3005 (kube-scheduler)
   CGroup: /system.slice/kube-scheduler.service
           └─3005 /opt/kubernetes/bin/kube-scheduler --logtostderr=true --v=4 --master=127.0.0.1:8080 --lea...

Jul 22 01:08:48 k8s-master kube-scheduler[3005]: I0722 01:08:48.837036    3005 leaderelection.go:199] s...uler
Jul 22 01:08:50 k8s-master kube-scheduler[3005]: I0722 01:08:50.843928    3005 leaderelection.go:199] s...uler
Jul 22 01:08:52 k8s-master kube-scheduler[3005]: I0722 01:08:52.851267    3005 leaderelection.go:199] s...uler
Jul 22 01:08:54 k8s-master kube-scheduler[3005]: I0722 01:08:54.871356    3005 leaderelection.go:199] s...uler
Jul 22 01:08:56 k8s-master kube-scheduler[3005]: I0722 01:08:56.902362    3005 leaderelection.go:199] s...uler
Jul 22 01:08:58 k8s-master kube-scheduler[3005]: I0722 01:08:58.926616    3005 leaderelection.go:199] s...uler
Jul 22 01:09:00 k8s-master kube-scheduler[3005]: I0722 01:09:00.955438    3005 leaderelection.go:199] s...uler
Jul 22 01:09:02 k8s-master kube-scheduler[3005]: I0722 01:09:02.976637    3005 leaderelection.go:199] s...uler
Jul 22 01:09:04 k8s-master kube-scheduler[3005]: I0722 01:09:04.997611    3005 leaderelection.go:199] s...uler
Jul 22 01:09:07 k8s-master kube-scheduler[3005]: I0722 01:09:07.018786    3005 leaderelection.go:199] s...uler
Hint: Some lines were ellipsized, use -l to show in full.

#查看进程
[root@k8s-master scripts]# ps -ef|grep kube-scheduler
root       3005      1  0 01:08 ?        00:00:00 /opt/kubernetes/bin/kube-scheduler --logtostderr=true --v=4 --master=127.0.0.1:8080 --leader-elect

#查看端口
[root@k8s-master scripts]# netstat -nplt|grep kube-scheduler
tcp6       0      0 :::10251                :::*                    LISTEN      3005/kube-scheduler

授于角色权限

#双向证书认证，node节点在请求的时候需要master节点认证并颁发证书
[root@k8s-master scripts]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
clusterrolebinding "kubelet-bootstrap" created

查看集群状态

[root@k8s-master scripts]# kubectl get cs
NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok                   
controller-manager   Healthy   ok                   
etcd-1               Healthy   {"health": "true"}   
etcd-2               Healthy   {"health": "true"}   
etcd-0               Healthy   {"health": "true"}