正文
二进制部署k8s
小程序：扫一扫查出行【扫一扫了解最新限行尾号】
复制小程序
一、二进制部署 k8s集群
1）参考文章
博客: https://blog.qikqiak.com
文章: https://www.qikqiak.com/post/manual-install-high-available-kubernetes-cluster/
2）环境架构
master:
192.168.10.12
192.168.10.22 etcd：类似于数据库，尽量使用高可用
192.168.10.12(etcd01)
192.168.10.22(etcd01)
二、创建证书
1）hosts 文件修改
[root@master01 ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
:: localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.10.12 k8s-api.virtual.local
k8s-api.virtual.local 为后期设计的高可用的访问地址。现在临时设置
2）环境变量定义
[root@master01 ~]# head -c /dev/urandom | od -An -t x | tr -d ' '
0b340751863956f119cbc624465db92b [root@master01 ~]# cat env.sh
BOOTSTRAP_TOKEN="0b340751863956f119cbc624465db92b"
SERVICE_CIDR="10.254.0.0/16"
CLUSTER_CIDR="172.30.0.0/16"
NODE_PORT_RANGE="30000-32766"
ETCD_ENDPOINTS="https://192.168.10.12:2379,https://192.168.10.22:2379"
FLANNEL_ETCD_PREFIX="/kubernetes/network"
CLUSTER_KUBERNETES_SVC_IP="10.254.0.1"
CLUSTER_DNS_SVC_IP="10.254.0.2"
CLUSTER_DNS_DOMAIN="cluster.local."
MASTER_URL="k8s-api.virtual.local"
[root@master01 ~]# mkdir -p /usr/k8s/bin
[root@master01 ~]# mv env.sh /usr/k8s/bin/
env.sh
head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成token值，每次都不一样
3）创建CA 证书和密钥
3.1）下载创建证书的命令
[root@master01 ~]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
[root@master01 ~]# chmod +x cfssl_linux-amd64
[root@master01 ~]# mv cfssl_linux-amd64 /usr/k8s/bin/cfssl [root@master01 ~]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
[root@master01 ~]# chmod +x cfssljson_linux-amd64
[root@master01 ~]# mv cfssljson_linux-amd64 /usr/k8s/bin/cfssljson [root@master01 ~]# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
[root@master01 ~]# chmod +x cfssl-certinfo_linux-amd64
[root@master01 ~]# mv cfssl-certinfo_linux-amd64 /usr/k8s/bin/cfssl-certinfo
[root@master01 ~]# chmod +x /usr/k8s/bin/cfssl*
3.2）为了方便使用命令，添加环境变量
[root@master01 bin]# ls /usr/k8s/bin/
cfssl cfssl-certinfo cfssljson env.sh
[root@master01 ~]# export PATH=/usr/k8s/bin/:$PATH
[root@master01 ~]# echo $PATH
/usr/k8s/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
环境变量写入配置文件永久生效
[root@master01 ~]# cat .bash_profile
# .bash_profile # Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi # User specific environment and startup programs PATH=$PATH:$HOME/bin
export PATH=/usr/k8s/bin/:$PATH # 新增
source /usr/k8s/bin/env.sh # 新增，后面需要
export PATH
3.3 ）创建默认证书文件
[root@master01 ~]# mkdir ssl
[root@master01 ~]# cd ssl/
[root@master01 ssl]# cfssl print-defaults config > config.json
[root@master01 ssl]# cfssl print-defaults csr > csr.json
[root@master01 ssl]# ls
config.json csr.json
3.5）改为需要的证书文件。ca-csr.json 和 ca-config.json
[root@master01 ssl]# cp config.json config.json.bak
[root@master01 ssl]# mv config.json ca-config.json
[root@master01 ssl]# cat ca-config.json # 修改为需要的
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
} [root@master01 ssl]# cp csr.json ca-csr.json
[root@master01 ssl]# mv csr.json csr.json.bak
[root@master01 ssl]# cat ca-csr.json # 修改为需要的
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
[root@master01 ssl]# ls
ca-config.json ca-csr.json config.json.bak csr.json.bak
3.6）根据证书生成证书秘钥对
[root@master01 ssl]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca
// :: [INFO] generating a new CA key and certificate from CSR
// :: [INFO] generate received request
// :: [INFO] received CSR
// :: [INFO] generating key: rsa-
// :: [INFO] encoded CSR
// :: [INFO] signed certificate with serial number
[root@master01 ssl]# ls ca*
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
备注：ca.pem 为私钥
ca-key.pem 为公钥
3.7）将证书拷贝到 所有的节点的 k8s的指定目录
[root@master01 ssl]# mkdir -p /etc/kubernetes/ssl
[root@master01 ssl]# cp ca* /etc/kubernetes/ssl
[root@master01 ssl]# ls /etc/kubernetes/ssl
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
强调：所有 k8s 节点
二、创建 etcd 集群
1）环境变量生效
export NODE_NAME=etcd01
export NODE_IP=192.168.10.12
export NODE_IPS="192.168.10.12 192.168.10.22"
export ETCD_NODES=etcd01=https://192.168.10.12:2380,etcd02=https://192.168.10.22:2380
执行过程
[root@master01 ssl]# source /usr/k8s/bin/env.sh
[root@master01 ssl]# echo $ETCD_ENDPOINTS # 检验变量
https://192.168.10.12:2379,https://192.168.10.22:2379
[root@master01 ssl]# export NODE_NAME=etcd01
[root@master01 ssl]# export NODE_IP=192.168.10.12
[root@master01 ssl]# export NODE_IPS="192.168.10.12 192.168.10.22"
[root@master01 ssl]# export ETCD_NODES=etcd01=https://192.168.10.12:2380,etcd02=https://192.168.10.22:2380
[root@master01 ssl]# echo $NODE_NAME
etcd01
2）导入etcd的命令。从github下载
https://github.com/coreos/etcd 找 releases 包
https://github.com/etcd-io/etcd/releases
[root@master01 ~]# wget https://github.com/etcd-io/etcd/releases/download/v3.3.13/etcd-v3.3.13-linux-amd64.tar.gz
[root@master01 ~]# tar xf etcd-v3.3.13-linux-amd64.tar.gz
[root@master01 ~]# ls etcd-v3.3.13-linux-amd64
Documentation etcd etcdctl README-etcdctl.md README.md READMEv2-etcdctl.md
[root@master01 ~]# cp etcd-v3.3.13-linux-amd64/etcd* /usr/k8s/bin/
3）创建etcd需要的json文件
[root@master01 ~]# cd /root/ssl/
[root@master01 ~]# cat > etcd-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"${NODE_IP}"
],
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
注意："${NODE_IP}" 根据环境变量替换为当前ip
[root@master01 ssl]# cat etcd-csr.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.10.12"
],
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
4）创建etcd秘钥文件
过程简略
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
-ca-key=/etc/kubernetes/ssl/ca-key.pem \
-config=/etc/kubernetes/ssl/ca-config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd
$ ls etcd*
etcd.csr etcd-csr.json etcd-key.pem etcd.pem
$ sudo mkdir -p /etc/etcd/ssl
$ sudo mv etcd*.pem /etc/etcd/ssl/
执行过程
[root@master01 ssl]# cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
> -ca-key=/etc/kubernetes/ssl/ca-key.pem \
> -config=/etc/kubernetes/ssl/ca-config.json \
> -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
// :: [INFO] generate received request
// :: [INFO] received CSR
// :: [INFO] generating key: rsa-
// :: [INFO] encoded CSR
// :: [INFO] signed certificate with serial number
// :: [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1., from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2. ("Information Requirements").
[root@master01 ssl]# ls etcd*
etcd.csr etcd-csr.json etcd-key.pem etcd.pem
备注：etcd-key.pem 私钥文件
etcd.pem 证书文件
[root@master01 ssl]# mkdir -p /etc/etcd/ssl
[root@master01 ssl]# mv etcd*.pem /etc/etcd/ssl/
[root@master01 ssl]# ll /etc/etcd/ssl/
总用量
-rw------- root root 5月 : etcd-key.pem
-rw-r--r-- root root 5月 : etcd.pem
5）创建etcd 的systemd unit 文件
$ sudo mkdir -p /var/lib/etcd # 必须要先创建工作目录
$ cat > etcd.service <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos [Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/k8s/bin/etcd \\
--name=${NODE_NAME} \\
--cert-file=/etc/etcd/ssl/etcd.pem \\
--key-file=/etc/etcd/ssl/etcd-key.pem \\
--peer-cert-file=/etc/etcd/ssl/etcd.pem \\
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \\
--trusted-ca-file=/etc/kubernetes/ssl/ca.pem \\
--peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \\
--initial-advertise-peer-urls=https://${NODE_IP}:2380 \\
--listen-peer-urls=https://${NODE_IP}:2380 \\
--listen-client-urls=https://${NODE_IP}:2379,http://127.0.0.1:2379 \\
--advertise-client-urls=https://${NODE_IP}:2379 \\
--initial-cluster-token=etcd-cluster- \\
--initial-cluster=${ETCD_NODES} \\
--initial-cluster-state=new \\
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=
LimitNOFILE= [Install]
WantedBy=multi-user.target
EOF
6.1）将创建的etcd.service文件放到指定位置，启动
$ sudo mv etcd.service /etc/systemd/system
$ sudo systemctl daemon-reload
$ sudo systemctl enable etcd
$ sudo systemctl start etcd
$ sudo systemctl status etcd
6.2）启动报错处理
[root@master01 ssl]# systemctl start etcd
Job for etcd.service failed because a timeout was exceeded. See "systemctl status etcd.service" and "journalctl -xe" for details.
处理错误过程
[root@master01 ssl]# systemctl status etcd.service -l
........
5月 :: master01 etcd[]: health check for peer 21e3841b92f796be could not connect: dial tcp 192.168.10.22:: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")
5月 :: master01 etcd[]: 1af68d968c7e3f22 is starting a new election at term
5月 :: master01 etcd[]: 1af68d968c7e3f22 became candidate at term
5月 :: master01 etcd[]: 1af68d968c7e3f22 received MsgVoteResp from 1af68d968c7e3f22 at term
5月 :: master01 etcd[]: 1af68d968c7e3f22 [logterm: , index: ] sent MsgVote request to 21e3841b92f796be at term
5月 :: master01 etcd[]: 1af68d968c7e3f22 is starting a new election at term
5月 :: master01 etcd[]: 1af68d968c7e3f22 became candidate at term
5月 :: master01 etcd[]: 1af68d968c7e3f22 received MsgVoteResp from 1af68d968c7e3f22 at term
5月 :: master01 etcd[]: 1af68d968c7e3f22 [logterm: , index: ] sent MsgVote request to 21e3841b92f796be at term
5月 :: master01 etcd[]: publish error: etcdserver: request timed out
原因。需要在其他节点安装 etcd，并启动etcd。如果启动了，该错误就直接消失了
最先启动的etcd 进程会卡住一段时间，等待其他节点启动加入集群，在所有的etcd 节点重复上面的步骤，直到所有的机器etcd 服务都已经启动。
7）在其他节点安装etcd
7.1）拷贝文件到其他节点
[root@master01 usr]# zip -r k8s.zip k8s/
[root@master01 usr]# scp k8s.zip root@master02:/usr/
[root@master01 etc]# zip -r kubernetes.zip kubernetes/
[root@master01 etc]# scp kubernetes.zip root@master02:/etc/
去master02解压该文件
[root@master02 etc]# tree /usr/k8s/
/usr/k8s/
└── bin
├── cfssl
├── cfssl-certinfo
├── cfssljson
├── env.sh
├── etcd
└── etcdctl [root@master02 etc]# tree /etc/kubernetes/
/etc/kubernetes/
└── ssl
├── ca-config.json
├── ca.csr
├── ca-csr.json
├── ca-key.pem
└── ca.pem
7.2）其他服务器的节点变量生效
export NODE_NAME=etcd02
export NODE_IP=192.168.10.22
export NODE_IPS="192.168.10.12 192.168.10.22"
export ETCD_NODES=etcd01=https://192.168.10.12:2380,etcd02=https://192.168.10.22:2380 source /usr/k8s/bin/env.sh
export PATH=/usr/k8s/bin/:$PATH
7.3）给其他节点创建TLS的证书请求
[root@master02 ~]# mkdir ssl
[root@master02 ~]# cd ssl/
cat > etcd-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"${NODE_IP}"
],
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
生成etcd证书和私钥
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
-ca-key=/etc/kubernetes/ssl/ca-key.pem \
-config=/etc/kubernetes/ssl/ca-config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd
$ ls etcd*
etcd.csr etcd-csr.json etcd-key.pem etcd.pem
$ sudo mkdir -p /etc/etcd/ssl
$ sudo mv etcd*.pem /etc/etcd/ssl/
[root@master02 ssl]# ls /etc/etcd/ssl/
etcd-key.pem etcd.pem
7.4）创建etcd的systemd unit文件
$ sudo mkdir -p /var/lib/etcd # 必须要先创建工作目录
$ cat > etcd.service <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos [Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/k8s/bin/etcd \\
--name=${NODE_NAME} \\
--cert-file=/etc/etcd/ssl/etcd.pem \\
--key-file=/etc/etcd/ssl/etcd-key.pem \\
--peer-cert-file=/etc/etcd/ssl/etcd.pem \\
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \\
--trusted-ca-file=/etc/kubernetes/ssl/ca.pem \\
--peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \\
--initial-advertise-peer-urls=https://${NODE_IP}:2380 \\
--listen-peer-urls=https://${NODE_IP}:2380 \\
--listen-client-urls=https://${NODE_IP}:2379,http://127.0.0.1:2379 \\
--advertise-client-urls=https://${NODE_IP}:2379 \\
--initial-cluster-token=etcd-cluster- \\
--initial-cluster=${ETCD_NODES} \\
--initial-cluster-state=new \\
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=
LimitNOFILE= [Install]
WantedBy=multi-user.target
EOF
7.5）启动etcd服务
[root@master02 ssl]# mv etcd.service /etc/systemd/system
systemctl daemon-reload
systemctl enable etcd
systemctl start etcd
systemctl status etcd
现在再次进入master01，执行systemctl status etcd 查看状态
8）验证etcd集群，在任意集群的节点执行下面命令
for ip in ${NODE_IPS}; do
ETCDCTL_API= /usr/k8s/bin/etcdctl \
--endpoints=https://${ip}:2379 \
--cacert=/etc/kubernetes/ssl/ca.pem \
--cert=/etc/etcd/ssl/etcd.pem \
--key=/etc/etcd/ssl/etcd-key.pem \
endpoint health; done
输出
https://192.168.10.12:2379 is healthy: successfully committed proposal: took = 1.298547ms
https://192.168.10.22:2379 is healthy: successfully committed proposal: took = 2.740962ms
三、搭建master集群
1）下载文件
https://github.com/kubernetes/kubernetes/
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.9.md
wget https://dl.k8s.io/v1.9.10/kubernetes-server-linux-amd64.tar.gz 下载该包较大，且国内无法访问
2）拷贝kubernetes的命令到 k8s的bin目录下
[root@master01 ~]# tar xf kubernetes-server-linux-amd64.tar.gz
[root@master01 ~]# cd kubernetes
[root@master01 kubernetes]# cp -r server/bin/{kube-apiserver,kube-controller-manager,kube-scheduler} /usr/k8s/bin/
[root@master01 kubernetes]# ls /usr/k8s/bin/kube-*
/usr/k8s/bin/kube-apiserver /usr/k8s/bin/kube-controller-manager /usr/k8s/bin/kube-scheduler
3）创建kubernetes 证书
cat > kubernetes-csr.json <<EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"${NODE_IP}",
"${MASTER_URL}",
"${CLUSTER_KUBERNETES_SVC_IP}",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
------------
[root@master01 kubernetes]# cat kubernetes-csr.json
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.10.12",
"k8s-api.virtual.local",
"10.254.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
kubernetes-csr.json
mv kubernetes-csr.json ../ssl/
4）生成kubernetes 证书和私钥
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
-ca-key=/etc/kubernetes/ssl/ca-key.pem \
-config=/etc/kubernetes/ssl/ca-config.json \
-profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
$ ls kubernetes*
kubernetes.csr kubernetes-csr.json kubernetes-key.pem kubernetes.pem
$ sudo mkdir -p /etc/kubernetes/ssl/
$ sudo mv kubernetes*.pem /etc/kubernetes/ssl/
4.1）操作过程
[root@master01 ssl]# cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
> -ca-key=/etc/kubernetes/ssl/ca-key.pem \
> -config=/etc/kubernetes/ssl/ca-config.json \
> -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
// :: [INFO] generate received request
// :: [INFO] received CSR
// :: [INFO] generating key: rsa-
// :: [INFO] encoded CSR
// :: [INFO] signed certificate with serial number
// :: [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1., from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2. ("Information Requirements").
[root@master01 ssl]# ls kubernetes*
kubernetes.csr kubernetes-csr.json kubernetes-key.pem kubernetes.pem
[root@master01 ssl]# ls /etc/kubernetes/ssl/
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
[root@master01 ssl]# mv kubernetes*.pem /etc/kubernetes/ssl/
[root@master01 ssl]# ls /etc/kubernetes/ssl/
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem kubernetes-key.pem kubernetes.pem
5） 配置和启动kube-apiserver
5.1）创建kube-apiserver 使用的客户端token 文件
kubelet 首次启动时向kube-apiserver 发送TLS Bootstrapping 请求，kube-apiserver 验证请求中的token 是否与它配置的token.csv 一致，如果一致则自动为kubelet 生成证书和密钥。
$ # 导入的 environment.sh 文件定义了 BOOTSTRAP_TOKEN 变量
$ cat > token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,,"system:kubelet-bootstrap"
EOF
$ sudo mv token.csv /etc/kubernetes/
操作流程
[root@master01 ssl]# cat > token.csv <<EOF
> ${BOOTSTRAP_TOKEN},kubelet-bootstrap,,"system:kubelet-bootstrap"
> EOF
[root@master01 ssl]# cat token.csv
0b340751863956f119cbc624465db92b,kubelet-bootstrap,,"system:kubelet-bootstrap"
[root@master01 ssl]#
[root@master01 ssl]# mv token.csv /etc/kubernetes/
5.2）审查日志策略文件内容如下：（/etc/kubernetes/audit-policy.yaml）
apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
- "RequestReceived"
rules:
# Log pod changes at RequestResponse level
- level: RequestResponse
resources:
- group: ""
# Resource "pods" doesn't match requests to any subresource of pods,
# which is consistent with the RBAC policy.
resources: ["pods"]
# Log "pods/log", "pods/status" at Metadata level
- level: Metadata
resources:
- group: ""
resources: ["pods/log", "pods/status"] # Don't log requests to a configmap called "controller-leader"
- level: None
resources:
- group: ""
resources: ["configmaps"]
resourceNames: ["controller-leader"] # Don't log watch requests by the "system:kube-proxy" on endpoints or services
- level: None
users: ["system:kube-proxy"]
verbs: ["watch"]
resources:
- group: "" # core API group
resources: ["endpoints", "services"] # Don't log authenticated requests to certain non-resource URL paths.
- level: None
userGroups: ["system:authenticated"]
nonResourceURLs:
- "/api*" # Wildcard matching.
- "/version" # Log the request body of configmap changes in kube-system.
- level: Request
resources:
- group: "" # core API group
resources: ["configmaps"]
# This rule only applies to resources in the "kube-system" namespace.
# The empty string "" can be used to select non-namespaced resources.
namespaces: ["kube-system"] # Log configmap and secret changes in all other namespaces at the Metadata level.
- level: Metadata
resources:
- group: "" # core API group
resources: ["secrets", "configmaps"] # Log all other resources in core and extensions at the Request level.
- level: Request
resources:
- group: "" # core API group
- group: "extensions" # Version of group should NOT be included. # A catch-all rule to log all other requests at the Metadata level.
- level: Metadata
# Long-running requests like watches that fall under this rule will not
# generate an audit event in RequestReceived.
omitStages:
audit-policy.yaml
5.3）命令行启动测试
/usr/k8s/bin/kube-apiserver \
--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
--advertise-address=${NODE_IP} \
--bind-address=0.0.0.0 \
--insecure-bind-address=${NODE_IP} \
--authorization-mode=Node,RBAC \
--runtime-config=rbac.authorization.k8s.io/v1alpha1 \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--token-auth-file=/etc/kubernetes/token.csv \
--service-cluster-ip-range=${SERVICE_CIDR} \
--service-node-port-range=${NODE_PORT_RANGE} \
--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/etc/kubernetes/ssl/ca.pem \
--etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \
--etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \
--etcd-servers=${ETCD_ENDPOINTS} \
--enable-swagger-ui=true \
--allow-privileged=true \
--apiserver-count= \
--audit-log-maxage= \
--audit-log-maxbackup= \
--audit-log-maxsize= \
--audit-log-path=/var/lib/audit.log \
--audit-policy-file=/etc/kubernetes/audit-policy.yaml \
--event-ttl=1h \
--logtostderr=true \
--v=
5.4）创建kube-apiserver 的systemd unit文件。将上面命令行执行成功的文件换到下面的文件内
cat > kube-apiserver.service <<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target [Service]
ExecStart=/usr/k8s/bin/kube-apiserver \\
--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
--advertise-address=${NODE_IP} \\
--bind-address=0.0.0.0 \\
--insecure-bind-address=${NODE_IP} \\
--authorization-mode=Node,RBAC \\
--runtime-config=rbac.authorization.k8s.io/v1alpha1 \\
--kubelet-https=true \\
--enable-bootstrap-token-auth \\
--token-auth-file=/etc/kubernetes/token.csv \\
--service-cluster-ip-range=${SERVICE_CIDR} \\
--service-node-port-range=${NODE_PORT_RANGE} \\
--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \\
--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \\
--client-ca-file=/etc/kubernetes/ssl/ca.pem \\
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--etcd-cafile=/etc/kubernetes/ssl/ca.pem \\
--etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \\
--etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \\
--etcd-servers=${ETCD_ENDPOINTS} \\
--enable-swagger-ui=true \\
--allow-privileged=true \\
--apiserver-count= \\
--audit-log-maxage= \\
--audit-log-maxbackup= \\
--audit-log-maxsize= \\
--audit-log-path=/var/lib/audit.log \\
--audit-policy-file=/etc/kubernetes/audit-policy.yaml \\
--event-ttl=1h \\
--logtostderr=true \\
--v=
Restart=on-failure
RestartSec=
Type=notify
LimitNOFILE= [Install]
WantedBy=multi-user.target
EOF
kube-apiserver.service
查看文件
[root@master01 ssl]# cat /root/ssl/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target [Service]
ExecStart=/usr/k8s/bin/kube-apiserver \
--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
--advertise-address=192.168.10.12 \
--bind-address=0.0.0.0 \
--insecure-bind-address=192.168.10.12 \
--authorization-mode=Node,RBAC \
--runtime-config=rbac.authorization.k8s.io/v1alpha1 \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--token-auth-file=/etc/kubernetes/token.csv \
--service-cluster-ip-range=10.254.0.0/ \
--service-node-port-range=- \
--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/etc/kubernetes/ssl/ca.pem \
--etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \
--etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \
--etcd-servers=https://192.168.10.12:2379,https://192.168.10.22:2379 \
--enable-swagger-ui=true \
--allow-privileged=true \
--apiserver-count= \
--audit-log-maxage= \
--audit-log-maxbackup= \
--audit-log-maxsize= \
--audit-log-path=/var/lib/audit.log \
--audit-policy-file=/etc/kubernetes/audit-policy.yaml \
--event-ttl=1h \
--logtostderr=true \
--v=
Restart=on-failure
RestartSec=
Type=notify
LimitNOFILE= [Install]
WantedBy=multi-user.target
cat kube-apiserver.service
5.5) 设置api_server开机自启动。并启动服务
[root@master01 ssl]# mv kube-apiserver.service /etc/systemd/system/
[root@master01 ssl]# systemctl daemon-reload
[root@master01 ssl]# systemctl enable kube-apiserver
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /etc/systemd/system/kube-apiserver.service.
[root@master01 ssl]# systemctl start kube-apiserver
[root@master01 ssl]# systemctl status kube-apiserver
● kube-apiserver.service - Kubernetes API Server
Loaded: loaded (/etc/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)
Active: active (running) since 一 2019-05-27 20:52:20 CST; 15s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
Main PID: 37700 (kube-apiserver)
CGroup: /system.slice/kube-apiserver.service
└─37700 /usr/k8s/bin/kube-apiserver --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota --advertise-address=192.168.10.12 --bind-address=0....
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.558316 37700 handler.go:160] kube-aggregator: GET "/api/v1/namespaces/default/services/kubernetes" satisfied by nonGoRestful
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.558345 37700 pathrecorder.go:247] kube-aggregator: "/api/v1/namespaces/default/services/kubernetes" satisfied by prefix /api/
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.558363 37700 handler.go:150] kube-apiserver: GET "/api/v1/namespaces/default/services/kubernetes" satisfied by gorestfu...rvice /api/v1
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.563809 37700 wrap.go:42] GET /api/v1/namespaces/default/services/kubernetes: (5.592002ms) 200 [[kube-apiserver/v1.9.10 ....0.0.1:42872]
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.564282 37700 round_trippers.go:436] GET https://127.0.0.1:6443/api/v1/namespaces/default/services/kubernetes 200 OK in 6 milliseconds
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.565420 37700 handler.go:160] kube-aggregator: GET "/api/v1/namespaces/default/endpoints/kubernetes" satisfied by nonGoRestful
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.565545 37700 pathrecorder.go:247] kube-aggregator: "/api/v1/namespaces/default/endpoints/kubernetes" satisfied by prefix /api/
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.565631 37700 handler.go:150] kube-apiserver: GET "/api/v1/namespaces/default/endpoints/kubernetes" satisfied by gorestf...rvice /api/v1
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.572355 37700 wrap.go:42] GET /api/v1/namespaces/default/endpoints/kubernetes: (7.170178ms) 200 [[kube-apiserver/v1.9.10....0.0.1:42872]
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.572846 37700 round_trippers.go:436] GET https://127.0.0.1:6443/api/v1/namespaces/default/endpoints/kubernetes 200 OK in 8 milliseconds
[root@master01 ssl]# systemctl status kube-apiserver -l
6）配置和启动kube-controller-manager
6.1）先命令行测试启动
/usr/k8s/bin/kube-controller-manager \
--address=127.0.0.1 \
--master=http://${MASTER_URL}:8080 \
--allocate-node-cidrs=true \
--service-cluster-ip-range=${SERVICE_CIDR} \
--cluster-cidr=${CLUSTER_CIDR} \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--leader-elect=true \
--v=
执行没有错误，创建文件
$ cat > kube-controller-manager.service <<EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service]
ExecStart=/usr/k8s/bin/kube-controller-manager \\
--address=127.0.0.1 \\
--master=http://${MASTER_URL}:8080 \\
--allocate-node-cidrs=true \\
--service-cluster-ip-range=${SERVICE_CIDR} \\
--cluster-cidr=${CLUSTER_CIDR} \\
--cluster-name=kubernetes \\
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \\
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--root-ca-file=/etc/kubernetes/ssl/ca.pem \\
--leader-elect=true \\
--v=
Restart=on-failure
RestartSec= [Install]
WantedBy=multi-user.target
EOF
kube-controller-manager.servic
查看该文件
[root@master01 ssl]# cat kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service]
ExecStart=/usr/k8s/bin/kube-controller-manager \
--address=127.0.0.1 \
--master=http://k8s-api.virtual.local:8080 \
--allocate-node-cidrs=true \
--service-cluster-ip-range=10.254.0.0/ \
--cluster-cidr=172.30.0.0/ \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--leader-elect=true \
--v=
Restart=on-failure
RestartSec= [Install]
WantedBy=multi-user.target
cat kube-controller-manager.service
6.2）设置kube-controller-manager.service 开机自启动
[root@master01 ssl]# mv kube-controller-manager.service /etc/systemd/system
[root@master01 ssl]# systemctl daemon-reload
[root@master01 ssl]# systemctl enable kube-controller-manager.service
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /etc/systemd/system/kube-controller-manager.service. [root@master01 ssl]# systemctl start kube-controller-manager.service
[root@master01 ssl]# systemctl status kube-controller-manager.service
● kube-controller-manager.service - Kubernetes Controller Manager
Loaded: loaded (/etc/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)
Active: active (running) since 一 -- :: CST; 13s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
Main PID: (kube-controller)
CGroup: /system.slice/kube-controller-manager.service
└─ /usr/k8s/bin/kube-controller-manager --address=127.0.0.1 --master=http://k8s-api.virtual.local:8080 --allocate-node-cidrs=true --service-cluster-ip-range=10.254.0.0/16 --cluster-c... 5月 :: master01 kube-controller-manager[]: I0527 ::47.333573 controller_utils.go:] Caches are synced for cidrallocator controller
5月 :: master01 kube-controller-manager[]: I0527 ::47.353933 controller_utils.go:] Caches are synced for certificate controller
5月 :: master01 kube-controller-manager[]: I0527 ::47.407102 controller_utils.go:] Caches are synced for certificate controller
5月 :: master01 kube-controller-manager[]: I0527 ::47.424111 controller_utils.go:] Caches are synced for job controller
5月 :: master01 kube-controller-manager[]: I0527 ::47.490109 controller_utils.go:] Caches are synced for resource quota controller
5月 :: master01 kube-controller-manager[]: I0527 ::47.506266 controller_utils.go:] Caches are synced for garbage collector controller
5月 :: master01 kube-controller-manager[]: I0527 ::47.506311 garbagecollector.go:] Garbage collector: all resource monitors have synced. Proceeding to collect garbage
5月 :: master01 kube-controller-manager[]: I0527 ::47.524898 controller_utils.go:] Caches are synced for resource quota controller
5月 :: master01 kube-controller-manager[]: I0527 ::47.525473 controller_utils.go:] Caches are synced for persistent volume controller
5月 :: master01 kube-controller-manager[]: I0527 ::48.444031 garbagecollector.go:] syncing garbage collector with updated resources from discovery: map[{ v1...:{} {rbac.au
7）配置和启动kube-scheduler
7.1）命令行测试命令是否正常
/usr/k8s/bin/kube-scheduler \
--address=127.0.0.1 \
--master=http://${MASTER_URL}:8080 \
--leader-elect=true \
--v=
正常则创建kube-scheduler.service文件
$ cat > kube-scheduler.service <<EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service]
ExecStart=/usr/k8s/bin/kube-scheduler \\
--address=127.0.0.1 \\
--master=http://${MASTER_URL}:8080 \\
--leader-elect=true \\
--v=
Restart=on-failure
RestartSec= [Install]
WantedBy=multi-user.target
EOF
查看文件
[root@master01 ssl]# cat kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service]
ExecStart=/usr/k8s/bin/kube-scheduler \
--address=127.0.0.1 \
--master=http://k8s-api.virtual.local:8080 \
--leader-elect=true \
--v=
Restart=on-failure
RestartSec= [Install]
WantedBy=multi-user.target
cat kube-scheduler.service
7.2）设置kube-scheduler的开机启动
[root@master01 ssl]# mv kube-scheduler.service /etc/systemd/system
[root@master01 ssl]# systemctl daemon-reload
[root@master01 ssl]# systemctl enable kube-scheduler
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /etc/systemd/system/kube-scheduler.service. [root@master01 ssl]# systemctl start kube-scheduler
[root@master01 ssl]# systemctl status kube-scheduler
● kube-scheduler.service - Kubernetes Scheduler
Loaded: loaded (/etc/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)
Active: active (running) since 一 -- :: CST; 17s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
Main PID: (kube-scheduler)
CGroup: /system.slice/kube-scheduler.service
└─ /usr/k8s/bin/kube-scheduler --address=127.0.0.1 --master=http://k8s-api.virtual.local:8080 --leader-elect=true --v=2 5月 :: master01 systemd[]: Starting Kubernetes Scheduler...
5月 :: master01 kube-scheduler[]: W0527 ::42.436689 server.go:] WARNING: all flags than --config are deprecated. Please begin using a config file ASAP.
5月 :: master01 kube-scheduler[]: I0527 ::42.438090 server.go:] Version: v1.9.10
5月 :: master01 kube-scheduler[]: I0527 ::42.438222 factory.go:] Creating scheduler from algorithm provider 'DefaultProvider'
5月 :: master01 kube-scheduler[]: I0527 ::42.438231 factory.go:] Creating scheduler with fit predicates 'map[MaxEBSVolumeCount:{} MaxGCEPDVolumeCount:{} Ge... MatchInterPo
5月 :: master01 kube-scheduler[]: I0527 ::42.438431 server.go:] starting healthz server on 127.0.0.1:
5月 :: master01 kube-scheduler[]: I0527 ::43.243364 controller_utils.go:] Waiting for caches to sync for scheduler controller
5月 :: master01 kube-scheduler[]: I0527 ::43.345039 controller_utils.go:] Caches are synced for scheduler controller
5月 :: master01 kube-scheduler[]: I0527 ::43.345105 leaderelection.go:] attempting to acquire leader lease...
5月 :: master01 kube-scheduler[]: I0527 ::43.359717 leaderelection.go:] successfully acquired lease kube-system/kube-scheduler
Hint: Some lines were ellipsized, use -l to show in full.
8）配置kubectl 命令行工具
8.1）配置环境变量
$ export KUBE_APISERVER="https://${MASTER_URL}:6443"
环境变量定义在配置文件
[root@master01 ssl]# cat ~/.bash_profile
# .bash_profile # Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi # User specific environment and startup programs PATH=$PATH:$HOME/bin
export PATH=/usr/k8s/bin/:$PATH
export NODE_IP=192.168.10.12
source /usr/k8s/bin/env.sh
export KUBE_APISERVER="https://${MASTER_URL}:6443"
export PATH
cat ~/.bash_profile
8.2) 配置 kubectl命令
[root@master01 ~]# ls kubernetes/server/bin/kubectl
kubernetes/server/bin/kubectl
[root@master01 ~]# cp kubernetes/server/bin/kubectl /usr/k8s/bin/
[root@master01 ~]# kubectl version
Client Version: version.Info{Major:"", Minor:"", GitVersion:"v1.9.10", GitCommit:"098570796b32895c38a9a1c9286425fb1ececa18", GitTreeState:"clean", BuildDate:"2018-08-02T17:19:54Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
The connection to the server localhost: was refused - did you specify the right host or port?
8.3）配置admin证书
$ cat > admin-csr.json <<EOF
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
EOF
备注
后续kube-apiserver使用RBAC 对客户端(如kubelet、kube-proxy、Pod)请求进行授权
kube-apiserver 预定义了一些RBAC 使用的RoleBindings，如cluster-admin 将Group system:masters与Role cluster-admin绑定，该Role 授予了调用kube-apiserver所有API 的权限
O 指定了该证书的Group 为system:masters，kubectl使用该证书访问kube-apiserver时，由于证书被CA 签名，所以认证通过，同时由于证书用户组为经过预授权的system:masters，所以被授予访问所有API 的劝降
hosts 属性值为空列表
8.4）生成admin 证书和私钥
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
-ca-key=/etc/kubernetes/ssl/ca-key.pem \
-config=/etc/kubernetes/ssl/ca-config.json \
-profile=kubernetes admin-csr.json | cfssljson -bare admin
$ ls admin
admin.csr admin-csr.json admin-key.pem admin.pem
$ sudo mv admin*.pem /etc/kubernetes/ssl/
8.5）创建kubectl kubeconfig 文件
# 设置集群参数
$ kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER}
# 设置客户端认证参数
$ kubectl config set-credentials admin \
--client-certificate=/etc/kubernetes/ssl/admin.pem \
--embed-certs=true \
--client-key=/etc/kubernetes/ssl/admin-key.pem \
--token=${BOOTSTRAP_TOKEN}
# 设置上下文参数
$ kubectl config set-context kubernetes \
--cluster=kubernetes \
--user=admin
# 设置默认上下文
$ kubectl config use-context kubernetes
执行过程
[root@master01 ssl]# echo $KUBE_APISERVER
https://k8s-api.virtual.local:6443
[root@master01 ssl]# kubectl config set-cluster kubernetes \
> --certificate-authority=/etc/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${KUBE_APISERVER}
Cluster "kubernetes" set.
[root@master01 ssl]#
[root@master01 ssl]# echo $BOOTSTRAP_TOKEN
0b340751863956f119cbc624465db92b
[root@master01 ssl]# kubectl config set-credentials admin \
> --client-certificate=/etc/kubernetes/ssl/admin.pem \
> --embed-certs=true \
> --client-key=/etc/kubernetes/ssl/admin-key.pem \
> --token=${BOOTSTRAP_TOKEN}
User "admin" set.
[root@master01 ssl]# kubectl config set-context kubernetes \
> --cluster=kubernetes \
> --user=admin
Context "kubernetes" created.
[root@master01 ssl]# kubectl config use-context kubernetes
Switched to context "kubernetes".
8.6）此时验证kubect是否可以使用
[root@master01 ssl]# ls ~/.kube/
config
[root@master01 ssl]# ls ~/.kube/config
/root/.kube/config
[root@master01 ssl]# kubectl version
Client Version: version.Info{Major:"", Minor:"", GitVersion:"v1.9.10", GitCommit:"098570796b32895c38a9a1c9286425fb1ececa18", GitTreeState:"clean", BuildDate:"2018-08-02T17:19:54Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"", Minor:"", GitVersion:"v1.9.10", GitCommit:"098570796b32895c38a9a1c9286425fb1ececa18", GitTreeState:"clean", BuildDate:"2018-08-02T17:11:51Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
[root@master01 ssl]# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd- Healthy {"health":"true"}
etcd- Healthy {"health":"true"}
四、部署flannel网络（所有节点都需要安装）
1） 给node节点 创建环境变量
将master01的/usr/k8s/bin/env.sh 拷贝过来
[root@k8s01-node01 ~]# export NODE_IP=192.168.10.23
[root@k8s01-node01 ~]# cat ~/.bash_profile
# .bash_profile # Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi # User specific environment and startup programs PATH=$PATH:$HOME/bin
PATH=/usr/k8s/bin:$PATH
source /usr/k8s/bin/env.sh
export NODE_IP=192.168.10.23
export PATH
2）从master01导入ca相关证书
[root@k8s01-node01 ~]# mkdir -p /etc/kubernetes/ssl
[root@master01 ssl]# ll ca* # 将master01节点的ca证书拷贝到node01中
-rw-r--r-- root root 5月 : ca-config.json
-rw-r--r-- root root 5月 : ca.csr
-rw-r--r-- root root 5月 : ca-csr.json
-rw------- root root 5月 : ca-key.pem
-rw-r--r-- root root 5月 : ca.pem
[root@master01 ssl]# scp ca* root@n1:/etc/kubernetes/ssl/
ca-config.json % .6KB/s :
ca.csr % .8KB/s :
ca-csr.json % .2KB/s :
ca-key.pem % .6KB/s :
ca.pem [root@k8s01-node01 ~]# ll /etc/kubernetes/ssl 查看nodes的ca证书
总用量
-rw-r--r-- root root 5月 : ca-config.json
-rw-r--r-- root root 5月 : ca.csr
-rw-r--r-- root root 5月 : ca-csr.json
-rw------- root root 5月 : ca-key.pem
-rw-r--r-- root root 5月 : ca.pem
3）创建flanneld 证书签名请求
cat > flanneld-csr.json <<EOF
{
"CN": "flanneld",
"hosts": [],
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
flanneld-csr.json
3.1）生成flanneld 证书和私钥
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
-ca-key=/etc/kubernetes/ssl/ca-key.pem \
-config=/etc/kubernetes/ssl/ca-config.json \
-profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
$ ls flanneld*
flanneld.csr flanneld-csr.json flanneld-key.pem flanneld.pem
$ sudo mkdir -p /etc/flanneld/ssl
$ sudo mv flanneld*.pem /etc/flanneld/ssl
3.2）实际操作（证书在master生成，发布给nodes）
node01操作
[root@k8s01-node01 ~]# mkdir -p /etc/flanneld/ssl
master01
创建flanneld 证书签名请求
$ cat > flanneld-csr.json <<EOF
{
"CN": "flanneld",
"hosts": [],
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
生成flanneld 证书和私钥：
[root@master01 ssl]# cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
> -ca-key=/etc/kubernetes/ssl/ca-key.pem \
> -config=/etc/kubernetes/ssl/ca-config.json \
> -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
// :: [INFO] generate received request
// :: [INFO] received CSR
// :: [INFO] generating key: rsa-
// :: [INFO] encoded CSR
// :: [INFO] signed certificate with serial number
// :: [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1., from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2. ("Information Requirements").
[root@master01 ssl]# ls flanneld*
flanneld.csr flanneld-csr.json flanneld-key.pem flanneld.pem
[root@master01 ssl]# scp flanneld* root@n1:/etc/flanneld/ssl/
flanneld.csr % .5KB/s :
flanneld-csr.json % .1KB/s :
flanneld-key.pem % .4MB/s :
flanneld.pem % .4MB/s :
[root@master01 ssl]#
node01查看
[root@k8s01-node01 ~]# ll /etc/flanneld/ssl
总用量
-rw-r--r-- root root 5月 : flanneld.csr
-rw-r--r-- root root 5月 : flanneld-csr.json
-rw------- root root 5月 : flanneld-key.pem
-rw-r--r-- root root 5月 : flanneld.pem
4）向etcd 写入集群Pod 网段信息。该步骤只需在第一次部署Flannel 网络时执行，后续在其他节点上部署Flanneld 时无需再写入该信息
[root@master01 ssl]# mkdir -p /etc/flanneld/ssl
[root@master01 ssl]# mv flanneld*.pem /etc/flanneld/ssl/
$ etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/flanneld/ssl/flanneld.pem \
--key-file=/etc/flanneld/ssl/flanneld-key.pem \
set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'
# 得到如下反馈信息
{"Network":"172.30.0.0/16", "SubnetLen": , "Backend": {"Type": "vxlan"}}
写入的 Pod 网段(${CLUSTER_CIDR}，172.30.0.0/16) 必须与kube-controller-manager 的 --cluster-cidr 选项值一致；
4.1）实际操作
[root@master01 ssl]# mkdir -p /etc/flanneld/ssl
[root@master01 ssl]# mv flanneld*.pem /etc/flanneld/ssl/
[root@master01 ssl]# ll /etc/flanneld/ssl/
总用量
-rw------- root root 5月 : flanneld-key.pem
-rw-r--r-- root root 5月 : flanneld.pem
[root@master01 ssl]# echo $FLANNEL_ETCD_PREFIX
/kubernetes/network
[root@master01 ssl]# echo $CLUSTER_CIDR
172.30.0.0/
[root@master01 ssl]# echo $ETCD_ENDPOINTS
https://192.168.10.12:2379,https://192.168.10.22:2379
[root@master01 ssl]# etcdctl \
> --endpoints=${ETCD_ENDPOINTS} \
> --ca-file=/etc/kubernetes/ssl/ca.pem \
> --cert-file=/etc/flanneld/ssl/flanneld.pem \
> --key-file=/etc/flanneld/ssl/flanneld-key.pem \
> set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'
{"Network":"172.30.0.0/16", "SubnetLen": , "Backend": {"Type": "vxlan"}}
[root@master01 ssl]# cat /etc/systemd/system/kube-controller-manager.service |grep "cluster-cidr"
--cluster-cidr=172.30.0.0/ \
5）下载最新版的flanneld 二进制文件。
https://github.com/coreos/flannel/releases
wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
[root@k8s01-node01 ~]# ls flannel-v0.11.0-linux-amd64.tar.gz
flannel-v0.11.0-linux-amd64.tar.gz
[root@k8s01-node01 ~]# tar xf flannel-v0.11.0-linux-amd64.tar.gz
[root@k8s01-node01 ~]# cp flanneld /usr/k8s/bin/
[root@k8s01-node01 ~]# cp mk-docker-opts.sh /usr/k8s/bin/
[root@k8s01-node01 ~]# ls /usr/k8s/bin/
env.sh flanneld mk-docker-opts.sh
6）创建flanneld的systemd unit 文件
6.1）测试命令是否能正常执行（node01操作）
/usr/k8s/bin/flanneld \
-etcd-cafile=/etc/kubernetes/ssl/ca.pem \
-etcd-certfile=/etc/flanneld/ssl/flanneld.pem \
-etcd-keyfile=/etc/flanneld/ssl/flanneld-key.pem \
-etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-prefix=${FLANNEL_ETCD_PREFIX}
6.2）node01创建flanneld.service文件
$ cat > flanneld.service << EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service [Service]
Type=notify
ExecStart=/usr/k8s/bin/flanneld \\
-etcd-cafile=/etc/kubernetes/ssl/ca.pem \\
-etcd-certfile=/etc/flanneld/ssl/flanneld.pem \\
-etcd-keyfile=/etc/flanneld/ssl/flanneld-key.pem \\
-etcd-endpoints=${ETCD_ENDPOINTS} \\
-etcd-prefix=${FLANNEL_ETCD_PREFIX}
ExecStartPost=/usr/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure [Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF
flanneld.service
6.3) 设置 flanneld 的开机启动
[root@k8s01-node01 ~]# mv flanneld.service /etc/systemd/system
[root@k8s01-node01 ~]# systemctl daemon-reload
[root@k8s01-node01 ~]# systemctl enable flanneld
Created symlink from /etc/systemd/system/multi-user.target.wants/flanneld.service to /etc/systemd/system/flanneld.service.
Created symlink from /etc/systemd/system/docker.service.requires/flanneld.service to /etc/systemd/system/flanneld.service.
[root@k8s01-node01 ~]# systemctl start flanneld
[root@k8s01-node01 ~]# systemctl status flanneld
● flanneld.service - Flanneld overlay address etcd agent
Loaded: loaded (/etc/systemd/system/flanneld.service; enabled; vendor preset: disabled)
Active: active (running) since 二 -- :: CST; 21s ago
Process: ExecStartPost=/usr/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker (code=exited, status=/SUCCESS)
Main PID: (flanneld)
Memory: 10.3M
CGroup: /system.slice/flanneld.service
└─ /usr/k8s/bin/flanneld -etcd-cafile=/etc/kubernetes/ssl/ca.pem -etcd-certfile=/etc/flanneld/ssl/flanneld.pem -etcd-keyfile=/etc/flanneld/ssl/flanneld-key.pem -etcd-endpoints=https:... 5月 :: k8s01-node01 flanneld[]: I0528 ::18.747119 main.go:] Created subnet manager: Etcd Local Manager with Previous Subnet: 172.30.23.0/
5月 :: k8s01-node01 flanneld[]: I0528 ::18.747123 main.go:] Installing signal handlers
5月 :: k8s01-node01 flanneld[]: I0528 ::18.819160 main.go:] Found network config - Backend type: vxlan
5月 :: k8s01-node01 flanneld[]: I0528 ::18.819293 vxlan.go:] VXLAN config: VNI= Port= GBP=false DirectRouting=false
5月 :: k8s01-node01 flanneld[]: I0528 ::18.830870 local_manager.go:] Found lease (172.30.23.0/) for current IP (192.168.10.23), reusing
5月 :: k8s01-node01 flanneld[]: I0528 ::18.844660 main.go:] Wrote subnet file to /run/flannel/subnet.env
5月 :: k8s01-node01 flanneld[]: I0528 ::18.844681 main.go:] Running backend.
5月 :: k8s01-node01 flanneld[]: I0528 ::18.844739 vxlan_network.go:] watching for new subnet leases
5月 :: k8s01-node01 flanneld[]: I0528 ::18.851989 main.go:] Waiting for 22h59m59.998588039s to renew lease
5月 :: k8s01-node01 systemd[]: Started Flanneld overlay address etcd agent.
查看网络
[root@k8s01-node01 ~]# ifconfig flannel.
flannel.: flags=<UP,BROADCAST,RUNNING,MULTICAST> mtu
inet 172.30.23.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::b4e5:d4ff:fe53:16da prefixlen scopeid 0x20<link>
ether b6:e5:d4:::da txqueuelen (Ethernet)
RX packets bytes (0.0 B)
RX errors dropped overruns frame
TX packets bytes (0.0 B)
TX errors dropped overruns carrier collisions
ifconfig flannel.1
6.4）其他node节点，同样的方法安装flanneld网络
7）master节点查看nodes网络节点信息
$ etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/flanneld/ssl/flanneld.pem \
--key-file=/etc/flanneld/ssl/flanneld-key.pem \
ls ${FLANNEL_ETCD_PREFIX}/subnets /kubernetes/network/subnets/172.30.23.0- # 为node01节点的网络。可在node01使用ifconfig flannel.1查看
五、部署node节点
1）配置环境变量，已经hosts文件
[root@k8s01-node01 ~]# export KUBE_APISERVER="https://${MASTER_URL}:6443"
[root@k8s01-node01 ~]# echo $KUBE_APISERVER
https://k8s-api.virtual.local:6443
[root@k8s01-node01 ~]# cat .bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
PATH=/usr/k8s/bin:$PATH
source /usr/k8s/bin/env.sh
export NODE_IP=192.168.10.23
export KUBE_APISERVER="https://${MASTER_URL}:6443"
export PATH
[root@k8s01-node01 ~]# cat /etc/hosts|grep k8s-api.virtual.local
192.168.10.12 k8s-api.virtual.local
2）开启路由转发
net.ipv4.ip_forward=1
[root@k8s01-node01 ~]# cat /etc/sysctl.conf
# sysctl settings are defined through files in
# /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/.
#
# Vendors settings live in /usr/lib/sysctl.d/.
# To override a whole file, create a new file with the same in
# /etc/sysctl.d/ and put new settings there. To override
# only specific settings, add a file with a lexically later
# name in /etc/sysctl.d/ and put new settings there.
#
# For more information, see sysctl.conf() and sysctl.d(). net.ipv4.ip_forward=
[root@k8s01-node01 ~]# sysctl -p
net.ipv4.ip_forward = 1
3）根据官网进行docker的安装
https://docs.docker.com/install/linux/docker-ce/centos # docker 官网安装
先安装仓库
yum install -y yum-utils \
device-mapper-persistent-data \
lvm2
安装源
yum-config-manager \
--add-repo \
https://download.docker.com/linux/centos/docker-ce.repo
安装docker
yum install docker-ce docker-ce-cli containerd.io -y [root@k8s01-node01 ~]# docker version
Client:
Version: 18.09.
API version: 1.39
Go version: go1.10.8
Git commit: 481bc77156
Built: Sat May ::
OS/Arch: linux/amd64
Experimental: false Server: Docker Engine - Community
Engine:
Version: 18.09.
API version: 1.39 (minimum version 1.12)
Go version: go1.10.8
Git commit: 481bc77
Built: Sat May ::
OS/Arch: linux/amd64
Experimental: false
4）修改docker 的systemd unit 文件
vim /usr/lib/systemd/system/docker.service
.....
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
EnvironmentFile=-/run/flannel/docker # 新增
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --log-level=info $DOCKER_NETWORK_OPTIONS # 新增 --log-level=info $DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=
RestartSec=
Restart=always
.....................
查看文件内容
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
BindsTo=containerd.service
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket [Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --log-level=info $DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=
RestartSec=
Restart=always # Note that StartLimit* options were moved from "Service" to "Unit" in systemd .
# Both the old, and new location are accepted by systemd and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst= # Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd .
# Both the old, and new name are accepted by systemd and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s # Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity # Comment TasksMax if your systemd version does not supports it.
# Only systemd and above support this option.
TasksMax=infinity # set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes # kill only the docker process, not all processes in the cgroup
KillMode=process [Install]
WantedBy=multi-user.target
cat /usr/lib/systemd/system/docker.service
5）为了加快 pull image 的速度，可以使用国内的仓库镜像服务器，同时增加下载的并发数。(如果 dockerd 已经运行，则需要重启 dockerd 生效。)
[root@k8s01-node01 ~]# cat /etc/docker/daemon.json
{
"max-concurrent-downloads":
}
6）启动docker
[root@k8s01-node01 ~]# systemctl daemon-reload
[root@k8s01-node01 ~]# systemctl enable docker
[root@k8s01-node01 ~]# systemctl start docker
[root@k8s01-node01 ~]# systemctl status docker
内核优化
[root@k8s01-node01 ~]# cat /etc/sysctl.conf
# sysctl settings are defined through files in
# /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/.
#
# Vendors settings live in /usr/lib/sysctl.d/.
# To override a whole file, create a new file with the same in
# /etc/sysctl.d/ and put new settings there. To override
# only specific settings, add a file with a lexically later
# name in /etc/sysctl.d/ and put new settings there.
#
# For more information, see sysctl.conf() and sysctl.d(). net.ipv4.ip_forward=
net.bridge.bridge-nf-call-iptables=
net.bridge.bridge-nf-call-ip6tables=
sysctl -p
[root@k8s01-node01 ~]# sysctl -p
net.ipv4.ip_forward =
net.bridge.bridge-nf-call-iptables =
net.bridge.bridge-nf-call-ip6tables =
[root@k8s01-node01 ~]# systemctl restart docker
7）防火墙问题
$ sudo systemctl daemon-reload
$ sudo systemctl stop firewalld
$ sudo systemctl disable firewalld
$ sudo iptables -F && sudo iptables -X && sudo iptables -F -t nat && sudo iptables -X -t nat #清空以前的防火墙
$ sudo systemctl enable docker
$ sudo systemctl start docker
查看防火墙
[root@k8s01-node01 ~]# iptables -L -n
Chain INPUT (policy ACCEPT)
target prot opt source destination Chain FORWARD (policy ACCEPT)
target prot opt source destination
DOCKER-USER all -- 0.0.0.0/ 0.0.0.0/
DOCKER-ISOLATION-STAGE- all -- 0.0.0.0/ 0.0.0.0/
ACCEPT all -- 0.0.0.0/ 0.0.0.0/ ctstate RELATED,ESTABLISHED
DOCKER all -- 0.0.0.0/ 0.0.0.0/
ACCEPT all -- 0.0.0.0/ 0.0.0.0/
ACCEPT all -- 0.0.0.0/ 0.0.0.0/
ACCEPT all -- 172.30.0.0/ 0.0.0.0/
ACCEPT all -- 0.0.0.0/ 172.30.0.0/ Chain OUTPUT (policy ACCEPT)
target prot opt source destination Chain DOCKER ( references)
target prot opt source destination Chain DOCKER-ISOLATION-STAGE- ( references)
target prot opt source destination
DOCKER-ISOLATION-STAGE- all -- 0.0.0.0/ 0.0.0.0/
RETURN all -- 0.0.0.0/ 0.0.0.0/ Chain DOCKER-ISOLATION-STAGE- ( references)
target prot opt source destination
DROP all -- 0.0.0.0/ 0.0.0.0/
RETURN all -- 0.0.0.0/ 0.0.0.0/ Chain DOCKER-USER ( references)
target prot opt source destination
RETURN all -- 0.0.0.0/ 0.0.0.0/
iptables -L -n
六、安装和配置kubelet
1）node1节点安装kubelet命令
node01执行
[root@k8s01-node01 ~]# ls /usr/k8s/bin/ # 查看kubectl是否有
env.sh flanneld mk-docker-opts.sh
[root@k8s01-node01 ~]# mkdir .kube master01传输文件
[root@master01 ~]# scp /usr/k8s/bin/kubectl root@n1:/usr/k8s/bin/
[root@master01 ~]# scp .kube/config root@n1:/root/.kube/ node01查看
[root@k8s01-node01 ~]# ls /usr/k8s/bin/
env.sh flanneld kubectl mk-docker-opts.sh
[root@k8s01-node01 ~]# ls .kube/
config
[root@k8s01-node01 ~]# kubectl version
Client Version: version.Info{Major:"", Minor:"", GitVersion:"v1.14.2", GitCommit:"66049e3b21efe110454d67df4fa62b08ea79a19b", GitTreeState:"clean", BuildDate:"2019-05-16T16:23:09Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"", Minor:"", GitVersion:"v1.9.10", GitCommit:"098570796b32895c38a9a1c9286425fb1ececa18", GitTreeState:"clean", BuildDate:"2018-08-02T17:11:51Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
2）给特殊用户赋予规则
kubelet 启动时向kube-apiserver 发送TLS bootstrapping 请求，需要先将bootstrap token 文件中的kubelet-bootstrap
用户赋予system:node-bootstrapper 角色，然后kubelet 才有权限创建认证请求(certificatesigningrequests)：
[root@k8s01-node01 ~]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created 另外1. 版本中还需要为Node 请求创建一个RBAC 授权规则：
[root@k8s01-node01 ~]# kubectl create clusterrolebinding kubelet-nodes --clusterrole=system:node --group=system:nodes
clusterrolebinding.rbac.authorization.k8s.io/kubelet-nodes created
3）下载最新的kubelet 和kube-proxy 二进制文件（前面下载kubernetes 目录下面其实也有）
$ wget https://dl.k8s.io/v1.8.2/kubernetes-server-linux-amd64.tar.gz
$ tar -xzvf kubernetes-server-linux-amd64.tar.gz
$ cd kubernetes
$ tar -xzvf kubernetes-src.tar.gz
$ sudo cp -r ./server/bin/{kube-proxy,kubelet} /usr/k8s/bin/
方法二（实际操作）
[root@master01 ~]# ls kubernetes/server/bin/kubelet
kubernetes/server/bin/kubelet
[root@master01 ~]# ls kubernetes/server/bin/kube-proxy
kubernetes/server/bin/kube-proxy
[root@master01 ~]# scp kubernetes/server/bin/kube-proxy root@n1:/usr/k8s/bin/
[root@master01 ~]# scp kubernetes/server/bin/kubelet root@n1:/usr/k8s/bin/
4）创建kubelet bootstapping kubeconfig 文件
$ # 设置集群参数
$ kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=bootstrap.kubeconfig
$ # 设置客户端认证参数
$ kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=bootstrap.kubeconfig
$ # 设置上下文参数
$ kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig
$ # 设置默认上下文
$ kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
$ mv bootstrap.kubeconfig /etc/kubernetes/
操作过程
[root@k8s01-node01 ~]# echo $KUBE_APISERVER
https://k8s-api.virtual.local:6443
[root@k8s01-node01 ~]# kubectl config set-cluster kubernetes \
> --certificate-authority=/etc/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${KUBE_APISERVER} \
> --kubeconfig=bootstrap.kubeconfig
Cluster "kubernetes" set.
[root@k8s01-node01 ~]# echo $BOOTSTRAP_TOKEN
0b340751863956f119cbc624465db92b
[root@k8s01-node01 ~]# kubectl config set-credentials kubelet-bootstrap \
> --token=${BOOTSTRAP_TOKEN} \
> --kubeconfig=bootstrap.kubeconfig
User "kubelet-bootstrap" set.
[root@k8s01-node01 ~]# kubectl config set-context default \
> --cluster=kubernetes \
> --user=kubelet-bootstrap \
> --kubeconfig=bootstrap.kubeconfig
Context "default" created.
[root@k8s01-node01 ~]# kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
Switched to context "default".
[root@k8s01-node01 ~]# ls bootstrap.kubeconfig
bootstrap.kubeconfig
[root@k8s01-node01 ~]# mv bootstrap.kubeconfig /etc/kubernetes/
5）创建kubelet 的systemd unit 文件
5.1）命令行测试是否通过
/usr/k8s/bin/kubelet \
--fail-swap-on=false \
--cgroup-driver=cgroupfs \
--address=${NODE_IP} \
--hostname-override=${NODE_IP} \
--experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
--require-kubeconfig \
--cert-dir=/etc/kubernetes/ssl \
--cluster-dns=${CLUSTER_DNS_SVC_IP} \
--cluster-domain=${CLUSTER_DNS_DOMAIN} \
--hairpin-mode promiscuous-bridge \
--allow-privileged=true \
--serialize-image-pulls=false \
--logtostderr=true \
--v=
5.2）通过则创建文件
$ sudo mkdir /var/lib/kubelet # 必须先创建工作目录
$ cat > kubelet.service <<EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service [Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/k8s/bin/kubelet \\
--fail-swap-on=false \\
--cgroup-driver=cgroupfs \\
--address=${NODE_IP} \\
--hostname-override=${NODE_IP} \\
--experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \\
--require-kubeconfig \\
--cert-dir=/etc/kubernetes/ssl \\
--cluster-dns=${CLUSTER_DNS_SVC_IP} \\
--cluster-domain=${CLUSTER_DNS_DOMAIN} \\
--hairpin-mode promiscuous-bridge \\
--allow-privileged=true \\
--serialize-image-pulls=false \\
--logtostderr=true \\
--v=
Restart=on-failure
RestartSec= [Install]
WantedBy=multi-user.target
EOF
5.3）开机自启动
[root@k8s01-node01 ~]# mv kubelet.service /etc/systemd/system
[root@k8s01-node01 ~]# systemctl daemon-reload
[root@k8s01-node01 ~]# systemctl restart kubelet
[root@k8s01-node01 ~]# systemctl status kubelet
[root@k8s01-node01 ~]# systemctl status kubelet -l 详细情况查看
● kubelet.service - Kubernetes Kubelet
Loaded: loaded (/etc/systemd/system/kubelet.service; enabled; vendor preset: disabled)
Active: active (running) since 二 -- :: CST; 14s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes [root@k8s01-node01 ~]# systemctl enable kubelet # 问题
Failed to execute operation: File exists
6) 通过kubelet 的TLS 证书请求
6.1) 查看未通过时的状态
[root@k8s01-node01 ~]# kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg 29m kubelet-bootstrap Pending
[root@k8s01-node01 ~]# kubectl get nodes
No resources found.
6.2）通过CSR请求
[root@k8s01-node01 ~]# kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg 31m kubelet-bootstrap Pending
[root@k8s01-node01 ~]# kubectl certificate approve node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg # 认证通过的命令
certificatesigningrequest.certificates.k8s.io/node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg approved
[root@k8s01-node01 ~]# kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg 32m kubelet-bootstrap Approved,Issued
[root@k8s01-node01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
192.168.10.23 Ready <none> 3m15s v1.9.10
6.3）此时会生成一些秘钥
[root@k8s01-node01 ~]# ll /etc/kubernetes/ssl/kubelet*
-rw-r--r-- root root 5月 : /etc/kubernetes/ssl/kubelet-client.crt
-rw------- root root 5月 : /etc/kubernetes/ssl/kubelet-client.key
-rw-r--r-- root root 5月 : /etc/kubernetes/ssl/kubelet.crt
-rw------- root root 5月 : /etc/kubernetes/ssl/kubelet.key
七、配置kube-proxy
1）创建kube-proxy 证书签名请求（master操作）
$ cat > kube-proxy-csr.json <<EOF
{
"CN": "system:kube-proxy",
"hosts": [],
"key": {
"algo": "rsa",
"size":
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
2）生成kube-proxy 客户端证书和私钥（master操作）
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
-ca-key=/etc/kubernetes/ssl/ca-key.pem \
-config=/etc/kubernetes/ssl/ca-config.json \
-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
$ ls kube-proxy*
kube-proxy.csr kube-proxy-csr.json kube-proxy-key.pem kube-proxy.pem
$ sudo mv kube-proxy*.pem /etc/kubernetes/ssl/
2.1）实际操作过程。将生成的秘钥发送给了node01
[root@master01 ssl]# cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
> -ca-key=/etc/kubernetes/ssl/ca-key.pem \
> -config=/etc/kubernetes/ssl/ca-config.json \
> -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
// :: [INFO] generate received request
// :: [INFO] received CSR
// :: [INFO] generating key: rsa-
// :: [INFO] encoded CSR
// :: [INFO] signed certificate with serial number
// :: [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1., from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2. ("Information Requirements").
[root@master01 ssl]# ls kube-proxy*
kube-proxy.csr kube-proxy-csr.json kube-proxy-key.pem kube-proxy.pem
[root@master01 ssl]# ls kube-proxy*.pem
kube-proxy-key.pem kube-proxy.pem
[root@master01 ssl]# scp kube-proxy*.pem root@n1:/etc/kubernetes/ssl/
kube-proxy-key.pem % .9KB/s :
kube-proxy.pem
在node01查看
[root@k8s01-node01 ~]# ll /etc/kubernetes/ssl/kube-proxy*.pem
-rw------- root root 5月 : /etc/kubernetes/ssl/kube-proxy-key.pem
-rw-r--r-- root root 5月 : /etc/kubernetes/ssl/kube-proxy.pem
3）创建kube-proxy kubeconfig 文件 （node01操作）
$ # 设置集群参数
$ kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig
$ # 设置客户端认证参数
$ kubectl config set-credentials kube-proxy \
--client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
$ # 设置上下文参数
$ kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
$ # 设置默认上下文
$ kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
$ mv kube-proxy.kubeconfig /etc/kubernetes/
实际操作过程
[root@k8s01-node01 ~]# kubectl config set-cluster kubernetes \
> --certificate-authority=/etc/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${KUBE_APISERVER} \
> --kubeconfig=kube-proxy.kubeconfig
Cluster "kubernetes" set.
[root@k8s01-node01 ~]# kubectl config set-credentials kube-proxy \
> --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
> --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
> --embed-certs=true \
> --kubeconfig=kube-proxy.kubeconfig
User "kube-proxy" set.
[root@k8s01-node01 ~]# kubectl config set-context default \
> --cluster=kubernetes \
> --user=kube-proxy \
> --kubeconfig=kube-proxy.kubeconfig
Context "default" created.
[root@k8s01-node01 ~]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
Switched to context "default".
[root@k8s01-node01 ~]# ls kube-proxy.kubeconfig
kube-proxy.kubeconfig
[root@k8s01-node01 ~]# mv kube-proxy.kubeconfig /etc/kubernetes/
4）创建kube-proxy 的systemd unit 文件
4.1）命令行执行判断是否执行成功
/usr/k8s/bin/kube-proxy \
--bind-address=${NODE_IP} \
--hostname-override=${NODE_IP} \
--cluster-cidr=${SERVICE_CIDR} \
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \
--logtostderr=true \
--v=
4.2）创建该文件
$ sudo mkdir -p /var/lib/kube-proxy # 必须先创建工作目录
$ cat > kube-proxy.service <<EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target [Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/k8s/bin/kube-proxy \\
--bind-address=${NODE_IP} \\
--hostname-override=${NODE_IP} \\
--cluster-cidr=${SERVICE_CIDR} \\
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \\
--logtostderr=true \\
--v=
Restart=on-failure
RestartSec=
LimitNOFILE= [Install]
WantedBy=multi-user.target
EOF
5）设置开机自启动
[root@k8s01-node01 ~]# mv kube-proxy.service /etc/systemd/system
[root@k8s01-node01 ~]# systemctl daemon-reload
[root@k8s01-node01 ~]# systemctl enable kube-proxy
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /etc/systemd/system/kube-proxy.service.
[root@k8s01-node01 ~]# systemctl start kube-proxy
[root@k8s01-node01 ~]# systemctl status kube-proxy
● kube-proxy.service - Kubernetes Kube-Proxy Server
Loaded: loaded (/etc/systemd/system/kube-proxy.service; enabled; vendor preset: disabled)
Active: active (running) since 二 -- :: CST; 12s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
6）此时防火墙写入了新的规则
[root@k8s01-node01 ~]# iptables -L -n
Chain INPUT (policy ACCEPT)
target prot opt source destination
KUBE-SERVICES all -- 0.0.0.0/ 0.0.0.0/ /* kubernetes service portals */
KUBE-FIREWALL all -- 0.0.0.0/ 0.0.0.0/ Chain FORWARD (policy ACCEPT)
target prot opt source destination
KUBE-FORWARD all -- 0.0.0.0/ 0.0.0.0/ /* kubernetes forward rules */
DOCKER-USER all -- 0.0.0.0/ 0.0.0.0/
DOCKER-ISOLATION-STAGE- all -- 0.0.0.0/ 0.0.0.0/
ACCEPT all -- 0.0.0.0/ 0.0.0.0/ ctstate RELATED,ESTABLISHED
DOCKER all -- 0.0.0.0/ 0.0.0.0/
ACCEPT all -- 0.0.0.0/ 0.0.0.0/
ACCEPT all -- 0.0.0.0/ 0.0.0.0/
ACCEPT all -- 172.30.0.0/ 0.0.0.0/
ACCEPT all -- 0.0.0.0/ 172.30.0.0/ Chain OUTPUT (policy ACCEPT)
target prot opt source destination
KUBE-SERVICES all -- 0.0.0.0/ 0.0.0.0/ /* kubernetes service portals */
KUBE-FIREWALL all -- 0.0.0.0/ 0.0.0.0/ Chain DOCKER ( references)
target prot opt source destination Chain DOCKER-ISOLATION-STAGE- ( references)
target prot opt source destination
DOCKER-ISOLATION-STAGE- all -- 0.0.0.0/ 0.0.0.0/
RETURN all -- 0.0.0.0/ 0.0.0.0/ Chain DOCKER-ISOLATION-STAGE- ( references)
target prot opt source destination
DROP all -- 0.0.0.0/ 0.0.0.0/
RETURN all -- 0.0.0.0/ 0.0.0.0/ Chain DOCKER-USER ( references)
target prot opt source destination
RETURN all -- 0.0.0.0/ 0.0.0.0/ Chain KUBE-FIREWALL ( references)
target prot opt source destination
DROP all -- 0.0.0.0/ 0.0.0.0/ /* kubernetes firewall for dropping marked packets */ mark match 0x8000/0x8000 Chain KUBE-FORWARD ( references)
target prot opt source destination
ACCEPT all -- 0.0.0.0/ 0.0.0.0/ /* kubernetes forwarding rules */ mark match 0x4000/0x4000
ACCEPT all -- 10.254.0.0/ 0.0.0.0/ /* kubernetes forwarding conntrack pod source rule */ ctstate RELATED,ESTABLISHED
ACCEPT all -- 0.0.0.0/ 10.254.0.0/ /* kubernetes forwarding conntrack pod destination rule */ ctstate RELATED,ESTABLISHED Chain KUBE-SERVICES ( references)
target prot opt source destination
iptables -L -n
7）对节点是否显示主机名的问题
[root@k8s01-node01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
192.168.10.23 Ready <none> 50m v1.9.10
如果这里的名字想显示为主机名时：
[root@k8s01-node01 ~]# cat /etc/systemd/system/kubelet.service |grep hostname
--hostname-override=192.168.10.23 去掉该参数
[root@k8s01-node01 ~]# cat /etc/systemd/system/kube-proxy.service |grep hostname
--hostname-override=192.168.10.23 去掉该参数
八、验证节点功能
1）定义yaml 文件：（将下面内容保存为：nginx-ds.yaml）
apiVersion: v1
kind: Service
metadata:
name: nginx-ds
labels:
app: nginx-ds
spec:
type: NodePort
selector:
app: nginx-ds
ports:
- name: http
port:
targetPort:
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: nginx-ds
labels:
addonmanager.kubernetes.io/mode: Reconcile
spec:
template:
metadata:
labels:
app: nginx-ds
spec:
containers:
- name: my-nginx
image: nginx:1.7.
ports:
- containerPort:
nginx-ds.yaml
2）创建该pods
[root@k8s01-node01 ~]# kubectl create -f nginx-ds.yaml
service/nginx-ds created
daemonset.extensions/nginx-ds created
[root@k8s01-node01 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
nginx-ds-txxnb / ContainerCreating 2m35s
kind: DaemonSet 参数是在每个节点启动一个pod
[root@k8s01-node01 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
nginx-ds-txxnb / ContainerCreating 2m35s
[root@k8s01-node01 ~]# kubectl describe pod nginx-ds-txxnb
.......
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal SuccessfulMountVolume 7m25s kubelet, 192.168.10.23 MountVolume.SetUp succeeded for volume "default-token-7l55q"
Warning FailedCreatePodSandBox 15s (x16 over 7m10s) kubelet, 192.168.10.23 Failed create pod sandbox.
[root@k8s01-node01 ~]# journalctl -u kubelet -f 更详细的查看报错
5月 :: k8s01-node01 kubelet[]: E0528 ::29.924991
kuberuntime_manager.go:] createPodSandbox for pod "nginx-ds-txxnb_default(004fdd27-8148-11e9-8809-000c29a2d5b5)"
failed: rpc error: code = Unknown desc = failed pulling image "gcr.io/google_containers/pause-amd64:3.0":
Error response from daemon: Get https://gcr.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
3）原因，从国外源无法拉取镜像。更改镜像源
原因，从国外源无法拉取镜像
[root@k8s01-node01 ~]# vim /etc/systemd/system/kubelet.service 修改镜像地址
......
[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/k8s/bin/kubelet \
--fail-swap-on=false \
--cgroup-driver=cgroupfs \
--address=192.168.10.23 \
--hostname-override=192.168.10.23 \
--pod-infra-container-image=cnych/pause-amd64:3.0 \ # 新增内容
[root@k8s01-node01 ~]# systemctl daemon-reload
[root@k8s01-node01 ~]# systemctl restart kubelet
[root@k8s01-node01 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
cnych/pause-amd64 3.0 99e59f495ffa years ago 747kB
nginx 1.7. 84581e99d807 years ago .7MB
[root@k8s01-node01 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
nginx-ds-txxnb / Running 14h
4）访问
[root@k8s01-node01 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-ds-txxnb / Running 14h 172.30.23.2 192.168.10.23 <none> <none>
[root@k8s01-node01 ~]# curl 172.30.23.2
[root@k8s01-node01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.254.0.1 <none> /TCP 38h
nginx-ds NodePort 10.254.66.68 <none> :/TCP 14h
[root@k8s01-node01 ~]# netstat -lntup|grep
tcp6 ::: :::* LISTEN /kube-proxy
[root@k8s01-node01 ~]# curl 192.168.10.23:
九、部署kubedns 插件
1）下载，修改文件
wget https://raw.githubusercontent.com/kubernetes/kubernetes/v1.9.3/cluster/addons/dns/kube-dns.yaml.base
cp kube-dns.yaml.base kube-dns.yaml
改好的文件
# Copyright The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. # Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml
# in sync with this file. # __MACHINE_GENERATED_WARNING__ apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "KubeDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.254.0.2
ports:
- name: dns
port:
protocol: UDP
- name: dns-tcp
port:
protocol: TCP
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-dns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kube-dns
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: EnsureExists
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: kube-dns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
# replicas: not specified here:
# . In order to make Addon Manager do not reconcile this replicas parameter.
# . Default is .
# . Will be tuned in real time if DNS horizontal auto-scaling is turned on.
strategy:
rollingUpdate:
maxSurge: %
maxUnavailable:
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
volumes:
- name: kube-dns-config
configMap:
name: kube-dns
optional: true
containers:
- name: kubedns
image: cnych/k8s-dns-kube-dns-amd64:1.14.
resources:
# TODO: Set memory limits when we've profiled the container for large
# clusters, then set request = limit to keep this container in
# guaranteed class. Currently, this container falls into the
# "burstable" category so the kubelet doesn't backoff from restarting it.
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
livenessProbe:
httpGet:
path: /healthcheck/kubedns
port:
scheme: HTTP
initialDelaySeconds:
timeoutSeconds:
successThreshold:
failureThreshold:
readinessProbe:
httpGet:
path: /readiness
port:
scheme: HTTP
# we poll on pod startup for the Kubernetes master service and
# only setup the /readiness HTTP server once that's available.
initialDelaySeconds:
timeoutSeconds:
args:
- --domain=cluster.local.
- --dns-port=
- --config-dir=/kube-dns-config
- --v=
env:
- name: PROMETHEUS_PORT
value: ""
ports:
- containerPort:
name: dns-local
protocol: UDP
- containerPort:
name: dns-tcp-local
protocol: TCP
- containerPort:
name: metrics
protocol: TCP
volumeMounts:
- name: kube-dns-config
mountPath: /kube-dns-config
- name: dnsmasq
image: cnych/k8s-dns-dnsmasq-nanny-amd64:1.14.
livenessProbe:
httpGet:
path: /healthcheck/dnsmasq
port:
scheme: HTTP
initialDelaySeconds:
timeoutSeconds:
successThreshold:
failureThreshold:
args:
- -v=
- -logtostderr
- -configDir=/etc/k8s/dns/dnsmasq-nanny
- -restartDnsmasq=true
- --
- -k
- --cache-size=
- --no-negcache
- --log-facility=-
- --server=/cluster.local/127.0.0.1#
- --server=/in-addr.arpa/127.0.0.1#
- --server=/ip6.arpa/127.0.0.1#
ports:
- containerPort:
name: dns
protocol: UDP
- containerPort:
name: dns-tcp
protocol: TCP
# see: https://github.com/kubernetes/kubernetes/issues/29055 for details
resources:
requests:
cpu: 150m
memory: 20Mi
volumeMounts:
- name: kube-dns-config
mountPath: /etc/k8s/dns/dnsmasq-nanny
- name: sidecar
image: cnych/k8s-dns-sidecar-amd64:1.14.
livenessProbe:
httpGet:
path: /metrics
port:
scheme: HTTP
initialDelaySeconds:
timeoutSeconds:
successThreshold:
failureThreshold:
args:
- --v=
- --logtostderr
- --probe=kubedns,127.0.0.1:,kubernetes.default.svc.cluster.local.,,SRV
- --probe=dnsmasq,127.0.0.1:,kubernetes.default.svc.cluster.local.,,SRV
ports:
- containerPort:
name: metrics
protocol: TCP
resources:
requests:
memory: 20Mi
cpu: 10m
dnsPolicy: Default # Don't use cluster DNS.
serviceAccountName: kube-dns
kube-dns.yaml
对比区别
[root@k8s01-node01 ~]# diff kube-dns.yaml kube-dns.yaml.base
33c33
< clusterIP: 10.254.0.2
---
> clusterIP: __PILLAR__DNS__SERVER__
97c97
< image: cnych/k8s-dns-kube-dns-amd64:1.14.
---
> image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.
127c127
< - --domain=cluster.local.
---
> - --domain=__PILLAR__DNS__DOMAIN__.
148c148
< image: cnych/k8s-dns-dnsmasq-nanny-amd64:1.14.
---
> image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.
168c168
< - --server=/cluster.local/127.0.0.1#
---
> - --server=/__PILLAR__DNS__DOMAIN__/127.0.0.1#
187c187
< image: cnych/k8s-dns-sidecar-amd64:1.14.
---
> image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.
,201c200,
< - --probe=kubedns,127.0.0.1:,kubernetes.default.svc.cluster.local.,,SRV
< - --probe=dnsmasq,127.0.0.1:,kubernetes.default.svc.cluster.local.,,SRV
---
> - --probe=kubedns,127.0.0.1:,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__,,SRV
> - --probe=dnsmasq,127.0.0.1:,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__,,SRV
2）启动查看运行状态
[root@k8s01-node01 ~]# kubectl create -f kube-dns.yaml
service/kube-dns created
serviceaccount/kube-dns created
configmap/kube-dns created
deployment.extensions/kube-dns created
[root@k8s01-node01 ~]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
kube-dns-77d877c467-j5q2h / Running 44s
[root@k8s01-node01 ~]# kubectl describe pod kube-dns-77d877c467-j5q2h -n kube-system
[root@k8s01-node01 ~]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kube-dns ClusterIP 10.254.0.2 <none> /UDP,/TCP 2m21s
3）验证kubedns功能
3.1）新建一个Deployment
cat > my-nginx.yaml<<EOF
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: my-nginx
spec:
replicas:
template:
metadata:
labels:
run: my-nginx
spec:
containers:
- name: my-nginx
image: nginx:1.7.
ports:
- containerPort:
EOF
my-nginx.yaml
启动服务
[root@k8s01-node01 ~]# kubectl create -f my-nginx.yaml
deployment.extensions/my-nginx created
[root@k8s01-node01 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
my-nginx-56b48db847-945bx / Running 13s
my-nginx-56b48db847-tdlrd / Running 13s
Expose 该Deployment，生成my-nginx 服务
[root@k8s01-node01 ~]# kubectl expose deploy my-nginx
service/my-nginx exposed
[root@k8s01-node01 ~]# kubectl get svc|grep my-nginx
my-nginx ClusterIP 10.254.57.19 <none> /TCP 40s
3.2）创建另外一个Pod
然后创建另外一个Pod，查看/etc/resolv.conf是否包含kubelet配置的--cluster-dns 和--cluster-domain，
是否能够将服务my-nginx 解析到上面显示的CLUSTER-IP 10.254..19上
pod内容
$ cat > pod-nginx.yaml<<EOF
apiVersion: v1
kind: Pod
metadata:
name: nginx
spec:
containers:
- name: nginx
image: nginx:1.7.
ports:
- containerPort:
EOF
pod-nginx.yaml
创建pod，并验证dns服务
[root@k8s01-node01 ~]# kubectl create -f pod-nginx.yaml
pod/nginx created
[root@k8s01-node01 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
my-nginx-56b48db847-945bx / Running 3m31s
my-nginx-56b48db847-tdlrd / Running 3m31s
nginx / Running 35s
nginx-ds-txxnb / Running 37h
[root@k8s01-node01 ~]# kubectl exec nginx -i -t -- /bin/bash
root@nginx:/# cat /etc/resolv.conf
nameserver 10.254.0.2
search default.svc.cluster.local. svc.cluster.local. cluster.local.
options ndots:
十、部署Dashboard 插件
1）下载kubernetes-dashboard.yaml文件，修改文件
wget https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml
修改的文件
# Copyright The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. # ------------------- Dashboard Secret ------------------- # apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kube-system
type: Opaque ---
# ------------------- Dashboard Service Account ------------------- # apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system ---
# ------------------- Dashboard Role & Role Binding ------------------- # kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kubernetes-dashboard-minimal
namespace: kube-system
rules:
# Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create"]
# Allow Dashboard to create 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["create"]
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"]
verbs: ["get", "update", "delete"]
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
# Allow Dashboard to get metrics from heapster.
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:"]
verbs: ["get"] ---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubernetes-dashboard-minimal
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard-minimal
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kube-system ---
# ------------------- Dashboard Deployment ------------------- # kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
replicas:
revisionHistoryLimit:
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: cnych/kubernetes-dashboard-amd64:v1.8.3
ports:
- containerPort:
protocol: TCP
args:
- --auto-generate-certificates
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
# - --apiserver-host=http://my-address:port
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port:
initialDelaySeconds:
timeoutSeconds:
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule ---
# ------------------- Dashboard Service ------------------- # kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
type: NodePort
ports:
- port:
targetPort:
selector:
k8s-app: kubernetes-dashboard
kubernetes-dashboard.yaml
对比修改的内容
[root@k8s01-node01 ~]# diff kubernetes-dashboard.yaml kubernetes-dashboard.yaml.bak
112c112
< image: cnych/kubernetes-dashboard-amd64:v1.8.3
---
> image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1
158d157
< type: NodePort
2）启动服务
[root@k8s01-node01 ~]# kubectl create -f kubernetes-dashboard.yaml
secret/kubernetes-dashboard-certs created
serviceaccount/kubernetes-dashboard created
role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
deployment.apps/kubernetes-dashboard created
service/kubernetes-dashboard created
[root@k8s01-node01 ~]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
kube-dns-77d877c467-j5q2h / Running 27m
kubernetes-dashboard-6857cb46c4-bd4s5 / Running 35s
[root@k8s01-node01 ~]# kubectl get svc -n kube-system|grep kubernetes-dashboard
kubernetes-dashboard NodePort 10.254.207.13 <none> :/TCP 78s
3）火狐浏览器访问 https://192.168.10.23:31569/
4）创建身份认证文件。https://www.qikqiak.com/post/update-kubernetes-dashboard-more-secure/
[root@k8s01-node01 ~]# vim admin-sa.yaml
[root@k8s01-node01 ~]# cat admin-sa.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: admin
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: admin
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
admin-sa.yaml
创建该yaml文件
[root@k8s01-node01 ~]# kubectl create -f admin-sa.yaml
clusterrolebinding.rbac.authorization.k8s.io/admin created
serviceaccount/admin created
5）上面的admin用户创建完成后我们就可以获取到该用户对应的token了，如下命令：
$ kubectl get secret -n kube-system|grep admin-token
admin-token-d5jsg kubernetes.io/service-account-token 1d
$ kubectl get secret admin-token-d5jsg -o jsonpath={.data.token} -n kube-system |base64 -d
# 会生成一串很长的base64后的字符串
示例
[root@k8s01-node01 ~]# kubectl get secret -n kube-system|grep admin-token
admin-token-8nfs6 kubernetes.io/service-account-token 9m36s
[root@k8s01-node01 ~]# kubectl get secret admin-token-8nfs6 -o jsonpath={.data.token} -n kube-system |base64 -d
eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi10b2tlbi04bmZzNiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJhZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjQ3Y2FlNzg2LTgyODctMTFlOS04ODA5LTAwMGMyOWEyZDViNSIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTphZG1pbiJ9.wLxu87Kn5B4cD3obq8u8kB0cNT49l6om2zWqP14S3KcCigqbzMsvQkf07EG6cots1l9xYomgZQhrNuKR3_xiaksoRaLeREs_JLAbCqXeSPg10XfqHvVVEHIgL6d3r5ujBZza2dOGjN8L4V6q38Xkt2rIfykqX1gB8x9EkVeseAc25IN3h5l7l0SWpuEE_nrY4FZc0FjXxPu5zczjelLFQsUHLf3MwCw1VPpyiQadv_oy9yUCyWXvveCEhHKY3scGHkEr8YZ0zsZYerTe16_ncQn2AKutkwrJWNdBFjF8O53izq6LJ7a2E5a-hVohLB9XJtXJYqlQj3BIPBYJVd_6zQ[root@k8s01-node01 ~]#
拷贝很长的字符串秘钥
6）部署Heapster插件 。测试存在问题
https://github.com/kubernetes-retired/heapster/releases
wget https://github.com/kubernetes-retired/heapster/archive/v1.5.4.tar.gz
教学版本测试
wget https://raw.githubusercontent.com/kubernetes/dashboard/v1.8.2/src/deploy/recommended/kubernetes-dashboard.yaml
wget https://github.com/kubernetes-retired/heapster/archive/v1.5.1.tar.gz
）修改镜像
[root@master ~]# cat heapster-1.5./deploy/kube-config/influxdb/grafana.yaml |grep image
image: cnych/heapster-grafana-amd64:v4.4.3
[root@master ~]# cat heapster-1.5./deploy/kube-config/influxdb/heapster.yaml |grep image
image: cnych/heapster-amd64:v1.5.1
imagePullPolicy: IfNotPresent
[root@master ~]# cat heapster-1.5./deploy/kube-config/influxdb/influxdb.yaml |grep image
image: cnych/heapster-influxdb-amd64:v1.3.3
）调试
vim heapster-1.5./deploy/kube-config/influxdb/grafana.yaml
spec:
# In a production setup, we recommend accessing Grafana through an external Loadbalancer
# or through a public IP.
# type: LoadBalancer
# You could also use NodePort to expose the service at a randomly-generated port
type: NodePort # 去掉注释 [root@master dashboard]# vim kubernetes-dashboard.yaml
args:
- --auto-generate-certificates
- --heapster-host=http://heapster # 新增，前提是dashboard和heapster 在同一个名称空间
- --heapster-host=http://heapster.kube-system # 如果在其他名称空间
dashboard_v1.8.2和heapster_v1.5.1
6.1）修改镜像地址，启动
[root@k8s01-node01 ~]# tar xf v1.5.4.tar.gz
[root@k8s01-node01 kube-config]# pwd
/root/heapster-1.5./deploy/kube-config
[root@k8s01-node01 kube-config]# kubectl create -f rbac/heapster-rbac.yaml
clusterrolebinding "heapster" created
[root@k8s01-node01 kube-config]# kubectl apply -f influxdb/ # 需要修改镜像地址
deployment "monitoring-grafana" created
service "monitoring-grafana" created
serviceaccount "heapster" created
deployment "heapster" created
service "heapster" created
deployment "monitoring-influxdb" created
service "monitoring-influxdb" created
6.2）查看是否运行正常
[root@k8s01-node01 kube-config]# kubectl get deployments -n kube-system
NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
heapster 1m
kube-dns 8h
kubernetes-dashboard 8h
monitoring-grafana 1m
monitoring-influxdb 1m
[root@k8s01-node01 kube-config]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
heapster-b559f89b5-wfcz5 / Running 38s
kube-dns-77d877c467-j5q2h / Running 8h
kubernetes-dashboard-6857cb46c4-bd4s5 / Running 8h
monitoring-grafana-5d48df8df5-hgcz9 / Running 38s
monitoring-influxdb-6d88965c6f-t7tlj / Running 38s
6.3）修改kubernetes-dashboard.yaml 配合Heapster
[root@k8s01-node01 ~]# vim kubernetes-dashboard.yaml
..........
args:
- --auto-generate-certificates
- --heapster-host=http://heapster # 添加 [root@k8s01-node01 ~]# kubectl apply -f kubernetes-dashboard.yaml
查看效果
7）grafana改为type: NodePort
[root@k8s01-node01 kube-config]# vim influxdb/grafana.yaml
...........
spec:
# In a production setup, we recommend accessing Grafana through an external Loadbalancer
# or through a public IP.
# type: LoadBalancer
# You could also use NodePort to expose the service at a randomly-generated port
type: NodePort # 去掉注释
ports:
- port:
targetPort:
7.1）查看默认生成的端口
[root@k8s01-node01 kube-config]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
heapster ClusterIP 10.254.230.254 <none> /TCP 27m
kube-dns ClusterIP 10.254.0.2 <none> /UDP,/TCP 9h
kubernetes-dashboard NodePort 10.254.20.231 <none> :/TCP 17m
monitoring-grafana NodePort 10.254.87.101 <none> :/TCP 27m
monitoring-influxdb ClusterIP 10.254.58.122 <none> /TCP 27m
访问：http://192.168.10.23:31703
十一、部署traefik
1）简介
Traefik是一款开源的反向代理与负载均衡工具。它最大的优点是能够与常见的微服务系统直接整合，可以实现自动化动态配置。
目前支持Docker、Swarm、Mesos/Marathon、 Mesos、Kubernetes、Consul、Etcd、Zookeeper、BoltDB、Rest API等等后端模型。
2）编辑文件
apiVersion: v1
kind: ServiceAccount
metadata:
name: ingress
namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: ingress
subjects:
- kind: ServiceAccount
name: ingress
namespace: kube-system
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
---
kind: ConfigMap
apiVersion: v1
metadata:
name: traefik-conf
namespace: kube-system
data:
traefik-config: |-
defaultEntryPoints = ["http"]
[entryPoints]
[entryPoints.http]
address = ":80" ---
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
name: traefik-ingress
namespace: kube-system
labels:
k8s-app: traefik-ingress
spec:
template:
metadata:
labels:
k8s-app: traefik-ingress
name: traefik-ingress
spec:
terminationGracePeriodSeconds:
restartPolicy: Always
serviceAccountName: ingress
containers:
- image: traefik:1.5.
name: traefik-ingress
ports:
- name: http
containerPort:
hostPort:
- name: https
containerPort:
hostPort:
- name: admin
containerPort:
args:
- --configFile=/etc/traefik/traefik.toml
- -d
- --web
- --kubernetes
- --logLevel=DEBUG
volumeMounts:
- name: traefik-config-volume
mountPath: /etc/traefik
volumes:
- name: traefik-config-volume
configMap:
name: traefik-conf
items:
- key: traefik-config
path: traefik.toml
traefik-daemonset.yaml
3）启动服务
[root@k8s01-node01 ~]# kubectl create -f traefik-daemonset.yaml
serviceaccount "ingress" created
clusterrolebinding "ingress" created
configmap "traefik-conf" created
daemonset "traefik-ingress" created
[root@k8s01-node01 ~]# kubectl get pods -n kube-system|grep traefik
traefik-ingress-frtnn / Running 37s
