部署控制面组件

    kube-controller-manager

    2. $ kubectl config set-credentials system:kube-controller-manager --client-certificate=/etc/kubernetes/pki/kube-controller-manager.pem --client-key=/etc/kubernetes/pki/kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig
    3. $ kubectl config set-context default --cluster=openeuler-k8s --user=system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
    4. $ kubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig
    1. $ kubectl config set-cluster openeuler-k8s --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://127.0.0.1:6443 --kubeconfig=kube-scheduler.kubeconfig
    2. $ kubectl config set-credentials system:kube-scheduler --client-certificate=/etc/kubernetes/pki/kube-scheduler.pem --client-key=/etc/kubernetes/pki/kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig
    3. $ kubectl config set-context default  --cluster=openeuler-k8s --user=system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
    4. $ kubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig

    admin

    1. $ kubectl config set-cluster openeuler-k8s --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://127.0.0.1:6443 --kubeconfig=admin.kubeconfig
    2. $ kubectl config set-credentials admin --client-certificate=/etc/kubernetes/pki/admin.pem --client-key=/etc/kubernetes/pki/admin-key.pem --embed-certs=true --kubeconfig=admin.kubeconfig
    3. $ kubectl config set-context default --cluster=openeuler-k8s --user=admin --kubeconfig=admin.kubeconfig
    4. $ kubectl config use-context default --kubeconfig=admin.kubeconfig
    1. admin.kubeconfig kube-proxy.kubeconfig  kube-controller-manager.kubeconfig  kube-scheduler.kubeconfig

    生成密钥提供者的配置

    拷贝证书

    本文把所有组件使用的证书、密钥以及配置统一放到/etc/kubernetes/pki/目录下。

    1. # 准备证书目录
    2. $ mkdir -p /etc/kubernetes/pki/
    3. $ ls /etc/kubernetes/pki/
    4. admin-key.pem  encryption-config.yaml              kube-proxy-key.pem     kubernetes.pem             service-account-key.pem
    5. admin.pem      kube-controller-manager-key.pem     kube-proxy.kubeconfig  kube-scheduler-key.pem     service-account.pem
    6. ca-key.pem     kube-controller-manager.kubeconfig  kube-proxy.pem         kube-scheduler.kubeconfig
    7. ca.pem         kube-controller-manager.pem         kubernetes-key.pem     kube-scheduler.pem

    使能 admin role

    1. $ cat admin_cluster_role.yaml
    2. apiVersion: rbac.authorization.k8s.io/v1
    3. kind: ClusterRole
    4. metadata:
    5.   annotations:
    6.     rbac.authorization.kubernetes.io/autoupdate: "true"
    7.   labels:
    8.     kubernetes.io/bootstrapping: rbac-defaults
    9.   name: system:kube-apiserver-to-kubelet
    10. rules:
    11.   - apiGroups:
    12.       - ""
    13.     resources:
    14.       - nodes/proxy
    15.       - nodes/stats
    16.       - nodes/log
    17.       - nodes/spec
    18.       - nodes/metrics
    19.     verbs:
    20.       - "*"
    22. # 使能admin role 
    23. $ kubectl apply --kubeconfig admin.kubeconfig -f admin_cluster_role.yaml
    1. $ cat admin_cluster_rolebind.yaml
    2. apiVersion: rbac.authorization.k8s.io/v1
    3. kind: ClusterRoleBinding
    4.   name: system:kube-apiserver
    5.   namespace: ""
    6. roleRef:
    7.   apiGroup: rbac.authorization.k8s.io
    8.   kind: ClusterRole
    9.   name: system:kube-apiserver-to-kubelet
    10. subjects:
    11.   - apiGroup: rbac.authorization.k8s.io
    12.     kind: User
    13.     name: kubernetes
    15. # 绑定admin role
    16. $ kubectl apply --kubeconfig admin.kubeconfig -f admin_cluster_rolebind.yaml

    部署 api server 服务

    修改 apiserver 的 etc 配置文件：

    1. $ cat /etc/kubernetes/apiserver
    2. KUBE_ADVERTIS_ADDRESS="--advertise-address=192.168.122.154"
    3. KUBE_ALLOW_PRIVILEGED="--allow-privileged=true"
    4. KUBE_AUTHORIZATION_MODE="--authorization-mode=Node,RBAC"
    5. KUBE_ENABLE_ADMISSION_PLUGINS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
    6. KUBE_SECURE_PORT="--secure-port=6443"
    7. KUBE_ENABLE_BOOTSTRAP_TOKEN_AUTH="--enable-bootstrap-token-auth=true"
    8. KUBE_ETCD_CAFILE="--etcd-cafile=/etc/kubernetes/pki/ca.pem"
    9. KUBE_ETCD_CERTFILE="--etcd-certfile=/etc/kubernetes/pki/kubernetes.pem"
    10. KUBE_ETCD_KEYFILE="--etcd-keyfile=/etc/kubernetes/pki/kubernetes-key.pem"
    11. KUBE_ETCD_SERVERS="--etcd-servers=https://192.168.122.154:2379,https://192.168.122.155:2379,https://192.168.122.156:2379"
    12. KUBE_CLIENT_CA_FILE="--client-ca-file=/etc/kubernetes/pki/ca.pem"
    13. KUBE_KUBELET_CERT_AUTH="--kubelet-certificate-authority=/etc/kubernetes/pki/ca.pem"
    14. KUBE_KUBELET_CLIENT_CERT="--kubelet-client-certificate=/etc/kubernetes/pki/kubernetes.pem"
    15. KUBE_KUBELET_CLIENT_KEY="--kubelet-client-key=/etc/kubernetes/pki/kubernetes-key.pem"
    16. KUBE_KUBELET_HTTPS="--kubelet-https=true"
    17. KUBE_PROXY_CLIENT_CERT_FILE="--proxy-client-cert-file=/etc/kubernetes/pki/kube-proxy.pem"
    18. KUBE_PROXY_CLIENT_KEY_FILE="--proxy-client-key-file=/etc/kubernetes/pki/kube-proxy-key.pem"
    19. KUBE_TLS_CERT_FILE="--tls-cert-file=/etc/kubernetes/pki/kubernetes.pem"
    20. KUBE_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/etc/kubernetes/pki/kubernetes-key.pem"
    21. KUBE_SERVICE_CLUSTER_IP_RANGE="--service-cluster-ip-range=10.32.0.0/16"
    22. KUBE_SERVICE_ACCOUNT_ISSUER="--service-account-issuer=https://kubernetes.default.svc.cluster.local"
    23. KUBE_SERVICE_ACCOUNT_KEY_FILE="--service-account-key-file=/etc/kubernetes/pki/service-account.pem"
    24. KUBE_SERVICE_ACCOUNT_SIGN_KEY_FILE="--service-account-signing-key-file=/etc/kubernetes/pki/service-account-key.pem"
    25. KUBE_SERVICE_NODE_PORT_RANGE="--service-node-port-range=30000-32767"
    26. KUB_ENCRYPTION_PROVIDER_CONF="--encryption-provider-config=/etc/kubernetes/pki/encryption-config.yaml"
    27. KUBE_REQUEST_HEADER_ALLOWED_NAME="--requestheader-allowed-names=front-proxy-client"
    28. KUBE_REQUEST_HEADER_EXTRA_HEADER_PREF="--requestheader-extra-headers-prefix=X-Remote-Extra-"
    29. KUBE_REQUEST_HEADER_GROUP_HEADER="--requestheader-group-headers=X-Remote-Group"
    30. KUBE_REQUEST_HEADER_USERNAME_HEADER="--requestheader-username-headers=X-Remote-User"
    31. KUBE_API_ARGS=""

    所有apiserver的配置都/etc/kubernetes/config文件中定义，然后在后面的service文件中直接使用即可。

    • --service-cluster-ip-range该地址需要和后面的设置的clusterDNS需要一致；

    编写 apiserver 的 systemd 配置

    部署 controller-manager 服务

    修改 controller-manager 配置文件：

    1. $ cat /etc/kubernetes/controller-manager
    2. KUBE_BIND_ADDRESS="--bind-address=127.0.0.1"
    3. KUBE_CLUSTER_CIDR="--cluster-cidr=10.200.0.0/16"
    4. KUBE_CLUSTER_NAME="--cluster-name=kubernetes"
    5. KUBE_CLUSTER_SIGNING_CERT_FILE="--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem"
    6. KUBE_CLUSTER_SIGNING_KEY_FILE="--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem"
    7. KUBE_KUBECONFIG="--kubeconfig=/etc/kubernetes/pki/kube-controller-manager.kubeconfig"
    8. KUBE_LEADER_ELECT="--leader-elect=true"
    9. KUBE_ROOT_CA_FILE="--root-ca-file=/etc/kubernetes/pki/ca.pem"
    10. KUBE_SERVICE_ACCOUNT_PRIVATE_KEY_FILE="--service-account-private-key-file=/etc/kubernetes/pki/service-account-key.pem"
    11. KUBE_SERVICE_CLUSTER_IP_RANGE="--service-cluster-ip-range=10.32.0.0/24"
    12. KUBE_USE_SERVICE_ACCOUNT_CRED="--use-service-account-credentials=true"
    13. KUBE_CONTROLLER_MANAGER_ARGS="--v=2"
    1. $ cat /usr/lib/systemd/system/kube-controller-manager.service
    2. [Unit]
    3. Description=Kubernetes Controller Manager
    4. Documentation=https://kubernetes.io/docs/reference/generated/kube-controller-manager/
    6. EnvironmentFile=-/etc/kubernetes/config
    7. EnvironmentFile=-/etc/kubernetes/controller-manager
    8. ExecStart=/usr/bin/kube-controller-manager \
    10.         $KUBE_LOGTOSTDERR \
    11.         $KUBE_LOG_LEVEL \
    12.         $KUBE_CLUSTER_CIDR \
    13.         $KUBE_CLUSTER_NAME \
    14.         $KUBE_CLUSTER_SIGNING_CERT_FILE \
    15.         $KUBE_CLUSTER_SIGNING_KEY_FILE \
    16.         $KUBE_KUBECONFIG \
    17.         $KUBE_LEADER_ELECT \
    18.         $KUBE_ROOT_CA_FILE \
    19.         $KUBE_SERVICE_ACCOUNT_PRIVATE_KEY_FILE \
    20.         $KUBE_SERVICE_CLUSTER_IP_RANGE \
    21.         $KUBE_USE_SERVICE_ACCOUNT_CRED \
    22.         $KUBE_CONTROLLER_MANAGER_ARGS
    23. Restart=on-failure
    24. LimitNOFILE=65536
    26. [Install]
    27. WantedBy=multi-user.target

    修改 scheduler 配置文件：

    1. $ cat /etc/kubernetes/scheduler
    2. KUBE_CONFIG="--kubeconfig=/etc/kubernetes/pki/kube-scheduler.kubeconfig"
    3. KUBE_AUTHENTICATION_KUBE_CONF="--authentication-kubeconfig=/etc/kubernetes/pki/kube-scheduler.kubeconfig"
    4. KUBE_AUTHORIZATION_KUBE_CONF="--authorization-kubeconfig=/etc/kubernetes/pki/kube-scheduler.kubeconfig"
    5. KUBE_BIND_ADDR="--bind-address=127.0.0.1"
    6. KUBE_LEADER_ELECT="--leader-elect=true"
    7. KUBE_SCHEDULER_ARGS=""

    编写 scheduler 的 systemd 配置文件

    1. $ cat /usr/lib/systemd/system/kube-scheduler.service
    2. [Unit]
    3. Description=Kubernetes Scheduler Plugin
    4. Documentation=https://kubernetes.io/docs/reference/generated/kube-scheduler/
    6. [Service]
    7. EnvironmentFile=-/etc/kubernetes/config
    8. EnvironmentFile=-/etc/kubernetes/scheduler
    9. ExecStart=/usr/bin/kube-scheduler \
    10.         $KUBE_LOGTOSTDERR \
    11.         $KUBE_LOG_LEVEL \
    12.         $KUBE_CONFIG \
    13.         $KUBE_AUTHENTICATION_KUBE_CONF \
    14.         $KUBE_AUTHORIZATION_KUBE_CONF \
    15.         $KUBE_BIND_ADDR \
    16.         $KUBE_LEADER_ELECT \
    17.         $KUBE_SCHEDULER_ARGS
    18. Restart=on-failure
    19. LimitNOFILE=65536
    21. [Install]
    22. WantedBy=multi-user.target

    使能各组件

    基本功能验证

    1. $ curl --cacert /etc/kubernetes/pki/ca.pem https://192.168.122.154:6443/version
    2. {
    3.   "major": "1",
    4.   "minor": "20",
    5.   "gitVersion": "v1.20.2",
    6.   "gitCommit": "faecb196815e248d3ecfb03c680a4507229c2a56",
    7.   "gitTreeState": "archive",
    8.   "buildDate": "2021-03-02T07:26:14Z",
    9.   "goVersion": "go1.15.7",
    10.   "compiler": "gc",
    11.   "platform": "linux/arm64"