Hướng dẫn cách cài đặt Kubernetes

apt-get update -y && apt-get upgrade -y
apt-get -y install -y vim curl wget 
apt-get -y install byobu

cat << EOF > /etc/network/interfaces
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

source /etc/network/interfaces.d/*

# The loopback network interface
auto lo
iface lo inet loopback

# The primary network interface
auto ens4
iface ens4 inet static
address 172.16.68.130
netmask 255.255.255.0
gateway 172.16.68.1
dns-nameservers 8.8.8.8
EOF

cat << EOF > /etc/hosts
127.0.0.1       localhost k8s-master
172.16.68.130       k8s-master
172.16.68.131       k8s-k8s-node1
172.16.68.132       k8s-node2
EOF

echo k8s-master > /etc/hostname

init 6

apt-get update -y && apt-get upgrade -y
apt-get -y install -y vim curl wget 
apt-get -y install byobu

swapoff -a

cat << EOF > /etc/network/interfaces
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface
auto ens4
iface ens4 inet static
address 172.16.68.131
netmask 255.255.255.0
gateway 172.16.68.1
dns-nameservers 8.8.8.8
EOF

cat << EOF > /etc/hosts
127.0.0.1       localhost k8s-node1
172.16.68.130       k8s-master
172.16.68.131       k8s-node1
172.16.68.132       k8s-node2
EOF

echo k8s-node1 > /etc/hostname

init 6

apt-get update -y && apt-get upgrade -y
apt-get -y install -y vim curl wget 
apt-get -y install byobu

swapoff -a

cat << EOF > /etc/network/interfaces
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface
auto ens4
iface ens4 inet static
address 172.16.68.132
netmask 255.255.255.0
gateway 172.16.68.1
dns-nameservers 8.8.8.8
EOF

cat << EOF > /etc/hosts
127.0.0.1       localhost k8s-node2
172.16.68.130       k8s-master
172.16.68.131       k8s-node1
172.16.68.132       k8s-node2
EOF

echo k8s-node2 > /etc/hostname

init 6

apt-get -y update && apt-get -y install docker.io

apt-get update && apt-get install -y apt-transport-https

curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add 

cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb http://apt.kubernetes.io/ kubernetes-xenial main
EOF

apt-get update  -y
apt-get install -y kubelet kubeadm kubectl

kubeadm init --apiserver-advertise-address 172.16.68.130 --pod-network-cidr=10.244.0.0/16

root@k8s-master:~#   kubeadm init --apiserver-advertise-address 172.16.68.130 --pod-network-cidr=10.244.0.0/16
[init] Using Kubernetes version: v1.9.2
[init] Using Authorization modes: [Node RBAC]
[preflight] Running pre-flight checks.
        [WARNING FileExisting-crictl]: crictl not found in system path
[certificates] Generated ca certificate and key.
[certificates] Generated apiserver certificate and key.
[certificates] apiserver serving cert is signed for DNS names [k8s-master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 172.16.68.130]
[certificates] Generated apiserver-kubelet-client certificate and key.
[certificates] Generated sa key and public key.
[certificates] Generated front-proxy-ca certificate and key.
[certificates] Generated front-proxy-client certificate and key.
[certificates] Valid certificates and keys now exist in "/etc/kubernetes/pki"
[kubeconfig] Wrote KubeConfig file to disk: "admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "kubelet.conf"
[kubeconfig] Wrote KubeConfig file to disk: "controller-manager.conf"
[kubeconfig] Wrote KubeConfig file to disk: "scheduler.conf"
[controlplane] Wrote Static Pod manifest for component kube-apiserver to "/etc/kubernetes/manifests/kube-apiserver.yaml"
[controlplane] Wrote Static Pod manifest for component kube-controller-manager to "/etc/kubernetes/manifests/kube-controller-manager.yaml"
[controlplane] Wrote Static Pod manifest for component kube-scheduler to "/etc/kubernetes/manifests/kube-scheduler.yaml"
[etcd] Wrote Static Pod manifest for a local etcd instance to "/etc/kubernetes/manifests/etcd.yaml"
[init] Waiting for the kubelet to boot up the control plane as Static Pods from directory "/etc/kubernetes/manifests".
[init] This might take a minute or longer if the control plane images have to be pulled.
[apiclient] All control plane components are healthy after 56.503430 seconds
[uploadconfig] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[markk8s-master] Will mark node k8s-master as k8s-master by adding a label and a taint
[markk8s-master] k8s-master k8s-master tainted and labelled with key/value: node-role.kubernetes.io/k8s-master=""
[bootstraptoken] Using token: 1aeb1d.89a15bf6272c8274
[bootstraptoken] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstraptoken] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstraptoken] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstraptoken] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: kube-dns
[addons] Applied essential addon: kube-proxy

Your Kubernetes k8s-master has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of machines by running the following on each node
as root:

  kubeadm join --token 1aeb1d.89a15bf6272c8274 172.16.68.130:6443 --discovery-token-ca-cert-hash sha256:cb8e0cd1238dc8fe8b1b2f16fe02817425005f04a8ddd09a7c19db08b75f72eb

root@k8s-master:~#

export KUBECONFIG=/etc/kubernetes/admin.conf

- Bắt đầu từ đây có thể thao thác với K8S bằng lệnh `kubectl` để quản trị. 

```
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
```

  adduser ubuntu

  echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers

  su - ubuntu
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

	echo "source <(kubectl completion bash)" >> ~/.bashrc

	kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

	root@k8s-master:~# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
	clusterrole.rbac.authorization.k8s.io "flannel" created
	clusterrolebinding.rbac.authorization.k8s.io "flannel" created
	serviceaccount "flannel" created
	configmap "kube-flannel-cfg" created
	daemonset.extensions "kube-flannel-ds" created

   sudo kubeadm token create --print-join-command

    kubeadm join --token 150984.0da1fe160e5113f0 172.16.68.130:6443 --discovery-token-ca-cert-hash sha256:cb8e0cd1238dc8fe8b1b2f16fe02817425005f04a8ddd09a7c19db08b75f72eb

  kubeadm join --token 150984.0da1fe160e5113f0 172.16.68.130:6443 --discovery-token-ca-cert-hash sha256:cb8e0cd1238dc8fe8b1b2f16fe02817425005f04a8ddd09a7c19db08b75f72eb

  swapoff -a

  [preflight] Running pre-flight checks.
          [WARNING FileExisting-crictl]: crictl not found in system path
  [discovery] Trying to connect to API Server "172.16.68.130:6443"
  [discovery] Created cluster-info discovery client, requesting info from "https://172.16.68.130:6443"
  [discovery] Requesting info from "https://172.16.68.130:6443" again to validate TLS against the pinned public key
  [discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server "172.16.68.130:6443"
  [discovery] Successfully established connection with API Server "172.16.68.130:6443"

  This node has joined the cluster:
  * Certificate signing request was sent to k8s-master and a response
    was received.
  * The Kubelet was informed of the new secure connection details.

  Run 'kubectl get nodes' on the k8s-master to see this node join the cluster.
  root@k8s-node1:~#

  export KUBECONFIG=/etc/kubernetes/admin.conf
  kubectl get nodes

  NAME      STATUS    ROLES     AGE       VERSION
  k8s-master    Ready     k8s-master    9h        v1.9.2
  k8s-node1     Ready     <none>    8h        v1.9.2
  k8s-node2     Ready     <none>    6m        v1.9.2

  kubectl get pod --all-namespaces

  NAMESPACE     NAME                             READY     STATUS    RESTARTS   AGE
  kube-system   etcd-k8s-master                      1/1       Running   0          9h
  kube-system   kube-apiserver-k8s-master            1/1       Running   0          9h
  kube-system   kube-controller-manager-k8s-master   1/1       Running   0          9h
  kube-system   kube-dns-6f4fd4bdf-ctxx7         3/3       Running   0          9h
  kube-system   kube-flannel-ds-kjnhs            1/1       Running   0          9h
  kube-system   kube-flannel-ds-wz648            1/1       Running   0          8h
  kube-system   kube-flannel-ds-xtcj9            1/1       Running   0          36m
  kube-system   kube-proxy-5slwp                 1/1       Running   0          36m
  kube-system   kube-proxy-5trrj                 1/1       Running   0          9h
  kube-system   kube-proxy-b54bs                 1/1       Running   0          8h
  kube-system   kube-scheduler-k8s-master            1/1       Running   0          9h

	export KUBECONFIG=/etc/kubernetes/admin.conf
	
	kubectl describe pod kube-scheduler-k8s-master --namespace=kube-system

	kubectl run test-nginx --image=nginx --replicas=2 --port=80 

- Kết quả như sau:

		root@k8s-master:~# kubectl run test-nginx --image=nginx --replicas=2 --port=80
		deployment.apps "test-nginx" created
		root@k8s-master:~#

Tới đây, ta mới tạo ra các container và chỉ có thể truy cập từ các máy trong cụm cluster, bởi vì các container này chưa được mở các port để ánh xạ với các IP của các máy trọng cụm K8S.

	kubectl get pods -o wide

		root@k8s-master:~# kubectl get pods -o wide
		NAME                         READY     STATUS    RESTARTS   AGE       IP           NODE
		test-nginx-959dbd6b6-7tx8l   1/1       Running   0          1m        10.244.2.5   k8s-node2
		test-nginx-959dbd6b6-h7xgg   1/1       Running   0          1m        10.244.1.5   k8s-node1

- Trong kết quả trên, ta có thể quan sát thấy trạng thái các container  ở cột `STATUS` và ở  cột `NODE` - nời mà các container được phân phối, số lượng là container sẽ là 2 vì chúng ta đã có tùy chọn `--replicas=2`, việc phân phố số lượng container này một phần là do thành phần scheduler trong K8S thực hiện. Ngoài ra, trong các phần nâng cao sau của tài liệu này, chúng ta sẽ thực hành thêm việc thay đổi số lượng replicas (tạm hiểu là số lượng container) sau khi đã tạo chúng hoặc sau khi deploy các ứng dụng (điểm khá hay ho của container nói chung và K8S nói riêng).

	kubectl get deployment

- Kết quả như bên dưới (hãy quan sát cột `AVAILABLE`, giá trị này sẽ thay đổi khi ta thực hiện quá trình deployment các service này).

		root@k8s-master:~# kubectl get deployment
		NAME         DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
		test-nginx   2         2         2            0           27m

	kubectl expose deploy test-nginx --port 80 --target-port 80 --type NodePort

- Kết quả 

		root@k8s-master:~# kubectl expose deploy test-nginx --port 80 --target-port 80 --type NodePort
		service "test-nginx" exposed

- Ngoài các tùy chọn `--port 80` và `--target-port 80` thì ta lưu ý tùy chọn `--type NodePort`, đây là tùy chọn để ánh xạ port của máy cài K8S vào container vừa tạo, sử dụng các lệnh dưới để biết được port mà host ánh xạ là bao nhiêu ở bên dưới.

		root@k8s-master:~# kubectl describe service test-nginx
		Name:                     test-nginx
		Namespace:                default
		Labels:                   run=test-nginx
		Annotations:              <none>
		Selector:                 run=test-nginx
		Type:                     NodePort
		IP:                       10.107.71.150
		Port:                     <unset>  80/TCP
		TargetPort:               80/TCP
		NodePort:                 <unset>  32136/TCP
		Endpoints:                10.244.1.5:80,10.244.2.5:80
		Session Affinity:         None
		External Traffic Policy:  Cluster
		Events:                   <none>
		root@k8s-master:~#

- Trong kết quả này, chúng ta có thể thấy các tham số quan trọng và cần lưu ý như sau: 

  - `IP: 10.107.71.150`: là địa chỉ được cấp phát cho ứng dụng `test-nginx` vừa tạo ở trên, địa chỉ này có ý nghĩa local.
	
	- `Endpoints: 10.244.1.5:80,10.244.2.5:80`: Đây là địa chỉ của dải mạng nội tại và liên kết các container khi chúng thuộc một POD. Ta có thể đứng trên một trong các node của cụm cluster K8S và thực hiện lệnh curl để truy cập web, ví dụ: `curl 10.244.1.5` hoặc `curl 10.244.2.5`. Kết quả trả về html của web server.
	
	- `Port và TargetPort`: là các port nằm trong chỉ định ở lệnh khi ta deploy ứng dụng.
	- `NodePort: <unset>  32136/TCP`: Đây chính là port mà ta dùng để truy cập vào web server được tạo ở trên thông qua một trong các IP của các máy trong cụm cluser. Ta sẽ có các kiểm chứng dưới.

	curl 10.107.71.150 

	hoặc 

	curl 10.244.1.5

	hoặc 

	curl 10.244.2.5

		root@k8s-master:~# curl  10.244.1.5
		<!DOCTYPE html>
		<html>
		<head>
		<title>Welcome to nginx!</title>
		<style>
				body {
						width: 35em;
						margin: 0 auto;
						font-family: Tahoma, Verdana, Arial, sans-serif;
				}
		</style>
		</head>
		<body>
		<h1>Welcome to nginx!</h1>
		<p>If you see this page, the nginx web server is successfully installed and
		working. Further configuration is required.</p>

		<p>For online documentation and support please refer to
		<a href="http://nginx.org/">nginx.org</a>.<br/>
		Commercial support is available at
		<a href="http://nginx.com/">nginx.com</a>.</p>

		<p><em>Thank you for using nginx.</em></p>
		</body>
		</html>

	ss -lan | grep 32136

	root@k8s-master:~# ss -lan | grep 32136
	tcp    LISTEN     0      128      :::32136                :::*
	root@k8s-master:~#

	root@k8s-master:~# kubectl get deployment
	NAME         DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
	test-nginx   2         2         2            2           27m

	root@k8s-master:~# kubectl get services
	NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
	kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        2d
	test-nginx   NodePort    10.107.71.150   <none>        80:32136/TCP   24m

	kubectl delete service test-nginx

	kubectl delete deployment test-nginx

	kubectl get services
	kubectl get deployments

apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: apache2
spec:
  template:
    metadata:
      labels:
        name: apache2
    spec:
      containers:
      - name: apache2
        image: httpd
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: apache2
spec:
  selector:
    name: apache2
  ports:
    - port: 5555
      targetPort: 80
  type: NodePort

	kubectl create -f apache-app.yaml

	root@k8s-master:~# kubectl create -f apache-app.yaml
	deployment.apps "apache-app" created
	service "apache-app" created

	kubectl get services
	kubectl get deployments

- Sử dụng port ở kết quả để truy cập, với node port sẽ là `http://172.16.68.130:32316/` hoặc `http://172.16.68.131:32316/` hoặc `http://172.16.68.132:32316/`

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml

secret "kubernetes-dashboard-certs" created
serviceaccount "kubernetes-dashboard" created
role.rbac.authorization.k8s.io "kubernetes-dashboard-minimal" created
rolebinding.rbac.authorization.k8s.io "kubernetes-dashboard-minimal" created
deployment.apps "kubernetes-dashboard" configured
service "kubernetes-dashboard" created

kubectl proxy

kubectl -n kube-system edit service kubernetes-dashboard

kubectl -n kube-system get service kubernetes-dashboard

NAME                   TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
kubernetes-dashboard   NodePort   10.108.100.24   <none>        443:30826/TCP   19m

kubectl -n kube-system get secret

NAME                                             TYPE                                  DATA      AGE
attachdetach-controller-token-2qzmx              kubernetes.io/service-account-token   3         8d
bootstrap-signer-token-4xf4c                     kubernetes.io/service-account-token   3         8d
bootstrap-token-mp1gba                           bootstrap.kubernetes.io/token         6         17h
certificate-controller-token-hp4pb               kubernetes.io/service-account-token   3         8d
clusterrole-aggregation-controller-token-82525   kubernetes.io/service-account-token   3         8d
cronjob-controller-token-h4r4q                   kubernetes.io/service-account-token   3         8d
daemon-set-controller-token-7jnmg                kubernetes.io/service-account-token   3         8d
default-token-vbq5r                              kubernetes.io/service-account-token   3         8d
deployment-controller-token-hw9z6                kubernetes.io/service-account-token   3         8d
disruption-controller-token-w88np                kubernetes.io/service-account-token   3         8d
endpoint-controller-token-c7kd7                  kubernetes.io/service-account-token   3         8d
flannel-token-znjq2                              kubernetes.io/service-account-token   3         8d
generic-garbage-collector-token-jcswb            kubernetes.io/service-account-token   3         8d
heapster-token-7sk58                             kubernetes.io/service-account-token   3         18h
horizontal-pod-autoscaler-token-2gwqd            kubernetes.io/service-account-token   3         8d
job-controller-token-h58gr                       kubernetes.io/service-account-token   3         8d
kube-dns-token-nlsm9                             kubernetes.io/service-account-token   3         8d
kube-proxy-token-zwsp7                           kubernetes.io/service-account-token   3         8d
kubernetes-dashboard-certs                       Opaque                                1         17h
kubernetes-dashboard-key-holder                  Opaque                                2         3d
kubernetes-dashboard-token-6vwnt                 kubernetes.io/service-account-token   3         19h
metrics-server-token-zntp6                       kubernetes.io/service-account-token   3         18h
namespace-controller-token-h9t47                 kubernetes.io/service-account-token   3         8d
node-controller-token-qlct6                      kubernetes.io/service-account-token   3         8d
persistent-volume-binder-token-69h9d             kubernetes.io/service-account-token   3         8d
pod-garbage-collector-token-j9d9f                kubernetes.io/service-account-token   3         8d
pv-protection-controller-token-m8zvk             kubernetes.io/service-account-token   3         8d
pvc-protection-controller-token-2xm8w            kubernetes.io/service-account-token   3         8d
replicaset-controller-token-h92xk                kubernetes.io/service-account-token   3         5m
replication-controller-token-dtf66               kubernetes.io/service-account-token   3         8d
resourcequota-controller-token-nkc65             kubernetes.io/service-account-token   3         8d
service-account-controller-token-dtg8c           kubernetes.io/service-account-token   3         8d
service-controller-token-mq55l                   kubernetes.io/service-account-token   3         8d
statefulset-controller-token-54fwx               kubernetes.io/service-account-token   3         8d
token-cleaner-token-grlqf                        kubernetes.io/service-account-token   3         8d
ttl-controller-token-59tgf                       kubernetes.io/service-account-token   3         8d

kubectl describe secret cluster-admin-dashboard-sa-token-r4x48

Name:         cluster-admin-dashboard-sa-token-r4x48
Namespace:    default
Labels:       <none>
Annotations:  kubernetes.io/service-account.name=cluster-admin-dashboard-sa
              kubernetes.io/service-account.uid=b0264e18-5c9a-11e8-874a-525400fd9cfb

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1025 bytes
namespace:  7 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6ImNsdXN0ZXItYWRtaW4tZGFzaGJvYXJkLXNhLXRva2VuLXI0eDQ4Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImNsdXN0ZXItYWRtaW4tZGFzaGJvYXJkLXNhIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiYjAyNjRlMTgtNWM5YS0xMWU4LTg3NGEtNTI1NDAwZmQ5Y2ZiIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50OmRlZmF1bHQ6Y2x1c3Rlci1hZG1pbi1kYXNoYm9hcmQtc2EifQ.mHpm3XZd5NWA8KAp3gmj3Zi5TnNlwnw7JwG-aqE9mMtleBr-a4aBzbIE2KR-1TaNR7daNqZ0SOb7lv8577PVAdM-pwBwFCHc1rJW6kzaNLywnuuSzmlkRG_3VgNA2j4hifaK0kSqClp3m6XW9YQdGXi89-ClNZl1YtUsFfInniUCBlR3Fj5uxsrIXZl8BivCT0jGDLvNgUGRC5Uau334phRYQsFpnSdg1iRbUaG9QO6IvOPTtn-dFPmMyJcNiDcN4_wMBii_LaVKTdLnRmTLw_gZyThkyCKh9216GAUTK-hgoGmE98L_GdA8gaQCO0urriNYkXUNK803t2_Y_eBnZg