一、准备工作
1、创建一个k8s集群，搭建步骤参见我的另外一篇博文 k8s搭建文档

[root@kubernetes-master ~]# kubectl get nodes
NAME                STATUS   ROLES                  AGE    VERSION
kubernetes-master   Ready    control-plane,master   152d   v1.23.4
kubernetes-node1    Ready                     152d   v1.23.4
kubernetes-node2    Ready                     152d   v1.23.4

1.2、阿里云oss账号，并创建bucket。这个没什么好说的，参考阿里云官方文档

1.3、在每台机器上安装阿里云ossfs软件。这个软件必须要安装，因为pv/pvc如果想要用阿里云的oss的话，这是必须的软件。安装步骤参见阿里云oss安装文档，我的机器是三节点centos,下边我就写一下centos 该怎么安装ossfs。

# 下载安装包
wget https://gosspublic.alicdn.com/ossfs/ossfs_1.80.6_centos7.0_x86_64.rpm
yum install ossfs_1.80.6_centos7.0_x86_64.rpm

二、yaml文件准备
2.1、rbac.yaml

# This YAML file contains all RBAC objects that are necessary to run external
# CSI provisioner.
#
# In production, each CSI driver deployment has to be customized:
# - to avoid conflicts, use non-default namespace and different names
#   for non-namespaced entities like the ClusterRole
# - decide whether the deployment replicates the external CSI
#   provisioner, in which case leadership election must be enabled;
#   this influences the RBAC setup, see belowapiVersion: v1
kind: ServiceAccount
metadata:name: admin# replace with the same namespace name with pluginnamespace: kube-system---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: alicloud-csi-plugin
rules:- apiGroups: [""]resources: ["secrets"]verbs: ["get", "list"]- apiGroups: [""]resources: ["persistentvolumes"]verbs: ["get", "list", "watch", "update", "create", "delete"]- apiGroups: [""]resources: ["persistentvolumeclaims"]verbs: ["get", "list", "watch", "update"]- apiGroups: ["storage.k8s.io"]resources: ["storageclasses"]verbs: ["get", "list", "watch"]- apiGroups: ["storage.k8s.io"]resources: ["csinodes"]verbs: ["get", "list", "watch"]- apiGroups: [""]resources: ["events"]verbs: ["get", "list", "watch", "create", "update", "patch"]- apiGroups: [""]resources: ["endpoints"]verbs: ["get", "watch", "list", "delete", "update", "create"]- apiGroups: [""]resources: ["configmaps"]verbs: ["get", "watch", "list", "delete", "update", "create"]- apiGroups: [""]resources: ["nodes"]verbs: ["get", "list", "watch"]- apiGroups: ["csi.storage.k8s.io"]resources: ["csinodeinfos"]verbs: ["get", "list", "watch"]- apiGroups: ["storage.k8s.io"]resources: ["volumeattachments"]verbs: ["get", "list", "watch", "update"]- apiGroups: ["snapshot.storage.k8s.io"]resources: ["volumesnapshotclasses"]verbs: ["get", "list", "watch"]- apiGroups: ["snapshot.storage.k8s.io"]resources: ["volumesnapshotcontents"]verbs: ["create", "get", "list", "watch", "update", "delete"]- apiGroups: ["snapshot.storage.k8s.io"]resources: ["volumesnapshots"]verbs: ["get", "list", "watch", "update"]- apiGroups: ["apiextensions.k8s.io"]resources: ["customresourcedefinitions"]verbs: ["create", "list", "watch", "delete"]---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: alicloud-csi-plugin
subjects:- kind: ServiceAccountname: adminnamespace: kube-system
roleRef:kind: ClusterRolename: alicloud-csi-pluginapiGroup: rbac.authorization.k8s.io

2.2、oss-plugin.yaml

apiVersion: storage.k8s.io/v1beta1
kind: CSIDriver
metadata:name: ossplugin.csi.alibabacloud.com
spec:attachRequired: false
---
# This YAML defines all API objects to create RBAC roles for csi node plugin.
kind: DaemonSet
apiVersion: apps/v1
metadata:name: csi-osspluginnamespace: kube-system
spec:selector:matchLabels:app: csi-ossplugintemplate:metadata:labels:app: csi-osspluginspec:tolerations:- operator: ExistspriorityClassName: system-node-criticalserviceAccount: adminhostNetwork: truehostPID: truecontainers:- name: driver-registrarimage: registry.cn-hangzhou.aliyuncs.com/acs/csi-node-driver-registrar:v1.1.0imagePullPolicy: Alwayslifecycle:preStop:exec:command: ["/bin/sh", "-c", "rm -rf /registration/ossplugin.csi.alibabacloud.com /registration/ossplugin.csi.alibabacloud.com-reg.sock"]args:- "--v=5"- "--csi-address=/var/lib/kubelet/plugins/ossplugin.csi.alibabacloud.com/csi.sock"- "--kubelet-registration-path=/var/lib/kubelet/plugins/ossplugin.csi.alibabacloud.com/csi.sock"env:- name: KUBE_NODE_NAMEvalueFrom:fieldRef:apiVersion: v1fieldPath: spec.nodeNamevolumeMounts:- name: kubelet-dirmountPath: /var/lib/kubelet/- name: registration-dirmountPath: /registration- name: csi-osspluginsecurityContext:privileged: truecapabilities:add: ["SYS_ADMIN"]allowPrivilegeEscalation: trueimage: registry.cn-hangzhou.aliyuncs.com/acs/csi-plugin:v1.14.8.32-c77e277b-aliyunimagePullPolicy: "Always"args:- "--endpoint=$(CSI_ENDPOINT)"- "--v=5"- "--driver=ossplugin.csi.alibabacloud.com"- "--nodeid=$(KUBE_NODE_NAME)"env:- name: CSI_ENDPOINTvalue: unix://var/lib/kubelet/plugins/ossplugin.csi.alibabacloud.com/csi.sock- name: KUBE_NODE_NAMEvalueFrom:fieldRef:fieldPath: spec.nodeNamevolumeMounts:- name: kubelet-dirmountPath: /var/lib/kubelet/mountPropagation: "Bidirectional"- name: etcmountPath: /host/etc- mountPath: /var/log/name: host-log- mountPath: /host/usr/name: flexvolumedirvolumes:- name: kubelet-dirhostPath:path: /var/lib/kubelet/type: Directory- name: registration-dirhostPath:path: /var/lib/kubelet/plugins_registrytype: DirectoryOrCreate- name: etchostPath:path: /etc- name: flexvolumedirhostPath:path: /usr/- name: host-loghostPath:path: /var/log/updateStrategy:type: RollingUpdate

2.3、pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:name: oss-csi-pvlabels:alicloud-pvname: oss-csi-pv
spec:capacity:storage: 5GiaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: Retaincsi:driver: ossplugin.csi.alibabacloud.com# set volumeHandle same value pv namevolumeHandle: oss-csi-pvvolumeAttributes:bucket: "*****" #重要url: "******" #重要otherOpts: "-o max_stat_cache_size=0 -o allow_other"akId: "****" #重要akSecret: "*******" #重要path: "/"

说明：

• bucket：目前只支持挂载Bucket，不支持挂载Bucket下面的子目录或文件。
• url：OSS endpoint，挂载OSS的接入域名，挂载节点和bucket相同region时，可使用内网地址。
• akId：用户的access id值。
• akSecret：用户的access secret值。
• otherOpts：挂载OSS时支持定制化参数输入，格式为：-o *** -o ***。

2.4、pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: oss-pvc
spec:accessModes:- ReadWriteOnceresources:requests:storage: 5Giselector:matchLabels:alicloud-pvname: oss-csi-pv

2.5、deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:name: deployment-osslabels:app: nginx
spec:selector:matchLabels:app: nginxtemplate:metadata:labels:app: nginxspec:containers:- name: nginximage: nginx:1.7.9ports:- containerPort: 80volumeMounts:- name: oss-pvcmountPath: "/data"volumes:- name: oss-pvcpersistentVolumeClaim:claimName: oss-pvc

三、部署服务

#创建rbac权限
$ kubectl create -f ./rbac.yaml 
serviceaccount/admin created
clusterrole.rbac.authorization.k8s.io/alicloud-csi-plugin created
clusterrolebinding.rbac.authorization.k8s.io/alicloud-csi-plugin created#创建oss-plugin
$ kubectl create -f ./oss-plugin.yaml#检查创建情况
$ kubectl get pod -n kube-system | grep csi-oss
kube-system             csi-ossplugin-9jdhw                                  2/2     Running             0          55m
kube-system             csi-ossplugin-f7n5f                                  2/2     Running             0          55m
kube-system             csi-ossplugin-vgkcp                                  2/2     Running             0          55m#查验CSIDriver安装情况
$ kubectl get CSIDriver
NAME                             CREATED AT
ossplugin.csi.alibabacloud.com   2020-06-23T14:48:18Z#创建pv
$ kubectl create -f ./pv.yaml#创建pvc
$ kubectl create -f ./pvc.yaml#检验一下阿里云oss是否可以成功挂载到k8s集群中做pv使用
$ kubectl create -f ./deploy.yaml

四、验证

$ kubectl get pod
NAME                              READY   STATUS              RESTARTS   AGE
deployment-oss-795894886d-lhpsx   1/1     Running             0          11h#pod成功后通过kubectl exec 进入到pod中，你能看到你账号下bucket里边的所有文件。样例如下：
$ kubectl exec -it deployment-oss-795894886d-lhpsx -- sh
$ ls
bin  boot  data  dev  etc  home  lib  lib64  media  mnt  opt  proc  root  run  sbin  selinux  srv  sys	tmp  usr  var
$ cd data	
$ ls
osstest

五、参考文档
5.1、 K8S有状态服务-OSS存储使用最佳实践
5.2、 阿里云oss CSI安装步骤
5.3、 阿里云oss官方文档