1. k8s部署metersphere
参考资料：https://github.com/metersphere/helm-chart
1.1 添加helm仓库
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo add metersphere https://metersphere.github.io/helm-chart/
helm repo update # 从 chart 仓库中更新本地可用chart的信息
helm install metersphere metersphere/metersphere
#添加bitnami开源仓库
[root@master tools]# helm repo add bitnami https://charts.bitnami.com/bitnami
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
"bitnami" already exists with the same configuration, skipping
#已经添加过了，所以显示"bitnami" already exists
#添加metersphere 开源仓库
[root@master tools]# helm repo add metersphere https://metersphere.github.io/helm-chart/
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
"metersphere" has been added to your repositories
#"metersphere" has been added to your repositories 显示已经添加完成
查看metersphere版本
[root@master tools]# helm search repo |grep metersphere
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
metersphere/metersphere 1.1.0 1.20.0-lts A Helm chart to deploy MeterSphere
该版本为1.20.0
1.2 安装metersphere
#执行helm安装metersphere
[root@master tools]# helm install metersphere metersphere/metersphere
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
Error: INSTALLATION FAILED: cannot re-use a name that is still in use
这是因为之前已经使用helm安装metersphere的原因，需要先执行卸载
#卸载metersphere
[root@master tools]# helm uninstall metersphere
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
W0506 10:23:07.152881 8204 warnings.go:70] networking.k8s.io/v1beta1 Ingress is deprecated in v1.19+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
release "metersphere" uninstalled
1.2.1 下载失败
[root@master tools]# helm install metersphere metersphere/metersphere
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
Error: INSTALLATION FAILED: failed to download "metersphere/metersphere"
该报错可能是因为网络问题导致下载失败。
1.2.2 redis冲突
再次执行安装命令
[root@master tools]# helm install metersphere metersphere/metersphere
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
Error: INSTALLATION FAILED: rendered manifests contain a resource that already exists. Unable to continue with install: StatefulSet "redis" in namespace "devsecops" exists and cannot be importeion error: key "meta.helm.sh/release-name" must equal "metersphere": current value is "dongtai"
redis已经存在，并且是属于洞态的，所以这里需要将redis的名称修改一下，但是修改可能会带来metersphere无法访问redis的情况，需要将连接地址一并修改。
解决这个问题，需要先下载metersphere的values.yaml文件
[root@master tools]# helm pull metersphere/metersphere --version 1.1.0
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
Error: Get "https://github.com/metersphere/helm-chart/releases/download/metersphere-1.1.0/metersphere-1.1.0.tgz": dial tcp 20.205.243.166:443: connect: connection timed out
该报错也是因为网络问题，重新下载即可。
[root@master tools]# helm pull metersphere/metersphere --version 1.1.0
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
解压压缩包
[root@master tools]# tar -zxf metersphere-1.1.0.tgz
[root@master tools]# cd metersphere/
编辑templates/03-modules/redis.yaml文件
#修改
{{ if .Values.redis.enabled }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis #修改为 metersphere-redis
spec:
replicas: 1
serviceName: "metersphere-redis"
selector:
matchLabels:
app: metersphere-redis
template:
metadata:
labels:
app: metersphere-redis
spec:
containers:
- name: redis #修改为 metersphere-redis
image: {{.Values.common.imagePrefix}}{{.Values.redis.image}}:{{.Values.redis.imageTag}}
imagePullPolicy: {{.Values.common.imagePullPolicy}}
resources:
requests:
memory: 100Mi
cpu: 0.1
ports:
- name: tcp
containerPort: 6379
env:
- name: MS_REDIS_PASSWORD
value: {{.Values.common.redis.password}}
command: ["redis-server", "/etc/redis/redis.conf", "--requirepass", "$(MS_REDIS_PASSWORD)"]
volumeMounts:
- mountPath: /etc/redis/redis.conf
name: opt-metersphere-config
subPath: redis.conf
- mountPath: /data
name: redis-data
subPath: redis
{{if ne (.Values.common.imagePullSecrets | default "nil") "nil"}}
imagePullSecrets:
- name: {{.Values.common.imagePullSecrets}}
{{ end }}
volumes:
- configMap:
defaultMode: 420
name: redis-config #修改为 metersphere-redis-config
name: opt-metersphere-config
- name: redis-data
{{if .Values.redis.persistence.enabled}}
persistentVolumeClaim:
claimName: redis-data-pvc #修改为 metersphere-redis-data-pvc
{{ else }}
emptyDir: {}
{{ end }}
修改后的
{{ if .Values.redis.enabled }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: metersphere-redis
spec:
replicas: 1
serviceName: "metersphere-redis"
selector:
matchLabels:
app: metersphere-redis
template:
metadata:
labels:
app: metersphere-redis
spec:
containers:
- name: metersphere-redis
image: {{.Values.common.imagePrefix}}{{.Values.redis.image}}:{{.Values.redis.imageTag}}
imagePullPolicy: {{.Values.common.imagePullPolicy}}
resources:
requests:
memory: 100Mi
cpu: 0.1
ports:
- name: tcp
containerPort: 6379
env:
- name: MS_REDIS_PASSWORD
value: {{.Values.common.redis.password}}
command: ["redis-server", "/etc/redis/redis.conf", "--requirepass", "$(MS_REDIS_PASSWORD)"]
volumeMounts:
- mountPath: /etc/redis/redis.conf
name: opt-metersphere-config
subPath: redis.conf
- mountPath: /data
name: redis-data
subPath: redis
{{if ne (.Values.common.imagePullSecrets | default "nil") "nil"}}
imagePullSecrets:
- name: {{.Values.common.imagePullSecrets}}
{{ end }}
volumes:
- configMap:
defaultMode: 420
name: metersphere-redis-config
name: opt-metersphere-config
- name: redis-data
{{if .Values.redis.persistence.enabled}}
persistentVolumeClaim:
claimName: metersphere-redis-data-pvc
{{ else }}
emptyDir: {}
{{ end }}
修改templates/02-pvc/redis.yaml
{{ if and .Values.redis.enabled .Values.redis.persistence.enabled}}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: redis-data-pvc #修改为 metersphere-redis-data-pvc
spec:
accessModes:
- {{.Values.redis.persistence.accessModes}}
resources:
requests:
storage: {{.Values.redis.persistence.size}}
storageClassName: {{.Values.common.storageClass}}
{{end}}
修改后
{{ if and .Values.redis.enabled .Values.redis.persistence.enabled}}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: metersphere-redis-data-pvc
spec:
accessModes:
- {{.Values.redis.persistence.accessModes}}
resources:
requests:
storage: {{.Values.redis.persistence.size}}
storageClassName: {{.Values.common.storageClass}}
{{end}}
修改 templates/01-config-map/redis-config.yaml
{{ if .Values.redis.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: redis-config ##修改为 metersphere-redis-config
data:
redis.conf: |
bind 0.0.0.0
……………………#后续参数就不列出来
修改后
{{ if .Values.redis.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: metersphere-redis-config
data:
redis.conf: |
bind 0.0.0.0
……………………##后续参数就不列出来
执行install命令
[root@storage metersphere]# helm install metersphere .
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
W0506 14:39:24.683674 3636 warnings.go:70] networking.k8s.io/v1beta1 Ingress is deprecated in v1.19+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
W0506 14:39:24.823063 3636 warnings.go:70] networking.k8s.io/v1beta1 Ingress is deprecated in v1.19+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
NAME: metersphere
LAST DEPLOYED: Fri May 6 14:39:24 2022
NAMESPACE: devsecops
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Thanks for installing the MeterSphere.
Default username/password for MeterSphere is admin/metersphere
查看pods
[root@storage metersphere]# kubectl get pods|grep metersphere
metersphere-data-streaming-6477f6d7c4-wn7kj 0/1 Init:0/2 0 52s
metersphere-kafka-0 1/1 Running 1 52s
metersphere-node-controller-79f5659d65-2hf28 0/1 Pending 0 52s
metersphere-redis-0 0/1 Pending 0 52s
metersphere-server-54966778fb-gqrqg 0/1 Pending 0 52s
metersphere-zookeeper-0 1/1 Running 0 52s
1.2.3 修改STORAGECLASS
查看pvc
[root@storage metersphere]# kubectl get pvc|grep meter
metersphere-data-pvc Pending default 108s
metersphere-redis-data-pvc Pending default 108s
发现使用的是STORAGECLASS为default，因为已经配置了默认存储，默认的STORAGECLASS为nfs-storageclass，所以需要修改对应配置
编辑values.yaml
###将storageClass: default修改为空
storageClass: default
修改后：
storageClass:
重新安装metersphere
###卸载
[root@storage metersphere]# helm uninstall metersphere
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
W0506 14:44:12.744438 31139 warnings.go:70] networking.k8s.io/v1beta1 Ingress is deprecated in v1.19+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
release "metersphere" uninstalled
###安装
[root@storage metersphere]# helm install metersphere .
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
W0506 14:45:05.465076 6037 warnings.go:70] networking.k8s.io/v1beta1 Ingress is deprecated in v1.19+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
W0506 14:45:05.622877 6037 warnings.go:70] networking.k8s.io/v1beta1 Ingress is deprecated in v1.19+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
NAME: metersphere
LAST DEPLOYED: Fri May 6 14:45:05 2022
NAMESPACE: devsecops
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Thanks for installing the MeterSphere.
Default username/password for MeterSphere is admin/metersphere
查看pvc
[root@storage metersphere]# kubectl get pvc|grep meter
metersphere-data-pvc Bound pvc-a6ed356f-eeec-4d0e-ba9b-9d8318ddff66 10Gi RWO nfs-storageclass 2m5s
metersphere-redis-data-pvc Bound pvc-2b8c1561-383d-44ca-a094-c115b80cbb53 10Gi RWO nfs-storageclass 2m5s
1.3 修改ingress
登陆k8s平台查看发现ingress没有配置正确
编辑values.yaml文件
将原来的ingress配置成以下内容：
ingress:
enabled: true
host: test.infomings.com
annotations:
## example for ingress annotions.
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
kubernetes.io/ingress.provider: nginx
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-buffering: "off"
nginx.ingress.kubernetes.io/proxy-read-timeout: "900"
nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
https:
enabled: false
secretName: ""
修改前：
修改后：
执行重新部署：
#卸载
[root@storage metersphere]# helm uninstall metersphere
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
W0506 15:14:40.172571 32523 warnings.go:70] networking.k8s.io/v1beta1 Ingress is deprecated in v1.19+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
release "metersphere" uninstalled
[root@storage metersphere]#
[root@storage metersphere]#
[root@storage metersphere]#
[root@storage metersphere]#
##查看pvc
[root@storage metersphere]# kubectl get pvc |grep metersphere
metersphere-data-pvc Terminating pvc-a6ed356f-eeec-4d0e-ba9b-9d8318ddff66 10Gi RWO nfs-storageclass 29m
metersphere-redis-data-pvc Terminating pvc-2b8c1561-383d-44ca-a094-c115b80cbb53 10Gi RWO nfs-storageclass 29m
[root@storage metersphere]# kubectl get pvc |grep metersphere
[root@storage metersphere]# kubectl get pvc |grep metersphere
#部署
[root@storage metersphere]# helm install metersphere .
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
W0506 15:14:56.779208 4834 warnings.go:70] networking.k8s.io/v1beta1 Ingress is deprecated in v1.19+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
W0506 15:14:56.956215 4834 warnings.go:70] networking.k8s.io/v1beta1 Ingress is deprecated in v1.19+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
NAME: metersphere
LAST DEPLOYED: Fri May 6 15:14:56 2022
NAMESPACE: devsecops
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Thanks for installing the MeterSphere.
Default username/password for MeterSphere is admin/metersphere
在浏览器访问对应的地址
无法正常访问，修改ingressClassName
在rancher界面编辑ingress的yaml
将ingressClassName: nginx修改为ingressClassName: gitlab-nginx
保存，刷新浏览器，还是一样的界面
经过后续测试，其实是因为浏览器缓存的问题，清理了就可以了，不用修改ingressClassName也可以
Q.E.D.