Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support blobfuse-proxy on OpenShift #1045

Merged
merged 1 commit into from
Nov 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified charts/latest/blob-csi-driver-v0.0.0.tgz
Binary file not shown.
11 changes: 7 additions & 4 deletions charts/latest/blob-csi-driver/templates/csi-blob-node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ spec:
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
{{- if .Values.node.enableBlobfuseProxy }}
initContainers:
- name: install-blobfuse-proxy
{{- if hasPrefix "/" .Values.image.blob.repository }}
Expand All @@ -90,6 +89,8 @@ spec:
value: "{{ .Values.node.blobfuseProxy.installBlobfuse2 }}"
- name: BLOBFUSE2_VERSION
value: "{{ .Values.node.blobfuseProxy.blobfuse2Version }}"
- name: INSTALL_BLOBFUSE_PROXY
value: "{{ .Values.node.enableBlobfuseProxy }}"
- name: SET_MAX_OPEN_FILE_NUM
value: "{{ .Values.node.blobfuseProxy.setMaxOpenFileNum }}"
- name: MAX_FILE_NUM
Expand All @@ -99,9 +100,10 @@ spec:
volumeMounts:
- name: host-usr
mountPath: /host/usr
- name: host-usr-local
mountPath: /host/usr/local
Comment on lines 101 to +104

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this necessary if you have already /host/usr ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, for coreOS, /usr/ and /usr/local are different mount point.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think what @jsafrane mean is that since L102 is already mounting with /usr, why not resuse this mount point

Copy link
Member Author

@cvvz cvvz Nov 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Becase for coreOS, /usr/local is not a subpath of /usr, it's a different mount point, so we have to mount /usr/local and /usr seperately.

- name: host-etc
mountPath: /host/etc
{{- end }}
containers:
- name: liveness-probe
imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }}
Expand Down Expand Up @@ -266,14 +268,15 @@ spec:
name: mountpoint-dir
{{- end }}
volumes:
{{- if .Values.node.enableBlobfuseProxy }}
- name: host-usr
hostPath:
path: /usr
- name: host-usr-local
hostPath:
path: /usr/local
- name: host-etc
hostPath:
path: /etc
{{- end }}
- hostPath:
path: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }}
type: DirectoryOrCreate
Expand Down
2 changes: 1 addition & 1 deletion charts/latest/blob-csi-driver/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ node:
livenessProbe:
healthPort: 29633
logLevel: 5
enableBlobfuseProxy: false
enableBlobfuseProxy: true
blobfuseProxy:
installBlobfuse: true
blobfuseVersion: "1.4.5"
Expand Down
5 changes: 5 additions & 0 deletions deploy/csi-blob-node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ spec:
volumeMounts:
- name: host-usr
mountPath: /host/usr
- name: host-usr-local
mountPath: /host/usr/local
- name: host-etc
mountPath: /host/etc
containers:
Expand Down Expand Up @@ -178,6 +180,9 @@ spec:
- name: host-usr
hostPath:
path: /usr
- name: host-usr-local
hostPath:
path: /usr/local
- name: host-etc
hostPath:
path: /etc
Expand Down
119 changes: 21 additions & 98 deletions pkg/blobfuse-proxy/init.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,110 +25,32 @@ READ_AHEAD_KB=${READ_AHEAD_KB:-15380}
HOST_CMD="nsenter --mount=/proc/1/ns/mnt"

DISTRIBUTION=$($HOST_CMD cat /etc/os-release | grep ^ID= | cut -d'=' -f2 | tr -d '"')
echo "Linux distribution: $DISTRIBUTION"
ARCH=$($HOST_CMD uname -m)
echo "Linux Arch is $(uname -m)"

if [ "${DISTRIBUTION}" = "ubuntu" ] && { [ "${INSTALL_BLOBFUSE}" = "true" ] || [ "${INSTALL_BLOBFUSE2}" = "true" ]; }
then
release=$($HOST_CMD lsb_release -rs)
echo "Ubuntu release: $release"

if [ "$(expr "$release" \< "22.04")" -eq 1 ]
then
cp /blobfuse-proxy/packages-microsoft-prod-18.04.deb /host/etc/packages-microsoft-prod.deb
else
cp /blobfuse-proxy/packages-microsoft-prod-22.04.deb /host/etc/packages-microsoft-prod.deb
fi

# when running dpkg -i /etc/packages-microsoft-prod.deb, need to enter y to continue.
# refer to https://stackoverflow.com/questions/45349571/how-to-install-deb-with-dpkg-non-interactively
yes | $HOST_CMD dpkg -i /etc/packages-microsoft-prod.deb && $HOST_CMD apt update

pkg_list=""
if [ "${INSTALL_BLOBFUSE}" = "true" ] && [ "$(expr "$release" \< "22.04")" -eq 1 ]
then
pkg_list="${pkg_list} fuse"
# install blobfuse with latest version or specific version
if [ -z "${BLOBFUSE_VERSION}" ]; then
echo "install blobfuse with latest version"
pkg_list="${pkg_list} blobfuse"
else
pkg_list="${pkg_list} blobfuse=${BLOBFUSE_VERSION}"
fi
fi

if [ "${INSTALL_BLOBFUSE2}" = "true" ]
then
if [ "$(expr "$release" \< "22.04")" -eq 1 ]; then
echo "install fuse for blobfuse2"
pkg_list="${pkg_list} fuse"
else
echo "install fuse3 for blobfuse2, current release is $release"
pkg_list="${pkg_list} fuse3"
fi

# install blobfuse2 with latest version or specific version
if [ -z "${BLOBFUSE2_VERSION}" ]; then
echo "install blobfuse2 with latest version"
pkg_list="${pkg_list} blobfuse2"
else
pkg_list="${pkg_list} blobfuse2=${BLOBFUSE2_VERSION}"
fi
fi
echo "begin to install ${pkg_list}"
$HOST_CMD apt-get install -y $pkg_list
$HOST_CMD rm -f /etc/packages-microsoft-prod.deb
fi

updateBlobfuseProxy="true"
if [ -f "/host/usr/bin/blobfuse-proxy" ];then
old=$(sha256sum /host/usr/bin/blobfuse-proxy | awk '{print $1}')
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}')
if [ "$old" = "$new" ];then
updateBlobfuseProxy="false"
echo "no need to update blobfuse-proxy"
fi
fi

if [ "$updateBlobfuseProxy" = "true" ];then
echo "copy blobfuse-proxy...."
rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock
rm -rf /host/usr/bin/blobfuse-proxy
cp /blobfuse-proxy/blobfuse-proxy /host/usr/bin/blobfuse-proxy
chmod 755 /host/usr/bin/blobfuse-proxy
fi

updateService="true"
if [ -f "/host/usr/lib/systemd/system/blobfuse-proxy.service" ];then
old=$(sha256sum /host/usr/lib/systemd/system/blobfuse-proxy.service | awk '{print $1}')
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}')
if [ "$old" = "$new" ];then
updateService="false"
echo "no need to update blobfuse-proxy.service"
fi
fi

if [ "$updateService" = "true" ];then
echo "copy blobfuse-proxy.service...."
mkdir -p /host/usr/lib/systemd/system
cp /blobfuse-proxy/blobfuse-proxy.service /host/usr/lib/systemd/system/blobfuse-proxy.service
fi

if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then
if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then
echo "start blobfuse-proxy...."
$HOST_CMD systemctl daemon-reload
$HOST_CMD systemctl enable blobfuse-proxy.service
$HOST_CMD systemctl restart blobfuse-proxy.service
fi
fi

echo "Linux distribution: $DISTRIBUTION, Arch: $ARCH"

# install blobfuse-proxy and blobfuse/blofuse2 if needed
case "${DISTRIBUTION}" in
"ubuntu")
. ./blobfuse-proxy/install-proxy-ubuntu.sh
;;
"rhcos")
. ./blobfuse-proxy/install-proxy-rhcos.sh
;;
"mariner")
. ./blobfuse-proxy/install-proxy-mariner.sh
;;
*)
echo "Unsupported distribution: ${DISTRIBUTION}"
;;
esac

# set max open file num
if [ "${SET_MAX_OPEN_FILE_NUM}" = "true" ]
then
$HOST_CMD sysctl -w fs.file-max="${MAX_FILE_NUM}"
fi

# disable updatedb
updateDBConfigPath="/host/etc/updatedb.conf"
if [ "${DISABLE_UPDATEDB}" = "true" ] && [ -f ${updateDBConfigPath} ]
then
Expand All @@ -140,6 +62,7 @@ then
cat ${updateDBConfigPath}
fi

# set read ahead size
if [ "${SET_READ_AHEAD_SIZE}" = "true" ]
then
echo "set read ahead size to ${READ_AHEAD_KB}KB"
Expand Down
62 changes: 62 additions & 0 deletions pkg/blobfuse-proxy/install-proxy-mariner.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/bin/sh

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -xe

# install blobfuse/blobfuse2
echo "skip install blobfuse/blobfuse2 for mariner...."

# install blobfuse-proxy
updateBlobfuseProxy="true"
if [ -f "/host/usr/bin/blobfuse-proxy" ];then
old=$(sha256sum /host/usr/bin/blobfuse-proxy | awk '{print $1}')
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}')
if [ "$old" = "$new" ];then
updateBlobfuseProxy="false"
echo "no need to update blobfuse-proxy"
fi
fi
if [ "$updateBlobfuseProxy" = "true" ];then
echo "copy blobfuse-proxy...."
rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock
rm -rf /host/usr/bin/blobfuse-proxy
cp /blobfuse-proxy/blobfuse-proxy /host/usr/bin/blobfuse-proxy
chmod 755 /host/usr/bin/blobfuse-proxy
fi

updateService="true"
if [ -f "/host/usr/lib/systemd/system/blobfuse-proxy.service" ];then
old=$(sha256sum /host/usr/lib/systemd/system/blobfuse-proxy.service | awk '{print $1}')
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}')
if [ "$old" = "$new" ];then
updateService="false"
echo "no need to update blobfuse-proxy.service"
fi
fi
if [ "$updateService" = "true" ];then
echo "copy blobfuse-proxy.service...."
mkdir -p /host/usr/lib/systemd/system
cp /blobfuse-proxy/blobfuse-proxy.service /host/usr/lib/systemd/system/blobfuse-proxy.service
fi

if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then
if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then
echo "start blobfuse-proxy...."
$HOST_CMD systemctl daemon-reload
$HOST_CMD systemctl enable blobfuse-proxy.service
$HOST_CMD systemctl restart blobfuse-proxy.service
fi
fi
68 changes: 68 additions & 0 deletions pkg/blobfuse-proxy/install-proxy-rhcos.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#!/bin/sh

# Copyright 2019 The Kubernetes Authors.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: 2024

#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -xe

# in coreos, we could just copy the blobfuse2 binary to /usr/local/bin/blobfuse2
if [ "${INSTALL_BLOBFUSE}" = "true" ] || [ "${INSTALL_BLOBFUSE2}" = "true" ]
cvvz marked this conversation as resolved.
Show resolved Hide resolved
then
echo "copy blobfuse2...."
cp /usr/bin/blobfuse2 /host/usr/local/bin/blobfuse2
fi

# install blobfuse-proxy
updateBlobfuseProxy="true"
if [ -f "/host/usr/local/bin/blobfuse-proxy" ];then
old=$(sha256sum /host/usr/local/bin/blobfuse-proxy | awk '{print $1}')
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}')
if [ "$old" = "$new" ];then
updateBlobfuseProxy="false"
echo "no need to update blobfuse-proxy"
fi
fi
if [ "$updateBlobfuseProxy" = "true" ];then
echo "copy blobfuse-proxy...."
rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock
rm -rf /host/usr/local/bin/blobfuse-proxy
cp /blobfuse-proxy/blobfuse-proxy /host/usr/local/bin/blobfuse-proxy
chmod 755 /host/usr/local/bin/blobfuse-proxy
fi

updateService="true"
echo "change from /usr/bin/blobfuse-proxy to /usr/local/bin/blobfuse-proxy in blobfuse-proxy.service"
sed -i 's/\/usr\/bin\/blobfuse-proxy/\/usr\/local\/bin\/blobfuse-proxy/g' /blobfuse-proxy/blobfuse-proxy.service
if [ -f "/host/etc/systemd/system/blobfuse-proxy.service" ];then
old=$(sha256sum /host/etc/systemd/system/blobfuse-proxy.service | awk '{print $1}')
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}')
if [ "$old" = "$new" ];then
updateService="false"
echo "no need to update blobfuse-proxy.service"
fi
fi
if [ "$updateService" = "true" ];then
echo "copy blobfuse-proxy.service...."
mkdir -p /host/etc/systemd/system/
cp /blobfuse-proxy/blobfuse-proxy.service /host/etc/systemd/system/blobfuse-proxy.service
fi

if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then
if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then
echo "start blobfuse-proxy...."
$HOST_CMD systemctl daemon-reload
$HOST_CMD systemctl enable blobfuse-proxy.service
$HOST_CMD systemctl restart blobfuse-proxy.service
fi
fi
Loading
Loading