legacy-client-ds.yaml

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: client
  namespace: quobyte
spec:
  selector:
    matchLabels:
      role: client
  template:
    metadata:
      annotations:
        prometheus.io/scrape: 'true'
        prometheus.io/path: '/prometheus'
        prometheus.io/port: '55000'
      labels:
        role: client
        version: "2"
    spec:
      containers:
      - name: quobyte-client
        image: quay.io/quobyte/quobyte-client:2
        imagePullPolicy: Always
        env:
          - name: QUOBYTE_CLIENT_LOG_LEVEL
            value: INFO
          - name: QUOBYTE_REGISTRY
            value: registry.quobyte
          - name: QUOBYTE_MOUNT_POINT
            # Note that the mount point has to be a subdir of the volume(Mount)
            value: /var/lib/kubelet/plugins/kubernetes.io~quobyte
          - name: NODENAME
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
        ports:
          - name: http-port
            containerPort: 55000
            hostPort: 55000
            protocol: TCP
        readinessProbe:
          timeoutSeconds: 5
          httpGet:
            port: 55000
            path: /
        livenessProbe:
          initialDelaySeconds: 30
          timeoutSeconds: 5
          httpGet:
            port: 55000
            path: /
        command:
          - /bin/bash
          - -xec
          - |
            if [[ ! -f /etcfs/fuse.conf ]]; then
              echo "Copy fuse config to host"
              { echo -e '# Copied from Quobyte Client Container\n'; cat /etc/fuse.conf; } > /etcfs/fuse.conf
            fi
            if [[ $(grep "^[^#]" /etcfs/fuse.conf | grep -c "user_allow_other") -eq 0 ]]; then
              echo "user_allow_other" >> /etcfs/fuse.conf
            fi
            if cut -d" " -f2 /etcfs/mtab | grep -q ${QUOBYTE_MOUNT_POINT}; then
              echo "found existing mount point on host. Trying to umount ${QUOBYTE_MOUNT_POINT}"
              if ! /bin/nsenter -t 1 -m -- /bin/umount -f ${QUOBYTE_MOUNT_POINT}; then
                /bin/nsenter -t 1 -m -- /bin/umount -l ${QUOBYTE_MOUNT_POINT}
              fi
              sleep 1 # give os the chance to clean up everything.
            else
              if ! [[ $(ls `dirname ${QUOBYTE_MOUNT_POINT}`|egrep "^`basename ${QUOBYTE_MOUNT_POINT}`$") ]]; then
                echo "mount point ${QUOBYTE_MOUNT_POINT} does not exist, creating it..."
                mkdir -p ${QUOBYTE_MOUNT_POINT}
              fi
            fi

            # set the mount point immutable. As long as mount.quobyte does not run,
            # other processes cannot write data to this dir.
            chattr +i ${QUOBYTE_MOUNT_POINT}

            # Until k8s mountPropagation is stable, nsenter is used for the client
            # to mount Quobyte into the host's mount namespace. By using nsenter, the client process
            # will also use the host's dns resolve mechanisms, which most probably will not include
            # the k8s internal name resolution. So the client is not able to resolve k8s internal 'registry.quobyte' anymore.
            # Hence, for k8s internal Quobyte clusters, we need to pre-resolve the current ips, and let the client
            # work with ips and not dns names.
            if [ "$QUOBYTE_REGISTRY" == "registry" ] || [ "$QUOBYTE_REGISTRY" == "registry.quobyte" ] ; then
              registry_ips=
              for ip in $(nslookup ${QUOBYTE_REGISTRY} | grep ${QUOBYTE_REGISTRY} -A2 | grep Address | cut -d':' -f2 | awk '{print $1}'); do
                registry_ips="$ip $registry_ips"
              done
              ADDR=$(echo $registry_ips | tr ' ' ',')
              echo "Assuming to connect to k8s hosted Quobyte cluster. Resolved DNS to: '$ADDR'"
            else
              ADDR=${QUOBYTE_REGISTRY}
              echo "Assuming to connect to an external Quobyte cluster. Client will resolve DNS for: '$ADDR'"
            fi

            /bin/nsenter -t 1 --wd=. -m -- \
              lib/ld-linux-x86-64.so.2 \
              --library-path ./lib \
              /usr/bin/mount.quobyte --hostname ${NODENAME} --allow-usermapping-in-volumename \
              --http-port 55000 -f -l /dev/stdout -d ${QUOBYTE_CLIENT_LOG_LEVEL} ${OPTS} \
              ${ADDR}/ ${QUOBYTE_MOUNT_POINT}

        securityContext:
          privileged: true
        volumeMounts:
          - name: k8s-plugin-dir
            mountPath: /var/lib/kubelet/plugins/
          - name: etcfs
            mountPath: /etcfs
        lifecycle:
          preStop:
            exec:
              command:
                - /bin/bash
                - -xc
                - |
                  if ! /bin/nsenter -t 1 -m -- /bin/umount -f ${QUOBYTE_MOUNT_POINT}; then
                    /bin/nsenter -t 1 -m -- /bin/umount -l ${QUOBYTE_MOUNT_POINT}
                  fi
      hostPID: true
      nodeSelector:
        quobyte_client: "true"
      volumes:
      - name: k8s-plugin-dir
        hostPath:
          path: /var/lib/kubelet/plugins/
      - name: etcfs
        hostPath:
          path: /etc