From 9430c71ea79b677e01a9a33290d86ba07ebdc79a Mon Sep 17 00:00:00 2001 From: Pravesh Humane Date: Fri, 16 Jul 2021 09:52:42 -0700 Subject: [PATCH 1/7] added db2 patch for wkc, iis --- .../ibmcloud/cpd_install/scripts/install-wkc.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh b/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh index 663d050b..8b6d9235 100755 --- a/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh +++ b/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh @@ -30,7 +30,7 @@ cloudctl case launch --case ${CASE_PACKAGE_NAME} \ # switch to zen namespace -oc project zen +oc project ${NAMESPACE} # # Install wkc Customer Resource @@ -40,6 +40,9 @@ echo '*** executing **** oc create -f wkc-cr.yaml' result=$(oc create -f wkc-cr.yaml) echo $result +while [ "$(oc get sts -n ${NAMESPACE} | grep c-db2oltp-wkc-db2u)" = "" ]; do echo "waiting for c-db2oltp-wkc-db2u statefulset."; sleep 60; done +oc patch sts c-db2oltp-wkc-db2u -p='{"spec":{"template":{"spec":{"containers":[{"name":"db2u","tty":false}]}}}}}'; echo "patch for c-db2oltp-wkc-db2u is appied." + # check the wkc cr status ./../check-cr-status.sh wkc wkc-cr ${NAMESPACE} wkcStatus @@ -76,6 +79,10 @@ echo '*** executing **** oc create -f wkc-iis-cr.yaml' result=$(oc create -f wkc-iis-cr.yaml) echo $result +while [ "$(oc get sts -n ${NAMESPACE} | grep c-db2oltp-iis-db2u)" = "" ]; do echo "waiting for c-db2oltp-iis-db2u statefulset."; sleep 60; done +oc patch sts c-db2oltp-iis-db2u -p='{"spec":{"template":{"spec":{"containers":[{"name":"db2u","tty":false}]}}}}}'; echo "patch for c-db2oltp-iis-db2u is appied." + + # check the wkc cr status ./../check-cr-status.sh iis iis-cr ${NAMESPACE} iisStatus From a5d0f81e128dc831a227c0bc27b7db58eb3bd7f4 Mon Sep 17 00:00:00 2001 From: muthukba Date: Tue, 20 Jul 2021 17:50:49 +0530 Subject: [PATCH 2/7] Adding wkc troubleshooting steps readme --- TroubleshootReadme.md | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 TroubleshootReadme.md diff --git a/TroubleshootReadme.md b/TroubleshootReadme.md new file mode 100644 index 00000000..5fa90548 --- /dev/null +++ b/TroubleshootReadme.md @@ -0,0 +1,45 @@ +# Known Issues and Troubleshooting steps: + +## 1) Startup probe failed on OCP 4.6.34, 4.6.36 and 4.7.18 + +In the recent version of the openshift (4.6.34, 4.6.36 and 4.7.18) we have noticed the below issue on db2u pods. + +### Issue: + +```bash +Error: Pods c-db2oltp-wkc-db2u-0  & c-db2oltp-iis-db2u-0 pods will have  Startup probe failed with bellow error message +``` +```bash +Warning  Unhealthy       101m                    kubelet            Startup probe failed: time="2021-07-09T03:05:48-05:00" level=error msg="exec failed: open /dev/tty: no such device or address" +  Normal   Pulled          34m (x4 over 87m)       kubelet            Container image "cp.icr.io/cp/db2u.restricted@sha256:c1aaa5dc86b01c7706ee0bf5d46da51cee628c77758b955385325a574e451372" already present on machine +  Warning  Unhealthy       5m24s (x249 over 101m)  kubelet            (combined from similar events): Startup probe failed: time="2021-07-09T04:42:08-05:00" level=error msg="exec failed: open /dev/tty: no such device or address" +``` +### Workaround: + +We have added the patch recommended by DB2 team to patch the db2 sts, this will restart the db2u pod and allow the service to be proceed further. + +Patch example for wkc: +```bash + +oc patch sts c-db2oltp-wkc-db2u -p='{"spec":{"template":{"spec":{"containers":[{"name":"db2u","tty":false}]}}}}}' + +Once the db2u pod is up: + +oc rsh c-db2oltp-wkc-db2u-0  + +bash +ipclean -a && sleep 5 +db2start +db2 -v activate db LINEAGE + +``` + +For IIS, the below patch should make the install proceed further. + +``` bash +oc patch sts c-db2oltp-iis-db2u -p='{"spec":{"template":{"spec":{"containers":[{"name":"db2u","tty":false}]}}}}}' +``` + +For db2 instance creation in CP4D services like db2wh, db2oltp, dv ... that might be in progress for long time,if you see db2u pods in 0/1 state, describe pod for the respective db2u pod shows /dev/tty: no such device or address error + +Run the similar patch as above for the db2u sts that is in 0/1 state. From ffed920e49a0101debd4b469e27048eb903b2941 Mon Sep 17 00:00:00 2001 From: muthukba Date: Wed, 21 Jul 2021 12:07:52 +0530 Subject: [PATCH 3/7] Removed csv values for ocs and pwx operators --- selfmanaged-openshift/azure/ocs_module/deploy-with-olm.yaml | 1 - selfmanaged-openshift/azure/portworx_module/px-install.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/selfmanaged-openshift/azure/ocs_module/deploy-with-olm.yaml b/selfmanaged-openshift/azure/ocs_module/deploy-with-olm.yaml index b27f2c07..df6a1db0 100644 --- a/selfmanaged-openshift/azure/ocs_module/deploy-with-olm.yaml +++ b/selfmanaged-openshift/azure/ocs_module/deploy-with-olm.yaml @@ -27,4 +27,3 @@ spec: name: ocs-operator source: redhat-operators sourceNamespace: openshift-marketplace - startingCSV: ocs-operator.v4.6.4 diff --git a/selfmanaged-openshift/azure/portworx_module/px-install.yaml b/selfmanaged-openshift/azure/portworx_module/px-install.yaml index 7ec06a3b..4e9279fd 100644 --- a/selfmanaged-openshift/azure/portworx_module/px-install.yaml +++ b/selfmanaged-openshift/azure/portworx_module/px-install.yaml @@ -23,4 +23,3 @@ spec: name: portworx-certified source: certified-operators sourceNamespace: openshift-marketplace - startingCSV: portworx-operator.v1.4.4 From dd1d0cccaa62e1b4562aff31666a5ab526a4b815 Mon Sep 17 00:00:00 2001 From: Pravesh Humane Date: Wed, 21 Jul 2021 07:15:07 -0700 Subject: [PATCH 4/7] Revert "added db2 patch for wkc, iis" This reverts commit 9430c71ea79b677e01a9a33290d86ba07ebdc79a. --- .../ibmcloud/cpd_install/scripts/install-wkc.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh b/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh index 8b6d9235..663d050b 100755 --- a/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh +++ b/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh @@ -30,7 +30,7 @@ cloudctl case launch --case ${CASE_PACKAGE_NAME} \ # switch to zen namespace -oc project ${NAMESPACE} +oc project zen # # Install wkc Customer Resource @@ -40,9 +40,6 @@ echo '*** executing **** oc create -f wkc-cr.yaml' result=$(oc create -f wkc-cr.yaml) echo $result -while [ "$(oc get sts -n ${NAMESPACE} | grep c-db2oltp-wkc-db2u)" = "" ]; do echo "waiting for c-db2oltp-wkc-db2u statefulset."; sleep 60; done -oc patch sts c-db2oltp-wkc-db2u -p='{"spec":{"template":{"spec":{"containers":[{"name":"db2u","tty":false}]}}}}}'; echo "patch for c-db2oltp-wkc-db2u is appied." - # check the wkc cr status ./../check-cr-status.sh wkc wkc-cr ${NAMESPACE} wkcStatus @@ -79,10 +76,6 @@ echo '*** executing **** oc create -f wkc-iis-cr.yaml' result=$(oc create -f wkc-iis-cr.yaml) echo $result -while [ "$(oc get sts -n ${NAMESPACE} | grep c-db2oltp-iis-db2u)" = "" ]; do echo "waiting for c-db2oltp-iis-db2u statefulset."; sleep 60; done -oc patch sts c-db2oltp-iis-db2u -p='{"spec":{"template":{"spec":{"containers":[{"name":"db2u","tty":false}]}}}}}'; echo "patch for c-db2oltp-iis-db2u is appied." - - # check the wkc cr status ./../check-cr-status.sh iis iis-cr ${NAMESPACE} iisStatus From b768abcb69c3fc7151346043a591e8995b26c226 Mon Sep 17 00:00:00 2001 From: Pravesh Humane Date: Wed, 21 Jul 2021 07:17:22 -0700 Subject: [PATCH 5/7] Revert "minor changes 2" This reverts commit be1229806afcc1430a7eb1bfac7c16d9ac6575c4. --- managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml b/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml index d18704d0..43fc3031 100644 --- a/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml +++ b/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml @@ -3,7 +3,7 @@ kind: Dmcaddon metadata: name: dmcaddon-cr spec: - namespace: CPD_NAMESPACE + namespace: zen storageClass: "portworx-shared-gp3" pullPrefix: cp.icr.io/cp/cpd version: "4.0.0" From b7a1fabfbfba50b90d7ca86149ac50f814dd2b3e Mon Sep 17 00:00:00 2001 From: Pravesh Humane Date: Wed, 21 Jul 2021 07:17:48 -0700 Subject: [PATCH 6/7] Revert "minor changes" This reverts commit 1cd3549f3bafb79454005ec396a773bf81738251. --- managed-openshift/ibmcloud/cpd_install/cpd.tf | 6 +++--- .../ibmcloud/cpd_install/scripts/dmc-cr.yaml | 12 ------------ .../cpd_install/scripts/install-dmc.sh | 19 ++++++++++++++----- ...tor.sh => install_bedrock_zen_operator.sh} | 0 .../{install-ccs.sh => install_ccs.sh} | 0 .../{install-wsl.sh => install_wsl.sh} | 0 6 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml rename managed-openshift/ibmcloud/cpd_install/scripts/{install-bedrock-zen-operator.sh => install_bedrock_zen_operator.sh} (100%) rename managed-openshift/ibmcloud/cpd_install/scripts/{install-ccs.sh => install_ccs.sh} (100%) rename managed-openshift/ibmcloud/cpd_install/scripts/{install-wsl.sh => install_wsl.sh} (100%) diff --git a/managed-openshift/ibmcloud/cpd_install/cpd.tf b/managed-openshift/ibmcloud/cpd_install/cpd.tf index f096bca0..66d6bd45 100644 --- a/managed-openshift/ibmcloud/cpd_install/cpd.tf +++ b/managed-openshift/ibmcloud/cpd_install/cpd.tf @@ -20,7 +20,7 @@ resource "null_resource" "bedrock_zen_operator" { working_dir = "${path.module}/scripts/" interpreter = ["/bin/bash", "-c"] - command = "./install-bedrock-zen-operator.sh" + command = "./install_bedrock_zen_operator.sh" } depends_on = [ @@ -41,7 +41,7 @@ resource "null_resource" "install_ccs" { working_dir = "${path.module}/scripts/" interpreter = ["/bin/bash", "-c"] - command = "./install-ccs.sh" + command = "./install_ccs.sh" } depends_on = [ @@ -161,7 +161,7 @@ resource "null_resource" "install_wsl" { working_dir = "${path.module}/scripts/" interpreter = ["/bin/bash", "-c"] - command = "./install-wsl.sh" + command = "./install_wsl.sh" } depends_on = [ diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml b/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml deleted file mode 100644 index 43fc3031..00000000 --- a/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: dmc.databases.ibm.com/v1 -kind: Dmcaddon -metadata: - name: dmcaddon-cr -spec: - namespace: zen - storageClass: "portworx-shared-gp3" - pullPrefix: cp.icr.io/cp/cpd - version: "4.0.0" - license: - accept: true - license: Enterprise \ No newline at end of file diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/install-dmc.sh b/managed-openshift/ibmcloud/cpd_install/scripts/install-dmc.sh index 74f8d897..74b56e6c 100755 --- a/managed-openshift/ibmcloud/cpd_install/scripts/install-dmc.sh +++ b/managed-openshift/ibmcloud/cpd_install/scripts/install-dmc.sh @@ -29,11 +29,20 @@ sleep 5m oc project ${NAMESPACE} -# Create dmc CR: -sed -i -e s#CPD_NAMESPACE#${NAMESPACE}#g dmc-cr.yaml -echo '*** executing **** oc create -f dmc-cr.yaml' -result=$(oc create -f dmc-cr.yaml) -echo $result +cat << EOF | oc apply -f - +apiVersion: dmc.databases.ibm.com/v1 +kind: Dmcaddon +metadata: + name: dmcaddon-cr +spec: + namespace: zen + storageClass: "portworx-shared-gp3" + pullPrefix: cp.icr.io/cp/cpd + version: "4.0.0" + license: + accept: true + license: Enterprise +EOF # checking status of dmc-operator ./pod-status-check.sh ibm-dmc-controller ${OP_NAMESPACE} diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/install-bedrock-zen-operator.sh b/managed-openshift/ibmcloud/cpd_install/scripts/install_bedrock_zen_operator.sh similarity index 100% rename from managed-openshift/ibmcloud/cpd_install/scripts/install-bedrock-zen-operator.sh rename to managed-openshift/ibmcloud/cpd_install/scripts/install_bedrock_zen_operator.sh diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/install-ccs.sh b/managed-openshift/ibmcloud/cpd_install/scripts/install_ccs.sh similarity index 100% rename from managed-openshift/ibmcloud/cpd_install/scripts/install-ccs.sh rename to managed-openshift/ibmcloud/cpd_install/scripts/install_ccs.sh diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/install-wsl.sh b/managed-openshift/ibmcloud/cpd_install/scripts/install_wsl.sh similarity index 100% rename from managed-openshift/ibmcloud/cpd_install/scripts/install-wsl.sh rename to managed-openshift/ibmcloud/cpd_install/scripts/install_wsl.sh From a2feea972c413763967bb4dcec695a696aea658a Mon Sep 17 00:00:00 2001 From: Pravesh Humane Date: Wed, 21 Jul 2021 07:41:44 -0700 Subject: [PATCH 7/7] changes for dmc cr --- .../ibmcloud/cpd_install/scripts/dmc-cr.yaml | 12 ++++++++++++ .../cpd_install/scripts/install-dmc.sh | 19 +++++-------------- .../cpd_install/scripts/install-wkc.sh | 2 +- 3 files changed, 18 insertions(+), 15 deletions(-) create mode 100644 managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml b/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml new file mode 100644 index 00000000..b19f8f16 --- /dev/null +++ b/managed-openshift/ibmcloud/cpd_install/scripts/dmc-cr.yaml @@ -0,0 +1,12 @@ +apiVersion: dmc.databases.ibm.com/v1 +kind: Dmcaddon +metadata: + name: dmcaddon-cr +spec: + namespace: CPD_NAMESPACE + storageClass: "portworx-shared-gp3" + pullPrefix: cp.icr.io/cp/cpd + version: "4.0.0" + license: + accept: true + license: Enterprise \ No newline at end of file diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/install-dmc.sh b/managed-openshift/ibmcloud/cpd_install/scripts/install-dmc.sh index 74b56e6c..74f8d897 100755 --- a/managed-openshift/ibmcloud/cpd_install/scripts/install-dmc.sh +++ b/managed-openshift/ibmcloud/cpd_install/scripts/install-dmc.sh @@ -29,20 +29,11 @@ sleep 5m oc project ${NAMESPACE} -cat << EOF | oc apply -f - -apiVersion: dmc.databases.ibm.com/v1 -kind: Dmcaddon -metadata: - name: dmcaddon-cr -spec: - namespace: zen - storageClass: "portworx-shared-gp3" - pullPrefix: cp.icr.io/cp/cpd - version: "4.0.0" - license: - accept: true - license: Enterprise -EOF +# Create dmc CR: +sed -i -e s#CPD_NAMESPACE#${NAMESPACE}#g dmc-cr.yaml +echo '*** executing **** oc create -f dmc-cr.yaml' +result=$(oc create -f dmc-cr.yaml) +echo $result # checking status of dmc-operator ./pod-status-check.sh ibm-dmc-controller ${OP_NAMESPACE} diff --git a/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh b/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh index 663d050b..183785fd 100755 --- a/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh +++ b/managed-openshift/ibmcloud/cpd_install/scripts/install-wkc.sh @@ -30,7 +30,7 @@ cloudctl case launch --case ${CASE_PACKAGE_NAME} \ # switch to zen namespace -oc project zen +oc project ${NAMESPACE} # # Install wkc Customer Resource