Merge branch 'master' into Implementation-of-ocp-power-automation#220

OWNERS

@@ -1,10 +1,10 @@
 reviewers:
-  - mkumatag
   - Prajyot-Parab
   - sudeeshjohn
   - yussufsh
   - bpradipt
   - cs-zhang
 approvers:
   - bpradipt
+  - cs-zhang
   - yussufsh

docs/automation_host_prereqs.md

@@ -22,7 +22,7 @@ Install the following packages on the automation host. Select the appropriate in
 **Terraform >= 0.13.0**: Please refer to the [link](https://learn.hashicorp.com/terraform/getting-started/install.html) for instructions on installing Terraform. For validating the version run `terraform version` command after install.
 
 Install Terraform and providers for Power environment:
-1. Download the Terraform binary version 0.13.5 from https://www.power-devops.com/terraform and install it to /usr/local/bin.
+1. Download and install the Terraform binary (>= 0.13.0) for Linux/ppc64le from https://www.power-devops.com/terraform.
 2. Download the required Terraform providers for Power into your TF project directory:
 ```
 $ cd <path_to_TF_project>

docs/var.tfvars-doc.md

@@ -83,10 +83,11 @@ worker                      = {instance_type    = "<worker-compute-template>", i
 ```
 These set of variables specify the username and the SSH key to be used for accessing the bastion node.
 ```
-rhel_username               = "root"
+rhel_username               = "root"  #Set it to an appropriate username for non-root user access
 public_key_file             = "data/id_rsa.pub"
 private_key_file            = "data/id_rsa"
 ```
+rhel_username is set to root. rhel_username can be set to an appropriate username having superuser privileges with no password prompt.
 Please note that only OpenSSH formatted keys are supported. Refer to the following links for instructions on creating SSH key based on your platform.
 - Windows 10 - https://phoenixnap.com/kb/generate-ssh-key-windows-10
 - Mac OSX - https://www.techrepublic.com/article/how-to-generate-ssh-keys-on-macos-mojave/
@@ -139,10 +140,24 @@ If `cluster_if_prefix` is not set, the `cluster_id` will be used only without pr
 A random value will be used for `cluster_id` if not set.
 The total length of `cluster_id_prefix`.`cluster_id` should not exceed 14 characters.
 
+### FIPS Variable for OpenShift deployment
+
+These variables will be used for deploying OCP in FIPS mode.
+Change the values as per your requirement.
+```
+fips_compliant      = false
+```
+
 ### Misc Customizations
 
 These variables provides miscellaneous customizations. For common usage scenarios these are not required and should be left unchanged.
 
+The following variables are used to define the IP address for the preconfigured external DNS and the Load-balancer 
+```
+lb_ipaddr                       = ""
+ext_dns                         = ""
+```
+
 The following variable is used to set the network adapter type for the VMs. By default the VMs will use SEA. If SRIOV is required then uncomment the variable
 ```
 network_type                = "SRIOV"
@@ -215,6 +230,10 @@ total 13452
 This variable can be used to define a different source for the helm package, like a local web server. By default, the help package will be downloaded from the official internet source.
 ```
 helm_repo                  = "https://<HTTP SERVER>/python-modules/helm-latest-linux-ppc64le.tar.gz"
+
+This variable specify the MTU value for the private network interface on RHEL and RHCOS nodes. The CNI network will have <private_network_mtu> - 50 for OpenshiftSDN and <private_network_mtu> - 100 for OVNKubernetes network provider.
+```
+private_network_mtu         = 1450
 ```
 
 These variables can be used when debugging ansible playbooks
@@ -228,6 +247,16 @@ This variable specifies the external DNS servers to forward DNS queries that can
 dns_forwarders              = "1.1.1.1; 9.9.9.9"
 ```
 
+List of [day-1 kernel arguments](https://docs.openshift.com/container-platform/4.8/installing/install_config/installing-customizing.html#installation-special-config-kargs_installing-customizing) for the cluster nodes.
+To add kernel arguments to master or worker nodes, using MachineConfig object and inject that object into the set of manifest files used by Ignition during cluster setup.
+```
+rhcos_pre_kernel_options        = []
+```
+- Example 1
+  ```
+  rhcos_pre_kernel_options   = ["rd.multipath=default","root=/dev/disk/by-label/dm-mpath-root"]
+  ```
+
 List of [kernel arguments](https://docs.openshift.com/container-platform/4.4/nodes/nodes/nodes-nodes-working.html#nodes-nodes-kernel-arguments_nodes-nodes-working) for the cluster nodes.
 Note that this will be applied after the cluster is installed and all the nodes are in `Ready` status.
 ```
@@ -285,4 +314,7 @@ This variable is used to set the default Container Network Interface (CNI) netwo
 
 ```
 cni_network_provider       = "OpenshiftSDN"
+cluster_network_cidr        = "10.128.0.0/14"
+cluster_network_hostprefix  = "23"
+service_network             = "172.30.0.0/16"
 ```

modules/1_bastion/bastion.tf

@@ -93,18 +93,18 @@ resource "null_resource" "bastion_init" {
     }
     provisioner "file" {
         content = var.private_key
-        destination = "$HOME/.ssh/id_rsa"
+        destination = ".ssh/id_rsa"
     }
     provisioner "file" {
         content = var.public_key
-        destination = "$HOME/.ssh/id_rsa.pub"
+        destination = ".ssh/id_rsa.pub"
     }
     provisioner "remote-exec" {
         inline = [
-            "sudo chmod 600 $HOME/.ssh/id_rsa*",
+            "sudo chmod 600 .ssh/id_rsa*",
             "sudo sed -i.bak -e 's/^ - set_hostname/# - set_hostname/' -e 's/^ - update_hostname/# - update_hostname/' /etc/cloud/cloud.cfg",
-            "sudo hostnamectl set-hostname --static ${lower(var.cluster_id)}-bastion-${count.index}.${var.cluster_domain}",
-            "echo 'HOSTNAME=${lower(var.cluster_id)}-bastion-${count.index}.${var.cluster_domain}' | sudo tee -a /etc/sysconfig/network > /dev/null",
+            "sudo hostnamectl set-hostname --static ${lower(var.cluster_id)}-bastion-${count.index}.${lower(var.cluster_id)}.${var.cluster_domain}",
+            "echo 'HOSTNAME=${lower(var.cluster_id)}-bastion-${count.index}.${lower(var.cluster_id)}.${var.cluster_domain}' | sudo tee -a /etc/sysconfig/network > /dev/null",
             "sudo hostname -F /etc/hostname",
             "echo 'vm.max_map_count = 262144' | sudo tee --append /etc/sysctl.conf > /dev/null",
         ]
@@ -270,7 +270,7 @@ resource "null_resource" "bastion_packages" {
     }
     provisioner "remote-exec" {
         inline = [
-           "sudo yum install -y ansible"
+           "sudo yum install -y ansible-2.9.*"
         ]
     }
     provisioner "remote-exec" {
@@ -327,11 +327,11 @@ resource "null_resource" "setup_nfs_disk" {
     }
     provisioner "remote-exec" {
         inline = [
-            "rm -rf mkdir ${local.storage_path}; mkdir -p ${local.storage_path}; chmod -R 755 ${local.storage_path}",
+            "sudo rm -rf mkdir ${local.storage_path}; sudo mkdir -p ${local.storage_path}; sudo chmod -R 755 ${local.storage_path}",
             "sudo chmod +x /tmp/create_disk_link.sh",
             # Fix for copying file from Windows OS having CR
-            "sed -i 's/\r//g' /tmp/create_disk_link.sh",
-            "/tmp/create_disk_link.sh",
+            "sudo sed -i 's/\r//g' /tmp/create_disk_link.sh",
+            "sudo /tmp/create_disk_link.sh",
             "sudo mkfs.ext4 -F /dev/${local.disk_config.disk_name}",
             "echo '/dev/${local.disk_config.disk_name} ${local.storage_path} ext4 defaults 0 0' | sudo tee -a /etc/fstab > /dev/null",
             "sudo mount ${local.storage_path}",

modules/1_bastion/versions.tf

@@ -33,5 +33,5 @@ terraform {
       version = "~> 2.3"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }

modules/2_network/versions.tf

@@ -25,5 +25,5 @@ terraform {
       version = "~> 1.32"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }

modules/3_helpernode/helpernode.tf

@@ -38,6 +38,8 @@ locals {
         bastion_master_ip   = var.bastion_ip[0]
         bastion_backup_ip   = length(var.bastion_ip) > 1 ? slice(var.bastion_ip, 1, length(var.bastion_ip)) : []
         forwarders      = var.dns_forwarders
+        lb_ipaddr       = var.lb_ipaddr
+        ext_dns         = var.ext_dns
         gateway_ip      = var.gateway_ip
         netmask         = cidrnetmask(var.cidr)
         broadcast       = cidrhost(var.cidr,-1)
@@ -72,7 +74,8 @@ locals {
         install_tarball          = var.openshift_install_tarball
     }
     helpernode_inventory = {
-        bastion_ip  = var.bastion_ip
+        rhel_username = var.rhel_username
+        bastion_ip    = var.bastion_ip
     }
 }
 
@@ -155,21 +158,21 @@ resource "null_resource" "config" {
 
     provisioner "file" {
         content     = templatefile("${path.module}/templates/helpernode_inventory", local.helpernode_inventory)
-        destination = "$HOME/ocp4-helpernode/inventory"
+        destination = "ocp4-helpernode/inventory"
     }
     provisioner "file" {
         content     = var.pull_secret
-        destination = "$HOME/.openshift/pull-secret"
+        destination = ".openshift/pull-secret"
     }
     provisioner "file" {
         content     = templatefile("${path.module}/templates/helpernode_vars.yaml", local.helpernode_vars)
-        destination = "$HOME/ocp4-helpernode/helpernode_vars.yaml"
+        destination = "ocp4-helpernode/helpernode_vars.yaml"
     }
     provisioner "remote-exec" {
         inline = [
             "sed -i \"/^helper:.*/a \\ \\ networkifacename: $(ip r | grep \"${var.cidr} dev\" | awk '{print $3}')\" ocp4-helpernode/helpernode_vars.yaml",
             "echo 'Running ocp4-helpernode playbook...'",
-            "cd ocp4-helpernode && ansible-playbook  -i inventory -e @helpernode_vars.yaml tasks/main.yml ${var.ansible_extra_options}"
+            "cd ocp4-helpernode && ansible-playbook  -i inventory -e @helpernode_vars.yaml tasks/main.yml ${var.ansible_extra_options} --become"
         ]
     }
 }

modules/3_helpernode/templates/helpernode_inventory

@@ -1,4 +1,4 @@
 [vmhost]
 %{ for ip in bastion_ip ~}
-${ip} ansible_connection=ssh ansible_user=root
+${ip} ansible_connection=ssh ansible_user=${rhel_username}
 %{ endfor ~}

modules/3_helpernode/templates/helpernode_vars.yaml

@@ -21,10 +21,16 @@ dns:
   domain: "${cluster_domain}"
   clusterid: "${cluster_id}"
   forwarder1: "${forwarders}"
+%{ if lb_ipaddr != "" }
+  lb_ipaddr: "${lb_ipaddr}"
+%{ endif }
 dhcp:
   router: "${gateway_ip}"
   bcast: "${broadcast}"
   netmask: "${netmask}"
+%{ if ext_dns != "" }
+  dns: "${ext_dns}"
+%{ endif }
   ipid: "${ipid}"
   netmaskid: "${netmask}"
   poolstart: "${pool.start}"

modules/3_helpernode/variables.tf

@@ -29,6 +29,8 @@ variable "dns_forwarders" {
     default   = "8.8.8.8; 9.9.9.9"
 }
 
+variable "lb_ipaddr" {}
+variable "ext_dns" {}
 variable "gateway_ip" {}
 variable "cidr" {}
 variable "allocation_pools" {}

modules/3_helpernode/versions.tf

@@ -25,5 +25,5 @@ terraform {
       version = "~> 2.1"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }

modules/4_nodes/versions.tf

@@ -33,5 +33,5 @@ terraform {
       version = "~> 2.3"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }

modules/5_install/install.tf

@@ -24,7 +24,10 @@ locals {
 
     ocp_release_repo    = "ocp4/openshift4"
 
+    bastion_count = lookup(var.bastion, "count", 1)
+
     install_inventory = {
+        rhel_username   = var.rhel_username
         bastion_hosts   = [for ix in range(length(var.bastion_ip)) : "${var.cluster_id}-bastion-${ix}"]
         bootstrap_host  = var.bootstrap_ip == "" ? "" : "bootstrap"
         master_hosts    = [for ix in range(length(var.master_ips)) : "master-${ix}"]
@@ -40,28 +43,36 @@ locals {
     local_registry_ocp_image = "registry.${var.cluster_id}.${local.cluster_domain}:5000/${local.ocp_release_repo}:${var.ocp_release_tag}"
 
     install_vars = {
-        bastion_vip             = var.bastion_vip
-        cluster_id              = var.cluster_id
-        cluster_domain          = local.cluster_domain
-        pull_secret             = var.pull_secret
-        public_ssh_key          = var.public_key
-        storage_type            = var.storage_type
-        log_level               = var.log_level
-        release_image_override  = var.enable_local_registry ? local.local_registry_ocp_image : var.release_image_override
-        enable_local_registry   = var.enable_local_registry
-        node_connection_timeout = 60 * var.connection_timeout
-        rhcos_kernel_options    = var.rhcos_kernel_options
-        sysctl_tuned_options    = var.sysctl_tuned_options
-        sysctl_options          = var.sysctl_options
-        match_array             = indent(2,var.match_array)
-        setup_squid_proxy       = var.setup_squid_proxy
-        squid_source_range      = var.cidr
-        proxy_url               = local.proxy.server == "" ? "" : "http://${local.proxy.user_pass}${local.proxy.server}:${local.proxy.port}"
-        no_proxy                = var.cidr
-        chrony_config           = var.chrony_config
-        chrony_config_servers   = var.chrony_config_servers
-        chrony_allow_range      = var.cidr
-        cni_network_provider    = var.cni_network_provider
+        bastion_vip                = var.bastion_vip
+        cluster_id                 = var.cluster_id
+        cluster_domain             = local.cluster_domain
+        pull_secret                = var.pull_secret
+        public_ssh_key             = var.public_key
+        storage_type               = var.storage_type
+        log_level                  = var.log_level
+        release_image_override     = var.enable_local_registry ? local.local_registry_ocp_image : var.release_image_override
+        enable_local_registry      = var.enable_local_registry
+        fips_compliant             = var.fips_compliant
+        node_connection_timeout    = 60 * var.connection_timeout
+        rhcos_pre_kernel_options   = var.rhcos_pre_kernel_options
+        rhcos_kernel_options       = var.rhcos_kernel_options
+        sysctl_tuned_options       = var.sysctl_tuned_options
+        sysctl_options             = var.sysctl_options
+        match_array                = indent(2,var.match_array)
+        setup_squid_proxy          = var.setup_squid_proxy
+        squid_source_range         = var.cidr
+        proxy_url                  = local.proxy.server == "" ? "" : "http://${local.proxy.user_pass}${local.proxy.server}:${local.proxy.port}"
+        no_proxy                   = var.cidr
+        chrony_config              = var.chrony_config
+        chrony_config_servers      = var.chrony_config_servers
+        chrony_allow_range         = var.cidr
+        cni_network_provider       = var.cni_network_provider
+        cluster_network_cidr       = var.cluster_network_cidr
+        cluster_network_hostprefix = var.cluster_network_hostprefix
+        service_network            = var.service_network
+        # Set CNI network MTU to MTU - 100 for OVNKubernetes and MTU - 50 for OpenShiftSDN(default).
+        # Add new conditions here when we have more network providers
+        cni_network_mtu = var.cni_network_provider == "OVNKubernetes" ? var.private_network_mtu - 100 : var.private_network_mtu - 50
     }
 
     upgrade_vars = {
@@ -125,8 +136,34 @@ resource "null_resource" "prep_playbooks_tools_curl" {
     }
 }
 
+resource "null_resource" "pre_install" {
+  count      = local.bastion_count
+
+  connection {
+    type        = "ssh"
+    user        = var.rhel_username
+    host        = var.bastion_ip[count.index]
+    private_key = var.private_key
+    agent       = var.ssh_agent
+    timeout     = "${var.connection_timeout}m"
+    bastion_host = var.jump_host
+  }
+
+  # DHCP config for setting MTU; Since helpernode DHCP template does not support MTU setting
+  provisioner "remote-exec" {
+    inline = [
+      # Set specified mtu for private interface.
+      "sudo ip link set dev $(ip r | grep \"${var.cidr} dev\" | awk '{print $3}') mtu ${var.private_network_mtu}",
+      "echo MTU=${var.private_network_mtu} | sudo tee -a /etc/sysconfig/network-scripts/ifcfg-$(ip r | grep ${var.cidr} | awk '{print $3}')",
+      # DHCP config for setting MTU;
+      "sed -i.mtubak '/option routers/i option interface-mtu ${var.private_network_mtu};' /etc/dhcp/dhcpd.conf",
+      "sudo systemctl restart dhcpd.service"
+    ]
+  }
+}
+
 resource "null_resource" "install" {
-    depends_on = [null_resource.prep_playbooks_tools_git, null_resource.prep_playbooks_tools_curl]
+    depends_on = [null_resource.prep_playbooks_tools_git, null_resource.prep_playbooks_tools_curl, null_resource.pre_install]
     triggers = {
         worker_count    = length(var.worker_ips)
     }
@@ -143,11 +180,11 @@ resource "null_resource" "install" {
 
     provisioner "file" {
         content     = templatefile("${path.module}/templates/install_inventory", local.install_inventory)
-        destination = "$HOME/ocp4-playbooks/inventory"
+        destination = "ocp4-playbooks/inventory"
     }
     provisioner "file" {
         content     = templatefile("${path.module}/templates/install_vars.yaml", local.install_vars)
-        destination = "$HOME/ocp4-playbooks/install_vars.yaml"
+        destination = "ocp4-playbooks/install_vars.yaml"
     }
     provisioner "remote-exec" {
         inline = [
@@ -173,7 +210,7 @@ resource "null_resource" "upgrade" {
 
     provisioner "file" {
         content     = templatefile("${path.module}/templates/upgrade_vars.yaml", local.upgrade_vars)
-        destination = "$HOME/ocp4-playbooks/upgrade_vars.yaml"
+        destination = "ocp4-playbooks/upgrade_vars.yaml"
     }
     provisioner "remote-exec" {
         inline = [

modules/5_install/templates/install_inventory

@@ -1,6 +1,6 @@
 [bastion]
 %{ for bastion in bastion_hosts ~}
-${bastion} ansible_connection=ssh ansible_user=root
+${bastion} ansible_connection=ssh ansible_user=${rhel_username}
 %{ endfor ~}
 
 %{ if bootstrap_host != "" ~}