Support Photon OS autoinstall (#221)

* Support Photon OS autoinstall

Signed-off-by: Qi Zhang <qiz@vmware.com>

README.md

@@ -72,7 +72,7 @@ This project supports below scenarios for end-to-end guest OS validation testing
 | SUSE Linux Enterprise 15 SP3, SP4               | :heavy_check_mark:               |                          | :heavy_check_mark:                 |
 | SUSE Linux Enterprise 12 SP5, 15 SP0/SP1/SP2    |                                  |                          | :heavy_check_mark:                 |
 | Photon OS 3.x                                   | :heavy_check_mark:               | :heavy_check_mark:       | :heavy_check_mark:                 |
-| Photon OS 4.0                                   |                                  | :heavy_check_mark:       | :heavy_check_mark:                 |
+| Photon OS 4.x                                   | :heavy_check_mark:               | :heavy_check_mark:       | :heavy_check_mark:                 |
 | Ubuntu 18.04, 18.10, 20.04, 20.10, 21.04, 21.10 live-server   | :heavy_check_mark:               |                          | :heavy_check_mark:                 |
 | Ubuntu 20.04, 20.10, 21.04, 21.10 cloud image                 |                                  | :heavy_check_mark:       | :heavy_check_mark:                 |
 | Ubuntu 18.04, 18.10, 20.04, 20.10, 21.04, 21.10 desktop       |                                  |                          | :heavy_check_mark:                 |

autoinstall/CentOS/7/ks.cfg

@@ -5,8 +5,8 @@ auth --enableshadow --passalgo=sha512
 cdrom
 # Use graphical install
 graphical
-# Run the Setup Agent on first boot
-firstboot --enable
+# Do not run gnome-initial-setupgnome-initial-setup on first boot
+firstboot --disable
 ignoredisk --only-use={{ boot_disk_name }}
 # Keyboard layouts
 keyboard --vckeymap=us --xlayouts='us'
@@ -91,4 +91,5 @@ sed -i 's/^ssh_pwauth:.*/ssh_pwauth: yes/' /etc/cloud/cloud.cfg
 # Add new user to sudoer
 echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 {% endif %}
+echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
 %end

autoinstall/CentOS/7/minimal/ks.cfg

@@ -5,8 +5,8 @@ auth --enableshadow --passalgo=sha512
 cdrom
 # Use graphical install
 graphical
-# Run the Setup Agent on first boot
-firstboot --enable
+# Do not run gnome-initial-setupgnome-initial-setup on first boot
+firstboot --disable
 ignoredisk --only-use={{ boot_disk_name }}
 # Keyboard layouts
 keyboard --vckeymap=us --xlayouts='us'
@@ -74,4 +74,5 @@ sed -i 's/^ssh_pwauth:.*/ssh_pwauth: yes/' /etc/cloud/cloud.cfg
 # Add new user to sudoer
 echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 {% endif %}
+echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
 %end

autoinstall/Photon/ks.cfg

@@ -0,0 +1,27 @@
+{
+    "hostname": "photon-autoinstall-machine",
+    "password": {
+        "crypted": true,
+        "text": "{{ vm_password_hash }}"
+        },
+    "disk": "/dev/{{ boot_disk_name }}",
+    "packagelist_file": "packages_ova.json",
+    "arch": "x86_64",
+    "additional_packages": ["vim","gawk","sudo"],
+    "install_linux_esx": true,
+    "eject_cdrom": true,
+    "postinstall": [
+        "#!/bin/sh",
+{% if new_user is defined and new_user != 'root' %}
+        "useradd -p '{{ vm_password_hash }}' -m {{ new_user }}",
+        "echo '{{ ssh_public_key }}' >/home/{{ new_user }}/.ssh/authorized_keys",
+        "echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}",
+{% endif %}
+        "tdnf --disablerepo=photon-updates install -y sg3_utils tar",
+        "sed -r -i 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/g' /etc/ssh/sshd_config",
+        "systemctl stop iptables",
+        "systemctl disable iptables",
+        "echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0"
+    ],
+    "public_key": "{{ ssh_public_key }}"
+}

autoinstall/README.md

@@ -8,3 +8,5 @@
 7. For Windows 10 or later unattend auto-install, please use files under Windows/win10.
 8. For Windows Server LTSC unattend auto-install, please use files under Windows/win_server.
 9. For Windows Server SAC unattend auto-install, please use files under Windows/win_server_sac.
+10. For Photon OS 3.0 or later unattend auto-install, please use file Photon/ks.cfg.
+10. For Ubuntu Server 20.04 or later unattend auto-install, please use file Ubuntu/Server/user-data.j2.

autoinstall/RHEL/7/server_with_GUI/ks.cfg

@@ -7,8 +7,8 @@ repo --name="Server-ResilientStorage" --baseurl=file:///run/install/repo/addons/
 cdrom
 # Use graphical install
 graphical
-# Run the Setup Agent on first boot
-firstboot --enable
+# Do not run gnome-initial-setupgnome-initial-setup on first boot
+firstboot --disable
 ignoredisk --only-use={{ boot_disk_name }}
 # Keyboard layouts
 keyboard --vckeymap=us --xlayouts='us'
@@ -106,4 +106,5 @@ sed -i 's/^ssh_pwauth:.*/ssh_pwauth: yes/' /etc/cloud/cloud.cfg
 # Add new user to sudoer
 echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 {% endif %}
+echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
 %end

autoinstall/RHEL/7/server_without_GUI/ks.cfg

@@ -7,8 +7,8 @@ repo --name="Server-ResilientStorage" --baseurl=file:///run/install/repo/addons/
 cdrom
 # Use graphical install
 graphical
-# Run the Setup Agent on first boot
-firstboot --enable
+# Do not run gnome-initial-setupgnome-initial-setup on first boot
+firstboot --disable
 ignoredisk --only-use={{ boot_disk_name }}
 # Keyboard layouts
 keyboard --vckeymap=us --xlayouts='us'
@@ -76,4 +76,5 @@ sed -i 's/^ssh_pwauth:.*/ssh_pwauth: yes/' /etc/cloud/cloud.cfg
 # Add new user to sudoer
 echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 {% endif %}
+echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
 %end

autoinstall/RHEL/8/server_with_GUI/ks.cfg

@@ -77,4 +77,5 @@ echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 sed -i '/\[daemon\]/a AutomaticLogin={{ new_user }}' /etc/gdm/custom.conf
 sed -i '/\[daemon\]/a AutomaticLoginEnable=True' /etc/gdm/custom.conf
 {% endif %}
+echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
 %end

autoinstall/RHEL/8/server_without_GUI/ks.cfg

@@ -74,4 +74,5 @@ sed -i 's/^#PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config
 # Add new user to sudoer
 echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 {% endif %}
+echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
 %end

autoinstall/SLE/15/SP3/SLED/autoinst.xml

@@ -1525,6 +1525,7 @@
 {% if new_user is defined and new_user != 'root' %}
         echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 {% endif %}
+        echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
         ]]>
        </source>
       </script>

autoinstall/SLE/15/SP3/SLES/autoinst.xml

@@ -1430,6 +1430,7 @@
 {% if new_user is defined and new_user != 'root' %}
         echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 {% endif %}
+        echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
         ]]>
        </source>
       </script>

autoinstall/SLE/15/SP3/SLES_Minimal/autoinst.xml

@@ -1008,6 +1008,7 @@
 {% if new_user is defined and new_user != 'root' %}
         echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 {% endif %}
+        echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
         ]]>
        </source>
       </script>

autoinstall/SLE/15/SP4/SLED/autoinst.xml

@@ -1519,6 +1519,7 @@
 {% if new_user is defined and new_user != 'root' %}
         echo '{{ new_user }} ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/{{ new_user }}
 {% endif %}
+        echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0
         ]]>
        </source>
       </script>

linux/deploy_vm/templates/ubuntu-iso-user-data.j2 → autoinstall/Ubuntu/Server/user-data.j2

@@ -2,20 +2,20 @@
 autoinstall:
   version: 1
   early-commands:
-    - echo "{{ autoinstall_start_msg }}" >/dev/ttyS0
+    - echo '{{ autoinstall_start_msg }}' >/dev/ttyS0
   refresh-installer:
     update: no
   locale: en_US
   keyboard:
     layout: en
     variant: uk
   user-data:
+    disable_root: false
+    ssh_pwauth: yes
     chpasswd:
       list: |
         root:{{ vm_password_hash }}
       expire: false
-    disable_root: false
-    ssh_pwauth: yes
 {% if new_user is defined and new_user != 'root' %}
     users:
       - name: {{ new_user }}
@@ -41,5 +41,6 @@ autoinstall:
     - sg3-utils
   late-commands:
     - rm -f /etc/cloud/cloud.cfg.d/*-installer.cfg 2>/dev/null
+    - echo 'Acquire::ForceIPv4 "true";' >>/etc/apt/apt.conf.d/99force-ipv4
     - sed -i 's/^#PermitRootLogin .*/PermitRootLogin yes/' /target/etc/ssh/sshd_config
-    - echo 'Ubuntu autoinstall is completed' >/target/var/log/autoinstall.log
+    - echo '{{ autoinstall_complete_msg }}' >/dev/ttyS0

common/README.md

@@ -83,8 +83,8 @@
 * vm_shell_in_guest.yml: Execute shell command in guest
 * vm_guest_file_operation.yml: Create/delete directory or fetch/copy files in guest with VMware tools installed
 
-### Tasks for vmware.log
-* vm_search_vmware_log.yml: Search vmware.log with a keyword
+### Tasks for VM log files
+* vm_wait_log_msg.yml: Check or wait for a message appear in VM's log file
 
 ### Tasks for VM vmx config
 * vm_get_extra_config.yml: Get all configs in the vmx file
@@ -114,6 +114,8 @@
 * esxi_get_datastores.yml: Get ESXi server datastores info
 * esxi_get_datastore_info.yml: Get specified datastore info on ESXi server
 * esxi_check_delete_datastore_file.yml: Get datastore file status or delete file
+* esxi_upload_datastore_file.yml: Upload file to ESXi datastore
+* esxi_download_datastore_file.yml: Download file from ESXi datastore
 
 ### Tasks for server information
 * esxi_get_host_facts.yml: Get ESXi server host facts
@@ -131,8 +133,8 @@
 
 ## Localhost tasks
 ### Tasks for NFS mount and unmount
-* nfs_mount.yml: Mount an NFS share folder on localhost
-* nfs_unmount.yml: Unmount a NFS share folder
+* local_mount.yml: Mount an NFS share folder or ISO image on localhost
+* local_unmount.yml: Unmount a NFS share folder or ISO image from localhost
 * delete_local_file.yml: Delete local file or directory
 * skip_test_case.yml: Tasks for skipping testcase and ending play
 

common/create_temp_file_dir.yml

@@ -3,6 +3,7 @@
 ---
 # Create a temporary file or directory
 # Parameters:
+#   tmp_dir: (Optional) the location where temporary file or diretotry will be created.
 #   tmp_state: file or directory. By default, temporary file will be created
 #   tmp_prefix: the prefix of temporary file or directory
 #   tmp_suffix: the suffix of temporar file or directory
@@ -16,6 +17,7 @@
 
 - name: "Create temporary {{ tmp_state }}"
   tempfile:
+    path: "{{ tmp_dir | default(omit) }}"
     state: "{{ tmp_state }}"
     prefix: "{{ tmp_prefix | default(omit) }}"
     suffix: "{{ tmp_suffix | default(omit) }}"

common/esxi_download_datastore_file.yml

@@ -0,0 +1,41 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: BSD-2-Clause
+---
+# Download file from ESXi datastore
+# Parameters:
+#   src_datastore: the datastore name of the file. e.g. datastore1
+#   src_file_path: the relative file path in datastore. e.g. vm_name/vmware.log
+#   dest_file_path: the downloaded file path at localhost. e.g. /tmp/downloaded_vmware.log
+#   download_file_timeout: timeout in seconds for downloading datastore file. Default is 300s.
+
+- include_tasks: esxi_check_delete_datastore_file.yml
+  vars:
+    file_in_datastore_ops: file
+    file_in_datastore: "{{ src_datastore }}"
+    file_in_datastore_path: "{{ src_file_path }}"
+    file_in_datastore_failed_ignore: False
+
+- name: "Check datastore file URL exists"
+  assert:
+    that:
+      - ds_file_result.url is defined
+      - ds_file_result.url
+    fail_msg: "There is no URL for downloading '{{ src_file_path }}' from datastore '{{ src_datastore }}'"
+
+- name: "Set fact of downloading URL for the src datastore file"
+  set_fact:
+    datastore_file_url: "{{ ds_file_result.url }}"
+
+- name: "Download datastore file"
+  get_url:
+    url: "{{ datastore_file_url }}"
+    dest: "{{ dest_file_path }}"
+    url_username: "{{ vsphere_host_user }}"
+    url_password: "{{ vsphere_host_user_password }}"
+    validate_certs: "{{ validate_certs | default(False) }}"
+    mode: "0644"
+    timeout: "{{ download_file_timeout | default(300) }}"
+  register: datastore_file_download_result
+
+- name: "Print datastore file download result"
+  debug: var=datastore_file_download_result

common/esxi_upload_datastore_file.yml

@@ -0,0 +1,25 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: BSD-2-Clause
+---
+# Upload a local file to ESXi datastore
+# Parameters
+#   src_file_path: the file path on localhost. e.g. /tmp/temp_file
+#   dest_datastore: the datastore name where to upload file. e.g. datastore1
+#   dest_file_path: the relative file path on ESXi server on datastore. e.g. vm_name/uploaded_file_name
+#   upload_file_timeout: timeout in seconds to upload the file. Default is 300s.
+
+- name: "Upload local file to ESXi datastore"
+  community.vmware.vsphere_copy:
+    hostname: "{{ vsphere_host_name }}"
+    username: "{{ vsphere_host_user }}"
+    password: "{{ vsphere_host_user_password }}"
+    validate_certs: "{{ validate_certs | default(False) }}"
+    datacenter: "{{ vsphere_host_datacenter }}"
+    datastore: "{{ dest_datastore | default(datastore) }}"
+    src: "{{ src_file_path }}"
+    path: "{{ dest_file_path }}"
+    timeout: "{{ upload_file_timeout | default(300) }}"
+  register: upload_file_result
+
+- name: "Print the result of uploading file to ESXi datastore"
+  debug: var=upload_file_result

common/local_mount.yml

@@ -0,0 +1,24 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: BSD-2-Clause
+---
+# Mount a NFS storage or an ISO image on local machine
+# Parameters:
+#   mount_src: The mount point source, which could be an ISO image path or NFS server path
+#   mount_path: The local path to mount the source
+#   mount_fstype: The fstype of mount point, such as nfs, iso9660, etc
+#   mount_opts: options for mounting filesystem
+#
+- name: "Mount {{ mount_src }} to {{ mount_path }} on localhost"
+  mount:
+    path: "{{ mount_path }}"
+    src: "{{ mount_src }}"
+    opts: "{{ mount_opts | default('ro,nolock') }}"
+    fstab: "/tmp/temp.fstab"
+    fstype: "{{ mount_fstype }}"
+    boot: false
+    state: mounted
+  register: mount_result
+  become: yes
+
+- name: "Display the mount result"
+  debug: var=mount_result

common/local_unmount.yml

@@ -0,0 +1,17 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: BSD-2-Clause
+---
+# Unmount a NFS storage or an ISO image from local machine
+# Parameters:
+#   mount_path: The local mount point
+#
+- name: "Unmount {{ mount_path }} on localhost"
+  mount:
+    path: "{{ mount_path }}"
+    fstab: "/tmp/temp.fstab"
+    state: absent
+  become: yes
+  register: unmount_result
+
+- name: "Display the unmount result"
+  debug: var=unmount_result

common/vm_add_serial_port.yml

@@ -9,7 +9,7 @@
 #   
 - name: "Set default serial port output file"
   set_fact:
-    vm_serial_port_file_path: "[{{ datastore }}] {{ vm_name }}/serial.log"
+    vm_serial_port_file_path: "{{ vm_files_path_ds }}/serial-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}.log"
   when: vm_serial_port_file_path is undefined or not vm_serial_port_file_path
 
 - name: "Add a serial port using output file"