From ec4dd5128f893d64d62e3b0325d773ac4ff83f8c Mon Sep 17 00:00:00 2001 From: Delphix Engineering Date: Sat, 27 Apr 2024 04:15:40 +0000 Subject: [PATCH] Version: 24.1.3-0ubuntu1~20.04.1 -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 Format: 3.0 (quilt) Source: cloud-init Binary: cloud-init Architecture: all Version: 24.1.3-0ubuntu1~20.04.1 Maintainer: Ubuntu Developers Homepage: https://cloud-init.io/ Standards-Version: 4.4.1 Vcs-Browser: https://github.com/canonical/cloud-init/tree/ubuntu/devel Vcs-Git: https://github.com/canonical/cloud-init -b ubuntu/devel Build-Depends: debhelper (>= 9.20160709), dh-python, iproute2, po-debconf, python3, python3-configobj, python3-debconf, python3-jinja2, python3-jsonpatch, python3-jsonschema, python3-mock, python3-netifaces, python3-oauthlib, python3-pytest, python3-pytest-mock, python3-requests, python3-serial, python3-setuptools, python3-yaml, python3-responses, python3-passlib Package-List: cloud-init deb admin optional arch=all Checksums-Sha1: a4e3711a745748f6b711033b3b5e9fdd24dcf591 1715393 cloud-init_24.1.3.orig.tar.gz 80471be2982a8b7c4992b69d8161f05c9b7a29d3 96204 cloud-init_24.1.3-0ubuntu1~20.04.1.debian.tar.xz Checksums-Sha256: e06636cccc1b1766cfbcaf1ae422fce9aee8b869bb9af0d4388f6729c63a8916 1715393 cloud-init_24.1.3.orig.tar.gz c318b18f3be3475ff110018a7ca8d14ec02e5ed04c3c92821a0622fe3ca035de 96204 cloud-init_24.1.3-0ubuntu1~20.04.1.debian.tar.xz Files: d87eee25fcc11a55b77ec3cbf93fc752 1715393 cloud-init_24.1.3.orig.tar.gz 56e553c7afb264fb08291f87e9318711 96204 cloud-init_24.1.3-0ubuntu1~20.04.1.debian.tar.xz Python-Version: all -----BEGIN PGP SIGNATURE----- iQGzBAEBCgAdFiEEsyC5UhjFbFFc1+BoZ0j1UGIrpdQFAmYUJewACgkQZ0j1UGIr pdSnWwwAi09IRqmPPwAojOAg8/n01wIDjnCevLwIdREG9zQH8Jc9D9c5oIsZCDmS oEqCvQnDCKKpES6wVsw0VUWpkdhBVk+oImK7oNLMRr5/Fa1lhlK9GW+qgL5JXuXk R3jLcUnNeBZlTHoP6auxnePi5/uTBF3HJU30bIBcZAVSO0ZIULjKiMcfW8kwIK35 BqngoVw9JdFe0XsCCL9Y5SHRQxCQG8Q5ywxRk/JPjusubo1+5PpSQ6MN92/frQOu gsyzgbK5JDOdVaJgoomDjPSR5yKpbYqKRaVadpRwaEDkFwvCMCyIQLDiuamRd1pv /zQnZhgoykfzJzSuGKiSns0dln25JZlIZDQJXxKaVqEt5yi9HT4f17Z5IPfQX54T sLeeXNT4xihEroD2GDofoJQzkaWKYFjsYRb6y28Uk0ilrvfB3IG0TNJw8TAAQnEi rlnrDaUOJwoTKY2KMePCO5/6goJ3n/Slelh5FDDGQMHy1oFvfYQ7Zfmpqq8TSioz 7TMjlciY =okcW -----END PGP SIGNATURE----- --- .github/workflows/check_format.yml | 6 +- .github/workflows/integration.yml | 2 +- .github/workflows/linkcheck.yml | 2 + .github/workflows/unit.yml | 18 +- .pc/applied-patches | 3 + .../cloudinit/features.py | 2 +- .../unittests/config/test_cc_set_passwords.py | 9 +- .../config/cloud.cfg.tmpl | 372 ++ .../cloudinit/net/netplan.py | 8 +- .../tests/unittests/distros/test_netconfig.py | 37 - .../tests/unittests/test_net.py | 655 +++- .../cloudinit/features.py | 2 +- .../cloudinit/sources/DataSourceEc2.py | 1212 ++++++ .../cloudinit/features.py | 2 +- .../tests/unittests/distros/test_netconfig.py | 37 - .../config/cloud.cfg.tmpl | 8 +- .../cloudinit/cloud.py | 116 + .../cloudinit/config/cc_lxd.py | 531 +++ .../cloudinit/config/cc_snap.py | 199 + .../cloudinit/config/cc_ubuntu_autoinstall.py | 147 + .../cloudinit/util.py | 3306 +++++++++++++++++ .../systemd/cloud-config.service.tmpl | 21 + .../modules/test_frequency_override.py | 45 + .../tests/unittests/config/test_cc_lxd.py | 481 +++ .../tests/unittests/config/test_cc_snap.py | 323 ++ .../config/test_cc_ubuntu_autoinstall.py | 160 + .../tests/unittests/util.py | 199 + .../cloudinit/cmd/status.py | 478 +-- .../tests/unittests/cmd/test_status.py | 307 +- .../cloudinit/cmd/status.py | 477 +-- .../tests/unittests/cmd/test_status.py | 311 +- .pylintrc | 2 +- ChangeLog | 281 ++ Makefile | 43 +- bash_completion/cloud-init | 8 +- cloudinit/analyze/show.py | 6 +- cloudinit/apport.py | 121 +- cloudinit/cmd/clean.py | 3 +- cloudinit/cmd/cloud_id.py | 10 +- cloudinit/cmd/devel/hotplug_hook.py | 67 +- cloudinit/cmd/devel/logs.py | 15 +- cloudinit/cmd/devel/render.py | 8 + cloudinit/cmd/main.py | 41 +- cloudinit/cmd/query.py | 26 +- cloudinit/cmd/status.py | 477 +-- cloudinit/config/cc_ansible.py | 6 +- cloudinit/config/cc_apt_configure.py | 122 +- cloudinit/config/cc_apt_pipelining.py | 2 +- cloudinit/config/cc_bootcmd.py | 8 +- cloudinit/config/cc_ca_certs.py | 12 +- cloudinit/config/cc_disk_setup.py | 10 +- cloudinit/config/cc_final_message.py | 6 +- cloudinit/config/cc_growpart.py | 86 +- cloudinit/config/cc_install_hotplug.py | 49 +- cloudinit/config/cc_keys_to_console.py | 8 +- cloudinit/config/cc_lxd.py | 2 +- cloudinit/config/cc_mcollective.py | 2 +- cloudinit/config/cc_migrator.py | 102 - cloudinit/config/cc_mounts.py | 16 +- cloudinit/config/cc_ntp.py | 2 +- .../cc_package_update_upgrade_install.py | 13 +- cloudinit/config/cc_phone_home.py | 10 +- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/config/cc_puppet.py | 4 +- cloudinit/config/cc_reset_rmc.py | 2 +- cloudinit/config/cc_resizefs.py | 36 +- cloudinit/config/cc_rh_subscription.py | 2 +- cloudinit/config/cc_rightscale_userdata.py | 130 - cloudinit/config/cc_seed_random.py | 13 +- cloudinit/config/cc_set_hostname.py | 11 +- cloudinit/config/cc_snap.py | 3 +- cloudinit/config/cc_ssh.py | 39 +- cloudinit/config/cc_ubuntu_autoinstall.py | 2 +- ...c_ubuntu_advantage.py => cc_ubuntu_pro.py} | 210 +- cloudinit/config/cc_update_hostname.py | 9 + cloudinit/config/cc_wireguard.py | 2 +- cloudinit/config/cc_write_files.py | 4 +- cloudinit/config/cc_zypper_add_repo.py | 2 +- cloudinit/config/modules.py | 49 +- cloudinit/config/schema.py | 445 ++- .../schemas/schema-cloud-config-v1.json | 288 +- .../schemas/schema-network-config-v1.json | 29 +- cloudinit/distros/__init__.py | 279 +- cloudinit/distros/alpine.py | 11 +- cloudinit/distros/amazon.py | 6 + cloudinit/distros/arch.py | 134 +- cloudinit/distros/bsd.py | 12 +- cloudinit/distros/bsd_utils.py | 4 +- cloudinit/distros/debian.py | 18 +- cloudinit/distros/freebsd.py | 14 +- cloudinit/distros/gentoo.py | 137 +- cloudinit/distros/mariner.py | 5 +- cloudinit/distros/net_util.py | 187 - cloudinit/distros/netbsd.py | 14 +- cloudinit/distros/networking.py | 2 +- cloudinit/distros/openbsd.py | 6 +- cloudinit/distros/opensuse.py | 10 +- cloudinit/distros/package_management/apt.py | 37 +- .../package_management/package_manager.py | 4 + cloudinit/distros/package_management/snap.py | 3 + cloudinit/distros/parsers/ifconfig.py | 6 + cloudinit/distros/photon.py | 9 +- cloudinit/distros/rhel.py | 13 +- cloudinit/distros/rhel_util.py | 2 +- cloudinit/distros/ubuntu.py | 3 +- cloudinit/distros/ug_util.py | 2 +- cloudinit/dmi.py | 89 +- cloudinit/features.py | 2 +- cloudinit/gpg.py | 2 +- cloudinit/handlers/__init__.py | 4 +- cloudinit/handlers/boot_hook.py | 10 +- cloudinit/handlers/jinja_template.py | 21 +- cloudinit/helpers.py | 21 +- cloudinit/log.py | 17 +- cloudinit/mergers/__init__.py | 2 +- cloudinit/mergers/m_list.py | 2 +- cloudinit/net/__init__.py | 11 +- cloudinit/net/activators.py | 36 + cloudinit/net/bsd.py | 2 +- cloudinit/net/cmdline.py | 2 +- cloudinit/net/dhcp.py | 708 +++- cloudinit/net/eni.py | 2 +- cloudinit/net/ephemeral.py | 29 +- cloudinit/net/netplan.py | 8 +- cloudinit/net/network_manager.py | 246 +- cloudinit/net/network_state.py | 4 +- cloudinit/net/networkd.py | 13 +- cloudinit/net/sysconfig.py | 6 +- cloudinit/persistence.py | 2 +- cloudinit/settings.py | 3 + cloudinit/sources/DataSourceAliYun.py | 2 + cloudinit/sources/DataSourceAltCloud.py | 8 +- cloudinit/sources/DataSourceAzure.py | 44 +- cloudinit/sources/DataSourceBigstep.py | 2 +- cloudinit/sources/DataSourceCloudStack.py | 117 +- cloudinit/sources/DataSourceConfigDrive.py | 6 +- cloudinit/sources/DataSourceEc2.py | 239 +- cloudinit/sources/DataSourceGCE.py | 8 +- cloudinit/sources/DataSourceIBMCloud.py | 4 +- cloudinit/sources/DataSourceMAAS.py | 2 +- cloudinit/sources/DataSourceOVF.py | 2 +- cloudinit/sources/DataSourceOpenNebula.py | 2 +- cloudinit/sources/DataSourceOpenStack.py | 4 +- cloudinit/sources/DataSourceOracle.py | 127 +- cloudinit/sources/DataSourceRbxCloud.py | 8 +- cloudinit/sources/DataSourceScaleway.py | 33 +- cloudinit/sources/DataSourceSmartOS.py | 2 +- cloudinit/sources/DataSourceVMware.py | 28 +- cloudinit/sources/DataSourceWSL.py | 261 ++ cloudinit/sources/__init__.py | 49 +- cloudinit/sources/azure/errors.py | 33 +- cloudinit/sources/azure/imds.py | 4 +- cloudinit/sources/helpers/azure.py | 139 +- cloudinit/sources/helpers/openstack.py | 8 +- .../sources/helpers/vmware/imc/config.py | 2 +- .../vmware/imc/config_custom_script.py | 6 +- .../helpers/vmware/imc/guestcust_util.py | 8 +- cloudinit/sources/helpers/vultr.py | 2 +- cloudinit/ssh_util.py | 15 +- cloudinit/stages.py | 78 +- cloudinit/subp.py | 98 +- cloudinit/templater.py | 73 +- cloudinit/url_helper.py | 52 +- cloudinit/user_data.py | 6 +- cloudinit/util.py | 268 +- cloudinit/version.py | 2 +- config/clean.d/README | 18 - config/cloud.cfg.d/05_logging.cfg | 4 +- config/cloud.cfg.tmpl | 8 +- conftest.py | 14 +- debian/apport-general-hook.py | 91 +- debian/changelog | 51 +- debian/cloud-init.logrotate | 11 + debian/cloud-init.maintscript | 1 + debian/cloud-init.postinst | 9 + debian/cloud-init.templates | 6 +- debian/patches/expire-on-hashed-users.patch | 4 +- .../keep-dhclient-as-priority-client.patch | 15 + .../netplan99-cannot-use-default.patch | 24 +- ...retain-ec2-default-net-update-events.patch | 23 + .../retain-netplan-world-readable.patch | 4 +- ...560d-cloud-config-after-snap-seeding.patch | 338 ++ debian/patches/series | 3 + ...status-do-not-remove-duplicated-data.patch | 23 +- ...s-retain-recoverable-error-exit-code.patch | 6 +- debian/po/templates.pot | 8 +- doc-requirements.txt | 1 + doc/examples/cloud-config-lxd.txt | 2 +- doc/examples/cloud-config-ssh-keys.txt | 18 +- doc/examples/cloud-config.txt | 25 +- doc/man/cloud-init.1 | 4 - doc/rtd/conf.py | 14 + doc/rtd/development/contribute_docs.rst | 6 +- doc/rtd/development/datasource_creation.rst | 151 +- doc/rtd/development/debugging.rst | 327 -- doc/rtd/development/index.rst | 3 +- doc/rtd/development/internal_files.rst | 46 + doc/rtd/development/summit/2023_summit.rst | 4 +- doc/rtd/explanation/analyze.rst | 4 +- doc/rtd/explanation/boot.rst | 21 +- doc/rtd/explanation/events.rst | 9 +- doc/rtd/explanation/exported_errors.rst | 131 + doc/rtd/explanation/failure_states.rst | 78 + doc/rtd/explanation/format.rst | 5 +- doc/rtd/explanation/index.rst | 2 + doc/rtd/explanation/instancedata.rst | 10 +- doc/rtd/howto/debug_user_data.rst | 50 +- doc/rtd/howto/debugging.rst | 183 + doc/rtd/howto/identify_datasource.rst | 25 - doc/rtd/howto/index.rst | 9 +- doc/rtd/howto/locate_files.rst | 72 - doc/rtd/howto/predeploy_testing.rst | 141 - doc/rtd/howto/rerun_cloud_init.rst | 97 + doc/rtd/howto/run_cloud_init_locally.rst | 217 ++ doc/rtd/howto/status.rst | 121 + doc/rtd/howto/ubuntu_test_prerelease.rst | 62 + doc/rtd/reference/base_config_reference.rst | 36 +- doc/rtd/reference/cli.rst | 61 +- doc/rtd/reference/datasources.rst | 2 + doc/rtd/reference/datasources/ec2.rst | 11 +- doc/rtd/reference/datasources/lxd.rst | 4 +- doc/rtd/reference/datasources/nocloud.rst | 2 +- doc/rtd/reference/datasources/oracle.rst | 14 + doc/rtd/reference/datasources/scaleway.rst | 56 + doc/rtd/reference/datasources/smartos.rst | 2 +- doc/rtd/reference/datasources/vmware.rst | 12 +- doc/rtd/reference/datasources/wsl.rst | 208 ++ doc/rtd/reference/faq.rst | 48 - doc/rtd/reference/index.rst | 3 + doc/rtd/reference/merging.rst | 27 +- doc/rtd/reference/modules.rst | 4 +- .../reference/network-config-format-v1.rst | 13 +- .../reference/network-config-format-v2.rst | 42 +- doc/rtd/reference/performance_analysis.rst | 110 + .../ubuntu_stable_release_updates.rst | 52 + doc/rtd/reference/user_files.rst | 68 + doc/rtd/spelling.py | 79 + doc/rtd/spelling_regex_list.txt | 14 + doc/rtd/spelling_word_list.txt | 292 ++ doc/rtd/tutorial/qemu-script.sh | 1 - doc/rtd/tutorial/qemu.rst | 1 - integration-requirements.txt | 1 + packages/bddeb | 22 +- packages/brpm | 239 +- packages/debian/cloud-init.logrotate | 11 + packages/redhat/cloud-init.spec.in | 3 +- packages/suse/cloud-init.spec.in | 1 - pyproject.toml | 2 + requirements.txt | 2 +- setup.py | 12 +- systemd/cloud-final.service.tmpl | 2 +- systemd/cloud-init-hotplugd.service | 2 +- systemd/cloud-init-hotplugd.socket | 2 +- sysvinit/freebsd/cloudinitlocal.tmpl | 2 +- sysvinit/netbsd/cloudconfig.tmpl | 7 +- sysvinit/netbsd/cloudfinal.tmpl | 7 +- sysvinit/netbsd/cloudinit.tmpl | 7 +- sysvinit/netbsd/cloudinitlocal.tmpl | 7 +- sysvinit/netbsd/dsidentify.tmpl | 7 +- sysvinit/openbsd/cloudconfig.tmpl | 37 + sysvinit/openbsd/cloudfinal.tmpl | 35 + sysvinit/openbsd/cloudinit.tmpl | 35 + sysvinit/openbsd/cloudinitlocal.tmpl | 38 + sysvinit/{gentoo => openrc}/cloud-config | 5 +- sysvinit/{gentoo => openrc}/cloud-final | 5 +- sysvinit/{gentoo => openrc}/cloud-init | 6 +- sysvinit/openrc/cloud-init-ds-identify | 21 + sysvinit/openrc/cloud-init-hotplug | 23 + sysvinit/{gentoo => openrc}/cloud-init-local | 6 +- templates/chrony.conf.cos.tmpl | 2 +- templates/chrony.conf.debian.tmpl | 2 +- templates/chrony.conf.ubuntu.tmpl | 2 +- templates/hosts.alpine.tmpl | 9 +- templates/hosts.mariner.tmpl | 44 +- templates/ntp.conf.ubuntu.tmpl | 2 +- templates/sources.list.ubuntu.deb822.tmpl | 62 +- test-requirements.txt | 2 +- tests/data/merge_sources/expected9.yaml | 2 +- tests/data/merge_sources/source9-1.yaml | 2 +- tests/data/net/dhcp/enp24s0.lease | Bin 0 -> 321 bytes tests/data/net/dhcp/eth0.lease | Bin 0 -> 370 bytes tests/helpers.py | 21 + .../assets/enable_coverage.py | 28 + .../integration_tests/bugs/test_lp1835584.py | 4 +- tests/integration_tests/clouds.py | 28 +- tests/integration_tests/cmd/test_clean.py | 47 +- tests/integration_tests/cmd/test_schema.py | 102 + tests/integration_tests/cmd/test_status.py | 78 + tests/integration_tests/conftest.py | 159 +- .../datasources/test_azure.py | 47 + .../datasources/test_ec2_ipv6.py | 8 +- tests/integration_tests/decorators.py | 3 +- tests/integration_tests/instances.py | 93 +- .../integration_tests/integration_settings.py | 7 + .../modules/test_apt_functionality.py | 44 +- .../modules/test_ca_certs.py | 2 +- tests/integration_tests/modules/test_cli.py | 21 +- .../modules/test_combined.py | 9 +- .../integration_tests/modules/test_hotplug.py | 225 +- .../modules/test_keys_to_console.py | 10 +- .../modules/test_set_hostname.py | 91 +- .../modules/test_set_password.py | 9 + .../modules/test_ssh_auth_key_fingerprints.py | 1 - .../modules/test_ssh_generate.py | 2 - .../modules/test_ssh_keys_provided.py | 24 - ...ubuntu_advantage.py => test_ubuntu_pro.py} | 51 +- tests/integration_tests/net/test_dhcp.py | 91 + tests/integration_tests/releases.py | 4 + .../reporting/test_webhook_reporting.py | 2 +- .../test_kernel_commandline_match.py | 35 +- tests/integration_tests/test_networking.py | 429 +++ tests/integration_tests/test_upgrade.py | 29 +- tests/integration_tests/util.py | 21 +- tests/unittests/analyze/test_boot.py | 17 +- tests/unittests/analyze/test_show.py | 24 + .../unittests/cmd/devel/test_hotplug_hook.py | 109 +- tests/unittests/cmd/devel/test_logs.py | 67 +- tests/unittests/cmd/devel/test_render.py | 17 + tests/unittests/cmd/test_cloud_id.py | 38 +- tests/unittests/cmd/test_main.py | 23 +- tests/unittests/cmd/test_query.py | 67 +- tests/unittests/cmd/test_status.py | 311 +- tests/unittests/config/test_apt_conf_v1.py | 12 +- .../test_apt_configure_sources_list_v1.py | 39 +- .../test_apt_configure_sources_list_v3.py | 13 +- tests/unittests/config/test_apt_source_v1.py | 31 +- tests/unittests/config/test_apt_source_v3.py | 150 +- tests/unittests/config/test_cc_ansible.py | 20 +- .../unittests/config/test_cc_apk_configure.py | 24 +- .../unittests/config/test_cc_apt_configure.py | 101 +- tests/unittests/config/test_cc_bootcmd.py | 19 +- tests/unittests/config/test_cc_ca_certs.py | 19 +- tests/unittests/config/test_cc_chef.py | 35 +- tests/unittests/config/test_cc_disk_setup.py | 4 +- tests/unittests/config/test_cc_growpart.py | 112 +- .../config/test_cc_install_hotplug.py | 44 +- tests/unittests/config/test_cc_locale.py | 6 +- tests/unittests/config/test_cc_lxd.py | 2 +- tests/unittests/config/test_cc_mcollective.py | 12 +- tests/unittests/config/test_cc_mounts.py | 10 +- tests/unittests/config/test_cc_ntp.py | 23 +- .../test_cc_package_update_upgrade_install.py | 116 +- tests/unittests/config/test_cc_phone_home.py | 15 +- tests/unittests/config/test_cc_puppet.py | 4 +- tests/unittests/config/test_cc_resizefs.py | 52 +- tests/unittests/config/test_cc_resolv_conf.py | 2 +- tests/unittests/config/test_cc_rsyslog.py | 10 +- tests/unittests/config/test_cc_runcmd.py | 7 +- tests/unittests/config/test_cc_seed_random.py | 15 +- .../unittests/config/test_cc_set_hostname.py | 32 +- .../unittests/config/test_cc_set_passwords.py | 9 +- tests/unittests/config/test_cc_snap.py | 113 +- tests/unittests/config/test_cc_ssh.py | 15 +- tests/unittests/config/test_cc_timezone.py | 4 +- ...ntu_advantage.py => test_cc_ubuntu_pro.py} | 374 +- .../config/test_cc_update_etc_hosts.py | 4 +- tests/unittests/config/test_cc_write_files.py | 18 +- .../config/test_cc_write_files_deferred.py | 4 +- .../unittests/config/test_cc_yum_add_repo.py | 10 +- .../config/test_cc_zypper_add_repo.py | 16 +- tests/unittests/config/test_modules.py | 3 +- tests/unittests/config/test_schema.py | 380 +- tests/unittests/conftest.py | 33 +- .../distros/package_management/test_apt.py | 28 +- tests/unittests/distros/test__init__.py | 85 +- tests/unittests/distros/test_arch.py | 44 +- tests/unittests/distros/test_bsd_utils.py | 2 +- tests/unittests/distros/test_gentoo.py | 6 +- tests/unittests/distros/test_init.py | 130 +- tests/unittests/distros/test_netconfig.py | 37 - tests/unittests/distros/test_opensuse.py | 28 +- tests/unittests/distros/test_photon.py | 2 +- tests/unittests/filters/test_launch_index.py | 2 +- tests/unittests/helpers.py | 39 +- tests/unittests/net/test_dhcp.py | 791 ++-- tests/unittests/net/test_dns.py | 4 +- tests/unittests/net/test_init.py | 302 +- tests/unittests/net/test_network_state.py | 23 +- .../reporting/test_reporting_hyperv.py | 70 +- tests/unittests/runs/test_merge_run.py | 2 +- tests/unittests/runs/test_simple_run.py | 4 +- tests/unittests/sources/azure/test_errors.py | 36 +- tests/unittests/sources/azure/test_imds.py | 20 +- tests/unittests/sources/azure/test_kvp.py | 28 +- .../sources/helpers/test_openstack.py | 17 +- tests/unittests/sources/test_aliyun.py | 25 +- tests/unittests/sources/test_altcloud.py | 4 + tests/unittests/sources/test_azure.py | 123 +- tests/unittests/sources/test_azure_helper.py | 171 +- tests/unittests/sources/test_bigstep.py | 2 +- tests/unittests/sources/test_cloudstack.py | 435 ++- tests/unittests/sources/test_common.py | 5 +- tests/unittests/sources/test_ec2.py | 323 +- tests/unittests/sources/test_gce.py | 8 +- tests/unittests/sources/test_init.py | 41 +- tests/unittests/sources/test_opennebula.py | 2 +- tests/unittests/sources/test_openstack.py | 18 +- tests/unittests/sources/test_oracle.py | 116 +- tests/unittests/sources/test_scaleway.py | 29 +- tests/unittests/sources/test_smartos.py | 6 +- tests/unittests/sources/test_upcloud.py | 16 +- tests/unittests/sources/test_vmware.py | 2 +- tests/unittests/sources/test_wsl.py | 396 ++ .../sources/vmware/test_vmware_config_file.py | 3 +- tests/unittests/test__init__.py | 4 +- tests/unittests/test_builtin_handlers.py | 10 +- tests/unittests/test_cli.py | 43 +- tests/unittests/test_conftest.py | 16 +- tests/unittests/test_data.py | 34 +- tests/unittests/test_dmi.py | 42 +- tests/unittests/test_ds_identify.py | 256 +- tests/unittests/test_helpers.py | 9 +- tests/unittests/test_log.py | 1 - tests/unittests/test_merging.py | 10 +- tests/unittests/test_net.py | 655 +++- tests/unittests/test_net_activators.py | 103 + tests/unittests/test_render_template.py | 2 +- tests/unittests/test_ssh_util.py | 83 +- tests/unittests/test_stages.py | 45 +- tests/unittests/test_subp.py | 75 +- tests/unittests/test_templating.py | 140 +- tests/unittests/test_upgrade.py | 232 +- tests/unittests/test_url_helper.py | 155 +- tests/unittests/test_util.py | 290 +- tests/unittests/util.py | 26 +- tools/.github-cla-signers | 5 + tools/Z99-cloud-locale-test.sh | 2 +- tools/build-on-openbsd | 46 +- tools/ccfg-merge-debug | 2 +- tools/cloud-init-hotplugd | 23 + tools/ds-identify | 213 +- tools/hook-hotplug | 16 +- tools/make-tarball | 2 +- tools/migrate-lp-user-to-github | 4 +- tools/mock-meta.py | 2 - tools/read-dependencies | 4 +- tools/uncloud-init | 2 +- tools/xkvm | 2 +- tox.ini | 15 +- 439 files changed, 23872 insertions(+), 6993 deletions(-) create mode 100644 .pc/keep-dhclient-as-priority-client.patch/config/cloud.cfg.tmpl create mode 100644 .pc/retain-ec2-default-net-update-events.patch/cloudinit/sources/DataSourceEc2.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/cloud.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_lxd.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_snap.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_ubuntu_autoinstall.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/util.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/systemd/cloud-config.service.tmpl create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/tests/integration_tests/modules/test_frequency_override.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/tests/unittests/config/test_cc_lxd.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/tests/unittests/config/test_cc_snap.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/tests/unittests/config/test_cc_ubuntu_autoinstall.py create mode 100644 .pc/revert-551f560d-cloud-config-after-snap-seeding.patch/tests/unittests/util.py delete mode 100644 cloudinit/config/cc_migrator.py delete mode 100644 cloudinit/config/cc_rightscale_userdata.py rename cloudinit/config/{cc_ubuntu_advantage.py => cc_ubuntu_pro.py} (71%) delete mode 100644 cloudinit/distros/net_util.py create mode 100644 cloudinit/sources/DataSourceWSL.py delete mode 100644 config/clean.d/README create mode 100644 debian/cloud-init.logrotate create mode 100644 debian/patches/keep-dhclient-as-priority-client.patch create mode 100644 debian/patches/retain-ec2-default-net-update-events.patch create mode 100644 debian/patches/revert-551f560d-cloud-config-after-snap-seeding.patch delete mode 100644 doc/rtd/development/debugging.rst create mode 100644 doc/rtd/development/internal_files.rst create mode 100644 doc/rtd/explanation/exported_errors.rst create mode 100644 doc/rtd/explanation/failure_states.rst create mode 100644 doc/rtd/howto/debugging.rst delete mode 100644 doc/rtd/howto/locate_files.rst delete mode 100644 doc/rtd/howto/predeploy_testing.rst create mode 100644 doc/rtd/howto/rerun_cloud_init.rst create mode 100644 doc/rtd/howto/run_cloud_init_locally.rst create mode 100644 doc/rtd/howto/status.rst create mode 100644 doc/rtd/howto/ubuntu_test_prerelease.rst create mode 100644 doc/rtd/reference/datasources/scaleway.rst create mode 100644 doc/rtd/reference/datasources/wsl.rst create mode 100644 doc/rtd/reference/performance_analysis.rst create mode 100644 doc/rtd/reference/ubuntu_stable_release_updates.rst create mode 100644 doc/rtd/reference/user_files.rst create mode 100644 doc/rtd/spelling.py create mode 100644 doc/rtd/spelling_regex_list.txt create mode 100644 doc/rtd/spelling_word_list.txt create mode 100644 packages/debian/cloud-init.logrotate create mode 100755 sysvinit/openbsd/cloudconfig.tmpl create mode 100755 sysvinit/openbsd/cloudfinal.tmpl create mode 100755 sysvinit/openbsd/cloudinit.tmpl create mode 100755 sysvinit/openbsd/cloudinitlocal.tmpl rename sysvinit/{gentoo => openrc}/cloud-config (83%) rename sysvinit/{gentoo => openrc}/cloud-final (81%) rename sysvinit/{gentoo => openrc}/cloud-init (83%) create mode 100755 sysvinit/openrc/cloud-init-ds-identify create mode 100755 sysvinit/openrc/cloud-init-hotplug rename sysvinit/{gentoo => openrc}/cloud-init-local (83%) create mode 100644 tests/data/net/dhcp/enp24s0.lease create mode 100644 tests/data/net/dhcp/eth0.lease create mode 100644 tests/helpers.py create mode 100644 tests/integration_tests/assets/enable_coverage.py create mode 100644 tests/integration_tests/datasources/test_azure.py rename tests/integration_tests/modules/{test_ubuntu_advantage.py => test_ubuntu_pro.py} (80%) create mode 100644 tests/integration_tests/net/test_dhcp.py create mode 100644 tests/unittests/analyze/test_show.py rename tests/unittests/config/{test_cc_ubuntu_advantage.py => test_cc_ubuntu_pro.py} (81%) create mode 100644 tests/unittests/sources/test_wsl.py create mode 100755 tools/cloud-init-hotplugd diff --git a/.github/workflows/check_format.yml b/.github/workflows/check_format.yml index 1b762886e..70f04afa9 100644 --- a/.github/workflows/check_format.yml +++ b/.github/workflows/check_format.yml @@ -87,10 +87,12 @@ jobs: - name: "Install dependencies" run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update - sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox lintian + sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox - name: "Spellcheck" run: | - make check_spelling + tox + env: + TOXENV: doc-spelling - name: "Build docs" env: TOXENV: doc diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index cd78568c0..0f3206f79 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -17,7 +17,7 @@ defaults: shell: sh -ex {0} env: - RELEASE: bionic + RELEASE: focal jobs: package-build: diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index d8391dc91..c9bfa15b7 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -37,6 +37,8 @@ jobs: broken_count=$(grep -c "broken" output.txt) if [[ $broken_count -ge 5 ]]; then echo "Too many broken links detected: $broken_count" + broken_matches=$(grep "broken" output.txt) + echo "Broken links \n$broken_matches" exit 1 else echo "Number of broken links is below threshold: $broken_count" diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 10dde658b..ec054f86c 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -14,19 +14,28 @@ jobs: unittests: strategy: matrix: - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12" ] - toxenv: [ py3 ] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + toxenv: [py3] experimental: [false] + check-latest: [false] + continue-on-error: [false] include: - python-version: "3.6" toxenv: lowest-supported + continue-on-error: false + check-latest: false experimental: false + - python-version: "3.13-dev" + toxenv: py3 + check-latest: true + experimental: true + continue-on-error: true name: unittest / ${{ matrix.toxenv }} / python ${{matrix.python-version}} runs-on: ubuntu-20.04 continue-on-error: ${{ matrix.experimental }} steps: - name: "Checkout" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # Fetch all tags for tools/read-version fetch-depth: 0 @@ -34,9 +43,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{matrix.python-version}} + check-latest: ${{matrix.check-latest}} - name: Install tox run: pip install tox - name: Run unittest - env: - PYTEST_ADDOPTS: -v run: tox -e ${{ matrix.toxenv }} diff --git a/.pc/applied-patches b/.pc/applied-patches index 0005bd09d..9e7a80ce2 100644 --- a/.pc/applied-patches +++ b/.pc/applied-patches @@ -3,7 +3,10 @@ expire-on-hashed-users.patch retain-netplan-world-readable.patch netplan99-cannot-use-default.patch retain-old-groups.patch +keep-dhclient-as-priority-client.patch +revert-551f560d-cloud-config-after-snap-seeding.patch do-not-block-user-login.patch status-do-not-remove-duplicated-data.patch retain-apt-pre-deb822.patch status-retain-recoverable-error-exit-code.patch +retain-ec2-default-net-update-events.patch diff --git a/.pc/expire-on-hashed-users.patch/cloudinit/features.py b/.pc/expire-on-hashed-users.patch/cloudinit/features.py index d864fdf69..d661b940b 100644 --- a/.pc/expire-on-hashed-users.patch/cloudinit/features.py +++ b/.pc/expire-on-hashed-users.patch/cloudinit/features.py @@ -62,7 +62,7 @@ NETPLAN_CONFIG_ROOT_READ_ONLY = True """ If ``NETPLAN_CONFIG_ROOT_READ_ONLY`` is True, then netplan configuration will -be written as a single root readon-only file /etc/netplan/50-cloud-init.yaml. +be written as a single root read-only file /etc/netplan/50-cloud-init.yaml. This prevents wifi passwords in network v2 configuration from being world-readable. Prior to 23.1, netplan configuration is world-readable. diff --git a/.pc/expire-on-hashed-users.patch/tests/unittests/config/test_cc_set_passwords.py b/.pc/expire-on-hashed-users.patch/tests/unittests/config/test_cc_set_passwords.py index 1a9fcd3c6..ef34a8c60 100644 --- a/.pc/expire-on-hashed-users.patch/tests/unittests/config/test_cc_set_passwords.py +++ b/.pc/expire-on-hashed-users.patch/tests/unittests/config/test_cc_set_passwords.py @@ -12,7 +12,11 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import does_not_raise, skipUnlessJsonSchema +from tests.unittests.helpers import ( + SCHEMA_EMPTY_ERROR, + does_not_raise, + skipUnlessJsonSchema, +) from tests.unittests.util import get_cloud MODPATH = "cloudinit.config.cc_set_passwords." @@ -718,7 +722,8 @@ class TestSetPasswordsSchema: ( {"chpasswd": {"list": []}}, pytest.raises( - SchemaValidationError, match=r"\[\] is too short" + SchemaValidationError, + match=rf"\[\] {SCHEMA_EMPTY_ERROR}", ), ), ], diff --git a/.pc/keep-dhclient-as-priority-client.patch/config/cloud.cfg.tmpl b/.pc/keep-dhclient-as-priority-client.patch/config/cloud.cfg.tmpl new file mode 100644 index 000000000..00e3e5dc0 --- /dev/null +++ b/.pc/keep-dhclient-as-priority-client.patch/config/cloud.cfg.tmpl @@ -0,0 +1,372 @@ +## template:jinja +# The top level settings are used as module +# and base configuration. +{% set is_bsd = variant in ["dragonfly", "freebsd", "netbsd", "openbsd"] %} +{% set is_rhel = variant in ["almalinux", "centos", "cloudlinux", "eurolinux", + "miraclelinux", "rhel", "rocky", "virtuozzo"] %} +{% set gecos = ({"amazon": "EC2 Default User", "centos": "Cloud User", + "debian": "Debian", "dragonfly": "DragonFly", + "freebsd": "FreeBSD", "mariner": "MarinerOS", + "rhel": "Cloud User", "netbsd": "NetBSD", + "openbsd": "openBSD", "openmandriva": "OpenMandriva admin", + "photon": "PhotonOS", "ubuntu": "Ubuntu", + "unknown": "Ubuntu"}) %} +{% set groups = ({"alpine": "adm, wheel", "arch": "wheel, users", + "debian": "adm, audio, cdrom, dialout, dip, floppy, netdev, plugdev, sudo, video", + "gentoo": "users, wheel", "mariner": "wheel", + "photon": "wheel", + "openmandriva": "wheel, users, systemd-journal", + "suse": "cdrom, users", + "ubuntu": "adm, audio, cdrom, dialout, dip, floppy, lxd, netdev, plugdev, sudo, video", + "unknown": "adm, cdrom, dip, lxd, sudo"}) %} +{% set shells = ({"alpine": "/bin/ash", "dragonfly": "/bin/sh", + "freebsd": "/bin/tcsh", "netbsd": "/bin/sh", + "openbsd": "/bin/ksh"}) %} +{% set usernames = ({"amazon": "ec2-user", "centos": "cloud-user", + "openmandriva": "omv", "rhel": "cloud-user", + "unknown": "ubuntu"}) %} +{% if is_bsd %} +syslog_fix_perms: root:wheel +{% elif variant == "suse" %} +syslog_fix_perms: root:root +{% endif %} + +# A set of users which may be applied and/or used by various modules +# when a 'default' entry is found it will reference the 'default_user' +# from the distro configuration specified below +users: +{% if variant == "photon" %} + - name: root + lock_passwd: false +{% else %} + - default +{% endif %} + +{% if variant == "photon" %} +# VMware guest customization. +disable_vmware_customization: true +manage_etc_hosts: false + +{% endif -%} + +# If this is set, 'root' will not be able to ssh in and they +# will get a message to login instead as the default $user +{% if variant in ["freebsd", "photon"] %} +disable_root: false +{% else %} +disable_root: true +{% endif %} + +{%- if variant in ["alpine", "amazon", "fedora", "OpenCloudOS", "openeuler", + "openmandriva", "photon", "TencentOS"] or is_rhel %} + +{% if is_rhel %} +mount_default_fields: [~, ~, 'auto', 'defaults,nofail,x-systemd.requires=cloud-init.service,_netdev', '0', '2'] +{% else %} +mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] +{% endif %} +{% if variant == "amazon" %} +resize_rootfs: noblock +{% endif %} +{% if variant not in ["photon"] %} +resize_rootfs_tmp: /dev +ssh_pwauth: false +{% endif %} +{% endif %} + +# This will cause the set+update hostname module to not operate (if true) +preserve_hostname: false + +{% if variant == "debian" %} +apt: + # This prevents cloud-init from rewriting apt's sources.list file, + # which has been a source of surprise. + preserve_sources_list: true + +{% endif -%} + +# If you use datasource_list array, keep array items in a single line. +# If you use multi line array, ds-identify script won't read array items. +# Example datasource config +# datasource: +# Ec2: +# metadata_urls: [ 'blah.com' ] +# timeout: 5 # (defaults to 50 seconds) +# max_wait: 10 # (defaults to 120 seconds) + +{% if variant == "amazon" %} +# Amazon Linux relies on ec2-net-utils for network configuration +network: + config: disabled + +{% endif -%} + +{% if is_rhel %} +# Default redhat settings: +ssh_deletekeys: true +ssh_genkeytypes: ['rsa', 'ecdsa', 'ed25519'] +syslog_fix_perms: ~ +disable_vmware_customization: false +{% endif -%} + +# The modules that run in the 'init' stage +cloud_init_modules: +{% if variant not in ["netbsd"] %} + - seed_random +{% endif %} + - bootcmd + - write_files +{% if variant not in ["netbsd", "openbsd"] %} + - growpart + - resizefs +{% endif %} +{% if not is_bsd %} + - disk_setup + - mounts +{% endif %} + - set_hostname + - update_hostname + - update_etc_hosts +{% if variant in ["alpine", "photon"] %} + - resolv_conf +{% endif %} +{% if not is_bsd or variant not in ["photon"] %} + - ca_certs +{% endif %} + - rsyslog + - users_groups + - ssh + +# The modules that run in the 'config' stage +cloud_config_modules: +{% if variant in ["ubuntu"] %} + - wireguard +{% endif %} +{% if variant in ["debian", "ubuntu", "unknown"] %} + - snap +{% endif %} +{% if variant == "ubuntu" %} + - ubuntu_autoinstall +{% endif %} +{% if variant not in ["photon"] %} + - ssh_import_id +{% if not is_rhel %} + - keyboard +{% endif %} + - locale +{% endif %} + - set_passwords +{% if variant == "alpine" %} + - apk_configure +{% elif variant in ["debian", "ubuntu", "unknown"] %} + - grub_dpkg + - apt_pipelining + - apt_configure +{% if variant == "ubuntu" %} + - ubuntu_pro +{% endif %} +{% elif variant in ["fedora", "mariner", "openeuler", "openmandriva", + "photon"] or is_rhel %} +{% if is_rhel %} + - rh_subscription +{% endif %} +{% if variant not in ["mariner", "photon"] %} + - spacewalk +{% endif %} + - yum_add_repo +{% elif variant == "suse" %} + - zypper_add_repo +{% endif %} + - ntp + - timezone + - disable_ec2_metadata + - runcmd +{% if variant in ["debian", "ubuntu", "unknown"] %} + - byobu +{% endif %} + +# The modules that run in the 'final' stage +cloud_final_modules: + - package_update_upgrade_install +{% if variant in ["debian", "ubuntu", "unknown"] %} + - fan + - landscape + - lxd +{% endif %} +{% if variant in ["ubuntu", "unknown"] %} + - ubuntu_drivers +{% endif %} + - write_files_deferred + - puppet + - chef + - ansible + - mcollective + - salt_minion + - reset_rmc + - scripts_vendor + - scripts_per_once + - scripts_per_boot + - scripts_per_instance + - scripts_user + - ssh_authkey_fingerprints + - keys_to_console + - install_hotplug + - phone_home + - final_message + - power_state_change + +# System and/or distro specific settings +# (not accessible to handlers/transforms) +system_info: + # This will affect which distro class gets used +{% if variant in ["alpine", "amazon", "arch", "debian", "fedora", "freebsd", + "gentoo", "mariner", "netbsd", "openbsd", "OpenCloudOS", + "openeuler", "openmandriva", "photon", "suse", "TencentOS", + "ubuntu"] or is_rhel %} + distro: {{ variant }} +{% elif variant == "dragonfly" %} + distro: dragonflybsd +{% else %} + # Unknown/fallback distro. + distro: ubuntu +{% endif %} + # Default user name + that default users groups (if added/used) + default_user: +{% if variant in usernames %} + name: {{ usernames[variant] }} +{% else %} + name: {{ variant }} +{% endif %} +{% if variant in ["alpine", "amazon", "arch", "debian", "fedora", "gentoo", + "mariner", "OpenCloudOS", "openeuler", "openmandriva", + "photon", "suse", "TencentOS", "ubuntu", "unknown"] + or is_bsd or is_rhel %} + lock_passwd: True +{% endif %} +{% if variant in gecos %} + gecos: {{ gecos[variant] }} +{% else %} + gecos: {{ variant }} Cloud User +{% endif %} +{% if variant in groups %} + groups: [{{ groups[variant] }}] +{% elif is_bsd %} + groups: [wheel] +{% elif is_rhel %} + groups: [adm, systemd-journal] +{% else %} + groups: [wheel, adm, systemd-journal] +{% endif %} +{% if variant == "gentoo" %} + primary_group: users + no_user_group: true +{% endif %} +{% if variant in ["alpine", "freebsd", "openbsd"] %} +{% if variant in usernames %} + doas: + - permit nopass usernames[{{ variant }}] +{% else %} + doas: + - permit nopass {{ variant }} +{% endif %} +{% endif %} + sudo: ["ALL=(ALL) NOPASSWD:ALL"] +{% if variant in shells %} + shell: {{ shells[variant] }} +{% else %} + shell: /bin/bash +{% endif %} +{% if variant == "alpine" %} + network: + renderers: ['eni'] +{% elif variant == "debian" %} + network: + renderers: ['netplan', 'eni', 'networkd'] + activators: ['netplan', 'eni', 'network-manager', 'networkd'] +{% elif variant == "dragonfly" %} + network: + renderers: ['freebsd'] +{% elif variant == "fedora" %} + network: + renderers: ['netplan', 'network-manager', 'networkd', 'sysconfig', 'eni'] +{% elif variant in ["freebsd", "netbsd", "openbsd"] %} + network: + renderers: ['{{ variant }}'] +{% elif variant in ["mariner", "photon"] %} + network: + renderers: ['networkd'] +{% elif variant == "openmandriva" %} + network: + renderers: ['network-manager', 'networkd'] +{% elif variant in ["ubuntu", "unknown"] %} +{# SRU_BLOCKER: do not ship network renderers on Xenial, Bionic or Eoan #} + network: + dhcp_client_priority: [dhcpcd, dhclient, udhcpc] + renderers: ['netplan', 'eni', 'sysconfig'] + activators: ['netplan', 'eni', 'network-manager', 'networkd'] +{% elif is_rhel %} + network: + renderers: ['sysconfig', 'eni', 'netplan', 'network-manager', 'networkd'] +{% endif %} +{% if variant == "photon" %} + # If set to true, cloud-init will not use fallback network config. + # In Photon, we have default network settings, hence if network settings are + # not explicitly given in metadata, don't use fallback network config. + disable_fallback_netcfg: true +{% endif %} +{% if variant in ["debian", "ubuntu", "unknown"] %} + # Automatically discover the best ntp_client + ntp_client: auto +{% endif %} +{% if variant in ["alpine", "amazon", "arch", "debian", "fedora", "gentoo", + "mariner", "OpenCloudOS", "openeuler", "openmandriva", + "photon", "suse", "TencentOS", "ubuntu", "unknown"] + or is_rhel %} + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ +{% elif is_bsd %} + paths: + run_dir: /var/run/ +{% endif %} +{% if variant == "debian" %} + package_mirrors: + - arches: [default] + failsafe: + primary: https://deb.debian.org/debian + security: https://deb.debian.org/debian-security +{% elif variant in ["ubuntu", "unknown"] %} + package_mirrors: + - arches: [i386, amd64] + failsafe: + primary: http://archive.ubuntu.com/ubuntu + security: http://security.ubuntu.com/ubuntu + search: + primary: + - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ + - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ + - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ + security: [] + - arches: [arm64, armel, armhf] + failsafe: + primary: http://ports.ubuntu.com/ubuntu-ports + security: http://ports.ubuntu.com/ubuntu-ports + search: + primary: + - http://%(ec2_region)s.ec2.ports.ubuntu.com/ubuntu-ports/ + - http://%(availability_zone)s.clouds.ports.ubuntu.com/ubuntu-ports/ + - http://%(region)s.clouds.ports.ubuntu.com/ubuntu-ports/ + security: [] + - arches: [default] + failsafe: + primary: http://ports.ubuntu.com/ubuntu-ports + security: http://ports.ubuntu.com/ubuntu-ports +{% endif %} +{% if variant in ["debian", "ubuntu", "unknown"] %} + ssh_svcname: ssh +{% elif variant in ["alpine", "amazon", "arch", "fedora", "gentoo", + "mariner", "OpenCloudOS", "openeuler", "openmandriva", + "photon", "suse", "TencentOS"] or is_rhel %} + ssh_svcname: sshd +{% endif %} diff --git a/.pc/netplan99-cannot-use-default.patch/cloudinit/net/netplan.py b/.pc/netplan99-cannot-use-default.patch/cloudinit/net/netplan.py index 0b8419af2..9e36fe16a 100644 --- a/.pc/netplan99-cannot-use-default.patch/cloudinit/net/netplan.py +++ b/.pc/netplan99-cannot-use-default.patch/cloudinit/net/netplan.py @@ -18,6 +18,8 @@ ) from cloudinit.net.network_state import NET_CONFIG_TO_V2, NetworkState +CLOUDINIT_NETPLAN_FILE = "/etc/netplan/50-cloud-init.yaml" + KNOWN_SNAPD_CONFIG = b"""\ # This is the initial network config. # It can be overwritten by cloud-init or console-conf. @@ -210,7 +212,7 @@ def _clean_default(target=None): tpath = subp.target_path(target, "etc/netplan/00-snapd-config.yaml") if not os.path.isfile(tpath): return - content = util.load_file(tpath, decode=False) + content = util.load_binary_file(tpath) if content != KNOWN_SNAPD_CONFIG: return @@ -242,9 +244,7 @@ class Renderer(renderer.Renderer): def __init__(self, config=None): if not config: config = {} - self.netplan_path = config.get( - "netplan_path", "etc/netplan/50-cloud-init.yaml" - ) + self.netplan_path = config.get("netplan_path", CLOUDINIT_NETPLAN_FILE) self.netplan_header = config.get("netplan_header", None) self._postcmds = config.get("postcmds", False) self.clean_default = config.get("clean_default", True) diff --git a/.pc/netplan99-cannot-use-default.patch/tests/unittests/distros/test_netconfig.py b/.pc/netplan99-cannot-use-default.patch/tests/unittests/distros/test_netconfig.py index d72631fa0..358f60dd0 100644 --- a/.pc/netplan99-cannot-use-default.patch/tests/unittests/distros/test_netconfig.py +++ b/.pc/netplan99-cannot-use-default.patch/tests/unittests/distros/test_netconfig.py @@ -1024,43 +1024,6 @@ def netctl_path(self, iface): def netplan_path(self): return "/etc/netplan/50-cloud-init.yaml" - def test_apply_network_config_v1_without_netplan(self): - # Note that this is in fact an invalid netctl config: - # "Address=None/None" - # But this is what the renderer has been writing out for a long time, - # and the test's purpose is to assert that the netctl renderer is - # still being used in absence of netplan, not the correctness of the - # rendered netctl config. - expected_cfgs = { - self.netctl_path("eth0"): dedent( - """\ - Address=192.168.1.5/255.255.255.0 - Connection=ethernet - DNS=() - Gateway=192.168.1.254 - IP=static - Interface=eth0 - """ - ), - self.netctl_path("eth1"): dedent( - """\ - Address=None/None - Connection=ethernet - DNS=() - Gateway= - IP=dhcp - Interface=eth1 - """ - ), - } - - self._apply_and_verify( - self.distro.apply_network_config, - V1_NET_CFG, - expected_cfgs=expected_cfgs.copy(), - with_netplan=False, - ) - def test_apply_network_config_v1_with_netplan(self): expected_cfgs = { self.netplan_path(): dedent( diff --git a/.pc/netplan99-cannot-use-default.patch/tests/unittests/test_net.py b/.pc/netplan99-cannot-use-default.patch/tests/unittests/test_net.py index c5509536a..cb9919388 100644 --- a/.pc/netplan99-cannot-use-default.patch/tests/unittests/test_net.py +++ b/.pc/netplan99-cannot-use-default.patch/tests/unittests/test_net.py @@ -13,7 +13,7 @@ import pytest from yaml.serializer import Serializer -from cloudinit import distros, log, net +from cloudinit import distros, net from cloudinit import safeyaml as yaml from cloudinit import subp, temp_utils, util from cloudinit.net import ( @@ -646,6 +646,7 @@ may-fail=false address1=172.19.1.34/22 route1=0.0.0.0/0,172.19.3.254 +dns=172.19.0.12; """.lstrip(), ), @@ -1280,8 +1281,6 @@ """\ auto lo iface lo inet loopback - dns-nameservers 8.8.8.8 8.8.4.4 - dns-search wark.maas iface eth1 inet manual @@ -1477,11 +1476,11 @@ [ipv4] method=auto - may-fail=false + may-fail=true [ipv6] method=auto - may-fail=false + may-fail=true """ ), @@ -1650,11 +1649,11 @@ [ipv6] method=auto - may-fail=false + may-fail=true [ipv4] method=auto - may-fail=false + may-fail=true """ ), @@ -2797,6 +2796,8 @@ [ipv4] method=auto may-fail=false + dns=8.8.8.8;4.4.4.4;8.8.4.4; + dns-search=barley.maas;wark.maas;foobar.maas; """ ), @@ -2822,6 +2823,8 @@ method=manual may-fail=false address1=192.168.200.7/24 + dns=8.8.8.8;4.4.4.4;8.8.4.4; + dns-search=barley.maas;wark.maas;foobar.maas; """ ), @@ -2846,6 +2849,8 @@ [ipv4] method=auto may-fail=false + dns=8.8.8.8;4.4.4.4;8.8.4.4; + dns-search=barley.maas;wark.maas;foobar.maas; """ ), @@ -2930,12 +2935,15 @@ method=manual may-fail=false address1=192.168.14.2/24 + dns=8.8.8.8;4.4.4.4;8.8.4.4; + dns-search=barley.maas;wark.maas;foobar.maas; [ipv6] method=manual may-fail=false address1=2001:1::1/64 route1=::/0,2001:4800:78ff:1b::1 + dns-search=barley.maas;wark.maas;foobar.maas; """ ), @@ -2962,9 +2970,9 @@ may-fail=false address1=192.168.0.2/24 gateway=192.168.0.1 + address2=192.168.2.10/24 dns=192.168.0.10;10.23.23.134; dns-search=barley.maas;sacchromyces.maas;brettanomyces.maas; - address2=192.168.2.10/24 """ ), @@ -2990,6 +2998,7 @@ [ipv6] method=auto may-fail=false + dns-search=barley.maas;wark.maas;foobar.maas; """ ), @@ -4131,6 +4140,170 @@ ), }, }, + "v1-dns": { + "expected_networkd": textwrap.dedent( + """\ + [Address] + Address=192.168.1.20/16 + + [Match] + MACAddress=11:22:33:44:55:66 + Name=interface0 + + [Network] + DHCP=no + DNS=1.1.1.1 3.3.3.3 + Domains=aaaa cccc + + [Route] + Gateway=192.168.1.1 + """ + ), + "expected_eni": textwrap.dedent( + """\ + # This file is generated from information provided by the datasource. Changes + # to it will not persist across an instance reboot. To disable cloud-init's + # network configuration capabilities, write a file + # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: + # network: {config: disabled} + auto lo + iface lo inet loopback + dns-nameservers 2.2.2.2 + dns-search bbbb + + iface lo inet6 loopback + dns-nameservers FEDC::1 + dns-search bbbb + + auto interface0 + iface interface0 inet static + address 192.168.1.20/16 + dns-nameservers 1.1.1.1 3.3.3.3 + dns-search aaaa cccc + gateway 192.168.1.1 + """ # noqa: E501 + ), + "expected_netplan": textwrap.dedent( + """\ + # This file is generated from information provided by the datasource. Changes + # to it will not persist across an instance reboot. To disable cloud-init's + # network configuration capabilities, write a file + # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: + # network: {config: disabled} + network: + version: 2 + ethernets: + interface0: + addresses: + - 192.168.1.20/16 + match: + macaddress: 11:22:33:44:55:66 + nameservers: + addresses: + - 1.1.1.1 + - 3.3.3.3 + search: + - aaaa + - cccc + routes: + - to: default + via: 192.168.1.1 + set-name: interface0 + """ # noqa: E501 + ), + "expected_sysconfig_opensuse": { + "ifcfg-interface0": textwrap.dedent( + """\ + # Created by cloud-init automatically, do not edit. + # + BOOTPROTO=static + IPADDR=192.168.1.20 + LLADDR=11:22:33:44:55:66 + NETMASK=255.255.0.0 + STARTMODE=auto + """ + ) + }, + "expected_sysconfig_rhel": { + "ifcfg-eth0": textwrap.dedent( + """\ + # Created by cloud-init automatically, do not edit. + # + BOOTPROTO=none + DEFROUTE=yes + DEVICE=interface0 + DNS1=1.1.1.1 + DNS2=3.3.3.3 + DOMAIN=aaaa cccc + GATEWAY=192.168.1.1 + HWADDR=11:22:33:44:55:66 + IPADDR=192.168.1.20 + NETMASK=255.255.0.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """ + ), + }, + "expected_network_manager": { + "cloud-init-interface0.nmconnection": textwrap.dedent( + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init interface0 + uuid=8b6862ed-dbd6-5830-93f7-a91451c13828 + autoconnect-priority=120 + type=ethernet + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + mac-address=11:22:33:44:55:66 + + [ipv4] + method=manual + may-fail=false + address1=192.168.1.20/16 + gateway=192.168.1.1 + dns=3.3.3.3;1.1.1.1; + dns-search=cccc;aaaa; + + """ + ) + }, + "yaml": textwrap.dedent( + """\ + version: 1 + config: + - type: physical + name: interface0 + mac_address: "11:22:33:44:55:66" + subnets: + - type: static + address: 192.168.1.20/16 + gateway: 192.168.1.1 + dns_nameservers: + - 3.3.3.3 + dns_search: + - cccc + - type: nameserver + interface: interface0 + address: + - 1.1.1.1 + search: + - aaaa + - type: nameserver + address: + - 2.2.2.2 + - FEDC::1 + search: + - bbbb + """ + ), + }, "v2-dev-name-via-mac-lookup": { "expected_sysconfig_rhel": { "ifcfg-eth0": textwrap.dedent( @@ -4154,6 +4327,320 @@ """ ), }, + "v2-mixed-routes": { + "expected_network_manager": { + "cloud-init-eth0.nmconnection": textwrap.dedent( + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init eth0 + uuid=1dd9a779-d327-56e1-8454-c65e2556c12c + autoconnect-priority=120 + type=ethernet + interface-name=eth0 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + mtu=500 + + [ipv4] + method=auto + may-fail=true + route1=169.254.42.42/32,62.210.0.1 + route1_options=mtu=400 + route2=169.254.42.43/32,62.210.0.2 + route2_options=mtu=200 + address1=192.168.1.20/16 + dns=8.8.8.8; + dns-search=lab;home; + + [ipv6] + route1=::/0,fe80::dc00:ff:fe20:186 + route1_options=mtu=300 + route2=fe80::dc00:ff:fe20:188/64,fe80::dc00:ff:fe20:187 + route2_options=mtu=100 + method=auto + may-fail=true + address1=2001:bc8:1210:232:dc00:ff:fe20:185/64 + dns=FEDC::1; + dns-search=lab;home; + + """ + ) + }, + "yaml": textwrap.dedent( + """\ + version: 2 + ethernets: + eth0: + dhcp4: true + dhcp6: true + mtu: 500 + nameservers: + search: [lab, home] + addresses: [8.8.8.8, "FEDC::1"] + routes: + - to: 169.254.42.42/32 + via: 62.210.0.1 + mtu: 400 + - via: fe80::dc00:ff:fe20:186 + to: ::/0 + mtu: 300 + - to: 169.254.42.43/32 + via: 62.210.0.2 + mtu: 200 + - via: fe80::dc00:ff:fe20:187 + to: fe80::dc00:ff:fe20:188 + mtu: 100 + addresses: + - 192.168.1.20/16 + - 2001:bc8:1210:232:dc00:ff:fe20:185/64 + """ + ), + }, + "v2-dns": { + "expected_networkd": textwrap.dedent( + """\ + [Address] + Address=192.168.1.20/16 + + [Address] + Address=2001:bc8:1210:232:dc00:ff:fe20:185/64 + + [Match] + Name=eth0 + + [Network] + DHCP=no + DNS=8.8.8.8 FEDC::1 + Domains=lab home + """ + ), + "expected_eni": textwrap.dedent( + """\ + # This file is generated from information provided by the datasource. Changes + # to it will not persist across an instance reboot. To disable cloud-init's + # network configuration capabilities, write a file + # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: + # network: {config: disabled} + auto lo + iface lo inet loopback + + auto eth0 + iface eth0 inet static + address 192.168.1.20/16 + dns-nameservers 8.8.8.8 + dns-search lab home + + # control-alias eth0 + iface eth0 inet6 static + address 2001:bc8:1210:232:dc00:ff:fe20:185/64 + dns-nameservers FEDC::1 + dns-search lab home + """ # noqa: E501 + ), + "expected_sysconfig_opensuse": { + "ifcfg-eth0": textwrap.dedent( + """\ + # Created by cloud-init automatically, do not edit. + # + BOOTPROTO=static + IPADDR=192.168.1.20 + IPADDR6=2001:bc8:1210:232:dc00:ff:fe20:185/64 + NETMASK=255.255.0.0 + STARTMODE=auto + """ + ) + }, + "expected_sysconfig_rhel": { + "ifcfg-eth0": textwrap.dedent( + """\ + # Created by cloud-init automatically, do not edit. + # + BOOTPROTO=none + DEVICE=eth0 + DNS1=8.8.8.8 + DNS2=FEDC::1 + DOMAIN="lab home" + IPADDR=192.168.1.20 + IPV6ADDR=2001:bc8:1210:232:dc00:ff:fe20:185/64 + IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_FORCE_ACCEPT_RA=no + NETMASK=255.255.0.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """ + ) + }, + "expected_network_manager": { + "cloud-init-eth0.nmconnection": textwrap.dedent( + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init eth0 + uuid=1dd9a779-d327-56e1-8454-c65e2556c12c + autoconnect-priority=120 + type=ethernet + interface-name=eth0 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + + [ipv4] + method=manual + may-fail=false + address1=192.168.1.20/16 + dns=8.8.8.8; + dns-search=lab;home; + + [ipv6] + method=manual + may-fail=false + address1=2001:bc8:1210:232:dc00:ff:fe20:185/64 + dns=FEDC::1; + dns-search=lab;home; + + """ + ) + }, + "yaml": textwrap.dedent( + """\ + version: 2 + ethernets: + eth0: + nameservers: + search: [lab, home] + addresses: [8.8.8.8, "FEDC::1"] + addresses: + - 192.168.1.20/16 + - 2001:bc8:1210:232:dc00:ff:fe20:185/64 + """ + ), + }, + "v2-dns-no-if-ips": { + "expected_network_manager": { + "cloud-init-eth0.nmconnection": textwrap.dedent( + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init eth0 + uuid=1dd9a779-d327-56e1-8454-c65e2556c12c + autoconnect-priority=120 + type=ethernet + interface-name=eth0 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + + [ipv4] + method=auto + may-fail=true + dns=8.8.8.8; + dns-search=lab;home; + + [ipv6] + method=auto + may-fail=true + dns=FEDC::1; + dns-search=lab;home; + + """ + ) + }, + "yaml": textwrap.dedent( + """\ + version: 2 + ethernets: + eth0: + dhcp4: true + dhcp6: true + nameservers: + search: [lab, home] + addresses: [8.8.8.8, "FEDC::1"] + """ + ), + }, + "v2-dns-no-dhcp": { + "expected_network_manager": { + "cloud-init-eth0.nmconnection": textwrap.dedent( + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init eth0 + uuid=1dd9a779-d327-56e1-8454-c65e2556c12c + autoconnect-priority=120 + type=ethernet + interface-name=eth0 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + + """ + ) + }, + "yaml": textwrap.dedent( + """\ + version: 2 + ethernets: + eth0: + nameservers: + search: [lab, home] + addresses: [8.8.8.8, "FEDC::1"] + """ + ), + }, + "v2-route-no-gateway": { + "expected_network_manager": { + "cloud-init-eth0.nmconnection": textwrap.dedent( + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init eth0 + uuid=1dd9a779-d327-56e1-8454-c65e2556c12c + autoconnect-priority=120 + type=ethernet + interface-name=eth0 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + + [ipv4] + method=auto + may-fail=false + route1=0.0.0.0/0 + + """ + ) + }, + "yaml": textwrap.dedent( + """\ + version: 2 + ethernets: + eth0: + dhcp4: true + routes: + - to: "0.0.0.0/0" + """ + ), + }, } @@ -4339,6 +4826,7 @@ def test_device_driver_v2( "ethernets": { "eth0": { "dhcp4": True, + "dhcp6": True, "set-name": "eth0", "match": { "macaddress": "00:11:22:33:44:55", @@ -4423,6 +4911,9 @@ def test_device_driver( auto eth0 iface eth0 inet dhcp + +# control-alias eth0 +iface eth0 inet6 dhcp """ self.assertEqual(expected.lstrip(), contents.lstrip()) @@ -4512,6 +5003,9 @@ def test_hv_netvsc_vf_filter( auto eth1 iface eth1 inet dhcp + +# control-alias eth1 +iface eth1 inet6 dhcp """ self.assertEqual(expected.lstrip(), contents.lstrip()) @@ -4735,7 +5229,9 @@ def test_default_generation( # BOOTPROTO=dhcp DEVICE=eth1000 +DHCPV6C=yes HWADDR=07-1c-c6-75-a4-be +IPV6INIT=yes NM_CONTROLLED=no ONBOOT=yes TYPE=Ethernet @@ -5228,6 +5724,20 @@ def test_netplan_dhcp_false_disable_dhcp_in_state(self): self.assertEqual([], dhcp_found) + @pytest.mark.xfail(reason="sysconfig should render interface-level DNS") + def test_v1_dns(self): + entry = NETWORK_CONFIGS["v1-dns"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + # TODO: verify resolv.conf + + def test_v2_dns(self): + entry = NETWORK_CONFIGS["v2-dns"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + def test_netplan_dhcp_false_no_dhcp_in_sysconfig(self): """netplan cfg with dhcp[46]: False should not have bootproto=dhcp""" @@ -5503,7 +6013,6 @@ def test_from_v2_routes(self): """ # noqa: E501 ), } - log.setup_logging() found = self._render_and_read(network_config=v2_data) self._compare_files_to_expected(expected, found) @@ -5646,7 +6155,8 @@ def test_default_generation( expected_content = """ # Created by cloud-init automatically, do not edit. # -BOOTPROTO=dhcp4 +BOOTPROTO=dhcp +DHCLIENT6_MODE=managed LLADDR=07-1c-c6-75-a4-be STARTMODE=auto """.lstrip() @@ -5918,6 +6428,19 @@ def test_render_v6_and_v4(self): self._compare_files_to_expected(entry[self.expected_name], found) self._assert_headers(found) + def test_v1_dns(self): + entry = NETWORK_CONFIGS["v1-dns"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + # TODO: verify resolv.conf + + def test_v2_dns(self): + entry = NETWORK_CONFIGS["v2-dns"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected(entry[self.expected_name], found) + self._assert_headers(found) + @mock.patch( "cloudinit.net.is_openvswitch_internal_interface", @@ -6032,7 +6555,11 @@ def test_default_generation( [ipv4] method=auto - may-fail=false + may-fail=true + + [ipv6] + method=auto + may-fail=true """ ), @@ -6253,6 +6780,48 @@ def test_render_v6_and_v4(self): entry[self.expected_name], self.expected_conf_d, found ) + def test_v1_dns(self): + entry = NETWORK_CONFIGS["v1-dns"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected( + entry[self.expected_name], self.expected_conf_d, found + ) + + def test_v2_mixed_routes(self): + entry = NETWORK_CONFIGS["v2-mixed-routes"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected( + entry[self.expected_name], self.expected_conf_d, found + ) + + def test_v2_dns(self): + entry = NETWORK_CONFIGS["v2-dns"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected( + entry[self.expected_name], self.expected_conf_d, found + ) + + def test_v2_dns_no_ips(self): + entry = NETWORK_CONFIGS["v2-dns-no-if-ips"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected( + entry[self.expected_name], self.expected_conf_d, found + ) + + def test_v2_dns_no_dhcp(self): + entry = NETWORK_CONFIGS["v2-dns-no-dhcp"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected( + entry[self.expected_name], self.expected_conf_d, found + ) + + def test_v2_route_no_gateway(self): + entry = NETWORK_CONFIGS["v2-route-no-gateway"] + found = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self._compare_files_to_expected( + entry[self.expected_name], self.expected_conf_d, found + ) + @mock.patch( "cloudinit.net.is_openvswitch_internal_interface", @@ -6298,6 +6867,9 @@ def test_default_generation( auto eth1000 iface eth1000 inet dhcp + +# control-alias eth1000 +iface eth1000 inet6 dhcp """ self.assertEqual(expected.lstrip(), contents.lstrip()) @@ -6357,6 +6929,7 @@ class TestNetplanNetRendering: ethernets: eth1000: dhcp4: true + dhcp6: true match: macaddress: 07-1c-c6-75-a4-be set-name: eth1000 @@ -7418,6 +7991,17 @@ def testsimple_render_manual(self): files["/etc/netplan/50-cloud-init.yaml"].splitlines(), ) + @pytest.mark.xfail( + reason="netplan should render interface-level nameservers" + ) + def testsimple_render_v1_dns(self): + entry = NETWORK_CONFIGS["v1-dns"] + files = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self.assertEqual( + entry["expected_netplan"].splitlines(), + files["/etc/netplan/50-cloud-init.yaml"].splitlines(), + ) + def test_render_output_has_yaml_no_aliases(self): entry = { "yaml": V1_NAMESERVER_ALIAS, @@ -7792,6 +8376,24 @@ def testsimple_render_bond(self): files["/etc/network/interfaces"].splitlines(), ) + @pytest.mark.xfail(reason="GH-4219") + def test_v1_dns(self): + entry = NETWORK_CONFIGS["v1-dns"] + files = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self.assertEqual( + entry["expected_eni"].splitlines(), + files["/etc/network/interfaces"].splitlines(), + ) + + @pytest.mark.xfail(reason="GH-4219") + def test_v2_dns(self): + entry = NETWORK_CONFIGS["v2-dns"] + files = self._render_and_read(network_config=yaml.load(entry["yaml"])) + self.assertEqual( + entry["expected_eni"].splitlines(), + files["/etc/network/interfaces"].splitlines(), + ) + class TestNetworkdNetRendering(CiTestCase): def create_conf_dict(self, contents): @@ -7856,7 +8458,7 @@ def test_networkd_default_generation( Name=eth1000 MACAddress=07-1c-c6-75-a4-be [Network] - DHCP=ipv4""" + DHCP=yes""" ).rstrip(" ") expected = self.create_conf_dict(expected.splitlines()) @@ -8055,6 +8657,35 @@ def test_dhcpv6_reject_ra_config_v2(self, m_chown): self.compare_dicts(actual, expected) + @pytest.mark.xfail( + reason="DNS and Domains getting rendered on multiple lines" + ) + @mock.patch("cloudinit.net.util.chownbyname", return_value=True) + def test_v1_dns(self, m_chown): + nwk_fn = "/etc/systemd/network/10-cloud-init-eth0.network" + entry = NETWORK_CONFIGS["v1-dns"] + files = self._render_and_read(network_config=yaml.load(entry["yaml"])) + + actual = self.create_conf_dict(files[nwk_fn].splitlines()) + expected = self.create_conf_dict( + entry["expected_networkd"].splitlines() + ) + + self.compare_dicts(actual, expected) + + @mock.patch("cloudinit.net.util.chownbyname", return_value=True) + def test_v2_dns(self, m_chown): + nwk_fn = "/etc/systemd/network/10-cloud-init-eth0.network" + entry = NETWORK_CONFIGS["v2-dns"] + files = self._render_and_read(network_config=yaml.load(entry["yaml"])) + + actual = self.create_conf_dict(files[nwk_fn].splitlines()) + expected = self.create_conf_dict( + entry["expected_networkd"].splitlines() + ) + + self.compare_dicts(actual, expected) + class TestRenderersSelect: @pytest.mark.parametrize( diff --git a/.pc/retain-apt-pre-deb822.patch/cloudinit/features.py b/.pc/retain-apt-pre-deb822.patch/cloudinit/features.py index bc8d38dff..3fc22cff5 100644 --- a/.pc/retain-apt-pre-deb822.patch/cloudinit/features.py +++ b/.pc/retain-apt-pre-deb822.patch/cloudinit/features.py @@ -62,7 +62,7 @@ NETPLAN_CONFIG_ROOT_READ_ONLY = False """ If ``NETPLAN_CONFIG_ROOT_READ_ONLY`` is True, then netplan configuration will -be written as a single root readon-only file /etc/netplan/50-cloud-init.yaml. +be written as a single root read-only file /etc/netplan/50-cloud-init.yaml. This prevents wifi passwords in network v2 configuration from being world-readable. Prior to 23.1, netplan configuration is world-readable. diff --git a/.pc/retain-ec2-default-net-update-events.patch/cloudinit/sources/DataSourceEc2.py b/.pc/retain-ec2-default-net-update-events.patch/cloudinit/sources/DataSourceEc2.py new file mode 100644 index 000000000..1b81b21f0 --- /dev/null +++ b/.pc/retain-ec2-default-net-update-events.patch/cloudinit/sources/DataSourceEc2.py @@ -0,0 +1,1212 @@ +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# Author: Joshua Harlow +# +# This file is part of cloud-init. See LICENSE file for license information. + +import copy +import logging +import os +import time +from typing import Dict, List + +from cloudinit import dmi, net, sources +from cloudinit import url_helper as uhelp +from cloudinit import util, warnings +from cloudinit.distros import Distro +from cloudinit.event import EventScope, EventType +from cloudinit.net import activators +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralIPNetwork +from cloudinit.sources.helpers import ec2 + +LOG = logging.getLogger(__name__) + +SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND]) + +STRICT_ID_PATH = ("datasource", "Ec2", "strict_id") +STRICT_ID_DEFAULT = "warn" + + +class CloudNames: + ALIYUN = "aliyun" + AWS = "aws" + BRIGHTBOX = "brightbox" + ZSTACK = "zstack" + E24CLOUD = "e24cloud" + OUTSCALE = "outscale" + # UNKNOWN indicates no positive id. If strict_id is 'warn' or 'false', + # then an attempt at the Ec2 Metadata service will be made. + UNKNOWN = "unknown" + # NO_EC2_METADATA indicates this platform does not have a Ec2 metadata + # service available. No attempt at the Ec2 Metadata service will be made. + NO_EC2_METADATA = "no-ec2-metadata" + + +# Drop when LP: #1988157 tag handling is fixed +def skip_404_tag_errors(exception): + return exception.code == 404 and "meta-data/tags/" in exception.url + + +# Cloud platforms that support IMDSv2 style metadata server +IDMSV2_SUPPORTED_CLOUD_PLATFORMS = [CloudNames.AWS, CloudNames.ALIYUN] + +# Only trigger hook-hotplug on NICs with Ec2 drivers. Avoid triggering +# it on docker virtual NICs and the like. LP: #1946003 +_EXTRA_HOTPLUG_UDEV_RULES = """ +ENV{ID_NET_DRIVER}=="vif|ena|ixgbevf", GOTO="cloudinit_hook" +GOTO="cloudinit_end" +""" + + +class DataSourceEc2(sources.DataSource): + dsname = "Ec2" + # Default metadata urls that will be used if none are provided + # They will be checked for 'resolveability' and some of the + # following may be discarded if they do not resolve + metadata_urls = [ + "http://169.254.169.254", + "http://[fd00:ec2::254]", + "http://instance-data.:8773", + ] + + # The minimum supported metadata_version from the ec2 metadata apis + min_metadata_version = "2009-04-04" + + # Priority ordered list of additional metadata versions which will be tried + # for extended metadata content. IPv6 support comes in 2016-09-02. + # Tags support comes in 2021-03-23. + extended_metadata_versions: List[str] = [ + "2021-03-23", + "2018-09-24", + "2016-09-02", + ] + + # Setup read_url parameters per get_url_params. + url_max_wait = 120 + url_timeout = 50 + + _api_token = None # API token for accessing the metadata service + _network_config = sources.UNSET # Used to cache calculated network cfg v1 + + # Whether we want to get network configuration from the metadata service. + perform_dhcp_setup = False + + supported_update_events = { + EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, + EventType.BOOT_LEGACY, + EventType.HOTPLUG, + } + } + + default_update_events = { + EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.HOTPLUG, + } + } + + extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES + + def __init__(self, sys_cfg, distro, paths): + super(DataSourceEc2, self).__init__(sys_cfg, distro, paths) + self.metadata_address = None + + def _unpickle(self, ci_pkl_version: int) -> None: + super()._unpickle(ci_pkl_version) + self.extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES + + def _get_cloud_name(self): + """Return the cloud name as identified during _get_data.""" + return identify_platform() + + def _get_data(self): + strict_mode, _sleep = read_strict_mode( + util.get_cfg_by_path( + self.sys_cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT + ), + ("warn", None), + ) + + LOG.debug( + "strict_mode: %s, cloud_name=%s cloud_platform=%s", + strict_mode, + self.cloud_name, + self.platform, + ) + if strict_mode == "true" and self.cloud_name == CloudNames.UNKNOWN: + return False + elif self.cloud_name == CloudNames.NO_EC2_METADATA: + return False + + if self.perform_dhcp_setup: # Setup networking in init-local stage. + if util.is_FreeBSD(): + LOG.debug("FreeBSD doesn't support running dhclient with -sf") + return False + try: + with EphemeralIPNetwork( + self.distro, + self.distro.fallback_interface, + ipv4=True, + ipv6=True, + ) as netw: + state_msg = f" {netw.state_msg}" if netw.state_msg else "" + self._crawled_metadata = util.log_time( + logfunc=LOG.debug, + msg=f"Crawl of metadata service{state_msg}", + func=self.crawl_metadata, + ) + + except NoDHCPLeaseError: + return False + else: + self._crawled_metadata = util.log_time( + logfunc=LOG.debug, + msg="Crawl of metadata service", + func=self.crawl_metadata, + ) + if not self._crawled_metadata: + return False + self.metadata = self._crawled_metadata.get("meta-data", None) + self.userdata_raw = self._crawled_metadata.get("user-data", None) + self.identity = ( + self._crawled_metadata.get("dynamic", {}) + .get("instance-identity", {}) + .get("document", {}) + ) + return True + + def is_classic_instance(self): + """Report if this instance type is Ec2 Classic (non-vpc).""" + if not self.metadata: + # Can return False on inconclusive as we are also called in + # network_config where metadata will be present. + # Secondary call site is in packaging postinst script. + return False + ifaces_md = self.metadata.get("network", {}).get("interfaces", {}) + for _mac, mac_data in ifaces_md.get("macs", {}).items(): + if "vpc-id" in mac_data: + return False + return True + + @property + def launch_index(self): + if not self.metadata: + return None + return self.metadata.get("ami-launch-index") + + @property + def platform(self): + # Handle upgrade path of pickled ds + if not hasattr(self, "_platform_type"): + self._platform_type = DataSourceEc2.dsname.lower() + if not self._platform_type: + self._platform_type = DataSourceEc2.dsname.lower() + return self._platform_type + + # IMDSv2 related parameters from the ec2 metadata api document + @property + def api_token_route(self): + return "latest/api/token" + + @property + def imdsv2_token_ttl_seconds(self): + return "21600" + + @property + def imdsv2_token_put_header(self): + return "X-aws-ec2-metadata-token" + + @property + def imdsv2_token_req_header(self): + return self.imdsv2_token_put_header + "-ttl-seconds" + + @property + def imdsv2_token_redact(self): + return [self.imdsv2_token_put_header, self.imdsv2_token_req_header] + + def get_metadata_api_version(self): + """Get the best supported api version from the metadata service. + + Loop through all extended support metadata versions in order and + return the most-fully featured metadata api version discovered. + + If extended_metadata_versions aren't present, return the datasource's + min_metadata_version. + """ + # Assumes metadata service is already up + url_tmpl = "{0}/{1}/meta-data/instance-id" + headers = self._get_headers() + for api_ver in self.extended_metadata_versions: + url = url_tmpl.format(self.metadata_address, api_ver) + try: + resp = uhelp.readurl( + url=url, + headers=headers, + headers_redact=self.imdsv2_token_redact, + ) + except uhelp.UrlError as e: + LOG.debug("url %s raised exception %s", url, e) + else: + if resp.code == 200: + LOG.debug("Found preferred metadata version %s", api_ver) + return api_ver + elif resp.code == 404: + msg = "Metadata api version %s not present. Headers: %s" + LOG.debug(msg, api_ver, resp.headers) + return self.min_metadata_version + + def get_instance_id(self): + if self.cloud_name == CloudNames.AWS: + # Prefer the ID from the instance identity document, but fall back + if not getattr(self, "identity", None): + # If re-using cached datasource, it's get_data run didn't + # setup self.identity. So we need to do that now. + api_version = self.get_metadata_api_version() + self.identity = ec2.get_instance_identity( + api_version, + self.metadata_address, + headers_cb=self._get_headers, + headers_redact=self.imdsv2_token_redact, + exception_cb=self._refresh_stale_aws_token_cb, + ).get("document", {}) + return self.identity.get( + "instanceId", self.metadata["instance-id"] + ) + else: + return self.metadata["instance-id"] + + def _maybe_fetch_api_token(self, mdurls): + """Get an API token for EC2 Instance Metadata Service. + + On EC2. IMDS will always answer an API token, unless + the instance owner has disabled the IMDS HTTP endpoint or + the network topology conflicts with the configured hop-limit. + """ + if self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: + return + + urls = [] + url2base = {} + url_path = self.api_token_route + request_method = "PUT" + for url in mdurls: + cur = "{0}/{1}".format(url, url_path) + urls.append(cur) + url2base[cur] = url + + # use the self._imds_exception_cb to check for Read errors + LOG.debug("Fetching Ec2 IMDSv2 API Token") + + response = None + url = None + url_params = self.get_url_params() + try: + url, response = uhelp.wait_for_url( + urls=urls, + max_wait=url_params.max_wait_seconds, + timeout=url_params.timeout_seconds, + status_cb=LOG.warning, + headers_cb=self._get_headers, + exception_cb=self._imds_exception_cb, + request_method=request_method, + headers_redact=self.imdsv2_token_redact, + connect_synchronously=False, + ) + except uhelp.UrlError: + # We use the raised exception to interrupt the retry loop. + # Nothing else to do here. + pass + + if url and response: + self._api_token = response + return url2base[url] + + # If we get here, then wait_for_url timed out, waiting for IMDS + # or the IMDS HTTP endpoint is disabled + return None + + def wait_for_metadata_service(self): + mcfg = self.ds_cfg + + url_params = self.get_url_params() + if url_params.max_wait_seconds <= 0: + return False + + # Remove addresses from the list that wont resolve. + mdurls = mcfg.get("metadata_urls", self.metadata_urls) + filtered = [x for x in mdurls if util.is_resolvable_url(x)] + + if set(filtered) != set(mdurls): + LOG.debug( + "Removed the following from metadata urls: %s", + list((set(mdurls) - set(filtered))), + ) + + if len(filtered): + mdurls = filtered + else: + LOG.warning("Empty metadata url list! using default list") + mdurls = self.metadata_urls + + # try the api token path first + metadata_address = self._maybe_fetch_api_token(mdurls) + # When running on EC2, we always access IMDS with an API token. + # If we could not get an API token, then we assume the IMDS + # endpoint was disabled and we move on without a data source. + # Fallback to IMDSv1 if not running on EC2 + if ( + not metadata_address + and self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS + ): + # if we can't get a token, use instance-id path + urls = [] + url2base = {} + url_path = "{ver}/meta-data/instance-id".format( + ver=self.min_metadata_version + ) + request_method = "GET" + for url in mdurls: + cur = "{0}/{1}".format(url, url_path) + urls.append(cur) + url2base[cur] = url + + start_time = time.time() + url, _ = uhelp.wait_for_url( + urls=urls, + max_wait=url_params.max_wait_seconds, + timeout=url_params.timeout_seconds, + status_cb=LOG.warning, + headers_redact=self.imdsv2_token_redact, + headers_cb=self._get_headers, + request_method=request_method, + ) + + if url: + metadata_address = url2base[url] + + if metadata_address: + self.metadata_address = metadata_address + LOG.debug("Using metadata source: '%s'", self.metadata_address) + elif self.cloud_name in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: + LOG.warning("IMDS's HTTP endpoint is probably disabled") + else: + LOG.critical( + "Giving up on md from %s after %s seconds", + urls, + int(time.time() - start_time), + ) + + return bool(metadata_address) + + def device_name_to_device(self, name): + # Consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + if "block-device-mapping" not in self.metadata: + return None + + # Example: + # 'block-device-mapping': + # {'ami': '/dev/sda1', + # 'ephemeral0': '/dev/sdb', + # 'root': '/dev/sda1'} + found = None + bdm = self.metadata["block-device-mapping"] + if not isinstance(bdm, dict): + LOG.debug("block-device-mapping not a dictionary: '%s'", bdm) + return None + + for entname, device in bdm.items(): + if entname == name: + found = device + break + # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' + if entname == "ephemeral" and name == "ephemeral0": + found = device + + if found is None: + LOG.debug("Unable to convert %s to a device", name) + return None + + ofound = found + if not found.startswith("/"): + found = "/dev/%s" % found + + if os.path.exists(found): + return found + + remapped = self._remap_device(os.path.basename(found)) + if remapped: + LOG.debug("Remapped device name %s => %s", found, remapped) + return remapped + + # On t1.micro, ephemeral0 will appear in block-device-mapping from + # metadata, but it will not exist on disk (and never will) + # at this point, we've verified that the path did not exist + # in the special case of 'ephemeral0' return None to avoid bogus + # fstab entry (LP: #744019) + if name == "ephemeral0": + return None + return ofound + + @property + def availability_zone(self): + try: + if self.cloud_name == CloudNames.AWS: + return self.identity.get( + "availabilityZone", + self.metadata["placement"]["availability-zone"], + ) + else: + return self.metadata["placement"]["availability-zone"] + except KeyError: + return None + + @property + def region(self): + if self.cloud_name == CloudNames.AWS: + region = self.identity.get("region") + # Fallback to trimming the availability zone if region is missing + if self.availability_zone and not region: + region = self.availability_zone[:-1] + return region + else: + az = self.availability_zone + if az is not None: + return az[:-1] + return None + + def activate(self, cfg, is_new_instance): + if not is_new_instance: + return + if self.cloud_name == CloudNames.UNKNOWN: + warn_if_necessary( + util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT), + cfg, + ) + + @property + def network_config(self): + """Return a network config dict for rendering ENI or netplan files.""" + if self._network_config != sources.UNSET: + return self._network_config + + if self.metadata is None: + # this would happen if get_data hadn't been called. leave as UNSET + LOG.warning( + "Unexpected call to network_config when metadata is None." + ) + return None + + result = None + no_network_metadata_on_aws = bool( + "network" not in self.metadata + and self.cloud_name == CloudNames.AWS + ) + if no_network_metadata_on_aws: + LOG.debug( + "Metadata 'network' not present:" + " Refreshing stale metadata from prior to upgrade." + ) + util.log_time( + logfunc=LOG.debug, + msg="Re-crawl of metadata service", + func=self.get_data, + ) + + iface = self.distro.fallback_interface + net_md = self.metadata.get("network") + if isinstance(net_md, dict): + # SRU_BLOCKER: xenial, bionic and eoan should default + # apply_full_imds_network_config to False to retain original + # behavior on those releases. + result = convert_ec2_metadata_network_config( + net_md, + self.distro, + fallback_nic=iface, + full_network_config=util.get_cfg_option_bool( + self.ds_cfg, "apply_full_imds_network_config", True + ), + ) + + # Non-VPC (aka Classic) Ec2 instances need to rewrite the + # network config file every boot due to MAC address change. + if self.is_classic_instance(): + self.default_update_events = copy.deepcopy( + self.default_update_events + ) + self.default_update_events[EventScope.NETWORK].add( + EventType.BOOT + ) + self.default_update_events[EventScope.NETWORK].add( + EventType.BOOT_LEGACY + ) + else: + LOG.warning("Metadata 'network' key not valid: %s.", net_md) + self._network_config = result + + return self._network_config + + def crawl_metadata(self): + """Crawl metadata service when available. + + @returns: Dictionary of crawled metadata content containing the keys: + meta-data, user-data and dynamic. + """ + if not self.wait_for_metadata_service(): + return {} + api_version = self.get_metadata_api_version() + redact = self.imdsv2_token_redact + crawled_metadata = {} + if self.cloud_name in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: + exc_cb = self._refresh_stale_aws_token_cb + exc_cb_ud = self._skip_or_refresh_stale_aws_token_cb + skip_cb = None + elif self.cloud_name == CloudNames.OUTSCALE: + exc_cb = exc_cb_ud = None + skip_cb = skip_404_tag_errors + else: + exc_cb = exc_cb_ud = skip_cb = None + try: + raw_userdata = ec2.get_instance_userdata( + api_version, + self.metadata_address, + headers_cb=self._get_headers, + headers_redact=redact, + exception_cb=exc_cb_ud, + ) + crawled_metadata["user-data"] = util.maybe_b64decode(raw_userdata) + crawled_metadata["meta-data"] = ec2.get_instance_metadata( + api_version, + self.metadata_address, + headers_cb=self._get_headers, + headers_redact=redact, + exception_cb=exc_cb, + retrieval_exception_ignore_cb=skip_cb, + ) + if self.cloud_name == CloudNames.AWS: + identity = ec2.get_instance_identity( + api_version, + self.metadata_address, + headers_cb=self._get_headers, + headers_redact=redact, + exception_cb=exc_cb, + ) + crawled_metadata["dynamic"] = {"instance-identity": identity} + except Exception: + util.logexc( + LOG, + "Failed reading from metadata address %s", + self.metadata_address, + ) + return {} + crawled_metadata["_metadata_api_version"] = api_version + return crawled_metadata + + def _refresh_api_token(self, seconds=None): + """Request new metadata API token. + @param seconds: The lifetime of the token in seconds + + @return: The API token or None if unavailable. + """ + if self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: + return None + + if seconds is None: + seconds = self.imdsv2_token_ttl_seconds + + LOG.debug("Refreshing Ec2 metadata API token") + request_header = {self.imdsv2_token_req_header: seconds} + token_url = "{}/{}".format(self.metadata_address, self.api_token_route) + try: + response = uhelp.readurl( + token_url, + headers=request_header, + headers_redact=self.imdsv2_token_redact, + request_method="PUT", + ) + except uhelp.UrlError as e: + LOG.warning( + "Unable to get API token: %s raised exception %s", token_url, e + ) + return None + return response.contents + + def _skip_or_refresh_stale_aws_token_cb(self, msg, exception): + """Callback will not retry on SKIP_USERDATA_CODES or if no token + is available.""" + retry = ec2.skip_retry_on_codes( + ec2.SKIP_USERDATA_CODES, msg, exception + ) + if not retry: + return False # False raises exception + return self._refresh_stale_aws_token_cb(msg, exception) + + def _refresh_stale_aws_token_cb(self, msg, exception): + """Exception handler for Ec2 to refresh token if token is stale.""" + if isinstance(exception, uhelp.UrlError) and exception.code == 401: + # With _api_token as None, _get_headers will _refresh_api_token. + LOG.debug("Clearing cached Ec2 API token due to expiry") + self._api_token = None + return True # always retry + + def _imds_exception_cb(self, msg, exception=None): + """Fail quickly on proper AWS if IMDSv2 rejects API token request + + Guidance from Amazon is that if IMDSv2 had disabled token requests + by returning a 403, or cloud-init malformed requests resulting in + other 40X errors, we want the datasource detection to fail quickly + without retries as those symptoms will likely not be resolved by + retries. + + Exceptions such as requests.ConnectionError due to IMDS being + temporarily unroutable or unavailable will still retry due to the + callsite wait_for_url. + """ + if isinstance(exception, uhelp.UrlError): + # requests.ConnectionError will have exception.code == None + if exception.code and exception.code >= 400: + if exception.code == 403: + LOG.warning( + "Ec2 IMDS endpoint returned a 403 error. " + "HTTP endpoint is disabled. Aborting." + ) + else: + LOG.warning( + "Fatal error while requesting Ec2 IMDSv2 API tokens" + ) + raise exception + + def _get_headers(self, url=""): + """Return a dict of headers for accessing a url. + + If _api_token is unset on AWS, attempt to refresh the token via a PUT + and then return the updated token header. + """ + if self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: + return {} + # Request a 6 hour token if URL is api_token_route + request_token_header = { + self.imdsv2_token_req_header: self.imdsv2_token_ttl_seconds + } + if self.api_token_route in url: + return request_token_header + if not self._api_token: + # If we don't yet have an API token, get one via a PUT against + # api_token_route. This _api_token may get unset by a 403 due + # to an invalid or expired token + self._api_token = self._refresh_api_token() + if not self._api_token: + return {} + return {self.imdsv2_token_put_header: self._api_token} + + +class DataSourceEc2Local(DataSourceEc2): + """Datasource run at init-local which sets up network to query metadata. + + In init-local, no network is available. This subclass sets up minimal + networking with dhclient on a viable nic so that it can talk to the + metadata service. If the metadata service provides network configuration + then render the network configuration for that instance based on metadata. + """ + + perform_dhcp_setup = True # Use dhcp before querying metadata + + def get_data(self): + supported_platforms = (CloudNames.AWS, CloudNames.OUTSCALE) + if self.cloud_name not in supported_platforms: + LOG.debug( + "Local Ec2 mode only supported on %s, not %s", + supported_platforms, + self.cloud_name, + ) + return False + return super(DataSourceEc2Local, self).get_data() + + +def read_strict_mode(cfgval, default): + try: + return parse_strict_mode(cfgval) + except ValueError as e: + LOG.warning(e) + return default + + +def parse_strict_mode(cfgval): + # given a mode like: + # true, false, warn,[sleep] + # return tuple with string mode (true|false|warn) and sleep. + if cfgval is True: + return "true", None + if cfgval is False: + return "false", None + + if not cfgval: + return "warn", 0 + + mode, _, sleep = cfgval.partition(",") + if mode not in ("true", "false", "warn"): + raise ValueError( + "Invalid mode '%s' in strict_id setting '%s': " + "Expected one of 'true', 'false', 'warn'." % (mode, cfgval) + ) + + if sleep: + try: + sleep = int(sleep) + except ValueError as e: + raise ValueError( + "Invalid sleep '%s' in strict_id setting '%s': not an integer" + % (sleep, cfgval) + ) from e + else: + sleep = None + + return mode, sleep + + +def warn_if_necessary(cfgval, cfg): + try: + mode, sleep = parse_strict_mode(cfgval) + except ValueError as e: + LOG.warning(e) + return + + if mode == "false": + return + + warnings.show_warning("non_ec2_md", cfg, mode=True, sleep=sleep) + + +def identify_aliyun(data): + if data["product_name"] == "Alibaba Cloud ECS": + return CloudNames.ALIYUN + + +def identify_aws(data): + # data is a dictionary returned by _collect_platform_data. + if data["uuid"].startswith("ec2") and ( + data["uuid_source"] == "hypervisor" or data["uuid"] == data["serial"] + ): + return CloudNames.AWS + + return None + + +def identify_brightbox(data): + if data["serial"].endswith(".brightbox.com"): + return CloudNames.BRIGHTBOX + + +def identify_zstack(data): + if data["asset_tag"].endswith(".zstack.io"): + return CloudNames.ZSTACK + + +def identify_e24cloud(data): + if data["vendor"] == "e24cloud": + return CloudNames.E24CLOUD + + +def identify_outscale(data): + if ( + data["product_name"] == "3DS Outscale VM".lower() + and data["vendor"] == "3DS Outscale".lower() + ): + return CloudNames.OUTSCALE + + +def identify_platform(): + # identify the platform and return an entry in CloudNames. + data = _collect_platform_data() + checks = ( + identify_aws, + identify_brightbox, + identify_zstack, + identify_e24cloud, + identify_outscale, + identify_aliyun, + lambda x: CloudNames.UNKNOWN, + ) + for checker in checks: + try: + result = checker(data) + if result: + return result + except Exception as e: + LOG.warning( + "calling %s with %s raised exception: %s", checker, data, e + ) + + +def _collect_platform_data(): + """Returns a dictionary of platform info from dmi or /sys/hypervisor. + + Keys in the dictionary are as follows: + uuid: system-uuid from dmi or /sys/hypervisor + uuid_source: 'hypervisor' (/sys/hypervisor/uuid) or 'dmi' + serial: dmi 'system-serial-number' (/sys/.../product_serial) + asset_tag: 'dmidecode -s chassis-asset-tag' + vendor: dmi 'system-manufacturer' (/sys/.../sys_vendor) + product_name: dmi 'system-product-name' (/sys/.../system-manufacturer) + + On Ec2 instances experimentation is that product_serial is upper case, + and product_uuid is lower case. This returns lower case values for both. + """ + data = {} + try: + uuid = util.load_text_file("/sys/hypervisor/uuid").strip() + data["uuid_source"] = "hypervisor" + except Exception: + uuid = dmi.read_dmi_data("system-uuid") + data["uuid_source"] = "dmi" + + if uuid is None: + uuid = "" + data["uuid"] = uuid.lower() + + serial = dmi.read_dmi_data("system-serial-number") + if serial is None: + serial = "" + + data["serial"] = serial.lower() + + asset_tag = dmi.read_dmi_data("chassis-asset-tag") + if asset_tag is None: + asset_tag = "" + + data["asset_tag"] = asset_tag.lower() + + vendor = dmi.read_dmi_data("system-manufacturer") + data["vendor"] = (vendor if vendor else "").lower() + + product_name = dmi.read_dmi_data("system-product-name") + data["product_name"] = (product_name if product_name else "").lower() + + return data + + +def _build_nic_order( + macs_metadata: Dict[str, Dict], macs: List[str] +) -> Dict[str, int]: + """ + Builds a dictionary containing macs as keys nad nic orders as values, + taking into account `network-card` and `device-number` if present. + + Note that the first NIC will be the primary NIC as it will be the one with + [network-card] == 0 and device-number == 0 if present. + + @param macs_metadata: dictionary with mac address as key and contents like: + {"device-number": "0", "interface-id": "...", "local-ipv4s": ...} + @macs: list of macs to consider + + @return: Dictionary with macs as keys and nic orders as values. + """ + nic_order: Dict[str, int] = {} + if len(macs) == 0 or len(macs_metadata) == 0: + return nic_order + + valid_macs_metadata = filter( + # filter out nics without metadata (not a physical nic) + lambda mmd: mmd[1] is not None, + # filter by macs + map(lambda mac: (mac, macs_metadata.get(mac)), macs), + ) + + def _get_key_as_int_or(dikt, key, alt_value): + value = dikt.get(key, None) + if value is not None: + return int(value) + return alt_value + + # Sort by (network_card, device_index) as some instances could have + # multiple network cards with repeated device indexes. + # + # On platforms where network-card and device-number are not present, + # as AliYun, the order will be by mac, as before the introduction of this + # function. + return { + mac: i + for i, (mac, _mac_metadata) in enumerate( + sorted( + valid_macs_metadata, + key=lambda mmd: ( + _get_key_as_int_or( + mmd[1], "network-card", float("infinity") + ), + _get_key_as_int_or( + mmd[1], "device-number", float("infinity") + ), + ), + ) + ) + } + + +def _configure_policy_routing( + dev_config: dict, + *, + nic_name: str, + nic_metadata: dict, + distro: Distro, + is_ipv4: bool, + table: int, +) -> None: + """ + Configure policy-based routing on secondary NICs / secondary IPs to + ensure outgoing packets are routed via the correct interface. + + @param: dev_config: network cfg v2 to be updated inplace. + @param: nic_name: nic name. Only used if ipv4. + @param: nic_metadata: nic metadata from IMDS. + @param: distro: Instance of Distro. Only used if ipv4. + @param: is_ipv4: Boolean indicating if we are acting over ipv4 or not. + @param: table: Routing table id. + """ + if not dev_config.get("routes"): + dev_config["routes"] = [] + if is_ipv4: + subnet_prefix_routes = nic_metadata["subnet-ipv4-cidr-block"] + ips = nic_metadata["local-ipv4s"] + try: + lease = distro.dhcp_client.dhcp_discovery(nic_name, distro=distro) + gateway = lease["routers"] + except NoDHCPLeaseError as e: + LOG.warning( + "Could not perform dhcp discovery on %s to find its " + "gateway. Not adding default route via the gateway. " + "Error: %s", + nic_name, + e, + ) + else: + # Add default route via the NIC's gateway + dev_config["routes"].append( + { + "to": "0.0.0.0/0", + "via": gateway, + "table": table, + }, + ) + else: + subnet_prefix_routes = nic_metadata["subnet-ipv6-cidr-blocks"] + ips = nic_metadata["ipv6s"] + + subnet_prefix_routes = ( + [subnet_prefix_routes] + if isinstance(subnet_prefix_routes, str) + else subnet_prefix_routes + ) + for prefix_route in subnet_prefix_routes: + dev_config["routes"].append( + { + "to": prefix_route, + "table": table, + }, + ) + + if not dev_config.get("routing-policy"): + dev_config["routing-policy"] = [] + # Packets coming from any IP associated with the current NIC + # will be routed using `table` routing table + ips = [ips] if isinstance(ips, str) else ips + for ip in ips: + dev_config["routing-policy"].append( + { + "from": ip, + "table": table, + }, + ) + + +def convert_ec2_metadata_network_config( + network_md, + distro, + macs_to_nics=None, + fallback_nic=None, + full_network_config=True, +): + """Convert ec2 metadata to network config version 2 data dict. + + @param: network_md: 'network' portion of EC2 metadata. + generally formed as {"interfaces": {"macs": {}} where + 'macs' is a dictionary with mac address as key and contents like: + {"device-number": "0", "interface-id": "...", "local-ipv4s": ...} + @param: distro: instance of Distro. + @param: macs_to_nics: Optional dict of mac addresses and nic names. If + not provided, get_interfaces_by_mac is called to get it from the OS. + @param: fallback_nic: Optionally provide the primary nic interface name. + This nic will be guaranteed to minimally have a dhcp4 configuration. + @param: full_network_config: Boolean set True to configure all networking + presented by IMDS. This includes rendering secondary IPv4 and IPv6 + addresses on all NICs and rendering network config on secondary NICs. + If False, only the primary nic will be configured and only with dhcp + (IPv4/IPv6). + + @return A dict of network config version 2 based on the metadata and macs. + """ + netcfg = {"version": 2, "ethernets": {}} + if not macs_to_nics: + macs_to_nics = net.get_interfaces_by_mac() + macs_metadata = network_md["interfaces"]["macs"] + + if not full_network_config: + for mac, nic_name in macs_to_nics.items(): + if nic_name == fallback_nic: + break + dev_config = { + "dhcp4": True, + "dhcp6": False, + "match": {"macaddress": mac.lower()}, + "set-name": nic_name, + } + nic_metadata = macs_metadata.get(mac) + if nic_metadata.get("ipv6s"): # Any IPv6 addresses configured + dev_config["dhcp6"] = True + netcfg["ethernets"][nic_name] = dev_config + return netcfg + # Apply network config for all nics and any secondary IPv4/v6 addresses + is_netplan = distro.network_activator == activators.NetplanActivator + macs = sorted(macs_to_nics.keys()) + nic_order = _build_nic_order(macs_metadata, macs) + for mac in macs: + nic_name = macs_to_nics[mac] + nic_metadata = macs_metadata.get(mac) + if not nic_metadata: + continue # Not a physical nic represented in metadata + nic_idx = nic_order[mac] + is_primary_nic = nic_idx == 0 + # nic_idx + 1 to start route_metric at 100 (nic_idx is 0-indexed) + dhcp_override = {"route-metric": (nic_idx + 1) * 100} + dev_config = { + "dhcp4": True, + "dhcp4-overrides": dhcp_override, + "dhcp6": False, + "match": {"macaddress": mac.lower()}, + "set-name": nic_name, + } + # This config only works on systems using Netplan because Networking + # config V2 does not support `routing-policy`, but this config is + # passed through on systems using Netplan. + # See: https://github.com/canonical/cloud-init/issues/4862 + # + # If device-number is not present (AliYun or other ec2-like platforms), + # do not configure source-routing as we cannot determine which is the + # primary NIC. + table = 100 + nic_idx + if ( + is_netplan + and nic_metadata.get("device-number") + and not is_primary_nic + ): + dhcp_override["use-routes"] = True + _configure_policy_routing( + dev_config, + distro=distro, + nic_name=nic_name, + nic_metadata=nic_metadata, + is_ipv4=True, + table=table, + ) + if nic_metadata.get("ipv6s"): # Any IPv6 addresses configured + dev_config["dhcp6"] = True + dev_config["dhcp6-overrides"] = dhcp_override + if ( + is_netplan + and nic_metadata.get("device-number") + and not is_primary_nic + ): + _configure_policy_routing( + dev_config, + distro=distro, + nic_name=nic_name, + nic_metadata=nic_metadata, + is_ipv4=False, + table=table, + ) + dev_config["addresses"] = get_secondary_addresses(nic_metadata, mac) + if not dev_config["addresses"]: + dev_config.pop("addresses") # Since we found none configured + + netcfg["ethernets"][nic_name] = dev_config + # Remove route-metric dhcp overrides and routes / routing-policy if only + # one nic configured + if len(netcfg["ethernets"]) == 1: + for nic_name in netcfg["ethernets"].keys(): + netcfg["ethernets"][nic_name].pop("dhcp4-overrides") + netcfg["ethernets"][nic_name].pop("dhcp6-overrides", None) + netcfg["ethernets"][nic_name].pop("routes", None) + netcfg["ethernets"][nic_name].pop("routing-policy", None) + return netcfg + + +def get_secondary_addresses(nic_metadata, mac): + """Parse interface-specific nic metadata and return any secondary IPs + + :return: List of secondary IPv4 or IPv6 addresses to configure on the + interface + """ + ipv4s = nic_metadata.get("local-ipv4s") + ipv6s = nic_metadata.get("ipv6s") + addresses = [] + # In version < 2018-09-24 local_ipv4s or ipv6s is a str with one IP + if bool(isinstance(ipv4s, list) and len(ipv4s) > 1): + addresses.extend( + _get_secondary_addresses( + nic_metadata, "subnet-ipv4-cidr-block", mac, ipv4s, "24" + ) + ) + if bool(isinstance(ipv6s, list) and len(ipv6s) > 1): + addresses.extend( + _get_secondary_addresses( + nic_metadata, "subnet-ipv6-cidr-block", mac, ipv6s, "128" + ) + ) + return sorted(addresses) + + +def _get_secondary_addresses(nic_metadata, cidr_key, mac, ips, default_prefix): + """Return list of IP addresses as CIDRs for secondary IPs + + The CIDR prefix will be default_prefix if cidr_key is absent or not + parseable in nic_metadata. + """ + addresses = [] + cidr = nic_metadata.get(cidr_key) + prefix = default_prefix + if not cidr or len(cidr.split("/")) != 2: + ip_type = "ipv4" if "ipv4" in cidr_key else "ipv6" + LOG.warning( + "Could not parse %s %s for mac %s. %s network" + " config prefix defaults to /%s", + cidr_key, + cidr, + mac, + ip_type, + prefix, + ) + else: + prefix = cidr.split("/")[1] + # We know we have > 1 ips for in metadata for this IP type + for ip in ips[1:]: + addresses.append("{ip}/{prefix}".format(ip=ip, prefix=prefix)) + return addresses + + +# Used to match classes to dependencies +datasources = [ + (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)), # Run at init-local + (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/.pc/retain-netplan-world-readable.patch/cloudinit/features.py b/.pc/retain-netplan-world-readable.patch/cloudinit/features.py index cf6f5b27a..259b9f772 100644 --- a/.pc/retain-netplan-world-readable.patch/cloudinit/features.py +++ b/.pc/retain-netplan-world-readable.patch/cloudinit/features.py @@ -62,7 +62,7 @@ NETPLAN_CONFIG_ROOT_READ_ONLY = True """ If ``NETPLAN_CONFIG_ROOT_READ_ONLY`` is True, then netplan configuration will -be written as a single root readon-only file /etc/netplan/50-cloud-init.yaml. +be written as a single root read-only file /etc/netplan/50-cloud-init.yaml. This prevents wifi passwords in network v2 configuration from being world-readable. Prior to 23.1, netplan configuration is world-readable. diff --git a/.pc/retain-netplan-world-readable.patch/tests/unittests/distros/test_netconfig.py b/.pc/retain-netplan-world-readable.patch/tests/unittests/distros/test_netconfig.py index 7ba430f29..48690d712 100644 --- a/.pc/retain-netplan-world-readable.patch/tests/unittests/distros/test_netconfig.py +++ b/.pc/retain-netplan-world-readable.patch/tests/unittests/distros/test_netconfig.py @@ -1012,43 +1012,6 @@ def netctl_path(self, iface): def netplan_path(self): return "/etc/netplan/50-cloud-init.yaml" - def test_apply_network_config_v1_without_netplan(self): - # Note that this is in fact an invalid netctl config: - # "Address=None/None" - # But this is what the renderer has been writing out for a long time, - # and the test's purpose is to assert that the netctl renderer is - # still being used in absence of netplan, not the correctness of the - # rendered netctl config. - expected_cfgs = { - self.netctl_path("eth0"): dedent( - """\ - Address=192.168.1.5/255.255.255.0 - Connection=ethernet - DNS=() - Gateway=192.168.1.254 - IP=static - Interface=eth0 - """ - ), - self.netctl_path("eth1"): dedent( - """\ - Address=None/None - Connection=ethernet - DNS=() - Gateway= - IP=dhcp - Interface=eth1 - """ - ), - } - - self._apply_and_verify( - self.distro.apply_network_config, - V1_NET_CFG, - expected_cfgs=expected_cfgs.copy(), - with_netplan=False, - ) - def test_apply_network_config_v1_with_netplan(self): expected_cfgs = { self.netplan_path(): dedent( diff --git a/.pc/retain-old-groups.patch/config/cloud.cfg.tmpl b/.pc/retain-old-groups.patch/config/cloud.cfg.tmpl index de0bf7bb4..e21770326 100644 --- a/.pc/retain-old-groups.patch/config/cloud.cfg.tmpl +++ b/.pc/retain-old-groups.patch/config/cloud.cfg.tmpl @@ -111,7 +111,6 @@ disable_vmware_customization: false # The modules that run in the 'init' stage cloud_init_modules: - - migrator {% if variant not in ["netbsd"] %} - seed_random {% endif %} @@ -164,7 +163,7 @@ cloud_config_modules: - apt_pipelining - apt_configure {% if variant == "ubuntu" %} - - ubuntu_advantage + - ubuntu_pro {% endif %} {% elif variant in ["fedora", "mariner", "openeuler", "openmandriva", "photon"] or is_rhel %} @@ -204,7 +203,6 @@ cloud_final_modules: - mcollective - salt_minion - reset_rmc - - rightscale_userdata - scripts_vendor - scripts_per_once - scripts_per_boot @@ -303,6 +301,7 @@ system_info: {% elif variant in ["ubuntu", "unknown"] %} {# SRU_BLOCKER: do not ship network renderers on Xenial, Bionic or Eoan #} network: + dhcp_client_priority: [dhcpcd, dhclient, udhcpc] renderers: ['netplan', 'eni', 'sysconfig'] activators: ['netplan', 'eni', 'network-manager', 'networkd'] {% elif is_rhel %} @@ -327,6 +326,9 @@ system_info: paths: cloud_dir: /var/lib/cloud/ templates_dir: /etc/cloud/templates/ +{% elif is_bsd %} + paths: + run_dir: /var/run/ {% endif %} {% if variant == "debian" %} package_mirrors: diff --git a/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/cloud.py b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/cloud.py new file mode 100644 index 000000000..ae079d485 --- /dev/null +++ b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/cloud.py @@ -0,0 +1,116 @@ +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# This file is part of cloud-init. See LICENSE file for license information. + +import copy +import logging +import os +from typing import Optional + +from cloudinit.distros import Distro +from cloudinit.helpers import Paths, Runners +from cloudinit.reporting import events +from cloudinit.sources import DataSource + +LOG = logging.getLogger(__name__) + +# This class is the high level wrapper that provides +# access to cloud-init objects without exposing the stage objects +# to handler and or module manipulation. It allows for cloud +# init to restrict what those types of user facing code may see +# and or adjust (which helps avoid code messing with each other) +# +# It also provides util functions that avoid having to know +# how to get a certain member from this submembers as well +# as providing a backwards compatible object that can be maintained +# while the stages/other objects can be worked on independently... + + +class Cloud: + def __init__( + self, + datasource: DataSource, + paths: Paths, + cfg: dict, + distro: Distro, + runners: Runners, + reporter: Optional[events.ReportEventStack] = None, + ): + self.datasource = datasource + self.paths = paths + self.distro = distro + self._cfg = cfg + self._runners = runners + if reporter is None: + reporter = events.ReportEventStack( + name="unnamed-cloud-reporter", + description="unnamed-cloud-reporter", + reporting_enabled=False, + ) + self.reporter = reporter + + @property + def cfg(self): + # Ensure that cfg is not indirectly modified + return copy.deepcopy(self._cfg) + + def run(self, name, functor, args, freq=None, clear_on_fail=False): + """Run a function gated by a named semaphore for a desired frequency. + + The typical case for this method would be to limit running of the + provided func to a single well-defined frequency: + PER_INSTANCE, PER_BOOT or PER_ONCE + + The semaphore provides a gate that persists across cloud-init + boot stage boundaries so multiple modules can share this state + even if they happen to be run in different boot stages or across + reboots. + """ + return self._runners.run(name, functor, args, freq, clear_on_fail) + + def get_template_filename(self, name): + fn = self.paths.template_tpl % (name) + if not os.path.isfile(fn): + LOG.warning( + "No template found in %s for template named %s", + os.path.dirname(fn), + name, + ) + return None + return fn + + # The rest of these are just useful proxies + def get_userdata(self, apply_filter=True): + return self.datasource.get_userdata(apply_filter) + + def get_instance_id(self): + return self.datasource.get_instance_id() + + @property + def launch_index(self): + return self.datasource.launch_index + + def get_public_ssh_keys(self): + return self.datasource.get_public_ssh_keys() + + def get_locale(self): + return self.datasource.get_locale() + + def get_hostname(self, fqdn=False, metadata_only=False): + return self.datasource.get_hostname( + fqdn=fqdn, metadata_only=metadata_only + ) + + def device_name_to_device(self, name): + return self.datasource.device_name_to_device(name) + + def get_ipath_cur(self, name=None): + return self.paths.get_ipath_cur(name) + + def get_cpath(self, name=None): + return self.paths.get_cpath(name) + + def get_ipath(self, name=None): + return self.paths.get_ipath(name) diff --git a/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_lxd.py b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_lxd.py new file mode 100644 index 000000000..2fdb8e55f --- /dev/null +++ b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_lxd.py @@ -0,0 +1,531 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Wesley Wiedenmeier +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""LXD: configure lxd with ``lxd init`` and optionally lxd-bridge""" + +import logging +import os +from textwrap import dedent +from typing import List, Tuple + +from cloudinit import safeyaml, subp, util +from cloudinit.cloud import Cloud +from cloudinit.config import Config +from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + +_DEFAULT_NETWORK_NAME = "lxdbr0" + + +MODULE_DESCRIPTION = """\ +This module configures lxd with user specified options using ``lxd init``. +If lxd is not present on the system but lxd configuration is provided, then +lxd will be installed. If the selected storage backend userspace utility is +not installed, it will be installed. If network bridge configuration is +provided, then lxd-bridge will be configured accordingly. +""" + +distros = ["ubuntu"] + +meta: MetaSchema = { + "id": "cc_lxd", + "name": "LXD", + "title": "Configure LXD with ``lxd init`` and optionally lxd-bridge", + "description": MODULE_DESCRIPTION, + "distros": distros, + "examples": [ + dedent( + """\ + # Simplest working directory backed LXD configuration + lxd: + init: + storage_backend: dir + """ + ), + dedent( + """\ + # LXD init showcasing cloud-init's LXD config options + lxd: + init: + network_address: 0.0.0.0 + network_port: 8443 + storage_backend: zfs + storage_pool: datapool + storage_create_loop: 10 + bridge: + mode: new + mtu: 1500 + name: lxdbr0 + ipv4_address: 10.0.8.1 + ipv4_netmask: 24 + ipv4_dhcp_first: 10.0.8.2 + ipv4_dhcp_last: 10.0.8.3 + ipv4_dhcp_leases: 250 + ipv4_nat: true + ipv6_address: fd98:9e0:3744::1 + ipv6_netmask: 64 + ipv6_nat: true + domain: lxd + """ + ), + dedent( + """\ + # For more complex non-iteractive LXD configuration of networks, + # storage_pools, profiles, projects, clusters and core config, + # `lxd:preseed` config will be passed as stdin to the command: + # lxd init --preseed + # See https://documentation.ubuntu.com/lxd/en/latest/howto/initialize/#non-interactive-configuration or + # run: lxd init --dump to see viable preseed YAML allowed. + # + # Preseed settings configuring the LXD daemon for HTTPS connections + # on 192.168.1.1 port 9999, a nested profile which allows for + # LXD nesting on containers and a limited project allowing for + # RBAC approach when defining behavior for sub projects. + lxd: + preseed: | + config: + core.https_address: 192.168.1.1:9999 + networks: + - config: + ipv4.address: 10.42.42.1/24 + ipv4.nat: true + ipv6.address: fd42:4242:4242:4242::1/64 + ipv6.nat: true + description: "" + name: lxdbr0 + type: bridge + project: default + storage_pools: + - config: + size: 5GiB + source: /var/snap/lxd/common/lxd/disks/default.img + description: "" + name: default + driver: zfs + profiles: + - config: {} + description: Default LXD profile + devices: + eth0: + name: eth0 + network: lxdbr0 + type: nic + root: + path: / + pool: default + type: disk + name: default + - config: {} + security.nesting: true + devices: + eth0: + name: eth0 + network: lxdbr0 + type: nic + root: + path: / + pool: default + type: disk + name: nested + projects: + - config: + features.images: true + features.networks: true + features.profiles: true + features.storage.volumes: true + description: Default LXD project + name: default + - config: + features.images: false + features.networks: true + features.profiles: false + features.storage.volumes: false + description: Limited Access LXD project + name: limited + + + """ # noqa: E501 + ), + ], + "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["lxd"], +} + +__doc__ = get_meta_doc(meta) + + +def supplemental_schema_validation( + init_cfg: dict, bridge_cfg: dict, preseed_str: str +): + """Validate user-provided lxd network and bridge config option values. + + @raises: ValueError describing invalid values provided. + """ + errors = [] + if not isinstance(init_cfg, dict): + errors.append( + f"lxd.init config must be a dictionary. found a" + f" '{type(init_cfg).__name__}'", + ) + + if not isinstance(bridge_cfg, dict): + errors.append( + f"lxd.bridge config must be a dictionary. found a" + f" '{type(bridge_cfg).__name__}'", + ) + + if not isinstance(preseed_str, str): + errors.append( + f"lxd.preseed config must be a string. found a" + f" '{type(preseed_str).__name__}'", + ) + if preseed_str and (init_cfg or bridge_cfg): + incompat_cfg = ["lxd.init"] if init_cfg else [] + incompat_cfg += ["lxd.bridge"] if bridge_cfg else [] + + errors.append( + "Unable to configure LXD. lxd.preseed config can not be provided" + f" with key(s): {', '.join(incompat_cfg)}" + ) + if errors: + raise ValueError(". ".join(errors)) + + +def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: + # Get config + lxd_cfg = cfg.get("lxd") + if not lxd_cfg: + LOG.debug( + "Skipping module named %s, not present or disabled by cfg", name + ) + return + if not isinstance(lxd_cfg, dict): + raise ValueError( + f"lxd config must be a dictionary. found a" + f" '{type(lxd_cfg).__name__}'" + ) + + util.wait_for_snap_seeded(cloud) + # Grab the configuration + init_cfg = lxd_cfg.get("init", {}) + preseed_str = lxd_cfg.get("preseed", "") + bridge_cfg = lxd_cfg.get("bridge", {}) + supplemental_schema_validation(init_cfg, bridge_cfg, preseed_str) + + if not subp.which("lxd"): + try: + subp.subp(["snap", "install", "lxd"]) + except subp.ProcessExecutionError as e: + raise RuntimeError( + "Failed to install lxd from snap: %s" % e + ) from e + packages = get_required_packages(init_cfg, preseed_str) + if len(packages): + try: + cloud.distro.install_packages(packages) + except subp.ProcessExecutionError as exc: + LOG.warning("failed to install packages %s: %s", packages, exc) + return + + subp.subp(["lxd", "waitready", "--timeout=300"]) + if preseed_str: + subp.subp(["lxd", "init", "--preseed"], data=preseed_str) + return + # Set up lxd if init config is given + if init_cfg: + # type is known, number of elements is not + # in the case of the ubuntu+lvm backend workaround + init_keys: Tuple[str, ...] = ( + "network_address", + "network_port", + "storage_backend", + "storage_create_device", + "storage_create_loop", + "storage_pool", + "trust_password", + ) + + # Bug https://bugs.launchpad.net/ubuntu/+source/linux-kvm/+bug/1982780 + kernel = util.system_info()["uname"][2] + if init_cfg["storage_backend"] == "lvm" and not os.path.exists( + f"/lib/modules/{kernel}/kernel/drivers/md/dm-thin-pool.ko" + ): + LOG.warning( + "cloud-init doesn't use thinpool by default on Ubuntu due to " + "LP #1982780. This behavior will change in the future.", + ) + subp.subp( + [ + "lxc", + "storage", + "create", + "default", + "lvm", + "lvm.use_thinpool=false", + ] + ) + + # Since we're manually setting use_thinpool=false + # filter it from the lxd init commands, don't configure + # storage twice + init_keys = tuple( + key for key in init_keys if key != "storage_backend" + ) + + cmd = ["lxd", "init", "--auto"] + for k in init_keys: + if init_cfg.get(k): + cmd.extend( + ["--%s=%s" % (k.replace("_", "-"), str(init_cfg[k]))] + ) + subp.subp(cmd) + + # Set up lxd-bridge if bridge config is given + dconf_comm = "debconf-communicate" + if bridge_cfg: + net_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME) + if os.path.exists("/etc/default/lxd-bridge") and subp.which( + dconf_comm + ): + # Bridge configured through packaging + + debconf = bridge_to_debconf(bridge_cfg) + + # Update debconf database + try: + LOG.debug("Setting lxd debconf via %s", dconf_comm) + data = ( + "\n".join( + ["set %s %s" % (k, v) for k, v in debconf.items()] + ) + + "\n" + ) + subp.subp(["debconf-communicate"], data=data) + except Exception: + util.logexc( + LOG, "Failed to run '%s' for lxd with" % dconf_comm + ) + + # Remove the existing configuration file (forces re-generation) + util.del_file("/etc/default/lxd-bridge") + + # Run reconfigure + LOG.debug("Running dpkg-reconfigure for lxd") + subp.subp(["dpkg-reconfigure", "lxd", "--frontend=noninteractive"]) + else: + # Built-in LXD bridge support + cmd_create, cmd_attach = bridge_to_cmd(bridge_cfg) + maybe_cleanup_default( + net_name=net_name, + did_init=bool(init_cfg), + create=bool(cmd_create), + attach=bool(cmd_attach), + ) + if cmd_create: + LOG.debug("Creating lxd bridge: %s", " ".join(cmd_create)) + _lxc(cmd_create) + + if cmd_attach: + LOG.debug( + "Setting up default lxd bridge: %s", " ".join(cmd_attach) + ) + _lxc(cmd_attach) + + elif bridge_cfg: + raise RuntimeError( + "Unable to configure lxd bridge without %s." + dconf_comm + ) + + +def bridge_to_debconf(bridge_cfg): + debconf = {} + + if bridge_cfg.get("mode") == "none": + debconf["lxd/setup-bridge"] = "false" + debconf["lxd/bridge-name"] = "" + + elif bridge_cfg.get("mode") == "existing": + debconf["lxd/setup-bridge"] = "false" + debconf["lxd/use-existing-bridge"] = "true" + debconf["lxd/bridge-name"] = bridge_cfg.get("name") + + elif bridge_cfg.get("mode") == "new": + debconf["lxd/setup-bridge"] = "true" + if bridge_cfg.get("name"): + debconf["lxd/bridge-name"] = bridge_cfg.get("name") + + if bridge_cfg.get("ipv4_address"): + debconf["lxd/bridge-ipv4"] = "true" + debconf["lxd/bridge-ipv4-address"] = bridge_cfg.get("ipv4_address") + debconf["lxd/bridge-ipv4-netmask"] = bridge_cfg.get("ipv4_netmask") + debconf["lxd/bridge-ipv4-dhcp-first"] = bridge_cfg.get( + "ipv4_dhcp_first" + ) + debconf["lxd/bridge-ipv4-dhcp-last"] = bridge_cfg.get( + "ipv4_dhcp_last" + ) + debconf["lxd/bridge-ipv4-dhcp-leases"] = bridge_cfg.get( + "ipv4_dhcp_leases" + ) + debconf["lxd/bridge-ipv4-nat"] = bridge_cfg.get("ipv4_nat", "true") + + if bridge_cfg.get("ipv6_address"): + debconf["lxd/bridge-ipv6"] = "true" + debconf["lxd/bridge-ipv6-address"] = bridge_cfg.get("ipv6_address") + debconf["lxd/bridge-ipv6-netmask"] = bridge_cfg.get("ipv6_netmask") + debconf["lxd/bridge-ipv6-nat"] = bridge_cfg.get( + "ipv6_nat", "false" + ) + + if bridge_cfg.get("domain"): + debconf["lxd/bridge-domain"] = bridge_cfg.get("domain") + + else: + raise RuntimeError('invalid bridge mode "%s"' % bridge_cfg.get("mode")) + + return debconf + + +def bridge_to_cmd(bridge_cfg): + if bridge_cfg.get("mode") == "none": + return None, None + + bridge_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME) + cmd_create = [] + cmd_attach = ["network", "attach-profile", bridge_name, "default", "eth0"] + + if bridge_cfg.get("mode") == "existing": + return None, cmd_attach + + if bridge_cfg.get("mode") != "new": + raise RuntimeError('invalid bridge mode "%s"' % bridge_cfg.get("mode")) + + cmd_create = ["network", "create", bridge_name] + + if bridge_cfg.get("ipv4_address") and bridge_cfg.get("ipv4_netmask"): + cmd_create.append( + "ipv4.address=%s/%s" + % (bridge_cfg.get("ipv4_address"), bridge_cfg.get("ipv4_netmask")) + ) + + if bridge_cfg.get("ipv4_nat", "true") == "true": + cmd_create.append("ipv4.nat=true") + + if bridge_cfg.get("ipv4_dhcp_first") and bridge_cfg.get( + "ipv4_dhcp_last" + ): + dhcp_range = "%s-%s" % ( + bridge_cfg.get("ipv4_dhcp_first"), + bridge_cfg.get("ipv4_dhcp_last"), + ) + cmd_create.append("ipv4.dhcp.ranges=%s" % dhcp_range) + else: + cmd_create.append("ipv4.address=none") + + if bridge_cfg.get("ipv6_address") and bridge_cfg.get("ipv6_netmask"): + cmd_create.append( + "ipv6.address=%s/%s" + % (bridge_cfg.get("ipv6_address"), bridge_cfg.get("ipv6_netmask")) + ) + + if bridge_cfg.get("ipv6_nat") == "true": + cmd_create.append("ipv6.nat=true") + + else: + cmd_create.append("ipv6.address=none") + + if bridge_cfg.get("domain"): + cmd_create.append("dns.domain=%s" % bridge_cfg.get("domain")) + + # if the default schema value is passed (-1) don't pass arguments + # to LXD. Use LXD defaults unless user manually sets a number + mtu = bridge_cfg.get("mtu", -1) + if mtu != -1: + cmd_create.append(f"bridge.mtu={mtu}") + + return cmd_create, cmd_attach + + +def _lxc(cmd): + env = { + "LC_ALL": "C", + "HOME": os.environ.get("HOME", "/root"), + "USER": os.environ.get("USER", "root"), + } + subp.subp(["lxc"] + list(cmd) + ["--force-local"], update_env=env) + + +def maybe_cleanup_default( + net_name, did_init, create, attach, profile="default", nic_name="eth0" +): + """Newer versions of lxc (3.0.1+) create a lxdbr0 network when + 'lxd init --auto' is run. Older versions did not. + + By removing any that lxd-init created, we simply leave the add/attach + code intact. + + https://github.com/canonical/lxd/issues/4649""" + if net_name != _DEFAULT_NETWORK_NAME or not did_init: + return + + fail_assume_enoent = "failed. Assuming it did not exist." + succeeded = "succeeded." + if create: + msg = "Detach of lxd network '%s' from profile '%s' %s" + try: + _lxc(["network", "detach-profile", net_name, profile]) + LOG.debug(msg, net_name, profile, succeeded) + except subp.ProcessExecutionError as e: + if e.exit_code != 1: + raise e + LOG.debug(msg, net_name, profile, fail_assume_enoent) + else: + msg = "Deletion of lxd network '%s' %s" + _lxc(["network", "delete", net_name]) + LOG.debug(msg, net_name, succeeded) + + if attach: + msg = "Removal of device '%s' from profile '%s' %s" + try: + _lxc(["profile", "device", "remove", profile, nic_name]) + LOG.debug(msg, nic_name, profile, succeeded) + except subp.ProcessExecutionError as e: + if e.exit_code != 1: + raise e + LOG.debug(msg, nic_name, profile, fail_assume_enoent) + + +def get_required_packages(init_cfg: dict, preseed_str: str) -> List[str]: + """identify required packages for install""" + packages = [] + # binary for pool creation must be available for the requested backend: + # zfs, lvcreate, mkfs.btrfs + storage_drivers: List[str] = [] + preseed_cfg: dict = {} + if "storage_backend" in init_cfg: + storage_drivers.append(init_cfg["storage_backend"]) + if preseed_str and "storage_pools" in preseed_str: + # Assume correct YAML preseed format + try: + preseed_cfg = safeyaml.load(preseed_str) + except (safeyaml.YAMLError, TypeError, ValueError): + LOG.warning( + "lxd.preseed string value is not YAML. " + " Unable to determine required storage driver packages to" + " support storage_pools config." + ) + for storage_pool in preseed_cfg.get("storage_pools", []): + if storage_pool.get("driver"): + storage_drivers.append(storage_pool["driver"]) + if "zfs" in storage_drivers and not subp.which("zfs"): + packages.append("zfsutils-linux") + if "lvm" in storage_drivers and not subp.which("lvcreate"): + packages.append("lvm2") + if "btrfs" in storage_drivers and not subp.which("mkfs.btrfs"): + packages.append("btrfs-progs") + return packages diff --git a/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_snap.py b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_snap.py new file mode 100644 index 000000000..b798b09d7 --- /dev/null +++ b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_snap.py @@ -0,0 +1,199 @@ +# Copyright (C) 2018 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Snap: Install, configure and manage snapd and snap packages.""" + +import logging +import os +from textwrap import dedent + +from cloudinit import subp, util +from cloudinit.cloud import Cloud +from cloudinit.config import Config +from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.settings import PER_INSTANCE +from cloudinit.subp import prepend_base_command + +distros = ["ubuntu"] +frequency = PER_INSTANCE + +LOG = logging.getLogger(__name__) + +meta: MetaSchema = { + "id": "cc_snap", + "name": "Snap", + "title": "Install, configure and manage snapd and snap packages", + "description": dedent( + """\ + This module provides a simple configuration namespace in cloud-init to + both setup snapd and install snaps. + + .. note:: + Both ``assertions`` and ``commands`` values can be either a + dictionary or a list. If these configs are provided as a + dictionary, the keys are only used to order the execution of the + assertions or commands and the dictionary is merged with any + vendor-data snap configuration provided. If a list is provided by + the user instead of a dict, any vendor-data snap configuration is + ignored. + + The ``assertions`` configuration option is a dictionary or list of + properly-signed snap assertions which will run before any snap + ``commands``. They will be added to snapd's assertion database by + invoking ``snap ack ``. + + Snap ``commands`` is a dictionary or list of individual snap + commands to run on the target system. These commands can be used to + create snap users, install snaps and provide snap configuration. + + .. note:: + If 'side-loading' private/unpublished snaps on an instance, it is + best to create a snap seed directory and seed.yaml manifest in + **/var/lib/snapd/seed/** which snapd automatically installs on + startup. + """ + ), + "distros": distros, + "examples": [ + dedent( + """\ + snap: + assertions: + 00: | + signed_assertion_blob_here + 02: | + signed_assertion_blob_here + commands: + 00: snap create-user --sudoer --known @mydomain.com + 01: snap install canonical-livepatch + 02: canonical-livepatch enable + """ + ), + dedent( + """\ + # Convenience: the snap command can be omitted when specifying commands + # as a list and 'snap' will automatically be prepended. + # The following commands are equivalent: + snap: + commands: + 00: ['install', 'vlc'] + 01: ['snap', 'install', 'vlc'] + 02: snap install vlc + 03: 'snap install vlc' + """ + ), + dedent( + """\ + # You can use a list of commands + snap: + commands: + - ['install', 'vlc'] + - ['snap', 'install', 'vlc'] + - snap install vlc + - 'snap install vlc' + """ + ), + dedent( + """\ + # You can use a list of assertions + snap: + assertions: + - signed_assertion_blob_here + - | + signed_assertion_blob_here + """ + ), + ], + "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["snap"], +} + + +__doc__ = get_meta_doc(meta) + +SNAP_CMD = "snap" + + +def add_assertions(assertions, assertions_file): + """Import list of assertions. + + Import assertions by concatenating each assertion into a + string separated by a '\n'. Write this string to a instance file and + then invoke `snap ack /path/to/file` and check for errors. + If snap exits 0, then all assertions are imported. + """ + if not assertions: + return + LOG.debug("Importing user-provided snap assertions") + if isinstance(assertions, dict): + assertions = assertions.values() + elif not isinstance(assertions, list): + raise TypeError( + "assertion parameter was not a list or dict: {assertions}".format( + assertions=assertions + ) + ) + + snap_cmd = [SNAP_CMD, "ack", assertions_file] + combined = "\n".join(assertions) + + for asrt in assertions: + LOG.debug("Snap acking: %s", asrt.split("\n")[0:2]) + + util.write_file(assertions_file, combined.encode("utf-8")) + subp.subp(snap_cmd, capture=True) + + +def run_commands(commands): + """Run the provided commands provided in snap:commands configuration. + + Commands are run individually. Any errors are collected and reported + after attempting all commands. + + @param commands: A list or dict containing commands to run. Keys of a + dict will be used to order the commands provided as dict values. + """ + if not commands: + return + LOG.debug("Running user-provided snap commands") + if isinstance(commands, dict): + # Sort commands based on dictionary key + commands = [v for _, v in sorted(commands.items())] + elif not isinstance(commands, list): + raise TypeError( + "commands parameter was not a list or dict: {commands}".format( + commands=commands + ) + ) + + fixed_snap_commands = prepend_base_command("snap", commands) + + cmd_failures = [] + for command in fixed_snap_commands: + shell = isinstance(command, str) + try: + subp.subp(command, shell=shell) + except subp.ProcessExecutionError as e: + cmd_failures.append(str(e)) + if cmd_failures: + msg = "Failures running snap commands:\n{cmd_failures}".format( + cmd_failures=cmd_failures + ) + util.logexc(LOG, msg) + raise RuntimeError(msg) + + +def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: + cfgin = cfg.get("snap", {}) + if not cfgin: + LOG.debug( + "Skipping module named %s, no 'snap' key in configuration", name + ) + return + util.wait_for_snap_seeded(cloud) + add_assertions( + cfgin.get("assertions", []), + os.path.join(cloud.paths.get_ipath_cur(), "snapd.assertions"), + ) + run_commands(cfgin.get("commands", [])) diff --git a/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_ubuntu_autoinstall.py b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_ubuntu_autoinstall.py new file mode 100644 index 000000000..c75f7a979 --- /dev/null +++ b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/config/cc_ubuntu_autoinstall.py @@ -0,0 +1,147 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Autoinstall: Support ubuntu live-server autoinstall syntax.""" + +import logging +import re +from textwrap import dedent + +from cloudinit import util +from cloudinit.cloud import Cloud +from cloudinit.config import Config +from cloudinit.config.schema import ( + MetaSchema, + SchemaProblem, + SchemaValidationError, + get_meta_doc, +) +from cloudinit.settings import PER_ONCE +from cloudinit.subp import subp + +LOG = logging.getLogger(__name__) + +distros = ["ubuntu"] + +meta: MetaSchema = { + "id": "cc_ubuntu_autoinstall", + "name": "Ubuntu Autoinstall", + "title": "Support Ubuntu live-server install syntax", + "description": dedent( + """\ + Ubuntu's autoinstall YAML supports single-system automated installs + in either the live-server install, via the ``subiquity`` snap, or the + next generation desktop installer, via `ubuntu-desktop-install` snap. + When "autoinstall" directives are provided in either + ``#cloud-config`` user-data or ``/etc/cloud/cloud.cfg.d`` validate + minimal autoinstall schema adherence and emit a warning if the + live-installer is not present. + + The live-installer will use autoinstall directives to seed answers to + configuration prompts during system install to allow for a + "touchless" or non-interactive Ubuntu system install. + + For more details on Ubuntu's autoinstaller: + https://ubuntu.com/server/docs/install/autoinstall + """ + ), + "distros": distros, + "examples": [ + dedent( + """\ + # Tell the live-server installer to provide dhcp6 network config + # and LVM on a disk matching the serial number prefix CT + autoinstall: + version: 1 + network: + version: 2 + ethernets: + enp0s31f6: + dhcp6: yes + storage: + layout: + name: lvm + match: + serial: CT* + """ + ) + ], + "frequency": PER_ONCE, + "activate_by_schema_keys": ["autoinstall"], +} + +__doc__ = get_meta_doc(meta) + + +LIVE_INSTALLER_SNAPS = ("subiquity", "ubuntu-desktop-installer") + + +def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: + + if "autoinstall" not in cfg: + LOG.debug( + "Skipping module named %s, no 'autoinstall' key in configuration", + name, + ) + return + + util.wait_for_snap_seeded(cloud) + snap_list, _ = subp(["snap", "list"]) + installer_present = None + for snap_name in LIVE_INSTALLER_SNAPS: + if re.search(snap_name, snap_list): + installer_present = snap_name + if not installer_present: + LOG.warning( + "Skipping autoinstall module. Expected one of the Ubuntu" + " installer snap packages to be present: %s", + ", ".join(LIVE_INSTALLER_SNAPS), + ) + return + validate_config_schema(cfg) + LOG.debug( + "Valid autoinstall schema. Config will be processed by %s", + installer_present, + ) + + +def validate_config_schema(cfg): + """Supplemental runtime schema validation for autoinstall yaml. + + Schema validation issues currently result in a warning log currently which + can be easily ignored because warnings do not bubble up to cloud-init + status output. + + In the case of the live-installer, we want cloud-init to raise an error + to set overall cloud-init status to 'error' so it is more discoverable + in installer environments. + + # TODO(Drop this validation When cloud-init schema is strict and errors) + + :raise: SchemaValidationError if any known schema values are present. + """ + autoinstall_cfg = cfg["autoinstall"] + if not isinstance(autoinstall_cfg, dict): + raise SchemaValidationError( + [ + SchemaProblem( + "autoinstall", + "Expected dict type but found:" + f" {type(autoinstall_cfg).__name__}", + ) + ] + ) + + if "version" not in autoinstall_cfg: + raise SchemaValidationError( + [SchemaProblem("autoinstall", "Missing required 'version' key")] + ) + elif not isinstance(autoinstall_cfg.get("version"), int): + raise SchemaValidationError( + [ + SchemaProblem( + "autoinstall.version", + f"Expected int type but found:" + f" {type(autoinstall_cfg['version']).__name__}", + ) + ] + ) diff --git a/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/util.py b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/util.py new file mode 100644 index 000000000..c9b7287ae --- /dev/null +++ b/.pc/revert-551f560d-cloud-config-after-snap-seeding.patch/cloudinit/util.py @@ -0,0 +1,3306 @@ +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This file is part of cloud-init. See LICENSE file for license information. + +import binascii +import contextlib +import copy as obj_copy +import email +import functools +import glob +import grp +import gzip +import hashlib +import io +import json +import logging +import os +import os.path +import platform +import pwd +import random +import re +import shlex +import shutil +import socket +import stat +import string +import subprocess +import sys +import time +from base64 import b64decode +from collections import deque, namedtuple +from contextlib import contextmanager, suppress +from errno import ENOENT +from functools import lru_cache, total_ordering +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Deque, + Dict, + Generator, + List, + Mapping, + Optional, + Sequence, + TypeVar, + Union, +) +from urllib import parse + +from cloudinit import ( + features, + importer, + mergers, + net, + safeyaml, + settings, + subp, + temp_utils, + type_utils, + url_helper, + version, +) +from cloudinit.settings import CFG_BUILTIN, PER_ONCE + +if TYPE_CHECKING: + # Avoid circular import + from cloudinit.helpers import Paths + +_DNS_REDIRECT_IP = None +LOG = logging.getLogger(__name__) + +# Helps cleanup filenames to ensure they aren't FS incompatible +FN_REPLACEMENTS = { + os.sep: "_", +} +FN_ALLOWED = "_-.()" + string.digits + string.ascii_letters + +TRUE_STRINGS = ("true", "1", "on", "yes") +FALSE_STRINGS = ("off", "0", "no", "false") + + +def kernel_version(): + return tuple(map(int, os.uname().release.split(".")[:2])) + + +@lru_cache() +def get_dpkg_architecture(): + """Return the sanitized string output by `dpkg --print-architecture`. + + N.B. This function is wrapped in functools.lru_cache, so repeated calls + won't shell out every time. + """ + out = subp.subp(["dpkg", "--print-architecture"], capture=True) + return out.stdout.strip() + + +@lru_cache() +def lsb_release(): + fmap = { + "Codename": "codename", + "Description": "description", + "Distributor ID": "id", + "Release": "release", + } + + data = {} + try: + out = subp.subp(["lsb_release", "--all"], capture=True) + for line in out.stdout.splitlines(): + fname, _, val = line.partition(":") + if fname in fmap: + data[fmap[fname]] = val.strip() + missing = [k for k in fmap.values() if k not in data] + if len(missing): + LOG.warning( + "Missing fields in lsb_release --all output: %s", + ",".join(missing), + ) + + except subp.ProcessExecutionError as err: + LOG.warning("Unable to get lsb_release --all: %s", err) + data = dict((v, "UNAVAILABLE") for v in fmap.values()) + + return data + + +def decode_binary(blob: Union[str, bytes], encoding="utf-8") -> str: + # Converts a binary type into a text type using given encoding. + return blob if isinstance(blob, str) else blob.decode(encoding=encoding) + + +def encode_text(text: Union[str, bytes], encoding="utf-8") -> bytes: + # Converts a text string into a binary type using given encoding. + return text if isinstance(text, bytes) else text.encode(encoding=encoding) + + +def maybe_b64decode(data: bytes) -> bytes: + """base64 decode data + + If data is base64 encoded bytes, return b64decode(data). + If not, return data unmodified. + + @param data: data as bytes. TypeError is raised if not bytes. + """ + if not isinstance(data, bytes): + raise TypeError("data is '%s', expected bytes" % type(data)) + try: + return b64decode(data, validate=True) + except binascii.Error: + return data + + +def fully_decoded_payload(part): + # In Python 3, decoding the payload will ironically hand us a bytes object. + # 'decode' means to decode according to Content-Transfer-Encoding, not + # according to any charset in the Content-Type. So, if we end up with + # bytes, first try to decode to str via CT charset, and failing that, try + # utf-8 using surrogate escapes. + cte_payload = part.get_payload(decode=True) + if part.get_content_maintype() == "text" and isinstance( + cte_payload, bytes + ): + charset = part.get_charset() + if charset and charset.input_codec: + encoding = charset.input_codec + else: + encoding = "utf-8" + return cte_payload.decode(encoding, "surrogateescape") + return cte_payload + + +class SeLinuxGuard: + def __init__(self, path, recursive=False): + # Late import since it might not always + # be possible to use this + try: + self.selinux = importer.import_module("selinux") + except ImportError: + self.selinux = None + self.path = path + self.recursive = recursive + + def __enter__(self): + if self.selinux and self.selinux.is_selinux_enabled(): + return True + else: + return False + + def __exit__(self, excp_type, excp_value, excp_traceback): + if not self.selinux or not self.selinux.is_selinux_enabled(): + return + if not os.path.lexists(self.path): + return + + path = os.path.realpath(self.path) + try: + stats = os.lstat(path) + self.selinux.matchpathcon(path, stats[stat.ST_MODE]) + except OSError: + return + + LOG.debug( + "Restoring selinux mode for %s (recursive=%s)", + path, + self.recursive, + ) + try: + self.selinux.restorecon(path, recursive=self.recursive) + except OSError as e: + LOG.warning( + "restorecon failed on %s,%s maybe badness? %s", + path, + self.recursive, + e, + ) + + +class MountFailedError(Exception): + pass + + +class DecompressionError(Exception): + pass + + +def fork_cb(child_cb, *args, **kwargs): + fid = os.fork() + if fid == 0: + try: + child_cb(*args, **kwargs) + os._exit(0) + except Exception: + logexc( + LOG, + "Failed forking and calling callback %s", + type_utils.obj_name(child_cb), + ) + os._exit(1) + else: + LOG.debug( + "Forked child %s who will run callback %s", + fid, + type_utils.obj_name(child_cb), + ) + + +def is_true(val, addons=None): + if isinstance(val, (bool)): + return val is True + check_set = TRUE_STRINGS + if addons: + check_set = list(check_set) + addons + if str(val).lower().strip() in check_set: + return True + return False + + +def is_false(val, addons=None): + if isinstance(val, (bool)): + return val is False + check_set = FALSE_STRINGS + if addons: + check_set = list(check_set) + addons + if str(val).lower().strip() in check_set: + return True + return False + + +def translate_bool(val, addons=None): + if not val: + # This handles empty lists and false and + # other things that python believes are false + return False + # If its already a boolean skip + if isinstance(val, (bool)): + return val + return is_true(val, addons) + + +def rand_str(strlen=32, select_from=None): + r = random.SystemRandom() + if not select_from: + select_from = string.ascii_letters + string.digits + return "".join([r.choice(select_from) for _x in range(strlen)]) + + +def rand_dict_key(dictionary, postfix=None): + if not postfix: + postfix = "" + while True: + newkey = rand_str(strlen=8) + "_" + postfix + if newkey not in dictionary: + break + return newkey + + +def read_conf(fname, *, instance_data_file=None) -> Dict: + """Read a yaml config with optional template, and convert to dict""" + # Avoid circular import + from cloudinit.handlers.jinja_template import ( + JinjaLoadError, + JinjaSyntaxParsingException, + NotJinjaError, + render_jinja_payload_from_file, + ) + + try: + config_file = load_text_file(fname) + except FileNotFoundError: + return {} + + if instance_data_file and os.path.exists(instance_data_file): + try: + config_file = render_jinja_payload_from_file( + config_file, + fname, + instance_data_file, + ) + LOG.debug( + "Applied instance data in '%s' to " + "configuration loaded from '%s'", + instance_data_file, + fname, + ) + except JinjaSyntaxParsingException as e: + LOG.warning( + "Failed to render templated yaml config file '%s'. %s", + fname, + e, + ) + except NotJinjaError: + # A log isn't appropriate here as we generally expect most + # cloud.cfgs to not be templated. The other path is logged + pass + except JinjaLoadError as e: + LOG.warning( + "Could not apply Jinja template '%s' to '%s'. " + "Exception: %s", + instance_data_file, + config_file, + repr(e), + ) + if config_file is None: + return {} + return load_yaml(config_file, default={}) # pyright: ignore + + +# Merges X lists, and then keeps the +# unique ones, but orders by sort order +# instead of by the original order +def uniq_merge_sorted(*lists): + return sorted(uniq_merge(*lists)) + + +# Merges X lists and then iterates over those +# and only keeps the unique items (order preserving) +# and returns that merged and uniqued list as the +# final result. +# +# Note: if any entry is a string it will be +# split on commas and empty entries will be +# evicted and merged in accordingly. +def uniq_merge(*lists): + combined_list = [] + for a_list in lists: + if isinstance(a_list, str): + a_list = a_list.strip().split(",") + # Kickout the empty ones + a_list = [a for a in a_list if a] + combined_list.extend(a_list) + return uniq_list(combined_list) + + +def clean_filename(fn): + for k, v in FN_REPLACEMENTS.items(): + fn = fn.replace(k, v) + removals = [] + for k in fn: + if k not in FN_ALLOWED: + removals.append(k) + for k in removals: + fn = fn.replace(k, "") + fn = fn.strip() + return fn + + +def decomp_gzip(data, quiet=True, decode=True): + try: + buf = io.BytesIO(encode_text(data)) + with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh: + # E1101 is https://github.com/PyCQA/pylint/issues/1444 + if decode: + return decode_binary(gh.read()) # pylint: disable=E1101 + else: + return gh.read() # pylint: disable=E1101 + except Exception as e: + if quiet: + return data + else: + raise DecompressionError(str(e)) from e + + +def extract_usergroup(ug_pair): + if not ug_pair: + return (None, None) + ug_parted = ug_pair.split(":", 1) + u = ug_parted[0].strip() + if len(ug_parted) == 2: + g = ug_parted[1].strip() + else: + g = None + if not u or u == "-1" or u.lower() == "none": + u = None + if not g or g == "-1" or g.lower() == "none": + g = None + return (u, g) + + +def get_modules_from_dir(root_dir: str) -> dict: + entries = dict() + for fname in glob.glob(os.path.join(root_dir, "*.py")): + if not os.path.isfile(fname): + continue + modname = os.path.basename(fname)[0:-3] + modname = modname.strip() + if modname and modname.find(".") == -1: + entries[fname] = modname + return entries + + +def write_to_console(conpath, text): + with open(conpath, "w") as wfh: + wfh.write(text) + wfh.flush() + + +def multi_log( + text, + console=True, + stderr=True, + log=None, + log_level=logging.DEBUG, + fallback_to_stdout=True, +): + if stderr: + sys.stderr.write(text) + if console: + conpath = "/dev/console" + writing_to_console_worked = False + if os.path.exists(conpath): + try: + write_to_console(conpath, text) + writing_to_console_worked = True + except OSError: + console_error = "Failed to write to /dev/console" + sys.stdout.write(f"{console_error}\n") + if log: + log.log(logging.WARNING, console_error) + + if fallback_to_stdout and not writing_to_console_worked: + # A container may lack /dev/console (arguably a container bug). + # Additionally, /dev/console may not be writable to on a VM (again + # likely a VM bug or virtualization bug). + # + # If either of these is the case, then write output to stdout. + # This will result in duplicate stderr and stdout messages if + # stderr was True. + # + # even though systemd might have set up output to go to + # /dev/console, the user may have configured elsewhere via + # cloud-config 'output'. If there is /dev/console, messages will + # still get there. + sys.stdout.write(text) + if log: + if text[-1] == "\n": + log.log(log_level, text[:-1]) + else: + log.log(log_level, text) + + +@lru_cache() +def is_Linux(): + return "Linux" in platform.system() + + +@lru_cache() +def is_BSD(): + if "BSD" in platform.system(): + return True + if platform.system() == "DragonFly": + return True + return False + + +@lru_cache() +def is_FreeBSD(): + return system_info()["variant"] == "freebsd" + + +@lru_cache() +def is_DragonFlyBSD(): + return system_info()["variant"] == "dragonfly" + + +@lru_cache() +def is_NetBSD(): + return system_info()["variant"] == "netbsd" + + +@lru_cache() +def is_OpenBSD(): + return system_info()["variant"] == "openbsd" + + +def get_cfg_option_bool(yobj, key, default=False): + if key not in yobj: + return default + return translate_bool(yobj[key]) + + +def get_cfg_option_str(yobj, key, default=None): + if key not in yobj: + return default + val = yobj[key] + if not isinstance(val, str): + val = str(val) + return val + + +def get_cfg_option_int(yobj, key, default=0): + return int(get_cfg_option_str(yobj, key, default=default)) + + +def _parse_redhat_release(release_file=None): + """Return a dictionary of distro info fields from /etc/redhat-release. + + Dict keys will align with /etc/os-release keys: + ID, VERSION_ID, VERSION_CODENAME + """ + + if not release_file: + release_file = "/etc/redhat-release" + if not os.path.exists(release_file): + return {} + redhat_release = load_text_file(release_file) + redhat_regex = ( + r"(?P.+) release (?P[\d\.]+) " + r"\((?P[^)]+)\)" + ) + + # Virtuozzo deviates here + if "Virtuozzo" in redhat_release: + redhat_regex = r"(?P.+) release (?P[\d\.]+)" + + match = re.match(redhat_regex, redhat_release) + if match: + group = match.groupdict() + + # Virtuozzo has no codename in this file + if "Virtuozzo" in group["name"]: + group["codename"] = group["name"] + + group["name"] = group["name"].lower().partition(" linux")[0] + if group["name"] == "red hat enterprise": + group["name"] = "redhat" + return { + "ID": group["name"], + "VERSION_ID": group["version"], + "VERSION_CODENAME": group["codename"], + } + return {} + + +@lru_cache() +def get_linux_distro(): + distro_name = "" + distro_version = "" + flavor = "" + os_release = {} + os_release_rhel = False + if os.path.exists("/etc/os-release"): + os_release = load_shell_content(load_text_file("/etc/os-release")) + if not os_release: + os_release_rhel = True + os_release = _parse_redhat_release() + if os_release: + distro_name = os_release.get("ID", "") + distro_version = os_release.get("VERSION_ID", "") + if "sles" in distro_name or "suse" in distro_name: + # RELEASE_BLOCKER: We will drop this sles divergent behavior in + # the future so that get_linux_distro returns a named tuple + # which will include both version codename and architecture + # on all distributions. + flavor = platform.machine() + elif distro_name == "alpine" or distro_name == "photon": + flavor = os_release.get("PRETTY_NAME", "") + elif distro_name == "virtuozzo" and not os_release_rhel: + # Only use this if the redhat file is not parsed + flavor = os_release.get("PRETTY_NAME", "") + else: + flavor = os_release.get("VERSION_CODENAME", "") + if not flavor: + match = re.match( + r"[^ ]+ \((?P[^)]+)\)", + os_release.get("VERSION", ""), + ) + if match: + flavor = match.groupdict()["codename"] + if distro_name == "rhel": + distro_name = "redhat" + elif is_BSD(): + distro_name = platform.system().lower() + distro_version = platform.release() + else: + dist = ("", "", "") + try: + # Was removed in 3.8 + dist = platform.dist() # pylint: disable=W1505,E1101 + except Exception: + pass + finally: + found = None + for entry in dist: + if entry: + found = 1 + if not found: + LOG.warning( + "Unable to determine distribution, template " + "expansion may have unexpected results" + ) + return dist + + return (distro_name, distro_version, flavor) + + +def _get_variant(info): + system = info["system"].lower() + variant = "unknown" + if system == "linux": + linux_dist = info["dist"][0].lower() + if linux_dist in ( + "almalinux", + "alpine", + "arch", + "centos", + "cloudlinux", + "debian", + "eurolinux", + "fedora", + "mariner", + "miraclelinux", + "openeuler", + "opencloudos", + "openmandriva", + "photon", + "rhel", + "rocky", + "suse", + "tencentos", + "virtuozzo", + ): + variant = linux_dist + elif linux_dist in ("ubuntu", "linuxmint", "mint"): + variant = "ubuntu" + elif linux_dist == "redhat": + variant = "rhel" + elif linux_dist in ( + "opensuse", + "opensuse-leap", + "opensuse-microos", + "opensuse-tumbleweed", + "sle_hpc", + "sle-micro", + "sles", + ): + variant = "suse" + else: + variant = "linux" + elif system in ( + "windows", + "darwin", + "freebsd", + "netbsd", + "openbsd", + "dragonfly", + ): + variant = system + + return variant + + +@lru_cache() +def system_info(): + info = { + "platform": platform.platform(), + "system": platform.system(), + "release": platform.release(), + "python": platform.python_version(), + "uname": list(platform.uname()), + "dist": get_linux_distro(), + } + info["variant"] = _get_variant(info) + return info + + +def get_cfg_option_list(yobj, key, default=None): + """ + Gets the C{key} config option from C{yobj} as a list of strings. If the + key is present as a single string it will be returned as a list with one + string arg. + + @param yobj: The configuration object. + @param key: The configuration key to get. + @param default: The default to return if key is not found. + @return: The configuration option as a list of strings or default if key + is not found. + """ + if key not in yobj: + return default + if yobj[key] is None: + return [] + val = yobj[key] + if isinstance(val, (list)): + cval = [v for v in val] + return cval + if not isinstance(val, str): + val = str(val) + return [val] + + +# get a cfg entry by its path array +# for f['a']['b']: get_cfg_by_path(mycfg,('a','b')) +def get_cfg_by_path(yobj, keyp, default=None): + """Return the value of the item at path C{keyp} in C{yobj}. + + example: + get_cfg_by_path({'a': {'b': {'num': 4}}}, 'a/b/num') == 4 + get_cfg_by_path({'a': {'b': {'num': 4}}}, 'c/d') == None + + @param yobj: A dictionary. + @param keyp: A path inside yobj. it can be a '/' delimited string, + or an iterable. + @param default: The default to return if the path does not exist. + @return: The value of the item at keyp." + is not found.""" + + if isinstance(keyp, str): + keyp = keyp.split("/") + cur = yobj + for tok in keyp: + if tok not in cur: + return default + cur = cur[tok] + return cur + + +def fixup_output(cfg, mode): + (outfmt, errfmt) = get_output_cfg(cfg, mode) + redirect_output(outfmt, errfmt) + return (outfmt, errfmt) + + +# redirect_output(outfmt, errfmt, orig_out, orig_err) +# replace orig_out and orig_err with filehandles specified in outfmt or errfmt +# fmt can be: +# > FILEPATH +# >> FILEPATH +# | program [ arg1 [ arg2 [ ... ] ] ] +# +# with a '|', arguments are passed to shell, so one level of +# shell escape is required. +# +# if _CLOUD_INIT_SAVE_STDOUT is set in environment to a non empty and true +# value then output input will not be closed (useful for debugging). +# +def redirect_output(outfmt, errfmt, o_out=None, o_err=None): + if is_true(os.environ.get("_CLOUD_INIT_SAVE_STDOUT")): + LOG.debug("Not redirecting output due to _CLOUD_INIT_SAVE_STDOUT") + return + + if not o_out: + o_out = sys.stdout + if not o_err: + o_err = sys.stderr + + # pylint: disable=subprocess-popen-preexec-fn + def set_subprocess_umask_and_gid(): + """Reconfigure umask and group ID to create output files securely. + + This is passed to subprocess.Popen as preexec_fn, so it is executed in + the context of the newly-created process. It: + + * sets the umask of the process so created files aren't world-readable + * if an adm group exists in the system, sets that as the process' GID + (so that the created file(s) are owned by root:adm) + """ + os.umask(0o037) + try: + group_id = grp.getgrnam("adm").gr_gid + except KeyError: + # No adm group, don't set a group + pass + else: + os.setgid(group_id) + + if outfmt: + LOG.debug("Redirecting %s to %s", o_out, outfmt) + (mode, arg) = outfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen( + arg, + shell=True, + stdin=subprocess.PIPE, + preexec_fn=set_subprocess_umask_and_gid, + ) + new_fp = proc.stdin + else: + raise TypeError("Invalid type for output format: %s" % outfmt) + + if o_out: + os.dup2(new_fp.fileno(), o_out.fileno()) + + if errfmt == outfmt: + LOG.debug("Redirecting %s to %s", o_err, outfmt) + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + if errfmt: + LOG.debug("Redirecting %s to %s", o_err, errfmt) + (mode, arg) = errfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen( + arg, + shell=True, + stdin=subprocess.PIPE, + preexec_fn=set_subprocess_umask_and_gid, + ) + new_fp = proc.stdin + else: + raise TypeError("Invalid type for error format: %s" % errfmt) + + if o_err: + os.dup2(new_fp.fileno(), o_err.fileno()) + + +def mergemanydict(sources: Sequence[Mapping], reverse=False) -> dict: + """Merge multiple dicts according to the dict merger rules. + + Dict merger rules can be found in cloud-init documentation. If no mergers + have been specified, entries will be recursively added, but no values + get replaced if they already exist. Functionally, this means that the + highest priority keys must be specified first. + + Example: + a = { + "a": 1, + "b": 2, + "c": [1, 2, 3], + "d": { + "a": 1, + "b": 2, + }, + } + + b = { + "a": 10, + "c": [4], + "d": { + "a": 3, + "f": 10, + }, + "e": 20, + } + + mergemanydict([a, b]) results in: + { + 'a': 1, + 'b': 2, + 'c': [1, 2, 3], + 'd': { + 'a': 1, + 'b': 2, + 'f': 10, + }, + 'e': 20, + } + """ + if reverse: + sources = list(reversed(sources)) + merged_cfg: dict = {} + for cfg in sources: + if cfg: + # Figure out which mergers to apply... + mergers_to_apply = mergers.dict_extract_mergers(cfg) + if not mergers_to_apply: + mergers_to_apply = mergers.default_mergers() + merger = mergers.construct(mergers_to_apply) + merged_cfg = merger.merge(merged_cfg, cfg) + return merged_cfg + + +@contextlib.contextmanager +def chdir(ndir): + curr = os.getcwd() + try: + os.chdir(ndir) + yield ndir + finally: + os.chdir(curr) + + +@contextlib.contextmanager +def umask(n_msk): + old = os.umask(n_msk) + try: + yield old + finally: + os.umask(old) + + +def center(text, fill, max_len): + return "{0:{fill}{align}{size}}".format( + text, fill=fill, align="^", size=max_len + ) + + +def del_dir(path): + LOG.debug("Recursively deleting %s", path) + shutil.rmtree(path) + + +# read_optional_seed +# returns boolean indicating success or failure (presence of files) +# if files are present, populates 'fill' dictionary with 'user-data' and +# 'meta-data' entries +def read_optional_seed(fill, base="", ext="", timeout=5): + try: + (md, ud, vd) = read_seeded(base, ext, timeout) + fill["user-data"] = ud + fill["vendor-data"] = vd + fill["meta-data"] = md + return True + except url_helper.UrlError as e: + if e.code == url_helper.NOT_FOUND: + return False + raise + + +def fetch_ssl_details(paths=None): + ssl_details = {} + # Lookup in these locations for ssl key/cert files + if not paths: + ssl_cert_paths = [ + "/var/lib/cloud/data/ssl", + "/var/lib/cloud/instance/data/ssl", + ] + else: + ssl_cert_paths = [ + os.path.join(paths.get_ipath_cur("data"), "ssl"), + os.path.join(paths.get_cpath("data"), "ssl"), + ] + ssl_cert_paths = uniq_merge(ssl_cert_paths) + ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)] + cert_file = None + for d in ssl_cert_paths: + if os.path.isfile(os.path.join(d, "cert.pem")): + cert_file = os.path.join(d, "cert.pem") + break + key_file = None + for d in ssl_cert_paths: + if os.path.isfile(os.path.join(d, "key.pem")): + key_file = os.path.join(d, "key.pem") + break + if cert_file and key_file: + ssl_details["cert_file"] = cert_file + ssl_details["key_file"] = key_file + elif cert_file: + ssl_details["cert_file"] = cert_file + return ssl_details + + +def load_yaml(blob, default=None, allowed=(dict,)): + loaded = default + blob = decode_binary(blob) + try: + LOG.debug( + "Attempting to load yaml from string " + "of length %s with allowed root types %s", + len(blob), + allowed, + ) + converted = safeyaml.load(blob) + if converted is None: + LOG.debug("loaded blob returned None, returning default.") + converted = default + elif not isinstance(converted, allowed): + # Yes this will just be caught, but thats ok for now... + raise TypeError( + "Yaml load allows %s root types, but got %s instead" + % (allowed, type_utils.obj_name(converted)) + ) + loaded = converted + except (safeyaml.YAMLError, TypeError, ValueError) as e: + msg = "Failed loading yaml blob" + mark = None + if hasattr(e, "context_mark") and getattr(e, "context_mark"): + mark = getattr(e, "context_mark") + elif hasattr(e, "problem_mark") and getattr(e, "problem_mark"): + mark = getattr(e, "problem_mark") + if mark: + msg += ( + '. Invalid format at line {line} column {col}: "{err}"'.format( + line=mark.line + 1, col=mark.column + 1, err=e + ) + ) + else: + msg += ". {err}".format(err=e) + LOG.warning(msg) + return loaded + + +def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): + if base.find("%s") >= 0: + ud_url = base.replace("%s", "user-data" + ext) + vd_url = base.replace("%s", "vendor-data" + ext) + md_url = base.replace("%s", "meta-data" + ext) + else: + if features.NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH: + if base[-1] != "/" and parse.urlparse(base).query == "": + # Append fwd slash when no query string and no %s + base += "/" + ud_url = "%s%s%s" % (base, "user-data", ext) + vd_url = "%s%s%s" % (base, "vendor-data", ext) + md_url = "%s%s%s" % (base, "meta-data", ext) + md_resp = url_helper.read_file_or_url( + md_url, timeout=timeout, retries=retries + ) + md = None + if md_resp.ok(): + md = load_yaml(decode_binary(md_resp.contents), default={}) + + ud_resp = url_helper.read_file_or_url( + ud_url, timeout=timeout, retries=retries + ) + ud = None + if ud_resp.ok(): + ud = ud_resp.contents + + vd = None + try: + vd_resp = url_helper.read_file_or_url( + vd_url, timeout=timeout, retries=retries + ) + except url_helper.UrlError as e: + LOG.debug("Error in vendor-data response: %s", e) + else: + if vd_resp.ok(): + vd = vd_resp.contents + else: + LOG.debug("Error in vendor-data response") + + return (md, ud, vd) + + +def read_conf_d(confd, *, instance_data_file=None) -> dict: + """Read configuration directory.""" + # Get reverse sorted list (later trumps newer) + confs = sorted(os.listdir(confd), reverse=True) + + # Remove anything not ending in '.cfg' + confs = [f for f in confs if f.endswith(".cfg")] + + # Remove anything not a file + confs = [f for f in confs if os.path.isfile(os.path.join(confd, f))] + + # Load them all so that they can be merged + cfgs = [] + for fn in confs: + path = os.path.join(confd, fn) + try: + cfgs.append( + read_conf( + path, + instance_data_file=instance_data_file, + ) + ) + except PermissionError: + LOG.warning( + "REDACTED config part %s, insufficient permissions", path + ) + except OSError as e: + LOG.warning("Error accessing file %s: [%s]", path, e) + + return mergemanydict(cfgs) + + +def read_conf_with_confd(cfgfile, *, instance_data_file=None) -> dict: + """Read yaml file along with optional ".d" directory, return merged config + + Given a yaml file, load the file as a dictionary. Additionally, if there + exists a same-named directory with .d extension, read all files from + that directory in order and return the merged config. The template + file is optional and will be applied to any applicable jinja file + in the configs. + + For example, this function can read both /etc/cloud/cloud.cfg and all + files in /etc/cloud/cloud.cfg.d and merge all configs into a single dict. + """ + cfgs: Deque[Dict] = deque() + cfg: dict = {} + try: + cfg = read_conf(cfgfile, instance_data_file=instance_data_file) + except PermissionError: + LOG.warning( + "REDACTED config part %s, insufficient permissions", cfgfile + ) + except OSError as e: + LOG.warning("Error accessing file %s: [%s]", cfgfile, e) + else: + cfgs.append(cfg) + + confd = "" + if "conf_d" in cfg: + confd = cfg["conf_d"] + if confd: + if not isinstance(confd, str): + raise TypeError( + "Config file %s contains 'conf_d' with non-string type %s" + % (cfgfile, type_utils.obj_name(confd)) + ) + else: + confd = str(confd).strip() + elif os.path.isdir(f"{cfgfile}.d"): + confd = f"{cfgfile}.d" + + if confd and os.path.isdir(confd): + # Conf.d settings override input configuration + confd_cfg = read_conf_d(confd, instance_data_file=instance_data_file) + cfgs.appendleft(confd_cfg) + + return mergemanydict(cfgs) + + +def read_conf_from_cmdline(cmdline=None): + # return a dictionary of config on the cmdline or None + return load_yaml(read_cc_from_cmdline(cmdline=cmdline)) + + +def read_cc_from_cmdline(cmdline=None): + # this should support reading cloud-config information from + # the kernel command line. It is intended to support content of the + # format: + # cc: [end_cc] + # this would include: + # cc: ssh_import_id: [smoser, kirkland]\\n + # cc: ssh_import_id: [smoser, bob]\\nruncmd: [ [ ls, -l ], echo hi ] end_cc + # cc:ssh_import_id: [smoser] end_cc cc:runcmd: [ [ ls, -l ] ] end_cc + # cc:ssh_import_id: %5Bsmoser%5D end_cc + if cmdline is None: + cmdline = get_cmdline() + + cmdline = f" {cmdline}" + tag_begin = " cc:" + tag_end = "end_cc" + begin_l = len(tag_begin) + end_l = len(tag_end) + clen = len(cmdline) + tokens = [] + begin = cmdline.find(tag_begin) + while begin >= 0: + end = cmdline.find(tag_end, begin + begin_l) + if end < 0: + end = clen + tokens.append( + parse.unquote(cmdline[begin + begin_l : end].lstrip()).replace( + "\\n", "\n" + ) + ) + begin = cmdline.find(tag_begin, end + end_l) + + return "\n".join(tokens) + + +def dos2unix(contents): + # find first end of line + pos = contents.find("\n") + if pos <= 0 or contents[pos - 1] != "\r": + return contents + return contents.replace("\r\n", "\n") + + +HostnameFqdnInfo = namedtuple( + "HostnameFqdnInfo", + ["hostname", "fqdn", "is_default"], +) + + +def get_hostname_fqdn(cfg, cloud, metadata_only=False): + """Get hostname and fqdn from config if present and fallback to cloud. + + @param cfg: Dictionary of merged user-data configuration (from init.cfg). + @param cloud: Cloud instance from init.cloudify(). + @param metadata_only: Boolean, set True to only query cloud meta-data, + returning None if not present in meta-data. + @return: a namedtuple of + , , (str, str, bool). + Values can be none when + metadata_only is True and no cfg or metadata provides hostname info. + is_default is a bool and + it's true only if hostname is localhost and was + returned by util.get_hostname() as a default. + This is used to differentiate with a user-defined + localhost hostname. + """ + is_default = False + if "fqdn" in cfg: + # user specified a fqdn. Default hostname then is based off that + fqdn = cfg["fqdn"] + hostname = get_cfg_option_str(cfg, "hostname", fqdn.split(".")[0]) + else: + if "hostname" in cfg and cfg["hostname"].find(".") > 0: + # user specified hostname, and it had '.' in it + # be nice to them. set fqdn and hostname from that + fqdn = cfg["hostname"] + hostname = cfg["hostname"][: fqdn.find(".")] + else: + # no fqdn set, get fqdn from cloud. + # get hostname from cfg if available otherwise cloud + fqdn = cloud.get_hostname( + fqdn=True, metadata_only=metadata_only + ).hostname + if "hostname" in cfg: + hostname = cfg["hostname"] + else: + hostname, is_default = cloud.get_hostname( + metadata_only=metadata_only + ) + return HostnameFqdnInfo(hostname, fqdn, is_default) + + +def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): + """ + For each host a single line should be present with + the following information: + + IP_address canonical_hostname [aliases...] + + Fields of the entry are separated by any number of blanks and/or tab + characters. Text from a "#" character until the end of the line is a + comment, and is ignored. Host names may contain only alphanumeric + characters, minus signs ("-"), and periods ("."). They must begin with + an alphabetic character and end with an alphanumeric character. + Optional aliases provide for name changes, alternate spellings, shorter + hostnames, or generic hostnames (for example, localhost). + """ + fqdn = None + try: + for line in load_text_file(filename).splitlines(): + hashpos = line.find("#") + if hashpos >= 0: + line = line[0:hashpos] + line = line.strip() + if not line: + continue + + # If there there is less than 3 entries + # (IP_address, canonical_hostname, alias) + # then ignore this line + toks = line.split() + if len(toks) < 3: + continue + + if hostname in toks[2:]: + fqdn = toks[1] + break + except IOError: + pass + return fqdn + + +def is_resolvable(url) -> bool: + """determine if a url's network address is resolvable, return a boolean + This also attempts to be resilent against dns redirection. + + Note, that normal nsswitch resolution is used here. So in order + to avoid any utilization of 'search' entries in /etc/resolv.conf + we have to append '.'. + + The top level 'invalid' domain is invalid per RFC. And example.com + should also not exist. The '__cloud_init_expected_not_found__' entry will + be resolved inside the search list. + """ + global _DNS_REDIRECT_IP + parsed_url = parse.urlparse(url) + name = parsed_url.hostname + if _DNS_REDIRECT_IP is None: + badips = set() + badnames = ( + "does-not-exist.example.com.", + "example.invalid.", + "__cloud_init_expected_not_found__", + ) + badresults: dict = {} + for iname in badnames: + try: + result = socket.getaddrinfo( + iname, None, 0, 0, socket.SOCK_STREAM, socket.AI_CANONNAME + ) + badresults[iname] = [] + for _fam, _stype, _proto, cname, sockaddr in result: + badresults[iname].append("%s: %s" % (cname, sockaddr[0])) + badips.add(sockaddr[0]) + except (socket.gaierror, socket.error): + pass + _DNS_REDIRECT_IP = badips + if badresults: + LOG.debug("detected dns redirection: %s", badresults) + + try: + # ip addresses need no resolution + with suppress(ValueError): + if net.is_ip_address(parsed_url.netloc.strip("[]")): + return True + result = socket.getaddrinfo(name, None) + # check first result's sockaddr field + addr = result[0][4][0] + return addr not in _DNS_REDIRECT_IP + except (socket.gaierror, socket.error): + return False + + +def get_hostname(): + hostname = socket.gethostname() + return hostname + + +def gethostbyaddr(ip): + try: + return socket.gethostbyaddr(ip)[0] + except socket.herror: + return None + + +def is_resolvable_url(url): + """determine if this url is resolvable (existing or ip).""" + return log_time( + logfunc=LOG.debug, + msg="Resolving URL: " + url, + func=is_resolvable, + args=(url,), + ) + + +def search_for_mirror(candidates): + """ + Search through a list of mirror urls for one that works + This needs to return quickly. + """ + if candidates is None: + return None + + LOG.debug("search for mirror in candidates: '%s'", candidates) + for cand in candidates: + try: + if is_resolvable_url(cand): + LOG.debug("found working mirror: '%s'", cand) + return cand + except Exception: + pass + return None + + +def close_stdin(): + """ + reopen stdin as /dev/null so even subprocesses or other os level things get + /dev/null as input. + + if _CLOUD_INIT_SAVE_STDIN is set in environment to a non empty and true + value then input will not be closed (useful for debugging). + """ + if is_true(os.environ.get("_CLOUD_INIT_SAVE_STDIN")): + return + with open(os.devnull) as fp: + os.dup2(fp.fileno(), sys.stdin.fileno()) + + +def find_devs_with_freebsd( + criteria=None, oformat="device", tag=None, no_cache=False, path=None +): + devlist = [] + if not criteria: + return glob.glob("/dev/msdosfs/*") + glob.glob("/dev/iso9660/*") + if criteria.startswith("LABEL="): + label = criteria.lstrip("LABEL=") + devlist = [ + p + for p in ["/dev/msdosfs/" + label, "/dev/iso9660/" + label] + if os.path.exists(p) + ] + elif criteria == "TYPE=vfat": + devlist = glob.glob("/dev/msdosfs/*") + elif criteria == "TYPE=iso9660": + devlist = glob.glob("/dev/iso9660/*") + return devlist + + +def find_devs_with_netbsd( + criteria=None, oformat="device", tag=None, no_cache=False, path=None +): + devlist = [] + label = None + _type = None + mscdlabel_out = "" + if criteria: + if criteria.startswith("LABEL="): + label = criteria.lstrip("LABEL=") + if criteria.startswith("TYPE="): + _type = criteria.lstrip("TYPE=") + out = subp.subp(["sysctl", "-n", "hw.disknames"], rcs=[0]) + for dev in out.stdout.split(): + if label or _type: + mscdlabel_out, _ = subp.subp(["mscdlabel", dev], rcs=[0, 1]) + if label and ('label "%s"' % label) not in mscdlabel_out: + continue + if _type == "iso9660" and "ISO filesystem" not in mscdlabel_out: + continue + if _type == "vfat" and "ISO filesystem" in mscdlabel_out: + continue + devlist.append("/dev/" + dev) + return devlist + + +def find_devs_with_openbsd( + criteria=None, oformat="device", tag=None, no_cache=False, path=None +): + out = subp.subp(["sysctl", "-n", "hw.disknames"], rcs=[0]) + devlist = [] + for entry in out.stdout.rstrip().split(","): + if not entry.endswith(":"): + # ffs partition with a serial, not a config-drive + continue + if entry == "fd0:": + continue + devlist.append(entry[:-1] + "a") + if not entry.startswith("cd"): + devlist.append(entry[:-1] + "i") + return ["/dev/" + i for i in devlist] + + +def find_devs_with_dragonflybsd( + criteria=None, oformat="device", tag=None, no_cache=False, path=None +): + out = subp.subp(["sysctl", "-n", "kern.disks"], rcs=[0]) + devlist = [ + i + for i in sorted(out.stdout.split(), reverse=True) + if not i.startswith("md") and not i.startswith("vn") + ] + + if criteria == "TYPE=iso9660": + devlist = [i for i in devlist if i.startswith(("cd", "acd"))] + elif criteria in ["LABEL=CONFIG-2", "TYPE=vfat"]: + devlist = [i for i in devlist if not (i.startswith(("cd", "acd")))] + elif criteria: + LOG.debug("Unexpected criteria: %s", criteria) + return ["/dev/" + i for i in devlist] + + +def find_devs_with( + criteria=None, oformat="device", tag=None, no_cache=False, path=None +): + """ + find devices matching given criteria (via blkid) + criteria can be *one* of: + TYPE= + LABEL=