docker run -v $(pwd)/data:/data -ti generate/creds:azure
# this will ask you to launch a web browser to authenticate to azure
# it will result in ./data/azure.env
# deployname can only consist of lowercase letters and numbers, and must be less than 18 characters long
DEPLOYNAME=$(date +%Y%m%d%H%M) # could be a branch+gitcommit etc
# all data for the deploy will be stored in it's own directory
docker run -ti -v $(pwd)/data/$DEPLOYNAME:/cncf/data --env-file ./data/azure.env create/azure deploy $DEPLOYNAMEbastion-fqdn = bastion201703230324.westus.cloudapp.azure.com
fqdn_k8s = k8s201703230324.westus.cloudapp.azure.com
ssh-key-setup = eval $(ssh-agent) ; ssh-add /cncf/data/.ssh/id_rsa
ssh-via-bastion = ssh -At cncf@bastion201703230324.westus.cloudapp.azure.com ssh cncf@master1.cncf.demo
real 3m2.509s
user 0m4.460s
sys 0m0.668s
# takes about 3-8 minutes to deploy
sudo chown -R $(whoami):$(whoami) $(pwd)/data/${DEPLOYNAME}
export KUBECONFIG=$(pwd)/data/${DEPLOYNAME}/kubeconfig
kubectl get nodes
NAME STATUS AGE
etcd-master1 Ready,SchedulingDisabled 13m
etcd-master2 Ready,SchedulingDisabled 13m
etcd-master3 Ready,SchedulingDisabled 14m
worker-node1 Ready 10m
worker-node2 Ready 13m
worker-node3 Ready 13m
...
kubectl proxy etc
# to destroy
docker run -ti -v $(pwd)/data/$DEPLOYNAME:/cncf/data --env-file ./data/azure.env terminate/azure destroy $DEPLOYNAME
Using DEPLOYNAME allows for multiple concurrent deploys and to easily.
Adding var-name=value to ./data/terraform.tfvars will allow you to override many settings for this deploy.
name= "azure"
location= "westus"
internal_tld= "cncf.demo"
master_node_count= "3"
worker_node_count= "3"
master_vm_size= "Standard_A2"
worker_vm_size= "Standard_A2"
bastion_vm_size= "Standard_A2"
kubelet_aci= "quay.io/coreos/hyperkube"
kubelet_image_tag= "v1.4.7_coreos.0"
image_publisher= "CoreOS"
image_offer= "CoreOS"
image_sku= "Stable"
image_version= "1298.6.0"- Store private keys in storage rather than cloud-init
- Look into using Centos and Debian/Ubuntu
- Deploy onto AWS and Azure at the same time
- Setup some CI to deploy on every commit to both clouds
- Start on GCE
Specifically azurerm_dns_zone name_servers only provides the server names, while azurerm_network_interface requires a list of IPs. We’ll do a cleaner maping later, but the current hack got us IPs and
We had a couple places where CNAMEs behaved unexpectedly when using Azure dns zones. Specifically CNAME records when queried with DIG wouldn’t refer.
This affects our ability to bootstrap etcd with an unknown number of nodes beforehand. If we stick with three (or any number) it’s not a problem.
We should be able to scale up our workers at some point in the future.
When using –cloud-provider=azure not only must you use –cloud-config=azure.json, it seems you have to provide all the optional settings as well. Failure to do so results in a panic.
instances are looked up via nodeName and if they don’t match, kubelet will not start.
kubelet_node_status.go:69] Unable to construct api.Node object for kubelet:
failed to get external ID from cloud provider: instance not found
$ docker run -v $(pwd)/data:/data -ti generate/creds:azure
To sign in, use a web browser to open the page https://aka.ms/devicelogin and enter the code GY7W7BMRZ to authenticate.
Name CloudName SubscriptionId State IsDefault
------------- ----------- ------------------------------------ -------- -----------
Free Trial AzureCloud 5358e673-95e7-4cd8-9791-ca28dd5e3cbb Disabled True
Pay-As-You-Go AzureCloud 70693672-7c0d-485f-ac08-06d458c80f0e Enabled
Please enter the Name of the account you wish to use. If you do not see
a valid account in the list press Ctrl+C to abort and create one.
If you leave this blank we will use the Current account.
> Pay-As-You-Go
Using subscription_id: 70693672-7c0d-485f-ac08-06d458c80f0e
Using tenant_id: 9996322a-93ac-43ae-80be-887a3e8194a1
==> Creating service principal
Retrying role assignment creation: 1/36
Retrying role assignment creation: 2/36
./data/azure.env created
$ cat ./data/azure.env
export ARM_SUBSCRIPTION_ID=70693672-XXXX-4858-ac08-06888888880e
export ARM_TENANT_ID=9896828a-93ac-43ae-YYYY-887a3e8898a1
export ARM_CLIENT_ID=968448ae-f9f9-ZZZZ-bf43-5c081da88975
export ARM_CLIENT_SECRET=BBBBBBBB-8eaa-AAAA-aafe-75b02ad4cebadocker run -v $(pwd)/.azure:/root/.azure azuresdk/azure-cli-python az account list -o table
az account list -o table
az account set --subscription Pay-As-You-Go#+RESULTS[eb0d69eb1ea1b9a005604b3dd37889127d19f76b]: az account list
| Name | CloudName | SubscriptionId | State | IsDefault |
| ------------- | ----------- | ------------------------------------ | -------- | ----------- |
| Free | Trial | AzureCloud | 5358e673-95e7-4cd8-9791-ca28dd5e3cbb | Disabled |
| Pay-As-You-Go | AzureCloud | 70693672-7c0d-485f-ac08-06d458c80f0e | Enabled | True |
az account show -o table#+RESULTS[00afff595364da643372e54234a45a775c1539ef]: az account show table
| EnvironmentName | IsDefault | Name | State | TenantId |
| ----------------- | ----------- | ------------- | ------- | ------------------------------------ |
| AzureCloud | True | Pay-As-You-Go | Enabled | 9996322a-93ac-43ae-80be-887a3e8194a1 |
az account show#+RESULTS[97a6b7ba7839519d9223a4e67e27ced7ed78f0b9]: az_account_show_json
{
"environmentName": "AzureCloud",
"id": "70693672-7c0d-485f-ac08-06d458c80f0e",
"isDefault": true,
"name": "Pay-As-You-Go",
"state": "Enabled",
"tenantId": "9996322a-93ac-43ae-80be-887a3e8194a1",
"user": {
"name": "azure@ii.coop",
"type": "user"
}
}ARM_SUBSCRIPTION_ID=$( az account show | jq -r .id )
CREDS_JSON=$( az ad sp create-for-rbac --name cncfdemos )
ARM_TENANT_ID=$( echo ${CREDS_JSON} | jq -r .tenant )
ARM_CLIENT_ID=$( echo ${CREDS_JSON} | jq -r .appId )
ARM_CLIENT_SECRET=$( echo ${CREDS_JSON} | jq -r .password )
echo export ARM_SUBSCRIPTION_ID=$ARM_SUBSCRIPTION_ID
echo export ARM_TENANT_ID=$ARM_TENANT_ID
echo export ARM_CLIENT_ID=$ARM_CLIENT_ID
echo export ARM_CLIENT_SECRET=$ARM_CLIENT_SECRET