Kubernetes on AzureStackHub with AKS-Engine and external AD/DNS integration

What is AKS Engine

Check your Quotas

Create a Service Priciple (service account)

az ad sp create-for-rbac --name $AKS-SP

Create a Virtual Network

az network vnet create --address-prefixes 10.222.0.0/16 --name $k8sVnetName --resource-group $RG-NETWORK --subnet-name $KubernetesMasterSubnet --subnet-prefixes 10.222.1.0/25

Create a Subnet

az network vnet subnet create -g $RG-NETWORK --vnet-name $k8sVnetName -n $KubernetesMasterSubnet --address-prefixes 10.222.1.0/25
az network vnet subnet create -g $RG-NETWORK --vnet-name $k8sVnetName -n $KubernetesWorkerSubnet --address-prefixes 10.222.1.128/25

Create a Ressource Group

az group create -l $LOCATION -g $RG-NAME-k8s

Create the deployment file

Deploymentfile without a custom DNS-Server

{
"apiVersion": "vlabs",
"location": "$LOCATION",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.17",
"orchestratorVersion": "1.17.5",
"kubernetesConfig": {
"cloudProviderBackoff": true,
"cloudProviderBackoffRetries": 1,
"cloudProviderBackoffDuration": 30,
"cloudProviderRateLimit": true,
"cloudProviderRateLimitQPS": 3,
"cloudProviderRateLimitBucket": 10,
"cloudProviderRateLimitQPSWrite": 3,
"cloudProviderRateLimitBucketWrite": 10,
"kubernetesImageBase": "mcr.microsoft.com/k8s/azurestack/core/",
"useInstanceMetadata": false,
"networkPlugin": "kubenet",
"kubeletConfig": {
"--node-status-update-frequency": "1m",
"--kube-reserved": "memory=500Mi",
"--system-reserved": "memory=500Mi",
"--eviction-hard": "memory.available<500Mi"
},
"controllerManagerConfig": {
"--node-monitor-grace-period": "5m",
"--pod-eviction-timeout": "5m",
"--route-reconciliation-period": "1m"
},
"privateCluster": {
"enabled": true
},
"etcdDiskSizeGB": "32"
}
},
"customCloudProfile": {
"portalURL": "https://$AzureStackHubURL"
},
"featureFlags": {
"enableTelemetry": true
},
"masterProfile": {
"dnsPrefix": "master",
"distro": "aks-ubuntu-16.04",
"count": 3,
"vmSize": "Standard_DS2_v2",
"vnetSubnetId": "/subscriptions/$SUBSCRIPTIONID/resourceGroups/$RG-NETWORK/providers/Microsoft.Network/virtualNetworks/$k8sVnetName/subnets/$KubernetesMasterSubnet",
"firstConsecutiveStaticIP": "..REDACTED...",
"customVMTags": {
"Application": "k8s Test"
}
},
"agentPoolProfiles": [
{
"name": "worker",
"count": 3,
"vmSize": "Standard_DS2_v2",
"distro": "aks-ubuntu-16.04",
"availabilityProfile": "AvailabilitySet",
"AcceleratedNetworkingEnabled": false,
"vnetSubnetId": "/subscriptions/$SUBSCRIPTIONID/resourceGroups/$RG-NETWORK/providers/Microsoft.Network/virtualNetworks/$k8sVnetName/subnets/$KubernetesWorkerSubnet",
"customVMTags": {
"Application": "k8s Test"
}
}
],
"linuxProfile": {
"adminUsername": "$ROOTUserName",
"ssh": {
"publicKeys": [
{
"keyData": "$ROOTUserSSHKey"
}
]
}
},
"servicePrincipalProfile": {
"clientId": "$SPID",
"secret": "$SPPassword"
}
}
}

Deploymentfile with custom DNS-Server

{
"apiVersion": "vlabs",
"location": "$LOCATION",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.17",
"orchestratorVersion": "1.17.5",
"kubernetesConfig": {
"cloudProviderBackoff": true,
"cloudProviderBackoffRetries": 1,
"cloudProviderBackoffDuration": 30,
"cloudProviderRateLimit": true,
"cloudProviderRateLimitQPS": 3,
"cloudProviderRateLimitBucket": 10,
"cloudProviderRateLimitQPSWrite": 3,
"cloudProviderRateLimitBucketWrite": 10,
"kubernetesImageBase": "mcr.microsoft.com/k8s/azurestack/core/",
"useInstanceMetadata": false,
"networkPlugin": "kubenet",
"kubeletConfig": {
"--node-status-update-frequency": "1m",
"--kube-reserved": "memory=500Mi",
"--system-reserved": "memory=500Mi",
"--eviction-hard": "memory.available<500Mi"
},
"controllerManagerConfig": {
"--node-monitor-grace-period": "5m",
"--pod-eviction-timeout": "5m",
"--route-reconciliation-period": "1m"
},
"privateCluster": {
"enabled": true
},
"etcdDiskSizeGB": "32"
}
},
"customCloudProfile": {
"portalURL": "https://$AzureStackHubURL"
},
"featureFlags": {
"enableTelemetry": true
},
"masterProfile": {
"dnsPrefix": "master",
"distro": "aks-ubuntu-16.04",
"count": 3,
"vmSize": "Standard_DS2_v2",
"vnetSubnetId": "/subscriptions/$SUBSCRIPTIONID/resourceGroups/$RG-NETWORK/providers/Microsoft.Network/virtualNetworks/$k8sVnetName/subnets/$KubernetesMasterSubnet",
"firstConsecutiveStaticIP": "..REDACTED...",
"customVMTags": {
"Application": "k8s Test"
}
},
"agentPoolProfiles": [
{
"name": "worker",
"count": 3,
"vmSize": "Standard_DS2_v2",
"distro": "aks-ubuntu-16.04",
"availabilityProfile": "AvailabilitySet",
"AcceleratedNetworkingEnabled": false,
"vnetSubnetId": "/subscriptions/$SUBSCRIPTIONID/resourceGroups/$RG-NETWORK/providers/Microsoft.Network/virtualNetworks/$k8sVnetName/subnets/$KubernetesWorkerSubnet",
"customVMTags": {
"Application": "k8s Test"
}
}
],
"linuxProfile": {
"adminUsername": "$ROOTUserName",
"ssh": {
"publicKeys": [
{
"keyData": "$ROOTUserSSHKey"
}
]
},
"customSearchDomain": {
"name": "$ADDOMAINNAME",
"realmUser": "$ADSERVICEACCOUNT",
"realmPassword": "$ADSERVICEACCOUNTPASSWORD"
},
"customNodesDNS": {
"dnsServer": "$CustomDNSIP"
}
},
"servicePrincipalProfile": {
"clientId": "$SPID",
"secret": "$SPPassword"
}
}
}

Deployment

aks-engine deploy -f --azure-env AzureStackCloud --api-model kubernetes-azurestack.json --location $LOCATION --resource-group $RG-NAME-k8s --client-id $SPID --client-secret $SPPassword --subscription-id $SUBSCRIPTIONID --output-directory $RG-NAME-k8s

Hotfix

cat <<EOF > /opt/azure/containers/setup-custom-search-domains.sh
#!/bin/bash
set -x
source /opt/azure/containers/provision_source.shecho " dns-search $ADDOMAINNAME" | tee -a /etc/network/interfaces.d/50-cloud-init.cfg
systemctl_restart 20 5 10 networking
wait_for_apt_locks
retrycmd 10 5 120 apt-get update
wait_for_apt_locks
retrycmd 10 5 120 apt-get -y install realmd sssd sssd-tools samba-common samba samba-common python2.7 samba-libs packagekit
wait_for_apt_locks
echo "$ADSERVICEACCOUNTPASSWORD" | realm join -U $ADSERVICEACCOUNT@$(echo "$ADDOMAINNAME" | tr /a-z/ /A-Z/) $(echo "$ADDOMAINNAME" | tr /a-z/ /A-Z/)
#EOF
EOF

If your deployment was successful

Problem or Feature i’m not sure

Suggestions welcome

don’t trust me…

--

--

--

Passionate DevOps engineer with a preference for Infrastructure as Code, Automation, Kubernetes, Go, Azure- and HetznerCloud.

Love podcasts or audiobooks? Learn on the go with our new app.

Recommended from Medium

HackTheBox — Admirer

INTERFACING SEVEN-SEGMENT DISPLAY TO THE AT89C52 MICROCONTROLLER

ATMEL 89C52 MIRCROCONTROLLER

Running Windows 11 on the VMWare Player

Hierarchical Data Model with MongoDB

Hierarchical Data Model with MongoDB

Teeslion customer care helpline number 6299325726//6299325726 help refund ke liye call karein thank…

4 Fun Reasons to Avoid Using “Primitive” Swift Types in Your Code

Tapalcatl 2 Updates

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
0hlov3

0hlov3

Passionate DevOps engineer with a preference for Infrastructure as Code, Automation, Kubernetes, Go, Azure- and HetznerCloud.

More from Medium

Here are Ant Media’s Highlights of 2021 — Now it’s time for 2022! — Ant Media Server

Finding .Net Assemblies

Host Blazor on ECS Part 2 —Configure Load Balancer with SSL

ACM — request certificate

Automating AKS + Linkerd deployment with Terraform.