feat: add hosts config agent support for Windows nodes

Azure · Jul 7, 2020 · 56a665a · 56a665a
1 parent 4521ec4
commit 56a665a
Show file tree

Hide file tree

Showing 12 changed files with 269 additions and 20 deletions.
diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1
@@ -143,6 143,9 @@ $global:TelemetryKey = "{{WrapAsVariable "applicationInsightsKey" }}";
 $global:EnableCsiProxy = [System.Convert]::ToBoolean("{{WrapAsVariable "windowsEnableCSIProxy" }}");
 $global:CsiProxyUrl = "{{WrapAsVariable "windowsCSIProxyURL" }}";
 
 # Hosts Config Agent settings
 $global:EnableHostsConfigAgent = [System.Convert]::ToBoolean("{{WrapAsVariable "enableHostsConfigAgent" }}");
 
 $global:ProvisioningScriptsPackageUrl = "{{WrapAsVariable "windowsProvisioningScriptsPackageURL" }}";
 
 # Base64 representation of ZIP archive
@@ -161,6 164,7 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\"
 . c:\AzureData\k8s\windowscsiproxyfunc.ps1
 . c:\AzureData\k8s\windowsinstallopensshfunc.ps1
 . c:\AzureData\k8s\windowscontainerdfunc.ps1
 . c:\AzureData\k8s\windowshostsconfigagentfunc.ps1
 
 $useContainerD = ($global:ContainerRuntime -eq "containerd")
 $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json"
@@ -326,6 330,11 @@ try
  -AgentKey $AgentKey `
  -AgentCertificate $global:AgentCertificate
 
  if ($global:EnableHostsConfigAgent) {
  Write-Log "Starting hosts config agent"
  New-HostsConfigService
  }
 
  Write-Log "Create the Pause Container kubletwin/pause"
  $infraContainerTimer = [System.Diagnostics.Stopwatch]::StartNew()
  New-InfraContainer -KubeDir $global:KubeDir -ContainerRuntime $global:ContainerRuntime

diff --git a/parts/k8s/windowshostsconfigagentfunc.ps1 b/parts/k8s/windowshostsconfigagentfunc.ps1
@@ -0,0 1,21 @@
 function New-HostsConfigService {
  $HostsConfigParameters = [io.path]::Combine($KubeDir, "hostsconfigagent.ps1")
 
  & "$KubeDir\nssm.exe" install hosts-config-agent C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppDirectory "$KubeDir" | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppParameters $HostsConfigParameters | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppRestartDelay 5000 | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent Description hosts-config-agent | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent Start SERVICE_DEMAND_START | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent ObjectName LocalSystem | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent Type SERVICE_WIN32_OWN_PROCESS | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppThrottle 1500 | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppStdout "$KubeDir\hosts-config-agent.log" | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppStderr "$KubeDir\hosts-config-agent.err.log" | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppStdoutCreationDisposition 4 | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppStderrCreationDisposition 4 | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppRotateFiles 1 | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppRotateOnline 1 | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppRotateSeconds 86400 | RemoveNulls
  & "$KubeDir\nssm.exe" set hosts-config-agent AppRotateBytes 10485760 | RemoveNulls
 }
diff --git a/parts/k8s/windowskubeletfunc.ps1 b/parts/k8s/windowskubeletfunc.ps1
@@ -322,6 322,9 @@ New-NSSMService {
  if ($global:EnableCsiProxy) {
  $kubeletDependOnServices  = " csi-proxy"
  }
  if ($global:EnableHostsConfigAgent) {
  $kubeletDependOnServices  = " hosts-config-agent"
  }
 
  # setup kubelet
  & "$KubeDir\nssm.exe" install Kubelet C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe | RemoveNulls

diff --git a/pkg/api/types.go b/pkg/api/types.go
@@ -1906,6 1906,14 @@ func (o *OrchestratorProfile) IsPrivateCluster() bool {
  return o.KubernetesConfig != nil && o.KubernetesConfig.PrivateCluster != nil && to.Bool(o.KubernetesConfig.PrivateCluster.Enabled)
 }
 
 // IsHostsConfigAgentEnabled returns true if hosts config agent is enabled
 func (o *OrchestratorProfile) IsHostsConfigAgentEnabled() bool {
  if !o.IsKubernetes() {
  return false
  }
  return o.KubernetesConfig != nil && o.KubernetesConfig.PrivateCluster != nil && to.Bool(o.KubernetesConfig.PrivateCluster.EnableHostsConfigAgent)
 }
 
 // GetPodInfraContainerSpec returns the sandbox image as a string (ex: k8s.gcr.io/pause-amd64:3.1)
 func (o *OrchestratorProfile) GetPodInfraContainerSpec() string {
  return o.KubernetesConfig.MCRKubernetesImageBase   GetK8sComponentsByVersionMap(o.KubernetesConfig)[o.OrchestratorVersion][common.PauseComponentName]

diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go
@@ -3401,6 3401,73 @@ func TestIsAzureCNI(t *testing.T) {
  }
 }
 
 func TestIsHostsConfigAgentEnabled(t *testing.T) {
  cases := []struct {
  p Properties
  expected bool
  }{
  {
  p: Properties{
  OrchestratorProfile: &OrchestratorProfile{
  OrchestratorType: DCOS,
  },
  },
  expected: false,
  },
  {
  p: Properties{
  OrchestratorProfile: &OrchestratorProfile{
  OrchestratorType: Kubernetes,
  },
  },
  expected: false,
  },
  {
  p: Properties{
  OrchestratorProfile: &OrchestratorProfile{
  OrchestratorType: Kubernetes,
  KubernetesConfig: &KubernetesConfig{
  PrivateCluster: &PrivateCluster{
  EnableHostsConfigAgent: to.BoolPtr(true),
  },
  },
  },
  },
  expected: true,
  },
  {
  p: Properties{
  OrchestratorProfile: &OrchestratorProfile{
  OrchestratorType: Kubernetes,
  KubernetesConfig: &KubernetesConfig{
  PrivateCluster: &PrivateCluster{
  EnableHostsConfigAgent: to.BoolPtr(false),
  },
  },
  },
  },
  expected: false,
  },
  {
  p: Properties{
  OrchestratorProfile: &OrchestratorProfile{
  OrchestratorType: Kubernetes,
  KubernetesConfig: &KubernetesConfig{
  PrivateCluster: &PrivateCluster{},
  },
  },
  },
  expected: false,
  },
  }
 
  for _, c := range cases {
  if c.p.OrchestratorProfile.IsHostsConfigAgentEnabled() != c.expected {
  t.Fatalf("expected IsHostsConfigAgentEnabled() to return %t but instead got %t", c.expected, c.p.OrchestratorProfile.IsHostsConfigAgentEnabled())
  }
  }
 }
 
 func TestOrchestrator(t *testing.T) {
  cases := []struct {
  p Properties

diff --git a/pkg/engine/armvariables.go b/pkg/engine/armvariables.go
@@ -385,6 385,7 @@ func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error)
  }
  }
  masterVars["primaryScaleSetName"] = cs.Properties.GetPrimaryScaleSetName()
  masterVars["enableHostsConfigAgent"] = cs.Properties.OrchestratorProfile.IsHostsConfigAgentEnabled()
 
  if isHostedMaster {
  masterVars["kubernetesAPIServerIP"] = "[parameters('kubernetesEndpoint')]"

diff --git a/pkg/engine/armvariables_test.go b/pkg/engine/armvariables_test.go
@@ -97,6 97,7 @@ func TestK8sVars(t *testing.T) {
  "applicationInsightsKey": "c92d8284-b550-4b06-b7ba-e80fd7178faa", // should be DefaultApplicationInsightsKey,
  "clusterKeyVaultName": "",
  "contributorRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]",
  "enableHostsConfigAgent": false,
  "enableTelemetry": false,
  "etcdCaFilepath": "/etc/kubernetes/certs/ca.crt",
  "etcdClientCertFilepath": "/etc/kubernetes/certs/etcdclient.crt",
@@ -700,6 701,7 @@ func TestK8sVars(t *testing.T) {
  "apiVersionStorage": "2017-10-01",
  "clusterKeyVaultName": "",
  "contributorRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]",
  "enableHostsConfigAgent": false,
  "enableTelemetry": false,
  "etcdCaFilepath": "/etc/kubernetes/certs/ca.crt",
  "etcdClientCertFilepath": "/etc/kubernetes/certs/etcdclient.crt",
@@ -950,6 952,7 @@ func TestK8sVarsMastersOnly(t *testing.T) {
  "contributorRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]",
  "customCloudAuthenticationMethod": "client_secret",
  "customCloudIdentifySystem": "azure_ad",
  "enableHostsConfigAgent": false,
  "enableTelemetry": false,
  "etcdCaFilepath": "/etc/kubernetes/certs/ca.crt",
  "etcdClientCertFilepath": "/etc/kubernetes/certs/etcdclient.crt",

diff --git a/pkg/engine/const.go b/pkg/engine/const.go
@@ -77,15 77,16 @@ const (
 const (
  kubeConfigJSON = "k8s/kubeconfig.json"
  // Windows custom scripts. These should all be listed in template_generator.go:func GetKubernetesWindowsAgentFunctions
- kubernetesWindowsAgentCustomDataPS1 = "k8s/kuberneteswindowssetup.ps1"
- kubernetesWindowsAgentFunctionsPS1 = "k8s/kuberneteswindowsfunctions.ps1"
- kubernetesWindowsConfigFunctionsPS1 = "k8s/windowsconfigfunc.ps1"
- kubernetesWindowsContainerdFunctionsPS1 = "k8s/windowscontainerdfunc.ps1"
- kubernetesWindowsCsiProxyFunctionsPS1 = "k8s/windowscsiproxyfunc.ps1"
- kubernetesWindowsKubeletFunctionsPS1 = "k8s/windowskubeletfunc.ps1"
- kubernetesWindowsCniFunctionsPS1 = "k8s/windowscnifunc.ps1"
- kubernetesWindowsAzureCniFunctionsPS1 = "k8s/windowsazurecnifunc.ps1"
- kubernetesWindowsOpenSSHFunctionPS1 = "k8s/windowsinstallopensshfunc.ps1"
  kubernetesWindowsAgentCustomDataPS1 = "k8s/kuberneteswindowssetup.ps1"
  kubernetesWindowsAgentFunctionsPS1 = "k8s/kuberneteswindowsfunctions.ps1"
  kubernetesWindowsConfigFunctionsPS1 = "k8s/windowsconfigfunc.ps1"
  kubernetesWindowsContainerdFunctionsPS1 = "k8s/windowscontainerdfunc.ps1"
  kubernetesWindowsCsiProxyFunctionsPS1 = "k8s/windowscsiproxyfunc.ps1"
  kubernetesWindowsKubeletFunctionsPS1 = "k8s/windowskubeletfunc.ps1"
  kubernetesWindowsCniFunctionsPS1 = "k8s/windowscnifunc.ps1"
  kubernetesWindowsAzureCniFunctionsPS1 = "k8s/windowsazurecnifunc.ps1"
  kubernetesWindowsHostsConfigAgentFunctionsPS1 = "k8s/windowshostsconfigagentfunc.ps1"
  kubernetesWindowsOpenSSHFunctionPS1 = "k8s/windowsinstallopensshfunc.ps1"
 )
 
 // cloud-init (i.e. ARM customData) source file references

diff --git a/pkg/engine/template_generator.go b/pkg/engine/template_generator.go
@@ -361,9 361,7 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap {
  return cs.Properties.OrchestratorProfile.IsPrivateCluster()
  },
  "EnableHostsConfigAgent": func() bool {
- return cs.Properties.OrchestratorProfile.KubernetesConfig != nil &&
- cs.Properties.OrchestratorProfile.KubernetesConfig.PrivateCluster != nil &&
- to.Bool(cs.Properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.EnableHostsConfigAgent)
  return cs.Properties.OrchestratorProfile.IsHostsConfigAgentEnabled()
  },
  "ProvisionJumpbox": func() bool {
  return cs.Properties.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision()
@@ -513,6 511,7 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap {
  kubernetesWindowsKubeletFunctionsPS1,
  kubernetesWindowsCniFunctionsPS1,
  kubernetesWindowsAzureCniFunctionsPS1,
  kubernetesWindowsHostsConfigAgentFunctionsPS1,
  kubernetesWindowsOpenSSHFunctionPS1,
  }
 

diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go