tests: k8s: inject agent policy failures (part2)

Auto-generate the policy and then simulate attacks from the K8s control plane by modifying the test yaml files. The policy then detects and blocks those changes. These test cases are using K8s Replication Controllers. Additional policy failures will be injected using other types of K8s resources - e.g., using Pods and/or Jobs - in separate PRs. Fixes: kata-containers#9463 Signed-off-by: Dan Mihai <[email protected]>
apostasie · Apr 11, 2024 · e51cbdc · e51cbdc
1 parent 6b2d655
commit e51cbdc
Show file tree

Hide file tree

Showing 3 changed files with 211 additions and 0 deletions.
diff --git a/tests/integration/kubernetes/k8s-policy-rc.bats b/tests/integration/kubernetes/k8s-policy-rc.bats
@@ -0,0  1,178 @@
 #!/usr/bin/env bats
 #
 # Copyright (c) 2024 Microsoft.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
 
 load "${BATS_TEST_DIRNAME}/../../common.bash"
 load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 setup() {
     policy_tests_enabled || skip "Policy tests are disabled."
 
     replication_name="policy-rc-test"
     app_name="policy-nginx-rc"
 
     get_pod_config_dir
 
     correct_yaml="${pod_config_dir}/test-k8s-policy-rc.yaml"
     incorrect_yaml="${pod_config_dir}/test-k8s-policy-rc-incorrect.yaml"
 
     # Save some time by executing genpolicy a single time.
     if [ "${BATS_TEST_NUMBER}" == "1" ]; then
         # Create the correct yaml file
         nginx_version="${docker_images_nginx_version}"
         nginx_image="nginx:$nginx_version"
 
         sed -e "s/\${nginx_version}/${nginx_image}/" \
             "${pod_config_dir}/k8s-policy-rc.yaml" > "${correct_yaml}"
 
         # Add policy to the correct yaml file
         policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
         add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
         auto_generate_policy "${policy_settings_dir}" "${correct_yaml}"
     fi
 
     # Start each test case with a copy of the correct yaml file.
     cp "${correct_yaml}" "${incorrect_yaml}"
 
     # teardown() parses this string for pod names and prints the output of "kubectl describe" for these pods.
     declare -a launched_pods=()
 }
 
 # Common function for all test cases from this bats script.
 test_rc_policy() {
     expect_denied_create_container=$1
 
     # Create replication controller
     if [ "${expect_denied_create_container}" = "true" ]; then
         kubectl create -f "${incorrect_yaml}"
     else 
         kubectl create -f "${correct_yaml}"
     fi
 
     # Check replication controller
     local cmd="kubectl describe rc ${replication_name} | grep replication-controller"
     info "Waiting for: ${cmd}"
     waitForProcess "$wait_time" "$sleep_time" "$cmd"
 
     number_of_replicas=$(kubectl get rc ${replication_name} \
         --output=jsonpath='{.spec.replicas}')
     [ "${number_of_replicas}" -gt 0 ]
 
     # The replicas pods can be in running, waiting, succeeded or failed
     # status. We need them all on running state before proceeding.
     cmd="kubectl describe rc ${replication_name}"
     cmd =" | grep \"Pods Status\" | grep \"${number_of_replicas} Running\""
     info "Waiting for: ${cmd}"
     waitForProcess "$wait_time" "$sleep_time" "$cmd"
 
     # Check that the number of pods created for the replication controller
     # is equal to the number of replicas that we defined.
     launched_pods=($(kubectl get pods "--selector=app=${app_name}" \
         --output=jsonpath={.items..metadata.name}))
     [ "${#launched_pods[@]}" -eq "${number_of_replicas}" ]
 
     # Check pod creation
     for pod_name in ${launched_pods[@]}; do
         if [ "${expect_denied_create_container}" = "true" ]; then
             wait_for_blocked_request "CreateContainerRequest" "${pod_name}"
         else
             cmd="kubectl wait --for=condition=Ready --timeout=${timeout} pod ${pod_name}"
             info "Waiting for: ${cmd}"
             waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
         fi
     done
 }
 
 @test "Successful replication controller with auto-generated policy" {
     test_rc_policy false
 }
 
 @test "Policy failure: unexpected container command" {
     # Changing the template spec after generating its policy will cause CreateContainer to be denied.
     yq write -i \
             "${incorrect_yaml}" \
             "spec.template.spec.containers[0].command.[ ]" \
             "ls"
 
     test_rc_policy true
 }
 
 @test "Policy failure: unexpected volume mountPath" {
     # Changing the template spec after generating its policy will cause CreateContainer to be denied.
     yq write -i \
             "${incorrect_yaml}" \
             "spec.template.spec.containers[0].volumeMounts[0].mountPath" \
             "/host/unexpected"
 
     test_rc_policy true
 }
 
 @test "Policy failure: unexpected host device mapping" {
     # Changing the template spec after generating its policy will cause CreateContainer to be denied.
     yq write -i \
             "${incorrect_yaml}" \
             "spec.template.spec.containers[0].volumeMounts.[ ].mountPath" \
             "/dev/ttyS0"
 
     yq write -i \
             "${incorrect_yaml}" \
             "spec.template.spec.containers[0].volumeMounts.[-1].name" \
             "dev-ttys0"
 
     yq write -i \
             "${incorrect_yaml}" \
             "spec.template.spec.volumes.[ ].name" \
             "dev-ttys0"
 
     yq write -i \
             "${incorrect_yaml}" \
             "spec.template.spec.volumes.[-1].hostPath.path" \
             "/dev/ttyS0"
 
     test_rc_policy true
 }
 
 @test "Policy failure: unexpected securityContext.allowPrivilegeEscalation" {
     # Changing the template spec after generating its policy will cause CreateContainer to be denied.
     yq write -i \
             "${incorrect_yaml}" \
             "spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation" \
             "false"
 
     test_rc_policy true
 }
 
 @test "Policy failure: unexpected capability" {
     # Changing the template spec after generating its policy will cause CreateContainer to be denied.
     yq write -i \
             "${incorrect_yaml}" \
             "spec.template.spec.containers[0].securityContext.capabilities.add.[ ]" \
             "CAP_SYS_CHROOT"
 
     test_rc_policy true
 }
 
 teardown() {
     policy_tests_enabled || skip "Policy tests are disabled."
 
     # Debugging information
     kubectl describe rc "${replication_name}"
 
     for pod_name in ${launched_pods[@]}; do
         info "Pod ${pod_name}:"
         kubectl describe pod "${pod_name}"
     done
 
     # Clean-up
     kubectl delete rc "${replication_name}"
 
     info "Deleting ${incorrect_yaml}"
     rm -f "${incorrect_yaml}"
 
     if [ "${BATS_TEST_NUMBER}" == "1" ]; then
         delete_tmp_policy_settings_dir "${policy_settings_dir}"
     fi
 }
diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh
@@ -52,6  52,7 @@ else
 		"k8s-pid-ns.bats" \
 		"k8s-pod-quota.bats" \
 		"k8s-policy-job.bats" \
 		"k8s-policy-rc.bats" \
 		"k8s-policy-set-keys.bats" \
 		"k8s-port-forward.bats" \
 		"k8s-projected-volume.bats" \

diff --git a/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-rc.yaml b/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-rc.yaml
@@ -0,0  1,32 @@
 #
 # Copyright (c) 2024 Microsoft
 #
 # SPDX-License-Identifier: Apache-2.0
 #
 apiVersion: v1
 kind: ReplicationController
 metadata:
   name: policy-rc-test
 spec:
   replicas: 1
   selector:
     app: policy-nginx-rc
   template:
     metadata:
       name: nginx
       labels:
         app: policy-nginx-rc
     spec:
       terminationGracePeriodSeconds: 0
       runtimeClassName: kata
       containers:
       - name: nginxtest
         image: quay.io/sjenning/${nginx_version}
         ports:
         - containerPort: 80
         volumeMounts:
           - name: host-empty-vol
             mountPath: "/host/cache"
       volumes:
       - name: host-empty-vol
         emptyDir: {}