AWS Lambda code to connect with EKS cluster

Question

I have a lambda code in python (v3.13) which is trying to connect to an AWS EKS cluster to run a job. The lambda and AWS EKS are in same VPC, subnets, and have same security groups so that there will not be any issue related to communication.

Lambda code in python3.13:

import os
import boto3
import kubernetes
from kubernetes import client, config
from botocore.exceptions import ClientError

def lambda_handler(event, context):
    # Step 1: Get the cluster name 
    cluster_name = "app1-eks-cluster"
    
    # Step 2: Retrieve the EKS cluster details to get the kubeconfig
    eks_client = boto3.client('eks')
    try:
        # Get EKS cluster information
        print("Getting cluster info...")
        cluster_info = eks_client.describe_cluster(name=cluster_name)
        cluster_endpoint = cluster_info['cluster']['endpoint']
        cluster_certificate = cluster_info['cluster']['certificateAuthority']['data']
        print("cluster_endpoint=" + cluster_endpoint)
    except ClientError as e:
        print(f"Error retrieving cluster details: {e}")
        raise e
    
    # Step 3: Create Kubernetes configuration object to connect to the EKS cluster
    try:
        kube_config = {
            'api_version': 'v1',
            'clusters': [{
                'cluster': {
                    'server': cluster_endpoint,
                    'certificate-authority-data': cluster_certificate
                },
                'name': cluster_name
            }],
            'contexts': [{
                'context': {
                    'cluster': cluster_name,
                    'user': cluster_name
                },
                'name': cluster_name
            }],
            'current-context': cluster_name,
            'kind': 'Config',
            'users': [{
                'name': cluster_name,
                'user': {
                    'exec': {
                        'apiVersion': 'client.authentication.k8s.io/v1beta1',
                        'command': 'aws',
                        'args': ['eks', 'get-token', '--cluster-name', cluster_name]
                    }
                }
            }]
        }
        
        # Set the kubeconfig
        config.load_kube_config_from_dict(kube_config)
        
    except Exception as e:
        print(f"Error setting up Kubernetes client: {e}")
        raise e
    
    # Step 4: Run a Kubernetes job (or other action) within the EKS cluster
    try:
        # Example: Create a simple Job in Kubernetes
        batch_v1 = client.BatchV1Api()
        job = client.V1Job(
            metadata=client.V1ObjectMeta(name="example-job"),
            spec=client.V1JobSpec(
                template=client.V1PodTemplateSpec(
                    spec=client.V1PodSpec(
                        containers=[client.V1Container(name="example-container", image="busybox", command=["/bin/sh", "-c", "echo Hello Kubernetes"])],
                        restart_policy="Never"
                    )
                ),
                backoff_limit=4
            )
        )
        
        # Submit the job to the Kubernetes cluster
        api_response = batch_v1.create_namespaced_job(
            body=job,
            namespace="f-design"
        )
        print(f"Job created. Status: {api_response.status}")
    
    except Exception as e:
        print(f"Error running job on EKS: {e}")
        raise e

    return {
        'statusCode': 200,
        'body': 'Job successfully submitted to the EKS cluster'
    }

Error:

Response:
{
  "errorMessage": "(403)\nReason: Forbidden\nHTTP response headers: HTTPHeaderDict({'Audit-Id': '21cfdf80-e0f7-45db-1234-65a3429a90ff', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Content-Type-Options': 'nosniff', 'X-Kubernetes-Pf-Flowschema-Uid': 'e60b20c4-5970-5678-1234-856e48a23731', 'X-Kubernetes-Pf-Prioritylevel-Uid': '4a2c47d9-bc5c-1234-5678-d36bb38878f0', 'Date': 'Fri, 27 Dec 2024 07:05:37 GMT', 'Content-Length': '305'})\nHTTP response body: {\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"jobs.batch is forbidden: User \\\"system:anonymous\\\" cannot create resource \\\"jobs\\\" in API group \\\"batch\\\" in the namespace \\\"f-design\\\"\",\"reason\":\"Forbidden\",\"details\":{\"group\":\"batch\",\"kind\":\"jobs\"},\"code\":403}\n\n",
  "errorType": "ApiException",
  "requestId": "4f536051-0e16-4ddd-1234-5e6799767e78",
  "stackTrace": [
    "  File \"/var/task/lambda_function.py\", line 89, in lambda_handler\n    raise e\n",
    "  File \"/var/task/lambda_function.py\", line 81, in lambda_handler\n    api_response = batch_v1.create_namespaced_job(\n",
    "  File \"/var/task/kubernetes/client/api/batch_v1_api.py\", line 210, in create_namespaced_job\n    return self.create_namespaced_job_with_http_info(namespace, body, **kwargs)  # noqa: E501\n",
    "  File \"/var/task/kubernetes/client/api/batch_v1_api.py\", line 309, in create_namespaced_job_with_http_info\n    return self.api_client.call_api(\n",
    "  File \"/var/task/kubernetes/client/api_client.py\", line 348, in call_api\n    return self.__call_api(resource_path, method,\n",
    "  File \"/var/task/kubernetes/client/api_client.py\", line 180, in __call_api\n    response_data = self.request(\n",
    "  File \"/var/task/kubernetes/client/api_client.py\", line 391, in request\n    return self.rest_client.POST(url,\n",
    "  File \"/var/task/kubernetes/client/rest.py\", line 279, in POST\n    return self.request(\"POST\", url,\n",
    "  File \"/var/task/kubernetes/client/rest.py\", line 238, in request\n    raise ApiException(http_resp=r)\n"
  ]
}

Function Logs:
START RequestId: 4f536051-0e16-4ddd-1234-5e6799767e78 Version: $LATEST
Getting cluster info...
cluster_endpoint=https://12333333333333332.gr8.ap-south-1.eks.amazonaws.com
[ERROR] 2024-12-27T07:05:37.187Z    4f536051-0e16-4ddd-1234-5e6799767e78    [Errno 2] No such file or directory: 'aws'
Error running job on EKS: (403)
Reason: Forbidden
HTTP response headers: HTTPHeaderDict({'Audit-Id': '21cfdf80-e0f7-45db-1234-65a3429a90ff', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Content-Type-Options': 'nosniff', 'X-Kubernetes-Pf-Flowschema-Uid': 'e60b20c4-5970-5678-1234-856e48a23731', 'X-Kubernetes-Pf-Prioritylevel-Uid': '4a2c47d9-bc5c-1234-5678-d36bb38878f0', 'Date': 'Fri, 27 Dec 2024 07:05:37 GMT', 'Content-Length': '305'})
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"jobs.batch is forbidden: User \"system:anonymous\" cannot create resource \"jobs\" in API group \"batch\" in the namespace \"f-design\"","reason":"Forbidden","details":{"group":"batch","kind":"jobs"},"code":403}
LAMBDA_WARNING: Unhandled exception. The most likely cause is an issue in the function code. However, in rare cases, a Lambda runtime update can cause unexpected function behavior. For functions using managed runtimes, runtime updates can be triggered by a function change, or can be applied automatically. To determine if the runtime has been updated, check the runtime version in the INIT_START log entry. If this error correlates with a change in the runtime version, you may be able to mitigate this error by temporarily rolling back to the previous runtime version. For more information, see https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html
[ERROR] ApiException: (403)
Reason: Forbidden
HTTP response headers: HTTPHeaderDict({'Audit-Id': '21cfdf80-e0f7-45db-1234-65a3429a90ff', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Content-Type-Options': 'nosniff', 'X-Kubernetes-Pf-Flowschema-Uid': 'e60b20c4-5970-5678-1234-856e48a23731', 'X-Kubernetes-Pf-Prioritylevel-Uid': '4a2c47d9-bc5c-1234-5678-d36bb38878f0', 'Date': 'Fri, 27 Dec 2024 07:05:37 GMT', 'Content-Length': '305'})

I have updated aws-auth-config and granted system:master permission to the lambda role for the lambda function that I have created. Something like this

mapRoles: |
- groups:
- system:masters
rolearn: arn:aws:iam::111122223333:role/lambdarole
username: lambdafunctionname

but still getting the same 403 forbidden error from eks.

Any help would be greatly appreciated!.

Answer 1

The primary error:

[Errno 2] No such file or directory: 'aws'

This appears before the 403 error and this is a indication that the Lambda function can't find the AWS CLI, which is needed for the token generation in your kubeconfig. This is why you're being identified as system:anonymous

Hence Suggest to removed the AWS CLI dependency in the kube_config dictionary under the users section

Please Replace this with direct boto3 token generation

'users': [{
    'name': cluster_name,
    'user': {
        'exec': {  # This part tries to execute the AWS CLI
            'apiVersion': 'client.authentication.k8s.io/v1beta1',
            'command': 'aws',  # <-- This is trying to use the AWS CLI command
            'args': ['eks', 'get-token', '--cluster-name', cluster_name]
        }
    }
}]