I have setup my lambda function to send traces to APM server and I am seeing these errors in the cloudwatch logs

opentelemetry.exporter.otlp.proto.http.metric_exporter - Failed to export batch code: 404, reason: 404 page not found

What I don't understand is why I am seeing errors from http when in my config file I am using only grpc and how should I fix this.

My setup is as follows

# Dockerfile

ARG PYTHON_VERSION=3.12

# Install ADOT layer
FROM amazonlinux:2 AS lambda-layer-builder

# Authentication is needed to download the layer
ARG AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-"eu-west-1"}
ARG AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-""}
ARG AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-""}
ARG AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-""}

RUN yum install -y unzip python3 python3-pip && \
  python3 -m pip install --pre --no-cache-dir awscli

RUN mkdir -p /tmp/otel-python-layer
RUN curl $(aws lambda get-layer-version-by-arn \
  --arn arn:aws:lambda:${AWS_DEFAULT_REGION}:901920570463:layer:aws-otel-python-amd64-ver-1-29-0:1 \
  --query 'Content.Location' --output text) \
  --output /tmp/otel-python-layer/layer.zip

WORKDIR /tmp/otel-python-layer
RUN unzip layer.zip -d .
RUN rm layer.zip


FROM public.ecr.aws/lambda/python:${PYTHON_VERSION}


COPY --from=lambda-layer-builder /tmp/otel-python-layer/ /opt/

COPY ./opentelemetry-collector.yaml /opt/opentelemetry-collector.yaml
ENV OPENTELEMETRY_COLLECTOR_CONFIG_URI=/opt/opentelemetry-collector.yaml
ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument

COPY requirements.txt ${LAMBDA_TASK_ROOT}
RUN python3 -m pip install --no-cache-dir -r requirements.txt
COPY lambda_function.py ${LAMBDA_TASK_ROOT}

CMD [ "lambda_function.handler" ]

# lambda.py

import os
import json
import random
import time


def handler(event, context):
    print("Lambda function invoked without custom tracing")
    print(json.dumps(event))

    latency = random.uniform(0.1, 0.5)
    print(f"Simulating latency of {latency} seconds")
    time.sleep(latency)

    return {
        "statusCode": 200,
        "body": json.dumps(
            {
                "message": "Hello, World!",
                "latency_seconds": latency,
                "function_name": name,
                "function_version": version,
            }
        ),
    }

# opentelemetry config file
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: "localhost:4317"
      http:
        endpoint: "localhost:4318"

exporters:
  otlp/grpc:
    endpoint: "xxx.dev.service.io:8200"
    headers:
      Authorization: "Bearer ${env:APM_TOKEN}"

  debug:
    verbosity: detailed

service:
  pipelines:
    traces:
      receivers: [otlp]
      exporters: [debug, otlp/grpc]

I have checked questions here but none of them helpful 1 2