Bug: PydanticSerializer can't save results with UUID, datetime or Enum fields, so retries run the function again #8569

import os
from datetime import datetime, timezone
from enum import Enum
from uuid import UUID, uuid4

from pydantic import BaseModel

from aws_lambda_powertools.utilities.idempotency import (
    DynamoDBPersistenceLayer,
    IdempotencyConfig,
    idempotent_function,
)
from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import PydanticSerializer
from aws_lambda_powertools.utilities.typing import LambdaContext

persistence_layer = DynamoDBPersistenceLayer(table_name=os.environ["IDEMPOTENCY_TABLE"])
config = IdempotencyConfig(event_key_jmespath="order_id")


class PaymentStatus(Enum):
    CHARGED = "charged"


class Order(BaseModel):
    order_id: str
    amount: int


class Payment(BaseModel):
    payment_id: UUID
    charged_at: datetime
    status: PaymentStatus


@idempotent_function(
    data_keyword_argument="order",
    config=config,
    persistence_store=persistence_layer,
    output_serializer=PydanticSerializer,
)
def charge(order: Order) -> Payment:
    # call the payment provider here
    return Payment(payment_id=uuid4(), charged_at=datetime.now(timezone.utc), status=PaymentStatus.CHARGED)


def lambda_handler(event: dict, context: LambdaContext):
    config.register_lambda_context(context)
    payment = charge(order=Order(**event))
    return payment.model_dump(mode="json")

def to_dict(self, data: BaseModel) -> dict:
    return data.model_dump(mode="json")

import json
from datetime import datetime, timezone
from uuid import UUID, uuid4

from pydantic import BaseModel

from aws_lambda_powertools.shared.json_encoder import Encoder
from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import PydanticSerializer


class Payment(BaseModel):
    payment_id: UUID
    charged_at: datetime


payment = Payment(payment_id=uuid4(), charged_at=datetime.now(timezone.utc))
json.dumps(PydanticSerializer(Payment).to_dict(payment), cls=Encoder, sort_keys=True)  # what save_success does
# TypeError: Object of type datetime is not JSON serializable

Traceback (most recent call last):
  File ".../aws_lambda_powertools/utilities/idempotency/base.py", line 300, in _get_function_response
    self.persistence_store.save_success(data=self.data, result=serialized_response)
  File ".../aws_lambda_powertools/utilities/idempotency/persistence/base.py", line 302, in save_success
    response_data = json.dumps(result, cls=Encoder, sort_keys=True)
  ...
TypeError: Object of type datetime is not JSON serializable
when serializing dict item 'charged_at'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  ...
  File ".../aws_lambda_powertools/utilities/idempotency/base.py", line 302, in _get_function_response
    raise IdempotencyPersistenceLayerError(
aws_lambda_powertools.utilities.idempotency.exceptions.IdempotencyPersistenceLayerError: Failed to update record state to success in idempotency store - (Object of type datetime is not JSON serializable)