Merge pull request #255 from socotecio/fix/defaultValue

Fix/default value

django_socio_grpc/mixins.py

@@ -17,9 +17,8 @@
     StrTemplatePlaceholder,
 )
 from .grpc_actions.utils import get_serializer_base_name
-from .protobuf.json_format import message_to_dict
 from .settings import grpc_settings
-from .utils.constants import DEFAULT_LIST_FIELD_NAME, REQUEST_SUFFIX
+from .utils.constants import DEFAULT_LIST_FIELD_NAME, PARTIAL_UPDATE_FIELD_NAME, REQUEST_SUFFIX
 
 
 ############################################################
@@ -253,9 +252,23 @@ def get_default_message(model_name, fields="__all__"):
 def _get_partial_update_request(service):
     serializer_class = service.get_serializer_class()
 
-    class PartialUpdateRequest(serializer_class):
+    class PartialUpdateMetaClass(serializers.SerializerMetaclass):
+        """
+        This metaclass exists so we can set the PARTIAL_UPDATE_FIELD_NAME variable as an attribute name of PartialUpdateRequest.
+        This can be replaced by just declaring in PartialUpdateRequest:
         _partial_update_fields = serializers.ListField(child=serializers.CharField())
+        but this would not be dynamic if a constant changes or if we want it to be configurable in settings in the future.
+        This metaclass should inherit from DRF SerializerMetaclass as serializer has it's own metaclass to add _declared_fields attribute
+        Using PartialUpdateRequest.setattr is not enough as _declared_fields is done in metaclass so all fields should be declared before
+        """
 
+        def __new__(cls, name, bases, attrs):
+            attrs[PARTIAL_UPDATE_FIELD_NAME] = serializers.ListField(
+                child=serializers.CharField()
+            )
+            return super().__new__(cls, name, bases, attrs)
+
+    class PartialUpdateRequest(serializer_class, metaclass=PartialUpdateMetaClass):
         class Meta(serializer_class.Meta):
             ...
 
@@ -264,7 +277,7 @@ class Meta(serializer_class.Meta):
     if (fields := getattr(PartialUpdateRequest.Meta, "fields", None)) and not isinstance(
         fields, str
     ):
-        PartialUpdateRequest.Meta.fields = (*fields, "_partial_update_fields")
+        PartialUpdateRequest.Meta.fields = (*fields, PARTIAL_UPDATE_FIELD_NAME)
 
     return PartialUpdateRequest
 
@@ -281,18 +294,14 @@ def PartialUpdate(self, request, context):
         """
         Partial update a model instance.
 
-        Performs a partial update on the given `_partial_update_fields`.
+        Performs a partial update on the given PARTIAL_UPDATE_FIELD_NAME(`_partial_update_fields`).
         """
 
-        content = message_to_dict(request)
-
-        data = {k: v for k, v in content.items() if k in request._partial_update_fields}
-
         instance = self.get_object()
 
         # INFO - L.G. - 11/07/2022 - We use the data parameter instead of message
         # because we handle a dict not a grpc message.
-        serializer = self.get_serializer(instance, data=data, partial=True)
+        serializer = self.get_serializer(instance, message=request, partial=True)
         serializer.is_valid(raise_exception=True)
         self.perform_partial_update(serializer)
 
@@ -483,18 +492,14 @@ async def PartialUpdate(self, request, context):
         """
         Partial update a model instance.
 
-        Performs a partial update on the given `_partial_update_fields`.
+        Performs a partial update on the given PARTIAL_UPDATE_FIELD_NAME(`_partial_update_fields`).
         """
 
-        content = message_to_dict(request)
-
-        data = {k: v for k, v in content.items() if k in request._partial_update_fields}
-
         instance = await self.aget_object()
 
         # INFO - L.G. - 11/07/2022 - We use the data parameter instead of message
         # because we handle a dict not a grpc message.
-        serializer = await self.aget_serializer(instance, data=data, partial=True)
+        serializer = await self.aget_serializer(instance, message=request, partial=True)
         await sync_to_async(serializer.is_valid)(raise_exception=True)
         await self.aperform_partial_update(serializer)
 

django_socio_grpc/proto_serializers.py

@@ -2,8 +2,10 @@
 
 from asgiref.sync import sync_to_async
 from django.core.validators import MaxLengthValidator
+from django.db.models.fields import NOT_PROVIDED
 from django.utils.translation import gettext as _
 from rest_framework.exceptions import ValidationError
+from rest_framework.fields import empty
 from rest_framework.relations import SlugRelatedField
 from rest_framework.serializers import (
     LIST_SERIALIZER_KWARGS,
@@ -17,24 +19,116 @@
 from rest_framework.utils.formatting import lazy_format
 
 from django_socio_grpc.protobuf.json_format import message_to_dict, parse_dict
-from django_socio_grpc.utils.constants import DEFAULT_LIST_FIELD_NAME, LIST_ATTR_MESSAGE_NAME
+from django_socio_grpc.utils.constants import (
+    DEFAULT_LIST_FIELD_NAME,
+    LIST_ATTR_MESSAGE_NAME,
+    PARTIAL_UPDATE_FIELD_NAME,
+)
 
 LIST_PROTO_SERIALIZER_KWARGS = (*LIST_SERIALIZER_KWARGS, LIST_ATTR_MESSAGE_NAME, "message")
 
 
+def get_default_value(field_default):
+    if callable(field_default):
+        return field_default()
+    else:
+        return field_default
+
+
 class BaseProtoSerializer(BaseSerializer):
     def __init__(self, *args, **kwargs):
         message = kwargs.pop("message", None)
         self.stream = kwargs.pop("stream", None)
         self.message_list_attr = kwargs.pop(LIST_ATTR_MESSAGE_NAME, DEFAULT_LIST_FIELD_NAME)
+        # INFO - AM - 04/01/2023 - Need to manually define partial before the super().__init__ as it's used in populate_dict_with_none_if_not_required that is used in message_to_data that is call before the super init
+        self.partial = kwargs.get("partial", False)
         if message is not None:
             self.initial_message = message
             kwargs["data"] = self.message_to_data(message)
         super().__init__(*args, **kwargs)
 
     def message_to_data(self, message):
         """Protobuf message -> Dict of python primitive datatypes."""
-        return message_to_dict(message)
+        data_dict = message_to_dict(message)
+        data_dict = self.populate_dict_with_none_if_not_required(data_dict, message=message)
+        return data_dict
+
+    def populate_dict_with_none_if_not_required(self, data_dict, message=None):
+        """
+        This method allow to populate the data dictionary with None for optional field that allow_null and not send in the request.
+        It's also allow to deal with partial update correctly.
+        This is mandatory for having null value received in request as DRF expect to have None value for field that are required.
+        We can't rely only on required True/False as in DSG if a field is required it will have the default value of it's type (empty string for string type) and not None
+
+        When refactoring serializer to only use message we will be able to determine the default value of the field depending of the same logic followed here
+
+        set default value for field except if optional or partial update
+        """
+        # INFO - AM - 04/01/2024 - If we are in a partial serializer with a message we need to have the PARTIAL_UPDATE_FIELD_NAME in the data_dict. If not we raise an exception
+        if self.partial and PARTIAL_UPDATE_FIELD_NAME not in data_dict:
+            raise ValidationError(
+                {
+                    PARTIAL_UPDATE_FIELD_NAME: [
+                        f"Field {PARTIAL_UPDATE_FIELD_NAME} not set in message when using partial=True"
+                    ]
+                },
+                code="missing_partial_message_attribute",
+            )
+
+        is_update_process = (
+            hasattr(self.Meta, "model") and self.Meta.model._meta.pk.name in data_dict
+        )
+        for field in self.fields.values():
+            # INFO - AM - 04/01/2024 - If we are in a partial serializer we only need to have field specified in PARTIAL_UPDATE_FIELD_NAME attribute in the data. Meaning deleting fields that should not be here and not adding None to allow_null field that are not specified
+            if self.partial and field.field_name not in data_dict.get(
+                PARTIAL_UPDATE_FIELD_NAME, {}
+            ):
+                if field.field_name in data_dict:
+                    del data_dict[field.field_name]
+                continue
+            # INFO - AM - 04/01/2024 - if field already existing in the data_dict we do not need to do something else
+            if field.field_name in data_dict:
+                continue
+
+            # INFO - AM - 04/01/2024 - if field is not in the data_dict but in PARTIAL_UPDATE_FIELD_NAME we need to set the default value if existing or raise exception to avoid having default grpc value by mistake
+            if self.partial and field.field_name in data_dict.get(
+                PARTIAL_UPDATE_FIELD_NAME, {}
+            ):
+                if field.allow_null:
+                    data_dict[field.field_name] = None
+                    continue
+                if field.default not in [None, empty]:
+                    data_dict[field.field_name] = get_default_value(field.default)
+                    continue
+
+                # INFO - AM - 11/03/2024 - Here we set the default value especially for the blank authorized data. We debated about raising a ValidaitonError but prefered this behavior. Can be changed if it create issue with users
+                data_dict[field.field_name] = message.DESCRIPTOR.fields_by_name[
+                    field.field_name
+                ].default_value
+
+            if field.allow_null or (field.default in [None, empty] and field.required is True):
+                if is_update_process:
+                    data_dict[field.field_name] = None
+                    continue
+
+                if field.default not in [None, empty]:
+                    data_dict[field.field_name] = None
+                    continue
+
+                if (
+                    hasattr(self, "Meta")
+                    and hasattr(self.Meta, "model")
+                    and hasattr(self.Meta.model, field.field_name)
+                ):
+                    deferred_attribute = getattr(self.Meta.model, field.field_name)
+                    if deferred_attribute.field.default != NOT_PROVIDED:
+                        data_dict[field.field_name] = get_default_value(
+                            deferred_attribute.field.default
+                        )
+                        continue
+
+                data_dict[field.field_name] = None
+        return data_dict
 
     def data_to_message(self, data):
         """Protobuf message <- Dict of python primitive datatypes."""

django_socio_grpc/protobuf/json_format.py

@@ -4,36 +4,16 @@
 from google.protobuf.json_format import MessageToDict, ParseDict
 
 
-def _is_field_optional(field):
-    """
-    Checks if a field is optional.
-
-    Under the hood, Optional fields are OneOf fields with only one field with the name of the OneOf
-    prefixed with an underscore.
-    """
-
-    if not (co := field.containing_oneof):
-        return False
-
-    return len(co.fields) == 1 and co.name == f"_{field.name}"
-
-
 def message_to_dict(message, **kwargs):
     """
     Converts a protobuf message to a dictionary.
     Uses the default `google.protobuf.json_format.MessageToDict` function.
     Adds None values for optional fields that are not set.
     """
-
     kwargs.setdefault("including_default_value_fields", True)
     kwargs.setdefault("preserving_proto_field_name", True)
 
-    result_dict = MessageToDict(message, **kwargs)
-    optional_fields = {
-        field.name: None for field in message.DESCRIPTOR.fields if _is_field_optional(field)
-    }
-
-    return {**optional_fields, **result_dict}
+    return MessageToDict(message, **kwargs)
 
 
 def parse_dict(js_dict, message, **kwargs):

django_socio_grpc/protobuf/proto_classes.py

@@ -20,7 +20,7 @@
 from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from rest_framework import serializers
-from rest_framework.fields import HiddenField
+from rest_framework.fields import HiddenField, empty
 from rest_framework.utils.model_meta import RelationInfo, get_field_info
 
 from django_socio_grpc.protobuf.message_name_constructor import MessageNameConstructor
@@ -89,7 +89,18 @@ def field_line(self) -> str:
 
     @classmethod
     def _get_cardinality(self, field: serializers.Field):
-        return FieldCardinality.OPTIONAL if field.allow_null else FieldCardinality.NONE
+        ProtoGeneratorPrintHelper.print("field.default: ", field.default)
+        """
+        INFO - AM - 04/01/2023
+        If field can be null -> optional
+        if field is not required -> optional. Since DRF 3.0 When using model default, only required is set to False. The model default is not set into the field as just passing None will result in model default. https://github.com/encode/django-rest-framework/issues/2683
+        if field.default is set (meaning not None or empty) -> optional
+
+        Not dealing with field.allow_blank now as it doesn't seem to be related to OPTIONAl and more about validation and only exist for charfield
+        """
+        if field.allow_null or not field.required or field.default not in [None, empty]:
+            return FieldCardinality.OPTIONAL
+        return FieldCardinality.NONE
 
     @classmethod
     def from_field_dict(cls, field_dict: FieldDict) -> "ProtoField":

django_socio_grpc/tests/assets/generated_protobuf_files_old_way.py

@@ -141,6 +141,14 @@
     rpc Destroy(RecursiveTestModelDestroyRequest) returns (google.protobuf.Empty) {}
 }
 
+service DefaultValueModelController {
+    rpc List(DefaultValueModelListRequest) returns (DefaultValueModelListResponse) {}
+    rpc Create(DefaultValueModel) returns (DefaultValueModel) {}
+    rpc Retrieve(DefaultValueModelRetrieveRequest) returns (DefaultValueModel) {}
+    rpc Update(DefaultValueModel) returns (DefaultValueModel) {}
+    rpc Destroy(DefaultValueModelDestroyRequest) returns (google.protobuf.Empty) {}
+}
+
 message UnitTestModel {
     int32 id = 1;
     string title = 2;
@@ -278,6 +286,44 @@
     string uuid = 1;
 }
 
+message DefaultValueModel {
+    string id = 1;
+    string string_required = 2;
+    string string_blank = 3;
+    string string_nullable = 4;
+    string string_default = 5;
+    string string_default_and_blank = 6;
+    string string_null_default_and_blank = 7;
+    string string_required_but_serializer_default = 8;
+    string string_default_but_serializer_default = 9;
+    string string_nullable_default_but_serializer_default = 10;
+    int32 int_required = 11;
+    int32 int_nullable = 12;
+    int32 int_default = 13;
+    int32 int_required_but_serializer_default = 14;
+    bool boolean_required = 15;
+    bool boolean_nullable = 16;
+    bool boolean_default_false = 17;
+    bool boolean_default_true = 18;
+    bool boolean_required_but_serializer_default = 19;
+}
+
+message DefaultValueModelListRequest {
+}
+
+message DefaultValueModelListResponse {
+    repeated DefaultValueModel results = 1;
+    int32 count = 2;
+}
+
+message DefaultValueModelRetrieveRequest {
+    string id = 1;
+}
+
+message DefaultValueModelDestroyRequest {
+    string id = 1;
+}
+
 """
 
 CUSTOM_APP_MODEL_GENERATED = """syntax = "proto3";