Add MISSING sentinel

.github/workflows/ci.yml

@@ -329,8 +329,8 @@ jobs:
       - uses: mymindstorm/setup-emsdk@v14
         with:
           # NOTE!: as per https://github.com/pydantic/pydantic-core/pull/149 this version needs to match the version
-          # in node_modules/pyodide/repodata.json, to get the version, run:
-          # `cat node_modules/pyodide/repodata.json | python -m json.tool | rg platform`
+          # in node_modules/pyodide/pyodide-lock.json, to get the version, run:
+          # `cat node_modules/pyodide/pyodide-lock.json | jq .info.platform`
           version: '3.1.58'
           actions-cache-folder: emsdk-cache
 

package.json

@@ -8,7 +8,7 @@
   "main": "tests/emscripten_runner.js",
   "dependencies": {
     "prettier": "^2.7.1",
-    "pyodide": "^0.26.3"
+    "pyodide": "^0.27.7"
   },
   "scripts": {
     "test": "node tests/emscripten_runner.js",

pyproject.toml

@@ -36,7 +36,7 @@ classifiers = [
     'Typing :: Typed',
 ]
 dependencies = [
-    'typing-extensions>=4.13.0',
+    'typing-extensions>=4.14.1',
 ]
 dynamic = ['license', 'readme', 'version']
 
@@ -160,3 +160,4 @@ fix = ["create", "fix"]
 # this ensures that `uv run` doesn't actually build the package; a `make`
 # command is needed to build
 package = false
+required-version = '>=0.7.2'

python/pydantic_core/__init__.py

@@ -3,6 +3,8 @@
 import sys as _sys
 from typing import Any as _Any
 
+from typing_extensions import Sentinel
+
 from ._pydantic_core import (
     ArgsKwargs,
     MultiHostUrl,
@@ -40,6 +42,7 @@
 
 __all__ = [
     '__version__',
+    'UNSET',
     'CoreConfig',
     'CoreSchema',
     'CoreSchemaType',
@@ -140,3 +143,29 @@ class MultiHostHost(_TypedDict):
     """The host part of this host, or `None`."""
     port: int | None
     """The port part of this host, or `None`."""
+
+
+MISSING = Sentinel('MISSING')
+"""A singleton indicating a field value was not provided during validation.
+
+This singleton can be used a default value, as an alternative to `None` when it has
+an explicit meaning. During serialization, any field with `MISSING` as a value is excluded
+from the output.
+
+Example:
+    ```python
+    from pydantic import BaseModel
+
+    from pydantic_core import MISSING
+
+
+    class Configuration(BaseModel):
+        timeout: int | None | MISSING = MISSING
+
+
+    # configuration defaults, stored somewhere else:
+    defaults = {'timeout': 200}
+
+    conf = Configuration.model_validate({...})
+    timeout = conf.timeout if timeout.timeout is not MISSING else defaults['timeout']
+"""

python/pydantic_core/core_schema.py

@@ -1346,6 +1346,16 @@ class Color(Enum):
     )
 
 
+class MissingSentinelSchema(TypedDict, total=False):
+    type: Required[Literal['missing-sentinel']]
+
+
+def missing_sentinel_schema() -> MissingSentinelSchema:
+    """Returns a schema for the `MISSING` sentinel."""
+
+    return {'type': 'missing-sentinel'}
+
+
 # must match input/parse_json.rs::JsonType::try_from
 JsonType = Literal['null', 'bool', 'int', 'float', 'str', 'list', 'dict']
 
@@ -4082,6 +4092,7 @@ def definition_reference_schema(
         DatetimeSchema,
         TimedeltaSchema,
         LiteralSchema,
+        MissingSentinelSchema,
         EnumSchema,
         IsInstanceSchema,
         IsSubclassSchema,
@@ -4140,6 +4151,7 @@ def definition_reference_schema(
     'datetime',
     'timedelta',
     'literal',
+    'missing-sentinel',
     'enum',
     'is-instance',
     'is-subclass',
@@ -4239,6 +4251,7 @@ def definition_reference_schema(
     'value_error',
     'assertion_error',
     'literal_error',
+    'missing_sentinel_error',
     'date_type',
     'date_parsing',
     'date_from_datetime_parsing',

src/common/missing_sentinel.rs

@@ -0,0 +1,16 @@
+use pyo3::intern;
+use pyo3::prelude::*;
+use pyo3::sync::GILOnceCell;
+
+static MISSING_SENTINEL_OBJECT: GILOnceCell<Py<PyAny>> = GILOnceCell::new();
+
+pub fn get_missing_sentinel_object(py: Python) -> &Bound<'_, PyAny> {
+    MISSING_SENTINEL_OBJECT
+        .get_or_init(py, || {
+            py.import(intern!(py, "pydantic_core"))
+                .and_then(|core_module| core_module.getattr(intern!(py, "MISSING")))
+                .unwrap()
+                .into()
+        })
+        .bind(py)
+}

src/common/mod.rs

@@ -1,2 +1,3 @@
+pub(crate) mod missing_sentinel;
 pub(crate) mod prebuilt;
 pub(crate) mod union;

src/errors/types.rs

@@ -316,6 +316,8 @@ error_types! {
         expected: {ctx_type: String, ctx_fn: field_from_context},
     },
     // ---------------------
+    // missing sentinel
+    MissingSentinelError {},
     // date errors
     DateType {},
     DateParsing {
@@ -531,6 +533,7 @@ impl ErrorType {
             Self::AssertionError {..} => "Assertion failed, {error}",
             Self::CustomError {..} => "",  // custom errors are handled separately
             Self::LiteralError {..} => "Input should be {expected}",
+            Self::MissingSentinelError { .. } => "Input should be the 'MISSING' sentinel",
             Self::DateType {..} => "Input should be a valid date",
             Self::DateParsing {..} => "Input should be a valid date in the format YYYY-MM-DD, {error}",
             Self::DateFromDatetimeParsing {..} => "Input should be a valid date or datetime, {error}",

src/serializers/computed_fields.rs

@@ -4,6 +4,7 @@ use pyo3::{intern, PyTraverseError, PyVisit};
 use serde::ser::SerializeMap;
 
 use crate::build_tools::py_schema_error_type;
+use crate::common::missing_sentinel::get_missing_sentinel_object;
 use crate::definitions::DefinitionsBuilder;
 use crate::py_gc::PyGcTraverse;
 use crate::serializers::filter::SchemaFilter;
@@ -148,6 +149,10 @@ impl ComputedFields {
             if extra.exclude_none && value.is_none() {
                 continue;
             }
+            let missing_sentinel = get_missing_sentinel_object(model.py());
+            if value.is(missing_sentinel) {
+                continue;
+            }
 
             let field_extra = Extra {
                 field_name: Some(&computed_field.property_name),

src/serializers/fields.rs

@@ -7,6 +7,7 @@ use ahash::AHashMap;
 use serde::ser::SerializeMap;
 use smallvec::SmallVec;
 
+use crate::common::missing_sentinel::get_missing_sentinel_object;
 use crate::serializers::extra::SerCheck;
 use crate::PydanticSerializationUnexpectedValue;
 
@@ -15,8 +16,7 @@ use super::errors::py_err_se_err;
 use super::extra::Extra;
 use super::filter::SchemaFilter;
 use super::infer::{infer_json_key, infer_serialize, infer_to_python, SerializeInfer};
-use super::shared::PydanticSerializer;
-use super::shared::{CombinedSerializer, TypeSerializer};
+use super::shared::{CombinedSerializer, PydanticSerializer, TypeSerializer};
 
 /// representation of a field for serialization
 #[derive(Debug)]
@@ -169,6 +169,7 @@ impl GeneralFieldsSerializer {
     ) -> PyResult<Bound<'py, PyDict>> {
         let output_dict = PyDict::new(py);
         let mut used_req_fields: usize = 0;
+        let missing_sentinel = get_missing_sentinel_object(py);
 
         // NOTE! we maintain the order of the input dict assuming that's right
         for result in main_iter {
@@ -178,6 +179,10 @@ impl GeneralFieldsSerializer {
             if extra.exclude_none && value.is_none() {
                 continue;
             }
+            if value.is(missing_sentinel) {
+                continue;
+            }
+
             let field_extra = Extra {
                 field_name: Some(key_str),
                 ..extra
@@ -253,9 +258,13 @@ impl GeneralFieldsSerializer {
 
         for result in main_iter {
             let (key, value) = result.map_err(py_err_se_err)?;
+            let missing_sentinel = get_missing_sentinel_object(value.py());
             if extra.exclude_none && value.is_none() {
                 continue;
             }
+            if value.is(missing_sentinel) {
+                continue;
+            }
             let key_str = key_str(&key).map_err(py_err_se_err)?;
             let field_extra = Extra {
                 field_name: Some(key_str),
@@ -347,6 +356,7 @@ impl TypeSerializer for GeneralFieldsSerializer {
         extra: &Extra,
     ) -> PyResult<PyObject> {
         let py = value.py();
+        let missing_sentinel = get_missing_sentinel_object(py);
         // If there is already a model registered (from a dataclass, BaseModel)
         // then do not touch it
         // If there is no model, we (a TypedDict) are the model
@@ -368,6 +378,9 @@ impl TypeSerializer for GeneralFieldsSerializer {
                 if extra.exclude_none && value.is_none() {
                     continue;
                 }
+                if value.is(missing_sentinel) {
+                    continue;
+                }
                 if let Some((next_include, next_exclude)) = self.filter.key_filter(&key, include, exclude)? {
                     let value = match &self.extra_serializer {
                         Some(serializer) => {
@@ -401,7 +414,7 @@ impl TypeSerializer for GeneralFieldsSerializer {
             extra.warnings.on_fallback_ser::<S>(self.get_name(), value, extra)?;
             return infer_serialize(value, serializer, include, exclude, extra);
         };
-
+        let missing_sentinel = get_missing_sentinel_object(value.py());
         // If there is already a model registered (from a dataclass, BaseModel)
         // then do not touch it
         // If there is no model, we (a TypedDict) are the model
@@ -428,6 +441,9 @@ impl TypeSerializer for GeneralFieldsSerializer {
                 if extra.exclude_none && value.is_none() {
                     continue;
                 }
+                if value.is(missing_sentinel) {
+                    continue;
+                }
                 let filter = self.filter.key_filter(&key, include, exclude).map_err(py_err_se_err)?;
                 if let Some((next_include, next_exclude)) = filter {
                     let output_key = infer_json_key(&key, extra).map_err(py_err_se_err)?;

src/serializers/shared.rs

@@ -142,6 +142,7 @@ combined_serializer! {
         Union: super::type_serializers::union::UnionSerializer;
         TaggedUnion: super::type_serializers::union::TaggedUnionSerializer;
         Literal: super::type_serializers::literal::LiteralSerializer;
+        MissingSentinel: super::type_serializers::missing_sentinel::MissingSentinelSerializer;
         Enum: super::type_serializers::enum_::EnumSerializer;
         Recursive: super::type_serializers::definitions::DefinitionRefSerializer;
         Tuple: super::type_serializers::tuple::TupleSerializer;
@@ -343,6 +344,7 @@ impl PyGcTraverse for CombinedSerializer {
             CombinedSerializer::Union(inner) => inner.py_gc_traverse(visit),
             CombinedSerializer::TaggedUnion(inner) => inner.py_gc_traverse(visit),
             CombinedSerializer::Literal(inner) => inner.py_gc_traverse(visit),
+            CombinedSerializer::MissingSentinel(inner) => inner.py_gc_traverse(visit),
             CombinedSerializer::Enum(inner) => inner.py_gc_traverse(visit),
             CombinedSerializer::Recursive(inner) => inner.py_gc_traverse(visit),
             CombinedSerializer::Tuple(inner) => inner.py_gc_traverse(visit),

src/serializers/type_serializers/missing_sentinel.rs

@@ -0,0 +1,76 @@
+// This serializer is defined so that building a schema serializer containing an
+// 'missing-sentinel' core schema doesn't crash. In practice, the serializer isn't
+// used for model-like classes, as the 'fields' serializer takes care of omitting
+// the fields from the output (the serializer can still be used if the 'missing-sentinel'
+// core schema is used standalone (e.g. with a Pydantic type adapter), but this isn't
+// something we explicitly support.
+
+use std::borrow::Cow;
+
+use pyo3::prelude::*;
+use pyo3::types::PyDict;
+
+use serde::ser::Error;
+
+use crate::common::missing_sentinel::get_missing_sentinel_object;
+use crate::definitions::DefinitionsBuilder;
+use crate::PydanticSerializationUnexpectedValue;
+
+use super::{BuildSerializer, CombinedSerializer, Extra, TypeSerializer};
+
+#[derive(Debug)]
+pub struct MissingSentinelSerializer {}
+
+impl BuildSerializer for MissingSentinelSerializer {
+    const EXPECTED_TYPE: &'static str = "missing-sentinel";
+
+    fn build(
+        _schema: &Bound<'_, PyDict>,
+        _config: Option<&Bound<'_, PyDict>>,
+        _definitions: &mut DefinitionsBuilder<CombinedSerializer>,
+    ) -> PyResult<CombinedSerializer> {
+        Ok(Self {}.into())
+    }
+}
+
+impl_py_gc_traverse!(MissingSentinelSerializer {});
+
+impl TypeSerializer for MissingSentinelSerializer {
+    fn to_python(
+        &self,
+        value: &Bound<'_, PyAny>,
+        _include: Option<&Bound<'_, PyAny>>,
+        _exclude: Option<&Bound<'_, PyAny>>,
+        _extra: &Extra,
+    ) -> PyResult<PyObject> {
+        let missing_sentinel = get_missing_sentinel_object(value.py());
+
+        if value.is(missing_sentinel) {
+            Ok(missing_sentinel.to_owned().into())
+        } else {
+            Err(
+                PydanticSerializationUnexpectedValue::new_from_msg(Some("Expected 'MISSING' sentinel".to_string()))
+                    .to_py_err(),
+            )
+        }
+    }
+
+    fn json_key<'a>(&self, key: &'a Bound<'_, PyAny>, extra: &Extra) -> PyResult<Cow<'a, str>> {
+        self.invalid_as_json_key(key, extra, Self::EXPECTED_TYPE)
+    }
+
+    fn serde_serialize<S: serde::ser::Serializer>(
+        &self,
+        _value: &Bound<'_, PyAny>,
+        _serializer: S,
+        _include: Option<&Bound<'_, PyAny>>,
+        _exclude: Option<&Bound<'_, PyAny>>,
+        _extra: &Extra,
+    ) -> Result<S::Ok, S::Error> {
+        Err(Error::custom("'MISSING' can't be serialized to JSON".to_string()))
+    }
+
+    fn get_name(&self) -> &str {
+        Self::EXPECTED_TYPE
+    }
+}

src/serializers/type_serializers/mod.rs

@@ -15,6 +15,7 @@ pub mod json;
 pub mod json_or_python;
 pub mod list;
 pub mod literal;
+pub mod missing_sentinel;
 pub mod model;
 pub mod nullable;
 pub mod other;