fix(core/protocols): support for $unknown union members in schema-serde (#1820)

Author: kuhe

.changeset/chatty-dogs-rule.md

@@ -0,0 +1,6 @@
+---
+"@smithy/types": minor
+"@smithy/core": minor
+---
+
+add static union schema as a new type

packages/core/src/submodules/cbor/CborCodec.spec.ts

@@ -1,5 +1,11 @@
 import { NormalizedSchema } from "@smithy/core/schema";
-import type { StaticSimpleSchema, StaticStructureSchema, StringSchema, TimestampDefaultSchema } from "@smithy/types";
+import type {
+  StaticSimpleSchema,
+  StaticStructureSchema,
+  StaticUnionSchema,
+  StringSchema,
+  TimestampDefaultSchema,
+} from "@smithy/types";
 import { describe, expect, it } from "vitest";
 
 import { cbor } from "./cbor";
@@ -94,6 +100,32 @@ describe(CborShapeSerializer.name, () => {
         },
       });
     });
+
+    it("can serialize the $unknown union convention", async () => {
+      const schema = [
+        3,
+        "ns",
+        "Struct",
+        0,
+        ["union"],
+        [[4, "ns", "Union", 0, ["a", "b", "c"], [0, 0, 0]] satisfies StaticUnionSchema],
+      ] satisfies StaticStructureSchema;
+
+      const ns = NormalizedSchema.of(schema);
+      const input = {
+        union: {
+          $unknown: ["d", {}],
+        },
+      };
+      serializer.write(ns, input);
+      const serialization = serializer.flush();
+      const objectEquivalent = cbor.deserialize(serialization);
+      expect(objectEquivalent).toEqual({
+        union: {
+          d: {},
+        },
+      });
+    });
   });
 
   describe("deserialization", () => {
@@ -128,5 +160,30 @@ describe(CborShapeSerializer.name, () => {
         timestamp: new Date(1),
       });
     });
+
+    it("deserializes unknown union members to the $unknown conventional property", async () => {
+      const schema = [
+        3,
+        "ns",
+        "Struct",
+        0,
+        ["union"],
+        [[4, "ns", "Union", 0, ["a", "b", "c"], [0, 0, 0]] satisfies StaticUnionSchema],
+      ] satisfies StaticStructureSchema;
+      const ns = NormalizedSchema.of(schema);
+      const receivedData = {
+        union: {
+          __type: "ns.Union",
+          d: {},
+        },
+      };
+      const serialization = cbor.serialize(receivedData);
+      const deserialized = await deserializer.read(ns, serialization);
+      expect(deserialized).toEqual({
+        union: {
+          $unknown: ["d", {}],
+        },
+      } satisfies Record<string, unknown>);
+    });
   });
 });

packages/core/src/submodules/cbor/CborCodec.ts

@@ -96,6 +96,11 @@ export class CborShapeSerializer extends SerdeContext implements ShapeSerializer
             newObject[key] = value;
           }
         }
+        const isUnion = ns.isUnionSchema();
+        if (isUnion && Array.isArray(sourceObject.$unknown)) {
+          const [k, v] = sourceObject.$unknown;
+          newObject[k] = v;
+        }
       } else if (ns.isDocumentSchema()) {
         for (const key of Object.keys(sourceObject)) {
           newObject[key] = this.serialize(ns.getValueSchema(), sourceObject[key]);
@@ -199,11 +204,22 @@ export class CborShapeDeserializer extends SerdeContext implements ShapeDeserial
           }
         }
       } else if (ns.isStructSchema()) {
+        const isUnion = ns.isUnionSchema();
+        let keys: Set<string> | undefined;
+        if (isUnion) {
+          keys = new Set(Object.keys(value).filter((k) => k !== "__type"));
+        }
         for (const [key, memberSchema] of ns.structIterator()) {
-          const v = this.readValue(memberSchema, value[key]);
-          if (v != null) {
-            newObject[key] = v;
+          if (isUnion) {
+            keys!.delete(key);
           }
+          if (value[key] != null) {
+            newObject[key] = this.readValue(memberSchema, value[key]);
+          }
+        }
+        if (isUnion && keys?.size === 1 && Object.keys(newObject).length === 0) {
+          const k = keys!.values().next().value as string;
+          newObject.$unknown = [k, value[k]];
         }
       }
       return newObject;

packages/core/src/submodules/schema/schemas/NormalizedSchema.spec.ts

@@ -12,6 +12,7 @@ import type {
   StaticMapSchema,
   StaticSimpleSchema,
   StaticStructureSchema,
+  StaticUnionSchema,
   StreamingBlobSchema,
   StringSchema,
   TimestampDefaultSchema,
@@ -22,12 +23,25 @@ import { NormalizedSchema } from "./NormalizedSchema";
 import { translateTraits } from "./translateTraits";
 
 describe(NormalizedSchema.name, () => {
-  const [List, Map, Struct]: [StaticListSchema, StaticMapSchema, () => StaticStructureSchema] = [
+  const [List, Map, Struct, Union]: [
+    StaticListSchema,
+    StaticMapSchema,
+    () => StaticStructureSchema,
+    StaticUnionSchema,
+  ] = [
     [1, "ack", "List", { sparse: 1 }, 0] satisfies StaticListSchema,
     [2, "ack", "Map", 0, 0, 1] satisfies StaticMapSchema,
     () => schema,
+    [4, "ack", "Union", 0, ["a", "b", "c"], ["unit", 0, 128]],
+  ];
+  const schema: StaticStructureSchema = [
+    3,
+    "ack",
+    "Structure",
+    {},
+    ["list", "map", "struct", "union"],
+    [List, Map, Struct, Union],
   ];
-  const schema: StaticStructureSchema = [3, "ack", "Structure", {}, ["list", "map", "struct"], [List, Map, Struct]];
 
   const ns = NormalizedSchema.of(schema);
   const nsFromIndirect = NormalizedSchema.of(() => ns);
@@ -200,6 +214,22 @@ describe(NormalizedSchema.name, () => {
         );
       });
     });
+    describe("union member", () => {
+      it("is a union and a struct", () => {
+        const member = ns.getMemberSchema("union");
+        expect(member.getName(true)).toBe("ack#Union");
+        expect(member.isMemberSchema()).toBe(true);
+        expect(member.isListSchema()).toBe(false);
+        expect(member.isMapSchema()).toBe(false);
+        expect(member.isStructSchema()).toBe(true);
+        expect(member.isUnionSchema()).toBe(true);
+        expect(member.getMemberName()).toBe("union");
+
+        expect(member.getMemberSchema("a").isUnitSchema()).toBe(true);
+        expect(member.getMemberSchema("b").isStringSchema()).toBe(true);
+        expect(member.getMemberSchema("c").isMapSchema()).toBe(true);
+      });
+    });
   });
 
   describe("iteration", () => {

packages/core/src/submodules/schema/schemas/NormalizedSchema.ts

@@ -22,6 +22,7 @@ import type {
   StaticSchemaIdList,
   StaticSchemaIdMap,
   StaticSchemaIdStruct,
+  StaticSchemaIdUnion,
   StaticSimpleSchema,
   StaticStructureSchema,
   StreamingBlobSchema,
@@ -194,14 +195,26 @@ export class NormalizedSchema implements INormalizedSchema {
       : (sc as StaticSchema)[0] === (2 satisfies StaticSchemaIdMap);
   }
 
+  /**
+   * To simplify serialization logic, static union schemas are considered a specialization
+   * of structs in the TypeScript typings and JS runtime, as well as static error schemas
+   * which have a different identifier.
+   */
   public isStructSchema(): boolean {
     const sc = this.getSchema();
+    const id = (sc as StaticSchema)[0];
     return (
-      (sc as StaticSchema)[0] === (3 satisfies StaticSchemaIdStruct) ||
-      (sc as StaticSchema)[0] === (-3 satisfies StaticSchemaIdError)
+      id === (3 satisfies StaticSchemaIdStruct) ||
+      id === (-3 satisfies StaticSchemaIdError) ||
+      id === (4 satisfies StaticSchemaIdUnion)
     );
   }
 
+  public isUnionSchema(): boolean {
+    const sc = this.getSchema();
+    return (sc as StaticSchema)[0] === (4 satisfies StaticSchemaIdUnion);
+  }
+
   public isBlobSchema(): boolean {
     const sc = this.getSchema();
     return sc === (21 satisfies BlobSchema) || sc === (42 satisfies StreamingBlobSchema);

packages/types/src/schema/static-schemas.ts

@@ -25,6 +25,11 @@ export type StaticSchemaIdMap = 2;
  */
 export type StaticSchemaIdStruct = 3;
 
+/**
+ * @public
+ */
+export type StaticSchemaIdUnion = 4;
+
 /**
  * @public
  */
@@ -43,6 +48,7 @@ export type StaticSchema =
   | StaticListSchema
   | StaticMapSchema
   | StaticStructureSchema
+  | StaticUnionSchema
   | StaticErrorSchema
   | StaticOperationSchema;
 
@@ -83,6 +89,18 @@ export type StaticStructureSchema = [
   $SchemaRef[], // member schema list
 ];
 
+/**
+ * @public
+ */
+export type StaticUnionSchema = [
+  StaticSchemaIdUnion,
+  ShapeNamespace,
+  ShapeName,
+  SchemaTraits,
+  string[], // member name list
+  $SchemaRef[], // member schema list
+];
+
 /**
  * @public
  */