Fix entity generation

Makefile

@@ -40,6 +40,15 @@ check_license_headers:
 .PHONY: clean
 clean:
 	rm -rf build generated/elasticsearch/composable/component experimental/generated/elasticsearch/composable/component
+	# Clean generated documentation files
+	@echo "Removing generated documentation files..."
+	@rm -f docs/reference/index.md docs/reference/ecs-field-reference.md docs/reference/ecs-otel-alignment-details.md docs/reference/ecs-otel-alignment-overview.md
+	@for schema in $$(ls schemas/*.yml 2>/dev/null | sed 's/schemas\///' | sed 's/\.yml$$//'); do \
+		if [ -f "docs/reference/ecs-$$schema.md" ]; then \
+			echo "Removing docs/reference/ecs-$$schema.md"; \
+			rm -f "docs/reference/ecs-$$schema.md"; \
+		fi; \
+	done
 
 # Build and serve the docs
 .PHONY: docs
@@ -107,7 +116,7 @@ generate: generator
 # Run the new generator
 .PHONY: generator
 generator: ve
-	$(PYTHON) scripts/generator.py --strict --include "${INCLUDE}" --subset "${SUBSETS_DIR}" --semconv-version "${SEMCONV_VERSION}" --force-docs
+	$(PYTHON) scripts/generator.py --strict $(if $(INCLUDE),--include "$(INCLUDE)") --subset "${SUBSETS_DIR}" --semconv-version "${SEMCONV_VERSION}" --force-docs
 
 # Check Makefile format.
 .PHONY: makelint

docs/reference/ecs-entity.md

@@ -17,16 +17,17 @@ The entity fields provide a standardized way to represent and categorize differe
 | --- | --- | --- |
 | $$$field-entity-attributes$$$ [entity.attributes](#field-entity-attributes) | _This field is beta and subject to change._ A set of static or semi-static attributes of the entity. Usually boolean or keyword field data types. Use this field set when you need to track static or semi-static characteristics of an entity for advanced searching and correlation of normalized values across different providers/sources and entity types.<br><br>type: object | extended |
 | $$$field-entity-behavior$$$ [entity.behavior](#field-entity-behavior) | _This field is beta and subject to change._ A set of ephemeral characteristics of the entity, derived from observed behaviors during a specific time period. Usually boolean field data type. Use this field set when you need to capture and track ephemeral characteristics of an entity for advanced searching, correlation of normalized values across different providers/sources and entity types.<br><br>type: object | extended |
-| $$$field-entity-display_name$$$ [entity.display_name](#field-entity-display_name) | _This field is beta and subject to change._ An optional field used when a pretty name is desired for entity-centric operations. This field should not be used for correlation with `*.name` fields for entities with dedicated field sets (e.g., `host`).<br><br>type: keyword<br><br>Multi-fields:<br><br>* entity.display_name.text (type: text) | extended |
+| $$$field-entity-display-name$$$ [entity.display_name](#field-entity-display-name) | _This field is beta and subject to change._ An optional field used when a pretty name is desired for entity-centric operations. This field should not be used for correlation with `*.name` fields for entities with dedicated field sets (e.g., `host`).<br><br>type: keyword<br><br>Multi-fields:<br><br>* entity.display_name.text (type: text) | extended |
 | $$$field-entity-id$$$ [entity.id](#field-entity-id) | A unique identifier for the entity. When multiple identifiers exist, this should be the most stable and commonly used identifier that: 1) persists across the entity's lifecycle, 2) ensures uniqueness within its scope, 3) is commonly used for queries and correlation, and 4) is readily available in most observations (logs/events). For entities with dedicated field sets (e.g., host, user), this value should match the corresponding *.id field. Alternative identifiers (e.g., ARNs values in AWS, URLs) can be preserved in the raw field.<br><br>type: keyword | core |
-| $$$field-entity-last_seen_timestamp$$$ [entity.last_seen_timestamp](#field-entity-last_seen_timestamp) | _This field is beta and subject to change._ Indicates the date/time when this entity was last "seen," usually based upon the last event/log that is initiated by this entity.<br><br>type: date | extended |
+| $$$field-entity-last-seen-timestamp$$$ [entity.last_seen_timestamp](#field-entity-last-seen-timestamp) | _This field is beta and subject to change._ Indicates the date/time when this entity was last "seen," usually based upon the last event/log that is initiated by this entity.<br><br>type: date | extended |
 | $$$field-entity-lifecycle$$$ [entity.lifecycle](#field-entity-lifecycle) | _This field is beta and subject to change._ A set of temporal characteristics of the entity. Usually date field data type. Use this field set when you need to track temporal characteristics of an entity for advanced searching and correlation of normalized values across different providers/sources and entity types.<br><br>type: object | extended |
 | $$$field-entity-metrics$$$ [entity.metrics](#field-entity-metrics) | _This field is beta and subject to change._ Field set for any fields containing numeric entity metrics. These use dynamic field data type mapping.<br><br>type: object | extended |
 | $$$field-entity-name$$$ [entity.name](#field-entity-name) | _This field is beta and subject to change._ The name of the entity. The keyword field enables exact matches for filtering and aggregations, while the text field enables full-text search. For entities with dedicated field sets (e.g., `host`), this field should mirrors the corresponding *.name value.<br><br>type: keyword<br><br>Multi-fields:<br><br>* entity.name.text (type: text) | core |
 | $$$field-entity-raw$$$ [entity.raw](#field-entity-raw) | _This field is beta and subject to change._ Original, unmodified fields from the source system. Usually flattened field data type. While the attributes field should be used for normalized fields requiring advanced queries, this field preserves all source metadata with basic search capabilities.<br><br>type: object | extended |
 | $$$field-entity-reference$$$ [entity.reference](#field-entity-reference) | _This field is beta and subject to change._ A URI, URL, or other direct reference to access or locate the entity in its source system. This could be an API endpoint, web console URL, or other addressable location. Format may vary by entity type and source system.<br><br>type: keyword | extended |
 | $$$field-entity-source$$$ [entity.source](#field-entity-source) | _This field is beta and subject to change._ The module or integration that provided this entity data (similar to event.module).<br><br>type: keyword | core |
-| $$$field-entity-type$$$ [entity.type](#field-entity-type) | _This field is beta and subject to change._ A standardized high-level classification of the entity. This provides a normalized way to group similar entities across different providers or systems. Example values: `bucket`, `database`, `container`, `function`, `queue`, `host`, `user`, `application`, `session`, etc.<br><br>type: keyword<br><br>example: `host` | core |
+| $$$field-entity-sub-type$$$ [entity.sub_type](#field-entity-sub-type) | _This field is beta and subject to change._ The specific type designation for the entity as defined by its provider or system. This field provides more granular classification than the type field. Examples: `aws_s3_bucket`, `gcp_cloud_storage_bucket`, `azure_blob_container` would all map to entity type `bucket`.  `hardware` , `virtual` , `container` , `node` , `cloud_instance` would all map to entity type `host`.<br><br>type: keyword<br><br>example: `aws_s3_bucket` | extended |
+| $$$field-entity-type$$$ [entity.type](#field-entity-type) | _This field is beta and subject to change._ A standardized high-level classification of the entity. This provides a normalized way to group similar entities across different providers or systems. Example values: `bucket`, `database`, `container`, `function`, `queue`, `host`, `user`, `application`, `session`, etc.<br><br>type: keyword<br><br>Note: This field should contain an array of values.<br><br>**Important:** The field value must be one of the following:<br><br>bucket, database, container, function, queue, host, user, application, service, session<br><br>To learn more about when to use which value, visit the page [allowed values for entity.type](/reference/ecs-allowed-values-entity-type.md)<br> | core |
 
 ## Field reuse [_field_reuse]
 

docs/reference/ecs-field-reference.md

@@ -38,6 +38,7 @@ For a single page representation of all fields, please see the [generated CSV of
 | [ECS](/reference/ecs-ecs.md) | Meta-information specific to ECS. |
 | [ELF Header](/reference/ecs-elf.md) | These fields contain Linux Executable Linkable Format (ELF) metadata. |
 | [Email](/reference/ecs-email.md) | Describes an email transaction. |
+| [Entity](/reference/ecs-entity.md) | Fields to describe various types of entities across IT environments. |
 | [Error](/reference/ecs-error.md) | Fields about errors of any kind. |
 | [Event](/reference/ecs-event.md) | Fields breaking down the event details. |
 | [FaaS](/reference/ecs-faas.md) | Fields describing functions as a service. |

docs/reference/ecs-otel-alignment-overview.md

@@ -48,6 +48,7 @@ The following table summarizes the alignment status by namespaces between ECS in
 | ELF Header | [38](/reference/ecs-elf.md) | · | · | · | · | · | · | · | · |
 | Email | [19](/reference/ecs-email.md) | · | · | · | · | · | · | · | · |
 | End User | · | [2](https://opentelemetry.io/docs/specs/semconv/attributes-registry/enduser) | · | · | · | · | · | · |  |
+| Entity | [13](/reference/ecs-entity.md) | · | · | · | · | · | · | · | · |
 | Error | [5](/reference/ecs-error.md) | [2](https://opentelemetry.io/docs/specs/semconv/attributes-registry/error) | 1 | 2 | · | · | · | · | · |
 | Event | [26](/reference/ecs-event.md) | · | · | · | · | · | · | · | · |
 | Exception | · | [3](https://opentelemetry.io/docs/specs/semconv/attributes-registry/exception) | · | · | · | · | · | · |  |

experimental/generated/beats/fields.ecs.yml

@@ -2188,6 +2188,137 @@
         original email message.
       example: Spambot v2.5
       default_field: false
+  - name: entity
+    title: Entity
+    group: 2
+    description: The entity fields provide a standardized way to represent and categorize
+      different types of components within an IT environment, including those that
+      don't have dedicated field sets in ECS. An entity represents a discrete, identifiable
+      component that can be described by a set of attributes and maintains its identity
+      over time.
+    type: group
+    default_field: true
+    fields:
+    - name: attributes
+      level: extended
+      type: object
+      description: A set of static or semi-static attributes of the entity. Usually
+        boolean or keyword field data types. Use this field set when you need to track
+        static or semi-static characteristics of an entity for advanced searching
+        and correlation of normalized values across different providers/sources and
+        entity types.
+      default_field: false
+    - name: behavior
+      level: extended
+      type: object
+      description: A set of ephemeral characteristics of the entity, derived from
+        observed behaviors during a specific time period. Usually boolean field data
+        type. Use this field set when you need to capture and track ephemeral characteristics
+        of an entity for advanced searching, correlation of normalized values across
+        different providers/sources and entity types.
+      default_field: false
+    - name: display_name
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      multi_fields:
+      - name: text
+        type: text
+        norms: false
+      description: An optional field used when a pretty name is desired for entity-centric
+        operations. This field should not be used for correlation with `*.name` fields
+        for entities with dedicated field sets (e.g., `host`).
+      default_field: false
+    - name: id
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: 'A unique identifier for the entity. When multiple identifiers
+        exist, this should be the most stable and commonly used identifier that: 1)
+        persists across the entity''s lifecycle, 2) ensures uniqueness within its
+        scope, 3) is commonly used for queries and correlation, and 4) is readily
+        available in most observations (logs/events). For entities with dedicated
+        field sets (e.g., host, user), this value should match the corresponding *.id
+        field. Alternative identifiers (e.g., ARNs values in AWS, URLs) can be preserved
+        in the raw field.'
+      default_field: false
+    - name: last_seen_timestamp
+      level: extended
+      type: date
+      description: Indicates the date/time when this entity was last "seen," usually
+        based upon the last event/log that is initiated by this entity.
+      default_field: false
+    - name: lifecycle
+      level: extended
+      type: object
+      description: A set of temporal characteristics of the entity. Usually date field
+        data type. Use this field set when you need to track temporal characteristics
+        of an entity for advanced searching and correlation of normalized values across
+        different providers/sources and entity types.
+      default_field: false
+    - name: metrics
+      level: extended
+      type: object
+      description: Field set for any fields containing numeric entity metrics. These
+        use dynamic field data type mapping.
+      default_field: false
+    - name: name
+      level: core
+      type: keyword
+      ignore_above: 1024
+      multi_fields:
+      - name: text
+        type: text
+        norms: false
+      description: The name of the entity. The keyword field enables exact matches
+        for filtering and aggregations, while the text field enables full-text search.
+        For entities with dedicated field sets (e.g., `host`), this field should mirrors
+        the corresponding *.name value.
+      default_field: false
+    - name: raw
+      level: extended
+      type: object
+      description: Original, unmodified fields from the source system. Usually flattened
+        field data type. While the attributes field should be used for normalized
+        fields requiring advanced queries, this field preserves all source metadata
+        with basic search capabilities.
+      default_field: false
+    - name: reference
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: A URI, URL, or other direct reference to access or locate the entity
+        in its source system. This could be an API endpoint, web console URL, or other
+        addressable location. Format may vary by entity type and source system.
+      default_field: false
+    - name: source
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: The module or integration that provided this entity data (similar
+        to event.module).
+      default_field: false
+    - name: sub_type
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'The specific type designation for the entity as defined by its
+        provider or system. This field provides more granular classification than
+        the type field. Examples: `aws_s3_bucket`, `gcp_cloud_storage_bucket`, `azure_blob_container`
+        would all map to entity type `bucket`.  `hardware` , `virtual` , `container`
+        , `node` , `cloud_instance` would all map to entity type `host`.'
+      example: aws_s3_bucket
+      default_field: false
+    - name: type
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: 'A standardized high-level classification of the entity. This provides
+        a normalized way to group similar entities across different providers or systems.
+        Example values: `bucket`, `database`, `container`, `function`, `queue`, `host`,
+        `user`, `application`, `session`, etc.'
+      example: host
+      default_field: false
   - name: error
     title: Error
     group: 2

experimental/generated/csv/fields.csv

@@ -260,6 +260,21 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.3.0-dev+exp,true,email,email.subject.text,match_only_text,extended,,Please see this important message.,The subject of the email message.
 9.3.0-dev+exp,true,email,email.to.address,keyword,extended,array,user1@example.com,Email address of recipient
 9.3.0-dev+exp,true,email,email.x_mailer,keyword,extended,,Spambot v2.5,Application that drafted email.
+9.3.0-dev+exp,true,entity,entity.attributes,object,extended,,,A set of static or semi-static attributes of the entity.
+9.3.0-dev+exp,true,entity,entity.behavior,object,extended,,,"A set of ephemeral characteristics of the entity, derived from observed behaviors during a specific time period."
+9.3.0-dev+exp,true,entity,entity.display_name,keyword,extended,,,An optional field used when a pretty name is desired for entity-centric operations.
+9.3.0-dev+exp,true,entity,entity.display_name.text,text,extended,,,An optional field used when a pretty name is desired for entity-centric operations.
+9.3.0-dev+exp,true,entity,entity.id,keyword,core,,,Unique identifier for the entity.
+9.3.0-dev+exp,true,entity,entity.last_seen_timestamp,date,extended,,,"Indicates the date/time when this entity was last ""seen."""
+9.3.0-dev+exp,true,entity,entity.lifecycle,object,extended,,,A set of temporal characteristics of the entity.
+9.3.0-dev+exp,true,entity,entity.metrics,object,extended,,,Field set for any fields containing numeric entity metrics.
+9.3.0-dev+exp,true,entity,entity.name,keyword,core,,,The name of the entity.
+9.3.0-dev+exp,true,entity,entity.name.text,text,core,,,The name of the entity.
+9.3.0-dev+exp,true,entity,entity.raw,object,extended,,,"Original, unmodified fields from the source system."
+9.3.0-dev+exp,true,entity,entity.reference,keyword,extended,,,"A URI, URL, or other direct reference to access or locate the entity."
+9.3.0-dev+exp,true,entity,entity.source,keyword,core,,,Source module or integration that provided the entity data.
+9.3.0-dev+exp,true,entity,entity.sub_type,keyword,extended,,aws_s3_bucket,The specific type designation for the entity as defined by its provider or system.
+9.3.0-dev+exp,true,entity,entity.type,keyword,core,array,host,Standardized high-level classification of the entity.
 9.3.0-dev+exp,true,error,error.code,keyword,core,,,Error code describing the error.
 9.3.0-dev+exp,true,error,error.id,keyword,core,,,Unique identifier for the error.
 9.3.0-dev+exp,true,error,error.message,match_only_text,core,,,Error message.