Support system tests otel input

internal/testrunner/runners/system/tester.go

@@ -145,6 +145,7 @@ var (
 type fieldValidationMethod int
 
 const (
+	// Required to allow setting `fields` as an option via environment variable
 	fieldsMethod fieldValidationMethod = iota
 	mappingsMethod
 )
@@ -1013,6 +1014,8 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, stackC
 	}
 	scenario.policyTemplateInput = policyTemplate.Input
 
+	scenario.policyTemplateInput = policyTemplate.Input
+
 	policyToEnrollOrCurrent, policyToTest, err := r.createOrGetKibanaPolicies(ctx, serviceStateData, stackConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create kibana policies: %w", err)
@@ -1614,14 +1617,18 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re
 		return result.WithErrorf("creating fields validator for data stream failed (path: %s): %w", r.dataStreamPath, err)
 	}
 
-	if errs := validateFields(scenario.docs, fieldsValidator); len(errs) > 0 {
-		return result.WithError(testrunner.ErrTestCaseFailed{
-			Reason:  fmt.Sprintf("one or more errors found in documents stored in %s data stream", scenario.dataStream),
-			Details: errs.Error(),
-		})
+	if r.isTestUsingOTELCollectorInput(scenario) {
+		logger.Debug("Skip field validation for OTEL Collector input")
+	} else {
+		if errs := validateFields(scenario.docs, fieldsValidator); len(errs) > 0 {
+			return result.WithError(testrunner.ErrTestCaseFailed{
+				Reason:  fmt.Sprintf("one or more errors found in documents stored in %s data stream", scenario.dataStream),
+				Details: errs.Error(),
+			})
+		}
 	}
 
-	if r.fieldValidationMethod == mappingsMethod {
+	if r.isTestUsingOTELCollectorInput(scenario) || r.fieldValidationMethod == mappingsMethod {
 		logger.Debug("Performing validation based on mappings")
 		exceptionFields := listExceptionFields(scenario.docs, fieldsValidator)
 
@@ -1797,6 +1804,19 @@ func (r *tester) runTest(ctx context.Context, config *testConfig, stackConfig st
 	return r.validateTestScenario(ctx, result, scenario, config)
 }
 
+func (r *tester) isTestUsingOTELCollectorInput(scenario *scenarioTest) bool {
+	// Just supported for input packages currently
+	if r.pkgManifest.Type != "input" {
+		return false
+	}
+
+	if scenario.policyTemplateInput != otelCollectorInputName {
+		return false
+	}
+
+	return true
+}
+
 func dumpScenarioDocs(docs any) error {
 	timestamp := time.Now().Format("20060102150405")
 	path := filepath.Join(os.TempDir(), fmt.Sprintf("elastic-package-test-docs-dump-%s.json", timestamp))
@@ -1960,16 +1980,14 @@ func createInputPackageDatastream(
 			PolicyTemplate: policyTemplate.Name,
 			Enabled:        true,
 			Vars:           kibana.Vars{},
+			Type:           policyTemplate.Input,
 		},
 	}
 
-	streamInput := policyTemplate.Input
-	r.Inputs[0].Type = streamInput
-
 	dataset := fmt.Sprintf("%s.%s", pkg.Name, policyTemplate.Name)
 	streams := []kibana.Stream{
 		{
-			ID:      fmt.Sprintf("%s-%s.%s", streamInput, pkg.Name, policyTemplate.Name),
+			ID:      fmt.Sprintf("%s-%s.%s", policyTemplate.Input, pkg.Name, policyTemplate.Name),
 			Enabled: true,
 			DataStream: kibana.DataStream{
 				Type:    policyTemplate.Type,

test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml

@@ -1,6 +1,6 @@
-skip:
-  reason: Not supported system tests with input type otelcol.
-  link: https://github.com/elastic/elastic-package/issues/2835
+# skip:
+#   reason: Not supported system tests with input type otelcol.
+#   link: https://github.com/elastic/elastic-package/issues/2835
 service: web
 vars:
   period: 1s

test/packages/parallel/httpcheck/fields/fields.yml

@@ -0,0 +1,49 @@
+# - name: "_metric_names_hash"
+#   type: keyword
+# - name: "attributes.http"
+#   type: group
+#   fields:
+#     - name: method
+#       type: keyword
+#     - name: status_class
+#       type: keyword
+#     - name: status_code
+#       type: long
+#     - name: url
+#       type: keyword
+- name: "http"
+  type: group
+  fields:
+    - name: method
+      type: keyword
+    - name: status_class
+      type: keyword
+    - name: status_code
+      type: long
+    - name: url
+      type: keyword
+- name: httpcheck
+  type: group
+  fields:
+    - name: duration
+      type: long
+    - name: status
+      type: long
+# - name: metrics.httpcheck
+#   type: group
+#   fields:
+#     - name: duration
+#       type: long
+#     - name: status
+#       type: long
+# - name: scope
+#   type: group
+#   fields:
+#     - name: name
+#       type: keyword
+#     - name: version
+#       type: keyword
+# - name: start_timestamp
+#   type: date
+# - name: unit
+#   type: keyword