From 88f989eb68ceb146ab15aa059585a6f2291e142d Mon Sep 17 00:00:00 2001 From: Oleksandr Zhelezniak Date: Mon, 22 Sep 2025 11:09:45 +0000 Subject: [PATCH 1/3] feat(QTDI-1291): support guess schema as service --- .../sdk/component/server/service/PropertiesService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index a876e08d3308d..4436d66fb38a0 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -35,6 +35,7 @@ import javax.inject.Inject; import javax.json.bind.Jsonb; +import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.runtime.internationalization.ParameterBundle; import org.talend.sdk.component.runtime.manager.ParameterMeta; import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.ValidationParameterEnricher; @@ -94,6 +95,9 @@ private Stream buildProperties(final List Date: Tue, 23 Sep 2025 08:36:40 +0000 Subject: [PATCH 2/3] feat(QTDI-1291): extend error payload --- .../front/model/error/ErrorPayload.java | 6 ++++ .../server/front/ActionResourceImpl.java | 35 +++++++++++-------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java index eff31953100ec..ce2807bec22c7 100644 --- a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java +++ b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java @@ -28,5 +28,11 @@ public class ErrorPayload { private ErrorDictionary code; + private String subCode; + private String description; + + public ErrorPayload(final ErrorDictionary code, final String description) { + this(code, null, description); + } } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java index 9af4e05fa49fa..0cdd993a8e17c 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java @@ -42,6 +42,8 @@ import javax.ws.rs.core.Response; import org.talend.sdk.component.api.exception.ComponentException; +import org.talend.sdk.component.api.exception.ComponentException.ErrorOrigin; +import org.talend.sdk.component.api.exception.DiscoverSchemaException; import org.talend.sdk.component.runtime.manager.ComponentManager; import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry; import org.talend.sdk.component.runtime.manager.ServiceMeta; @@ -212,29 +214,34 @@ private CompletableFuture doExecuteLocalAction(final String family, fi private Response onError(final Throwable re) { log.warn(re.getMessage(), re); - if (WebApplicationException.class.isInstance(re.getCause())) { - return WebApplicationException.class.cast(re.getCause()).getResponse(); + if (re.getCause() instanceof WebApplicationException webException) { + return webException.getResponse(); } - if (ComponentException.class.isInstance(re)) { - final ComponentException ce = (ComponentException) re; + final String description = "Action execution failed with: " + ofNullable(re.getMessage()) + .orElseGet(() -> re instanceof NullPointerException + ? "unexpected null" + : "no error message"); + if (re instanceof final DiscoverSchemaException eSchema) { + // we send reason to recognize the error on client side + final String subCode = eSchema.getPossibleHandleErrorWith().toString(); throw new WebApplicationException(Response - .status(ce.getErrorOrigin() == ComponentException.ErrorOrigin.USER ? 400 - : ce.getErrorOrigin() == ComponentException.ErrorOrigin.BACKEND ? 456 : 520, + .status(400, subCode) + .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, subCode, description)) + .build()); + } else if (re instanceof final ComponentException eComponent) { + throw new WebApplicationException(Response + .status(eComponent.getErrorOrigin() == ErrorOrigin.USER + ? 400 + : eComponent.getErrorOrigin() == ErrorOrigin.BACKEND ? 456 : 520, "Unexpected callback error") - .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, - "Action execution failed with: " + ofNullable(re.getMessage()) - .orElseGet(() -> NullPointerException.class.isInstance(re) ? "unexpected null" - : "no error message"))) + .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, description)) .build()); } throw new WebApplicationException(Response .status(520, "Unexpected callback error") - .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, - "Action execution failed with: " + ofNullable(re.getMessage()) - .orElseGet(() -> NullPointerException.class.isInstance(re) ? "unexpected null" - : "no error message"))) + .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, description)) .build()); } From de0f9e607ae60d29d473a9cb00cb07459efcc2cd Mon Sep 17 00:00:00 2001 From: Oleksandr Zhelezniak Date: Tue, 23 Sep 2025 14:58:00 +0000 Subject: [PATCH 3/3] feat(QTDI-1291): clean up --- .../runtime/di/schema/TaCoKitGuessSchema.java | 309 +----------------- .../di/schema/TaCoKitGuessSchemaTest.java | 297 +---------------- .../runtime/di/schema/TypeConversionTest.java | 2 +- 3 files changed, 17 insertions(+), 591 deletions(-) diff --git a/component-studio/component-runtime-di/src/main/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchema.java b/component-studio/component-runtime-di/src/main/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchema.java index b8a93b5228fb9..3b7f6166aee06 100644 --- a/component-studio/component-runtime-di/src/main/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchema.java +++ b/component-studio/component-runtime-di/src/main/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchema.java @@ -17,10 +17,7 @@ import static java.lang.reflect.Modifier.isStatic; import static java.util.Optional.ofNullable; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; import static org.talend.sdk.component.api.exception.DiscoverSchemaException.HandleErrorWith.EXCEPTION; -import static org.talend.sdk.component.api.exception.DiscoverSchemaException.HandleErrorWith.EXECUTE_LIFECYCLE; import static org.talend.sdk.component.api.record.SchemaProperty.IS_KEY; import static org.talend.sdk.component.api.record.SchemaProperty.ORIGIN_TYPE; import static org.talend.sdk.component.api.record.SchemaProperty.PATTERN; @@ -36,10 +33,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.IntStream; @@ -65,12 +60,8 @@ import org.talend.sdk.component.runtime.di.OutputsHandler; import org.talend.sdk.component.runtime.input.Input; import org.talend.sdk.component.runtime.input.Mapper; -import org.talend.sdk.component.runtime.manager.ComponentFamilyMeta; import org.talend.sdk.component.runtime.manager.ComponentManager; import org.talend.sdk.component.runtime.manager.ComponentManager.AllServices; -import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry; -import org.talend.sdk.component.runtime.manager.ParameterMeta; -import org.talend.sdk.component.runtime.manager.ServiceMeta; import org.talend.sdk.component.runtime.manager.chain.ChainedMapper; import org.talend.sdk.component.runtime.manager.xbean.converter.SchemaConverter; import org.talend.sdk.component.runtime.output.InputFactory; @@ -84,23 +75,15 @@ public class TaCoKitGuessSchema { public static final String STRING_ESCAPE = "\""; - public static final String NO_COMPONENT = "No component "; - - public static final String TCOMP_CONFIGURATIONTYPE_TYPE = "tcomp::configurationtype::type"; - - public static final String DATASET = "dataset"; - public static final String ERROR_THROUGH_ACTION = "Can't guess schema through action."; public static final String ERROR_NO_AVAILABLE_SCHEMA_FOUND = "There is no available schema found."; - public static final String ERROR_INSTANCE_SCHEMA = "Result is not an instance of Talend Component Kit Schema."; - private static final String NO_COLUMN_FOUND_BY_GUESS_SCHEMA = "No column found by guess schema action"; private ComponentManager componentManager; - private JavaTypesManager javaTypesManager; + private final JavaTypesManager javaTypesManager; private PrintStream out; @@ -122,18 +105,12 @@ public class TaCoKitGuessSchema { private String componentName; - private String action; - private final Integer version; - private static final String SCHEMA_TYPE = "schema"; - - private static final String SCHEMA_EXTENDED_TYPE = "schema_extended"; - private static final String EMPTY = ""; //$NON-NLS-1$ public TaCoKitGuessSchema(final PrintStream out, final Map configuration, final String plugin, - final String family, final String componentName, final String action, final String version) { + final String family, final String componentName, final String version) { this.out = out; this.lineLimit = 50; this.lineCount = -1; @@ -143,7 +120,6 @@ public TaCoKitGuessSchema(final PrintStream out, final Map confi this.plugin = plugin; this.family = family; this.componentName = componentName; - this.action = action; this.columns = new LinkedHashMap<>(); this.keysNoTypeYet = new HashSet<>(); this.javaTypesManager = new JavaTypesManager(); @@ -159,7 +135,7 @@ public TaCoKitGuessSchema() { private void initClass2JavaTypeMap() { class2JavaTypeMap = new HashMap<>(); - JavaType javaTypes[] = javaTypesManager.getJavaTypes(); + JavaType[] javaTypes = javaTypesManager.getJavaTypes(); for (JavaType javaType : javaTypes) { Class nullableClass = javaType.getNullableClass(); if (nullableClass != null) { @@ -194,11 +170,8 @@ private DiscoverSchemaException handleException(final Exception e) throws Except return discoverSchemaException; } - public void guessInputComponentSchema(final Schema schema) throws Exception { + public void guessInputComponentSchema() throws Exception { try { - if (guessSchemaThroughAction(schema)) { - return; - } if (guessInputComponentSchemaThroughResult()) { return; } @@ -208,274 +181,17 @@ public void guessInputComponentSchema(final Schema schema) throws Exception { throw handleException(new Exception(ERROR_NO_AVAILABLE_SCHEMA_FOUND)); } - public void guessComponentSchema(final Schema incomingSchema, final String outgoingBranch, - final boolean isStartOfJob) throws Exception { + /** + * When a processor is the start of a studio job and dev explicitly set the handleError to Lifecycle exec + */ + public void guessComponentSchemaByLifecycle() throws Exception { try { - executeDiscoverSchemaExtendedAction(incomingSchema, outgoingBranch); - } catch (Exception e) { - final DiscoverSchemaException dse = transformException(e); - // When a processor is the start of a studio job and dev explicitly set the handleError to Lifecycle exec - if (isStartOfJob && EXECUTE_LIFECYCLE == dse.getPossibleHandleErrorWith()) { - try { - guessOutputComponentSchemaThroughResult(); - } catch (Exception er) { - throw handleException(e); - } - } else { - throw handleException(e); - } + guessOutputComponentSchemaThroughResult(); + } catch (Exception er) { + throw handleException(er); } } - public void guessComponentSchema(final Schema incomingSchema, final String outgoingBranch) throws Exception { - guessComponentSchema(incomingSchema, outgoingBranch, false); - } - - private void executeDiscoverSchemaExtendedAction(final Schema schema, final String branch) throws Exception { - final Collection services = getPluginServices(); - ServiceMeta.ActionMeta actionRef = services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && - a.getType().equals(SCHEMA_EXTENDED_TYPE) && - componentName.equals(a.getAction())) - .findFirst() - .orElse(null); - // did not find action named like componentName, trying to find one matching action... - if (actionRef == null) { - actionRef = services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getType().equals(SCHEMA_EXTENDED_TYPE)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No action " + family + "#" + SCHEMA_EXTENDED_TYPE)); - } - final Object schemaResult = - actionRef.getInvoker().apply(buildActionConfig(actionRef, configuration, schema, branch)); - if (schemaResult instanceof Schema) { - final Schema result = (Schema) schemaResult; - if (result.getEntries().isEmpty()) { - throw new DiscoverSchemaException(ERROR_NO_AVAILABLE_SCHEMA_FOUND, EXCEPTION); - } else { - fromSchema(Schema.class.cast(schemaResult)); - } - } - } - - private Map buildActionConfig(final ServiceMeta.ActionMeta action, - final Map configuration, final Schema schema, final String branch) { - final String schemaPath = action.getParameters() - .get() - .stream() - .filter(p -> Schema.class.isAssignableFrom((Class) p.getJavaType())) - .map(p -> p.getPath()) - .findFirst() - .orElse(""); - final String branchPath = action.getParameters() - .get() - .stream() - .filter(p -> String.class.isAssignableFrom((Class) p.getJavaType())) - .map(ParameterMeta::getPath) - .findFirst() - .orElse(""); - - final Map mapped = new HashMap<>(); - if (!schemaPath.isEmpty()) { - try (final Jsonb jsonb = JsonbBuilder.create()) { - mapped.put(schemaPath, jsonb.toJson(schema)); - } catch (final Exception e) { - throw new IllegalStateException(e); - } - } - if (!branchPath.isEmpty()) { - mapped.put(branchPath, branch); - } - if (configuration == null || configuration.isEmpty()) { - return mapped; - } - final String prefix = action - .getParameters() - .get() - .stream() - .filter(s -> !s.getPath().equals(schemaPath) && !s.getPath().equals(branchPath)) - .map(ParameterMeta::getPath) - .findFirst() - .orElse(null); - if (prefix == null) { - return mapped; - } - mapped.putAll(configuration - .entrySet() - .stream() - .filter(e -> isChildParameter(e.getKey(), prefix) || prefix.equals(e.getKey())) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))); - return mapped; - } - - private Map buildActionConfig(final ServiceMeta.ActionMeta action, - final Map configuration) { - if (configuration == null || configuration.isEmpty()) { - return configuration; // no-mapping - } - - final String prefix = action - .getParameters() - .get() - .stream() - .filter(param -> param.getMetadata().containsKey(TCOMP_CONFIGURATIONTYPE_TYPE) - && DATASET.equals(param.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE))) - .findFirst() - .map(ParameterMeta::getPath) - .orElse(null); - - if (prefix == null) { // no mapping to do - return configuration; - } - - final ParameterMeta dataSet = findDataset(action) - .orElseThrow(() -> new IllegalArgumentException("Dataset not found for " + action.getAction())); - - final String dataSetPath = dataSet.getPath(); - return configuration - .entrySet() - .stream() - .filter(e -> isChildParameter(e.getKey(), dataSetPath)) - .collect(toMap(e -> prefix + e.getKey().substring(dataSetPath.length()), Map.Entry::getValue)); - } - - private boolean isChildParameter(final String path, final String parentPath) { - return path.startsWith(parentPath) && path.substring(parentPath.length()).startsWith("."); - } - - private Optional findDataset(final ServiceMeta.ActionMeta action) { - final ComponentFamilyMeta familyMeta = findFamily(); - final ComponentFamilyMeta.BaseMeta componentMeta = findComponent(familyMeta); - - // dataset name should be the same as DiscoverSchema action name - final Collection metas = toStream(componentMeta.getParameterMetas().get()).collect(toList()); - return ofNullable(metas - .stream() - .filter(p -> DATASET.equals(p.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE)) - && action.getAction().equals(p.getMetadata().get("tcomp::configurationtype::name"))) - .findFirst() - .orElseGet(() -> { - // find and use single dataset - final Iterator iterator = metas - .stream() - .filter(p -> DATASET.equals(p.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE))) - .iterator(); - if (iterator.hasNext()) { - final ParameterMeta value = iterator.next(); - if (!iterator.hasNext()) { - return value; - } - log - .warn("Multiple potential datasets for {}:{}, ignoring parameters", action.getType(), - action.getAction()); - } - return null; - })); - } - - private ComponentFamilyMeta.BaseMeta findComponent(final ComponentFamilyMeta familyMeta) { - return Stream - .concat(familyMeta.getPartitionMappers().entrySet().stream(), - familyMeta.getProcessors().entrySet().stream()) - .filter(e -> e.getKey().equals(componentName)) - .map(Map.Entry::getValue) - .findFirst() - .orElseThrow(() -> new IllegalStateException(NO_COMPONENT + componentName)); - } - - private ComponentFamilyMeta findFamily() { - return componentManager - .findPlugin(plugin) - .orElseThrow(() -> new IllegalArgumentException("No component family " + plugin)) - .get(ContainerComponentRegistry.class) - .getComponents() - .get(family); - } - - private Stream toStream(final Collection parameterMetas) { - return Stream - .concat(parameterMetas.stream(), - parameterMetas - .stream() - .map(ParameterMeta::getNestedParameters) - .filter(Objects::nonNull) - .flatMap(this::toStream)); - } - - private Optional findFirstComponentDataSetName() { - final ComponentFamilyMeta familyMeta = findFamily(); - final ComponentFamilyMeta.BaseMeta componentMeta = findComponent(familyMeta); - return toStream(componentMeta.getParameterMetas().get()) - .filter(p -> DATASET.equals(p.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE))) - .findFirst() - .map(p -> p.getMetadata().get("tcomp::configurationtype::name")); - } - - public boolean guessSchemaThroughAction(final Schema schema) { - final Collection services = getPluginServices(); - - ServiceMeta.ActionMeta actionRef; - if (action == null || action.isEmpty()) { - // dataset name should be the same as DiscoverSchema action name so let's try to guess from the component - actionRef = findFirstComponentDataSetName() - .flatMap(datasetName -> services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getType().equals(SCHEMA_TYPE)) - .filter(a -> a.getAction().equals(datasetName)) - .findFirst()) - .orElse(null); - if (actionRef == null) { - // let's try DiscoverSchemaExtended action name - actionRef = findFirstComponentDataSetName() - .flatMap(datasetName -> services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getType().equals(SCHEMA_EXTENDED_TYPE)) - .filter(a -> a.getAction().equals(datasetName)) - .findFirst()) - .orElse(null); - } - } else { - actionRef = services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getAction().equals(action) - && a.getType().equals(SCHEMA_TYPE)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No action " + family + "#" + SCHEMA_TYPE + "#" + action)); - } - if (actionRef == null) { - return false; - } - final Map actionConfiguration = - SCHEMA_TYPE.equals(actionRef.getType()) ? buildActionConfig(actionRef, configuration) - : buildActionConfig(actionRef, configuration, schema, "INPUT"); - final Object schemaResult = actionRef.getInvoker().apply(actionConfiguration); - - if (schemaResult instanceof Schema) { - return fromSchema(Schema.class.cast(schemaResult)); - - } else { - log.error(ERROR_INSTANCE_SCHEMA); - return false; - } - } - - private Collection getPluginServices() { - return componentManager - .findPlugin(plugin) - .orElseThrow(() -> new IllegalArgumentException(NO_COMPONENT + plugin)) - .get(ContainerComponentRegistry.class) - .getServices(); - } - private boolean fromSchema(final Schema schema) { final Collection entries = schema.getEntries(); if (entries == null || entries.isEmpty()) { @@ -492,8 +208,7 @@ private boolean fromSchema(final Schema schema) { public Collection getFixedSchema(final String execute) { SchemaConverter sc = new SchemaConverter(); Object o = sc.toObjectImpl(execute); - if (o instanceof Schema) { - final Schema schema = Schema.class.cast(o); + if (o instanceof final Schema schema) { final Collection entries = schema.getEntries(); if (entries == null || entries.isEmpty()) { log.info(NO_COLUMN_FOUND_BY_GUESS_SCHEMA); diff --git a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java index 054bc1f7c308f..65cfa29bc974b 100644 --- a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java +++ b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java @@ -22,10 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.talend.sdk.component.api.record.SchemaProperty.IS_KEY; -import static org.talend.sdk.component.api.record.SchemaProperty.PATTERN; import static org.talend.sdk.component.api.record.SchemaProperty.SCALE; import static org.talend.sdk.component.api.record.SchemaProperty.SIZE; -import static org.talend.sdk.component.api.record.SchemaProperty.STUDIO_TYPE; import java.io.ByteArrayOutputStream; import java.io.File; @@ -65,7 +63,6 @@ import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.api.record.Schema.Entry; -import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.Service; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.api.service.schema.DiscoverSchemaExtended; @@ -116,18 +113,11 @@ void guessSchemaUseVersion(String version) throws Exception { Collections.singletonMap("para1", "bla"), "test-classes", "TaCoKitGuessSchemaTest", - "inputDi", null, version) { - @Override - public boolean guessSchemaThroughAction(final Schema schema) { - // stub to invoke: guessInputComponentSchemaThroughResult - return false; - } - }; - guessSchema.guessInputComponentSchema(null); + guessSchema.guessInputComponentSchema(); guessSchema.close(); assertTrue(byteArrayOutputStream.size() > 0); @@ -156,20 +146,13 @@ void guessSchemaUseVersionNOK(final String version) throws Exception { Collections.singletonMap("para1", "bla"), "test-classes", "TaCoKitGuessSchemaTest", - "inputDi", null, version) { - @Override - public boolean guessSchemaThroughAction(final Schema schema) { - // stub to invoke: guessInputComponentSchemaThroughResult - return false; - } - }; final DiscoverSchemaException exception = Assertions.assertThrows(DiscoverSchemaException.class, - () -> guessSchema.guessInputComponentSchema(null)); + () -> guessSchema.guessInputComponentSchema()); assertEquals(EXPECTED_ERROR_MESSAGE, exception.getMessage()); assertEquals(HandleErrorWith.EXCEPTION, exception.getPossibleHandleErrorWith()); } @@ -191,75 +174,18 @@ public boolean guessSchemaThroughAction(final Schema schema) { .withType(Schema.Type.BOOLEAN) .build(); - @Test - void guessProcessorSchema() throws Exception { - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - final Entry f1 = factory.newEntryBuilder() - .withName("f1") - .withType(Schema.Type.STRING) - .withProp(SchemaProperty.ORIGIN_TYPE, "VARCHAR") - .build(); - final Entry f2 = factory.newEntryBuilder() - .withName("f2") - .withType(Schema.Type.LONG) - .withDefaultValue(11l) - .withProp(SchemaProperty.ORIGIN_TYPE, "LONGINT") - .build(); - final Entry f3 = factory.newEntryBuilder() - .withName("f3") - .withType(Schema.Type.BOOLEAN) - .withProp(SchemaProperty.ORIGIN_TYPE, "BOOLEAN") - .build(); - final Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD) - .withProp("aprop", "a property!") - .withEntry(f1) - .withEntry(f2) - .withEntry(f3) - .build(); - Map config = new HashMap<>(); - config.put("configuration.param1", "parameter one"); - config.put("configuration.param2", "parameter two"); - final TaCoKitGuessSchema guessSchema = - new TaCoKitGuessSchema(out, config, "test-classes", "TaCoKitGuessSchemaTest", "outputDi", null, - "1"); - redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", false); - guessSchema.close(); - restoreStdout(); - final String flattened = flatten(byteArrayOutputStream); - final String expected = - "[{\"label\":\"f1\",\"nullable\":false,\"originalDbColumnName\":\"f1\",\"sourceType\":\"VARCHAR\",\"talendType\":\"id_String\"},{\"default\":\"11\",\"defaut\":\"11\",\"label\":\"f2\",\"nullable\":false,\"originalDbColumnName\":\"f2\",\"sourceType\":\"LONGINT\",\"talendType\":\"id_Long\"},{\"label\":\"f3\",\"nullable\":false,\"originalDbColumnName\":\"f3\",\"sourceType\":\"BOOLEAN\",\"talendType\":\"id_Boolean\"},{\"comment\":\"branch name\",\"label\":\"out\",\"nullable\":false,\"originalDbColumnName\":\"out\",\"talendType\":\"id_String\"}]"; - final Matcher schemaMatcher = schemaPattern.matcher(flattened); - assertFalse(errorPattern.matcher(flattened).find()); - assertTrue(schemaMatcher.find()); - assertEquals(expected, schemaMatcher.group()); - } - } - - @Test - void guessProcessorSchemaRecordBuilderFactoryImpl() throws Exception { - guessProcessorSchemaWithRecordBuilderFactory(null); - } - - @Test - void guessProcessorSchemaAvroRecordBuilderFactory() throws Exception { - guessProcessorSchemaWithRecordBuilderFactory(factory); - } - @Test void guessProcessorSchemaInStartOfJob() throws Exception { try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); PrintStream out = new PrintStream(byteArrayOutputStream)) { - final Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD).build(); Map config = new HashMap<>(); config.put("configuration.shouldActionFail", "true"); config.put("configuration.failWith", "EXECUTE_LIFECYCLE"); final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(out, config, "test-classes", - "TaCoKitGuessSchemaTest", "outputDi", null, "1"); + "TaCoKitGuessSchemaTest", null, "1"); // guess schema action will fail and as start of job is true, it should use processor lifecycle redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", true); + guessSchema.guessComponentSchemaByLifecycle(); guessSchema.close(); restoreStdout(); final String expected = @@ -272,221 +198,6 @@ void guessProcessorSchemaInStartOfJob() throws Exception { } } - @Test - void guessProcessorSchemaInStartWithMockExecution() throws Exception { - final Schema sin = new RecordBuilderFactoryImpl("test-classes").newSchemaBuilder(Schema.Type.RECORD).build(); - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - Map config = new HashMap<>(); - config.put("configuration.shouldActionFail", "true"); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(out, config, "test-classes", - "TaCoKitGuessSchemaTest", "outputDi", null, "1"); - try { - redirectStdout(out); - guessSchema.guessComponentSchema(sin, "out", true); - } catch (Exception e) { - guessSchema.close(); - } - restoreStdout(); - // same transformations as in Studio - final String flattened = flatten(byteArrayOutputStream); - final Matcher errorMatcher = errorPattern.matcher(flattened); - assertFalse(schemaPattern.matcher(flattened).find()); - assertTrue(errorMatcher.find()); - final DiscoverSchemaException de = jsonToException(errorMatcher.group()); - assertNotNull(de); - assertEquals("Cannot execute action.", de.getMessage()); - assertEquals(HandleErrorWith.EXECUTE_MOCK_JOB, de.getPossibleHandleErrorWith()); - } - } - - private void guessProcessorSchemaWithRecordBuilderFactory(RecordBuilderFactory facto) throws Exception { - final RecordBuilderFactory factory = facto == null ? new RecordBuilderFactoryImpl("test") : facto; - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD) - .withProp("aprop", "a property!") - .withEntry(f1) - .withEntry(f2) - .withEntry(f3) - .withEntry(factory.newEntryBuilder() - .withName("id") - .withType(Schema.Type.INT) - .withNullable(false) - .withRawName("id") - .withComment("hjk;ljkkj") - .withProp(STUDIO_TYPE, "id_Integer") - .withProp(IS_KEY, "true") - .withProp(SCALE, "0") - .withProp(SIZE, "10") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("name") - .withType(Schema.Type.STRING) - .withNullable(true) - .withRawName("name") - .withComment("hljkjhlk") - .withDefaultValue("toto") - .withProp(STUDIO_TYPE, "id_String") - .withProp(SCALE, "0") - .withProp(SIZE, "20") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("flag") - .withType(Schema.Type.STRING) - .withNullable(true) - .withRawName("flag") - .withProp(STUDIO_TYPE, "id_Character") - .withProp(SCALE, "0") - .withProp(SIZE, "4") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("female") - .withType(Schema.Type.BOOLEAN) - .withNullable(true) - .withRawName("female") - .withProp(STUDIO_TYPE, "id_Boolean") - .withProp(SCALE, "0") - .withProp(SIZE, "1") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("num1") - .withType(Schema.Type.BYTES) - .withNullable(true) - .withRawName("num1") - .withComment("hhhh") - .withProp(STUDIO_TYPE, "id_Byte") - .withProp(SCALE, "0") - .withProp(SIZE, "3") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("num2") - .withType(Schema.Type.INT) - .withNullable(true) - .withRawName("num2") - .withProp(STUDIO_TYPE, "id_Short") - .withProp(SCALE, "0") - .withProp(SIZE, "5") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("age") - .withType(Schema.Type.LONG) - .withNullable(true) - .withRawName("age") - .withProp(STUDIO_TYPE, "id_Long") - .withProp(SCALE, "0") - .withProp(SIZE, "19") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("bonus") - .withType(Schema.Type.FLOAT) - .withNullable(true) - .withRawName("bonus") - .withProp(STUDIO_TYPE, "id_Float") - .withProp(SCALE, "2") - .withProp(SIZE, "12") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("salary") - .withType(Schema.Type.DOUBLE) - .withNullable(true) - .withRawName("salary") - .withProp(STUDIO_TYPE, "id_Double") - .withProp(SCALE, "2") - .withProp(SIZE, "22") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("play") - .withType(Schema.Type.STRING) - .withNullable(true) - .withRawName("play") - .withProp(STUDIO_TYPE, "id_String") - .withProp(SCALE, "2") - .withProp(SIZE, "10") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("startdate") - .withType(Schema.Type.DATETIME) - .withNullable(true) - .withRawName("startdate") - .withProp(STUDIO_TYPE, "id_Date") - .withProp(PATTERN, "yyyy-MM-dd") - .build()) - .build(); - - Map config = new HashMap<>(); - config.put("configuration.param1", "parameter one"); - config.put("configuration.param2", "parameter two"); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema( - out, config, "test-classes", "TaCoKitGuessSchemaTest", - "outputDi", null, "1"); - redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", false); - guessSchema.close(); - restoreStdout(); - - final String flattened = flatten(byteArrayOutputStream); - final String expected = - "[{\"label\":\"f1\",\"nullable\":false,\"originalDbColumnName\":\"f1\",\"talendType\":\"id_String\"},{\"default\":\"11\",\"defaut\":\"11\",\"label\":\"f2\",\"nullable\":false,\"originalDbColumnName\":\"f2\",\"talendType\":\"id_Long\"},{\"label\":\"f3\",\"nullable\":false,\"originalDbColumnName\":\"f3\",\"talendType\":\"id_Boolean\"},{\"comment\":\"hjk;ljkkj\",\"key\":true,\"label\":\"id\",\"length\":10,\"nullable\":false,\"originalDbColumnName\":\"id\",\"precision\":0,\"talendType\":\"id_Integer\"},{\"comment\":\"hljkjhlk\",\"default\":\"toto\",\"defaut\":\"toto\",\"label\":\"name\",\"length\":20,\"nullable\":true,\"originalDbColumnName\":\"name\",\"precision\":0,\"talendType\":\"id_String\"},{\"label\":\"flag\",\"length\":4,\"nullable\":true,\"originalDbColumnName\":\"flag\",\"precision\":0,\"talendType\":\"id_Character\"},{\"label\":\"female\",\"length\":1,\"nullable\":true,\"originalDbColumnName\":\"female\",\"precision\":0,\"talendType\":\"id_Boolean\"},{\"comment\":\"hhhh\",\"label\":\"num1\",\"length\":3,\"nullable\":true,\"originalDbColumnName\":\"num1\",\"precision\":0,\"talendType\":\"id_Byte\"},{\"label\":\"num2\",\"length\":5,\"nullable\":true,\"originalDbColumnName\":\"num2\",\"precision\":0,\"talendType\":\"id_Short\"},{\"label\":\"age\",\"length\":19,\"nullable\":true,\"originalDbColumnName\":\"age\",\"precision\":0,\"talendType\":\"id_Long\"},{\"label\":\"bonus\",\"length\":12,\"nullable\":true,\"originalDbColumnName\":\"bonus\",\"precision\":2,\"talendType\":\"id_Float\"},{\"label\":\"salary\",\"length\":22,\"nullable\":true,\"originalDbColumnName\":\"salary\",\"precision\":2,\"talendType\":\"id_Double\"},{\"label\":\"play\",\"length\":10,\"nullable\":true,\"originalDbColumnName\":\"play\",\"precision\":2,\"talendType\":\"id_String\"},{\"label\":\"startdate\",\"nullable\":true,\"originalDbColumnName\":\"startdate\",\"pattern\":\"\\\"yyyy-MM-dd\\\"\",\"talendType\":\"id_Date\"},{\"comment\":\"branch name\",\"label\":\"out\",\"nullable\":false,\"originalDbColumnName\":\"out\",\"talendType\":\"id_String\"}]"; - final Matcher schemaMatcher = schemaPattern.matcher(flattened); - assertFalse(errorPattern.matcher(flattened).find()); - assertTrue(schemaMatcher.find()); - assertEquals(expected, schemaMatcher.group()); - } - } - - @Test - void testFromSchema() throws Exception { - final RecordBuilderFactory factory = new RecordBuilderFactoryImpl("test-classes"); - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD) - .withEntry(factory.newEntryBuilder() - .withName("name") - .withType(Schema.Type.STRING) - .withProp(STUDIO_TYPE, StudioTypes.STRING) - .build()) - .withEntry(factory.newEntryBuilder() - .withName("bit") - .withType(Schema.Type.BYTES) - .withProp(STUDIO_TYPE, StudioTypes.BYTE) - .build()) - .withEntry(factory.newEntryBuilder() - .withName("dynamic") - .withType(Schema.Type.RECORD) - .withProp(STUDIO_TYPE, StudioTypes.DYNAMIC) - .withProp(PATTERN, "dd/MM/YYYY") - .withElementSchema(factory.newSchemaBuilder(Schema.Type.RECORD).build()) - .build()) - .withEntry(factory.newEntryBuilder() - .withName("document") - .withType(Schema.Type.RECORD) - .withProp(STUDIO_TYPE, StudioTypes.DOCUMENT) - .withElementSchema(factory.newSchemaBuilder(Schema.Type.RECORD).build()) - .build()) - .build(); - - Map config = new HashMap<>(); - config.put("configuration.skipAssertions", "true"); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema( - out, config, "test-classes", "TaCoKitGuessSchemaTest", - "outputDi", null, "1"); - redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", false); - guessSchema.close(); - restoreStdout(); - - final String flattened = flatten(byteArrayOutputStream); - final String expected = - "[{\"label\":\"name\",\"nullable\":false,\"originalDbColumnName\":\"name\",\"talendType\":\"id_String\"},{\"label\":\"bit\",\"nullable\":false,\"originalDbColumnName\":\"bit\",\"talendType\":\"id_Byte\"},{\"label\":\"dynamic\",\"nullable\":true,\"originalDbColumnName\":\"dynamic\",\"pattern\":\"\\\"dd/MM/YYYY\\\"\",\"talendType\":\"id_Dynamic\"},{\"label\":\"document\",\"nullable\":true,\"originalDbColumnName\":\"document\",\"talendType\":\"id_Document\"},{\"comment\":\"branch name\",\"label\":\"out\",\"nullable\":false,\"originalDbColumnName\":\"out\",\"talendType\":\"id_String\"}]"; - assertTrue(byteArrayOutputStream.size() > 0); - final Matcher schemaMatcher = schemaPattern.matcher(flattened); - assertFalse(errorPattern.matcher(flattened).find()); - assertTrue(schemaMatcher.find()); - assertEquals(expected, schemaMatcher.group()); - } - } - @Test void serializeDiscoverSchemaException() throws Exception { final DiscoverSchemaException de = new DiscoverSchemaException( diff --git a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java index 6d2982a7bd9d2..fec1789debddd 100644 --- a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java +++ b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java @@ -31,7 +31,7 @@ class TypeConversionTest { @Test void jsonValueToTalendType() { JavaTypesManager javaTypesManager = new JavaTypesManager(); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(null, null, null, null, null, null, null); + final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(null, null, null, null, null, null); assertEquals(javaTypesManager.LONG.getId(), guessSchema.getTalendType(jsonProvider.createValue(1))); assertEquals(javaTypesManager.LONG.getId(), guessSchema.getTalendType(jsonProvider.createValue(1L))); assertEquals(javaTypesManager.DOUBLE.getId(), guessSchema.getTalendType(jsonProvider.createValue(1.1d)));