diff --git a/silk-plugins/silk-plugins-spatial-temporal/src/main/scala/de/fuberlin/wiwiss/silk/plugins/spatial/transformer/GeometryTransformer.scala b/silk-plugins/silk-plugins-spatial-temporal/src/main/scala/de/fuberlin/wiwiss/silk/plugins/spatial/transformer/GeometryTransformer.scala index 4fd38b6ee6..04f6d1e638 100644 --- a/silk-plugins/silk-plugins-spatial-temporal/src/main/scala/de/fuberlin/wiwiss/silk/plugins/spatial/transformer/GeometryTransformer.scala +++ b/silk-plugins/silk-plugins-spatial-temporal/src/main/scala/de/fuberlin/wiwiss/silk/plugins/spatial/transformer/GeometryTransformer.scala @@ -15,7 +15,7 @@ package org.silkframework.plugins.spatial.transformer import org.silkframework.plugins.spatial.utils._ -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin /** @@ -30,7 +30,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin categories = Array("Spatial"), label = "Transform geometry", description = "Trasforms a geometry expressed in GeoSPARQL, stSPARQL or W3C Geo vocabulary from any serialization (WKT or GML) and any Coordinate Reference System (CRS) to WKT and WGS 84 (latitude-longitude).") -case class GeometryTransformer() extends Transformer { +case class GeometryTransformer() extends InlineTransformer { override final def apply(values: Seq[Seq[String]]): Seq[String] = { diff --git a/silk-plugins/silk-plugins-spatial-temporal/src/main/scala/de/fuberlin/wiwiss/silk/plugins/spatial/transformer/PointsToCentroidTransformer.scala b/silk-plugins/silk-plugins-spatial-temporal/src/main/scala/de/fuberlin/wiwiss/silk/plugins/spatial/transformer/PointsToCentroidTransformer.scala index a59233f73d..887965a725 100644 --- a/silk-plugins/silk-plugins-spatial-temporal/src/main/scala/de/fuberlin/wiwiss/silk/plugins/spatial/transformer/PointsToCentroidTransformer.scala +++ b/silk-plugins/silk-plugins-spatial-temporal/src/main/scala/de/fuberlin/wiwiss/silk/plugins/spatial/transformer/PointsToCentroidTransformer.scala @@ -17,7 +17,7 @@ package org.silkframework.plugins.spatial.transformer import java.util.logging.Logger import org.silkframework.plugins.spatial.utils._ -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin /** @@ -32,7 +32,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin categories = Array("Spatial"), label = "Points-to-centroid", description = "Transforms a cluster of points expressed in W3C Geo vocabulary to their centroid expressed in WKT and WGS 84 (latitude-longitude).") -case class PointsToCentroidTransformer() extends Transformer { +case class PointsToCentroidTransformer() extends InlineTransformer { override final def apply(values: Seq[Seq[String]]): Seq[String] = { diff --git a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/LinkingSerializers.scala b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/LinkingSerializers.scala index 34d6db44e7..16ac0f1574 100644 --- a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/LinkingSerializers.scala +++ b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/LinkingSerializers.scala @@ -64,7 +64,7 @@ object LinkingSerializers { case _ => // Link does not provide evaluation details, need to evaluate it first for (linkingRule <- rule; entities <- link.entities) { - val details = DetailedEvaluator(linkingRule, entities).details + val details = DetailedEvaluator(linkingRule.execution(), entities).details json += (RULE_VALUES -> ConfidenceJsonFormat.write(details)) } } diff --git a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/transformer/JsonTransformer.scala b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/transformer/JsonTransformer.scala index b2e65596a9..4002f9abd6 100644 --- a/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/transformer/JsonTransformer.scala +++ b/silk-plugins/silk-serialization-json/src/main/scala/org/silkframework/serialization/json/transformer/JsonTransformer.scala @@ -2,7 +2,7 @@ package org.silkframework.serialization.json.transformer import org.silkframework.config.{Task, TaskSpec} import org.silkframework.rule.TaskContext -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.{Transformer, TransformerExecution} import org.silkframework.rule.plugins.transformer.value.ConstantTransformer import org.silkframework.runtime.activity.UserContext import org.silkframework.runtime.plugin.PluginContext @@ -28,7 +28,7 @@ trait JsonTransformer extends Transformer { def getJson(inputTask: Task[_ <: TaskSpec], project: ProjectTrait) (implicit pluginContext: PluginContext): JsValue - override def withContext(taskContext: TaskContext): Transformer = { + override def execution(taskContext: TaskContext): TransformerExecution = { val inputTask = taskContext.inputTasks.headOption.getOrElse(throw new ValidationException("This task does not have an input")) val project = taskContext.pluginContext.projectId match { case Some(projectId) => @@ -47,10 +47,6 @@ trait JsonTransformer extends Transformer { } } - def apply(values: Seq[Seq[String]]): Seq[String] = { - throw new ValidationException("No input task available.") - } - @tailrec private def navigatePath(json: JsValue, path: String): JsLookupResult = { val parts = path.split('/') diff --git a/silk-plugins/silk-serialization-json/src/test/scala/org/silkframework/serialization/json/transformer/InputTaskJsonTransformerTest.scala b/silk-plugins/silk-serialization-json/src/test/scala/org/silkframework/serialization/json/transformer/InputTaskJsonTransformerTest.scala index 0fa36b7fcf..cb2c052bff 100644 --- a/silk-plugins/silk-serialization-json/src/test/scala/org/silkframework/serialization/json/transformer/InputTaskJsonTransformerTest.scala +++ b/silk-plugins/silk-serialization-json/src/test/scala/org/silkframework/serialization/json/transformer/InputTaskJsonTransformerTest.scala @@ -42,7 +42,7 @@ class InputTaskJsonTransformerTest extends AnyFlatSpec with Matchers with TestWo def retrieve(path: String): String = { val taskContext = TaskContext(Seq(dataset), PluginContext.fromProject(project)) - val transformer = InputTaskAttributesTransformer(path).withContext(taskContext) + val transformer = InputTaskAttributesTransformer(path).execution(taskContext) transformer(Seq.empty).head } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/LinkageRule.scala b/silk-rules/src/main/scala/org/silkframework/rule/LinkageRule.scala index 84e56ecbb3..8c67343635 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/LinkageRule.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/LinkageRule.scala @@ -15,7 +15,7 @@ package org.silkframework.rule import org.silkframework.entity.{Entity, Index} -import org.silkframework.rule.similarity.SimilarityOperator +import org.silkframework.rule.similarity.{SimilarityOperator, SimilarityOperatorExecution} import org.silkframework.runtime.plugin.PluginObjectParameterNoSchema import org.silkframework.runtime.serialization._ import org.silkframework.runtime.validation.ValidationIssue @@ -40,24 +40,40 @@ case class LinkageRule(operator: Option[SimilarityOperator] = None, operator.foreach(_.validateIds()) /** - * Generates a copy of this rule that has been configured with a given task context. - * This is relevant for operators whose results are based on the input task(s), i.e., the file hash transformer. + * Validates this rule. + * This should cover non-fatal issues that should be fixed by the user after rule creation. + * Issues that lead to an inconsistent and unusable rule should not be checked here, but instead throw an exception in the constructor. */ - def withContext(taskContext: TaskContext): LinkageRule = { - copy(operator = operator.map(_.withContext(taskContext))) + def validate(): Seq[ValidationIssue] = { + operator.toSeq.flatMap(_.validate()) } + /** + * Builds a runtime executor for this rule resolved against the given task context. + */ + def execution(taskContext: TaskContext = TaskContext.empty): LinkageRuleExecution = { + new LinkageRuleExecution(this, operator.map(_.execution(taskContext))) + } +} + +/** + * Runtime executor for a [[LinkageRule]]. Holds the contextualized + * [[SimilarityOperatorExecution]] used to evaluate the rule against entities. + */ +final class LinkageRuleExecution(val operator: LinkageRule, + val operatorExecution: Option[SimilarityOperatorExecution]) extends Serializable { + /** * Computes the similarity between two entities. * * @param entities The entities to be compared. * @param limit If the confidence is below this limit, it will be capped to -1.0. - * @return The confidence as a value between -1.0 and 1.0. + * @return The confidence as a value between -1.0 and 1.0. * -1.0 for definitive non-matches. * +1.0 for definitive matches. */ def apply(entities: DPair[Entity], limit: Double = 0.0): Double = { - operator match { + operatorExecution match { case Some(op) => op(entities, limit).getOrElse(-1.0) case None => -1.0 } @@ -67,24 +83,16 @@ case class LinkageRule(operator: Option[SimilarityOperator] = None, * Indexes an entity. * * @param entity The entity to be indexed + * @param sourceOrTarget If true, the index will be for the source entity. If false, the index will be for the target entity. * @param limit The confidence limit - * @return A set of (multidimensional) indexes. Entities within the threshold will always get the same index. + * @return A set of (multidimensional) indexes. Entities within the threshold will always get the same index. */ def index(entity: Entity, sourceOrTarget: Boolean, limit: Double = 0.0): Index = { - operator match { + operatorExecution match { case Some(op) => op.index(entity, sourceOrTarget, limit) case None => Index.empty } } - - /** - * Validates this rule. - * This should cover non-fatal issues that should be fixed by the user after rule creation. - * Issues that lead to an inconsistent and unusable rule should not be checked here, but instead throw an exception in the constructor. - */ - def validate(): Seq[ValidationIssue] = { - operator.toSeq.flatMap(_.validate()) - } } /** diff --git a/silk-rules/src/main/scala/org/silkframework/rule/LinkageRuleIndex.scala b/silk-rules/src/main/scala/org/silkframework/rule/LinkageRuleIndex.scala index e80ddf3607..c29047af15 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/LinkageRuleIndex.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/LinkageRuleIndex.scala @@ -53,7 +53,7 @@ object LinkageRuleIndex { case BooleanNot(child) => LinkageRuleIndexNot(convert(child, entity, sourceOrTarget)) case BooleanComparisonOperator(id, sourceInput, targetInput, comparison) => val inputId = if(sourceOrTarget) sourceInput.inputOperator.id else targetInput.inputOperator.id - val index = comparison.index(entity, sourceOrTarget, limit = 0.0) + val index = comparison.execution(TaskContext.empty).index(entity, sourceOrTarget, limit = 0.0) LinkageRuleIndexComparison(id, LinkageRuleIndexInput(inputId, index.flatten)) } } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/Operator.scala b/silk-rules/src/main/scala/org/silkframework/rule/Operator.scala index 5bf06a93b5..3ede1a3ed0 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/Operator.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/Operator.scala @@ -47,10 +47,11 @@ trait Operator { def withChildren(newChildren: Seq[Operator]): Operator /** - * Generates a copy of this operator tree that has been configured with a given task context. - * This is relevant for operators whose results are based on the input task(s), i.e., the file hash transformer. + * Builds an executor that performs this operator's runtime work for a given task context. + * The returned [[OperatorExecution]] holds any state that depends on the task context + * (e.g. resolved input task data). Pure operators may return themselves. */ - def withContext(taskContext: TaskContext): Operator = this + def execution(taskContext: TaskContext = TaskContext.empty): OperatorExecution /** * Asserts that all identifiers in this rule tree are unique. @@ -67,6 +68,18 @@ trait Operator { } +/** + * Runtime form of an [[Operator]], contextualized for a specific [[TaskContext]]. + * Each [[Operator]] subclass has a corresponding [[OperatorExecution]] sub-trait + * (e.g. [[org.silkframework.rule.input.InputExecution]], + * [[org.silkframework.rule.similarity.SimilarityOperatorExecution]]) + * that declares the actual execution methods. + */ +trait OperatorExecution extends Serializable { + /** Backlink to the operator that produced this execution. */ + def operator: Operator +} + /** * Operator companion object. */ diff --git a/silk-rules/src/main/scala/org/silkframework/rule/TaskContext.scala b/silk-rules/src/main/scala/org/silkframework/rule/TaskContext.scala index 72f08558cb..757787b3c0 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/TaskContext.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/TaskContext.scala @@ -10,4 +10,34 @@ import org.silkframework.runtime.plugin.PluginContext * If the task is executed within a workflow, those are the connected input task(s). * If the task is executed standalone, those are the configured default input(s). */ -case class TaskContext(inputTasks: Seq[Task[_ <: TaskSpec]], pluginContext: PluginContext) \ No newline at end of file +case class TaskContext(inputTasks: Seq[Task[_ <: TaskSpec]], pluginContext: PluginContext) + +object TaskContext { + + /** + * The empty task context. + */ + def empty: TaskContext = TaskContext(Seq.empty, PluginContext.empty) + + /** + * Creates a task context for a single input task. + */ + def forInput(inputTask: Task[_ <: TaskSpec])(implicit pluginContext: PluginContext): TaskContext = { + TaskContext(Seq(inputTask), pluginContext) + } + + /** + * Creates a task context for the given input tasks. + */ + def forInputs(inputTasks: Seq[Task[_ <: TaskSpec]])(implicit pluginContext: PluginContext): TaskContext = { + TaskContext(inputTasks, pluginContext) + } + + /** + * Creates a task context for no input. + */ + def noInput(implicit pluginContext: PluginContext): TaskContext = { + TaskContext(Seq.empty, pluginContext) + } + +} \ No newline at end of file diff --git a/silk-rules/src/main/scala/org/silkframework/rule/TransformRule.scala b/silk-rules/src/main/scala/org/silkframework/rule/TransformRule.scala index fd8f797153..52ef611f49 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/TransformRule.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/TransformRule.scala @@ -8,7 +8,7 @@ import org.silkframework.entity.paths.{TypedPath, UntypedPath} import org.silkframework.rule.MappingRules.MappingRulesFormat import org.silkframework.rule.MappingTarget.MappingTargetFormat import org.silkframework.rule.TransformRule.RDF_TYPE -import org.silkframework.rule.input.{Input, OperatorEvaluationError, PathInput, TransformInput, Value} +import org.silkframework.rule.input.{Input, InputExecution, OperatorEvaluationError, PathInput, TransformInput, Value} import org.silkframework.rule.plugins.transformer.combine.ConcatTransformer import org.silkframework.rule.plugins.transformer.normalize.{UriFixTransformer, UrlEncodeTransformer} import org.silkframework.rule.plugins.transformer.value.{ConstantTransformer, ConstantUriTransformer, EmptyValueTransformer} @@ -62,20 +62,6 @@ sealed trait TransformRule extends Operator with HasMetaData { assert(rules.allRules.forall(!_.isInstanceOf[RootMappingRule]), "No root mapping rule allowed as child of another rule!") - /** - * Generates the transformed values. - * - * @param entity The source entity. - * @return The transformed values. - * @throws ValidationException If a value failed to be transformed or a generated value doesn't match the target type. - */ - def apply(entity: Entity): Value = { - val values = operator(entity) - // Validate values - target.foreach(_.validate(values.values)) - values - } - /** * Generates a label for this rule. * Will return the user-defined label, if any is defined. @@ -140,13 +126,39 @@ sealed trait TransformRule extends Operator with HasMetaData { } } - override def withContext(taskContext: TaskContext): TransformRule = this + override def execution(taskContext: TaskContext = TaskContext.empty): TransformRuleExecution def representsDefaultUriRule: Boolean = { false } } +/** + * Runtime executor of a [[TransformRule]]. + */ +sealed trait TransformRuleExecution extends OperatorExecution { + + /** The originating rule. */ + override def operator: TransformRule + + /** Resolved executions of the rule's direct child rules. Empty for value-producing rules. */ + def childExecutions: Seq[TransformRuleExecution] + + /** The contextualized input that produces this rule's values for an entity. */ + def inputExecution: InputExecution + + /** + * Generates the transformed values. + * + * @throws ValidationException If a value failed to be transformed or a generated value doesn't match the target type. + */ + def apply(entity: Entity): Value = { + val values = inputExecution(entity) + operator.target.foreach(_.validate(values.values)) + values + } +} + /** * Base trait for all rules that can have child rules. */ @@ -161,11 +173,25 @@ sealed trait ContainerTransformRule extends TransformRule { } } - override def withContext(taskContext: TaskContext): TransformRule = { - withChildren(rules.map(_.withContext(taskContext))) + override def execution(taskContext: TaskContext = TaskContext.empty): ContainerTransformRuleExecution = { + new ContainerTransformRuleExecution(this, rules.map(_.execution(taskContext)), operator.execution(taskContext)) } } +/** + * Runtime executor for container transform rules. + * + * Child executions are built lazily: when a container appears as a property of its parent, + * the parent only needs this execution's [[inputExecution]] (to generate the URIs linking to + * the child entities). Its grandchildren are contextualized in their own schema iteration, + * so deferring this expansion avoids building those executions twice. + */ +class ContainerTransformRuleExecution(override val operator: ContainerTransformRule, + childExecs: => Seq[TransformRuleExecution], + override val inputExecution: InputExecution) extends TransformRuleExecution { + override lazy val childExecutions: Seq[TransformRuleExecution] = childExecs +} + /** * Base trait for all rule that generate a value and do not have any child rules. */ @@ -188,11 +214,22 @@ sealed trait ValueTransformRule extends TransformRule { override def withId(newId: Identifier): ValueTransformRule - override def withContext(taskContext: TaskContext): ValueTransformRule = { - this + override def execution(taskContext: TaskContext = TaskContext.empty): ValueTransformRuleExecution = { + new ValueTransformRuleExecution(this, operator.execution(taskContext)) } } +/** + * Runtime executor for value-producing transform rules (non-container). Holds the contextualized + * [[InputExecution]] used to evaluate the rule on an entity. + */ +class ValueTransformRuleExecution(override val operator: ValueTransformRule, + override val inputExecution: InputExecution) extends TransformRuleExecution { + + /** Value rules have no child rules. */ + override def childExecutions: Seq[TransformRuleExecution] = Seq.empty +} + /** * The root mapping rule. * @@ -368,18 +405,26 @@ case class ComplexUriMapping(id: Identifier = "complexURI", override def withMetaData(metaData: MetaData): TransformRule = this.copy(metaData = metaData) - override def withContext(taskContext: TaskContext): ComplexUriMapping = { - copy(operator = operator.withContext(taskContext)) + override def execution(taskContext: TaskContext = TaskContext.empty): ValueTransformRuleExecution = { + new ComplexUriMappingExecution(this, operator.execution(taskContext)) } +} + +/** + * Runtime executor for [[ComplexUriMapping]] that additionally validates the generated URIs. + */ +final class ComplexUriMappingExecution(override val operator: ComplexUriMapping, + inputExecution: InputExecution) + extends ValueTransformRuleExecution(operator, inputExecution) { override def apply(entity: Entity): Value = { val v = super.apply(entity) val invalidUri = v.values.find(uri => !Uri(uri).isValidUri) if(invalidUri.isDefined && v.errors.isEmpty) { - // The URI rule has generated an invalid URI - return v.copy(errors = Seq(OperatorEvaluationError(id, new ValidationException(s"URI rule has generated an invalid URI: '${invalidUri.get}'!")))) + v.copy(errors = Seq(OperatorEvaluationError(operator.id, new ValidationException(s"URI rule has generated an invalid URI: '${invalidUri.get}'!")))) + } else { + v } - v } } @@ -426,9 +471,6 @@ case class ComplexMapping(id: Identifier = "mapping", override def withMetaData(metaData: MetaData): TransformRule = this.copy(metaData = metaData) - override def withContext(taskContext: TaskContext): ComplexMapping = { - copy(operator = operator.withContext(taskContext)) - } } /** diff --git a/silk-rules/src/main/scala/org/silkframework/rule/TransformSpec.scala b/silk-rules/src/main/scala/org/silkframework/rule/TransformSpec.scala index d075983077..8183aa2072 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/TransformSpec.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/TransformSpec.scala @@ -388,11 +388,25 @@ object TransformSpec { } } - def withContext(taskContext: TaskContext): RuleSchemata = { - copy(transformRule = transformRule.withContext(taskContext)) + /** + * Builds a [[RuleSchemataExecution]] for the given task context, contextualizing the + * underlying transform rule. + */ + def execution(taskContext: TaskContext): RuleSchemataExecution = { + RuleSchemataExecution(transformRule.execution(taskContext), inputSchema, outputSchema) } } + /** + * Runtime form of a [[RuleSchemata]] with the contained transform rule contextualized for + * a specific task context. + */ + case class RuleSchemataExecution(transformRuleExecution: TransformRuleExecution, + inputSchema: EntitySchema, + outputSchema: EntitySchema) { + def transformRule: TransformRule = transformRuleExecution.operator + } + object RuleSchemata { def create(rule: TransformRule, selection: DatasetSelection, sourceSubPath: UntypedPath, targetSubPath: UntypedPath): RuleSchemata = { val inputSchema = EntitySchema( diff --git a/silk-rules/src/main/scala/org/silkframework/rule/TransformedDataSource.scala b/silk-rules/src/main/scala/org/silkframework/rule/TransformedDataSource.scala index d6ebea2aa9..441ad08c3a 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/TransformedDataSource.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/TransformedDataSource.scala @@ -56,7 +56,7 @@ class TransformedDataSource(source: DataSource, inputSchema: EntitySchema, trans val sourceEntities = source.retrieve(inputSchema, limit).entities val taskContext = new ActivityMonitor[TransformReport](task.id, None) val reportBuilder = new TransformReportBuilder(task, taskContext) - val transformedEntities = new TransformedEntities(task, sourceEntities, transformRule.label(), transformRule, + val transformedEntities = new TransformedEntities(task, sourceEntities, transformRule.label(), transformRule.execution(TaskContext.noInput), entitySchema, isRequestedSchema = true, abortIfErrorsOccur = false, report = reportBuilder).iterator GenericEntityTable(transformedEntities, entitySchema, underlyingTask) } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/evaluation/DetailedEvaluator.scala b/silk-rules/src/main/scala/org/silkframework/rule/evaluation/DetailedEvaluator.scala index 9acf2325c9..b5d55c204b 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/evaluation/DetailedEvaluator.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/evaluation/DetailedEvaluator.scala @@ -15,24 +15,29 @@ package org.silkframework.rule.evaluation import org.silkframework.entity.Entity -import org.silkframework.rule.input.{Input, PathInput, TransformInput} -import org.silkframework.rule.similarity.{Aggregation, Comparison, SimilarityOperator} -import org.silkframework.rule.{ComplexUriMapping, LinkageRule, TransformRule} +import org.silkframework.rule.input.{InputExecution, PathInput, TransformInputExecution} +import org.silkframework.rule.similarity.{AggregationExecution, ComparisonExecution, SimilarityOperatorExecution} +import org.silkframework.rule.{ComplexUriMapping, LinkageRuleExecution, TransformRuleExecution} import org.silkframework.runtime.validation.ValidationException import org.silkframework.util.{DPair, Uri} +import scala.collection.immutable.ArraySeq import scala.util.control.NonFatal +/** + * Walks a pre-built rule execution tree to produce a detailed evaluation result + */ object DetailedEvaluator { /** - * Evaluates a linkage rule. + * Evaluates a pre-built linkage rule execution against an entity pair. */ - def apply(rule: LinkageRule, entities: DPair[Entity], limit: Double): Option[EvaluatedLink] = { - rule.operator match { - case Some(op) => + def apply(ruleExec: LinkageRuleExecution, entities: DPair[Entity], limit: Double): Option[EvaluatedLink] = { + val rule = ruleExec.operator + ruleExec.operatorExecution match { + case Some(opExec) => if(!rule.excludeSelfReferences || entities.source.uri != entities.target.uri) { - val confidence = evaluateOperator(op, entities, limit) + val confidence = evaluateOperator(opExec, entities, limit) if (confidence.score.getOrElse(-1.0) >= limit) { Some(new EvaluatedLink(entities.source.uri, entities.target.uri, entities, confidence)) } else { @@ -52,12 +57,12 @@ object DetailedEvaluator { } /** - * Evaluates a linkage rule. - */ - def apply(rule: LinkageRule, entities: DPair[Entity]): EvaluatedLink = { - rule.operator match { - case Some(op) => - val confidence = evaluateOperator(op, entities, -1.0) + * Evaluates a pre-built linkage rule execution against an entity pair. + */ + def apply(ruleExec: LinkageRuleExecution, entities: DPair[Entity]): EvaluatedLink = { + ruleExec.operatorExecution match { + case Some(opExec) => + val confidence = evaluateOperator(opExec, entities, -1.0) new EvaluatedLink(entities.source.uri, entities.target.uri, entities, confidence) case None => new EvaluatedLink(entities.source.uri, entities.target.uri, entities, SimpleConfidence(Some(-1.0))) @@ -65,25 +70,25 @@ object DetailedEvaluator { } /** - * Evaluates a set of transform rules. + * Evaluates a set of pre-built transform rule executions for a single entity. */ - def apply(rules: Seq[TransformRule], entity: Entity): DetailedEntity = { - val subjectRule = rules.find(_.target.isEmpty) - val uris = subjectRule match { - case Some(rule) => rule(entity).values + def apply(ruleExecs: Seq[TransformRuleExecution], entity: Entity): DetailedEntity = { + val subjectExec = ruleExecs.find(_.operator.target.isEmpty) + val uris = subjectExec match { + case Some(exec) => exec(entity).values case None => Seq(entity.uri.toString) } - val values = for(rule <- rules) yield apply(rule, entity) - DetailedEntity(uris, values, rules) + val values = ruleExecs.map(re => apply(re, entity)) + DetailedEntity(uris, values, ruleExecs.map(_.operator)) } /** - * Evaluates a single transform rule. + * Evaluates a single pre-built transform rule execution for an entity. */ - def apply(rule: TransformRule, entity: Entity): Value = { - val result = evaluateInput(rule.operator, entity) - // Validate values - for(target <- rule.target) { + def apply(ruleExec: TransformRuleExecution, entity: Entity): Value = { + val result = evaluateInput(ruleExec.inputExecution, entity) + // Validate against the rule's mapping target + for(target <- ruleExec.operator.target) { try { target.validate(result.values) } catch { @@ -92,52 +97,47 @@ object DetailedEvaluator { } } // Complex URI mapping rules need to be validated separately - if(rule.isInstanceOf[ComplexUriMapping]) { - val invalidUri = result.values.find(uri => !Uri(uri).isValidUri) + if(ruleExec.operator.isInstanceOf[ComplexUriMapping]) { + val invalidUri = result.values.find(uri => !Uri(uri).isValidUri) if(invalidUri.isDefined) { // The URI rule has generated an invalid URI return result.withError(new ValidationException(s"URI rule of object mapping has generated an invalid URI: '${invalidUri.get}'!")) } } - // Return validated result result } - private def evaluateOperator(operator: SimilarityOperator, entities: DPair[Entity], threshold: Double) = operator match { - case aggregation: Aggregation => evaluateAggregation(aggregation, entities, threshold) - case comparison: Comparison => evaluateComparison(comparison, entities, threshold) + private def evaluateOperator(opExec: SimilarityOperatorExecution, entities: DPair[Entity], threshold: Double): Confidence = opExec match { + case agg: AggregationExecution => evaluateAggregation(agg, entities, threshold) + case cmp: ComparisonExecution => evaluateComparison(cmp, entities, threshold) } - private def evaluateAggregation(agg: Aggregation, entities: DPair[Entity], threshold: Double): AggregatorConfidence = { - val operatorValues = { - for (operator <- agg.operators) yield { - evaluateOperator(operator, entities, threshold) - } - } - - val aggregatedValue = agg.aggregator(agg.operators, entities, threshold) - AggregatorConfidence(aggregatedValue.score, agg, operatorValues) + private def evaluateAggregation(agg: AggregationExecution, entities: DPair[Entity], threshold: Double): AggregatorConfidence = { + val operatorValues = agg.operatorExecutions.map(opExec => evaluateOperator(opExec, entities, threshold)) + val aggregatedValue = agg.operator.aggregator(agg.operatorExecutions, entities, threshold) + AggregatorConfidence(aggregatedValue.score, agg.operator, operatorValues) } - private def evaluateComparison(comparison: Comparison, entities: DPair[Entity], threshold: Double): ComparisonConfidence = { + private def evaluateComparison(cmp: ComparisonExecution, entities: DPair[Entity], threshold: Double): ComparisonConfidence = { ComparisonConfidence( - score = comparison.apply(entities, threshold), - comparison = comparison, - sourceValue = evaluateInput(comparison.inputs.source, entities.source), - targetValue = evaluateInput(comparison.inputs.target, entities.target) + score = cmp(entities, threshold), + comparison = cmp.operator, + sourceValue = evaluateInput(cmp.sourceInput, entities.source), + targetValue = evaluateInput(cmp.targetInput, entities.target) ) } - private def evaluateInput(input: Input, entity: Entity): Value = input match { - case ti: TransformInput => - val children = ti.inputs.map(i => evaluateInput(i, entity)) + private def evaluateInput(inputExec: InputExecution, entity: Entity): Value = inputExec match { + case ti: TransformInputExecution => + val children = ti.inputExecutions.map(child => evaluateInput(child, entity)) try { - TransformedValue(ti, ti.transformer(children.map(_.values)), children) + val transformed = ti.transformerExecution(ArraySeq.unsafeWrapArray(children.map(_.values).toArray[Seq[String]])) + TransformedValue(ti.operator, transformed, children) } catch { case NonFatal(ex) => - TransformedValue(ti, Seq.empty, children, Some(ex)) + TransformedValue(ti.operator, Seq.empty, children, Some(ex)) } - case pi: PathInput => InputValue(pi, input(entity).values) + case pi: PathInput => InputValue(pi, pi(entity).values) } -} \ No newline at end of file +} diff --git a/silk-rules/src/main/scala/org/silkframework/rule/evaluation/DetailedIndexer.scala b/silk-rules/src/main/scala/org/silkframework/rule/evaluation/DetailedIndexer.scala index 73d487d0dc..ee1ea7203f 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/evaluation/DetailedIndexer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/evaluation/DetailedIndexer.scala @@ -38,7 +38,7 @@ object DetailedIndexer { } def indexComparison(cmp: Comparison, entity: Entity, sourceOrTarget: Boolean, limit: Double): ComparisonIndex = { - val values = if(sourceOrTarget) cmp.inputs.source(entity) else cmp.inputs.target(entity) + val values = if(sourceOrTarget) cmp.inputs.source.execution()(entity) else cmp.inputs.target.execution()(entity) val distanceLimit = cmp.threshold * (1.0 - limit) ComparisonIndex(cmp.metric.index(values.values, distanceLimit, sourceOrTarget), values.values, cmp) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/evaluation/LinkageRuleEvaluator.scala b/silk-rules/src/main/scala/org/silkframework/rule/evaluation/LinkageRuleEvaluator.scala index 142d8777ab..8964988358 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/evaluation/LinkageRuleEvaluator.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/evaluation/LinkageRuleEvaluator.scala @@ -3,12 +3,12 @@ package org.silkframework.rule.evaluation import org.silkframework.entity.{LinkDecision, ReferenceLink} import java.util.logging.Logger -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution object LinkageRuleEvaluator { val log: Logger = Logger.getLogger(this.getClass.getName) - def apply(rule: LinkageRule, + def apply(rule: LinkageRuleExecution, referenceLinks: Seq[ReferenceLink], threshold: Double): EvaluationResult = { var truePositives: Int = 0 @@ -38,7 +38,7 @@ object LinkageRuleEvaluator { new EvaluationResult(truePositives, trueNegatives, falsePositives, falseNegatives) } - def apply(rule: LinkageRule, + def apply(rule: LinkageRuleExecution, entity: ReferenceEntities, threshold: Double = 0.0, logFalseNegatives: Boolean = false, diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/EvaluateTransform.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/EvaluateTransform.scala index 7e29d2b876..1ef39cc84b 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/EvaluateTransform.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/EvaluateTransform.scala @@ -24,12 +24,13 @@ class EvaluateTransform(source: DataSource, private var cachedValues = Seq[DetailedEntity]() def execute()(implicit context: PluginContext): Seq[DetailedEntity] = { + val ruleExecs = rules.map(_.execution()) // Retrieve entities source.retrieve(entitySchema, Some(maxEntities)).entities.use { entities => // Read all entities for (entity <- entities) { // Transform entity - val transformedEntity = DetailedEvaluator(rules, entity) + val transformedEntity = DetailedEvaluator(ruleExecs, entity) // Write transformed entity to cache cachedValues = cachedValues :+ transformedEntity if (cachedValues.size >= maxEntities) return cachedValues diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/ExecuteTransform.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/ExecuteTransform.scala index af03c79894..88abb672c0 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/ExecuteTransform.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/ExecuteTransform.scala @@ -3,7 +3,7 @@ package org.silkframework.rule.execution import org.silkframework.config.{Prefixes, Task, TaskSpec} import org.silkframework.dataset.{DataSource, EntitySink} import org.silkframework.execution.local.ErrorOutputWriter -import org.silkframework.rule.TransformSpec.RuleSchemata +import org.silkframework.rule.TransformSpec.RuleSchemataExecution import org.silkframework.rule._ import org.silkframework.rule.execution.local.TransformedEntities import org.silkframework.runtime.activity.{Activity, ActivityContext, UserContext} @@ -63,7 +63,7 @@ class ExecuteTransform(task: Task[TransformSpec], context.status.updateMessage("Retrieving entities") try { for ((ruleSchemata, index) <- transform.ruleSchemataWithoutEmptyObjectRules.zipWithIndex) { - transformEntities(dataSource, ruleSchemata.withContext(taskContext), entitySink, errorEntitySink, report, context) + transformEntities(dataSource, ruleSchemata.execution(taskContext), entitySink, errorEntitySink, report, context) context.status.updateProgress((index + 1.0) / transform.ruleSchemataWithoutEmptyObjectRules.size) } } finally { @@ -73,7 +73,7 @@ class ExecuteTransform(task: Task[TransformSpec], } private def transformEntities(dataSource: DataSource, - rule: RuleSchemata, + rule: RuleSchemataExecution, entitySink: EntitySink, errorEntitySink: Option[EntitySink], reportBuilder: TransformReportBuilder, @@ -92,7 +92,7 @@ class ExecuteTransform(task: Task[TransformSpec], case NonFatal(ex) => throw new RuntimeException("Failed to retrieve input entities from data source.", ex) } - val transformedEntities = new TransformedEntities(task, entityTable.entities, rule.transformRule.label(), rule.transformRule, rule.outputSchema, + val transformedEntities = new TransformedEntities(task, entityTable.entities, rule.transformRule.label(), rule.transformRuleExecution, rule.outputSchema, isRequestedSchema = false, abortIfErrorsOccur = task.data.abortIfErrorsOccur, report = reportBuilder).iterator var count = 0 breakable { diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/ExecutionMethod.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/ExecutionMethod.scala index b2bff990b5..fec4ba0b4a 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/ExecutionMethod.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/ExecutionMethod.scala @@ -2,7 +2,7 @@ package org.silkframework.rule.execution import org.silkframework.cache.{EntityCache, Partition} import org.silkframework.entity.{Entity, Index} -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution import org.silkframework.util.DPair /** @@ -13,7 +13,7 @@ trait ExecutionMethod { /** * Generates an index for a single entity. */ - def indexEntity(entity: Entity, rule: LinkageRule, sourceOrTarget: Boolean): Index = Index.default + def indexEntity(entity: Entity, rule: LinkageRuleExecution, sourceOrTarget: Boolean): Index = Index.default /** * Iterates over all pairs of partitions that should be compared. diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/GenerateLinks.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/GenerateLinks.scala index 02191fccec..4efd4f710d 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/GenerateLinks.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/GenerateLinks.scala @@ -21,7 +21,7 @@ import org.silkframework.entity.{Entity, EntitySchema, Link} import org.silkframework.execution.ExecutionReport import org.silkframework.execution.report.{EntitySample, SampleEntities, SampleEntitiesSchema} import org.silkframework.rule.execution.rdb.RDBEntityIndex -import org.silkframework.rule.{LinkSpec, LinkageRule, LinkingExecutionBackend, RuntimeLinkingConfig} +import org.silkframework.rule.{LinkSpec, LinkageRuleExecution, LinkingExecutionBackend, RuntimeLinkingConfig} import org.silkframework.runtime.activity._ import org.silkframework.util.FileUtils._ import org.silkframework.util.{CollectLogs, DPair} @@ -38,14 +38,14 @@ class GenerateLinks(task: Task[LinkSpec], inputs: DPair[DataSource], output: Option[LinkSink], runtimeConfig: RuntimeLinkingConfig = RuntimeLinkingConfig(), - overrideLinkageRule: Option[LinkageRule] = None) + overrideLinkageRule: Option[LinkageRuleExecution] = None) (implicit prefixes: Prefixes) extends Activity[Linking] { private val log: Logger = Logger.getLogger(this.getClass.getName) - private val rule = overrideLinkageRule.getOrElse(task.data.rule) + private val rule = overrideLinkageRule.getOrElse(task.data.rule.execution()) - private val linkSpec = task.data.copy(rule = rule) + private val linkSpec = task.data.copy(rule = rule.operator) /** The warnings which occurred during execution */ @volatile private var warningLog: Seq[LogRecord] = Seq.empty @@ -62,11 +62,11 @@ class GenerateLinks(task: Task[LinkSpec], */ def warnings: Seq[LogRecord] = warningLog - override def initialValue: Option[Linking] = Some(Linking(task, rule)) + override def initialValue: Option[Linking] = Some(Linking(task, rule.operator)) override def run(context: ActivityContext[Linking]) (implicit userContext: UserContext): Unit = { - context.value.update(Linking(task, rule)) + context.value.update(Linking(task, rule.operator)) warningLog = CollectLogs() { if(RDBEntityIndex.configured() && runtimeConfig.executionBackend == LinkingExecutionBackend.rdb && false) { //FIXME CMEM-1408: Remove false to enable RDB feature @@ -98,7 +98,7 @@ class GenerateLinks(task: Task[LinkSpec], // Execute matching val sourceEqualsTarget = false // FIXME: CMEM-1975: Fix heuristic for this particular matching optimization val matcher = context.child(new Matcher(loaders, rule, caches, runtimeConfig, sourceEqualsTarget), 0.95) - val updateLinks = (result: MatcherResult) => context.value.update(Linking(task, rule, result.links, LinkingStatistics(entityCount = caches.map(_.size)), result.warnings)) + val updateLinks = (result: MatcherResult) => context.value.update(Linking(task, rule.operator, result.links, LinkingStatistics(entityCount = caches.map(_.size)), result.warnings)) matcher.value.subscribe(updateLinks) children ::= matcher matcher.startBlocking() @@ -110,7 +110,7 @@ class GenerateLinks(task: Task[LinkSpec], if(context.status.isCanceling) return // Filter links - val filterTask = new Filter(matcher.value().links, rule.filter) + val filterTask = new Filter(matcher.value().links, rule.operator.filter) var filteredLinks = context.child(filterTask, 0.03).startBlockingAndGetValue() if(context.status.isCanceling) return @@ -126,12 +126,12 @@ class GenerateLinks(task: Task[LinkSpec], } filteredLinks = filteredLinks.take(linkLimit) } - context.value.update(Linking(task, rule, filteredLinks, context.value().statistics, context.value().matcherWarnings, isDone = true, + context.value.update(Linking(task, rule.operator, filteredLinks, context.value().statistics, context.value().matcherWarnings, isDone = true, sampleOutputEntities = Seq(sampleEntities(filteredLinks)))) //Output links // TODO dont commit links to context if the task is not configured to hold links - val outputTask = new OutputWriter(context.value().links, rule.linkType, rule.inverseLinkType, output) + val outputTask = new OutputWriter(context.value().links, rule.operator.linkType, rule.operator.inverseLinkType, output) context.child(outputTask, 0.02).startBlocking() logStatistics(context) } @@ -190,8 +190,8 @@ class GenerateLinks(task: Task[LinkSpec], val sourceIndexFunction = (entity: Entity) => runtimeConfig.executionMethod.indexEntity(entity, rule, sourceOrTarget = true) val targetIndexFunction = (entity: Entity) => runtimeConfig.executionMethod.indexEntity(entity, rule, sourceOrTarget = false) - val sourceSchema = comparisonToRestrictionConverter.extendEntitySchemaWithLinkageRuleRestriction(entityDescs.source, rule, sourceOrTarget = true) - val targetSchema = comparisonToRestrictionConverter.extendEntitySchemaWithLinkageRuleRestriction(entityDescs.target, rule, sourceOrTarget = false) + val sourceSchema = comparisonToRestrictionConverter.extendEntitySchemaWithLinkageRuleRestriction(entityDescs.source, rule.operator, sourceOrTarget = true) + val targetSchema = comparisonToRestrictionConverter.extendEntitySchemaWithLinkageRuleRestriction(entityDescs.target, rule.operator, sourceOrTarget = false) if (runtimeConfig.useFileCache) { val cacheDir = new File(runtimeConfig.homeDir + "/entityCache/" + task.id + UUID.randomUUID().toString) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/Matcher.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/Matcher.scala index d55ba5c5d8..b59302df14 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/Matcher.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/Matcher.scala @@ -16,7 +16,7 @@ package org.silkframework.rule.execution import org.silkframework.cache.EntityCache import org.silkframework.entity.Link -import org.silkframework.rule.{LinkageRule, RuntimeLinkingConfig} +import org.silkframework.rule.{LinkageRuleExecution, RuntimeLinkingConfig} import org.silkframework.runtime.activity.Status.Waiting import org.silkframework.runtime.activity.{Activity, ActivityContext, ActivityControl, UserContext} import org.silkframework.runtime.execution.Execution @@ -37,7 +37,7 @@ import scala.math.min * @param sourceEqualsTarget Can be set to true if the source and the target cache are equal to enable faster matching in that case. */ class Matcher(loaders: DPair[ActivityControl[Unit]], - linkageRule: LinkageRule, + linkageRule: LinkageRuleExecution, caches: DPair[EntityCache], runtimeConfig: RuntimeLinkingConfig = RuntimeLinkingConfig(), sourceEqualsTarget: Boolean = false) extends Activity[MatcherResult] { @@ -283,7 +283,7 @@ class Matcher(loaders: DPair[ActivityControl[Unit]], val attachedEntities = if(runtimeConfig.generateLinksWithEntities) Some(entityPair) else None if(!runtimeConfig.indexingOnly) { - if(!linkageRule.excludeSelfReferences || entityPair.source.uri != entityPair.target.uri) { + if(!linkageRule.operator.excludeSelfReferences || entityPair.source.uri != entityPair.target.uri) { val confidence = linkageRule(entityPair, 0.0) if (confidence >= 0.0) { links = links :+ Link(entityPair.source.uri, entityPair.target.uri, Some(confidence), attachedEntities) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/local/LocalLinkSpecExecutor.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/local/LocalLinkSpecExecutor.scala index 773ed691e8..a4b7f8583d 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/local/LocalLinkSpecExecutor.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/local/LocalLinkSpecExecutor.scala @@ -42,7 +42,7 @@ class LocalLinkSpecExecutor extends Executor[LinkSpec, LocalExecution] { executionTimeout = Some(task.matchingExecutionTimeout * 1000L).filter(_ > 0) ) val taskContext = TaskContext(inputs.map(_.task), pluginContext) - val activity = new GenerateLinks(task, sources, None, linkConfig, Some(task.data.rule.withContext(taskContext))) + val activity = new GenerateLinks(task, sources, None, linkConfig, Some(linkSpec.rule.execution(taskContext))) var linking = context.child(activity, progressContribution = 1.0).startBlockingAndGetValue() if(adaptedLinkLimit < task.linkLimit) { linking = linking.copy(matcherWarnings = linking.matcherWarnings ++ Seq( diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/local/LocalTransformSpecExecutor.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/local/LocalTransformSpecExecutor.scala index 7e5f508e9d..5792127936 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/local/LocalTransformSpecExecutor.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/local/LocalTransformSpecExecutor.scala @@ -68,13 +68,13 @@ class LocalTransformSpecExecutor extends Executor[TransformSpec, LocalExecution] val (requestedRuleLabel, requestedRules, inputTable) = findMappingRulesMatchingRequestedOutputSchema(rules, ruleLabel, outputType, inputTables) addInputErrorsToTransformReport(inputTable, report) val transformedEntities = new TransformedEntities(task, inputTable.entities, requestedRuleLabel, - rule.withChildren(requestedRules).withContext(taskContext), activeOutputSchema, + rule.withChildren(requestedRules).execution(taskContext), activeOutputSchema, isRequestedSchema = true, abortIfErrorsOccur = task.data.abortIfErrorsOccur, report).iterator GenericEntityTable(transformedEntities, activeOutputSchema, task) case _ => // Else execute the complete mapping addInputErrorsToTransformReport(input, report) - val transformedEntities = new TransformedEntities(task, input.entities, ruleLabel, rule.withContext(taskContext), schemata.outputSchema, + val transformedEntities = new TransformedEntities(task, input.entities, ruleLabel, rule.execution(taskContext), schemata.outputSchema, isRequestedSchema = false, abortIfErrorsOccur = task.data.abortIfErrorsOccur, report).iterator GenericEntityTable(transformedEntities, schemata.outputSchema, task) } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/local/TransformedEntities.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/local/TransformedEntities.scala index 7cd9099ac9..597b907bca 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/local/TransformedEntities.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/local/TransformedEntities.scala @@ -7,7 +7,7 @@ import org.silkframework.execution.report.EntitySample import org.silkframework.execution.{AbortExecutionException, ExecutionException, ExecutionReport} import org.silkframework.failures.EntityException import org.silkframework.rule.execution.TransformReportBuilder -import org.silkframework.rule.{RootMappingRule, TransformRule, TransformSpec} +import org.silkframework.rule.{RootMappingRule, TransformRule, TransformRuleExecution, TransformSpec} import org.silkframework.runtime.iterator.CloseableIterator import org.silkframework.runtime.validation.ValidationException import org.silkframework.util.{Identifier, Uri} @@ -21,16 +21,15 @@ import scala.util.control.NonFatal * * @param task The transform task these entities originate from. * @param entities The source entities the transformation is applied to. - * @param rule The transformation rules that are applied against the source entities. + * @param ruleExecution The contextualized container rule that is applied against the source entities. * @param outputSchema The output schema of the transformation. * @param isRequestedSchema True, if the output schema was requested by the following task. False if this is the output * schema defined by the mapping itself. A requested schema is type agnostic. - * @param context The activity context. */ class TransformedEntities(task: Task[TransformSpec], entities: CloseableIterator[Entity], ruleLabel: String, - rule: TransformRule, + ruleExecution: TransformRuleExecution, outputSchema: EntitySchema, isRequestedSchema: Boolean, abortIfErrorsOccur: Boolean, @@ -38,18 +37,22 @@ class TransformedEntities(task: Task[TransformSpec], private val log: Logger = Logger.getLogger(this.getClass.getName) + private val rule: TransformRule = ruleExecution.operator + private val rules = rule.rules - private val subjectRule = rules.find(_.target.isEmpty) + private val ruleExecutions: Seq[TransformRuleExecution] = ruleExecution.childExecutions + + private val subjectRuleExecution = ruleExecutions.find(_.operator.target.isEmpty) - private val propertyRules = rules.filter(_.target.nonEmpty).toIndexedSeq + private val propertyRuleExecutions = ruleExecutions.filter(_.operator.target.nonEmpty).toIndexedSeq - // For each schema path, collect all rules that map to it - private val rulesPerPath = { + // For each schema path, collect all rule executions that map to it + private val ruleExecutionsPerPath = { if(isRequestedSchema) { - for(path <- outputSchema.typedPaths) yield propertyRules.filter(_.target.get.asPath() == path.asUntypedPath) + for(path <- outputSchema.typedPaths) yield propertyRuleExecutions.filter(_.operator.target.get.asPath() == path.asUntypedPath) } else { - for(path <- outputSchema.typedPaths) yield propertyRules.filter(_.target.get.asTypedPath() == path) + for(path <- outputSchema.typedPaths) yield propertyRuleExecutions.filter(_.operator.target.get.asTypedPath() == path) } } @@ -89,8 +92,8 @@ class TransformedEntities(task: Task[TransformSpec], errorFlag = false errors.clear() - val uris = subjectRule match { - case Some(rule) => evaluateRule(entity, rule, report).iterator + val uris = subjectRuleExecution match { + case Some(subjectExec) => evaluateRule(entity, subjectExec, report).iterator case None => Iterator(entity.uri.toString) } @@ -107,16 +110,16 @@ class TransformedEntities(task: Task[TransformSpec], Entity(entity.uri, values, entity.schema.copy(typedPaths = typedPaths)) } - def evalRule(rule: TransformRule): Seq[String] = { // evaluate rule on the correct entity representation - if (rule.representsDefaultUriRule) { - evaluateRule(objectEntity, rule, report) + def evalRule(exec: TransformRuleExecution): Seq[String] = { // evaluate rule on the correct entity representation + if (exec.operator.representsDefaultUriRule) { + evaluateRule(objectEntity, exec, report) } else { - evaluateRule(entity, rule, report) // This works even though there are still object paths mixed in, because they all are at the end + evaluateRule(entity, exec, report) // This works even though there are still object paths mixed in, because they all are at the end } } - val values = for (rules <- rulesPerPath) yield { - rules.flatMap(evalRule) + val values = for (rulesAtPath <- ruleExecutionsPerPath) yield { + rulesAtPath.flatMap(evalRule) } updateReport(report) @@ -124,8 +127,8 @@ class TransformedEntities(task: Task[TransformSpec], Entity(uri, values, outputSchema, metadata = buildErrorMetadata()) } } else { - for(uriRule <- subjectRule) { - report.addRuleError(uriRule, entity, new ValidationException("The URI pattern did not generate any URI for this entity.")) + for(uriRuleExec <- subjectRuleExecution) { + report.addRuleError(uriRuleExec.operator, entity, new ValidationException("The URI pattern did not generate any URI for this entity.")) } errorFlag = true Iterator.empty @@ -149,9 +152,10 @@ class TransformedEntities(task: Task[TransformSpec], } } - private def evaluateRule(entity: Entity, rule: TransformRule, report: TransformReportBuilder): Seq[String] = { + private def evaluateRule(entity: Entity, exec: TransformRuleExecution, report: TransformReportBuilder): Seq[String] = { + val rule = exec.operator try { - val result = rule(entity) + val result = exec(entity) for(error <- result.errors) { addError(report, rule, entity, error.error, Some(error.operatorId)) } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/Blocking.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/Blocking.scala index 327162135d..fb8a78086c 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/Blocking.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/Blocking.scala @@ -4,7 +4,7 @@ import org.silkframework.entity.paths.UntypedPath import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.execution.ExecutionMethod import org.silkframework.rule.plugins.transformer.linguistic.SoundexTransformer -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution import org.silkframework.rule.input.SimpleTransformer /** @@ -17,7 +17,7 @@ import org.silkframework.rule.input.SimpleTransformer * @param transformers A list of transformers that are applied to each value prior to blocking */ case class Blocking(sourceKey: UntypedPath, targetKey: UntypedPath, q: Int = 100, transformers: List[SimpleTransformer] = SoundexTransformer() :: Nil) extends ExecutionMethod { - override def indexEntity(entity: Entity, rule: LinkageRule, sourceOrTarget: Boolean): Index = { + override def indexEntity(entity: Entity, rule: LinkageRuleExecution, sourceOrTarget: Boolean): Index = { val key = if(sourceOrTarget) sourceKey else targetKey val values = entity.evaluate(key) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/CompositeBlocking.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/CompositeBlocking.scala index 944ef88a8f..304e945f4c 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/CompositeBlocking.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/CompositeBlocking.scala @@ -3,14 +3,14 @@ package org.silkframework.rule.execution.methods import org.silkframework.entity.paths.UntypedPath import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.execution.ExecutionMethod -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution /** * Blocking using a composite key built from two single keys. */ class CompositeBlocking(blockingKey1: UntypedPath, blockingKey2: UntypedPath) extends ExecutionMethod { - override def indexEntity(entity: Entity, rule: LinkageRule, sourceOrTarget: Boolean): Index = { + override def indexEntity(entity: Entity, rule: LinkageRuleExecution, sourceOrTarget: Boolean): Index = { val blocks = for(v1 <- entity.evaluate(blockingKey1); v2 <- entity.evaluate(blockingKey2)) yield { diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/Full.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/Full.scala index 005f924195..73d95b6e40 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/Full.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/Full.scala @@ -2,11 +2,11 @@ package org.silkframework.rule.execution.methods import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.execution.ExecutionMethod -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution /** * Full execution method. */ case class Full() extends ExecutionMethod { - override def indexEntity(entity: Entity, rule: LinkageRule, sourceOrTarget: Boolean): Index = Index.default + override def indexEntity(entity: Entity, rule: LinkageRuleExecution, sourceOrTarget: Boolean): Index = Index.default } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/MultiBlock.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/MultiBlock.scala index 1b24d902f0..787cd53b47 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/MultiBlock.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/MultiBlock.scala @@ -2,11 +2,11 @@ package org.silkframework.rule.execution.methods import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.execution.ExecutionMethod -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution /** * MultiBlock execution method. */ case class MultiBlock() extends ExecutionMethod { - override def indexEntity(entity: Entity, rule: LinkageRule, sourceOrTarget: Boolean): Index = rule.index(entity, sourceOrTarget, 0.0) + override def indexEntity(entity: Entity, rule: LinkageRuleExecution, sourceOrTarget: Boolean): Index = rule.index(entity, sourceOrTarget, 0.0) } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/MultiPassBlocking.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/MultiPassBlocking.scala index 7642f5ba5f..ec1af84620 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/MultiPassBlocking.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/MultiPassBlocking.scala @@ -3,7 +3,7 @@ package org.silkframework.rule.execution.methods import org.silkframework.entity.paths.UntypedPath import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.execution.ExecutionMethod -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution /** * Multi-pass blocking. @@ -12,7 +12,7 @@ import org.silkframework.rule.LinkageRule */ class MultiPassBlocking(blockingKeys: Set[UntypedPath]) extends ExecutionMethod { - override def indexEntity(entity: Entity, rule: LinkageRule, sourceOrTarget: Boolean): Index = { + override def indexEntity(entity: Entity, rule: LinkageRuleExecution, sourceOrTarget: Boolean): Index = { val values = blockingKeys.flatMap(key => entity.evaluate(key)) Index.oneDim(values.map(_.hashCode)) } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/QGrams.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/QGrams.scala index 0dbf985ecb..592452d456 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/QGrams.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/QGrams.scala @@ -3,7 +3,7 @@ package org.silkframework.rule.execution.methods import org.silkframework.entity.paths.UntypedPath import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.execution.ExecutionMethod -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution import scala.math._ @@ -22,7 +22,7 @@ case class QGrams(sourceKey: UntypedPath, targetKey: UntypedPath, q: Int = 2, t: require(q > 0, "q > 0") require(0.0 <= t && t < 1.0, "0 <= t < 1") - override def indexEntity(entity: Entity, rule: LinkageRule, sourceOrTarget: Boolean): Index = { + override def indexEntity(entity: Entity, rule: LinkageRuleExecution, sourceOrTarget: Boolean): Index = { val key = if(sourceOrTarget) sourceKey else targetKey val values = entity.evaluate(key) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/SortedBlocks.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/SortedBlocks.scala index e829863ffb..b7b46492e5 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/SortedBlocks.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/SortedBlocks.scala @@ -3,7 +3,7 @@ package org.silkframework.rule.execution.methods import org.silkframework.entity.paths.UntypedPath import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.execution.ExecutionMethod -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution import scala.math.{max, min, pow} @@ -15,7 +15,7 @@ case class SortedBlocks(sourceKey: UntypedPath, targetKey: UntypedPath, overlap: private val blockCount = pow(maxChar - minChar + 1, 2).toInt - override def indexEntity(entity: Entity, rule: LinkageRule, sourceOrTarget: Boolean): Index = { + override def indexEntity(entity: Entity, rule: LinkageRuleExecution, sourceOrTarget: Boolean): Index = { val key = if(sourceOrTarget) sourceKey else targetKey val values = entity.evaluate(key) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/StringMap.scala b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/StringMap.scala index 29085968ab..b4ea87f75a 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/StringMap.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/execution/methods/StringMap.scala @@ -3,7 +3,7 @@ package org.silkframework.rule.execution.methods import org.silkframework.cache.Partition import org.silkframework.entity.paths.UntypedPath import org.silkframework.entity.{Entity, Index} -import org.silkframework.rule.LinkageRule +import org.silkframework.rule.LinkageRuleExecution import org.silkframework.rule.execution.ExecutionMethod import org.silkframework.rule.execution.methods.StringMap.Mapper import org.silkframework.rule.plugins.distance.characterbased.LevenshteinDistance @@ -16,7 +16,7 @@ case class StringMap(sourceKey: UntypedPath, targetKey: UntypedPath, distThresho * Generates an index for a single entity. * StringMap don't uses the indexing. */ - override def indexEntity(entity: Entity, rule: LinkageRule, sourceOrTarget: Boolean): Index = Index.default + override def indexEntity(entity: Entity, rule: LinkageRuleExecution, sourceOrTarget: Boolean): Index = Index.default /** * Generates comparison pairs from two partitions. diff --git a/silk-rules/src/main/scala/org/silkframework/rule/input/Input.scala b/silk-rules/src/main/scala/org/silkframework/rule/input/Input.scala index d64f509b7d..80ecc2bb85 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/input/Input.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/input/Input.scala @@ -16,7 +16,7 @@ package org.silkframework.rule.input import org.silkframework.entity.paths.UntypedPath import org.silkframework.entity.Entity -import org.silkframework.rule.{Operator, TaskContext} +import org.silkframework.rule.{Operator, OperatorExecution, TaskContext} import org.silkframework.runtime.serialization.{ReadContext, WriteContext, XmlFormat, XmlSerialization} import scala.xml.Node @@ -25,6 +25,14 @@ import scala.xml.Node * An input that retrieves a set of values. */ trait Input extends Operator { + override def execution(taskContext: TaskContext = TaskContext.empty): InputExecution +} + +/** + * Runtime executor for an [[Input]] operator. + */ +trait InputExecution extends OperatorExecution { + override def operator: Input /** * Retrieves the values of this input for a given entity. @@ -33,8 +41,14 @@ trait Input extends Operator { * @return The values. */ def apply(entity: Entity): Value +} - override def withContext(taskContext: TaskContext): Input = this +/** + * An [[Input]] that has no task-context dependency and serves as its own executor. + */ +trait InlineInput extends Input with InputExecution { + override def operator: Input = this + override def execution(taskContext: TaskContext = TaskContext.empty): InputExecution = this } object Input { diff --git a/silk-rules/src/main/scala/org/silkframework/rule/input/PathInput.scala b/silk-rules/src/main/scala/org/silkframework/rule/input/PathInput.scala index e232a2a973..5b3f83db22 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/input/PathInput.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/input/PathInput.scala @@ -26,7 +26,7 @@ import scala.xml.Node /** * A PathInput retrieves values from a data item by a given RDF path and optionally applies a transform to them. */ -case class PathInput(id: Identifier = Operator.generateId, path: Path) extends Input { +case class PathInput(id: Identifier = Operator.generateId, path: Path) extends InlineInput { @volatile private var cachedPathIndex = -1 diff --git a/silk-rules/src/main/scala/org/silkframework/rule/input/SimpleTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/input/SimpleTransformer.scala index 709a347f4d..4b59b1b725 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/input/SimpleTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/input/SimpleTransformer.scala @@ -19,7 +19,7 @@ import scala.collection.immutable.ArraySeq /** * Simple transformer which transforms all values of all inputs. */ -abstract class SimpleTransformer extends Transformer { +abstract class SimpleTransformer extends InlineTransformer { override final def apply(values: Seq[Seq[String]]): Seq[String] = { var numberOfValues = 0 diff --git a/silk-rules/src/main/scala/org/silkframework/rule/input/TransformInput.scala b/silk-rules/src/main/scala/org/silkframework/rule/input/TransformInput.scala index 324d2ae243..8fbea6c05f 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/input/TransformInput.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/input/TransformInput.scala @@ -33,15 +33,41 @@ import scala.collection.immutable.ArraySeq * @param transformer The transformer used to transform values. * @param inputs The children operators from which input values are read. */ -case class TransformInput(id: Identifier = Operator.generateId, transformer: Transformer, inputs: IndexedSeq[Input] = IndexedSeq.empty) extends Input { +case class TransformInput(id: Identifier = Operator.generateId, + transformer: Transformer, + inputs: IndexedSeq[Input] = IndexedSeq.empty) extends Input { - def apply(entity: Entity): Value = { - val inputValues = new Array[Seq[String]](inputs.length) + override def children: Seq[Input] = inputs + + override def withId(newId: Identifier): Operator = copy(id = newId) + + override def withChildren(newChildren: Seq[Operator]): TransformInput = { + copy(inputs = newChildren.map(_.asInstanceOf[Input]).toIndexedSeq) + } + + override def execution(taskContext: TaskContext = TaskContext.empty): InputExecution = { + new TransformInputExecution( + operator = this, + inputExecutions = inputs.map(_.execution(taskContext)), + transformerExecution = transformer.execution(taskContext)) + } +} + +/** + * Runtime executor of a [[TransformInput]] with all child input executors and the + * transformer execution resolved against a specific [[TaskContext]]. + */ +final class TransformInputExecution(override val operator: TransformInput, + val inputExecutions: IndexedSeq[InputExecution], + val transformerExecution: TransformerExecution) extends InputExecution { + + override def apply(entity: Entity): Value = { + val inputValues = new Array[Seq[String]](inputExecutions.length) var errors = Seq[OperatorEvaluationError]() // Evaluate input operators - for(i <- inputs.indices) { - val result = inputs(i)(entity) + for(i <- inputExecutions.indices) { + val result = inputExecutions(i)(entity) inputValues(i) = result.values for(error <- result.errors) { errors +:= error @@ -50,27 +76,15 @@ case class TransformInput(id: Identifier = Operator.generateId, transformer: Tra // Evaluate transform try { - Value(transformer(ArraySeq.unsafeWrapArray(inputValues)), errors) + Value(transformerExecution(ArraySeq.unsafeWrapArray(inputValues)), errors) } catch { case ex: ExecutionException if ex.abortExecution => throw ex case NonFatal(ex) => - errors +:= OperatorEvaluationError(id, ex) + errors +:= OperatorEvaluationError(operator.id, ex) Value(Seq.empty, errors) } } - - override def children: Seq[Input] = inputs - - override def withId(newId: Identifier): Operator = copy(id = newId) - - override def withChildren(newChildren: Seq[Operator]): TransformInput = { - copy(inputs = newChildren.map(_.asInstanceOf[Input]).toIndexedSeq) - } - - override def withContext(taskContext: TaskContext): Input = { - copy(transformer = transformer.withContext(taskContext), inputs = inputs.map(_.withContext(taskContext))) - } } object TransformInput { diff --git a/silk-rules/src/main/scala/org/silkframework/rule/input/Transformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/input/Transformer.scala index 4be2342dd0..b107ebbe35 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/input/Transformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/input/Transformer.scala @@ -30,17 +30,11 @@ import org.silkframework.runtime.templating.TemplateVariableName ) trait Transformer extends AnyPlugin { - def withContext(taskContext: TaskContext): Transformer = this - /** - * Transforms a sequence of values - * - * @param values A sequence which contains as many elements as there are input operators for this transformation. - * For each input operator it contains a sequence of values. - * - * @return The transformed sequence of values. - */ - def apply(values: Seq[Seq[String]]): Seq[String] + * Returns an executor that performs the actual transformation. + * Simple transformers may return this object itself. + */ + def execution(taskContext: TaskContext = TaskContext.empty): TransformerExecution /** * The resources that are directly referenced by this transformer. @@ -66,3 +60,24 @@ trait Transformer extends AnyPlugin { } object Transformer extends PluginFactory[Transformer] + +trait TransformerExecution { + + /** + * Transforms a sequence of values + * + * @param values A sequence which contains as many elements as there are input operators for this transformation. + * For each input operator it contains a sequence of values. + * + * @return The transformed sequence of values. + */ + def apply(values: Seq[Seq[String]]): Seq[String] +} + +/** + * A Transformer whose instance is also its own executor. + * `withContext` returns `this`, so the same object serves as both descriptor and runtime executor. + */ +trait InlineTransformer extends Transformer with TransformerExecution { + override def execution(taskContext: TaskContext = TaskContext.empty): TransformerExecution = this +} \ No newline at end of file diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/aggegrator/MinimumAggregator.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/aggegrator/MinimumAggregator.scala index 46f4b571f7..b97ad022f6 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/aggegrator/MinimumAggregator.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/aggegrator/MinimumAggregator.scala @@ -16,7 +16,7 @@ package org.silkframework.rule.plugins.aggegrator import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.annotations.{AggregatorExample, AggregatorExamples} -import org.silkframework.rule.similarity.{Aggregator, SimilarityOperator, SimilarityScore} +import org.silkframework.rule.similarity.{Aggregator, SimilarityOperatorExecution, SimilarityScore} import org.silkframework.runtime.plugin.PluginCategories import org.silkframework.runtime.plugin.annotations.Plugin import org.silkframework.util.DPair @@ -54,7 +54,7 @@ case class MinimumAggregator() extends Aggregator { /** * Returns the minimum of the provided values. */ - override def apply(operators: Seq[SimilarityOperator], entities: DPair[Entity], limit: Double): SimilarityScore = { + override def apply(operators: Seq[SimilarityOperatorExecution], entities: DPair[Entity], limit: Double): SimilarityScore = { var minScore = Double.MaxValue if(operators.isEmpty) { return SimilarityScore.none diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatMultipleValuesTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatMultipleValuesTransformer.scala index bd51f29fce..ed3e0e5b98 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatMultipleValuesTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatMultipleValuesTransformer.scala @@ -3,7 +3,7 @@ package org.silkframework.rule.plugins.transformer.combine import org.silkframework.rule.annotations.{TransformExample, TransformExamples} import java.util.regex.Pattern -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} /** @@ -51,7 +51,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} output = Array("a\n\t\\b\n\t\\c") ) )) -case class ConcatMultipleValuesTransformer(glue: String = "", removeDuplicates: Boolean = false) extends Transformer { +case class ConcatMultipleValuesTransformer(glue: String = "", removeDuplicates: Boolean = false) extends InlineTransformer { // glue with escaped char sequences (\\, \n, \t) converted to actual character. lazy val parsedGlue: String = ConcatTransformer.parseGlue(glue) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatPairwiseTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatPairwiseTransformer.scala index 77beb8732d..f5c89e1b73 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatPairwiseTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatPairwiseTransformer.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.combine import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} @Plugin( @@ -67,7 +67,7 @@ import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginRefere case class ConcatPairwiseTransformer( @Param("Separator to be inserted between two concatenated strings. The text can contain escaped characters \\n, \\t and" + " \\\\ that are replaced by a newline, tab or backslash respectively.") - glue: String = "") extends Transformer { + glue: String = "") extends InlineTransformer { // glue with escaped char sequences (\\, \n, \t) converted to actual character. private lazy val parsedGlue: String = ConcatTransformer.parseGlue(glue) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatTransformer.scala index 2159957bc4..9b87a44ae7 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ConcatTransformer.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.combine import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} @Plugin( @@ -97,7 +97,7 @@ case class ConcatTransformer( @Param(ConcatTransformer.glueDescription) glue: String = "", @Param("Handle missing values as empty strings.") - missingValuesAsEmptyStrings: Boolean = false) extends Transformer { + missingValuesAsEmptyStrings: Boolean = false) extends InlineTransformer { // glue with escaped char sequences (\\, \n, \t) converted to actual character. lazy val parsedGlue: String = ConcatTransformer.parseGlue(glue) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/MergeTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/MergeTransformer.scala index bfa3d3eed2..39e2be5789 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/MergeTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/MergeTransformer.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.combine import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin @Plugin( @@ -34,7 +34,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin output = Array("a", "b", "c") ) )) -case class MergeTransformer() extends Transformer { +case class MergeTransformer() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { if(values.nonEmpty) { values.reduce(_ concat _) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ZipTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ZipTransformer.scala index 6496e38265..3cfcd0f43a 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ZipTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/combine/ZipTransformer.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.combine import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} @Plugin( @@ -64,7 +64,7 @@ case class ZipTransformer(@Param(value = "Placeholder to be used if the first in @Param(value = "Placeholder to be used if the second input provides fewer values than the first one.") secondPlaceholder: String = "", @Param(value = ConcatTransformer.glueDescription) - glue: String = "") extends Transformer { + glue: String = "") extends InlineTransformer { // glue with escaped char sequences (\\, \n, \t) converted to actual character. lazy val parsedGlue: String = ConcatTransformer.parseGlue(glue) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/ContainsAllOf.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/ContainsAllOf.scala index 40806111b0..ce9f38adc3 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/ContainsAllOf.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/ContainsAllOf.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.conditional import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin @Plugin( @@ -47,7 +47,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin throwsException = classOf[java.lang.IllegalArgumentException] ) )) -case class ContainsAllOf() extends Transformer { +case class ContainsAllOf() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { require(values.size == 2, "The containsAllOf transformation accepts two inputs") val input1 = values.head diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/ContainsAnyOf.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/ContainsAnyOf.scala index a511c09ebe..783607cd20 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/ContainsAnyOf.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/ContainsAnyOf.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.conditional import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin @Plugin( @@ -47,7 +47,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin throwsException = classOf[java.lang.IllegalArgumentException] ) )) -case class ContainsAnyOf() extends Transformer { +case class ContainsAnyOf() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { require(values.size == 2, "The containsAnyOf transformation accepts exactly two inputs") val input1 = values.head diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfContains.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfContains.scala index b421b2dbef..e41acd4d6b 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfContains.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfContains.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.conditional import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin @Plugin( @@ -31,7 +31,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin output = Array("this is no match") ) )) -case class IfContains(search: String) extends Transformer { +case class IfContains(search: String) extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { require(values.size >= 2, "The ifContains transformation accepts two or three inputs") if(values.head.exists(_.contains(search))) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfExists.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfExists.scala index 6252fca879..4eee417d0e 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfExists.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfExists.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.conditional import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin @Plugin( @@ -29,7 +29,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin output = Array() ) )) -case class IfExists() extends Transformer { +case class IfExists() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { require(values.size >= 2, "The ifExists transformation requires at least two inputs") if(values.head.nonEmpty) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfMatchesRegexTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfMatchesRegexTransformer.scala index 3eded81be1..39c7adabc2 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfMatchesRegexTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/IfMatchesRegexTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.conditional import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.extraction.RegexExtractionTransformer import org.silkframework.rule.plugins.transformer.selection.RegexSelectTransformer import org.silkframework.rule.plugins.transformer.validation.ValidateRegex @@ -54,7 +54,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} output = Array() ), )) -case class IfMatchesRegexTransformer(regex: String, negate: Boolean = false) extends Transformer { +case class IfMatchesRegexTransformer(regex: String, negate: Boolean = false) extends InlineTransformer { val r = regex.r override def apply(values: Seq[Seq[String]]): Seq[String] = { require(values.size >= 2, "ifMatchesRegex operator needs at least two inputs!") diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/Negate.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/Negate.scala index f340b722cd..3aec2010be 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/Negate.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/conditional/Negate.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.conditional import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin @Plugin( @@ -24,7 +24,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin throwsException = classOf[java.lang.IllegalArgumentException] ) )) -case class Negate() extends Transformer { +case class Negate() extends InlineTransformer { def negate(value: String): String ={ value.trim.toLowerCase match{ diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/CompareDatesTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/CompareDatesTransformer.scala index 1077da969c..f384ee01ba 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/CompareDatesTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/CompareDatesTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.date import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.ComparatorEnum import org.silkframework.util.StringUtils.XSDDateLiteral import org.silkframework.rule.plugins.transformer.ComparatorEnum._ @@ -67,7 +67,7 @@ import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginRefere output = Array("0") ) )) -case class CompareDatesTransformer(comparator: ComparatorEnum = ComparatorEnum.less) extends Transformer { +case class CompareDatesTransformer(comparator: ComparatorEnum = ComparatorEnum.less) extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { // Collect all dates in milliseconds diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/CurrentDateTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/CurrentDateTransformer.scala index 2b61d1a3d0..b399f6077a 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/CurrentDateTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/CurrentDateTransformer.scala @@ -4,7 +4,7 @@ import java.util.GregorianCalendar import javax.xml.datatype.DatatypeFactory import org.silkframework.rule.plugins.transformer.date.CurrentDateTransformer._ -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} @Plugin( @@ -19,7 +19,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} ) ) ) -case class CurrentDateTransformer() extends Transformer { +case class CurrentDateTransformer() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { val currentDate = datatypeFactory.newXMLGregorianCalendar(new GregorianCalendar).toXMLFormat diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/DurationTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/DurationTransformer.scala index 5094b23b46..bd39f4bd8b 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/DurationTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/DurationTransformer.scala @@ -2,7 +2,7 @@ package org.silkframework.rule.plugins.transformer.date import javax.xml.bind.DatatypeConverter import javax.xml.datatype.DatatypeFactory -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} @Plugin( @@ -17,7 +17,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} ) ) ) -case class DurationTransformer() extends Transformer { +case class DurationTransformer() extends InlineTransformer { private val datatypeFactory = DatatypeFactory.newInstance() diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/ParseDateTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/ParseDateTransformer.scala index f3f6a1593f..70312b91e3 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/ParseDateTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/date/ParseDateTransformer.scala @@ -17,7 +17,7 @@ package org.silkframework.rule.plugins.transformer.date import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} import org.silkframework.runtime.plugin.types.LocaleOptionParameter import org.silkframework.runtime.plugin.types.autoComlpetionProviders.LocaleParameterAutoCompletionProvider @@ -85,7 +85,7 @@ case class ParseDateTransformer( lenient: Boolean = false, @Param(value = "Optional locale for the date format. If not set the system's locale will be used.", autoCompletionProvider = classOf[LocaleParameterAutoCompletionProvider]) - locale: LocaleOptionParameter = LocaleOptionParameter(None)) extends Transformer with Serializable { + locale: LocaleOptionParameter = LocaleOptionParameter(None)) extends InlineTransformer with Serializable { def apply(values: Seq[Seq[String]]): Seq[String] = { values.flatten.flatMap(parse) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/extraction/RegexExtractionTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/extraction/RegexExtractionTransformer.scala index 115e3306b5..fe480c8cae 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/extraction/RegexExtractionTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/extraction/RegexExtractionTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.extraction import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.conditional.IfMatchesRegexTransformer import org.silkframework.rule.plugins.transformer.replace.RegexReplaceTransformer import org.silkframework.rule.plugins.transformer.selection.RegexSelectTransformer @@ -91,7 +91,7 @@ case class RegexExtractionTransformer( @Param("Regular expression") regex: String, @Param("If true, all matches are extracted. If false, only the first match is extracted (default).") - extractAll: Boolean = false) extends Transformer { + extractAll: Boolean = false) extends InlineTransformer { lazy val r = regex.r diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/FilterByLength.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/FilterByLength.scala index 3526130ca3..9fb9bb17f0 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/FilterByLength.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/FilterByLength.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.plugins.transformer.filter -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin /** @@ -12,7 +12,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin label = "Filter by length", description = "Removes all strings that are shorter than 'min' characters and longer than 'max' characters." ) -case class FilterByLength(min: Int = 0, max: Int = Int.MaxValue) extends Transformer { +case class FilterByLength(min: Int = 0, max: Int = Int.MaxValue) extends InlineTransformer { override def apply(values: Seq[Seq[String]]) = { values.head.filterNot(str => str.length < min || str.length > max) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/FilterByRegex.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/FilterByRegex.scala index fc3cbf0ad2..012d77e0ec 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/FilterByRegex.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/FilterByRegex.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.plugins.transformer.filter -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.selection.RegexSelectTransformer import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} @@ -19,7 +19,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} ) ) ) -case class FilterByRegex(regex: String, negate: Boolean = false) extends Transformer { +case class FilterByRegex(regex: String, negate: Boolean = false) extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { if(!negate) { diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/RemoveEmptyValues.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/RemoveEmptyValues.scala index 39a7c22c61..985de64db9 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/RemoveEmptyValues.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/RemoveEmptyValues.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.filter import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.value.EmptyValueTransformer import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} @@ -45,7 +45,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} output = Array() ) )) -case class RemoveEmptyValues() extends Transformer { +case class RemoveEmptyValues() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { values.head.filter(!_.isEmpty) } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/RemoveValues.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/RemoveValues.scala index 9ed9fbfa14..cf0dc11319 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/RemoveValues.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/filter/RemoveValues.scala @@ -14,7 +14,7 @@ package org.silkframework.rule.plugins.transformer.filter -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.normalize.RemoveDuplicates import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} @@ -34,7 +34,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} ) ) ) -case class RemoveValues(blacklist: String) extends Transformer { +case class RemoveValues(blacklist: String) extends InlineTransformer { val filterValues = blacklist.split(",").map(_.toLowerCase).toSet override def apply(values: Seq[Seq[String]]) = { diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/linguistic/SpotlightTextVectorTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/linguistic/SpotlightTextVectorTransformer.scala index 060efc6d10..98b6890ffc 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/linguistic/SpotlightTextVectorTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/linguistic/SpotlightTextVectorTransformer.scala @@ -2,7 +2,7 @@ package org.silkframework.rule.plugins.transformer.linguistic import java.net.{HttpURLConnection, URL, URLEncoder} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin import org.silkframework.util.HttpURLConnectionUtils._ @@ -23,7 +23,7 @@ import scala.xml.{Elem, XML} label = "Spotlight", description = "Concatenates all values to a string and gets a weighted entity vector from the Spotlight service." ) -case class SpotlightTextVectorTransformer() extends Transformer { +case class SpotlightTextVectorTransformer() extends InlineTransformer { def apply(values: Seq[Seq[String]]): Seq[String] = { val stringSet = values.reduce(_ concat _) if(stringSet.isEmpty) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/metadata/FileHashTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/metadata/FileHashTransformer.scala index e8ae00cc83..ca27fce538 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/metadata/FileHashTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/metadata/FileHashTransformer.scala @@ -2,7 +2,7 @@ package org.silkframework.rule.plugins.transformer.metadata import org.silkframework.dataset.{DatasetSpec, ResourceBasedDataset} import org.silkframework.rule.TaskContext -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.{InlineTransformer, TransformerExecution} import org.silkframework.rule.plugins.transformer.value.HashAlgorithmAutoCompletionProvider import org.silkframework.runtime.plugin.annotations.{Param, Plugin} import org.silkframework.runtime.plugin.types.ResourceOption @@ -27,13 +27,13 @@ case class FileHashTransformer(@Param(value = "File for which the hash sum will file: ResourceOption = None, @Param(value = "The hash algorithm to be used.", autoCompletionProvider = classOf[HashAlgorithmAutoCompletionProvider], allowOnlyAutoCompletedValues = true) - algorithm: String = "SHA256") extends Transformer { + algorithm: String = "SHA256") extends InlineTransformer { require(algorithm.trim.nonEmpty, "Algorithm must not be empty. Please specify an algorithm, such as 'SHA256'.") private val cache = file.resource.map(new Cache(_, algorithm)) - override def withContext(taskContext: TaskContext): Transformer = { + override def execution(taskContext: TaskContext): TransformerExecution = { taskContext.inputTasks.headOption.map(_.data) match { case Some(DatasetSpec(ds: ResourceBasedDataset, _, _)) if file.isEmpty => FileHashTransformer(Some(ds.file), algorithm) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/metadata/InputFileAttributesTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/metadata/InputFileAttributesTransformer.scala index 1a892eb677..5e2876c4c1 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/metadata/InputFileAttributesTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/metadata/InputFileAttributesTransformer.scala @@ -2,7 +2,7 @@ package org.silkframework.rule.plugins.transformer.metadata import org.silkframework.dataset.{DatasetSpec, ResourceBasedDataset} import org.silkframework.rule.TaskContext -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.{Transformer, TransformerExecution} import org.silkframework.rule.plugins.transformer.value.ConstantTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin} import org.silkframework.runtime.resource.{Resource, ResourceManager} @@ -19,19 +19,15 @@ case class InputFileAttributesTransformer(@Param("File attribute to be retrieved attribute: FileAttributeEnum = FileAttributeEnum.name) extends Transformer { - override def withContext(taskContext: TaskContext): Transformer = { + override def execution(taskContext: TaskContext): TransformerExecution = { taskContext.inputTasks.headOption.map(_.data) match { case Some(DatasetSpec(ds: ResourceBasedDataset, _, _)) => ConstantTransformer(getAttribute(ds.file, taskContext.pluginContext.resources)) case _ => - this + throw new ValidationException("No resource supplied by input dataset.") } } - override def apply(values: Seq[Seq[String]]): Seq[String] = { - throw new ValidationException("No resource supplied by input dataset.") - } - private def getAttribute(file: Resource, resourceManager: ResourceManager): String = { attribute match { case FileAttributeEnum.name => file.name diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/normalize/RemoveDuplicates.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/normalize/RemoveDuplicates.scala index b9a0c9b689..344a824cf2 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/normalize/RemoveDuplicates.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/normalize/RemoveDuplicates.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.plugins.transformer.normalize -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.filter.RemoveValues import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} @@ -16,7 +16,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} ) ) ) -case class RemoveDuplicates() extends Transformer { +case class RemoveDuplicates() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { values.flatten.distinct diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/AggregateNumbersTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/AggregateNumbersTransformer.scala index 53223c85e3..0297518eb1 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/AggregateNumbersTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/AggregateNumbersTransformer.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.numeric import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} import org.silkframework.util.StringUtils.DoubleLiteral @@ -126,7 +126,7 @@ import org.silkframework.util.StringUtils.DoubleLiteral case class AggregateNumbersTransformer( @Param("The aggregation operation to be applied to all values. One of `+`, `*`, `min`, `max`, `average`.") operator: String -) extends Transformer { +) extends InlineTransformer { require(Set("+", "*", "min", "max", "average") contains operator, "Operator must be one of '+', '*', 'min', 'max', 'average'") diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/CompareNumbersTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/CompareNumbersTransformer.scala index 0043c5db83..78155b4c82 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/CompareNumbersTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/CompareNumbersTransformer.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.plugins.transformer.numeric -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.ComparatorEnum import org.silkframework.rule.plugins.transformer.date.CompareDatesTransformer import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} @@ -25,7 +25,7 @@ For instance, {1,2} < {2,3} yields 0 as not all numbers in the first set are sma ) ) ) -case class CompareNumbersTransformer(comparator: ComparatorEnum = ComparatorEnum.less) extends Transformer { +case class CompareNumbersTransformer(comparator: ComparatorEnum = ComparatorEnum.less) extends InlineTransformer { def apply(values: Seq[Seq[String]]): Seq[String] = { // Collect all numbers diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/CountTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/CountTransformer.scala index f3a9b6bb2d..e15f081d33 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/CountTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/CountTransformer.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.numeric import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin /** @@ -40,7 +40,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin output = Array("2") ) )) -case class CountTransformer() extends Transformer { +case class CountTransformer() extends InlineTransformer { def apply(values: Seq[Seq[String]]): Seq[String] = { Seq(values.flatten.size.toString) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/NumOperationTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/NumOperationTransformer.scala index 4adb9bcc6b..252cad2baa 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/NumOperationTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/NumOperationTransformer.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.numeric import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} import org.silkframework.runtime.validation.ValidationException import org.silkframework.util.StringUtils.DoubleLiteral @@ -100,7 +100,7 @@ import org.silkframework.util.StringUtils.DoubleLiteral case class NumOperationTransformer( @Param("The operator to be applied to all values. One of `+`, `-`, `*`, `/`") operator: String -) extends Transformer { +) extends InlineTransformer { require(Set("+", "-", "*", "/") contains operator, "Operator must be one of '+', '-', '*', '/'") diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/PhysicalQuantityExtractor.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/PhysicalQuantityExtractor.scala index 2a9fa431e8..9b597d1613 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/PhysicalQuantityExtractor.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/numeric/PhysicalQuantityExtractor.scala @@ -2,7 +2,7 @@ package org.silkframework.rule.plugins.transformer.numeric import java.text.NumberFormat import java.util.Locale -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} import scala.util.matching.Regex @@ -32,7 +32,7 @@ case class PhysicalQuantityExtractor(@Param("The symbol of the dimension, e.g., @Param("Only extracts from values that contain the given regex (case-insensitive).") filter: String = "", @Param("If there are multiple matches, retrieve the value with the given index (zero-based).") - index: Int = 0) extends Transformer { + index: Int = 0) extends InlineTransformer { private val unitPrefixes = Map( "p" -> 0.000000000001, diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/replace/MapTransformerWithDefaultInput.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/replace/MapTransformerWithDefaultInput.scala index a47b7d5e37..8092e66e30 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/replace/MapTransformerWithDefaultInput.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/replace/MapTransformerWithDefaultInput.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.plugins.transformer.replace -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.value.InputHashTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} @@ -28,7 +28,7 @@ import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginRefere ) ) case class MapTransformerWithDefaultInput(@Param(value = "A map of values", example = "A:1,B:2,C:3") - map: Map[String, String]) extends Transformer { + map: Map[String, String]) extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { if(values.size != 2) { throw new IllegalArgumentException("MapDefaultInput takes exactly two inputs! Instead found " + values.size + " inputs.") diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/selection/CoalesceTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/selection/CoalesceTransformer.scala index 079d402e5d..4af649ff06 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/selection/CoalesceTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/selection/CoalesceTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.selection import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin @TransformExamples(Array( @@ -43,7 +43,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin categories = Array("Selection"), description = "Forwards the first non-empty input, i.e. for which any value(s) exist. A single empty string is considered a value." ) -case class CoalesceTransformer() extends Transformer { +case class CoalesceTransformer() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { values.find(_.nonEmpty).getOrElse(Seq.empty) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/selection/RegexSelectTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/selection/RegexSelectTransformer.scala index 0751975607..33e9b9002d 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/selection/RegexSelectTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/selection/RegexSelectTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.selection import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.conditional.IfMatchesRegexTransformer import org.silkframework.rule.plugins.transformer.extraction.RegexExtractionTransformer import org.silkframework.rule.plugins.transformer.filter.FilterByRegex @@ -77,7 +77,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} output = Array("output", "", "") ), )) -case class RegexSelectTransformer(oneOnly: Boolean = false) extends Transformer { +case class RegexSelectTransformer(oneOnly: Boolean = false) extends InlineTransformer { override def apply(inputs: Seq[Seq[String]]): Seq[String] = { require(inputs.size == 3, "The ") require(inputs.head.nonEmpty, "The first input needs to have at least one value!") diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/GetValueByIndexTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/GetValueByIndexTransformer.scala index 0dd04392a3..8ac9ca75e1 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/GetValueByIndexTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/GetValueByIndexTransformer.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.plugins.transformer.sequence -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin /** @@ -21,7 +21,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin ) case class GetValueByIndexTransformer(index: Int, failIfNotFound: Boolean = false, - emptyStringToEmptyResult: Boolean = false) extends Transformer { + emptyStringToEmptyResult: Boolean = false) extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { values.flatMap { vs => vs.drop(index).headOption match { diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/SortTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/SortTransformer.scala index ee6234851f..954f9fc54b 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/SortTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/SortTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.sequence import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin @Plugin( @@ -24,7 +24,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin output = Array("Hamburg", "Hans", "Hansa") ) )) -case class SortTransformer() extends Transformer { +case class SortTransformer() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { values.flatten.sorted diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/ValuesToIndexesTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/ValuesToIndexesTransformer.scala index c925870f5f..b9ee5f791e 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/ValuesToIndexesTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/sequence/ValuesToIndexesTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.sequence import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin /** @@ -22,7 +22,7 @@ import org.silkframework.runtime.plugin.annotations.Plugin output = Array("0", "1", "2") )) ) -case class ValuesToIndexesTransformer() extends Transformer { +case class ValuesToIndexesTransformer() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { values.flatten.zipWithIndex map { _._2.toString } } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/tokenization/CamelCaseTokenizer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/tokenization/CamelCaseTokenizer.scala index ef8de8269a..03acc84f65 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/tokenization/CamelCaseTokenizer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/tokenization/CamelCaseTokenizer.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.tokenization import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} import scala.collection.mutable.ArrayBuffer @@ -42,7 +42,7 @@ import scala.collection.mutable.ArrayBuffer output = Array("nocamelcase") ) )) -case class CamelCaseTokenizer() extends Transformer { +case class CamelCaseTokenizer() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { values.reduce(_ ++ _).flatMap(splitOnCamelCase) } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/tokenization/Tokenizer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/tokenization/Tokenizer.scala index 77d95ee882..84b709c756 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/tokenization/Tokenizer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/tokenization/Tokenizer.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.tokenization import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.PluginCategories import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} @@ -46,7 +46,7 @@ import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginRefere )) case class Tokenizer( @Param("The regular expression used to split values.") - regex: String = "\\s") extends Transformer { + regex: String = "\\s") extends InlineTransformer { private[this] val compiledRegex = regex.r diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/validation/ValidateDateAfter.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/validation/ValidateDateAfter.scala index 133c715c35..cb92c6b666 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/validation/ValidateDateAfter.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/validation/ValidateDateAfter.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.validation import javax.xml.datatype.DatatypeFactory -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import ValidateDateAfter._ import org.silkframework.rule.annotations.{TransformExample, TransformExamples} import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} @@ -59,7 +59,7 @@ import org.silkframework.runtime.validation.ValidationException )) case class ValidateDateAfter( @Param(value = "Allow both dates to be equal.") - allowEqual: Boolean = false) extends Transformer { + allowEqual: Boolean = false) extends InlineTransformer { def apply(values: Seq[Seq[String]]): Seq[String] = { require(values.length == 2, "ValidateDateOrder accepts exactly two inputs") diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/validation/ValidateNumberOValues.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/validation/ValidateNumberOValues.scala index a1e4c8d868..3067825024 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/validation/ValidateNumberOValues.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/validation/ValidateNumberOValues.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.plugins.transformer.validation import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin} import org.silkframework.runtime.validation.ValidationException @@ -42,7 +42,7 @@ case class ValidateNumberOValues( @Param(value = "Minimum allowed number of values") min: Int = 0, @Param(value = "Maximum allowed number of values") - max: Int = 1) extends Transformer { + max: Int = 1) extends InlineTransformer { def apply(values: Seq[Seq[String]]): Seq[String] = { val allValues = values.flatten diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/ConstantTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/ConstantTransformer.scala index 644ede3077..370fe02b5b 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/ConstantTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/ConstantTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.value import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin} import org.silkframework.runtime.plugin.PluginCategories @@ -20,7 +20,7 @@ import org.silkframework.runtime.plugin.PluginCategories )) case class ConstantTransformer( @Param("The constant value to be generated") - value: String = "") extends Transformer { + value: String = "") extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { Seq(value) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/ConstantUriTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/ConstantUriTransformer.scala index 1846c9a341..61fd70d815 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/ConstantUriTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/ConstantUriTransformer.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.plugins.transformer.value -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin} import org.silkframework.util.Uri @@ -12,7 +12,7 @@ import org.silkframework.util.Uri ) case class ConstantUriTransformer( @Param("The constant URI to be generated") - value: Uri = Uri("http://www.w3.org/2002/07/owl#Class")) extends Transformer { + value: Uri = Uri("http://www.w3.org/2002/07/owl#Class")) extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { Seq(value.toString) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/DefaultValueTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/DefaultValueTransformer.scala index 9eebbdf038..5c6b8ea5f3 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/DefaultValueTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/DefaultValueTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.value import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.PluginCategories import org.silkframework.runtime.plugin.annotations.{Param, Plugin} @@ -26,7 +26,7 @@ import org.silkframework.runtime.plugin.annotations.{Param, Plugin} )) case class DefaultValueTransformer( @Param("The default value to be generated, if input values are empty") - value: String = "default") extends Transformer { + value: String = "default") extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { val allValues = values.flatten diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/EmptyValueTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/EmptyValueTransformer.scala index ad3cd3b6a2..5b0c5354b0 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/EmptyValueTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/EmptyValueTransformer.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.plugins.transformer.value -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.filter.RemoveEmptyValues import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} @@ -16,7 +16,7 @@ import org.silkframework.runtime.plugin.annotations.{Plugin, PluginReference} ) ) ) -case class EmptyValueTransformer() extends Transformer { +case class EmptyValueTransformer() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { Seq.empty diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/GenerateUUID.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/GenerateUUID.scala index e2bb12f319..2b753f19f7 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/GenerateUUID.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/GenerateUUID.scala @@ -3,7 +3,7 @@ package org.silkframework.rule.plugins.transformer.value import java.nio.charset.Charset import java.util.UUID -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.Plugin import GenerateUUID._ import org.silkframework.rule.annotations.{TransformExample, TransformExamples} @@ -29,7 +29,7 @@ Each input value will generate a separate UUID. For building a UUID from multipl output = Array("690802dd-a317-335f-807c-e4e1e32b7b5b", "925cbd7f-377b-3fbd-8f4c-ca41529b74ad") ) )) -case class GenerateUUID() extends Transformer { +case class GenerateUUID() extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { if(values.isEmpty) { diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/InputHashTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/InputHashTransformer.scala index 535860a8ee..b77e95ba22 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/InputHashTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/InputHashTransformer.scala @@ -1,7 +1,7 @@ package org.silkframework.rule.plugins.transformer.value import org.silkframework.rule.annotations.{TransformExample, TransformExamples} -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.rule.plugins.transformer.replace.MapTransformerWithDefaultInput import org.silkframework.runtime.plugin.annotations.{Param, Plugin, PluginReference} import org.silkframework.runtime.plugin.{AutoCompletionResult, ParamValue, PluginContext, PluginParameterAutoCompletionProvider} @@ -32,7 +32,7 @@ import scala.jdk.CollectionConverters.IterableHasAsScala )) case class InputHashTransformer(@Param(value = "The hash algorithm to be used.", autoCompletionProvider = classOf[HashAlgorithmAutoCompletionProvider], allowOnlyAutoCompletedValues = true) - algorithm: String = "SHA256") extends Transformer { + algorithm: String = "SHA256") extends InlineTransformer { require(algorithm.trim.nonEmpty, "Algorithm must not be empty. Please specify an algorithm, such as 'SHA256'.") diff --git a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/RandomNumberTransformer.scala b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/RandomNumberTransformer.scala index 068739465c..df442ea456 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/RandomNumberTransformer.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/plugins/transformer/value/RandomNumberTransformer.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.plugins.transformer.value -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin} import scala.util.Random @@ -19,7 +19,7 @@ case class RandomNumberTransformer( @Param("The minimum number of values to generate in each set.") minCount: Int = 1, @Param("The maximum number of values to generate in each set.") - maxCount: Int = 1) extends Transformer { + maxCount: Int = 1) extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { val count = minCount + Random.nextInt(1 + maxCount - minCount) diff --git a/silk-rules/src/main/scala/org/silkframework/rule/similarity/Aggregation.scala b/silk-rules/src/main/scala/org/silkframework/rule/similarity/Aggregation.scala index ae4a0d4469..240756d13b 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/similarity/Aggregation.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/similarity/Aggregation.scala @@ -36,36 +36,6 @@ case class Aggregation(id: Identifier = Operator.generateId, def indexing: Boolean = operators.exists(_.indexing) - /** - * Computes the similarity between two entities. - * - * @param entities The entities to be compared. - * @param limit The similarity threshold. - * @return The similarity as a value between -1.0 and 1.0. - * None, if no similarity could be computed. - */ - override def apply(entities: DPair[Entity], limit: Double): Option[Double] = { - aggregator(operators, entities, limit).score - } - - /** - * Indexes an entity. - * - * @param entity The entity to be indexed - * @param threshold The similarity threshold. - * @return A set of (multidimensional) indexes. Entities within the threshold will always get the same index. - */ - override def index(entity: Entity, sourceOrTarget: Boolean, threshold: Double): Index = { - val indexSets = { - for (op <- operators if op.indexing) yield { - val index = op.index(entity, sourceOrTarget, threshold) - index - } - } - - aggregator.aggregateIndexes(indexSets) - } - override def validate(): Seq[ValidationIssue] = { operators.flatMap(_.validate()) } @@ -80,8 +50,29 @@ case class Aggregation(id: Identifier = Operator.generateId, copy(operators = newChildren.map(_.asInstanceOf[SimilarityOperator])) } - override def withContext(taskContext: TaskContext): Aggregation = { - copy(operators = operators.map(_.withContext(taskContext))) + override def execution(taskContext: TaskContext = TaskContext.empty): SimilarityOperatorExecution = { + new AggregationExecution(this, operators.map(_.execution(taskContext))) + } +} + +/** + * Runtime executor for an [[Aggregation]] with all child similarity operator executions resolved. + */ +final class AggregationExecution(override val operator: Aggregation, + val operatorExecutions: Seq[SimilarityOperatorExecution]) extends SimilarityOperatorExecution { + + override def apply(entities: DPair[Entity], limit: Double): Option[Double] = { + operator.aggregator(operatorExecutions, entities, limit).score + } + + override def index(entity: Entity, sourceOrTarget: Boolean, threshold: Double): Index = { + val indexSets = { + for (op <- operatorExecutions if op.operator.indexing) yield { + op.index(entity, sourceOrTarget, threshold) + } + } + + operator.aggregator.aggregateIndexes(indexSets) } } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/similarity/Aggregator.scala b/silk-rules/src/main/scala/org/silkframework/rule/similarity/Aggregator.scala index b66a84103b..a1fdcac901 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/similarity/Aggregator.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/similarity/Aggregator.scala @@ -26,7 +26,7 @@ import org.silkframework.util.DPair ) trait Aggregator extends AnyPlugin { - def apply(operators: Seq[SimilarityOperator], entities: DPair[Entity], limit: Double): SimilarityScore + def apply(operators: Seq[SimilarityOperatorExecution], entities: DPair[Entity], limit: Double): SimilarityScore /** * Aggregates or manipulates the child indexes. The default implementation leaves the indexes as they are. diff --git a/silk-rules/src/main/scala/org/silkframework/rule/similarity/Comparison.scala b/silk-rules/src/main/scala/org/silkframework/rule/similarity/Comparison.scala index c8a34e9436..c69ed657f3 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/similarity/Comparison.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/similarity/Comparison.scala @@ -16,7 +16,7 @@ package org.silkframework.rule.similarity import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.{Operator, TaskContext} -import org.silkframework.rule.input.Input +import org.silkframework.rule.input.{Input, InputExecution} import org.silkframework.rule.similarity.Comparison.distanceToScore import org.silkframework.runtime.plugin.PluginBackwardCompatibility import org.silkframework.runtime.serialization.{ReadContext, WriteContext, XmlFormat, XmlSerialization} @@ -39,15 +39,52 @@ case class Comparison(id: Identifier = Operator.generateId, require(weight > 0, "weight > 0") - private val sourceInput = inputs.source - private val targetInput = inputs.target + override def validate(): Seq[ValidationIssue] = { + for(message <- metric.validateThreshold(threshold).toSeq) yield { + ValidationWarning(message, Some(id), Some("Comparison")) + } + } + + override def children: Seq[Input] = inputs.toSeq + + override def withId(newId: Identifier): Operator = { + copy(id = newId) + } + + override def withChildren(newChildren: Seq[Operator]): Comparison = { + copy(inputs = DPair.fromSeq(newChildren.collect{ case input: Input => input })) + } + + override def execution(taskContext: TaskContext = TaskContext.empty): SimilarityOperatorExecution = { + // Each input should receive only the task that it refers to (source or target). + // For non-linking contexts (e.g. detailed evaluation in the workbench) the task + // context may be empty; in that case the same context is propagated to both inputs. + taskContext.inputTasks match { + case Seq(sourceTask, targetTask) => + new ComparisonExecution( + operator = this, + sourceInput = inputs.source.execution(taskContext.copy(inputTasks = Seq(sourceTask))), + targetInput = inputs.target.execution(taskContext.copy(inputTasks = Seq(targetTask)))) + case Seq() => + new ComparisonExecution( + operator = this, + sourceInput = inputs.source.execution(taskContext), + targetInput = inputs.target.execution(taskContext)) + case _ => + throw new ValidationException("Comparison operator expects exactly two inputs") + } + } +} + +/** + * Runtime executor for a [[Comparison]] with both child input executors resolved. + */ +final class ComparisonExecution(override val operator: Comparison, + val sourceInput: InputExecution, + val targetInput: InputExecution) extends SimilarityOperatorExecution { /** * Computes the similarity between two entities. - * - * @param entities The entities to be compared. - * @param limit The confidence limit. - * @return The confidence as a value between -1.0 and 1.0. */ override def apply(entities: DPair[Entity], limit: Double): Option[Double] = { val values1 = @@ -68,53 +105,21 @@ case class Comparison(id: Identifier = Operator.generateId, if (values1.isEmpty || values2.isEmpty) { None } else { - val distance = metric(values1, values2, threshold * (1.0 - limit)) - Some(distanceToScore(distance, threshold)) + val distance = operator.metric(values1, values2, operator.threshold * (1.0 - limit)) + Some(distanceToScore(distance, operator.threshold)) } } /** - * Indexes an entity. - * - * @param entity The entity to be indexed - * @param limit The similarity threshold. - * @param sourceOrTarget If true the value comes from the source, else from the target. - * @return A set of (multidimensional) indexes. Entities within the threshold will always get the same index. - */ + * Indexes an entity. + */ override def index(entity: Entity, sourceOrTarget: Boolean, limit: Double): Index = { - val values = Try(inputs.select(sourceOrTarget)(entity).values).getOrElse(Seq.empty) - - val distanceLimit = threshold * (1.0 - limit) - - metric.index(values, distanceLimit, sourceOrTarget) - } - - override def validate(): Seq[ValidationIssue] = { - for(message <- metric.validateThreshold(threshold).toSeq) yield { - ValidationWarning(message, Some(id), Some("Comparison")) - } - } - - override def children: Seq[Input] = inputs.toSeq + val input = if (sourceOrTarget) sourceInput else targetInput + val values = Try(input(entity).values).getOrElse(Seq.empty) - override def withId(newId: Identifier): Operator = { - copy(id = newId) - } + val distanceLimit = operator.threshold * (1.0 - limit) - override def withChildren(newChildren: Seq[Operator]): Comparison = { - copy(inputs = DPair.fromSeq(newChildren.collect{ case input: Input => input })) - } - - override def withContext(taskContext: TaskContext): Comparison = { - // Each input should receive only the task that it refers to (source or target) - taskContext.inputTasks match { - case Seq(sourceTask, targetTask) => - val newSourceInput = inputs.source.withContext(taskContext.copy(inputTasks = Seq(sourceTask))) - val newTargetInput = inputs.target.withContext(taskContext.copy(inputTasks = Seq(targetTask))) - copy(inputs = DPair(newSourceInput, newTargetInput)) - case _ => - throw new ValidationException("Comparison operator expects exactly two inputs") - } + operator.metric.index(values, distanceLimit, sourceOrTarget) } } diff --git a/silk-rules/src/main/scala/org/silkframework/rule/similarity/SimilarityOperator.scala b/silk-rules/src/main/scala/org/silkframework/rule/similarity/SimilarityOperator.scala index b0ea4ddde2..1097bf1582 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/similarity/SimilarityOperator.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/similarity/SimilarityOperator.scala @@ -15,7 +15,7 @@ package org.silkframework.rule.similarity import org.silkframework.entity.{Entity, Index} -import org.silkframework.rule.{Operator, TaskContext} +import org.silkframework.rule.{Operator, OperatorExecution, TaskContext} import org.silkframework.runtime.serialization.XmlSerialization._ import org.silkframework.runtime.serialization.{ReadContext, WriteContext, XmlFormat} import org.silkframework.runtime.validation.ValidationIssue @@ -33,6 +33,25 @@ trait SimilarityOperator extends Operator { def indexing: Boolean + /** + * Validates this rule operator. + * This should cover non-fatal issues that should be fixed by the user after rule creation. + * Issues that lead to an inconsistent and unusable rule should not be checked here, but instead throw an exception in the constructor. + */ + def validate(): Seq[ValidationIssue] = { + Seq.empty + } + + override def execution(taskContext: TaskContext = TaskContext.empty): SimilarityOperatorExecution +} + +/** + * Runtime executor for a [[SimilarityOperator]]. + */ +trait SimilarityOperatorExecution extends OperatorExecution { + + override def operator: SimilarityOperator + /** * Computes the similarity between two entities. * @@ -51,19 +70,14 @@ trait SimilarityOperator extends Operator { * @return A set of (multidimensional) indexes. Entities within the threshold will always get the same index. */ def index(entity: Entity, sourceOrTarget: Boolean, limit: Double): Index +} - /** - * Validates this rule operator. - * This should cover non-fatal issues that should be fixed by the user after rule creation. - * Issues that lead to an inconsistent and unusable rule should not be checked here, but instead throw an exception in the constructor. - */ - def validate(): Seq[ValidationIssue] = { - Seq.empty - } - - override def withContext(taskContext: TaskContext): SimilarityOperator = { - this - } +/** + * A [[SimilarityOperator]] that has no task-context dependency and serves as its own executor. + */ +trait InlineSimilarityOperator extends SimilarityOperator with SimilarityOperatorExecution { + override def operator: SimilarityOperator = this + override def execution(taskContext: TaskContext = TaskContext.empty): SimilarityOperatorExecution = this } object SimilarityOperator { diff --git a/silk-rules/src/main/scala/org/silkframework/rule/similarity/SimpleAggregator.scala b/silk-rules/src/main/scala/org/silkframework/rule/similarity/SimpleAggregator.scala index b18ae690e5..acecf2e607 100644 --- a/silk-rules/src/main/scala/org/silkframework/rule/similarity/SimpleAggregator.scala +++ b/silk-rules/src/main/scala/org/silkframework/rule/similarity/SimpleAggregator.scala @@ -7,11 +7,11 @@ import scala.collection.mutable.ArrayBuffer trait SimpleAggregator extends Aggregator { - def apply(operators: Seq[SimilarityOperator], entities: DPair[Entity], limit: Double): SimilarityScore = { + def apply(operators: Seq[SimilarityOperatorExecution], entities: DPair[Entity], limit: Double): SimilarityScore = { val weightedValues = new ArrayBuffer[WeightedSimilarityScore](operators.size) for(op <- operators) { val score = op(entities, limit) - weightedValues += WeightedSimilarityScore(score, op.weight) + weightedValues += WeightedSimilarityScore(score, op.operator.weight) } evaluate(weightedValues.toSeq) diff --git a/silk-rules/src/test/scala/org/silkframework/rule/RuleTraverserTest.scala b/silk-rules/src/test/scala/org/silkframework/rule/RuleTraverserTest.scala index d6a4b354a0..67e71314f8 100644 --- a/silk-rules/src/test/scala/org/silkframework/rule/RuleTraverserTest.scala +++ b/silk-rules/src/test/scala/org/silkframework/rule/RuleTraverserTest.scala @@ -35,6 +35,7 @@ class RuleTraverserTest extends AnyFlatSpec with Matchers { case class Node(id: Identifier, children: Seq[Node] = Seq.empty) extends Operator { override def withId(newId: Identifier): Operator = copy(id = newId) override def withChildren(newChildren: Seq[Operator]): Operator = copy(children = newChildren.map(_.asInstanceOf[Node])) + override def execution(taskContext: TaskContext): OperatorExecution = new OperatorExecution { override def operator: Operator = Node.this } } } diff --git a/silk-rules/src/test/scala/org/silkframework/rule/execution/ExecuteTransformTest.scala b/silk-rules/src/test/scala/org/silkframework/rule/execution/ExecuteTransformTest.scala index be1a8718b7..1641874fb8 100644 --- a/silk-rules/src/test/scala/org/silkframework/rule/execution/ExecuteTransformTest.scala +++ b/silk-rules/src/test/scala/org/silkframework/rule/execution/ExecuteTransformTest.scala @@ -11,7 +11,7 @@ import org.silkframework.entity.{Entity, EntitySchema} import org.silkframework.execution.local.GenericEntityTable import org.silkframework.plugins.dataset.InternalDataset import org.silkframework.rule._ -import org.silkframework.rule.input.{PathInput, TransformInput, Transformer} +import org.silkframework.rule.input.{InlineTransformer, PathInput, TransformInput, Transformer} import org.silkframework.runtime.activity.{ActivityContext, StatusHolder, UserContext, ValueHolder} import org.silkframework.runtime.iterator.CloseableIterator import org.silkframework.runtime.plugin.PluginContext @@ -53,7 +53,7 @@ class ExecuteTransformTest extends AnyFlatSpec with Matchers with MockitoSugar { } private def transformerWithExceptions(): Transformer = { - new Transformer { + new InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { values.flatten map { v => if(v == "invalid") { diff --git a/silk-rules/src/test/scala/org/silkframework/rule/execution/local/LocalTransformSpecExecutorTest.scala b/silk-rules/src/test/scala/org/silkframework/rule/execution/local/LocalTransformSpecExecutorTest.scala index 874c1c479c..7bdbf29691 100644 --- a/silk-rules/src/test/scala/org/silkframework/rule/execution/local/LocalTransformSpecExecutorTest.scala +++ b/silk-rules/src/test/scala/org/silkframework/rule/execution/local/LocalTransformSpecExecutorTest.scala @@ -10,7 +10,7 @@ import org.silkframework.entity.paths.{TypedPath, UntypedPath} import org.silkframework.execution.local.{GenericEntityTable, LocalExecution, MultiEntityTable} import org.silkframework.execution.{ExecutionReport, ExecutorOutput, ExecutorRegistry} import org.silkframework.rule._ -import org.silkframework.rule.input.{PathInput, TransformInput, Transformer} +import org.silkframework.rule.input.{InlineTransformer, PathInput, TransformInput, Transformer, TransformerExecution} import org.silkframework.runtime.activity.{ActivityContext, ActivityMonitor, TestUserContextTrait} import org.silkframework.runtime.iterator.CloseableIterator import org.silkframework.runtime.plugin.PluginContext @@ -160,17 +160,13 @@ class LocalTransformSpecExecutorTest extends AnyFlatSpec with Matchers with Exec case class TestTransformer() extends Transformer { - override def withContext(taskContext: TaskContext): Transformer = { + override def execution(taskContext: TaskContext): TransformerExecution = { TransformerWithContext(taskContext) } - override def apply(values: Seq[Seq[String]]): Seq[String] = { - Seq.empty - } - } -case class TransformerWithContext(taskContext: TaskContext) extends Transformer { +case class TransformerWithContext(taskContext: TaskContext) extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = { Seq(taskContext.inputTasks.head.id.toString) diff --git a/silk-rules/src/test/scala/org/silkframework/rule/plugins/transformer/metadata/FileHashTransformerTest.scala b/silk-rules/src/test/scala/org/silkframework/rule/plugins/transformer/metadata/FileHashTransformerTest.scala index ae1ed60e1f..9c323b7c8b 100644 --- a/silk-rules/src/test/scala/org/silkframework/rule/plugins/transformer/metadata/FileHashTransformerTest.scala +++ b/silk-rules/src/test/scala/org/silkframework/rule/plugins/transformer/metadata/FileHashTransformerTest.scala @@ -27,7 +27,7 @@ class FileHashTransformerTest extends AnyFlatSpec with Matchers { val dataset = PlainTask("input", DatasetSpec(CsvDataset(file = resource))) val taskContext = TaskContext(Seq(dataset), PluginContext.empty) - val transformer = FileHashTransformer(None).withContext(taskContext) + val transformer = FileHashTransformer(None).execution(taskContext) transformer(Seq.empty) shouldBe Seq("b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78") } diff --git a/silk-rules/src/test/scala/org/silkframework/rule/plugins/transformer/metadata/InputFileAttributesTransformerTest.scala b/silk-rules/src/test/scala/org/silkframework/rule/plugins/transformer/metadata/InputFileAttributesTransformerTest.scala index 6e4a5032a2..4ca7d51b25 100644 --- a/silk-rules/src/test/scala/org/silkframework/rule/plugins/transformer/metadata/InputFileAttributesTransformerTest.scala +++ b/silk-rules/src/test/scala/org/silkframework/rule/plugins/transformer/metadata/InputFileAttributesTransformerTest.scala @@ -28,7 +28,7 @@ class InputFileAttributesTransformerTest extends AnyFlatSpec with Matchers with def retrieve(attribute: FileAttributeEnum): String = { val taskContext = TaskContext(Seq(dataset), PluginContext(Prefixes.empty, resources, UserContext.Empty)) - val transformer = InputFileAttributesTransformer(attribute).withContext(taskContext) + val transformer = InputFileAttributesTransformer(attribute).execution(taskContext) transformer(Seq.empty).head } diff --git a/silk-rules/src/test/scala/org/silkframework/rule/similarity/AggregationTest.scala b/silk-rules/src/test/scala/org/silkframework/rule/similarity/AggregationTest.scala index ae5bf01e23..ee1876c31b 100644 --- a/silk-rules/src/test/scala/org/silkframework/rule/similarity/AggregationTest.scala +++ b/silk-rules/src/test/scala/org/silkframework/rule/similarity/AggregationTest.scala @@ -19,7 +19,7 @@ import org.silkframework.config.Prefixes import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.plugins.aggegrator.AverageAggregator import org.silkframework.rule.Operator -import org.silkframework.rule.similarity.{Aggregation, SimilarityOperator} +import org.silkframework.rule.similarity.{Aggregation, InlineSimilarityOperator, SimilarityOperator, SimilarityOperatorExecution} import org.silkframework.testutil.approximatelyEqualToOption import org.silkframework.util.{DPair, Identifier} @@ -62,22 +62,22 @@ class AggregationTest extends AnyFlatSpec with Matchers { } private def eval(ops: Seq[SimilarityOperator]) = { - Aggregation(operators = ops, aggregator = aggregator).apply(null) + Aggregation(operators = ops, aggregator = aggregator).execution().apply(null) } private def index(ops: Seq[SimilarityOperator]) = { - Aggregation(operators = ops, aggregator = aggregator).index(null, true, 0.0) + Aggregation(operators = ops, aggregator = aggregator).execution().index(null, true, 0.0) } private def operator(_weight: Int = 1, _required: Boolean = false, value: Option[Double] = None, indices: Index = Index.default) = { - new SimilarityOperator { + new InlineSimilarityOperator { val id: Identifier = Identifier.random val weight: Int = _weight val indexing: Boolean = true - def apply(entities: DPair[Entity], limit: Double): Option[Double] = value - def index(entity: Entity, sourceOrTarget: Boolean, limit: Double): Index = indices + override def apply(entities: DPair[Entity], limit: Double): Option[Double] = value + override def index(entity: Entity, sourceOrTarget: Boolean, limit: Double): Index = indices def toXML(implicit prefixes: Prefixes): Node = null - def children = Seq.empty + def children: Seq[Operator] = Seq.empty override def withId(newId: Identifier): Operator = ??? def withChildren(newChildren: Seq[Operator]): Operator = ??? } diff --git a/silk-rules/src/test/scala/org/silkframework/rule/similarity/ComparisonTest.scala b/silk-rules/src/test/scala/org/silkframework/rule/similarity/ComparisonTest.scala index 745a9fa978..ff2f219349 100644 --- a/silk-rules/src/test/scala/org/silkframework/rule/similarity/ComparisonTest.scala +++ b/silk-rules/src/test/scala/org/silkframework/rule/similarity/ComparisonTest.scala @@ -18,7 +18,7 @@ package org.silkframework.rule.plugins.similarity import org.silkframework.config.Prefixes import org.silkframework.entity.Entity import org.silkframework.rule.Operator -import org.silkframework.rule.input.{Input, Value} +import org.silkframework.rule.input.{InlineInput, Input, Value} import org.silkframework.rule.similarity.{Comparison, DistanceMeasure} import org.silkframework.testutil.approximatelyEqualTo import org.silkframework.util.{DPair, Identifier} @@ -44,15 +44,15 @@ class ComparisonTest extends AnyFlatSpec with Matchers { threshold = threshold, metric = new DistanceMeasure { def apply(values1: Seq[String], values2: Seq[String], limit: Double) = distance }, inputs = DPair.fill(DummyInput) - ).apply(DPair.fill(null), -1.0).get + ).execution().apply(DPair.fill(null), -1.0).get } - private object DummyInput extends Input { + private object DummyInput extends InlineInput { val id = Identifier.random - def apply(entity: Entity): Value = Value(Seq("dummy")) + override def apply(entity: Entity): Value = Value(Seq("dummy")) def toXML(implicit prefixes: Prefixes): Node = null - def children = Seq.empty + def children: Seq[Operator] = Seq.empty def withId(newId: Identifier): Operator = ??? - def withChildren(newChildren: Seq[Operator]) = ??? + def withChildren(newChildren: Seq[Operator]): Operator = ??? } } diff --git a/silk-rules/src/test/scala/org/silkframework/rule/test/AggregatorTest.scala b/silk-rules/src/test/scala/org/silkframework/rule/test/AggregatorTest.scala index 6182f20dc4..34ea8ce830 100644 --- a/silk-rules/src/test/scala/org/silkframework/rule/test/AggregatorTest.scala +++ b/silk-rules/src/test/scala/org/silkframework/rule/test/AggregatorTest.scala @@ -2,7 +2,7 @@ package org.silkframework.rule.test import org.silkframework.entity.{Entity, Index} import org.silkframework.rule.Operator -import org.silkframework.rule.similarity.{Aggregator, AggregatorExampleValue, SimilarityOperator} +import org.silkframework.rule.similarity.{Aggregator, AggregatorExampleValue, InlineSimilarityOperator, SimilarityOperator} import org.silkframework.runtime.plugin.{AnyPlugin, ClassPluginDescription, ParameterValues, PluginContext, TestPluginContext} import org.silkframework.test.PluginTest import org.silkframework.util.{DPair, Identifier} @@ -95,7 +95,7 @@ abstract class AggregatorTest[T <: Aggregator : ClassTag] extends PluginTest { } } - private case class DummyOperator(score: Option[Double], weight: Int) extends SimilarityOperator { + private case class DummyOperator(score: Option[Double], weight: Int) extends InlineSimilarityOperator { override def id: Identifier = "dummy" diff --git a/silk-rules/src/test/scala/org/silkframework/rule/test/TransformerTest.scala b/silk-rules/src/test/scala/org/silkframework/rule/test/TransformerTest.scala index 5b1da78c2f..7da2131849 100644 --- a/silk-rules/src/test/scala/org/silkframework/rule/test/TransformerTest.scala +++ b/silk-rules/src/test/scala/org/silkframework/rule/test/TransformerTest.scala @@ -1,6 +1,6 @@ package org.silkframework.rule.test -import org.silkframework.rule.input.{TransformExampleValue, Transformer} +import org.silkframework.rule.input.{InlineTransformer, TransformExampleValue} import org.silkframework.runtime.plugin.{AnyPlugin, ClassPluginDescription, ParameterValues, PluginContext, TestPluginContext} import org.silkframework.test.PluginTest @@ -8,7 +8,7 @@ import java.util.logging.Logger import scala.reflect.ClassTag import scala.util.control.NonFatal -abstract class TransformerTest[T <: Transformer : ClassTag] extends PluginTest { +abstract class TransformerTest[T <: InlineTransformer : ClassTag] extends PluginTest { /** Numeric values may differ slightly from their expected values. */ private val epsilon = 0.0001 diff --git a/silk-tools/silk-freetext-preprocessing/src/main/scala/org/silkframework/preprocessing/transformer/Ngrams.scala b/silk-tools/silk-freetext-preprocessing/src/main/scala/org/silkframework/preprocessing/transformer/Ngrams.scala index 630d28d011..2ebd0d3c61 100644 --- a/silk-tools/silk-freetext-preprocessing/src/main/scala/org/silkframework/preprocessing/transformer/Ngrams.scala +++ b/silk-tools/silk-freetext-preprocessing/src/main/scala/org/silkframework/preprocessing/transformer/Ngrams.scala @@ -7,7 +7,7 @@ package org.silkframework.preprocessing.transformer * @param lower The lower bound * @param upper The upper bound */ -class Ngrams(lower: Int, upper: Int) extends Transformer{ +class Ngrams(lower: Int, upper: Int) extends InlineTransformer{ def organize(ss: List[String]): String = { ss.reduce(_+" "+_) diff --git a/silk-tools/silk-freetext-preprocessing/src/main/scala/org/silkframework/preprocessing/transformer/Rounder.scala b/silk-tools/silk-freetext-preprocessing/src/main/scala/org/silkframework/preprocessing/transformer/Rounder.scala index 5ada577b56..199114c288 100644 --- a/silk-tools/silk-freetext-preprocessing/src/main/scala/org/silkframework/preprocessing/transformer/Rounder.scala +++ b/silk-tools/silk-freetext-preprocessing/src/main/scala/org/silkframework/preprocessing/transformer/Rounder.scala @@ -6,7 +6,7 @@ package org.silkframework.preprocessing.transformer * * @param round Takes floor or ceil as function */ -case class Rounder(round: Double => Double) extends Transformer{ +case class Rounder(round: Double => Double) extends InlineTransformer{ private[this] val compiledRegex = "^[+-]?\\d+(\\.\\d+)?$".r private val empty = "" diff --git a/silk-workbench/silk-workbench-core/test/cacheUpdater/CacheUpdaterIntegrationTest.scala b/silk-workbench/silk-workbench-core/test/cacheUpdater/CacheUpdaterIntegrationTest.scala index ff96a3bfda..67be647b09 100644 --- a/silk-workbench/silk-workbench-core/test/cacheUpdater/CacheUpdaterIntegrationTest.scala +++ b/silk-workbench/silk-workbench-core/test/cacheUpdater/CacheUpdaterIntegrationTest.scala @@ -10,7 +10,7 @@ import org.scalatest.time.Span import org.silkframework.dataset.DatasetSpec import org.silkframework.dataset.DatasetSpec.GenericDatasetSpec import org.silkframework.plugins.dataset.json.{JsonDataset, JsonSink} -import org.silkframework.rule.input.{TransformInput, Transformer} +import org.silkframework.rule.input.{InlineTransformer, TransformInput} import org.silkframework.rule._ import org.silkframework.runtime.resource.Resource import org.silkframework.util.ConfigTestTrait @@ -75,7 +75,7 @@ class CacheUpdaterIntegrationTest() extends AnyFlatSpec with IntegrationTestTrai /** * Counts each update of the provided file resource. */ -case class CountingTransformer(file: Resource) extends Transformer { +case class CountingTransformer(file: Resource) extends InlineTransformer { val updateCounter = new AtomicInteger() diff --git a/silk-workbench/silk-workbench-rules/app/controllers/linking/EvaluateLinkingController.scala b/silk-workbench/silk-workbench-rules/app/controllers/linking/EvaluateLinkingController.scala index 9d9ba3fff3..a837ac78b9 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/linking/EvaluateLinkingController.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/linking/EvaluateLinkingController.scala @@ -51,9 +51,10 @@ class EvaluateLinkingController @Inject() (implicit system: ActorSystem, if(showLinks) { val referenceLinks = task.data.referenceLinks + val ruleExec = task.data.rule.execution() def links = for (link <- linking.links.view) yield { - val detailedLink = DetailedEvaluator(task.data.rule, link.entities.get) + val detailedLink = DetailedEvaluator(ruleExec, link.entities.get) if (referenceLinks.positive.contains(link)) new EvalLink(detailedLink, Correct, Generated) else if (referenceLinks.negative.contains(link)) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/linking/LinkingEditor.scala b/silk-workbench/silk-workbench-rules/app/controllers/linking/LinkingEditor.scala index cae0146c34..07c4cc28fa 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/linking/LinkingEditor.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/linking/LinkingEditor.scala @@ -9,6 +9,7 @@ import org.silkframework.util.DPair import org.silkframework.workbench.Context import org.silkframework.workbench.workspace.WorkbenchAccessMonitor import org.silkframework.workspace.WorkspaceFactory +import org.silkframework.workspace.activity.linking.LinkingTaskUtils._ import org.silkframework.workspace.activity.linking.{LinkingPathsCache, ReferenceEntitiesCache} import play.api.mvc.{Action, AnyContent, InjectedController} @@ -69,7 +70,7 @@ class LinkingEditor @Inject() (implicit accessMonitor: WorkbenchAccessMonitor, w // If everything needed for computing a score is available } else { try { - val result = LinkageRuleEvaluator(task.data.rule, entitiesCache.value()) + val result = LinkageRuleEvaluator(task.ruleWithContext, entitiesCache.value()) val score = f"Precision: ${result.precision}%.2f | Recall: ${result.recall}%.2f | F-measure: ${result.fMeasure}%.2f" Ok(views.html.editor.score(score)) } catch { diff --git a/silk-workbench/silk-workbench-rules/app/controllers/linking/LinkingTaskApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/linking/LinkingTaskApi.scala index d979c12e7e..d6fefb6b5e 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/linking/LinkingTaskApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/linking/LinkingTaskApi.scala @@ -21,7 +21,7 @@ import org.silkframework.entity.paths.{TypedPath, UntypedPath} import org.silkframework.plugins.path.{PathMetaDataPlugin, StandardMetaDataPlugin} import org.silkframework.rule.evaluation._ import org.silkframework.rule.execution.{Linking, GenerateLinks => GenerateLinksActivity} -import org.silkframework.rule.{DatasetSelection, LinkSpec, LinkageRule, RuntimeLinkingConfig} +import org.silkframework.rule.{DatasetSelection, LinkSpec, LinkageRule, LinkageRuleExecution, RuntimeLinkingConfig} import org.silkframework.runtime.activity.{Activity, UserContext} import org.silkframework.runtime.serialization.{ReadContext, WriteContext, XmlSerialization} import org.silkframework.runtime.validation._ @@ -679,13 +679,13 @@ class LinkingTaskApi @Inject() (accessMonitor: WorkbenchAccessMonitor) extends I } private def serializeLinks(entities: Iterable[DPair[Entity]], - linkageRule: LinkageRule, + linkageRule: LinkageRuleExecution, withEntitiesAndSchema: Boolean = false) (implicit writeContext: WriteContext[JsValue]): JsValue = { JsArray( for(entities <- entities.toSeq) yield { val link = new FullLink(entities.source.uri, entities.target.uri, linkageRule(entities), entities) - new LinkJsonFormat(Some(linkageRule), writeEntities = withEntitiesAndSchema, writeEntitySchema = withEntitiesAndSchema).write(link) + new LinkJsonFormat(Some(linkageRule.operator), writeEntities = withEntitiesAndSchema, writeEntitySchema = withEntitiesAndSchema).write(link) } ) } @@ -739,7 +739,7 @@ class LinkingTaskApi @Inject() (accessMonitor: WorkbenchAccessMonitor) extends I val rule = task.ruleWithContext val referenceEntityCacheValue = updateAndGetReferenceEntityCacheValue(task, refreshCache = true) - val evaluationResult: LinkageRuleEvaluationResult = LinkingTaskApiUtils.referenceLinkEvaluationScore(task.data.rule, referenceEntityCacheValue) + val evaluationResult: LinkageRuleEvaluationResult = LinkingTaskApiUtils.referenceLinkEvaluationScore(rule, referenceEntityCacheValue) implicit val writeContext: WriteContext[JsValue] = WriteContext.fromProject[JsValue](project) val result = @@ -789,7 +789,7 @@ class LinkingTaskApi @Inject() (accessMonitor: WorkbenchAccessMonitor) extends I implicit val prefixes: Prefixes = project.config.prefixes SerializationUtils.deserializeCompileTime[LinkageRule](defaultMimeType = SerializationUtils.APPLICATION_JSON) { linkageRule => - val ruleWithContext = linkageRule.withContext(task.taskContext) + val ruleWithContext = linkageRule.execution(task.taskContext) val referenceEntityCacheValue = updateAndGetReferenceEntityCacheValue(task, refreshCache = false) implicit val writeContext: WriteContext[JsValue] = WriteContext.fromProject[JsValue](project) def serialize(links: Iterable[DPair[Entity]]): JsValue = { @@ -967,7 +967,7 @@ class LinkingTaskApi @Inject() (accessMonitor: WorkbenchAccessMonitor) extends I SerializationUtils.deserializeCompileTime[LinkageRule](defaultMimeType = SerializationUtils.APPLICATION_JSON) { linkageRule => val runtimeConfig = RuntimeLinkingConfig(executionTimeout = Some(timeoutInMs), linkLimit = Some(linkLimit), generateLinksWithEntities = true, includeReferenceLinks = includeReferenceLinks) - val linksActivity = new GenerateLinksActivity(task, sources, None, runtimeConfig, Some(linkageRule.withContext(task.taskContext))) + val linksActivity = new GenerateLinksActivity(task, sources, None, runtimeConfig, Some(linkageRule.execution(task.taskContext))) val control = Activity(linksActivity) control.startBlocking() control.value.get match { @@ -1159,6 +1159,7 @@ class LinkingTaskApi @Inject() (accessMonitor: WorkbenchAccessMonitor) extends I if (evaluationActivity.control.status.get.isEmpty) { evaluationActivity.control.startBlocking() } + val ruleExec = linkTask.data.rule.execution() for(link <- evaluationActivity.value().links) yield { val evaluatedLink = link match { @@ -1167,7 +1168,7 @@ class LinkingTaskApi @Inject() (accessMonitor: WorkbenchAccessMonitor) extends I case link: Link => link.entities match { case Some(entities) => - DetailedEvaluator(linkTask.data.rule, entities) + DetailedEvaluator(ruleExec, entities) case None => throw new IllegalArgumentException("Evaluation links are missing entities.") } @@ -1179,8 +1180,9 @@ class LinkingTaskApi @Inject() (accessMonitor: WorkbenchAccessMonitor) extends I private def retrieveReferenceLinksSafe(linkTask: ProjectTask[LinkSpec]): Seq[EvaluatedLinkWithDecision] = { val referenceEntityCache = linkTask.activity[ReferenceEntitiesCache].value() val DPair(sourceEntitySchema, targetEntitySchema) = linkTask.data.entityDescriptions + val ruleExec = linkTask.data.rule.execution() for(link <- referenceEntityCache.toReferenceLinksSafe(sourceEntitySchema, targetEntitySchema)) yield { - val evaluatedLink = DetailedEvaluator(linkTask.data.rule, link.linkEntities) + val evaluatedLink = DetailedEvaluator(ruleExec, link.linkEntities) evaluatedLink.withDecision(link.decision) } } diff --git a/silk-workbench/silk-workbench-rules/app/controllers/linking/ReferenceLinksManager.scala b/silk-workbench/silk-workbench-rules/app/controllers/linking/ReferenceLinksManager.scala index c651e7b51a..4e32cdb453 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/linking/ReferenceLinksManager.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/linking/ReferenceLinksManager.scala @@ -33,6 +33,7 @@ class ReferenceLinksManager @Inject() (implicit workspaceReact: WorkspaceReact) val referenceLinks = task.data.referenceLinks def linkSpec = task.data def linkageRule = linkSpec.rule + val ruleExec = linkageRule.execution() def entities = task.activity[ReferenceEntitiesCache].value() val linkSorter = LinkSorter.fromId(sorting) val linkResolvers = LinkResolver.forLinkingTask(task) @@ -47,7 +48,7 @@ class ReferenceLinksManager @Inject() (implicit workspaceReact: WorkspaceReact) case "positive" => { for (link <- referenceLinks.positive.toSeq.view) yield entities.positiveLinkToEntities(link) match { case Some(entities) if hasPaths(entities) => { - val evaluatedLink = DetailedEvaluator(linkageRule, entities, -1.0).get + val evaluatedLink = DetailedEvaluator(ruleExec, entities, -1.0).get new EvalLink( link = evaluatedLink, @@ -69,7 +70,7 @@ class ReferenceLinksManager @Inject() (implicit workspaceReact: WorkspaceReact) case "negative" => { for (link <- referenceLinks.negative.toSeq.view) yield entities.negativeLinkToEntities(link) match { case Some(entities) if hasPaths(entities) => { - val evaluatedLink = DetailedEvaluator(linkageRule, entities, -1.0).get + val evaluatedLink = DetailedEvaluator(ruleExec, entities, -1.0).get new EvalLink( link = evaluatedLink, @@ -91,7 +92,7 @@ class ReferenceLinksManager @Inject() (implicit workspaceReact: WorkspaceReact) case "unlabeled" => { for (link <- referenceLinks.unlabeled.toSeq.view) yield entities.unlabeledLinkToEntities(link) match { case Some(entities) if hasPaths(entities) => { - val evaluatedLink = DetailedEvaluator(linkageRule, entities, -1.0).get + val evaluatedLink = DetailedEvaluator(ruleExec, entities, -1.0).get new EvalLink( link = evaluatedLink, diff --git a/silk-workbench/silk-workbench-rules/app/controllers/linking/linkingTask/LinkingTaskApiUtils.scala b/silk-workbench/silk-workbench-rules/app/controllers/linking/linkingTask/LinkingTaskApiUtils.scala index cf9044b401..e24f37cbdc 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/linking/linkingTask/LinkingTaskApiUtils.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/linking/linkingTask/LinkingTaskApiUtils.scala @@ -8,7 +8,7 @@ import org.silkframework.plugins.path.PathMetaDataPlugin import org.silkframework.rule.evaluation.{EvaluationResult, LinkageRuleEvaluator, ReferenceEntities} import org.silkframework.rule.input.Transformer import org.silkframework.rule.similarity.{Aggregator, DistanceMeasure} -import org.silkframework.rule.{LinkSpec, LinkageRule} +import org.silkframework.rule.{LinkSpec, LinkageRuleExecution} import org.silkframework.runtime.activity.UserContext import org.silkframework.runtime.plugin.{PluginContext, PluginRegistry} import org.silkframework.runtime.serialization.WriteContext @@ -156,13 +156,13 @@ object LinkingTaskApiUtils { } /** Score of reference links for a given linking rule. */ - def referenceLinkEvaluationScore(linkageRule: LinkageRule, referenceEntityCacheValue: ReferenceEntities): LinkageRuleEvaluationResult = { + def referenceLinkEvaluationScore(linkageRule: LinkageRuleExecution, referenceEntityCacheValue: ReferenceEntities): LinkageRuleEvaluationResult = { val score = LinkageRuleEvaluator(linkageRule, referenceEntityCacheValue) linkRuleEvaluationResult(score) } /** Score of reference links for a given linking rule. */ - def referenceLinkEvaluationScore(linkageRule: LinkageRule, referenceLinks: Seq[ReferenceLink], threshold: Double = 0.0): LinkageRuleEvaluationResult = { + def referenceLinkEvaluationScore(linkageRule: LinkageRuleExecution, referenceLinks: Seq[ReferenceLink], threshold: Double = 0.0): LinkageRuleEvaluationResult = { val score = LinkageRuleEvaluator(linkageRule, referenceLinks, threshold) linkRuleEvaluationResult(score) } diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/EvaluateTransformApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/EvaluateTransformApi.scala index 87da3da0b6..6118d8ff03 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/EvaluateTransformApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/EvaluateTransformApi.scala @@ -194,7 +194,7 @@ class EvaluateTransformApi @Inject()(implicit accessMonitor: WorkbenchAccessMoni )) } - private def evaluatedRulesJson(ruleSchema: TransformSpec.RuleSchemata) + private def evaluatedRulesJson(ruleSchema: TransformSpec.RuleSchemataExecution) (implicit writeContext: WriteContext[JsValue]): Seq[JsValue] = { ruleSchema.transformRule.rules.allRules .map(r => { @@ -215,13 +215,13 @@ class EvaluateTransformApi @Inject()(implicit accessMonitor: WorkbenchAccessMoni } private def ruleSchemaById(task: ProjectTask[TransformSpec], ruleId: String) - (implicit pluginContext: PluginContext): TransformSpec.RuleSchemata = { + (implicit pluginContext: PluginContext): TransformSpec.RuleSchemataExecution = { val objectMappingId = task.data.objectMappingIdOfRule(ruleId).getOrElse(ruleId) task.data.ruleSchemata .find(_.transformRule.id.toString == objectMappingId) .getOrElse(throw new NotFoundException(s"Mapping rule '$ruleId' is not part of task '${task.fullLabel}' in project '${task.project.fullLabel}'. " + s"Available rules: ${task.data.ruleSchemata.map(_.transformRule.id).mkString(", ")}")) - .withContext(task.taskContext) + .execution(task.taskContext) } private def evaluateRule(task: ProjectTask[TransformSpec], parentRuleId: Identifier, transformRule: TransformRule, limit: Int) @@ -230,11 +230,11 @@ class EvaluateTransformApi @Inject()(implicit accessMonitor: WorkbenchAccessMoni val ruleSchema = ruleSchemaById(task, parentRuleId) val inputSchema = ruleSchema.inputSchema.copy(typedPaths = transformRule.sourcePaths.toIndexedSeq) - val ruleWithContext = transformRule.withContext(task.taskContext) val entities = task.dataSource.retrieve(inputSchema, Some(limit)).entities.take(limit) + val ruleExec = transformRule.execution() for(entity <- entities) yield { - DetailedEvaluator(ruleWithContext, entity) + DetailedEvaluator(ruleExec, entity) } } diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/PeakTransformApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/PeakTransformApi.scala index 02f340804e..a4d5f3d4ee 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/PeakTransformApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/PeakTransformApi.scala @@ -20,7 +20,8 @@ import org.silkframework.entity.paths.{Path, UntypedPath} import org.silkframework.plugins.dataset.rdf.executors.{LocalSparqlSelectExecutor, LocalSparqlSelectIterator} import org.silkframework.plugins.dataset.rdf.tasks.SparqlSelectCustomTask import org.silkframework.rule.TransformSpec.RuleSchemata -import org.silkframework.rule.{ComplexUriMapping, TaskContext, TransformRule, TransformSpec} +import org.silkframework.rule.input.Value +import org.silkframework.rule.{ComplexUriMapping, TaskContext, TransformRule, TransformRuleExecution, TransformSpec, ValueTransformRuleExecution} import org.silkframework.runtime.activity.UserContext import org.silkframework.runtime.plugin.PluginContext import org.silkframework.runtime.serialization.ReadContext @@ -201,7 +202,7 @@ class PeakTransformApi @Inject() () extends InjectedController with UserContextA case peakDataSource: PeakDataSource => try { peakDataSource.peak(ruleSchemata.inputSchema, maxTryEntities).use { exampleEntities => - generateMappingPreviewResponse(ruleSchemata.transformRule.withContext(TaskContext(Seq(inputTask), PluginContext.fromProject(project))), exampleEntities, limit) + generateMappingPreviewResponse(ruleSchemata.transformRule.execution(TaskContext.forInput(inputTask)), exampleEntities, limit) } } catch { case pe: PeakException => @@ -243,7 +244,7 @@ class PeakTransformApi @Inject() () extends InjectedController with UserContextA val entityDatasource = EntityDatasource(datasetTask, entities, sparqlSelectTask.outputSchema) try { entityDatasource.peak(ruleSchemata.inputSchema, maxTryEntities).use { exampleEntities => - generateMappingPreviewResponse(ruleSchemata.transformRule, exampleEntities, limit) + generateMappingPreviewResponse(ruleSchemata.transformRule.execution(TaskContext.noInput), exampleEntities, limit) } } catch { case pe: PeakException => @@ -258,11 +259,12 @@ class PeakTransformApi @Inject() () extends InjectedController with UserContextA } // Generate the HTTP response for the mapping transformation preview - private def generateMappingPreviewResponse(rule: TransformRule, + private def generateMappingPreviewResponse(ruleExecution: TransformRuleExecution, exampleEntities: Iterator[Entity], limit: Int) (implicit prefixes: Prefixes) = { - val (tryCounter, errorCounter, errorMessage, sourceAndTargetResults) = collectTransformationExamples(rule, exampleEntities, limit) + val rule = ruleExecution.operator + val (tryCounter, errorCounter, errorMessage, sourceAndTargetResults) = collectTransformationExamples(ruleExecution, exampleEntities, limit) if (sourceAndTargetResults.nonEmpty && errorMessage.nonEmpty) { Ok(Json.toJson(PeakResults(Some(rule.sourcePaths.map(serializePath)), Some(sourceAndTargetResults), status = PeakStatus("with exceptions", errorMessage)))) @@ -310,13 +312,16 @@ object PeakTransformApi { final val NOT_SUPPORTED_STATUS_MSG = "not supported" /** - * - * @param rule The transformation rule to execute on the example entities. - * @param exampleEntities Entities to try executing the tranform rule on - * @param limit Limit of examples to return - * @return - */ - def collectTransformationExamples(rule: TransformRule, exampleEntities: Iterator[Entity], limit: Int): (Int, Int, String, Seq[PeakResult]) = { + * @param ruleExecution The contextualized transformation rule to execute on the example entities. + * @param exampleEntities Entities to try executing the transform rule on + * @param limit Limit of examples to return + */ + def collectTransformationExamples(ruleExecution: TransformRuleExecution, exampleEntities: Iterator[Entity], limit: Int): (Int, Int, String, Seq[PeakResult]) = { + val ruleApply: Entity => Value = ruleExecution match { + case ve: ValueTransformRuleExecution => ve.apply + case other => + throw new IllegalArgumentException(s"Cannot generate a mapping preview for non-value rule '${other.operator.id}'.") + } // Number of examples collected var exampleCounter = 0 // Number of exceptions occurred @@ -330,7 +335,7 @@ object PeakTransformApi { tryCounter += 1 val entity = exampleEntities.next() try { - val transformResult = rule(entity) + val transformResult = ruleApply(entity) for(error <- transformResult.errors) { errorCounter += 1 if (errorMessage.isEmpty) { diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PeakTransformApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PeakTransformApiTest.scala index a8ed9240ed..045a1a7e5a 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PeakTransformApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PeakTransformApiTest.scala @@ -1,7 +1,6 @@ package controllers.transform import helper.IntegrationTestTrait - import org.silkframework.entity.paths.UntypedPath import org.silkframework.entity.{Entity, EntitySchema} import org.silkframework.rule.input.{PathInput, TransformInput, Transformer} @@ -9,7 +8,7 @@ import org.silkframework.rule.plugins.transformer.combine.ConcatTransformer import org.silkframework.rule.plugins.transformer.date.DateToTimestampTransformer import org.silkframework.rule.plugins.transformer.normalize.LowerCaseTransformer import org.silkframework.rule.plugins.transformer.tokenization.CamelCaseTokenizer -import org.silkframework.rule.{ComplexMapping, PatternUriMapping, TransformRule} +import org.silkframework.rule.{ComplexMapping, PatternUriMapping, TransformRule, TransformRuleExecution} import org.silkframework.serialization.json.JsonSerializers.TransformRuleJsonFormat import org.silkframework.serialization.json.{JsonHelpers, JsonSerialization} import org.silkframework.util.Uri @@ -51,10 +50,10 @@ class PeakTransformApiTest extends AnyFlatSpec with SingleProjectWorkspaceProvid ) } - private def transformRule(transformer: Transformer): ComplexMapping = { + private def transformRule(transformer: Transformer): TransformRuleExecution = { val transformation = TransformInput(transformer = transformer, inputs = IndexedSeq(PathInput("p", UntypedPath("a")), PathInput("p", UntypedPath("b")))) - ComplexMapping(operator = transformation) + ComplexMapping(operator = transformation).execution() } it should "collect transformation examples skipping empty transformation results" in { diff --git a/silk-workbench/silk-workbench-workspace/test/controllers/workspace/WorkspaceApiTest.scala b/silk-workbench/silk-workbench-workspace/test/controllers/workspace/WorkspaceApiTest.scala index c74e65b6d4..ee5712c7f2 100644 --- a/silk-workbench/silk-workbench-workspace/test/controllers/workspace/WorkspaceApiTest.scala +++ b/silk-workbench/silk-workbench-workspace/test/controllers/workspace/WorkspaceApiTest.scala @@ -18,7 +18,7 @@ import org.silkframework.plugins.dataset.rdf.datasets.InMemoryDataset import org.silkframework.plugins.dataset.rdf.tasks.SparqlUpdateCustomTask import org.silkframework.plugins.dataset.xml.XSLTOperator import org.silkframework.rule._ -import org.silkframework.rule.input.{TransformInput, Transformer} +import org.silkframework.rule.input.{InlineTransformer, TransformInput} import org.silkframework.rule.plugins.distance.equality.EqualityMetric import org.silkframework.rule.similarity.Comparison import org.silkframework.runtime.resource.Resource @@ -184,7 +184,7 @@ class WorkspaceApiTest extends PlaySpec with IntegrationTestTrait with Matchers } object WorkspaceApiTest { - case class TestTransformer(referencedResource: Resource) extends Transformer { + case class TestTransformer(referencedResource: Resource) extends InlineTransformer { override def apply(values: Seq[Seq[String]]): Seq[String] = Seq.empty override def referencedResources: Seq[Resource] = { diff --git a/silk-workspace/src/main/scala/org/silkframework/plugins/transformer/value/ReadParameter.scala b/silk-workspace/src/main/scala/org/silkframework/plugins/transformer/value/ReadParameter.scala index d7aa3f9a88..5052839bd4 100644 --- a/silk-workspace/src/main/scala/org/silkframework/plugins/transformer/value/ReadParameter.scala +++ b/silk-workspace/src/main/scala/org/silkframework/plugins/transformer/value/ReadParameter.scala @@ -4,7 +4,7 @@ import java.time.Instant import java.util.Properties import java.util.logging.Logger -import org.silkframework.rule.input.Transformer +import org.silkframework.rule.input.InlineTransformer import org.silkframework.runtime.plugin.annotations.{Param, Plugin} import org.silkframework.runtime.resource.Resource import org.silkframework.runtime.validation.ValidationException @@ -20,7 +20,7 @@ case class ReadParameter( @Param(value = "The Java properties file to read the parameter from.", autoCompletionProvider = classOf[ResourceAutoCompletionProvider], allowOnlyAutoCompletedValues = true) resource: Resource, @Param("The name of the parameter.") - parameter: String) extends Transformer { + parameter: String) extends InlineTransformer { private val logger = Logger.getLogger(classOf[ReadParameter].getName) diff --git a/silk-workspace/src/main/scala/org/silkframework/workspace/activity/linking/LinkingTaskUtils.scala b/silk-workspace/src/main/scala/org/silkframework/workspace/activity/linking/LinkingTaskUtils.scala index b9994e7bba..3154687f29 100644 --- a/silk-workspace/src/main/scala/org/silkframework/workspace/activity/linking/LinkingTaskUtils.scala +++ b/silk-workspace/src/main/scala/org/silkframework/workspace/activity/linking/LinkingTaskUtils.scala @@ -2,7 +2,7 @@ package org.silkframework.workspace.activity.linking import org.silkframework.dataset.DatasetSpec.GenericDatasetSpec import org.silkframework.dataset.{DataSource, Dataset, DatasetSpec, EmptySource, LinkSink} -import org.silkframework.rule.{DatasetSelection, LinkSpec, LinkageRule, TaskContext, TransformSpec} +import org.silkframework.rule.{DatasetSelection, LinkSpec, LinkageRuleExecution, TaskContext, TransformSpec} import org.silkframework.runtime.activity.UserContext import org.silkframework.runtime.plugin.PluginContext import org.silkframework.util.DPair @@ -56,10 +56,10 @@ object LinkingTaskUtils { } /** - * Returns the linking rule with the task context. + * Returns the linking rule executor resolved against the task context. */ - def ruleWithContext(implicit userContext: UserContext): LinkageRule = { - task.data.rule.withContext(taskContext) + def ruleWithContext(implicit userContext: UserContext): LinkageRuleExecution = { + task.data.rule.execution(taskContext) } }