Add Jinja templating support to SPARQL operators

build.sbt

@@ -151,13 +151,29 @@ lazy val workspace = (project in file("silk-workspace"))
 // Plugins
 //////////////////////////////////////////////////////////////////////////////
 
+lazy val pluginsTemplatingJinja = (project in file("silk-plugins/silk-plugins-templating-jinja"))
+  .dependsOn(rules % "compile->compile;test->test")
+  .settings(commonSettings *)
+  .settings(
+    name := "Silk Plugins Templating Jinja",
+    libraryDependencies += "com.hubspot.jinjava" % "jinjava" % "2.8.3"
+  )
+
+lazy val pluginsTemplatingVelocity = (project in file("silk-plugins/silk-plugins-templating-velocity"))
+  .dependsOn(rules % "compile->compile;test->test")
+  .settings(commonSettings *)
+  .settings(
+    name := "Silk Plugins Templating Velocity",
+    libraryDependencies += "org.apache.velocity" % "velocity-engine-core" % "2.4.1"
+  )
+
 lazy val pluginsRdf = (project in file("silk-plugins/silk-plugins-rdf"))
-  .dependsOn(rules, workspace % "test->test;compile->compile", core % "test->test;compile->compile", pluginsCsv % "test->compile")
+  .dependsOn(rules, workspace % "test->test;compile->compile", core % "test->test;compile->compile", pluginsCsv % "test->compile",
+             pluginsTemplatingJinja % "test->compile", pluginsTemplatingVelocity % "test->compile")
   .settings(commonSettings *)
   .settings(
     name := "Silk Plugins RDF",
-    libraryDependencies += "org.apache.jena" % "jena-fuseki-main" % "5.6.0" % "test",
-    libraryDependencies += "org.apache.velocity" % "velocity-engine-core" % "2.4.1"
+    libraryDependencies += "org.apache.jena" % "jena-fuseki-main" % "5.6.0" % "test"
 )
 
 lazy val pluginsCsv = (project in file("silk-plugins/silk-plugins-csv"))
@@ -226,8 +242,8 @@ lazy val persistentCaching = (project in file("silk-plugins/silk-persistent-cach
 
 // Aggregate all plugins
 lazy val plugins = (project in file("silk-plugins"))
-  .dependsOn(pluginsRdf, pluginsCsv, pluginsXml, pluginsJson, pluginsAsian, serializationJson, persistentCaching)
-  .aggregate(pluginsRdf, pluginsCsv, pluginsXml, pluginsJson, pluginsAsian, serializationJson, persistentCaching)
+  .dependsOn(pluginsRdf, pluginsCsv, pluginsXml, pluginsJson, pluginsAsian, serializationJson, persistentCaching, pluginsTemplatingJinja, pluginsTemplatingVelocity)
+  .aggregate(pluginsRdf, pluginsCsv, pluginsXml, pluginsJson, pluginsAsian, serializationJson, persistentCaching, pluginsTemplatingJinja, pluginsTemplatingVelocity)
   .settings(commonSettings *)
   .settings(
     name := "Silk Plugins"
@@ -369,7 +385,7 @@ lazy val workbenchCore = (project in file("silk-workbench/silk-workbench-core"))
 
 lazy val workbenchWorkspace = (project in file("silk-workbench/silk-workbench-workspace"))
   .enablePlugins(PlayScala)
-  .dependsOn(workbenchCore % "compile->compile;test->test", pluginsRdf, pluginsCsv % "test->compile", pluginsXml % "test->compile")
+  .dependsOn(workbenchCore % "compile->compile;test->test", pluginsRdf, pluginsCsv % "test->compile", pluginsXml % "test->compile", pluginsTemplatingJinja % "test->compile")
   .aggregate(workbenchCore)
   .settings(commonSettings *)
   .settings(

silk-core/src/main/scala/org/silkframework/config/Prefixes.scala

@@ -87,11 +87,9 @@ case class Prefixes(prefixMap: immutable.HashMap[String, String]) extends Serial
   }
 
   def toSparql: String = {
-    var sparql = ""
-    for ((key, value) <- prefixMap) {
-      sparql += "PREFIX " + key + ": <" + value + "> "
-    }
-    sparql
+    prefixMap.toSeq.sortBy(_._1).map { case (key, value) =>
+      s"PREFIX $key: <$value>"
+    }.mkString("\n")
   }
 
 }

silk-core/src/main/scala/org/silkframework/runtime/plugin/ParameterType.scala

@@ -752,7 +752,7 @@ object StringParameterType {
   private object SparqlCodeParameterType extends CodeParameterType[SparqlCodeParameter] {
     override def codeMode: String = "sparql"
 
-    override def fromString(str: String)(implicit context: PluginContext): SparqlCodeParameter = SparqlCodeParameter(str)
+    override def fromString(str: String)(implicit context: PluginContext): SparqlCodeParameter = SparqlCodeParameter(str, Some(context.templateVariables))
   }
 
   private object SqlCodeParameterType extends CodeParameterType[SqlCodeParameter] {

silk-core/src/main/scala/org/silkframework/runtime/plugin/types/CodeParameters.scala

@@ -1,7 +1,9 @@
 package org.silkframework.runtime.plugin.types
 
-import scala.language.implicitConversions
+import org.silkframework.runtime.templating.TemplateVariablesReader
+
 import scala.collection.immutable.ArraySeq
+import scala.language.implicitConversions
 
 sealed trait CodeParameter {
   def str: String
@@ -23,7 +25,13 @@ object Jinja2CodeParameter {
 
 case class JsonCodeParameter(var str: String) extends CodeParameter
 
-case class SparqlCodeParameter(var str: String) extends CodeParameter
+/**
+ * A SPARQL code parameter that might contain Jinja template variables.
+ *
+ * @param str The SPARQL query
+ * @param variables That variables that are available at creation time
+ */
+case class SparqlCodeParameter(var str: String, val variables: Option[TemplateVariablesReader] = None) extends CodeParameter
 
 object SparqlCodeParameter {
   implicit def str2parameter(str: String): SparqlCodeParameter = SparqlCodeParameter(str)

silk-core/src/main/scala/org/silkframework/runtime/templating/CombinedTemplateVariablesReader.scala

@@ -6,7 +6,7 @@ case class CombinedTemplateVariablesReader(readers: Seq[TemplateVariablesReader]
   /**
     * The available variable scopes.
     */
-  override def scopes: Set[String] = {
+  override def scopes: Set[Seq[String]] = {
     readers.flatMap(_.scopes).toSet
   }
 

silk-core/src/main/scala/org/silkframework/runtime/templating/GlobalTemplateVariablesConfig.scala

@@ -75,7 +75,7 @@ object GlobalTemplateVariables extends TemplateVariablesReader with Serializable
   /**
     * The available variable scopes.
     */
-  override def scopes: Set[String] = Set(TemplateVariableScopes.global)
+  override def scopes: Set[Seq[String]] = Set(TemplateVariableScopes.global)
 
   /**
     * Retrieves all template variables.

silk-core/src/main/scala/org/silkframework/runtime/templating/InMemoryTemplateVariablesReader.scala

@@ -1,3 +1,3 @@
 package org.silkframework.runtime.templating
 
-case class InMemoryTemplateVariablesReader(override val all: TemplateVariables, override val scopes: Set[String]) extends TemplateVariablesReader
+case class InMemoryTemplateVariablesReader(override val all: TemplateVariables, override val scopes: Set[Seq[String]]) extends TemplateVariablesReader

silk-core/src/main/scala/org/silkframework/runtime/templating/TemplateEngine.scala

@@ -30,6 +30,14 @@ trait CompiledTemplate {
     */
   def variables: Option[Seq[TemplateVariableName]] = None
 
+  /**
+    * Returns all method usages on a given variable in the template.
+    * Each usage contains the method name and its string parameter value.
+    * Only methods with a single string constant parameter are returned.
+    * Returns an empty sequence by default if not supported by the template engine.
+    */
+  def methodUsages(variableName: String): Seq[TemplateMethodUsage] = Seq.empty
+
   /**
    * Evaluates this template using a map of variable values.
    */
@@ -43,40 +51,19 @@ trait CompiledTemplate {
   def evaluate(values: Seq[TemplateVariableValue], writer: Writer, evaluationConfig: EvaluationConfig = EvaluationConfig()): Unit
 
   /**
-    * Evaluates this template using a provided entity.
-    *
-    * @throws TemplateEvaluationException If the evaluation failed.
-    */
-  def evaluate(entity: Entity, writer: Writer): Unit = {
-    evaluate(entityToMap(entity), writer)
-  }
-
-  /**
-   * Converts an entity to a sequence of template variables.
-   */
-  protected def entityToMap(entity: Entity): Seq[TemplateVariableValue] = {
-    for((path, value) <- entity.schema.typedPaths zip entity.values if value.nonEmpty) yield {
-       new TemplateVariableValue(path.normalizedSerialization, "", value)
-    }
-  }
-
-  /**
-   * Converts template values to a Java Map
+   * Converts template variable values to a nested Java-compatible map.
+   * Variables with an empty scope are placed at the top level.
+   * Variables with a scope are placed in nested maps corresponding to each scope element,
+   * e.g., scope Seq("project", "meta") produces Map("project" -> Map("meta" -> Map(name -> value))).
    */
   protected def convertValues(value: Seq[TemplateVariableValue]): Map[String, AnyRef] = {
-    value.groupBy(_.scope).flatMap { case (scope, values) =>
-      if (scope.isEmpty) {
-        for (value <- values) yield {
-          (value.name, IterableTemplateValues.fromValues(value.values))
-        }
-      } else {
-        val nestedValues =
-          for (value <- values) yield {
-            (value.name, IterableTemplateValues.fromValues(value.values))
-          }
-        Seq((scope, nestedValues.toMap.asJava))
-      }
+    val (flatVars, scopedVars) = value.partition(_.scope.isEmpty)
+    val flatEntries = flatVars.map(v => v.name -> IterableTemplateValues.fromValues(v.values).asInstanceOf[AnyRef])
+    val scopedEntries = scopedVars.groupBy(_.scope.head).map { case (topScope, vars) =>
+      val shallowVars = vars.map(v => new TemplateVariableValue(v.name, v.scope.tail, v.values))
+      topScope -> convertValues(shallowVars).asJava.asInstanceOf[AnyRef]
     }
+    (flatEntries ++ scopedEntries).toMap
   }
 }
 
@@ -85,4 +72,12 @@ trait CompiledTemplate {
   * @param ignoreUnboundVariables If an unbound variable is found then instead of throwing an error the variable evaluates
   *                               to the variable name itself.
   */
-case class EvaluationConfig(ignoreUnboundVariables: Boolean = false)
+case class EvaluationConfig(ignoreUnboundVariables: Boolean = false)
+
+/**
+  * Represents a method invocation on a template variable with a single string parameter.
+  *
+  * @param methodName     The name of the invoked method.
+  * @param parameterValue The string constant passed as parameter.
+  */
+case class TemplateMethodUsage(methodName: String, parameterValue: String)

silk-core/src/main/scala/org/silkframework/runtime/templating/TemplateEngineAutocompletionProvider.scala

@@ -12,10 +12,10 @@ class TemplateEngineAutocompletionProvider extends PluginParameterAutoCompletion
                            (implicit context: PluginContext): Iterable[AutoCompletionResult] = {
     val multiSearchWords = extractSearchTerms(searchQuery)
     TemplateEngines.availableEngines
-      .filter(_ != DisabledTemplateEngine.id) // Disabled template engine should not be suggested to the user
-      .filter(_ != UnresolvedTemplateEngine.id) // Unresolved template engine should not be suggested to the user
-      .filter(r => matchesSearchTerm(multiSearchWords, r.toLowerCase))
-      .map(r => AutoCompletionResult(r, None))
+      .filter(_.id.toString != DisabledTemplateEngine.id) // Disabled template engine should not be suggested to the user
+      .filter(_.id.toString != UnresolvedTemplateEngine.id) // Unresolved template engine should not be suggested to the user
+      .filter(engine => matchesSearchTerm(multiSearchWords, engine.id.toLowerCase))
+      .map(engine => AutoCompletionResult(engine.id, Some(engine.label)))
   }
 
   /** Returns the label if exists for the given auto-completion value. This is needed if a value should

silk-core/src/main/scala/org/silkframework/runtime/templating/TemplateEngines.scala

@@ -1,6 +1,6 @@
 package org.silkframework.runtime.templating
 
-import org.silkframework.runtime.plugin.{PluginContext, PluginRegistry}
+import org.silkframework.runtime.plugin.{PluginContext, PluginDescription, PluginRegistry}
 
 /**
   * Manages available template engines.
@@ -10,8 +10,8 @@ object TemplateEngines {
   /**
     * Returns a list of all available template engines.
     */
-  def availableEngines: Set[String] = {
-    PluginRegistry.availablePlugins[TemplateEngine].map(_.id.toString).toSet
+  def availableEngines: Seq[PluginDescription[TemplateEngine]] = {
+    PluginRegistry.availablePlugins[TemplateEngine]
   }
 
   /**
@@ -21,7 +21,7 @@ object TemplateEngines {
     */
   def create(id: String): TemplateEngine = {
     implicit val pluginContext: PluginContext = PluginContext.empty
-    PluginRegistry.create[TemplateEngine](id.toLowerCase)
+    PluginRegistry.create[TemplateEngine](id)
   }
 
 }

silk-core/src/main/scala/org/silkframework/runtime/templating/TemplateVariable.scala

@@ -7,13 +7,22 @@ import scala.xml.{Node, PCData}
 
 /**
   * A single template variable.
+  *
+  * @param name        The local name of the variable.
+  * @param value       The value of the variable.
+  * @param template    Optional template expression to compute the value dynamically.
+  * @param description Optional description for documentation.
+  * @param isSensitive True if the variable value should not be exposed to users.
+  * @param scope       The scope as a sequence of strings forming a prefix path. May be empty.
+  *                    For example, a variable with name "label" and scope Seq("project", "metaData")
+  *                    is addressed as "project.metaData.label".
   */
 case class TemplateVariable(override val name: String,
                             value: String,
                             template: Option[String] = None,
                             description: Option[String] = None,
                             isSensitive: Boolean = false,
-                            override val scope: String) extends TemplateVariableValue(name, scope, values = Seq(value)) {
+                            override val scope: Seq[String] = Seq.empty) extends TemplateVariableValue(name, scope, values = Seq(value)) {
 
   validate()
 
@@ -49,14 +58,14 @@ object TemplateVariable {
         template = Option((value \ "Template").text).filter(_.trim.nonEmpty),
         description = Option((value \ "Description").text).filter(_.trim.nonEmpty),
         isSensitive = (value \ "@isSensitive").text.toBoolean,
-        scope = (value \ "@scope").text,
+        scope = (value \ "@scope").text.split('.').filter(_.nonEmpty).toSeq,
       )
     }
 
     override def write(value: TemplateVariable)(implicit writeContext: WriteContext[Node]): Node = {
       <Variable name={value.name}
                 isSensitive={value.isSensitive.toString}
-                scope={value.scope}>
+                scope={value.scope.mkString(".")}>
         <Value xml:space="preserve">{PCData(value.value)}</Value>
         { value.template.toSeq.map(template => <Template xml:space="preserve">{PCData(template)}</Template>) }
         <Description xml:space="preserve">{value.description.getOrElse("")}</Description>

silk-core/src/main/scala/org/silkframework/runtime/templating/TemplateVariableConversions.scala

@@ -0,0 +1,44 @@
+package org.silkframework.runtime.templating
+
+import org.silkframework.config.{Prefixes, Task, TaskSpec}
+import org.silkframework.entity.Entity
+import org.silkframework.runtime.plugin.{ParameterValues, PluginContext, SimpleParameterValue}
+
+object TemplateVariableConversions {
+
+  /**
+   * Converts an entity to a sequence of template variables.
+   *
+   * @param entity The entity to convert.
+   * @param scope  The scope to assign to all resulting variables.
+   */
+  def fromEntity(entity: Entity, scope: Seq[String] = Seq.empty): Seq[TemplateVariableValue] = {
+    for((path, value) <- entity.schema.typedPaths zip entity.values if value.nonEmpty) yield {
+      new TemplateVariableValue(path.normalizedSerialization, scope, value)
+    }
+  }
+
+  /**
+   * Converts a task's parameters to a sequence of template variables.
+   * Nested plugin parameters are placed into nested scopes using the parameter key.
+   *
+   * @param task  The task whose parameters to convert.
+   * @param scope The base scope. Nested parameters extend this scope with the parameter key.
+   */
+  def fromTask(task: Task[_ <: TaskSpec], scope: Seq[String] = Seq("task"))(implicit pluginContext: PluginContext): Seq[TemplateVariableValue] = {
+    fromPluginParameters(task.data.parameters, scope)
+  }
+
+  private def fromPluginParameters(values: ParameterValues, scope: Seq[String] = Seq.empty)(implicit pluginContext: PluginContext): Seq[TemplateVariableValue] = {
+    for((key, value) <- values.values) yield {
+      value match {
+        case sv: SimpleParameterValue =>
+          Seq(new TemplateVariableValue(key, scope, Seq(sv.strValue)))
+        case nested: ParameterValues =>
+          fromPluginParameters(nested, scope :+ key)
+        case _ =>
+          Seq.empty
+      }
+    }
+  }.flatten.toSeq
+}

silk-core/src/main/scala/org/silkframework/runtime/templating/TemplateVariableName.scala

@@ -4,19 +4,18 @@ package org.silkframework.runtime.templating
   * Holds the full name of a template variable including it's scope.
   *
   * @param name  The local name of the variable.
-  * @param scope The scope. May be empty.
+  * @param scope The scope as a sequence of strings forming a prefix path. May be empty.
+  *              For example, a variable with name "label" and scope Seq("project", "metaData")
+  *              is addressed as "project.metaData.label".
   */
-class TemplateVariableName(val name: String, val scope: String) {
+class TemplateVariableName(val name: String, val scope: Seq[String] = Seq.empty) {
 
   /**
-    * The variable name including its scope, e.g., `project.var`
+    * The variable name including its scope as a dot-separated string, e.g., `project.var` or `project.metaData.var`.
+    * If the scope is empty, this is just the local name.
     */
   def scopedName: String = {
-    if (scope.nonEmpty) {
-      scope + "." + name
-    } else {
-      name
-    }
+    (scope :+ name).mkString(".")
   }
 
   override def toString: String = {
@@ -36,12 +35,17 @@ class TemplateVariableName(val name: String, val scope: String) {
 
 object TemplateVariableName {
 
+  /**
+   * Parses a dot-separated full variable name into a [[TemplateVariableName]].
+   * All segments except the last form the scope; the last segment is the local name.
+   * For example, "project.metaData.label" parses to name="label", scope=Seq("project","metaData").
+   */
   def parse(fullName: String): TemplateVariableName = {
-    val pointIndex = fullName.indexOf('.'.toInt)
-    if(pointIndex != -1) {
-      new TemplateVariableName(fullName.substring(pointIndex + 1), fullName.substring(0, pointIndex))
+    val parts = fullName.split('.')
+    if (parts.length > 1) {
+      new TemplateVariableName(parts.last, parts.dropRight(1).toSeq)
     } else {
-      new TemplateVariableName(fullName, "")
+      new TemplateVariableName(fullName, Seq.empty)
     }
   }