From 73e658682409333d54e5b729041713b9a9992474 Mon Sep 17 00:00:00 2001 From: Andrea Spina Date: Wed, 8 May 2019 15:23:54 +0200 Subject: [PATCH 1/2] updating jpmml version to 14 --- build.sbt | 2 +- .../flink/pmml/scala/api/PmmlModel.scala | 16 ++++++++-------- .../flink/pmml/scala/api/PmmlModelSpec.scala | 2 +- .../scala/models/prediction/PredictionSpec.scala | 2 +- .../pmml/scala/utils/PmmlEvaluatorKit.scala | 4 ++-- .../flink/pmml/scala/utils/PmmlLoaderKit.scala | 8 ++------ project/Dependencies.scala | 4 ++-- version.sbt | 2 +- 8 files changed, 18 insertions(+), 22 deletions(-) diff --git a/build.sbt b/build.sbt index 640e10d..4cbfdf1 100644 --- a/build.sbt +++ b/build.sbt @@ -36,7 +36,7 @@ lazy val root = project .enablePlugins(ScalaUnidocPlugin) .settings( name := "flink-jpmml", - crossScalaVersions := Seq("2.10.6", "2.11.11"), + crossScalaVersions := Seq("2.11.12"), unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(`flink-jpmml-examples`, `flink-jpmml-assets`) ) diff --git a/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/PmmlModel.scala b/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/PmmlModel.scala index 521fada..7c75650 100644 --- a/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/PmmlModel.scala +++ b/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/PmmlModel.scala @@ -26,11 +26,11 @@ import io.radicalbit.flink.pmml.scala.api.exceptions.{InputValidationException, import io.radicalbit.flink.pmml.scala.api.pipeline.Pipeline import io.radicalbit.flink.pmml.scala.api.reader.ModelReader import io.radicalbit.flink.pmml.scala.models.prediction.Prediction +import org.apache.commons.io.input.ReaderInputStream import org.apache.flink.ml.math.Vector import org.dmg.pmml.FieldName import org.jpmml.evaluator._ -import org.jpmml.model.{ImportFilter, JAXBUtil} -import org.xml.sax.InputSource +import org.jpmml.model.{PMMLUtil => MarshalUtil} import scala.collection.JavaConversions._ import scala.util.Try @@ -54,11 +54,11 @@ object PmmlModel { val readerFromFs = reader.buildDistributedPath val result = fromFilteredSource(readerFromFs) - new PmmlModel(Evaluator(evaluatorInstance.newModelEvaluator(JAXBUtil.unmarshalPMML(result)))) + new PmmlModel(Evaluator(evaluatorInstance.newModelEvaluator(result))) } private def fromFilteredSource(PMMLPath: String) = - JAXBUtil.createFilteredSource(new InputSource(new StringReader(PMMLPath)), new ImportFilter()) + MarshalUtil.unmarshal(new ReaderInputStream(new StringReader(PMMLPath))) /** It provides a new instance of the [[PmmlModel]] with [[EmptyEvaluator]] * @@ -93,7 +93,7 @@ class PmmlModel(private[api] val evaluator: Evaluator) extends Pipeline { * * - `validateInput` validates the input to be conform to PMML model size * - * - `prepareInput` prepares the input in full compliance to [[org.jpmml.evaluator.EvaluatorUtil.prepare]] JPMML method + * - `prepareInput` prepares the input in full compliance to [[InputField]] preparation JPMML method * * - `evaluateInput` evaluates the input against inner PMML model instance and returns a Java Map output * @@ -134,7 +134,7 @@ class PmmlModel(private[api] val evaluator: Evaluator) extends Pipeline { } /** Binds each field with input value and prepare the record to be evaluated - * by [[EvaluatorUtil.prepare]] method. + * by [[InputField]] preparation method. * * @param input Validated input as a [[Map]] keyed by field name * @param replaceNaN Optional replace value in case of missing values @@ -146,7 +146,7 @@ class PmmlModel(private[api] val evaluator: Evaluator) extends Pipeline { activeFields.map { field => val rawValue = input.get(field.getName.getValue).orElse(replaceNaN).orNull - prepareAndEmit(Try { EvaluatorUtil.prepare(field, rawValue) }, field.getName) + prepareAndEmit(Try { field.prepare(rawValue) }, field.getName) }.toMap } @@ -164,7 +164,7 @@ class PmmlModel(private[api] val evaluator: Evaluator) extends Pipeline { * @param evaluationResult outcome from JPMML evaluation * @return The prediction value as a [Double] */ - private[api] def extractTarget(evaluationResult: java.util.Map[FieldName, _]): Double = { + private[api] def extractTarget(evaluationResult: util.Map[FieldName, _]): Double = { val targets = extractTargetFields(evaluationResult) targets.headOption.flatMap { diff --git a/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/PmmlModelSpec.scala b/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/PmmlModelSpec.scala index 9e6273f..39f8262 100644 --- a/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/PmmlModelSpec.scala +++ b/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/PmmlModelSpec.scala @@ -313,7 +313,7 @@ class PmmlModelSpec extends WordSpec with Matchers with PmmlLoaderKit with PmmlE "Pipeline.prepareAndEmit" should { "emit valid prepared value if the input is correct" in { - val v = FieldValueUtil.create(null.asInstanceOf[DataType], null.asInstanceOf[OpType], 1.0) + val v = FieldValueUtil.create(1.0) val d = createDataField() model.prepareAndEmit(Success(v), d.getName) shouldBe (d.getName, v) } diff --git a/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/models/prediction/PredictionSpec.scala b/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/models/prediction/PredictionSpec.scala index 281c73a..8265b17 100644 --- a/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/models/prediction/PredictionSpec.scala +++ b/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/models/prediction/PredictionSpec.scala @@ -69,7 +69,7 @@ class PredictionSpec extends WordSpec with Matchers { } "return None if onFailedPrediction is active and EvaluationException" in { - throwableFunc(new EvaluationException) { + throwableFunc(new EvaluationException("")) { case e: Throwable => Prediction.onFailedPrediction(e) } shouldBe Prediction(EmptyScore) } diff --git a/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlEvaluatorKit.scala b/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlEvaluatorKit.scala index 3fd1528..779a9e2 100644 --- a/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlEvaluatorKit.scala +++ b/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlEvaluatorKit.scala @@ -40,8 +40,8 @@ trait PmmlEvaluatorKit { final protected def buildExpectedPreparedMap(in: Map[String, Any], keys: Seq[String], replaceValue: Option[Double]) = keys.map { - case k if in.contains(k) => new FieldName(k) -> FieldValueUtil.create(null, null, in(k)) - case emptyKey => new FieldName(emptyKey) -> FieldValueUtil.create(null, null, replaceValue.orNull) + case k if in.contains(k) => new FieldName(k) -> FieldValueUtil.create(in(k)) + case emptyKey => new FieldName(emptyKey) -> FieldValueUtil.create(replaceValue.orNull) } toMap } diff --git a/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlLoaderKit.scala b/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlLoaderKit.scala index 6852e5a..ce3681a 100644 --- a/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlLoaderKit.scala +++ b/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlLoaderKit.scala @@ -20,8 +20,7 @@ package io.radicalbit.flink.pmml.scala.utils import org.dmg.pmml.PMML -import org.jpmml.model.{ImportFilter, JAXBUtil} -import org.xml.sax.InputSource +import org.jpmml.model.PMMLUtil trait PmmlLoaderKit { @@ -42,9 +41,6 @@ trait PmmlLoaderKit { final protected def getPMMLSource(path: String): String = getClass.getResource(path).getPath - final protected def getPMMLResource(path: String): PMML = { - val source = scala.io.Source.fromURL(getClass.getResource(path)).reader() - JAXBUtil.unmarshalPMML(ImportFilter.apply(new InputSource(source))) - } + final protected def getPMMLResource(path: String): PMML = PMMLUtil.unmarshal(getClass.getResourceAsStream(path)) } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 19b85b5..162d16e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -65,7 +65,7 @@ object Dependencies { private object jpmml { lazy val namespace = "org.jpmml" - lazy val version = "1.3.9" + lazy val version = "1.4.6" lazy val evaluator = namespace % "pmml-evaluator" % version } @@ -84,7 +84,7 @@ object Dependencies { private object hadoop { lazy val namespace = "org.apache.hadoop" - lazy val version = "2.3.0" + lazy val version = "2.8.3" lazy val hdfs = namespace % "hadoop-hdfs" % version lazy val common = namespace % "hadoop-common" % version lazy val mincluster = namespace % "hadoop-minicluster" % version diff --git a/version.sbt b/version.sbt index 2a29371..bfd2dac 100644 --- a/version.sbt +++ b/version.sbt @@ -19,4 +19,4 @@ * */ -version in ThisBuild := "0.7.0-SNAPSHOT" \ No newline at end of file +version in ThisBuild := "0.7.1-SNAPSHOT" \ No newline at end of file From 4c534ccf2bd305fa00693ac7feb084fd76fec81e Mon Sep 17 00:00:00 2001 From: Andrea Spina Date: Thu, 9 May 2019 13:42:54 +0200 Subject: [PATCH 2/2] updating jpmml evalutor --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 162d16e..036fedd 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -65,7 +65,7 @@ object Dependencies { private object jpmml { lazy val namespace = "org.jpmml" - lazy val version = "1.4.6" + lazy val version = "1.4.7" lazy val evaluator = namespace % "pmml-evaluator" % version }