From 42c6e6e1c40b2d2e21e74d1bdf3a2bcf4a5e7fa1 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 8 Aug 2017 14:09:13 -0700 Subject: [PATCH 01/80] flat param API impl --- python/sparkdl/graph/utils.py | 52 +++-- python/sparkdl/transformers/param.py | 124 ++++++++-- python/sparkdl/transformers/tf_tensor.py | 94 ++++++++ python/tests/transformers/tf_tensor_test.py | 236 ++++++++++++++++++++ 4 files changed, 475 insertions(+), 31 deletions(-) create mode 100644 python/sparkdl/transformers/tf_tensor.py create mode 100644 python/tests/transformers/tf_tensor_test.py diff --git a/python/sparkdl/graph/utils.py b/python/sparkdl/graph/utils.py index 45d8b065..75dec230 100644 --- a/python/sparkdl/graph/utils.py +++ b/python/sparkdl/graph/utils.py @@ -95,31 +95,49 @@ def get_tensor(graph, tfobj_or_name): 'cannot locate tensor {} in current graph'.format(_tensor_name) return tnsr -def as_tensor_name(name): +def as_tensor_name(tfobj_or_name): """ Derive tf.Tensor name from an op/tensor name. - We do not check if the tensor exist (as no graph parameter is passed in). + If the input is a name, we do not check if the tensor exist + (as no graph parameter is passed in). - :param name: op name or tensor name + :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ - assert isinstance(name, six.string_types) - name_parts = name.split(":") - assert len(name_parts) <= 2, name_parts - if len(name_parts) < 2: - name += ":0" - return name + if isinstance(tfobj_or_name, six.string_types): + # If input is a string, assume it is a name and infer the corresponding tensor name. + # WARNING: this depends on TensorFlow's tensor naming convention + name = tfobj_or_name + name_parts = name.split(":") + assert len(name_parts) <= 2, name_parts + if len(name_parts) < 2: + name += ":0" + return name + elif hasattr(tfobj_or_name, 'graph'): + tfobj = tfobj_or_name + return get_tensor(tfobj.graph, tfobj).name + else: + raise TypeError('invalid tf.Tensor name query type {}'.format(type(tfobj_or_name))) -def as_op_name(name): +def as_op_name(tfobj_or_name): """ - Derive tf.Operation name from an op/tensor name - We do not check if the operation exist (as no graph parameter is passed in). + Derive tf.Operation name from an op/tensor name. + If the input is a name, we do not check if the operation exist + (as no graph parameter is passed in). - :param name: op name or tensor name + :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ - assert isinstance(name, six.string_types) - name_parts = name.split(":") - assert len(name_parts) <= 2, name_parts - return name_parts[0] + if isinstance(tfobj_or_name, six.string_types): + # If input is a string, assume it is a name and infer the corresponding operation name. + # WARNING: this depends on TensorFlow's operation naming convention + name = tfobj_or_name + name_parts = name.split(":") + assert len(name_parts) <= 2, name_parts + return name_parts[0] + elif hasattr(tfobj_or_name, 'graph'): + tfobj = tfobj_or_name + return get_op(tfobj.graph, tfobj).name + else: + raise TypeError('invalid tf.Operation name query type {}'.format(type(tfobj_or_name))) def op_name(graph, tfobj_or_name): """ diff --git a/python/sparkdl/transformers/param.py b/python/sparkdl/transformers/param.py index f3d3cbaf..eb3d4188 100644 --- a/python/sparkdl/transformers/param.py +++ b/python/sparkdl/transformers/param.py @@ -20,14 +20,19 @@ """ from functools import wraps +import six import keras import tensorflow as tf from pyspark.ml.param import Param, Params, TypeConverters +from sparkdl.graph.builder import GraphFunction, IsolatedSession +import sparkdl.graph.utils as tfx -# From pyspark +""" +Copied from PySpark for backward compatibility. First in Apache Spark version 2.1.1. +""" def keyword_only(func): """ @@ -50,7 +55,8 @@ class HasInputCol(Params): Mixin for param inputCol: input column name. """ - inputCol = Param(Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) + inputCol = Param(Params._dummy(), "inputCol", "input column name.", + typeConverter=TypeConverters.toString) def __init__(self): super(HasInputCol, self).__init__() @@ -73,7 +79,8 @@ class HasOutputCol(Params): Mixin for param outputCol: output column name. """ - outputCol = Param(Params._dummy(), "outputCol", "output column name.", typeConverter=TypeConverters.toString) + outputCol = Param(Params._dummy(), "outputCol", "output column name.", + typeConverter=TypeConverters.toString) def __init__(self): super(HasOutputCol, self).__init__() @@ -92,10 +99,44 @@ def getOutputCol(self): return self.getOrDefault(self.outputCol) -# New in sparkdl - +""" +TensorFlow Specific Parameters +""" class SparkDLTypeConverters(object): + @staticmethod + def toTFGraph(value): + if isinstance(value, tf.Graph): + return value + elif isinstance(value, GraphFunction): + with IsolatedSession() as issn: + issn.importGraphFunction(value, prefix='') + g = issn.graph + return g + else: + raise TypeError("Could not convert %s to TensorFlow Graph" % type(value)) + + @staticmethod + def asColumnToTensorMap(value): + if isinstance(value, dict): + strs_pair_seq = [(k, tfx.as_tensor_name(v)) for k, v in value.items()] + return sorted(strs_pair_seq) + raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) + + @staticmethod + def asTensorToColumnMap(value): + if isinstance(value, dict): + strs_pair_seq = [(tfx.as_tensor_name(k), v) for k, v in value.items()] + return sorted(strs_pair_seq) + raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) + + @staticmethod + def toTFHParams(value): + if isinstance(value, tf.contrib.training.HParams): + return value + else: + raise TypeError("Could not convert %s to TensorFlow HParams" % type(value)) + @staticmethod def toStringOrTFTensor(value): if isinstance(value, tf.Tensor): @@ -106,15 +147,6 @@ def toStringOrTFTensor(value): except TypeError: raise TypeError("Could not convert %s to tensorflow.Tensor or str" % type(value)) - @staticmethod - def toTFGraph(value): - # TODO: we may want to support tf.GraphDef in the future instead of tf.Graph since user - # is less likely to mess up using GraphDef vs Graph (e.g. constants vs variables). - if isinstance(value, tf.Graph): - return value - else: - raise TypeError("Could not convert %s to tensorflow.Graph type" % type(value)) - @staticmethod def supportedNameConverter(supportedList): def converter(value): @@ -122,3 +154,67 @@ def converter(value): return value else: raise TypeError("%s %s is not in the supported list." % type(value), str(value)) + return converter + + +class HasTFHParams(Params): + """ + Mixin for TensorFlow params + """ + hparam = Param(Params._dummy(), "hparams", "instance of :class:`tf.contrib.training.HParams`", + typeConverter=SparkDLTypeConverters.toTFHParams) + +# New in sparkdl + +class HasOutputMapping(Params): + """ + Mixin for param outputMapping: ordered list of ('outputTensorName', 'outputColName') pairs + """ + outputMapping = Param(Params._dummy(), "outputMapping", + "Mapping output :class:`tf.Tensor` objects to DataFrame column names", + typeConverter=SparkDLTypeConverters.asTensorToColumnMap) + + def __init__(self): + super(HasOutputMapping, self).__init__() + + def setOutputMapping(self, value): + return self._set(outputMapping=value) + + def getOutputMapping(self): + return self.getOrDefault(self.outputMapping) + + +class HasInputMapping(Params): + """ + Mixin for param inputMapping: ordered list of ('inputColName', 'inputTensorName') pairs + """ + inputMapping = Param(Params._dummy(), "inputMapping", + "Mapping input DataFrame column names to :class:`tf.Tensor` objects", + typeConverter=SparkDLTypeConverters.asColumnToTensorMap) + + def __init__(self): + super(HasInputMapping, self).__init__() + + def setInputMapping(self, value): + return self._set(inputMapping=value) + + def getInputMapping(self): + return self.getOrDefault(self.inputMapping) + + +class HasTFGraph(Params): + """ + Mixin for param tfGraph: the :class:`tf.Graph` object that represents a TensorFlow computation. + """ + tfGraph = Param(Params._dummy(), "tfGraph", + "TensorFlow Graph object", + typeConverter=SparkDLTypeConverters.toTFGraph) + + def __init__(self): + super(HasTFGraph, self).__init__() + + def setTFGraph(self, value): + return self._set(tfGraph=value) + + def getTFGraph(self): + return self.getOrDefault(self.tfGraph) diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py new file mode 100644 index 00000000..d2053c3d --- /dev/null +++ b/python/sparkdl/transformers/tf_tensor.py @@ -0,0 +1,94 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import absolute_import, division, print_function + +import logging +import numpy as np +import tensorflow as tf +import tensorframes as tfs + +from pyspark.ml import Transformer +from pyspark.ml.param import Param, Params +from pyspark.sql.functions import udf + +from sparkdl.graph.builder import IsolatedSession +import sparkdl.graph.utils as tfx +from sparkdl.transformers.param import ( + keyword_only, HasInputMapping, HasOutputMapping, SparkDLTypeConverters, + HasTFGraph, HasTFHParams) + +__all__ = ['TFTransformer'] + +logger = logging.getLogger('sparkdl') + +class TFTransformer(Transformer, HasTFGraph, HasTFHParams, HasInputMapping, HasOutputMapping): + """ + Applies the TensorFlow graph to the array column in DataFrame. + + Restrictions of the current API: + + We assume that + - All graphs have a "minibatch" dimension (i.e. an unknown leading + dimension) in the tensor shapes. + - Input DataFrame has an array column where all elements have the same length + """ + + @keyword_only + def __init__(self, inputMapping=None, outputMapping=None, tfGraph=None, hparams=None): + """ + __init__(self, inputMapping=None, outputMapping=None, tfGraph=None, hparams=None) + """ + super(TFTransformer, self).__init__() + kwargs = self._input_kwargs + self.setParams(**kwargs) + + @keyword_only + def setParams(self, inputMapping=None, outputMapping=None, tfGraph=None, hparams=None): + """ + setParams(self, inputMapping=None, outputMapping=None, tfGraph=None, hparams=None) + """ + super(TFTransformer, self).__init__() + kwargs = self._input_kwargs + return self._set(**kwargs) + + def _transform(self, dataset): + df = dataset + output_renaming = {} + + with IsolatedSession(graph=self.getTFGraph()) as issn: + feeds = [] + for input_colname, tnsr in self.getInputMapping(): + feeds.append(tfx.get_tensor(issn.graph, tnsr)) + tf_expected_colname = tfx.op_name(issn.graph, tnsr) + df = df.withColumnRenamed(input_colname, tf_expected_colname) + + fetches = [] + for tnsr, output_colname in self.getOutputMapping(): + fetches.append(tfx.get_tensor(issn.graph, tnsr)) + tf_expected_colname = tfx.op_name(issn.graph, tnsr) + output_renaming[tf_expected_colname] = output_colname + + gfn = issn.asGraphFunction(feeds, fetches, strip_and_freeze=True) + + analyzed_df = tfs.analyze(df) + + with IsolatedSession() as issn: + _, fetches = issn.importGraphFunction(gfn, prefix='') + out_df = tfs.map_blocks(fetches, analyzed_df) + + for old_colname, new_colname in output_renaming.items(): + out_df = out_df.withColumnRenamed(old_colname, new_colname) + + return out_df diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py new file mode 100644 index 00000000..495ff8e8 --- /dev/null +++ b/python/tests/transformers/tf_tensor_test.py @@ -0,0 +1,236 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import shutil +import tempfile + +from keras.layers import Conv1D, Dense, Flatten, MaxPool1D +import numpy as np +import tensorflow as tf +import tensorframes as tfs + +from pyspark.sql.types import Row + +from sparkdl.graph.builder import IsolatedSession +import sparkdl.graph.utils as tfx +from sparkdl.transformers.tf_tensor import TFTransformer + +from ..tests import SparkDLTestCase + +def grab_df_arr(df, output_col): + """ Stack the numpy array from a DataFrame column """ + return np.array([row.asDict()[output_col] + for row in df.select(output_col).toLocalIterator()]) + +class TFTransformerTest(SparkDLTestCase): + + def _get_rand_vec_df(self, num_rows, vec_size): + return self.session.createDataFrame( + Row(idx=idx, vec=np.random.randn(vec_size).tolist()) + for idx in range(num_rows)) + + def test_checkpoint_reload(self): + vec_size = 17 + num_vecs = 31 + df = self._get_rand_vec_df(num_vecs, vec_size) + analyzed_df = tfs.analyze(df) + input_col = 'vec' + output_col = 'outputCol' + + # Build the TensorFlow graph + model_temp_dir = tempfile.mkdtemp() + ckpt_dir = os.path.join(model_temp_dir, 'model_ckpt') + with tf.Session() as sess: + x = tf.placeholder(tf.float64, shape=[None, vec_size], name='tnsrIn') + w = tf.Variable(tf.random_normal([vec_size], dtype=tf.float64), + dtype=tf.float64, name='varW') + z = tf.reduce_mean(x * w, axis=1, name='tnsrOut') + sess.run(w.initializer) + saver = tf.train.Saver(var_list=[w]) + saved_path = saver.save(sess, ckpt_dir, global_step=2702) + + # Get the reference data + _results = [] + for row in df.rdd.toLocalIterator(): + arr = np.array(row.vec)[np.newaxis, :] + _results.append(sess.run(z, {x: arr})) + out_ref = np.hstack(_results) + + # Load the saved model checkpoint + # We want to clear device assignment in order to run it anywhere we want + with IsolatedSession() as issn: + saver = tf.train.import_meta_graph('{}.meta'.format(saved_path), clear_devices=True) + saver.restore(issn.sess, saved_path) + gfn = issn.asGraphFunction( + [tfx.get_tensor(issn.graph, 'tnsrIn')], + [tfx.get_tensor(issn.graph, 'tnsrOut')]) + + transformer = TFTransformer(tfGraph=gfn, + inputMapping={ + input_col: 'tnsrIn' + }, + outputMapping={ + 'tnsrOut': output_col + }) + final_df = transformer.transform(analyzed_df) + out_tgt = grab_df_arr(final_df, output_col) + + shutil.rmtree(model_temp_dir, ignore_errors=True) + self.assertTrue(np.allclose(out_ref, out_tgt)) + + def test_simple(self): + # Build a simple input DataFrame + vec_size = 17 + num_vecs = 31 + df = self._get_rand_vec_df(num_vecs, vec_size) + analyzed_df = tfs.analyze(df) + + # Build the TensorFlow graph + with tf.Session() as sess: + #x = tf.placeholder(tf.float64, shape=[None, vec_size]) + x = tfs.block(analyzed_df, 'vec') + z = tf.reduce_mean(x, axis=1) + graph = sess.graph + + # Get the reference data + _results = [] + for row in df.rdd.toLocalIterator(): + arr = np.array(row.vec)[np.newaxis, :] + _results.append(sess.run(z, {x: arr})) + out_ref = np.hstack(_results) + + # Apply the transform + transfomer = TFTransformer(tfGraph=graph, + inputMapping={ + 'vec': x + }, + outputMapping={ + z: 'outCol' + }) + final_df = transfomer.transform(analyzed_df) + out_tgt = grab_df_arr(final_df, 'outCol') + + self.assertTrue(np.allclose(out_ref, out_tgt)) + + + def test_multi_io(self): + # Build a simple input DataFrame + vec_size = 17 + num_vecs = 31 + _df = self._get_rand_vec_df(num_vecs, vec_size) + df_x = _df.withColumnRenamed('vec', 'vec_x') + _df = self._get_rand_vec_df(num_vecs, vec_size) + df_y = _df.withColumnRenamed('vec', 'vec_y') + df = df_x.join(df_y, on='idx', how='inner') + analyzed_df = tfs.analyze(df) + + # Build the TensorFlow graph + with tf.Session() as sess: + x = tfs.block(analyzed_df, 'vec_x') + y = tfs.block(analyzed_df, 'vec_y') + p = tf.reduce_mean(x + y, axis=1) + q = tf.reduce_mean(x - y, axis=1) + graph = sess.graph + + # Get the reference data + p_out_ref = [] + q_out_ref = [] + for row in df.rdd.toLocalIterator(): + arr_x = np.array(row['vec_x'])[np.newaxis, :] + arr_y = np.array(row['vec_y'])[np.newaxis, :] + p_val, q_val = sess.run([p, q], {x: arr_x, y: arr_y}) + p_out_ref.append(p_val) + q_out_ref.append(q_val) + p_out_ref = np.hstack(p_out_ref) + q_out_ref = np.hstack(q_out_ref) + + # Apply the transform + transfomer = TFTransformer(tfGraph=graph, + inputMapping={ + 'vec_x': x, + 'vec_y': y + }, + outputMapping={ + p: 'outcol_p', + q: 'outcol_q' + }) + final_df = transfomer.transform(analyzed_df) + p_out_tgt = grab_df_arr(final_df, 'outcol_p') + q_out_tgt = grab_df_arr(final_df, 'outcol_q') + + self.assertTrue(np.allclose(p_out_ref, p_out_tgt)) + self.assertTrue(np.allclose(q_out_ref, q_out_tgt)) + + def test_map_blocks_graph(self): + + vec_size = 17 + num_vecs = 137 + df = self._get_rand_vec_df(num_vecs, vec_size) + analyzed_df = tfs.analyze(df) + + input_col = 'vec' + output_col = 'outCol' + + # Build the graph: the output should have the same leading/batch dimension + with IsolatedSession(using_keras=True) as issn: + tnsr_in = tfs.block(analyzed_df, input_col) + inp = tf.expand_dims(tnsr_in, axis=2) + # Keras layers does not take tf.double + inp = tf.cast(inp, tf.float32) + conv = Conv1D(filters=4, kernel_size=2)(inp) + pool = MaxPool1D(pool_size=2)(conv) + flat = Flatten()(pool) + dense = Dense(1)(flat) + # We must keep the leading dimension of the output + redsum = tf.reduce_sum(dense, axis=1) + tnsr_out = tf.cast(redsum, tf.double, name='TnsrOut') + + # Initialize the variables + init_op = tf.global_variables_initializer() + issn.run(init_op) + # We could train the model ... but skip it here + gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) + + with IsolatedSession() as issn: + # Import the graph function object + feeds, fetches = issn.importGraphFunction(gfn, prefix='') + + # Rename the input column name to the feed op's name + orig_in_name = tfx.op_name(issn.graph, feeds[0]) + input_df = analyzed_df.withColumnRenamed(input_col, orig_in_name) + + # Do the actual computation + output_df = tfs.map_blocks(fetches, input_df) + + # Rename the output column (by default, the name of the fetch op's name) + orig_out_name = tfx.op_name(issn.graph, fetches[0]) + final_df = output_df.withColumnRenamed(orig_out_name, output_col) + + arr_ref = grab_df_arr(final_df, output_col) + + # Using the Transformer + transformer = TFTransformer(tfGraph=gfn, + inputMapping={ + input_col: gfn.input_names[0] + }, + outputMapping={ + gfn.output_names[0]: output_col + }) + transformed_df = transformer.transform(analyzed_df) + + arr_tgt = grab_df_arr(transformed_df, output_col) + + self.assertTrue(np.allclose(arr_ref, arr_tgt)) + From ecbefb948c8cbe8d66183493833732908e61d398 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 25 Aug 2017 00:44:16 -0700 Subject: [PATCH 02/80] support input graph scenarios --- python/sparkdl/transformers/param.py | 79 ++++++++- python/sparkdl/transformers/tf_tensor.py | 108 +++++++++--- python/tests/transformers/tf_tensor_test.py | 183 ++++++++++++++------ 3 files changed, 293 insertions(+), 77 deletions(-) diff --git a/python/sparkdl/transformers/param.py b/python/sparkdl/transformers/param.py index eb3d4188..e11789d0 100644 --- a/python/sparkdl/transformers/param.py +++ b/python/sparkdl/transformers/param.py @@ -119,14 +119,14 @@ def toTFGraph(value): @staticmethod def asColumnToTensorMap(value): if isinstance(value, dict): - strs_pair_seq = [(k, tfx.as_tensor_name(v)) for k, v in value.items()] + strs_pair_seq = [(k, tfx.as_op_name(v)) for k, v in value.items()] return sorted(strs_pair_seq) raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) @staticmethod def asTensorToColumnMap(value): if isinstance(value, dict): - strs_pair_seq = [(tfx.as_tensor_name(k), v) for k, v in value.items()] + strs_pair_seq = [(tfx.as_op_name(k), v) for k, v in value.items()] return sorted(strs_pair_seq) raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) @@ -161,8 +161,8 @@ class HasTFHParams(Params): """ Mixin for TensorFlow params """ - hparam = Param(Params._dummy(), "hparams", "instance of :class:`tf.contrib.training.HParams`", - typeConverter=SparkDLTypeConverters.toTFHParams) + tfHParms = Param(Params._dummy(), "hparams", "instance of :class:`tf.contrib.training.HParams`", + typeConverter=SparkDLTypeConverters.toTFHParams) # New in sparkdl @@ -202,6 +202,76 @@ def getInputMapping(self): return self.getOrDefault(self.inputMapping) +class HasTagSet(Params): + # TODO: add docs + tagSet = Param(Params._dummy(), "tagSet", + "signature def tag set", + typeConverter=TypeConverters.toString) + + def __init__(self): + super(HasTagSet, self).__init__() + # TODO: add default value + + def setTagSet(self, value): + return self._set(tagSet=value) + + def getTagSet(self): + return self.getOrDefault(self.tagSet) + + +class HasSignatureDefKey(Params): + # TODO: add docs + signatureDefKey = Param(Params._dummy(), "signatureDefKey", + "signature def", + typeConverter=TypeConverters.toString) + + def __init__(self): + super(HasSignatureDefKey, self).__init__() + # TODO: add default value + + def setSignatureDefKey(self, value): + return self._set(signatureDefKey=value) + + def getSignatureDefKey(self): + return self.getOrDefault(self.signatureDefKey) + + +class HasExportDir(Params): + """ + Mixin for param for constructing inputs + """ + exportDir = Param(Params._dummy(), "exportDir", + "Directory of saved model", + typeConverter=TypeConverters.toString) + + def __init__(self): + super(HasExportDir, self).__init__() + + def setExportDir(self, value): + return self._set(exportDir=value) + + def getExportDir(self): + return self.getOrDefault(self.exportDir) + + +class HasTFCheckpointDir(Params): + """ + Mixin for TensorFlow model checkpoint + """ + tfCheckpointDir = Param(Params._dummy(), "tfCheckpointDir", + "Directory that contains a model checkpoint", + typeConverter=TypeConverters.toString) + + def __init__(self): + super(HasTFCheckpointDir, self).__init__() + + def setTFCheckpointDir(self, value): + return self._set(tfCheckpointDir=value) + + def getTFCheckpointDir(self): + return self.getOrDefault(self.tfCheckpointDir) + + class HasTFGraph(Params): """ Mixin for param tfGraph: the :class:`tf.Graph` object that represents a TensorFlow computation. @@ -212,6 +282,7 @@ class HasTFGraph(Params): def __init__(self): super(HasTFGraph, self).__init__() + self._setDefault(tfGraph=None) def setTFGraph(self, value): return self._set(tfGraph=value) diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index d2053c3d..24ae4365 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -27,13 +27,15 @@ import sparkdl.graph.utils as tfx from sparkdl.transformers.param import ( keyword_only, HasInputMapping, HasOutputMapping, SparkDLTypeConverters, - HasTFGraph, HasTFHParams) + HasTFGraph, HasTFHParams, HasTFCheckpointDir, HasExportDir, HasTagSet, HasSignatureDefKey) __all__ = ['TFTransformer'] logger = logging.getLogger('sparkdl') -class TFTransformer(Transformer, HasTFGraph, HasTFHParams, HasInputMapping, HasOutputMapping): +class TFTransformer(Transformer, HasTFCheckpointDir, HasTFGraph, + HasExportDir, HasTagSet, HasSignatureDefKey, + HasTFHParams, HasInputMapping, HasOutputMapping): """ Applies the TensorFlow graph to the array column in DataFrame. @@ -46,49 +48,111 @@ class TFTransformer(Transformer, HasTFGraph, HasTFHParams, HasInputMapping, HasO """ @keyword_only - def __init__(self, inputMapping=None, outputMapping=None, tfGraph=None, hparams=None): + def __init__(self, tfCheckpointDir=None, tfGraph=None, + exportDir=None, tagSet=None, signatureDefKey=None, + inputMapping=None, outputMapping=None, tfHParms=None): """ - __init__(self, inputMapping=None, outputMapping=None, tfGraph=None, hparams=None) + __init__(self, tfCheckpointDir=None, tfGraph=None, + exportDir=None, tagSet=None, signatureDefKey=None, + inputMapping=None, outputMapping=None, tfHParms=None) """ super(TFTransformer, self).__init__() kwargs = self._input_kwargs self.setParams(**kwargs) + @keyword_only - def setParams(self, inputMapping=None, outputMapping=None, tfGraph=None, hparams=None): + def setParams(self, tfCheckpointDir=None, tfGraph=None, + exportDir=None, tagSet=None, signatureDefKey=None, + inputMapping=None, outputMapping=None, tfHParms=None): """ - setParams(self, inputMapping=None, outputMapping=None, tfGraph=None, hparams=None) + setParams(self, tfCheckpointDir=None, tfGraph=None, + exportDir=None, tagSet=None, signatureDefKey=None, + inputMapping=None, outputMapping=None, tfHParms=None) """ super(TFTransformer, self).__init__() kwargs = self._input_kwargs return self._set(**kwargs) - def _transform(self, dataset): - df = dataset - output_renaming = {} - with IsolatedSession(graph=self.getTFGraph()) as issn: + def _convertInternal(self): + assert self.isDefined(self.inputMapping) and self.isDefined(self.outputMapping), \ + "inputMapping and outputMapping must be defined" + + _maybe_graph = self.getTFGraph() + _maybe_meta_graph_def = None + with IsolatedSession(graph=_maybe_graph) as issn: + if self.isDefined(self.exportDir): + assert _maybe_graph is None + assert not self.isDefined(self.tfCheckpointDir) + tag_set = self.getTagSet().split(',') + _maybe_meta_graph_def = tf.saved_model.loader.load( + issn.sess, tag_set, self.getExportDir()) + elif self.isDefined(self.tfCheckpointDir): + assert _maybe_graph is None + ckpt_dir = self.getTFCheckpointDir() + ckpt_path = tf.train.latest_checkpoint(ckpt_dir) + print('using checkpoint path from {} as {}'.format(ckpt_dir, ckpt_path)) + saver = tf.train.import_meta_graph("{}.meta".format(ckpt_path), clear_devices=True) + saver.restore(issn.sess, ckpt_path) + _maybe_meta_graph_def = saver.export_meta_graph(clear_devices=True) + else: + assert _maybe_graph is not None + + sig_def = None + if self.isDefined(self.signatureDefKey): + sig_def_key = self.getSignatureDefKey() + if sig_def_key is not None: + meta_graph_def = _maybe_meta_graph_def + assert meta_graph_def is not None + #print('sigdef:', meta_graph_def.signature_def) + sig_def = tf.contrib.saved_model.get_signature_def_by_key( + meta_graph_def, sig_def_key) + assert sig_def is not None + feeds = [] - for input_colname, tnsr in self.getInputMapping(): + _input_mapping = {} + for input_colname, tnsr_or_sig in self.getInputMapping(): + if sig_def: + tnsr = sig_def.inputs[tnsr_or_sig].name + _input_mapping[input_colname] = tfx.op_name(issn.graph, tnsr) + else: + tnsr = tnsr_or_sig feeds.append(tfx.get_tensor(issn.graph, tnsr)) - tf_expected_colname = tfx.op_name(issn.graph, tnsr) - df = df.withColumnRenamed(input_colname, tf_expected_colname) + + if sig_def: + self.setInputMapping(_input_mapping) fetches = [] - for tnsr, output_colname in self.getOutputMapping(): + # By default the output columns will have the name of their + # corresponding `tf.Graph` operation names. + # We have to convert them to the user specified output names + self.output_renaming = {} + for tnsr_or_sig, output_colname in self.getOutputMapping(): + if sig_def: + tnsr = sig_def.outputs[tnsr_or_sig].name + else: + tnsr = tnsr_or_sig fetches.append(tfx.get_tensor(issn.graph, tnsr)) tf_expected_colname = tfx.op_name(issn.graph, tnsr) - output_renaming[tf_expected_colname] = output_colname + self.output_renaming[tf_expected_colname] = output_colname - gfn = issn.asGraphFunction(feeds, fetches, strip_and_freeze=True) + # Consolidate the input format into a serialized format + self.gfn = issn.asGraphFunction(feeds, fetches, strip_and_freeze=True) - analyzed_df = tfs.analyze(df) - with IsolatedSession() as issn: - _, fetches = issn.importGraphFunction(gfn, prefix='') - out_df = tfs.map_blocks(fetches, analyzed_df) + def _transform(self, dataset): + self._convertInternal() - for old_colname, new_colname in output_renaming.items(): - out_df = out_df.withColumnRenamed(old_colname, new_colname) + with IsolatedSession() as issn: + analyzed_df = tfs.analyze(dataset) + _, fetches = issn.importGraphFunction(self.gfn, prefix='') + feed_dict = dict([(tnsr_name, col_name) for col_name, tnsr_name in self.getInputMapping()]) + out_df = tfs.map_blocks(fetches, analyzed_df, feed_dict=feed_dict) + + # We still have to rename output columns + for old_colname, new_colname in self.output_renaming.items(): + if old_colname != new_colname: + out_df = out_df.withColumnRenamed(old_colname, new_colname) return out_df diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 495ff8e8..296709c8 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import absolute_import, division, print_function + import os import shutil import tempfile @@ -32,7 +34,7 @@ def grab_df_arr(df, output_col): """ Stack the numpy array from a DataFrame column """ return np.array([row.asDict()[output_col] - for row in df.select(output_col).toLocalIterator()]) + for row in df.select(output_col).collect()]) class TFTransformerTest(SparkDLTestCase): @@ -41,87 +43,167 @@ def _get_rand_vec_df(self, num_rows, vec_size): Row(idx=idx, vec=np.random.randn(vec_size).tolist()) for idx in range(num_rows)) - def test_checkpoint_reload(self): + def test_build_from_tf_graph(self): + # Build a simple input DataFrame vec_size = 17 num_vecs = 31 df = self._get_rand_vec_df(num_vecs, vec_size) analyzed_df = tfs.analyze(df) - input_col = 'vec' - output_col = 'outputCol' # Build the TensorFlow graph - model_temp_dir = tempfile.mkdtemp() - ckpt_dir = os.path.join(model_temp_dir, 'model_ckpt') with tf.Session() as sess: + #x = tf.placeholder(tf.float64, shape=[None, vec_size]) + x = tfs.block(analyzed_df, 'vec') + z = tf.reduce_mean(x, axis=1) + graph = sess.graph + + # Get the reference data + _results = [] + for row in df.collect(): + arr = np.array(row.vec)[np.newaxis, :] + _results.append(sess.run(z, {x: arr})) + out_ref = np.hstack(_results) + + # Apply the transform + transfomer = TFTransformer(tfGraph=graph, + inputMapping={ + 'vec': x + }, + outputMapping={ + z: 'outCol' + }) + final_df = transfomer.transform(analyzed_df) + out_tgt = grab_df_arr(final_df, 'outCol') + + self.assertTrue(np.allclose(out_ref, out_tgt)) + + + def test_build_from_saved_model(self): + # Setup dataset + vec_size = 17 + num_vecs = 31 + df = self._get_rand_vec_df(num_vecs, vec_size) + analyzed_df = tfs.analyze(df) + input_col = 'vec' + output_col = 'outputCol' + + # Setup saved model export directory + saved_model_root = tempfile.mkdtemp() + saved_model_dir = os.path.join(saved_model_root, 'saved_model') + serving_tag = "serving_tag" + serving_sigdef_key = 'prediction_signature' + + builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) + with tf.Session(graph=tf.Graph()) as sess: + # Model definition: begin x = tf.placeholder(tf.float64, shape=[None, vec_size], name='tnsrIn') + #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) w = tf.Variable(tf.random_normal([vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') z = tf.reduce_mean(x * w, axis=1, name='tnsrOut') + # Model definition ends + sess.run(w.initializer) - saver = tf.train.Saver(var_list=[w]) - saved_path = saver.save(sess, ckpt_dir, global_step=2702) + sig_inputs = { + 'input_sig': tf.saved_model.utils.build_tensor_info(x)} + sig_outputs = { + 'output_sig': tf.saved_model.utils.build_tensor_info(z)} + + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs=sig_inputs, + outputs=sig_outputs) + + builder.add_meta_graph_and_variables(sess, + [serving_tag], + signature_def_map={ + serving_sigdef_key: serving_sigdef + }) # Get the reference data _results = [] - for row in df.rdd.toLocalIterator(): + for row in df.collect(): arr = np.array(row.vec)[np.newaxis, :] _results.append(sess.run(z, {x: arr})) out_ref = np.hstack(_results) - # Load the saved model checkpoint - # We want to clear device assignment in order to run it anywhere we want - with IsolatedSession() as issn: - saver = tf.train.import_meta_graph('{}.meta'.format(saved_path), clear_devices=True) - saver.restore(issn.sess, saved_path) - gfn = issn.asGraphFunction( - [tfx.get_tensor(issn.graph, 'tnsrIn')], - [tfx.get_tensor(issn.graph, 'tnsrOut')]) - - transformer = TFTransformer(tfGraph=gfn, - inputMapping={ - input_col: 'tnsrIn' - }, - outputMapping={ - 'tnsrOut': output_col - }) - final_df = transformer.transform(analyzed_df) - out_tgt = grab_df_arr(final_df, output_col) - - shutil.rmtree(model_temp_dir, ignore_errors=True) - self.assertTrue(np.allclose(out_ref, out_tgt)) - - def test_simple(self): - # Build a simple input DataFrame + # Save the model + builder.save() + + # Build the transformer from exported serving model + # We are using signaures, thus must provide the keys + trans_with_sig = TFTransformer(exportDir=saved_model_dir, + signatureDefKey=serving_sigdef_key, + tagSet=serving_tag, + inputMapping={ + input_col: 'input_sig' + }, + outputMapping={ + 'output_sig': output_col + }) + + # Build the transformer from exported serving model + # We are not using signatures, thus must provide tensor/operation names + trans_no_sig = TFTransformer(exportDir=saved_model_dir, + signatureDefKey=None, + tagSet=serving_tag, + inputMapping={ + input_col: 'tnsrIn' + }, + outputMapping={ + 'tnsrOut': output_col + }) + + df_trans_with_sig = trans_with_sig.transform(analyzed_df) + df_trans_no_sig = trans_no_sig.transform(analyzed_df) + out_with_sig_tgt = grab_df_arr(df_trans_with_sig, output_col) + out_no_sig_tgt = grab_df_arr(df_trans_no_sig, output_col) + # Cleanup the resources + shutil.rmtree(saved_model_root, ignore_errors=True) + self.assertTrue(np.allclose(out_ref, out_with_sig_tgt)) + self.assertTrue(np.allclose(out_ref, out_no_sig_tgt)) + + + def test_build_from_checkpoint(self): vec_size = 17 num_vecs = 31 df = self._get_rand_vec_df(num_vecs, vec_size) analyzed_df = tfs.analyze(df) + input_col = 'vec' + output_col = 'outputCol' # Build the TensorFlow graph - with tf.Session() as sess: - #x = tf.placeholder(tf.float64, shape=[None, vec_size]) - x = tfs.block(analyzed_df, 'vec') - z = tf.reduce_mean(x, axis=1) - graph = sess.graph + model_ckpt_dir = tempfile.mkdtemp() + ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') + # Warning: please use a new graph for each test cases + # or the tests could affect one another + with tf.Session(graph=tf.Graph()) as sess: + x = tf.placeholder(tf.float64, shape=[None, vec_size], name='tnsrIn') + #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) + w = tf.Variable(tf.random_normal([vec_size], dtype=tf.float64), + dtype=tf.float64, name='varW') + z = tf.reduce_mean(x * w, axis=1, name='tnsrOut') + sess.run(w.initializer) + saver = tf.train.Saver(var_list=[w]) + _ = saver.save(sess, ckpt_path_prefix, global_step=2702) # Get the reference data _results = [] - for row in df.rdd.toLocalIterator(): + for row in df.collect(): arr = np.array(row.vec)[np.newaxis, :] _results.append(sess.run(z, {x: arr})) out_ref = np.hstack(_results) - # Apply the transform - transfomer = TFTransformer(tfGraph=graph, - inputMapping={ - 'vec': x - }, - outputMapping={ - z: 'outCol' - }) - final_df = transfomer.transform(analyzed_df) - out_tgt = grab_df_arr(final_df, 'outCol') + transformer = TFTransformer(tfCheckpointDir=model_ckpt_dir, + inputMapping={ + input_col: 'tnsrIn' + }, + outputMapping={ + 'tnsrOut': output_col + }) + final_df = transformer.transform(analyzed_df) + out_tgt = grab_df_arr(final_df, output_col) + shutil.rmtree(model_ckpt_dir, ignore_errors=True) self.assertTrue(np.allclose(out_ref, out_tgt)) @@ -147,7 +229,7 @@ def test_multi_io(self): # Get the reference data p_out_ref = [] q_out_ref = [] - for row in df.rdd.toLocalIterator(): + for row in df.collect(): arr_x = np.array(row['vec_x'])[np.newaxis, :] arr_y = np.array(row['vec_y'])[np.newaxis, :] p_val, q_val = sess.run([p, q], {x: arr_x, y: arr_y}) @@ -233,4 +315,3 @@ def test_map_blocks_graph(self): arr_tgt = grab_df_arr(transformed_df, output_col) self.assertTrue(np.allclose(arr_ref, arr_tgt)) - From ab89bd271e7ee964f6ed24a7b836e591eacf7fe7 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 8 Sep 2017 18:27:56 -0700 Subject: [PATCH 03/80] (WIP) new interface implementation --- python/sparkdl/transformers/param.py | 121 +++-------- python/sparkdl/transformers/tf_tensor.py | 227 ++++++++++++-------- python/tests/transformers/tf_tensor_test.py | 102 ++++----- 3 files changed, 219 insertions(+), 231 deletions(-) diff --git a/python/sparkdl/transformers/param.py b/python/sparkdl/transformers/param.py index e11789d0..90edbad0 100644 --- a/python/sparkdl/transformers/param.py +++ b/python/sparkdl/transformers/param.py @@ -58,9 +58,6 @@ class HasInputCol(Params): inputCol = Param(Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) - def __init__(self): - super(HasInputCol, self).__init__() - def setInputCol(self, value): """ Sets the value of :py:attr:`inputCol`. @@ -116,6 +113,16 @@ def toTFGraph(value): else: raise TypeError("Could not convert %s to TensorFlow Graph" % type(value)) + @staticmethod + def toTFInputGraph(value): + return value + # if isinstance(value, tf.Graph): + # return value.as_graph_def(add_shapes=True) + # elif isinstance(value, tf.GraphDef): + # return value + # else: + # raise TypeError("Could not convert %s to TFInputGraph" % type(value)) + @staticmethod def asColumnToTensorMap(value): if isinstance(value, dict): @@ -156,14 +163,6 @@ def converter(value): raise TypeError("%s %s is not in the supported list." % type(value), str(value)) return converter - -class HasTFHParams(Params): - """ - Mixin for TensorFlow params - """ - tfHParms = Param(Params._dummy(), "hparams", "instance of :class:`tf.contrib.training.HParams`", - typeConverter=SparkDLTypeConverters.toTFHParams) - # New in sparkdl class HasOutputMapping(Params): @@ -174,9 +173,6 @@ class HasOutputMapping(Params): "Mapping output :class:`tf.Tensor` objects to DataFrame column names", typeConverter=SparkDLTypeConverters.asTensorToColumnMap) - def __init__(self): - super(HasOutputMapping, self).__init__() - def setOutputMapping(self, value): return self._set(outputMapping=value) @@ -192,9 +188,6 @@ class HasInputMapping(Params): "Mapping input DataFrame column names to :class:`tf.Tensor` objects", typeConverter=SparkDLTypeConverters.asColumnToTensorMap) - def __init__(self): - super(HasInputMapping, self).__init__() - def setInputMapping(self, value): return self._set(inputMapping=value) @@ -202,90 +195,34 @@ def getInputMapping(self): return self.getOrDefault(self.inputMapping) -class HasTagSet(Params): - # TODO: add docs - tagSet = Param(Params._dummy(), "tagSet", - "signature def tag set", - typeConverter=TypeConverters.toString) - - def __init__(self): - super(HasTagSet, self).__init__() - # TODO: add default value - - def setTagSet(self, value): - return self._set(tagSet=value) - - def getTagSet(self): - return self.getOrDefault(self.tagSet) - - -class HasSignatureDefKey(Params): - # TODO: add docs - signatureDefKey = Param(Params._dummy(), "signatureDefKey", - "signature def", - typeConverter=TypeConverters.toString) - - def __init__(self): - super(HasSignatureDefKey, self).__init__() - # TODO: add default value - - def setSignatureDefKey(self, value): - return self._set(signatureDefKey=value) - - def getSignatureDefKey(self): - return self.getOrDefault(self.signatureDefKey) - - -class HasExportDir(Params): +class HasTFInputGraph(Params): """ - Mixin for param for constructing inputs - """ - exportDir = Param(Params._dummy(), "exportDir", - "Directory of saved model", - typeConverter=TypeConverters.toString) - - def __init__(self): - super(HasExportDir, self).__init__() - - def setExportDir(self, value): - return self._set(exportDir=value) - - def getExportDir(self): - return self.getOrDefault(self.exportDir) - - -class HasTFCheckpointDir(Params): - """ - Mixin for TensorFlow model checkpoint + Mixin for param tfGraph: the :class:`tf.Graph` object that represents a TensorFlow computation. """ - tfCheckpointDir = Param(Params._dummy(), "tfCheckpointDir", - "Directory that contains a model checkpoint", - typeConverter=TypeConverters.toString) + tfInputGraph = Param(Params._dummy(), "tfInputGraph", + "TensorFlow Graph object", + typeConverter=SparkDLTypeConverters.toTFInputGraph) def __init__(self): - super(HasTFCheckpointDir, self).__init__() + super(HasTFInputGraph, self).__init__() + self._setDefault(tfInputGraph=None) - def setTFCheckpointDir(self, value): - return self._set(tfCheckpointDir=value) + def setTFInputGraph(self, value): + return self._set(tfInputGraph=value) - def getTFCheckpointDir(self): - return self.getOrDefault(self.tfCheckpointDir) + def getTFInputGraph(self): + return self.getOrDefault(self.tfInputGraph) -class HasTFGraph(Params): +class HasTFHParams(Params): """ - Mixin for param tfGraph: the :class:`tf.Graph` object that represents a TensorFlow computation. + Mixin for TensorFlow model hyper-parameters """ - tfGraph = Param(Params._dummy(), "tfGraph", - "TensorFlow Graph object", - typeConverter=SparkDLTypeConverters.toTFGraph) - - def __init__(self): - super(HasTFGraph, self).__init__() - self._setDefault(tfGraph=None) + tfHParams = Param(Params._dummy(), "hparams", "instance of :class:`tf.contrib.training.HParams`", + typeConverter=SparkDLTypeConverters.toTFHParams) - def setTFGraph(self, value): - return self._set(tfGraph=value) + def setTFHParams(self, value): + return self._set(tfHParam=value) - def getTFGraph(self): - return self.getOrDefault(self.tfGraph) + def getTFHParams(self): + return self.getOrDefault(self.tfHParams) diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 24ae4365..54c05460 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -23,135 +23,182 @@ from pyspark.ml.param import Param, Params from pyspark.sql.functions import udf -from sparkdl.graph.builder import IsolatedSession +from sparkdl.graph.builder import GraphFunction, IsolatedSession import sparkdl.graph.utils as tfx from sparkdl.transformers.param import ( - keyword_only, HasInputMapping, HasOutputMapping, SparkDLTypeConverters, - HasTFGraph, HasTFHParams, HasTFCheckpointDir, HasExportDir, HasTagSet, HasSignatureDefKey) + keyword_only, SparkDLTypeConverters, HasInputMapping, + HasOutputMapping, HasTFInputGraph, HasTFHParams) __all__ = ['TFTransformer'] logger = logging.getLogger('sparkdl') -class TFTransformer(Transformer, HasTFCheckpointDir, HasTFGraph, - HasExportDir, HasTagSet, HasSignatureDefKey, - HasTFHParams, HasInputMapping, HasOutputMapping): +class TFInputGraph(object): + def __init__(self, graph_function, input_mapping, output_mapping): + # GraphFunction + self.graph_function = graph_function + # type: (str, str) list + if isinstance(input_mapping, dict): + input_mapping = input_mapping.items() + self.input_mapping = sorted(input_mapping) + # type: (str, str) list + if isinstance(output_mapping, dict): + output_mapping = output_mapping.items() + self.output_mapping = sorted(output_mapping) + +class TFInputGraphBuilder(object): """ - Applies the TensorFlow graph to the array column in DataFrame. - - Restrictions of the current API: - - We assume that - - All graphs have a "minibatch" dimension (i.e. an unknown leading - dimension) in the tensor shapes. - - Input DataFrame has an array column where all elements have the same length + Create a builder function so as to be able to compile graph for inference. + The actual compilation will be done at the time when the + inputs (feeds) and outputs (fetches) are provided. """ + def __init__(self, graph_import_fn): + # Return graph_def, input_mapping, output_mapping + self.graph_import_fn = graph_import_fn - @keyword_only - def __init__(self, tfCheckpointDir=None, tfGraph=None, - exportDir=None, tagSet=None, signatureDefKey=None, - inputMapping=None, outputMapping=None, tfHParms=None): - """ - __init__(self, tfCheckpointDir=None, tfGraph=None, - exportDir=None, tagSet=None, signatureDefKey=None, - inputMapping=None, outputMapping=None, tfHParms=None) - """ - super(TFTransformer, self).__init__() - kwargs = self._input_kwargs - self.setParams(**kwargs) - - - @keyword_only - def setParams(self, tfCheckpointDir=None, tfGraph=None, - exportDir=None, tagSet=None, signatureDefKey=None, - inputMapping=None, outputMapping=None, tfHParms=None): - """ - setParams(self, tfCheckpointDir=None, tfGraph=None, - exportDir=None, tagSet=None, signatureDefKey=None, - inputMapping=None, outputMapping=None, tfHParms=None) - """ - super(TFTransformer, self).__init__() - kwargs = self._input_kwargs - return self._set(**kwargs) - + def build(self, input_mapping, output_mapping): - def _convertInternal(self): - assert self.isDefined(self.inputMapping) and self.isDefined(self.outputMapping), \ - "inputMapping and outputMapping must be defined" - - _maybe_graph = self.getTFGraph() - _maybe_meta_graph_def = None - with IsolatedSession(graph=_maybe_graph) as issn: - if self.isDefined(self.exportDir): - assert _maybe_graph is None - assert not self.isDefined(self.tfCheckpointDir) - tag_set = self.getTagSet().split(',') - _maybe_meta_graph_def = tf.saved_model.loader.load( - issn.sess, tag_set, self.getExportDir()) - elif self.isDefined(self.tfCheckpointDir): - assert _maybe_graph is None - ckpt_dir = self.getTFCheckpointDir() - ckpt_path = tf.train.latest_checkpoint(ckpt_dir) - print('using checkpoint path from {} as {}'.format(ckpt_dir, ckpt_path)) - saver = tf.train.import_meta_graph("{}.meta".format(ckpt_path), clear_devices=True) - saver.restore(issn.sess, ckpt_path) - _maybe_meta_graph_def = saver.export_meta_graph(clear_devices=True) - else: - assert _maybe_graph is not None - - sig_def = None - if self.isDefined(self.signatureDefKey): - sig_def_key = self.getSignatureDefKey() - if sig_def_key is not None: - meta_graph_def = _maybe_meta_graph_def - assert meta_graph_def is not None - #print('sigdef:', meta_graph_def.signature_def) - sig_def = tf.contrib.saved_model.get_signature_def_by_key( - meta_graph_def, sig_def_key) - assert sig_def is not None + with IsolatedSession() as issn: + sig_def = self.graph_import_fn(issn.sess) + # Append feeds and input mapping feeds = [] _input_mapping = {} - for input_colname, tnsr_or_sig in self.getInputMapping(): + for input_colname, tnsr_or_sig in input_mapping.items(): if sig_def: tnsr = sig_def.inputs[tnsr_or_sig].name - _input_mapping[input_colname] = tfx.op_name(issn.graph, tnsr) else: tnsr = tnsr_or_sig + _input_mapping[input_colname] = tfx.op_name(issn.graph, tnsr) feeds.append(tfx.get_tensor(issn.graph, tnsr)) + input_mapping = _input_mapping - if sig_def: - self.setInputMapping(_input_mapping) - + # Append fetches and output mapping fetches = [] + _output_mapping = {} # By default the output columns will have the name of their # corresponding `tf.Graph` operation names. # We have to convert them to the user specified output names - self.output_renaming = {} - for tnsr_or_sig, output_colname in self.getOutputMapping(): + for tnsr_or_sig, requested_colname in output_mapping.items(): if sig_def: tnsr = sig_def.outputs[tnsr_or_sig].name else: tnsr = tnsr_or_sig fetches.append(tfx.get_tensor(issn.graph, tnsr)) - tf_expected_colname = tfx.op_name(issn.graph, tnsr) - self.output_renaming[tf_expected_colname] = output_colname + tf_output_colname = tfx.op_name(issn.graph, tnsr) + _output_mapping[tf_output_colname] = requested_colname + output_mapping = _output_mapping + + gfn = issn.asGraphFunction(feeds, fetches, strip_and_freeze=True) + + return TFInputGraph(gfn, input_mapping, output_mapping) + + @classmethod + def fromGraph(cls, graph): + assert isinstance(graph, tf.Graph), \ + ('expect tf.Graph type but got', type(graph)) + + def import_graph_fn(sess): + #graph.finalize() + gdef = graph.as_graph_def(add_shapes=True) + tf.import_graph_def(gdef, name='') + return None # no meta_graph_def + + return cls(import_graph_fn) + + @classmethod + def fromGraphDef(cls, graph_def): + assert isinstance(graph_def, tf.GraphDef), \ + ('expect tf.GraphDef type but got', type(graph_def)) + + def import_graph_fn(sess): + tf.import_graph_def(graph_def, name='') + return None + + return cls(import_graph_fn) + + @classmethod + def fromCheckpointDir(cls, checkpoint_dir, signature_def_key=None): - # Consolidate the input format into a serialized format - self.gfn = issn.asGraphFunction(feeds, fetches, strip_and_freeze=True) + def import_graph_fn(sess): + # Load checkpoint and import the graph + ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) + saver = tf.train.import_meta_graph("{}.meta".format(ckpt_path), clear_devices=True) + saver.restore(sess, ckpt_path) + meta_graph_def = saver.export_meta_graph(clear_devices=True) + sig_def = None + if signature_def_key is not None: + sig_def = tf.contrib.saved_model.get_signature_def_by_key( + meta_graph_def, signature_def_key) + + return sig_def + + return cls(import_graph_fn) + + @classmethod + def fromSavedModelDir(cls, saved_model_dir, tag_set, signature_def_key=None): + + def import_graph_fn(sess): + tag_sets = tag_set.split(',') + meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) + + sig_def = None + if signature_def_key is not None: + sig_def = tf.contrib.saved_model.get_signature_def_by_key( + meta_graph_def, signature_def_key) + + return sig_def + + return cls(import_graph_fn) + + +class TFTransformer(Transformer, HasTFInputGraph, HasTFHParams, HasInputMapping, HasOutputMapping): + """ + Applies the TensorFlow graph to the array column in DataFrame. + + Restrictions of the current API: + + We assume that + - All graphs have a "minibatch" dimension (i.e. an unknown leading + dimension) in the tensor shapes. + - Input DataFrame has an array column where all elements have the same length + """ + + @keyword_only + def __init__(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tfHParms=None): + """ + __init__(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tfHParms=None) + """ + super(TFTransformer, self).__init__() + kwargs = self._input_kwargs + gin = tfInputGraph.build(inputMapping, outputMapping) + kwargs['tfInputGraph'] = gin + self.setParams(**kwargs) + + @keyword_only + def setParams(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tfHParms=None): + """ + setParams(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tfHParms=None) + """ + super(TFTransformer, self).__init__() + kwargs = self._input_kwargs + return self._set(**kwargs) def _transform(self, dataset): - self._convertInternal() + gin = self.getTFInputGraph() + input_mapping = gin.input_mapping + output_mapping = gin.output_mapping with IsolatedSession() as issn: analyzed_df = tfs.analyze(dataset) - _, fetches = issn.importGraphFunction(self.gfn, prefix='') - feed_dict = dict([(tnsr_name, col_name) for col_name, tnsr_name in self.getInputMapping()]) + _, fetches = issn.importGraphFunction(gin.graph_function, prefix='') + feed_dict = dict([(tnsr_name, col_name) for col_name, tnsr_name in input_mapping]) + out_df = tfs.map_blocks(fetches, analyzed_df, feed_dict=feed_dict) # We still have to rename output columns - for old_colname, new_colname in self.output_renaming.items(): + for old_colname, new_colname in output_mapping: if old_colname != new_colname: out_df = out_df.withColumnRenamed(old_colname, new_colname) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 296709c8..a61d1b54 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -27,7 +27,7 @@ from sparkdl.graph.builder import IsolatedSession import sparkdl.graph.utils as tfx -from sparkdl.transformers.tf_tensor import TFTransformer +from sparkdl.transformers.tf_tensor import TFTransformer, TFInputGraphBuilder from ..tests import SparkDLTestCase @@ -65,13 +65,14 @@ def test_build_from_tf_graph(self): out_ref = np.hstack(_results) # Apply the transform - transfomer = TFTransformer(tfGraph=graph, - inputMapping={ - 'vec': x - }, - outputMapping={ - z: 'outCol' - }) + transfomer = TFTransformer( + tfInputGraph=TFInputGraphBuilder.fromGraph(graph), + inputMapping={ + 'vec': x + }, + outputMapping={ + z: 'outCol' + }) final_df = transfomer.transform(analyzed_df) out_tgt = grab_df_arr(final_df, 'outCol') @@ -131,27 +132,27 @@ def test_build_from_saved_model(self): # Build the transformer from exported serving model # We are using signaures, thus must provide the keys - trans_with_sig = TFTransformer(exportDir=saved_model_dir, - signatureDefKey=serving_sigdef_key, - tagSet=serving_tag, - inputMapping={ - input_col: 'input_sig' - }, - outputMapping={ - 'output_sig': output_col - }) + trans_with_sig = TFTransformer( + tfInputGraph=TFInputGraphBuilder.fromSavedModelDir( + saved_model_dir, tag_set=serving_tag, signature_def_key=serving_sigdef_key), + inputMapping={ + input_col: 'input_sig' + }, + outputMapping={ + 'output_sig': output_col + }) # Build the transformer from exported serving model # We are not using signatures, thus must provide tensor/operation names - trans_no_sig = TFTransformer(exportDir=saved_model_dir, - signatureDefKey=None, - tagSet=serving_tag, - inputMapping={ - input_col: 'tnsrIn' - }, - outputMapping={ - 'tnsrOut': output_col - }) + trans_no_sig = TFTransformer( + tfInputGraph=TFInputGraphBuilder.fromSavedModelDir( + saved_model_dir, tag_set=serving_tag, signature_def_key=None), + inputMapping={ + input_col: 'tnsrIn' + }, + outputMapping={ + 'tnsrOut': output_col + }) df_trans_with_sig = trans_with_sig.transform(analyzed_df) df_trans_no_sig = trans_no_sig.transform(analyzed_df) @@ -193,13 +194,14 @@ def test_build_from_checkpoint(self): _results.append(sess.run(z, {x: arr})) out_ref = np.hstack(_results) - transformer = TFTransformer(tfCheckpointDir=model_ckpt_dir, - inputMapping={ - input_col: 'tnsrIn' - }, - outputMapping={ - 'tnsrOut': output_col - }) + transformer = TFTransformer( + tfInputGraph=TFInputGraphBuilder.fromCheckpointDir(model_ckpt_dir), + inputMapping={ + input_col: 'tnsrIn' + }, + outputMapping={ + 'tnsrOut': output_col + }) final_df = transformer.transform(analyzed_df) out_tgt = grab_df_arr(final_df, output_col) @@ -239,15 +241,16 @@ def test_multi_io(self): q_out_ref = np.hstack(q_out_ref) # Apply the transform - transfomer = TFTransformer(tfGraph=graph, - inputMapping={ - 'vec_x': x, - 'vec_y': y - }, - outputMapping={ - p: 'outcol_p', - q: 'outcol_q' - }) + transfomer = TFTransformer( + tfInputGraph=TFInputGraphBuilder.fromGraph(graph), + inputMapping={ + 'vec_x': x, + 'vec_y': y + }, + outputMapping={ + p: 'outcol_p', + q: 'outcol_q' + }) final_df = transfomer.transform(analyzed_df) p_out_tgt = grab_df_arr(final_df, 'outcol_p') q_out_tgt = grab_df_arr(final_df, 'outcol_q') @@ -303,13 +306,14 @@ def test_map_blocks_graph(self): arr_ref = grab_df_arr(final_df, output_col) # Using the Transformer - transformer = TFTransformer(tfGraph=gfn, - inputMapping={ - input_col: gfn.input_names[0] - }, - outputMapping={ - gfn.output_names[0]: output_col - }) + transformer = TFTransformer( + tfInputGraph=TFInputGraphBuilder.fromGraphDef(gfn.graph_def), + inputMapping={ + input_col: gfn.input_names[0] + }, + outputMapping={ + gfn.output_names[0]: output_col + }) transformed_df = transformer.transform(analyzed_df) arr_tgt = grab_df_arr(transformed_df, output_col) From 8c7d72e629673f5ce4182214e2ae3495d4160cf9 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 8 Sep 2017 21:06:50 -0700 Subject: [PATCH 04/80] docs and cleanup --- python/sparkdl/transformers/tf_tensor.py | 63 +++++++++++++++------ python/tests/transformers/tf_tensor_test.py | 6 +- 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 54c05460..b655e85a 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -29,21 +29,37 @@ keyword_only, SparkDLTypeConverters, HasInputMapping, HasOutputMapping, HasTFInputGraph, HasTFHParams) -__all__ = ['TFTransformer'] +__all__ = ['TFTransformer', 'TFInputGraphBuilder'] logger = logging.getLogger('sparkdl') +def _assert_set_incl(seq_small, seq_large, msg): + set_small = set(seq_small) + set_large = set(seq_large) + assert set_small <= set_large, \ + 'set not inclusive: {} => diff items {}'.format(msg, set_small - set_large) + class TFInputGraph(object): + """ + An opaque serializable object containing TensorFlow graph. + """ + # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. def __init__(self, graph_function, input_mapping, output_mapping): # GraphFunction self.graph_function = graph_function - # type: (str, str) list + + _assert_set_incl(input_mapping.values(), graph_function.input_names, 'input names') if isinstance(input_mapping, dict): - input_mapping = input_mapping.items() + input_mapping = list(input_mapping.items()) + assert isinstance(input_mapping, list), \ + "output mapping must be a list of strings, found type {}".format(type(input_mapping)) self.input_mapping = sorted(input_mapping) - # type: (str, str) list + + _assert_set_incl(output_mapping.keys(), graph_function.output_names, 'output names') if isinstance(output_mapping, dict): - output_mapping = output_mapping.items() + output_mapping = list(output_mapping.items()) + assert isinstance(output_mapping, list), \ + "output mapping must be a list of strings, found type {}".format(type(output_mapping)) self.output_mapping = sorted(output_mapping) class TFInputGraphBuilder(object): @@ -51,13 +67,18 @@ class TFInputGraphBuilder(object): Create a builder function so as to be able to compile graph for inference. The actual compilation will be done at the time when the inputs (feeds) and outputs (fetches) are provided. + :param graph_import_fn: `tf.Session` -> `tf.signature_def`, load a graph to the provided session. + If the meta_graph contains a `signature_def`, return it. """ def __init__(self, graph_import_fn): - # Return graph_def, input_mapping, output_mapping + # Return signature_def if the underlying graph contains one self.graph_import_fn = graph_import_fn def build(self, input_mapping, output_mapping): - + """ + Create a serializable TensorFlow graph representation + :param input_mapping: dict, from input DataFrame column name to internal graph name. + """ with IsolatedSession() as issn: sig_def = self.graph_import_fn(issn.sess) @@ -95,31 +116,40 @@ def build(self, input_mapping, output_mapping): @classmethod def fromGraph(cls, graph): + """ + Construct a TFInputGraphBuilder from a in memory tf.Graph object + """ assert isinstance(graph, tf.Graph), \ ('expect tf.Graph type but got', type(graph)) def import_graph_fn(sess): - #graph.finalize() gdef = graph.as_graph_def(add_shapes=True) - tf.import_graph_def(gdef, name='') + with sess.as_default(): + tf.import_graph_def(gdef, name='') return None # no meta_graph_def return cls(import_graph_fn) @classmethod def fromGraphDef(cls, graph_def): + """ + Construct a TFInputGraphBuilder from a tf.GraphDef object + """ assert isinstance(graph_def, tf.GraphDef), \ ('expect tf.GraphDef type but got', type(graph_def)) def import_graph_fn(sess): - tf.import_graph_def(graph_def, name='') + with sess.as_default(): + tf.import_graph_def(graph_def, name='') return None return cls(import_graph_fn) @classmethod - def fromCheckpointDir(cls, checkpoint_dir, signature_def_key=None): - + def fromCheckpoint(cls, checkpoint_dir, signature_def_key=None): + """ + Construct a TFInputGraphBuilder from a model checkpoint + """ def import_graph_fn(sess): # Load checkpoint and import the graph ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) @@ -137,8 +167,10 @@ def import_graph_fn(sess): return cls(import_graph_fn) @classmethod - def fromSavedModelDir(cls, saved_model_dir, tag_set, signature_def_key=None): - + def fromSavedModel(cls, saved_model_dir, tag_set, signature_def_key=None): + """ + Construct a TFInputGraphBuilder from a SavedModel + """ def import_graph_fn(sess): tag_sets = tag_set.split(',') meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) @@ -172,8 +204,6 @@ def __init__(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tfH """ super(TFTransformer, self).__init__() kwargs = self._input_kwargs - gin = tfInputGraph.build(inputMapping, outputMapping) - kwargs['tfInputGraph'] = gin self.setParams(**kwargs) @keyword_only @@ -183,6 +213,7 @@ def setParams(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tf """ super(TFTransformer, self).__init__() kwargs = self._input_kwargs + kwargs['tfInputGraph'] = tfInputGraph.build(inputMapping, outputMapping) return self._set(**kwargs) def _transform(self, dataset): diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index a61d1b54..d9360c2d 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -133,7 +133,7 @@ def test_build_from_saved_model(self): # Build the transformer from exported serving model # We are using signaures, thus must provide the keys trans_with_sig = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromSavedModelDir( + tfInputGraph=TFInputGraphBuilder.fromSavedModel( saved_model_dir, tag_set=serving_tag, signature_def_key=serving_sigdef_key), inputMapping={ input_col: 'input_sig' @@ -145,7 +145,7 @@ def test_build_from_saved_model(self): # Build the transformer from exported serving model # We are not using signatures, thus must provide tensor/operation names trans_no_sig = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromSavedModelDir( + tfInputGraph=TFInputGraphBuilder.fromSavedModel( saved_model_dir, tag_set=serving_tag, signature_def_key=None), inputMapping={ input_col: 'tnsrIn' @@ -195,7 +195,7 @@ def test_build_from_checkpoint(self): out_ref = np.hstack(_results) transformer = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromCheckpointDir(model_ckpt_dir), + tfInputGraph=TFInputGraphBuilder.fromCheckpoint(model_ckpt_dir), inputMapping={ input_col: 'tnsrIn' }, From eb543c6cada6c757691789401f2c8810b32706a3 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 9 Sep 2017 20:43:42 -0700 Subject: [PATCH 05/80] using tensorflow API instead of our utilities --- python/sparkdl/transformers/tf_tensor.py | 49 +++++++++++------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index b655e85a..c053c0e6 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -20,10 +20,7 @@ import tensorframes as tfs from pyspark.ml import Transformer -from pyspark.ml.param import Param, Params -from pyspark.sql.functions import udf -from sparkdl.graph.builder import GraphFunction, IsolatedSession import sparkdl.graph.utils as tfx from sparkdl.transformers.param import ( keyword_only, SparkDLTypeConverters, HasInputMapping, @@ -33,29 +30,21 @@ logger = logging.getLogger('sparkdl') -def _assert_set_incl(seq_small, seq_large, msg): - set_small = set(seq_small) - set_large = set(seq_large) - assert set_small <= set_large, \ - 'set not inclusive: {} => diff items {}'.format(msg, set_small - set_large) - class TFInputGraph(object): """ An opaque serializable object containing TensorFlow graph. """ # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. - def __init__(self, graph_function, input_mapping, output_mapping): - # GraphFunction - self.graph_function = graph_function + def __init__(self, graph_def, input_mapping, output_mapping): + # tf.GraphDef + self.graph_def = graph_def - _assert_set_incl(input_mapping.values(), graph_function.input_names, 'input names') if isinstance(input_mapping, dict): input_mapping = list(input_mapping.items()) assert isinstance(input_mapping, list), \ "output mapping must be a list of strings, found type {}".format(type(input_mapping)) self.input_mapping = sorted(input_mapping) - _assert_set_incl(output_mapping.keys(), graph_function.output_names, 'output names') if isinstance(output_mapping, dict): output_mapping = list(output_mapping.items()) assert isinstance(output_mapping, list), \ @@ -79,19 +68,18 @@ def build(self, input_mapping, output_mapping): Create a serializable TensorFlow graph representation :param input_mapping: dict, from input DataFrame column name to internal graph name. """ - with IsolatedSession() as issn: - sig_def = self.graph_import_fn(issn.sess) + graph = tf.Graph() + with tf.Session(graph=graph) as sess: + sig_def = self.graph_import_fn(sess) # Append feeds and input mapping - feeds = [] _input_mapping = {} for input_colname, tnsr_or_sig in input_mapping.items(): if sig_def: tnsr = sig_def.inputs[tnsr_or_sig].name else: tnsr = tnsr_or_sig - _input_mapping[input_colname] = tfx.op_name(issn.graph, tnsr) - feeds.append(tfx.get_tensor(issn.graph, tnsr)) + _input_mapping[input_colname] = tfx.op_name(graph, tnsr) input_mapping = _input_mapping # Append fetches and output mapping @@ -105,14 +93,14 @@ def build(self, input_mapping, output_mapping): tnsr = sig_def.outputs[tnsr_or_sig].name else: tnsr = tnsr_or_sig - fetches.append(tfx.get_tensor(issn.graph, tnsr)) - tf_output_colname = tfx.op_name(issn.graph, tnsr) + fetches.append(tfx.get_tensor(graph, tnsr)) + tf_output_colname = tfx.op_name(graph, tnsr) _output_mapping[tf_output_colname] = requested_colname output_mapping = _output_mapping - gfn = issn.asGraphFunction(feeds, fetches, strip_and_freeze=True) + gdef = tfx.strip_and_freeze_until(fetches, graph, sess) - return TFInputGraph(gfn, input_mapping, output_mapping) + return TFInputGraph(gdef, input_mapping, output_mapping) @classmethod def fromGraph(cls, graph): @@ -221,10 +209,19 @@ def _transform(self, dataset): input_mapping = gin.input_mapping output_mapping = gin.output_mapping - with IsolatedSession() as issn: + graph = tf.Graph() + with tf.Session(graph=graph): analyzed_df = tfs.analyze(dataset) - _, fetches = issn.importGraphFunction(gin.graph_function, prefix='') - feed_dict = dict([(tnsr_name, col_name) for col_name, tnsr_name in input_mapping]) + + out_tnsr_op_names = [tfx.as_op_name(tnsr_op_name) + for tnsr_op_name, _ in output_mapping] + tf.import_graph_def(graph_def=gin.graph_def, + name='', + return_elements=out_tnsr_op_names) + + feed_dict = dict((tfx.op_name(graph, tnsr_op_name), col_name) + for col_name, tnsr_op_name in input_mapping) + fetches = [tfx.get_tensor(graph, tnsr_op_name) for tnsr_op_name in out_tnsr_op_names] out_df = tfs.map_blocks(fetches, analyzed_df, feed_dict=feed_dict) From 4743bb9818517b91d8eb3b0444bad8e86a25b796 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 9 Sep 2017 20:52:50 -0700 Subject: [PATCH 06/80] automatic type conversion --- python/sparkdl/transformers/param.py | 18 ++- python/sparkdl/transformers/tf_tensor.py | 149 +------------------- python/sparkdl/transformers/utils.py | 148 ++++++++++++++++++- python/tests/transformers/tf_tensor_test.py | 54 +++---- 4 files changed, 188 insertions(+), 181 deletions(-) diff --git a/python/sparkdl/transformers/param.py b/python/sparkdl/transformers/param.py index 90edbad0..827b7234 100644 --- a/python/sparkdl/transformers/param.py +++ b/python/sparkdl/transformers/param.py @@ -29,6 +29,7 @@ from sparkdl.graph.builder import GraphFunction, IsolatedSession import sparkdl.graph.utils as tfx +from sparkdl.transformers.utils import TFInputGraph, TFInputGraphBuilder """ Copied from PySpark for backward compatibility. First in Apache Spark version 2.1.1. @@ -115,13 +116,16 @@ def toTFGraph(value): @staticmethod def toTFInputGraph(value): - return value - # if isinstance(value, tf.Graph): - # return value.as_graph_def(add_shapes=True) - # elif isinstance(value, tf.GraphDef): - # return value - # else: - # raise TypeError("Could not convert %s to TFInputGraph" % type(value)) + if isinstance(value, TFInputGraph): + return value + elif isinstance(value, TFInputGraphBuilder): + return value + elif isinstance(value, tf.Graph): + return TFInputGraphBuilder.fromGraph(value) + elif isinstance(value, tf.GraphDef): + return TFInputGraphBuilder.fromGraphDef(value) + else: + raise TypeError("Could not convert %s to TFInputGraph" % type(value)) @staticmethod def asColumnToTensorMap(value): diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index c053c0e6..9aba6d88 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -15,13 +15,13 @@ from __future__ import absolute_import, division, print_function import logging -import numpy as np import tensorflow as tf import tensorframes as tfs from pyspark.ml import Transformer import sparkdl.graph.utils as tfx +from sparkdl.transformers.utils import TFInputGraphBuilder from sparkdl.transformers.param import ( keyword_only, SparkDLTypeConverters, HasInputMapping, HasOutputMapping, HasTFInputGraph, HasTFHParams) @@ -30,149 +30,6 @@ logger = logging.getLogger('sparkdl') -class TFInputGraph(object): - """ - An opaque serializable object containing TensorFlow graph. - """ - # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. - def __init__(self, graph_def, input_mapping, output_mapping): - # tf.GraphDef - self.graph_def = graph_def - - if isinstance(input_mapping, dict): - input_mapping = list(input_mapping.items()) - assert isinstance(input_mapping, list), \ - "output mapping must be a list of strings, found type {}".format(type(input_mapping)) - self.input_mapping = sorted(input_mapping) - - if isinstance(output_mapping, dict): - output_mapping = list(output_mapping.items()) - assert isinstance(output_mapping, list), \ - "output mapping must be a list of strings, found type {}".format(type(output_mapping)) - self.output_mapping = sorted(output_mapping) - -class TFInputGraphBuilder(object): - """ - Create a builder function so as to be able to compile graph for inference. - The actual compilation will be done at the time when the - inputs (feeds) and outputs (fetches) are provided. - :param graph_import_fn: `tf.Session` -> `tf.signature_def`, load a graph to the provided session. - If the meta_graph contains a `signature_def`, return it. - """ - def __init__(self, graph_import_fn): - # Return signature_def if the underlying graph contains one - self.graph_import_fn = graph_import_fn - - def build(self, input_mapping, output_mapping): - """ - Create a serializable TensorFlow graph representation - :param input_mapping: dict, from input DataFrame column name to internal graph name. - """ - graph = tf.Graph() - with tf.Session(graph=graph) as sess: - sig_def = self.graph_import_fn(sess) - - # Append feeds and input mapping - _input_mapping = {} - for input_colname, tnsr_or_sig in input_mapping.items(): - if sig_def: - tnsr = sig_def.inputs[tnsr_or_sig].name - else: - tnsr = tnsr_or_sig - _input_mapping[input_colname] = tfx.op_name(graph, tnsr) - input_mapping = _input_mapping - - # Append fetches and output mapping - fetches = [] - _output_mapping = {} - # By default the output columns will have the name of their - # corresponding `tf.Graph` operation names. - # We have to convert them to the user specified output names - for tnsr_or_sig, requested_colname in output_mapping.items(): - if sig_def: - tnsr = sig_def.outputs[tnsr_or_sig].name - else: - tnsr = tnsr_or_sig - fetches.append(tfx.get_tensor(graph, tnsr)) - tf_output_colname = tfx.op_name(graph, tnsr) - _output_mapping[tf_output_colname] = requested_colname - output_mapping = _output_mapping - - gdef = tfx.strip_and_freeze_until(fetches, graph, sess) - - return TFInputGraph(gdef, input_mapping, output_mapping) - - @classmethod - def fromGraph(cls, graph): - """ - Construct a TFInputGraphBuilder from a in memory tf.Graph object - """ - assert isinstance(graph, tf.Graph), \ - ('expect tf.Graph type but got', type(graph)) - - def import_graph_fn(sess): - gdef = graph.as_graph_def(add_shapes=True) - with sess.as_default(): - tf.import_graph_def(gdef, name='') - return None # no meta_graph_def - - return cls(import_graph_fn) - - @classmethod - def fromGraphDef(cls, graph_def): - """ - Construct a TFInputGraphBuilder from a tf.GraphDef object - """ - assert isinstance(graph_def, tf.GraphDef), \ - ('expect tf.GraphDef type but got', type(graph_def)) - - def import_graph_fn(sess): - with sess.as_default(): - tf.import_graph_def(graph_def, name='') - return None - - return cls(import_graph_fn) - - @classmethod - def fromCheckpoint(cls, checkpoint_dir, signature_def_key=None): - """ - Construct a TFInputGraphBuilder from a model checkpoint - """ - def import_graph_fn(sess): - # Load checkpoint and import the graph - ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) - saver = tf.train.import_meta_graph("{}.meta".format(ckpt_path), clear_devices=True) - saver.restore(sess, ckpt_path) - meta_graph_def = saver.export_meta_graph(clear_devices=True) - - sig_def = None - if signature_def_key is not None: - sig_def = tf.contrib.saved_model.get_signature_def_by_key( - meta_graph_def, signature_def_key) - - return sig_def - - return cls(import_graph_fn) - - @classmethod - def fromSavedModel(cls, saved_model_dir, tag_set, signature_def_key=None): - """ - Construct a TFInputGraphBuilder from a SavedModel - """ - def import_graph_fn(sess): - tag_sets = tag_set.split(',') - meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) - - sig_def = None - if signature_def_key is not None: - sig_def = tf.contrib.saved_model.get_signature_def_by_key( - meta_graph_def, signature_def_key) - - return sig_def - - return cls(import_graph_fn) - - class TFTransformer(Transformer, HasTFInputGraph, HasTFHParams, HasInputMapping, HasOutputMapping): """ Applies the TensorFlow graph to the array column in DataFrame. @@ -201,7 +58,9 @@ def setParams(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tf """ super(TFTransformer, self).__init__() kwargs = self._input_kwargs - kwargs['tfInputGraph'] = tfInputGraph.build(inputMapping, outputMapping) + _maybe_gin = SparkDLTypeConverters.toTFInputGraph(tfInputGraph) + if isinstance(_maybe_gin, TFInputGraphBuilder): + kwargs['tfInputGraph'] = _maybe_gin.build(inputMapping, outputMapping) return self._set(**kwargs) def _transform(self, dataset): diff --git a/python/sparkdl/transformers/utils.py b/python/sparkdl/transformers/utils.py index bb20ce2e..04ff3eff 100644 --- a/python/sparkdl/transformers/utils.py +++ b/python/sparkdl/transformers/utils.py @@ -15,11 +15,9 @@ import tensorflow as tf -from pyspark.ml.param import TypeConverters - +import sparkdl.graph.utils as tfx from sparkdl.image.imageIO import imageType - # image stuff IMAGE_INPUT_PLACEHOLDER_NAME = "sparkdl_image_input" @@ -36,3 +34,147 @@ class ImageNetConstants: class InceptionV3Constants: INPUT_SHAPE = (299, 299) NUM_OUTPUT_FEATURES = 131072 + + +class TFInputGraph(object): + """ + An opaque serializable object containing TensorFlow graph. + """ + # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. + def __init__(self, graph_def, input_mapping, output_mapping): + # tf.GraphDef + self.graph_def = graph_def + + if isinstance(input_mapping, dict): + input_mapping = list(input_mapping.items()) + assert isinstance(input_mapping, list), \ + "output mapping must be a list of strings, found type {}".format(type(input_mapping)) + self.input_mapping = sorted(input_mapping) + + if isinstance(output_mapping, dict): + output_mapping = list(output_mapping.items()) + assert isinstance(output_mapping, list), \ + "output mapping must be a list of strings, found type {}".format(type(output_mapping)) + self.output_mapping = sorted(output_mapping) + + +class TFInputGraphBuilder(object): + """ + Create a builder function so as to be able to compile graph for inference. + The actual compilation will be done at the time when the + inputs (feeds) and outputs (fetches) are provided. + :param graph_import_fn: `tf.Session` -> `tf.signature_def`, load a graph to the provided session. + If the meta_graph contains a `signature_def`, return it. + """ + def __init__(self, graph_import_fn): + # Return signature_def if the underlying graph contains one + self.graph_import_fn = graph_import_fn + + def build(self, input_mapping, output_mapping): + """ + Create a serializable TensorFlow graph representation + :param input_mapping: dict, from input DataFrame column name to internal graph name. + """ + graph = tf.Graph() + with tf.Session(graph=graph) as sess: + sig_def = self.graph_import_fn(sess) + + # Append feeds and input mapping + _input_mapping = {} + for input_colname, tnsr_or_sig in input_mapping.items(): + if sig_def: + tnsr = sig_def.inputs[tnsr_or_sig].name + else: + tnsr = tnsr_or_sig + _input_mapping[input_colname] = tfx.op_name(graph, tnsr) + input_mapping = _input_mapping + + # Append fetches and output mapping + fetches = [] + _output_mapping = {} + # By default the output columns will have the name of their + # corresponding `tf.Graph` operation names. + # We have to convert them to the user specified output names + for tnsr_or_sig, requested_colname in output_mapping.items(): + if sig_def: + tnsr = sig_def.outputs[tnsr_or_sig].name + else: + tnsr = tnsr_or_sig + fetches.append(tfx.get_tensor(graph, tnsr)) + tf_output_colname = tfx.op_name(graph, tnsr) + _output_mapping[tf_output_colname] = requested_colname + output_mapping = _output_mapping + + gdef = tfx.strip_and_freeze_until(fetches, graph, sess) + + return TFInputGraph(gdef, input_mapping, output_mapping) + + @classmethod + def fromGraph(cls, graph): + """ + Construct a TFInputGraphBuilder from a in memory tf.Graph object + """ + assert isinstance(graph, tf.Graph), \ + ('expect tf.Graph type but got', type(graph)) + + def import_graph_fn(sess): + gdef = graph.as_graph_def(add_shapes=True) + with sess.as_default(): + tf.import_graph_def(gdef, name='') + return None # no meta_graph_def + + return cls(import_graph_fn) + + @classmethod + def fromGraphDef(cls, graph_def): + """ + Construct a TFInputGraphBuilder from a tf.GraphDef object + """ + assert isinstance(graph_def, tf.GraphDef), \ + ('expect tf.GraphDef type but got', type(graph_def)) + + def import_graph_fn(sess): + with sess.as_default(): + tf.import_graph_def(graph_def, name='') + return None + + return cls(import_graph_fn) + + @classmethod + def fromCheckpoint(cls, checkpoint_dir, signature_def_key=None): + """ + Construct a TFInputGraphBuilder from a model checkpoint + """ + def import_graph_fn(sess): + # Load checkpoint and import the graph + ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) + saver = tf.train.import_meta_graph("{}.meta".format(ckpt_path), clear_devices=True) + saver.restore(sess, ckpt_path) + meta_graph_def = saver.export_meta_graph(clear_devices=True) + + sig_def = None + if signature_def_key is not None: + sig_def = tf.contrib.saved_model.get_signature_def_by_key( + meta_graph_def, signature_def_key) + + return sig_def + + return cls(import_graph_fn) + + @classmethod + def fromSavedModel(cls, saved_model_dir, tag_set, signature_def_key=None): + """ + Construct a TFInputGraphBuilder from a SavedModel + """ + def import_graph_fn(sess): + tag_sets = tag_set.split(',') + meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) + + sig_def = None + if signature_def_key is not None: + sig_def = tf.contrib.saved_model.get_signature_def_by_key( + meta_graph_def, signature_def_key) + + return sig_def + + return cls(import_graph_fn) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index d9360c2d..85df23a7 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -65,18 +65,19 @@ def test_build_from_tf_graph(self): out_ref = np.hstack(_results) # Apply the transform - transfomer = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromGraph(graph), - inputMapping={ - 'vec': x - }, - outputMapping={ - z: 'outCol' - }) - final_df = transfomer.transform(analyzed_df) - out_tgt = grab_df_arr(final_df, 'outCol') - - self.assertTrue(np.allclose(out_ref, out_tgt)) + gin_from_graph = TFInputGraphBuilder.fromGraph(graph) + for gin in [gin_from_graph, graph]: + transfomer = TFTransformer( + tfInputGraph=TFInputGraphBuilder.fromGraph(graph), + inputMapping={ + 'vec': x + }, + outputMapping={ + z: 'outCol' + }) + final_df = transfomer.transform(analyzed_df) + out_tgt = grab_df_arr(final_df, 'outCol') + self.assertTrue(np.allclose(out_ref, out_tgt)) def test_build_from_saved_model(self): @@ -258,7 +259,7 @@ def test_multi_io(self): self.assertTrue(np.allclose(p_out_ref, p_out_tgt)) self.assertTrue(np.allclose(q_out_ref, q_out_tgt)) - def test_map_blocks_graph(self): + def test_mixed_keras_graph(self): vec_size = 17 num_vecs = 137 @@ -306,16 +307,17 @@ def test_map_blocks_graph(self): arr_ref = grab_df_arr(final_df, output_col) # Using the Transformer - transformer = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromGraphDef(gfn.graph_def), - inputMapping={ - input_col: gfn.input_names[0] - }, - outputMapping={ - gfn.output_names[0]: output_col - }) - transformed_df = transformer.transform(analyzed_df) - - arr_tgt = grab_df_arr(transformed_df, output_col) - - self.assertTrue(np.allclose(arr_ref, arr_tgt)) + gin_from_gdef = TFInputGraphBuilder.fromGraphDef(gfn.graph_def) + for gin in [gin_from_gdef, gfn.graph_def]: + transformer = TFTransformer( + tfInputGraph=gin, + inputMapping={ + input_col: gfn.input_names[0] + }, + outputMapping={ + gfn.output_names[0]: output_col + }) + + transformed_df = transformer.transform(analyzed_df) + arr_tgt = grab_df_arr(transformed_df, output_col) + self.assertTrue(np.allclose(arr_ref, arr_tgt)) From 622c7884a9f670f7af33cbdebaf37d7447463266 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 9 Sep 2017 21:27:50 -0700 Subject: [PATCH 07/80] cleanup --- python/sparkdl/__init__.py | 8 ++++---- python/sparkdl/transformers/tf_tensor.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/sparkdl/__init__.py b/python/sparkdl/__init__.py index aa15059a..b1a921cb 100644 --- a/python/sparkdl/__init__.py +++ b/python/sparkdl/__init__.py @@ -17,11 +17,11 @@ from .transformers.keras_image import KerasImageFileTransformer from .transformers.named_image import DeepImagePredictor, DeepImageFeaturizer from .transformers.tf_image import TFImageTransformer -from .transformers.utils import imageInputPlaceholder +from .transformers.tf_tensor import TFTransformer +from .transformers.utils import imageInputPlaceholder, TFInputGraphBuilder __all__ = [ 'imageSchema', 'imageType', 'readImages', - 'TFImageTransformer', - 'DeepImagePredictor', 'DeepImageFeaturizer', - 'KerasImageFileTransformer', + 'TFImageTransformer', 'TFInputGraphBuilder', 'TFTransformer', + 'DeepImagePredictor', 'DeepImageFeaturizer', 'KerasImageFileTransformer', 'imageInputPlaceholder'] diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 9aba6d88..255c264a 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -26,7 +26,7 @@ keyword_only, SparkDLTypeConverters, HasInputMapping, HasOutputMapping, HasTFInputGraph, HasTFHParams) -__all__ = ['TFTransformer', 'TFInputGraphBuilder'] +__all__ = ['TFTransformer'] logger = logging.getLogger('sparkdl') From 07f1cec524b9ba604692be8e7bb7c2b12c7eb54c Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 11 Sep 2017 16:09:22 -0700 Subject: [PATCH 08/80] PR comments 1. Move `InputGraph` to its module. --- python/sparkdl/graph/input.py | 166 +++++++++++++++++++++++++++ python/sparkdl/transformers/utils.py | 145 ----------------------- 2 files changed, 166 insertions(+), 145 deletions(-) create mode 100644 python/sparkdl/graph/input.py diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py new file mode 100644 index 00000000..ea0d7502 --- /dev/null +++ b/python/sparkdl/graph/input.py @@ -0,0 +1,166 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import tensorflow as tf + +import sparkdl.graph.utils as tfx + + +class TFInputGraph(object): + """ + An opaque serializable object containing TensorFlow graph. + """ + + # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. + def __init__(self, graph_def, input_mapping, output_mapping): + # tf.GraphDef + self.graph_def = graph_def + + if isinstance(input_mapping, dict): + input_mapping = list(input_mapping.items()) + assert isinstance(input_mapping, list), \ + "output mapping must be a list of strings, found type {}".format(type(input_mapping)) + self.input_mapping = sorted(input_mapping) + + if isinstance(output_mapping, dict): + output_mapping = list(output_mapping.items()) + assert isinstance(output_mapping, list), \ + "output mapping must be a list of strings, found type {}".format(type(output_mapping)) + self.output_mapping = sorted(output_mapping) + + +class TFInputGraphBuilder(object): + """ + Create a builder function so as to be able to compile graph for inference. + The actual compilation will be done at the time when the + inputs (feeds) and outputs (fetches) are provided. + :param graph_import_fn: `tf.Session` -> `tf.signature_def`, load a graph to the provided session. + If the meta_graph contains a `signature_def`, return it. + """ + + def __init__(self, graph_import_fn): + # Return signature_def if the underlying graph contains one + self.graph_import_fn = graph_import_fn + + def build(self, input_mapping, output_mapping): + """ + Create a serializable TensorFlow graph representation + :param input_mapping: dict, from input DataFrame column name to internal graph name. + """ + graph = tf.Graph() + with tf.Session(graph=graph) as sess: + sig_def = self.graph_import_fn(sess) + + # Append feeds and input mapping + _input_mapping = {} + for input_colname, tnsr_or_sig in input_mapping.items(): + if sig_def: + tnsr = sig_def.inputs[tnsr_or_sig].name + else: + tnsr = tnsr_or_sig + _input_mapping[input_colname] = tfx.op_name(graph, tnsr) + input_mapping = _input_mapping + + # Append fetches and output mapping + fetches = [] + _output_mapping = {} + # By default the output columns will have the name of their + # corresponding `tf.Graph` operation names. + # We have to convert them to the user specified output names + for tnsr_or_sig, requested_colname in output_mapping.items(): + if sig_def: + tnsr = sig_def.outputs[tnsr_or_sig].name + else: + tnsr = tnsr_or_sig + fetches.append(tfx.get_tensor(graph, tnsr)) + tf_output_colname = tfx.op_name(graph, tnsr) + _output_mapping[tf_output_colname] = requested_colname + output_mapping = _output_mapping + + gdef = tfx.strip_and_freeze_until(fetches, graph, sess) + + return TFInputGraph(gdef, input_mapping, output_mapping) + + @classmethod + def fromGraph(cls, graph): + """ + Construct a TFInputGraphBuilder from a in memory tf.Graph object + """ + assert isinstance(graph, tf.Graph), \ + ('expect tf.Graph type but got', type(graph)) + + def import_graph_fn(sess): + gdef = graph.as_graph_def(add_shapes=True) + with sess.as_default(): + tf.import_graph_def(gdef, name='') + return None # no meta_graph_def + + return cls(import_graph_fn) + + @classmethod + def fromGraphDef(cls, graph_def): + """ + Construct a TFInputGraphBuilder from a tf.GraphDef object + """ + assert isinstance(graph_def, tf.GraphDef), \ + ('expect tf.GraphDef type but got', type(graph_def)) + + def import_graph_fn(sess): + with sess.as_default(): + tf.import_graph_def(graph_def, name='') + return None + + return cls(import_graph_fn) + + @classmethod + def fromCheckpoint(cls, checkpoint_dir, signature_def_key=None): + """ + Construct a TFInputGraphBuilder from a model checkpoint + """ + + def import_graph_fn(sess): + # Load checkpoint and import the graph + ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) + saver = tf.train.import_meta_graph("{}.meta".format(ckpt_path), clear_devices=True) + saver.restore(sess, ckpt_path) + meta_graph_def = saver.export_meta_graph(clear_devices=True) + + sig_def = None + if signature_def_key is not None: + sig_def = tf.contrib.saved_model.get_signature_def_by_key( + meta_graph_def, signature_def_key) + + return sig_def + + return cls(import_graph_fn) + + @classmethod + def fromSavedModel(cls, saved_model_dir, tag_set, signature_def_key=None): + """ + Construct a TFInputGraphBuilder from a SavedModel + """ + + def import_graph_fn(sess): + tag_sets = tag_set.split(',') + meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) + + sig_def = None + if signature_def_key is not None: + sig_def = tf.contrib.saved_model.get_signature_def_by_key( + meta_graph_def, signature_def_key) + + return sig_def + + return cls(import_graph_fn) diff --git a/python/sparkdl/transformers/utils.py b/python/sparkdl/transformers/utils.py index 04ff3eff..64373e6a 100644 --- a/python/sparkdl/transformers/utils.py +++ b/python/sparkdl/transformers/utils.py @@ -15,7 +15,6 @@ import tensorflow as tf -import sparkdl.graph.utils as tfx from sparkdl.image.imageIO import imageType # image stuff @@ -34,147 +33,3 @@ class ImageNetConstants: class InceptionV3Constants: INPUT_SHAPE = (299, 299) NUM_OUTPUT_FEATURES = 131072 - - -class TFInputGraph(object): - """ - An opaque serializable object containing TensorFlow graph. - """ - # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. - def __init__(self, graph_def, input_mapping, output_mapping): - # tf.GraphDef - self.graph_def = graph_def - - if isinstance(input_mapping, dict): - input_mapping = list(input_mapping.items()) - assert isinstance(input_mapping, list), \ - "output mapping must be a list of strings, found type {}".format(type(input_mapping)) - self.input_mapping = sorted(input_mapping) - - if isinstance(output_mapping, dict): - output_mapping = list(output_mapping.items()) - assert isinstance(output_mapping, list), \ - "output mapping must be a list of strings, found type {}".format(type(output_mapping)) - self.output_mapping = sorted(output_mapping) - - -class TFInputGraphBuilder(object): - """ - Create a builder function so as to be able to compile graph for inference. - The actual compilation will be done at the time when the - inputs (feeds) and outputs (fetches) are provided. - :param graph_import_fn: `tf.Session` -> `tf.signature_def`, load a graph to the provided session. - If the meta_graph contains a `signature_def`, return it. - """ - def __init__(self, graph_import_fn): - # Return signature_def if the underlying graph contains one - self.graph_import_fn = graph_import_fn - - def build(self, input_mapping, output_mapping): - """ - Create a serializable TensorFlow graph representation - :param input_mapping: dict, from input DataFrame column name to internal graph name. - """ - graph = tf.Graph() - with tf.Session(graph=graph) as sess: - sig_def = self.graph_import_fn(sess) - - # Append feeds and input mapping - _input_mapping = {} - for input_colname, tnsr_or_sig in input_mapping.items(): - if sig_def: - tnsr = sig_def.inputs[tnsr_or_sig].name - else: - tnsr = tnsr_or_sig - _input_mapping[input_colname] = tfx.op_name(graph, tnsr) - input_mapping = _input_mapping - - # Append fetches and output mapping - fetches = [] - _output_mapping = {} - # By default the output columns will have the name of their - # corresponding `tf.Graph` operation names. - # We have to convert them to the user specified output names - for tnsr_or_sig, requested_colname in output_mapping.items(): - if sig_def: - tnsr = sig_def.outputs[tnsr_or_sig].name - else: - tnsr = tnsr_or_sig - fetches.append(tfx.get_tensor(graph, tnsr)) - tf_output_colname = tfx.op_name(graph, tnsr) - _output_mapping[tf_output_colname] = requested_colname - output_mapping = _output_mapping - - gdef = tfx.strip_and_freeze_until(fetches, graph, sess) - - return TFInputGraph(gdef, input_mapping, output_mapping) - - @classmethod - def fromGraph(cls, graph): - """ - Construct a TFInputGraphBuilder from a in memory tf.Graph object - """ - assert isinstance(graph, tf.Graph), \ - ('expect tf.Graph type but got', type(graph)) - - def import_graph_fn(sess): - gdef = graph.as_graph_def(add_shapes=True) - with sess.as_default(): - tf.import_graph_def(gdef, name='') - return None # no meta_graph_def - - return cls(import_graph_fn) - - @classmethod - def fromGraphDef(cls, graph_def): - """ - Construct a TFInputGraphBuilder from a tf.GraphDef object - """ - assert isinstance(graph_def, tf.GraphDef), \ - ('expect tf.GraphDef type but got', type(graph_def)) - - def import_graph_fn(sess): - with sess.as_default(): - tf.import_graph_def(graph_def, name='') - return None - - return cls(import_graph_fn) - - @classmethod - def fromCheckpoint(cls, checkpoint_dir, signature_def_key=None): - """ - Construct a TFInputGraphBuilder from a model checkpoint - """ - def import_graph_fn(sess): - # Load checkpoint and import the graph - ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) - saver = tf.train.import_meta_graph("{}.meta".format(ckpt_path), clear_devices=True) - saver.restore(sess, ckpt_path) - meta_graph_def = saver.export_meta_graph(clear_devices=True) - - sig_def = None - if signature_def_key is not None: - sig_def = tf.contrib.saved_model.get_signature_def_by_key( - meta_graph_def, signature_def_key) - - return sig_def - - return cls(import_graph_fn) - - @classmethod - def fromSavedModel(cls, saved_model_dir, tag_set, signature_def_key=None): - """ - Construct a TFInputGraphBuilder from a SavedModel - """ - def import_graph_fn(sess): - tag_sets = tag_set.split(',') - meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) - - sig_def = None - if signature_def_key is not None: - sig_def = tf.contrib.saved_model.get_signature_def_by_key( - meta_graph_def, signature_def_key) - - return sig_def - - return cls(import_graph_fn) From 692b0ebeaacbcfaff22c3205e9a55df3256c5cac Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 11 Sep 2017 20:49:06 -0700 Subject: [PATCH 09/80] (WIP) address comments --- python/sparkdl/__init__.py | 4 +- python/sparkdl/graph/input.py | 57 +++++++++++++--- python/sparkdl/transformers/param.py | 56 +++++++++------ python/sparkdl/transformers/tf_tensor.py | 14 ++-- python/tests/transformers/tf_tensor_test.py | 75 ++++++++++++++++----- 5 files changed, 150 insertions(+), 56 deletions(-) diff --git a/python/sparkdl/__init__.py b/python/sparkdl/__init__.py index b1a921cb..c5c55ff3 100644 --- a/python/sparkdl/__init__.py +++ b/python/sparkdl/__init__.py @@ -13,12 +13,14 @@ # limitations under the License. # +from .graph.input import TFInputGraphBuilder from .image.imageIO import imageSchema, imageType, readImages from .transformers.keras_image import KerasImageFileTransformer from .transformers.named_image import DeepImagePredictor, DeepImageFeaturizer from .transformers.tf_image import TFImageTransformer from .transformers.tf_tensor import TFTransformer -from .transformers.utils import imageInputPlaceholder, TFInputGraphBuilder +from .transformers.utils import imageInputPlaceholder + __all__ = [ 'imageSchema', 'imageType', 'readImages', diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index ea0d7502..c495697d 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -14,13 +14,18 @@ # import tensorflow as tf +from tensorflow.core.protobuf import meta_graph_pb2 import sparkdl.graph.utils as tfx +__all__ = ["TFInputGraphBuilder", "get_params_from_checkpoint", "get_params_from_saved_model"] + class TFInputGraph(object): """ An opaque serializable object containing TensorFlow graph. + + [WARNING] This class should not be called by any user code. """ # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. @@ -41,6 +46,28 @@ def __init__(self, graph_def, input_mapping, output_mapping): self.output_mapping = sorted(output_mapping) +def _get_params_from(gin_builder, input_mapping, output_mapping): + gin = gin_builder.build(input_mapping, output_mapping) + imap = dict(gin.input_mapping) + assert len(imap) == len(gin.input_mapping) + omap = dict(gin.output_mapping) + assert len(omap) == len(gin.output_mapping) + return gin.graph_def, imap, omap + + +def get_params_from_checkpoint(checkpoint_dir, signature_def_key, input_mapping, output_mapping): + assert signature_def_key is not None + gin_builder = TFInputGraphBuilder.fromCheckpoint(checkpoint_dir, signature_def_key) + return _get_params_from(gin_builder, input_mapping, output_mapping) + + +def get_params_from_saved_model(saved_model_dir, tag_set, signature_def_key, input_mapping, + output_mapping): + assert signature_def_key is not None + gin_builder = TFInputGraphBuilder.fromSavedModel(saved_model_dir, tag_set, signature_def_key) + return _get_params_from(gin_builder, input_mapping, output_mapping) + + class TFInputGraphBuilder(object): """ Create a builder function so as to be able to compile graph for inference. @@ -65,7 +92,9 @@ def build(self, input_mapping, output_mapping): # Append feeds and input mapping _input_mapping = {} - for input_colname, tnsr_or_sig in input_mapping.items(): + if isinstance(input_mapping, dict): + input_mapping = input_mapping.items() + for input_colname, tnsr_or_sig in input_mapping: if sig_def: tnsr = sig_def.inputs[tnsr_or_sig].name else: @@ -79,7 +108,9 @@ def build(self, input_mapping, output_mapping): # By default the output columns will have the name of their # corresponding `tf.Graph` operation names. # We have to convert them to the user specified output names - for tnsr_or_sig, requested_colname in output_mapping.items(): + if isinstance(output_mapping, dict): + output_mapping = output_mapping.items() + for tnsr_or_sig, requested_colname in output_mapping: if sig_def: tnsr = sig_def.outputs[tnsr_or_sig].name else: @@ -132,15 +163,21 @@ def fromCheckpoint(cls, checkpoint_dir, signature_def_key=None): def import_graph_fn(sess): # Load checkpoint and import the graph - ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) - saver = tf.train.import_meta_graph("{}.meta".format(ckpt_path), clear_devices=True) - saver.restore(sess, ckpt_path) - meta_graph_def = saver.export_meta_graph(clear_devices=True) + with sess.as_default(): + ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) - sig_def = None - if signature_def_key is not None: - sig_def = tf.contrib.saved_model.get_signature_def_by_key( - meta_graph_def, signature_def_key) + # NOTE(phi-dbq): we must manually load meta_graph_def to get the signature_def + meta_graph_def = meta_graph_pb2.MetaGraphDef() + with open("{}.meta".format(ckpt_path), 'rb') as fin: + meta_graph_def.ParseFromString(fin.read()) + + saver = tf.train.import_meta_graph(meta_graph_def, clear_devices=True) + saver.restore(sess, ckpt_path) + + sig_def = None + if signature_def_key is not None: + sig_def = meta_graph_def.signature_def[signature_def_key] + # TODO: check if sig_def is valid return sig_def diff --git a/python/sparkdl/transformers/param.py b/python/sparkdl/transformers/param.py index 827b7234..233de707 100644 --- a/python/sparkdl/transformers/param.py +++ b/python/sparkdl/transformers/param.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - """ Some parts are copied from pyspark.ml.param.shared and some are complementary to pyspark.ml.param. The copy is due to some useful pyspark fns/classes being @@ -29,12 +28,12 @@ from sparkdl.graph.builder import GraphFunction, IsolatedSession import sparkdl.graph.utils as tfx -from sparkdl.transformers.utils import TFInputGraph, TFInputGraphBuilder - +from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder """ Copied from PySpark for backward compatibility. First in Apache Spark version 2.1.1. """ + def keyword_only(func): """ A decorator that forces keyword arguments in the wrapped method @@ -42,12 +41,14 @@ def keyword_only(func): .. note:: Should only be used to wrap a method where first arg is `self` """ + @wraps(func) def wrapper(self, *args, **kwargs): if len(args) > 0: raise TypeError("Method %s forces keyword arguments." % func.__name__) self._input_kwargs = kwargs return func(self, **kwargs) + return wrapper @@ -56,8 +57,8 @@ class HasInputCol(Params): Mixin for param inputCol: input column name. """ - inputCol = Param(Params._dummy(), "inputCol", "input column name.", - typeConverter=TypeConverters.toString) + inputCol = Param( + Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) def setInputCol(self, value): """ @@ -77,8 +78,8 @@ class HasOutputCol(Params): Mixin for param outputCol: output column name. """ - outputCol = Param(Params._dummy(), "outputCol", "output column name.", - typeConverter=TypeConverters.toString) + outputCol = Param( + Params._dummy(), "outputCol", "output column name.", typeConverter=TypeConverters.toString) def __init__(self): super(HasOutputCol, self).__init__() @@ -100,8 +101,9 @@ def getOutputCol(self): """ TensorFlow Specific Parameters """ -class SparkDLTypeConverters(object): + +class SparkDLTypeConverters(object): @staticmethod def toTFGraph(value): if isinstance(value, tf.Graph): @@ -165,17 +167,22 @@ def converter(value): return value else: raise TypeError("%s %s is not in the supported list." % type(value), str(value)) + return converter + # New in sparkdl + class HasOutputMapping(Params): """ - Mixin for param outputMapping: ordered list of ('outputTensorName', 'outputColName') pairs + Mixin for param outputMapping: ordered list of ('outputTensorOpName', 'outputColName') pairs """ - outputMapping = Param(Params._dummy(), "outputMapping", - "Mapping output :class:`tf.Tensor` objects to DataFrame column names", - typeConverter=SparkDLTypeConverters.asTensorToColumnMap) + outputMapping = Param( + Params._dummy(), + "outputMapping", + "Mapping output :class:`tf.Operation` names to DataFrame column names", + typeConverter=SparkDLTypeConverters.asTensorToColumnMap) def setOutputMapping(self, value): return self._set(outputMapping=value) @@ -186,11 +193,13 @@ def getOutputMapping(self): class HasInputMapping(Params): """ - Mixin for param inputMapping: ordered list of ('inputColName', 'inputTensorName') pairs + Mixin for param inputMapping: ordered list of ('inputColName', 'inputTensorOpName') pairs """ - inputMapping = Param(Params._dummy(), "inputMapping", - "Mapping input DataFrame column names to :class:`tf.Tensor` objects", - typeConverter=SparkDLTypeConverters.asColumnToTensorMap) + inputMapping = Param( + Params._dummy(), + "inputMapping", + "Mapping input DataFrame column names to :class:`tf.Operation` names", + typeConverter=SparkDLTypeConverters.asColumnToTensorMap) def setInputMapping(self, value): return self._set(inputMapping=value) @@ -203,9 +212,11 @@ class HasTFInputGraph(Params): """ Mixin for param tfGraph: the :class:`tf.Graph` object that represents a TensorFlow computation. """ - tfInputGraph = Param(Params._dummy(), "tfInputGraph", - "TensorFlow Graph object", - typeConverter=SparkDLTypeConverters.toTFInputGraph) + tfInputGraph = Param( + Params._dummy(), + "tfInputGraph", + "TensorFlow Graph object", + typeConverter=SparkDLTypeConverters.toTFInputGraph) def __init__(self): super(HasTFInputGraph, self).__init__() @@ -222,8 +233,11 @@ class HasTFHParams(Params): """ Mixin for TensorFlow model hyper-parameters """ - tfHParams = Param(Params._dummy(), "hparams", "instance of :class:`tf.contrib.training.HParams`", - typeConverter=SparkDLTypeConverters.toTFHParams) + tfHParams = Param( + Params._dummy(), + "hparams", + "instance of :class:`tf.contrib.training.HParams`", + typeConverter=SparkDLTypeConverters.toTFHParams) def setTFHParams(self, value): return self._set(tfHParam=value) diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 255c264a..21c9524a 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -21,10 +21,9 @@ from pyspark.ml import Transformer import sparkdl.graph.utils as tfx -from sparkdl.transformers.utils import TFInputGraphBuilder -from sparkdl.transformers.param import ( - keyword_only, SparkDLTypeConverters, HasInputMapping, - HasOutputMapping, HasTFInputGraph, HasTFHParams) +from sparkdl.graph.input import TFInputGraphBuilder +from sparkdl.transformers.param import (keyword_only, SparkDLTypeConverters, HasInputMapping, + HasOutputMapping, HasTFInputGraph, HasTFHParams) __all__ = ['TFTransformer'] @@ -72,11 +71,8 @@ def _transform(self, dataset): with tf.Session(graph=graph): analyzed_df = tfs.analyze(dataset) - out_tnsr_op_names = [tfx.as_op_name(tnsr_op_name) - for tnsr_op_name, _ in output_mapping] - tf.import_graph_def(graph_def=gin.graph_def, - name='', - return_elements=out_tnsr_op_names) + out_tnsr_op_names = [tfx.as_op_name(tnsr_op_name) for tnsr_op_name, _ in output_mapping] + tf.import_graph_def(graph_def=gin.graph_def, name='', return_elements=out_tnsr_op_names) feed_dict = dict((tfx.op_name(graph, tnsr_op_name), col_name) for col_name, tnsr_op_name in input_mapping) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 85df23a7..88eeb273 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -14,6 +14,7 @@ # from __future__ import absolute_import, division, print_function +from glob import glob import os import shutil import tempfile @@ -26,8 +27,9 @@ from pyspark.sql.types import Row from sparkdl.graph.builder import IsolatedSession +from sparkdl.graph.input import * import sparkdl.graph.utils as tfx -from sparkdl.transformers.tf_tensor import TFTransformer, TFInputGraphBuilder +from sparkdl.transformers.tf_tensor import TFTransformer from ..tests import SparkDLTestCase @@ -133,15 +135,15 @@ def test_build_from_saved_model(self): # Build the transformer from exported serving model # We are using signaures, thus must provide the keys - trans_with_sig = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromSavedModel( - saved_model_dir, tag_set=serving_tag, signature_def_key=serving_sigdef_key), - inputMapping={ - input_col: 'input_sig' - }, - outputMapping={ - 'output_sig': output_col - }) + tfInputGraph, inputMapping, outputMapping = get_params_from_saved_model( + saved_model_dir, serving_tag, serving_sigdef_key, + input_mapping={ + input_col: 'input_sig'}, + output_mapping={ + 'output_sig': output_col}) + trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, + inputMapping=inputMapping, + outputMapping=outputMapping) # Build the transformer from exported serving model # We are not using signatures, thus must provide tensor/operation names @@ -186,7 +188,31 @@ def test_build_from_checkpoint(self): z = tf.reduce_mean(x * w, axis=1, name='tnsrOut') sess.run(w.initializer) saver = tf.train.Saver(var_list=[w]) - _ = saver.save(sess, ckpt_path_prefix, global_step=2702) + ckpt_path = saver.save(sess, ckpt_path_prefix, global_step=2702) + + # Prepare the signature_def + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs={ + 'input_sig': tf.saved_model.utils.build_tensor_info(x) + }, + outputs={ + 'output_sig': tf.saved_model.utils.build_tensor_info(z) + }) + + # A rather contrived way to add signature def to a meta_graph + serving_sigdef_key = 'prediction_signature' + meta_graph_def = tf.train.export_meta_graph() + + # Find the meta_graph file (there should be only one) + _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) + self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) + ckpt_meta_fpath = _ckpt_meta_fpaths[0] + + # Add signature_def to the meta_graph and serialize it + # This will overwrite the existing meta_graph_def file + meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) + with open(ckpt_meta_fpath, mode='wb') as fout: + fout.write(meta_graph_def.SerializeToString()) # Get the reference data _results = [] @@ -195,7 +221,24 @@ def test_build_from_checkpoint(self): _results.append(sess.run(z, {x: arr})) out_ref = np.hstack(_results) - transformer = TFTransformer( + test_results = [] + def _add_test(transformer, msg, trs=test_results): + final_df = transformer.transform(analyzed_df) + out_tgt = grab_df_arr(final_df, output_col) + trs.append((np.allclose(out_ref, out_tgt), msg)) + + tfInputGraph, inputMapping, outputMapping = get_params_from_checkpoint( + model_ckpt_dir, serving_sigdef_key, + input_mapping={ + input_col: 'input_sig'}, + output_mapping={ + 'output_sig': output_col}) + trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, + inputMapping=inputMapping, + outputMapping=outputMapping) + _add_test(trans_with_sig, 'transformer built with signature_def') + + trans_no_sig = TFTransformer( tfInputGraph=TFInputGraphBuilder.fromCheckpoint(model_ckpt_dir), inputMapping={ input_col: 'tnsrIn' @@ -203,11 +246,13 @@ def test_build_from_checkpoint(self): outputMapping={ 'tnsrOut': output_col }) - final_df = transformer.transform(analyzed_df) - out_tgt = grab_df_arr(final_df, output_col) + _add_test(trans_no_sig, 'transformer built WITHOUT signature_def') + # First delete the resource shutil.rmtree(model_ckpt_dir, ignore_errors=True) - self.assertTrue(np.allclose(out_ref, out_tgt)) + # Then check each test result + for test_result, test_msg in test_results: + self.assertTrue(test_result, msg=test_msg) def test_multi_io(self): From 66d44e99dda76781275a53f639f7e1ba924f0226 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 13 Sep 2017 10:57:56 -0700 Subject: [PATCH 10/80] (WIP) respond to PR comments --- python/sparkdl/graph/input.py | 44 +++++++----------- python/sparkdl/transformers/param.py | 51 ++++++++++----------- python/sparkdl/transformers/tf_tensor.py | 33 +++++++++++-- python/tests/transformers/tf_tensor_test.py | 5 +- 4 files changed, 70 insertions(+), 63 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index c495697d..7590be13 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -29,43 +29,20 @@ class TFInputGraph(object): """ # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. - def __init__(self, graph_def, input_mapping, output_mapping): + def __init__(self, graph_def): # tf.GraphDef self.graph_def = graph_def - if isinstance(input_mapping, dict): - input_mapping = list(input_mapping.items()) - assert isinstance(input_mapping, list), \ - "output mapping must be a list of strings, found type {}".format(type(input_mapping)) - self.input_mapping = sorted(input_mapping) - - if isinstance(output_mapping, dict): - output_mapping = list(output_mapping.items()) - assert isinstance(output_mapping, list), \ - "output mapping must be a list of strings, found type {}".format(type(output_mapping)) - self.output_mapping = sorted(output_mapping) - - -def _get_params_from(gin_builder, input_mapping, output_mapping): - gin = gin_builder.build(input_mapping, output_mapping) - imap = dict(gin.input_mapping) - assert len(imap) == len(gin.input_mapping) - omap = dict(gin.output_mapping) - assert len(omap) == len(gin.output_mapping) - return gin.graph_def, imap, omap - - def get_params_from_checkpoint(checkpoint_dir, signature_def_key, input_mapping, output_mapping): assert signature_def_key is not None gin_builder = TFInputGraphBuilder.fromCheckpoint(checkpoint_dir, signature_def_key) - return _get_params_from(gin_builder, input_mapping, output_mapping) - + return gin_builder.build(input_mapping, output_mapping) def get_params_from_saved_model(saved_model_dir, tag_set, signature_def_key, input_mapping, output_mapping): assert signature_def_key is not None gin_builder = TFInputGraphBuilder.fromSavedModel(saved_model_dir, tag_set, signature_def_key) - return _get_params_from(gin_builder, input_mapping, output_mapping) + return gin_builder.build(input_mapping, output_mapping) class TFInputGraphBuilder(object): @@ -117,12 +94,19 @@ def build(self, input_mapping, output_mapping): tnsr = tnsr_or_sig fetches.append(tfx.get_tensor(graph, tnsr)) tf_output_colname = tfx.op_name(graph, tnsr) + # NOTE(phi-dbq): put the check here as it will be the entry point to construct + # a `TFInputGraph` object. + assert tf_output_colname not in _output_mapping, \ + "operation {} has multiple output tensors and ".format(tf_output_colname) + \ + "at least two of them are used in the output DataFrame. " + \ + "Operation names are used to name columns which leads to conflicts. " + \ + "You can apply `tf.identity` ops to each to avoid name conflicts." _output_mapping[tf_output_colname] = requested_colname output_mapping = _output_mapping gdef = tfx.strip_and_freeze_until(fetches, graph, sess) - return TFInputGraph(gdef, input_mapping, output_mapping) + return TFInputGraph(gdef), input_mapping, output_mapping @classmethod def fromGraph(cls, graph): @@ -167,6 +151,8 @@ def import_graph_fn(sess): ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) # NOTE(phi-dbq): we must manually load meta_graph_def to get the signature_def + # the current `import_graph_def` function seems to ignore + # any signature_def fields in a checkpoint's meta_graph_def. meta_graph_def = meta_graph_pb2.MetaGraphDef() with open("{}.meta".format(ckpt_path), 'rb') as fin: meta_graph_def.ParseFromString(fin.read()) @@ -177,7 +163,9 @@ def import_graph_fn(sess): sig_def = None if signature_def_key is not None: sig_def = meta_graph_def.signature_def[signature_def_key] - # TODO: check if sig_def is valid + assert sig_def, 'singnature_def_key {} provided, '.format(signature_def_key) + \ + 'but failed to find it from the meta_graph_def ' + \ + 'from checkpoint {}'.format(checkpoint_dir) return sig_def diff --git a/python/sparkdl/transformers/param.py b/python/sparkdl/transformers/param.py index 233de707..840a6d02 100644 --- a/python/sparkdl/transformers/param.py +++ b/python/sparkdl/transformers/param.py @@ -29,10 +29,10 @@ from sparkdl.graph.builder import GraphFunction, IsolatedSession import sparkdl.graph.utils as tfx from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder -""" -Copied from PySpark for backward compatibility. First in Apache Spark version 2.1.1. -""" +######################################################## +# Copied from PySpark for backward compatibility. First in Apache Spark version 2.1.1. +######################################################## def keyword_only(func): """ @@ -98,21 +98,15 @@ def getOutputCol(self): return self.getOrDefault(self.outputCol) -""" -TensorFlow Specific Parameters -""" - +######################################################## +# New in sparkdl: TensorFlow Specific Parameters +######################################################## class SparkDLTypeConverters(object): @staticmethod def toTFGraph(value): if isinstance(value, tf.Graph): return value - elif isinstance(value, GraphFunction): - with IsolatedSession() as issn: - issn.importGraphFunction(value, prefix='') - g = issn.graph - return g else: raise TypeError("Could not convert %s to TensorFlow Graph" % type(value)) @@ -120,12 +114,6 @@ def toTFGraph(value): def toTFInputGraph(value): if isinstance(value, TFInputGraph): return value - elif isinstance(value, TFInputGraphBuilder): - return value - elif isinstance(value, tf.Graph): - return TFInputGraphBuilder.fromGraph(value) - elif isinstance(value, tf.GraphDef): - return TFInputGraphBuilder.fromGraphDef(value) else: raise TypeError("Could not convert %s to TFInputGraph" % type(value)) @@ -171,9 +159,6 @@ def converter(value): return converter -# New in sparkdl - - class HasOutputMapping(Params): """ Mixin for param outputMapping: ordered list of ('outputTensorOpName', 'outputColName') pairs @@ -185,7 +170,11 @@ class HasOutputMapping(Params): typeConverter=SparkDLTypeConverters.asTensorToColumnMap) def setOutputMapping(self, value): - return self._set(outputMapping=value) + # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the + # serializable TFInputGraph object once the inputMapping and outputMapping + # parameters are provided. + raise NotImplementedError( + "Please use the Transformer's constructor to assign `outputMapping` field.") def getOutputMapping(self): return self.getOrDefault(self.outputMapping) @@ -202,7 +191,11 @@ class HasInputMapping(Params): typeConverter=SparkDLTypeConverters.asColumnToTensorMap) def setInputMapping(self, value): - return self._set(inputMapping=value) + # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the + # serializable TFInputGraph object once the inputMapping and outputMapping + # parameters are provided. + raise NotImplementedError( + "Please use the Transformer's constructor to assigne `inputMapping` field.") def getInputMapping(self): return self.getOrDefault(self.inputMapping) @@ -210,12 +203,12 @@ def getInputMapping(self): class HasTFInputGraph(Params): """ - Mixin for param tfGraph: the :class:`tf.Graph` object that represents a TensorFlow computation. + Mixin for param tfInputGraph: a serializable object derived from a TensorFlow computation graph. """ tfInputGraph = Param( Params._dummy(), "tfInputGraph", - "TensorFlow Graph object", + "A serializable object derived from a TensorFlow computation graph", typeConverter=SparkDLTypeConverters.toTFInputGraph) def __init__(self): @@ -223,7 +216,11 @@ def __init__(self): self._setDefault(tfInputGraph=None) def setTFInputGraph(self, value): - return self._set(tfInputGraph=value) + # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the + # serializable TFInputGraph object once the inputMapping and outputMapping + # parameters are provided. + raise NotImplementedError( + "Please use the Transformer's constructor to assign `tfInputGraph` field.") def getTFInputGraph(self): return self.getOrDefault(self.tfInputGraph) @@ -236,7 +233,7 @@ class HasTFHParams(Params): tfHParams = Param( Params._dummy(), "hparams", - "instance of :class:`tf.contrib.training.HParams`", + "instance of :class:`tf.contrib.training.HParams`, a key-value map-like object", typeConverter=SparkDLTypeConverters.toTFHParams) def setTFHParams(self, value): diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 21c9524a..2b091979 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -21,7 +21,7 @@ from pyspark.ml import Transformer import sparkdl.graph.utils as tfx -from sparkdl.graph.input import TFInputGraphBuilder +from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder from sparkdl.transformers.param import (keyword_only, SparkDLTypeConverters, HasInputMapping, HasOutputMapping, HasTFInputGraph, HasTFHParams) @@ -57,15 +57,38 @@ def setParams(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tf """ super(TFTransformer, self).__init__() kwargs = self._input_kwargs - _maybe_gin = SparkDLTypeConverters.toTFInputGraph(tfInputGraph) + # The set of parameters either come from some helper functions, + # in which case type(_maybe_gin) is already TFInputGraph. + _maybe_gin = tfInputGraph + if isinstance(_maybe_gin, TFInputGraph): + return self._set(**kwargs) + + # Otherwise, `_maybe_gin` needs to be converted to TFInputGraph + # We put all the conversion logic here rather than in SparkDLTypeConverters if isinstance(_maybe_gin, TFInputGraphBuilder): - kwargs['tfInputGraph'] = _maybe_gin.build(inputMapping, outputMapping) + gin = _maybe_gin + elif isinstance(_maybe_gin, tf.Graph): + gin = TFInputGraphBuilder.fromGraph(_maybe_gin) + elif isinstance(_maybe_gin, tf.GraphDef): + gin = TFInputGraphBuilder.fromGraphDef(_maybe_gin) + else: + raise TypeError("TFTransformer expect tfInputGraph convertible to TFInputGraph, " + \ + "but the given type {} cannot be converted, ".format(type(tfInputGraph)) + \ + "please provide `tf.Graph`, `tf.GraphDef` or use one of the " + \ + "`get_params_from_*` helper functions to build parameters") + + gin, input_mapping, output_mapping = gin.build(inputMapping, outputMapping) + kwargs['tfInputGraph'] = gin + kwargs['inputMapping'] = input_mapping + kwargs['outputMapping'] = output_mapping + + # Further conanonicalization, e.g. converting dict to sorted str pairs happens here return self._set(**kwargs) def _transform(self, dataset): gin = self.getTFInputGraph() - input_mapping = gin.input_mapping - output_mapping = gin.output_mapping + input_mapping = self.getInputMapping() + output_mapping = self.getOutputMapping() graph = tf.Graph() with tf.Session(graph=graph): diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 88eeb273..221c5f43 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -54,7 +54,6 @@ def test_build_from_tf_graph(self): # Build the TensorFlow graph with tf.Session() as sess: - #x = tf.placeholder(tf.float64, shape=[None, vec_size]) x = tfs.block(analyzed_df, 'vec') z = tf.reduce_mean(x, axis=1) graph = sess.graph @@ -68,9 +67,9 @@ def test_build_from_tf_graph(self): # Apply the transform gin_from_graph = TFInputGraphBuilder.fromGraph(graph) - for gin in [gin_from_graph, graph]: + for gin_or_graph in [gin_from_graph, graph]: transfomer = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromGraph(graph), + tfInputGraph=gin_or_graph, inputMapping={ 'vec': x }, From 9b3fe86e759bcf7a1a86595a3fa86ece60ca301e Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 13 Sep 2017 15:56:50 -0700 Subject: [PATCH 11/80] test refactor --- python/sparkdl/transformers/param.py | 5 +- python/tests/transformers/tf_tensor_test.py | 409 +++++++++----------- 2 files changed, 181 insertions(+), 233 deletions(-) diff --git a/python/sparkdl/transformers/param.py b/python/sparkdl/transformers/param.py index 840a6d02..a8c7a891 100644 --- a/python/sparkdl/transformers/param.py +++ b/python/sparkdl/transformers/param.py @@ -31,7 +31,8 @@ from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder ######################################################## -# Copied from PySpark for backward compatibility. First in Apache Spark version 2.1.1. +# Copied from PySpark for backward compatibility. +# They first appeared in Apache Spark version 2.1.1. ######################################################## def keyword_only(func): @@ -99,7 +100,7 @@ def getOutputCol(self): ######################################################## -# New in sparkdl: TensorFlow Specific Parameters +# New in sparkdl ######################################################## class SparkDLTypeConverters(object): diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 221c5f43..d6fe9296 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -14,6 +14,7 @@ # from __future__ import absolute_import, division, print_function +from contextlib import contextmanager from glob import glob import os import shutil @@ -33,77 +34,142 @@ from ..tests import SparkDLTestCase -def grab_df_arr(df, output_col): - """ Stack the numpy array from a DataFrame column """ - return np.array([row.asDict()[output_col] - for row in df.select(output_col).collect()]) class TFTransformerTest(SparkDLTestCase): - def _get_rand_vec_df(self, num_rows, vec_size): - return self.session.createDataFrame( - Row(idx=idx, vec=np.random.randn(vec_size).tolist()) - for idx in range(num_rows)) - - def test_build_from_tf_graph(self): - # Build a simple input DataFrame - vec_size = 17 - num_vecs = 31 - df = self._get_rand_vec_df(num_vecs, vec_size) + def setUp(self): + self.vec_size = 17 + self.num_vecs = 31 + + self.input_col = 'vec' + self.input_op_name = 'tnsrOpIn' + self.output_col = 'outputCol' + self.output_op_name = 'tnsrOpOut' + + self.input_mapping = {} + self.output_mapping = {} + + self.transformers = [] + self.test_case_results = [] + # Build a temporary directory, which might or might not be used by the test + self.model_output_root = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.model_output_root, ignore_errors=True) + + def build_standard_transformers(self, sess, gin_builder_convertible): + def _add_transformer(imap, omap): + trnsfmr = TFTransformer( + tfInputGraph=gin_builder_convertible, inputMapping=imap, outputMapping=omap) + self.transformers.append(trnsfmr) + + _add_transformer(self.input_mapping, self.output_mapping) + + imap = [(col, tfx.get_tensor(sess.graph, op_name)) + for col, op_name in self.input_mapping.items()] + omap = [(tfx.get_tensor(sess.graph, op_name), col) + for op_name, col in self.output_mapping.items()] + _add_transformer(imap, omap) + + @contextmanager + def run_test_in_tf_session(self, replica=1): + """ [THIS IS NOT A TEST]: encapsulate general test workflow """ + + if replica > 1: + for i in range(replica): + colname = '{}_replica{:03d}'.format(self.input_col, i) + tnsr_op_name = '{}_replica{:03d}'.format(self.input_op_name, i) + self.input_mapping[colname] = tnsr_op_name + + colname = '{}_replica{:03d}'.format(self.output_col, i) + tnsr_op_name = '{}_replica{:03d}'.format(self.output_op_name, i) + self.output_mapping[tnsr_op_name] = colname + else: + self.input_mapping = {self.input_col: self.input_op_name} + self.output_mapping = {self.output_op_name: self.output_col} + + # Build local features and DataFrame from it + local_features = [] + for idx in range(self.num_vecs): + _dict = {'idx': idx} + for colname, _ in self.input_mapping.items(): + _dict[colname] = np.random.randn(self.vec_size).tolist() + + local_features.append(Row(**_dict)) + + df = self.session.createDataFrame(local_features) analyzed_df = tfs.analyze(df) # Build the TensorFlow graph - with tf.Session() as sess: - x = tfs.block(analyzed_df, 'vec') - z = tf.reduce_mean(x, axis=1) - graph = sess.graph + graph = tf.Graph() + with tf.Session(graph=graph) as sess: + # Build test graph and transformers from here + yield sess # Get the reference data _results = [] - for row in df.collect(): - arr = np.array(row.vec)[np.newaxis, :] - _results.append(sess.run(z, {x: arr})) + for row in local_features: + fetches = [tfx.get_tensor(graph, tnsr_op_name) + for tnsr_op_name in self.output_mapping.keys()] + feed_dict = {} + for colname, tnsr_op_name in self.input_mapping.items(): + tnsr = tfx.get_tensor(graph, tnsr_op_name) + feed_dict[tnsr] = np.array(row[colname])[np.newaxis, :] + + curr_res = sess.run(fetches, feed_dict=feed_dict) + _results.append(np.ravel(curr_res)) + out_ref = np.hstack(_results) # Apply the transform - gin_from_graph = TFInputGraphBuilder.fromGraph(graph) - for gin_or_graph in [gin_from_graph, graph]: - transfomer = TFTransformer( - tfInputGraph=gin_or_graph, - inputMapping={ - 'vec': x - }, - outputMapping={ - z: 'outCol' - }) - final_df = transfomer.transform(analyzed_df) - out_tgt = grab_df_arr(final_df, 'outCol') - self.assertTrue(np.allclose(out_ref, out_tgt)) + for transfomer in self.transformers: + out_df = transfomer.transform(analyzed_df) + out_colnames = [] + for old_colname, new_colname in self.output_mapping.items(): + out_colnames.append(new_colname) + if old_colname != new_colname: + out_df = out_df.withColumnRenamed(old_colname, new_colname) + _results = [] + for row in out_df.select(out_colnames).collect(): + curr_res = [row[colname] for colname in out_colnames] + _results.append(np.ravel(curr_res)) + out_tgt = np.hstack(_results) + + self.assertTrue(np.allclose(out_ref, out_tgt), + msg=repr(transfomer)) + + + def test_build_from_tf_graph(self): + with self.run_test_in_tf_session() as sess: + # Begin building graph + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) + # End building graph + + # Begin building transformers + self.build_standard_transformers(sess, sess.graph) + self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) + gdef = sess.graph.as_graph_def() + self.build_standard_transformers(sess, gdef) + self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraphDef(gdef)) + # End building transformers - def test_build_from_saved_model(self): - # Setup dataset - vec_size = 17 - num_vecs = 31 - df = self._get_rand_vec_df(num_vecs, vec_size) - analyzed_df = tfs.analyze(df) - input_col = 'vec' - output_col = 'outputCol' + def test_build_from_saved_model(self): # Setup saved model export directory - saved_model_root = tempfile.mkdtemp() + saved_model_root = self.model_output_root saved_model_dir = os.path.join(saved_model_root, 'saved_model') serving_tag = "serving_tag" serving_sigdef_key = 'prediction_signature' - builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - with tf.Session(graph=tf.Graph()) as sess: + + with self.run_test_in_tf_session() as sess: # Model definition: begin - x = tf.placeholder(tf.float64, shape=[None, vec_size], name='tnsrIn') - #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - w = tf.Variable(tf.random_normal([vec_size], dtype=tf.float64), + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name='tnsrOut') + z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) # Model definition ends sess.run(w.initializer) @@ -120,74 +186,48 @@ def test_build_from_saved_model(self): builder.add_meta_graph_and_variables(sess, [serving_tag], signature_def_map={ - serving_sigdef_key: serving_sigdef - }) - # Get the reference data - _results = [] - for row in df.collect(): - arr = np.array(row.vec)[np.newaxis, :] - _results.append(sess.run(z, {x: arr})) - out_ref = np.hstack(_results) - - # Save the model - builder.save() - - # Build the transformer from exported serving model - # We are using signaures, thus must provide the keys - tfInputGraph, inputMapping, outputMapping = get_params_from_saved_model( - saved_model_dir, serving_tag, serving_sigdef_key, - input_mapping={ - input_col: 'input_sig'}, - output_mapping={ - 'output_sig': output_col}) - trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, - inputMapping=inputMapping, - outputMapping=outputMapping) - - # Build the transformer from exported serving model - # We are not using signatures, thus must provide tensor/operation names - trans_no_sig = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromSavedModel( - saved_model_dir, tag_set=serving_tag, signature_def_key=None), - inputMapping={ - input_col: 'tnsrIn' - }, - outputMapping={ - 'tnsrOut': output_col - }) - - df_trans_with_sig = trans_with_sig.transform(analyzed_df) - df_trans_no_sig = trans_no_sig.transform(analyzed_df) - out_with_sig_tgt = grab_df_arr(df_trans_with_sig, output_col) - out_no_sig_tgt = grab_df_arr(df_trans_no_sig, output_col) - # Cleanup the resources - shutil.rmtree(saved_model_root, ignore_errors=True) - self.assertTrue(np.allclose(out_ref, out_with_sig_tgt)) - self.assertTrue(np.allclose(out_ref, out_no_sig_tgt)) + serving_sigdef_key: serving_sigdef}) + builder.save() + + # Build the transformer from exported serving model + # We are using signaures, thus must provide the keys + tfInputGraph, inputMapping, outputMapping = get_params_from_saved_model( + saved_model_dir, serving_tag, serving_sigdef_key, + input_mapping={ + self.input_col: 'input_sig'}, + output_mapping={ + 'output_sig': self.output_col}) + trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, + inputMapping=inputMapping, + outputMapping=outputMapping) + self.transformers.append(trans_with_sig) + + # Build the transformer from exported serving model + # We are not using signatures, thus must provide tensor/operation names + gin_builder = TFInputGraphBuilder.fromSavedModel( + saved_model_dir, tag_set=serving_tag, signature_def_key=None) + self.build_standard_transformers(sess, gin_builder) def test_build_from_checkpoint(self): - vec_size = 17 - num_vecs = 31 - df = self._get_rand_vec_df(num_vecs, vec_size) - analyzed_df = tfs.analyze(df) - input_col = 'vec' - output_col = 'outputCol' - + """ + Test constructing a Transformer from a TensorFlow training checkpoint + """ # Build the TensorFlow graph - model_ckpt_dir = tempfile.mkdtemp() + model_ckpt_dir = self.model_output_root ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') + serving_sigdef_key = 'prediction_signature' # Warning: please use a new graph for each test cases # or the tests could affect one another - with tf.Session(graph=tf.Graph()) as sess: - x = tf.placeholder(tf.float64, shape=[None, vec_size], name='tnsrIn') + with self.run_test_in_tf_session() as sess: + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - w = tf.Variable(tf.random_normal([vec_size], dtype=tf.float64), + w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name='tnsrOut') + z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) sess.run(w.initializer) saver = tf.train.Saver(var_list=[w]) - ckpt_path = saver.save(sess, ckpt_path_prefix, global_step=2702) + _ = saver.save(sess, ckpt_path_prefix, global_step=2702) # Prepare the signature_def serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( @@ -199,7 +239,6 @@ def test_build_from_checkpoint(self): }) # A rather contrived way to add signature def to a meta_graph - serving_sigdef_key = 'prediction_signature' meta_graph_def = tf.train.export_meta_graph() # Find the meta_graph file (there should be only one) @@ -213,109 +252,42 @@ def test_build_from_checkpoint(self): with open(ckpt_meta_fpath, mode='wb') as fout: fout.write(meta_graph_def.SerializeToString()) - # Get the reference data - _results = [] - for row in df.collect(): - arr = np.array(row.vec)[np.newaxis, :] - _results.append(sess.run(z, {x: arr})) - out_ref = np.hstack(_results) + tfInputGraph, inputMapping, outputMapping = get_params_from_checkpoint( + model_ckpt_dir, serving_sigdef_key, + input_mapping={ + self.input_col: 'input_sig'}, + output_mapping={ + 'output_sig': self.output_col}) + trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, + inputMapping=inputMapping, + outputMapping=outputMapping) + self.transformers.append(trans_with_sig) - test_results = [] - def _add_test(transformer, msg, trs=test_results): - final_df = transformer.transform(analyzed_df) - out_tgt = grab_df_arr(final_df, output_col) - trs.append((np.allclose(out_ref, out_tgt), msg)) - - tfInputGraph, inputMapping, outputMapping = get_params_from_checkpoint( - model_ckpt_dir, serving_sigdef_key, - input_mapping={ - input_col: 'input_sig'}, - output_mapping={ - 'output_sig': output_col}) - trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, - inputMapping=inputMapping, - outputMapping=outputMapping) - _add_test(trans_with_sig, 'transformer built with signature_def') - - trans_no_sig = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromCheckpoint(model_ckpt_dir), - inputMapping={ - input_col: 'tnsrIn' - }, - outputMapping={ - 'tnsrOut': output_col - }) - _add_test(trans_no_sig, 'transformer built WITHOUT signature_def') - - # First delete the resource - shutil.rmtree(model_ckpt_dir, ignore_errors=True) - # Then check each test result - for test_result, test_msg in test_results: - self.assertTrue(test_result, msg=test_msg) + gin_builder = TFInputGraphBuilder.fromCheckpoint(model_ckpt_dir) + self.build_standard_transformers(sess, gin_builder) def test_multi_io(self): - # Build a simple input DataFrame - vec_size = 17 - num_vecs = 31 - _df = self._get_rand_vec_df(num_vecs, vec_size) - df_x = _df.withColumnRenamed('vec', 'vec_x') - _df = self._get_rand_vec_df(num_vecs, vec_size) - df_y = _df.withColumnRenamed('vec', 'vec_y') - df = df_x.join(df_y, on='idx', how='inner') - analyzed_df = tfs.analyze(df) - # Build the TensorFlow graph - with tf.Session() as sess: - x = tfs.block(analyzed_df, 'vec_x') - y = tfs.block(analyzed_df, 'vec_y') - p = tf.reduce_mean(x + y, axis=1) - q = tf.reduce_mean(x - y, axis=1) - graph = sess.graph + with self.run_test_in_tf_session(replica=2) as sess: + xs = [] + for tnsr_op_name in self.input_mapping.values(): + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) + xs.append(x) - # Get the reference data - p_out_ref = [] - q_out_ref = [] - for row in df.collect(): - arr_x = np.array(row['vec_x'])[np.newaxis, :] - arr_y = np.array(row['vec_y'])[np.newaxis, :] - p_val, q_val = sess.run([p, q], {x: arr_x, y: arr_y}) - p_out_ref.append(p_val) - q_out_ref.append(q_val) - p_out_ref = np.hstack(p_out_ref) - q_out_ref = np.hstack(q_out_ref) + zs = [] + for i, tnsr_op_name in enumerate(self.output_mapping.keys()): + z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) + zs.append(z) - # Apply the transform - transfomer = TFTransformer( - tfInputGraph=TFInputGraphBuilder.fromGraph(graph), - inputMapping={ - 'vec_x': x, - 'vec_y': y - }, - outputMapping={ - p: 'outcol_p', - q: 'outcol_q' - }) - final_df = transfomer.transform(analyzed_df) - p_out_tgt = grab_df_arr(final_df, 'outcol_p') - q_out_tgt = grab_df_arr(final_df, 'outcol_q') - - self.assertTrue(np.allclose(p_out_ref, p_out_tgt)) - self.assertTrue(np.allclose(q_out_ref, q_out_tgt)) + self.build_standard_transformers(sess, sess.graph) + self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) def test_mixed_keras_graph(self): - - vec_size = 17 - num_vecs = 137 - df = self._get_rand_vec_df(num_vecs, vec_size) - analyzed_df = tfs.analyze(df) - - input_col = 'vec' - output_col = 'outCol' - # Build the graph: the output should have the same leading/batch dimension with IsolatedSession(using_keras=True) as issn: - tnsr_in = tfs.block(analyzed_df, input_col) + tnsr_in = tf.placeholder( + tf.double, shape=[None, self.vec_size], name=self.input_op_name) inp = tf.expand_dims(tnsr_in, axis=2) # Keras layers does not take tf.double inp = tf.cast(inp, tf.float32) @@ -325,7 +297,7 @@ def test_mixed_keras_graph(self): dense = Dense(1)(flat) # We must keep the leading dimension of the output redsum = tf.reduce_sum(dense, axis=1) - tnsr_out = tf.cast(redsum, tf.double, name='TnsrOut') + tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) # Initialize the variables init_op = tf.global_variables_initializer() @@ -333,35 +305,10 @@ def test_mixed_keras_graph(self): # We could train the model ... but skip it here gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) - with IsolatedSession() as issn: - # Import the graph function object - feeds, fetches = issn.importGraphFunction(gfn, prefix='') - - # Rename the input column name to the feed op's name - orig_in_name = tfx.op_name(issn.graph, feeds[0]) - input_df = analyzed_df.withColumnRenamed(input_col, orig_in_name) - - # Do the actual computation - output_df = tfs.map_blocks(fetches, input_df) - - # Rename the output column (by default, the name of the fetch op's name) - orig_out_name = tfx.op_name(issn.graph, fetches[0]) - final_df = output_df.withColumnRenamed(orig_out_name, output_col) - - arr_ref = grab_df_arr(final_df, output_col) - - # Using the Transformer - gin_from_gdef = TFInputGraphBuilder.fromGraphDef(gfn.graph_def) - for gin in [gin_from_gdef, gfn.graph_def]: - transformer = TFTransformer( - tfInputGraph=gin, - inputMapping={ - input_col: gfn.input_names[0] - }, - outputMapping={ - gfn.output_names[0]: output_col - }) + with self.run_test_in_tf_session() as sess: + tf.import_graph_def(gfn.graph_def, name='') - transformed_df = transformer.transform(analyzed_df) - arr_tgt = grab_df_arr(transformed_df, output_col) - self.assertTrue(np.allclose(arr_ref, arr_tgt)) + self.build_standard_transformers(sess, sess.graph) + self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) + self.build_standard_transformers(sess, gfn.graph_def) + self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraphDef(gfn.graph_def)) From dbd9aaad4860d4185e67088cb7b34be9d66042fd Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 15 Sep 2017 20:05:39 -0700 Subject: [PATCH 12/80] (wip) consolidating params --- python/sparkdl/param/converters.py | 95 ++++++++++ python/sparkdl/param/shared_params.py | 158 +++++++++++------ python/sparkdl/transformers/param.py | 244 -------------------------- 3 files changed, 197 insertions(+), 300 deletions(-) create mode 100644 python/sparkdl/param/converters.py delete mode 100644 python/sparkdl/transformers/param.py diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py new file mode 100644 index 00000000..23fdd9dd --- /dev/null +++ b/python/sparkdl/param/converters.py @@ -0,0 +1,95 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from functools import wraps +import six + +import keras +import tensorflow as tf + +from sparkdl.graph.builder import GraphFunction, IsolatedSession +import sparkdl.graph.utils as tfx +from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder +import sparkdl.utils.keras_model as kmutil + +class SparkDLTypeConverters(object): + @staticmethod + def toTFGraph(value): + if isinstance(value, tf.Graph): + return value + else: + raise TypeError("Could not convert %s to TensorFlow Graph" % type(value)) + + @staticmethod + def toTFInputGraph(value): + if isinstance(value, TFInputGraph): + return value + else: + raise TypeError("Could not convert %s to TFInputGraph" % type(value)) + + @staticmethod + def asColumnToTensorMap(value): + if isinstance(value, dict): + strs_pair_seq = [(k, tfx.as_op_name(v)) for k, v in value.items()] + return sorted(strs_pair_seq) + raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) + + @staticmethod + def asTensorToColumnMap(value): + if isinstance(value, dict): + strs_pair_seq = [(tfx.as_op_name(k), v) for k, v in value.items()] + return sorted(strs_pair_seq) + raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) + + @staticmethod + def toTFHParams(value): + if isinstance(value, tf.contrib.training.HParams): + return value + else: + raise TypeError("Could not convert %s to TensorFlow HParams" % type(value)) + + @staticmethod + def toStringOrTFTensor(value): + if isinstance(value, tf.Tensor): + return value + else: + try: + return TypeConverters.toString(value) + except TypeError: + raise TypeError("Could not convert %s to tensorflow.Tensor or str" % type(value)) + + @staticmethod + def supportedNameConverter(supportedList): + def converter(value): + if value in supportedList: + return value + else: + raise TypeError("%s %s is not in the supported list." % type(value), str(value)) + + return converter + + @staticmethod + def toKerasLoss(value): + if kmutil.is_valid_loss_function(value): + return value + raise ValueError( + "Named loss not supported in Keras: {} type({})".format(value, type(value))) + + @staticmethod + def toKerasOptimizer(value): + if kmutil.is_valid_optimizer(value): + return value + raise TypeError( + "Named optimizer not supported in Keras: {} type({})".format(value, type(value))) diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index e169e891..8e1b9741 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -20,14 +20,21 @@ """ from functools import wraps +import six +import keras import tensorflow as tf -from pyspark.ml.param import Param, Params, TypeConverters - +from sparkdl.graph.builder import GraphFunction, IsolatedSession +import sparkdl.graph.utils as tfx +from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder import sparkdl.utils.keras_model as kmutil -# From pyspark + +######################################################## +# Copied from PySpark for backward compatibility. +# They first appeared in Apache Spark version 2.1.1. +######################################################## def keyword_only(func): """ @@ -36,12 +43,14 @@ def keyword_only(func): .. note:: Should only be used to wrap a method where first arg is `self` """ + @wraps(func) def wrapper(self, *args, **kwargs): if len(args) > 0: raise TypeError("Method %s forces keyword arguments." % func.__name__) self._input_kwargs = kwargs return func(self, **kwargs) + return wrapper @@ -50,10 +59,8 @@ class HasInputCol(Params): Mixin for param inputCol: input column name. """ - inputCol = Param(Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) - - def __init__(self): - super(HasInputCol, self).__init__() + inputCol = Param( + Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) def setInputCol(self, value): """ @@ -73,8 +80,8 @@ class HasOutputCol(Params): Mixin for param outputCol: output column name. """ - outputCol = Param(Params._dummy(), - "outputCol", "output column name.", typeConverter=TypeConverters.toString) + outputCol = Param( + Params._dummy(), "outputCol", "output column name.", typeConverter=TypeConverters.toString) def __init__(self): super(HasOutputCol, self).__init__() @@ -92,54 +99,9 @@ def getOutputCol(self): """ return self.getOrDefault(self.outputCol) -############################################ +######################################################## # New in sparkdl -############################################ - -class SparkDLTypeConverters(object): - - @staticmethod - def toStringOrTFTensor(value): - if isinstance(value, tf.Tensor): - return value - else: - try: - return TypeConverters.toString(value) - except TypeError: - raise TypeError("Could not convert %s to tensorflow.Tensor or str" % type(value)) - - @staticmethod - def toTFGraph(value): - # TODO: we may want to support tf.GraphDef in the future instead of tf.Graph since user - # is less likely to mess up using GraphDef vs Graph (e.g. constants vs variables). - if isinstance(value, tf.Graph): - return value - else: - raise TypeError("Could not convert %s to tensorflow.Graph type" % type(value)) - - @staticmethod - def supportedNameConverter(supportedList): - def converter(value): - if value in supportedList: - return value - else: - raise TypeError("%s %s is not in the supported list." % type(value), str(value)) - - return converter - - @staticmethod - def toKerasLoss(value): - if kmutil.is_valid_loss_function(value): - return value - raise ValueError( - "Named loss not supported in Keras: {} type({})".format(value, type(value))) - - @staticmethod - def toKerasOptimizer(value): - if kmutil.is_valid_optimizer(value): - return value - raise TypeError( - "Named optimizer not supported in Keras: {} type({})".format(value, type(value))) +######################################################## class HasOutputNodeName(Params): @@ -233,3 +195,87 @@ def seKerasLoss(self, value): def getKerasLoss(self): return self.getOrDefault(self.kerasLoss) + + +class HasOutputMapping(Params): + """ + Mixin for param outputMapping: ordered list of ('outputTensorOpName', 'outputColName') pairs + """ + outputMapping = Param( + Params._dummy(), + "outputMapping", + "Mapping output :class:`tf.Operation` names to DataFrame column names", + typeConverter=SparkDLTypeConverters.asTensorToColumnMap) + + def setOutputMapping(self, value): + # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the + # serializable TFInputGraph object once the inputMapping and outputMapping + # parameters are provided. + raise NotImplementedError( + "Please use the Transformer's constructor to assign `outputMapping` field.") + + def getOutputMapping(self): + return self.getOrDefault(self.outputMapping) + + +class HasInputMapping(Params): + """ + Mixin for param inputMapping: ordered list of ('inputColName', 'inputTensorOpName') pairs + """ + inputMapping = Param( + Params._dummy(), + "inputMapping", + "Mapping input DataFrame column names to :class:`tf.Operation` names", + typeConverter=SparkDLTypeConverters.asColumnToTensorMap) + + def setInputMapping(self, value): + # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the + # serializable TFInputGraph object once the inputMapping and outputMapping + # parameters are provided. + raise NotImplementedError( + "Please use the Transformer's constructor to assigne `inputMapping` field.") + + def getInputMapping(self): + return self.getOrDefault(self.inputMapping) + + +class HasTFInputGraph(Params): + """ + Mixin for param tfInputGraph: a serializable object derived from a TensorFlow computation graph. + """ + tfInputGraph = Param( + Params._dummy(), + "tfInputGraph", + "A serializable object derived from a TensorFlow computation graph", + typeConverter=SparkDLTypeConverters.toTFInputGraph) + + def __init__(self): + super(HasTFInputGraph, self).__init__() + self._setDefault(tfInputGraph=None) + + def setTFInputGraph(self, value): + # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the + # serializable TFInputGraph object once the inputMapping and outputMapping + # parameters are provided. + raise NotImplementedError( + "Please use the Transformer's constructor to assign `tfInputGraph` field.") + + def getTFInputGraph(self): + return self.getOrDefault(self.tfInputGraph) + + +class HasTFHParams(Params): + """ + Mixin for TensorFlow model hyper-parameters + """ + tfHParams = Param( + Params._dummy(), + "hparams", + "instance of :class:`tf.contrib.training.HParams`, a key-value map-like object", + typeConverter=SparkDLTypeConverters.toTFHParams) + + def setTFHParams(self, value): + return self._set(tfHParam=value) + + def getTFHParams(self): + return self.getOrDefault(self.tfHParams) diff --git a/python/sparkdl/transformers/param.py b/python/sparkdl/transformers/param.py deleted file mode 100644 index a8c7a891..00000000 --- a/python/sparkdl/transformers/param.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright 2017 Databricks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -Some parts are copied from pyspark.ml.param.shared and some are complementary -to pyspark.ml.param. The copy is due to some useful pyspark fns/classes being -private APIs. -""" - -from functools import wraps -import six - -import keras -import tensorflow as tf - -from pyspark.ml.param import Param, Params, TypeConverters - -from sparkdl.graph.builder import GraphFunction, IsolatedSession -import sparkdl.graph.utils as tfx -from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder - -######################################################## -# Copied from PySpark for backward compatibility. -# They first appeared in Apache Spark version 2.1.1. -######################################################## - -def keyword_only(func): - """ - A decorator that forces keyword arguments in the wrapped method - and saves actual input keyword arguments in `_input_kwargs`. - - .. note:: Should only be used to wrap a method where first arg is `self` - """ - - @wraps(func) - def wrapper(self, *args, **kwargs): - if len(args) > 0: - raise TypeError("Method %s forces keyword arguments." % func.__name__) - self._input_kwargs = kwargs - return func(self, **kwargs) - - return wrapper - - -class HasInputCol(Params): - """ - Mixin for param inputCol: input column name. - """ - - inputCol = Param( - Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) - - def setInputCol(self, value): - """ - Sets the value of :py:attr:`inputCol`. - """ - return self._set(inputCol=value) - - def getInputCol(self): - """ - Gets the value of inputCol or its default value. - """ - return self.getOrDefault(self.inputCol) - - -class HasOutputCol(Params): - """ - Mixin for param outputCol: output column name. - """ - - outputCol = Param( - Params._dummy(), "outputCol", "output column name.", typeConverter=TypeConverters.toString) - - def __init__(self): - super(HasOutputCol, self).__init__() - self._setDefault(outputCol=self.uid + '__output') - - def setOutputCol(self, value): - """ - Sets the value of :py:attr:`outputCol`. - """ - return self._set(outputCol=value) - - def getOutputCol(self): - """ - Gets the value of outputCol or its default value. - """ - return self.getOrDefault(self.outputCol) - - -######################################################## -# New in sparkdl -######################################################## - -class SparkDLTypeConverters(object): - @staticmethod - def toTFGraph(value): - if isinstance(value, tf.Graph): - return value - else: - raise TypeError("Could not convert %s to TensorFlow Graph" % type(value)) - - @staticmethod - def toTFInputGraph(value): - if isinstance(value, TFInputGraph): - return value - else: - raise TypeError("Could not convert %s to TFInputGraph" % type(value)) - - @staticmethod - def asColumnToTensorMap(value): - if isinstance(value, dict): - strs_pair_seq = [(k, tfx.as_op_name(v)) for k, v in value.items()] - return sorted(strs_pair_seq) - raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) - - @staticmethod - def asTensorToColumnMap(value): - if isinstance(value, dict): - strs_pair_seq = [(tfx.as_op_name(k), v) for k, v in value.items()] - return sorted(strs_pair_seq) - raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) - - @staticmethod - def toTFHParams(value): - if isinstance(value, tf.contrib.training.HParams): - return value - else: - raise TypeError("Could not convert %s to TensorFlow HParams" % type(value)) - - @staticmethod - def toStringOrTFTensor(value): - if isinstance(value, tf.Tensor): - return value - else: - try: - return TypeConverters.toString(value) - except TypeError: - raise TypeError("Could not convert %s to tensorflow.Tensor or str" % type(value)) - - @staticmethod - def supportedNameConverter(supportedList): - def converter(value): - if value in supportedList: - return value - else: - raise TypeError("%s %s is not in the supported list." % type(value), str(value)) - - return converter - - -class HasOutputMapping(Params): - """ - Mixin for param outputMapping: ordered list of ('outputTensorOpName', 'outputColName') pairs - """ - outputMapping = Param( - Params._dummy(), - "outputMapping", - "Mapping output :class:`tf.Operation` names to DataFrame column names", - typeConverter=SparkDLTypeConverters.asTensorToColumnMap) - - def setOutputMapping(self, value): - # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the - # serializable TFInputGraph object once the inputMapping and outputMapping - # parameters are provided. - raise NotImplementedError( - "Please use the Transformer's constructor to assign `outputMapping` field.") - - def getOutputMapping(self): - return self.getOrDefault(self.outputMapping) - - -class HasInputMapping(Params): - """ - Mixin for param inputMapping: ordered list of ('inputColName', 'inputTensorOpName') pairs - """ - inputMapping = Param( - Params._dummy(), - "inputMapping", - "Mapping input DataFrame column names to :class:`tf.Operation` names", - typeConverter=SparkDLTypeConverters.asColumnToTensorMap) - - def setInputMapping(self, value): - # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the - # serializable TFInputGraph object once the inputMapping and outputMapping - # parameters are provided. - raise NotImplementedError( - "Please use the Transformer's constructor to assigne `inputMapping` field.") - - def getInputMapping(self): - return self.getOrDefault(self.inputMapping) - - -class HasTFInputGraph(Params): - """ - Mixin for param tfInputGraph: a serializable object derived from a TensorFlow computation graph. - """ - tfInputGraph = Param( - Params._dummy(), - "tfInputGraph", - "A serializable object derived from a TensorFlow computation graph", - typeConverter=SparkDLTypeConverters.toTFInputGraph) - - def __init__(self): - super(HasTFInputGraph, self).__init__() - self._setDefault(tfInputGraph=None) - - def setTFInputGraph(self, value): - # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the - # serializable TFInputGraph object once the inputMapping and outputMapping - # parameters are provided. - raise NotImplementedError( - "Please use the Transformer's constructor to assign `tfInputGraph` field.") - - def getTFInputGraph(self): - return self.getOrDefault(self.tfInputGraph) - - -class HasTFHParams(Params): - """ - Mixin for TensorFlow model hyper-parameters - """ - tfHParams = Param( - Params._dummy(), - "hparams", - "instance of :class:`tf.contrib.training.HParams`, a key-value map-like object", - typeConverter=SparkDLTypeConverters.toTFHParams) - - def setTFHParams(self, value): - return self._set(tfHParam=value) - - def getTFHParams(self): - return self.getOrDefault(self.tfHParams) From 45722059bac739dc0dba2a2346aca27b842e5c49 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 16 Sep 2017 12:32:33 -0700 Subject: [PATCH 13/80] rebase upstream --- python/sparkdl/param/__init__.py | 3 ++- python/sparkdl/param/converters.py | 2 ++ python/sparkdl/param/shared_params.py | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/python/sparkdl/param/__init__.py b/python/sparkdl/param/__init__.py index 98a8f7dd..1080b29f 100644 --- a/python/sparkdl/param/__init__.py +++ b/python/sparkdl/param/__init__.py @@ -15,6 +15,7 @@ from sparkdl.param.shared_params import ( keyword_only, HasInputCol, HasOutputCol, HasLabelCol, HasKerasModel, - HasKerasLoss, HasKerasOptimizer, HasOutputNodeName, SparkDLTypeConverters) + HasKerasLoss, HasKerasOptimizer, HasOutputNodeName) +from sparkdl.param.converters import SparkDLTypeConverters from sparkdl.param.image_params import ( CanLoadImage, HasInputImageNodeName, HasOutputMode, OUTPUT_MODES) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index 23fdd9dd..d9b8e512 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -19,6 +19,8 @@ import keras import tensorflow as tf +from pyspark.ml.param import TypeConverters + from sparkdl.graph.builder import GraphFunction, IsolatedSession import sparkdl.graph.utils as tfx from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index 8e1b9741..64d4e6f1 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -25,9 +25,12 @@ import keras import tensorflow as tf +from pyspark.ml.param import Param, Params, TypeConverters + from sparkdl.graph.builder import GraphFunction, IsolatedSession import sparkdl.graph.utils as tfx from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder +from sparkdl.param.converters import SparkDLTypeConverters import sparkdl.utils.keras_model as kmutil From 1cc7591daafb15e8464f05edd3d3ceb4b02343a4 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 16 Sep 2017 13:00:40 -0700 Subject: [PATCH 14/80] import params fix --- python/sparkdl/param/__init__.py | 7 +++++-- python/sparkdl/transformers/tf_tensor.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/python/sparkdl/param/__init__.py b/python/sparkdl/param/__init__.py index 1080b29f..ca1a9121 100644 --- a/python/sparkdl/param/__init__.py +++ b/python/sparkdl/param/__init__.py @@ -14,8 +14,11 @@ # from sparkdl.param.shared_params import ( - keyword_only, HasInputCol, HasOutputCol, HasLabelCol, HasKerasModel, - HasKerasLoss, HasKerasOptimizer, HasOutputNodeName) + keyword_only, HasInputCol, HasOutputCol, HasLabelCol, + # TFTransformer Params + HasInputMapping, HasOutputMapping, HasTFInputGraph, HasTFHParams, + # Keras Estimator Params + HasKerasModel, HasKerasLoss, HasKerasOptimizer, HasOutputNodeName) from sparkdl.param.converters import SparkDLTypeConverters from sparkdl.param.image_params import ( CanLoadImage, HasInputImageNodeName, HasOutputMode, OUTPUT_MODES) diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 2b091979..027c6043 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -22,8 +22,8 @@ import sparkdl.graph.utils as tfx from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder -from sparkdl.transformers.param import (keyword_only, SparkDLTypeConverters, HasInputMapping, - HasOutputMapping, HasTFInputGraph, HasTFHParams) +from sparkdl.param import (keyword_only, SparkDLTypeConverters, HasInputMapping, + HasOutputMapping, HasTFInputGraph, HasTFHParams) __all__ = ['TFTransformer'] From 2fc6787bcd5246f93b589d96c9fcaa604f98d630 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 16 Sep 2017 16:10:22 -0700 Subject: [PATCH 15/80] (wip) TFInputGraph impl --- python/sparkdl/graph/input.py | 231 +++++++++++++++++++--------------- 1 file changed, 132 insertions(+), 99 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 7590be13..97e99a33 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -18,7 +18,7 @@ import sparkdl.graph.utils as tfx -__all__ = ["TFInputGraphBuilder", "get_params_from_checkpoint", "get_params_from_saved_model"] +__all__ = ["TFInputGraph"] class TFInputGraph(object): @@ -27,105 +27,36 @@ class TFInputGraph(object): [WARNING] This class should not be called by any user code. """ + def __init__(self): + raise NotImplementedError( + "Please do NOT construct TFInputGraph directly. Instead, use one of the helper functions") - # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. - def __init__(self, graph_def): - # tf.GraphDef - self.graph_def = graph_def - -def get_params_from_checkpoint(checkpoint_dir, signature_def_key, input_mapping, output_mapping): - assert signature_def_key is not None - gin_builder = TFInputGraphBuilder.fromCheckpoint(checkpoint_dir, signature_def_key) - return gin_builder.build(input_mapping, output_mapping) - -def get_params_from_saved_model(saved_model_dir, tag_set, signature_def_key, input_mapping, - output_mapping): - assert signature_def_key is not None - gin_builder = TFInputGraphBuilder.fromSavedModel(saved_model_dir, tag_set, signature_def_key) - return gin_builder.build(input_mapping, output_mapping) - - -class TFInputGraphBuilder(object): - """ - Create a builder function so as to be able to compile graph for inference. - The actual compilation will be done at the time when the - inputs (feeds) and outputs (fetches) are provided. - :param graph_import_fn: `tf.Session` -> `tf.signature_def`, load a graph to the provided session. - If the meta_graph contains a `signature_def`, return it. - """ - - def __init__(self, graph_import_fn): - # Return signature_def if the underlying graph contains one - self.graph_import_fn = graph_import_fn - - def build(self, input_mapping, output_mapping): - """ - Create a serializable TensorFlow graph representation - :param input_mapping: dict, from input DataFrame column name to internal graph name. - """ - graph = tf.Graph() - with tf.Session(graph=graph) as sess: - sig_def = self.graph_import_fn(sess) - - # Append feeds and input mapping - _input_mapping = {} - if isinstance(input_mapping, dict): - input_mapping = input_mapping.items() - for input_colname, tnsr_or_sig in input_mapping: - if sig_def: - tnsr = sig_def.inputs[tnsr_or_sig].name - else: - tnsr = tnsr_or_sig - _input_mapping[input_colname] = tfx.op_name(graph, tnsr) - input_mapping = _input_mapping - - # Append fetches and output mapping - fetches = [] - _output_mapping = {} - # By default the output columns will have the name of their - # corresponding `tf.Graph` operation names. - # We have to convert them to the user specified output names - if isinstance(output_mapping, dict): - output_mapping = output_mapping.items() - for tnsr_or_sig, requested_colname in output_mapping: - if sig_def: - tnsr = sig_def.outputs[tnsr_or_sig].name - else: - tnsr = tnsr_or_sig - fetches.append(tfx.get_tensor(graph, tnsr)) - tf_output_colname = tfx.op_name(graph, tnsr) - # NOTE(phi-dbq): put the check here as it will be the entry point to construct - # a `TFInputGraph` object. - assert tf_output_colname not in _output_mapping, \ - "operation {} has multiple output tensors and ".format(tf_output_colname) + \ - "at least two of them are used in the output DataFrame. " + \ - "Operation names are used to name columns which leads to conflicts. " + \ - "You can apply `tf.identity` ops to each to avoid name conflicts." - _output_mapping[tf_output_colname] = requested_colname - output_mapping = _output_mapping - - gdef = tfx.strip_and_freeze_until(fetches, graph, sess) - - return TFInputGraph(gdef), input_mapping, output_mapping + @classmethod + def _new_obj_internal(cls): + # pylint: disable=attribute-defined-outside-init + obj = object.__new__(cls) + # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. + obj.graph_def = None + obj.input_tensor_name_from_signature = None + obj.output_tensor_name_from_signature = None + return obj @classmethod - def fromGraph(cls, graph): + def fromGraph(cls, graph, sess, feed_names, fetch_names): """ Construct a TFInputGraphBuilder from a in memory tf.Graph object """ assert isinstance(graph, tf.Graph), \ ('expect tf.Graph type but got', type(graph)) - def import_graph_fn(sess): - gdef = graph.as_graph_def(add_shapes=True) - with sess.as_default(): - tf.import_graph_def(gdef, name='') - return None # no meta_graph_def + def import_graph_fn(_sess): + assert _sess == sess, 'must have the same session' + return _GinBuilderInfo() - return cls(import_graph_fn) + return _GinBuilder(import_graph_fn, sess, graph).build(feed_names, fetch_names) @classmethod - def fromGraphDef(cls, graph_def): + def fromGraphDef(cls, graph_def, feed_names, fetch_names): """ Construct a TFInputGraphBuilder from a tf.GraphDef object """ @@ -135,16 +66,33 @@ def fromGraphDef(cls, graph_def): def import_graph_fn(sess): with sess.as_default(): tf.import_graph_def(graph_def, name='') - return None + return _GinBuilderInfo() + + return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) + + @classmethod + def fromCheckpoint(cls, checkpoint_dir): + return cls._from_checkpoint_impl(checkpoint_dir, signature_def_key=None) + + @classmethod + def fromCheckpointWithSignature(cls, checkpoint_dir, signature_def_key): + assert signature_def_key is not None + return cls._from_checkpoint_impl(checkpoint_dir, signature_def_key) - return cls(import_graph_fn) + @classmethod + def fromSavedModel(cls, saved_model_dir, tag_set): + return cls._from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None) + + @classmethod + def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key): + assert signature_def_key is not None + return cls._from_saved_model_impl(saved_model_dir, tag_set, signature_def_key) @classmethod - def fromCheckpoint(cls, checkpoint_dir, signature_def_key=None): + def _from_checkpoint_impl(cls, checkpoint_dir, signature_def_key=None): """ Construct a TFInputGraphBuilder from a model checkpoint """ - def import_graph_fn(sess): # Load checkpoint and import the graph with sess.as_default(): @@ -167,16 +115,15 @@ def import_graph_fn(sess): 'but failed to find it from the meta_graph_def ' + \ 'from checkpoint {}'.format(checkpoint_dir) - return sig_def + return _GinBuilderInfo(sig_def=sig_def) - return cls(import_graph_fn) + return _GinBuilder(import_graph_fn).build() @classmethod - def fromSavedModel(cls, saved_model_dir, tag_set, signature_def_key=None): + def _from_saved_model_impl(cls, saved_model_dir, tag_set, signature_def_key=None): """ Construct a TFInputGraphBuilder from a SavedModel """ - def import_graph_fn(sess): tag_sets = tag_set.split(',') meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) @@ -186,6 +133,92 @@ def import_graph_fn(sess): sig_def = tf.contrib.saved_model.get_signature_def_by_key( meta_graph_def, signature_def_key) - return sig_def - - return cls(import_graph_fn) + return _GinBuilderInfo(sig_def=sig_def) + + return _GinBuilder(import_graph_fn).build() + + +class _GinBuilderInfo(object): + def __init__(self, sig_def=None): + self.sig_def = sig_def + + +class _GinBuilder(object): + def __init__(self, import_graph_fn, sess=None, graph=None): + self.import_graph_fn = import_graph_fn + assert (sess is None) == (graph is None) + self.graph = graph or tf.Graph() + if sess is not None: + self.sess = sess + self._should_clean = True + else: + self.sess = tf.Session(self.graph) + self._should_clean = False + + def _build_impl(self, feed_names, fetch_names): + # pylint: disable=protected-access,attribute-defined-outside-init + gin = TFInputGraph._new_obj_internal() + assert (feed_names is None) == (fetch_names is None) + must_have_sig_def = fetch_names is None + with self.sess.as_default(): + _ginfo = self.import_graph_fn(self.sess) + # TODO: extract signature mappings + if must_have_sig_def: + raise NotImplementedError("cannot extract mappings from sig_def at the moment") + for tnsr_name in feed_names: + assert tfx.get_op(self.graph, tnsr_name) + fetches = [tfx.get_tensor(self.graph, tnsr_name) for tnsr_name in fetch_names] + gin.graph_def = tfx.strip_and_freeze_until(fetches, self.graph, self.sess) + return gin + + def build(self, feed_names=None, fetch_names=None): + try: + gin = self._build_impl(feed_names, fetch_names) + finally: + if self._should_clean: + self.sess.close() + return gin + +# def the_rest(input_mapping, output_mapping): +# graph = tf.Graph() +# with tf.Session(graph=graph) as sess: +# # Append feeds and input mapping +# _input_mapping = {} +# if isinstance(input_mapping, dict): +# input_mapping = input_mapping.items() +# for input_colname, tnsr_or_sig in input_mapping: +# if sig_def: +# tnsr = sig_def.inputs[tnsr_or_sig].name +# else: +# tnsr = tnsr_or_sig +# _input_mapping[input_colname] = tfx.op_name(graph, tnsr) +# input_mapping = _input_mapping + +# # Append fetches and output mapping +# fetches = [] +# _output_mapping = {} +# # By default the output columns will have the name of their +# # corresponding `tf.Graph` operation names. +# # We have to convert them to the user specified output names +# if isinstance(output_mapping, dict): +# output_mapping = output_mapping.items() +# for tnsr_or_sig, requested_colname in output_mapping: +# if sig_def: +# tnsr = sig_def.outputs[tnsr_or_sig].name +# else: +# tnsr = tnsr_or_sig +# fetches.append(tfx.get_tensor(graph, tnsr)) +# tf_output_colname = tfx.op_name(graph, tnsr) +# # NOTE(phi-dbq): put the check here as it will be the entry point to construct +# # a `TFInputGraph` object. +# assert tf_output_colname not in _output_mapping, \ +# "operation {} has multiple output tensors and ".format(tf_output_colname) + \ +# "at least two of them are used in the output DataFrame. " + \ +# "Operation names are used to name columns which leads to conflicts. " + \ +# "You can apply `tf.identity` ops to each to avoid name conflicts." +# _output_mapping[tf_output_colname] = requested_colname +# output_mapping = _output_mapping + +# gdef = tfx.strip_and_freeze_until(fetches, graph, sess) + +# return TFInputGraph(gdef), input_mapping, output_mapping From 889df0aa6808eb491f5f7a79de0154c086d8351a Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 16 Sep 2017 17:16:39 -0700 Subject: [PATCH 16/80] (wip) moving to new API --- python/sparkdl/__init__.py | 4 +- python/sparkdl/graph/input.py | 46 ++- python/sparkdl/param/converters.py | 7 +- python/sparkdl/param/shared_params.py | 2 +- python/sparkdl/transformers/tf_tensor.py | 33 +- python/tests/transformers/tf_tensor_test.py | 353 ++++++++++---------- 6 files changed, 224 insertions(+), 221 deletions(-) diff --git a/python/sparkdl/__init__.py b/python/sparkdl/__init__.py index c5c55ff3..06b91bc8 100644 --- a/python/sparkdl/__init__.py +++ b/python/sparkdl/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. # -from .graph.input import TFInputGraphBuilder +from .graph.input import TFInputGraph from .image.imageIO import imageSchema, imageType, readImages from .transformers.keras_image import KerasImageFileTransformer from .transformers.named_image import DeepImagePredictor, DeepImageFeaturizer @@ -24,6 +24,6 @@ __all__ = [ 'imageSchema', 'imageType', 'readImages', - 'TFImageTransformer', 'TFInputGraphBuilder', 'TFTransformer', + 'TFImageTransformer', 'TFInputGraph', 'TFTransformer', 'DeepImagePredictor', 'DeepImageFeaturizer', 'KerasImageFileTransformer', 'imageInputPlaceholder'] diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 97e99a33..54375f6c 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import absolute_import, division, print_function import tensorflow as tf from tensorflow.core.protobuf import meta_graph_pb2 @@ -20,7 +21,6 @@ __all__ = ["TFInputGraph"] - class TFInputGraph(object): """ An opaque serializable object containing TensorFlow graph. @@ -141,30 +141,58 @@ def import_graph_fn(sess): class _GinBuilderInfo(object): def __init__(self, sig_def=None): self.sig_def = sig_def - + self.feed_names = None + self.feed_mapping = None + self.fetch_names = None + self.fetch_mapping = None + + def extract_signatures(self): + assert self.sig_def is not None, \ + "ask to find sigdef mapping, but not found any" + + self.feed_mapping = {} + self.feed_names = [] + for sigdef_key, tnsr_info in self.sig_def.inputs: + tnsr_name = tnsr_info.name + self.feed_mapping[sigdef_key] = tnsr_name + self.feed_names.append(tnsr_name) + + self.fetch_mapping = {} + self.fetch_names = [] + for sigdef_key, tnsr_info in self.sig_def.outputs: + tnsr_name = tnsr_info.name + self.feed_mapping[sigdef_key] = tnsr_name + self.fetch_names.append(tnsr_name) class _GinBuilder(object): def __init__(self, import_graph_fn, sess=None, graph=None): self.import_graph_fn = import_graph_fn assert (sess is None) == (graph is None) - self.graph = graph or tf.Graph() if sess is not None: + self.graph = graph self.sess = sess - self._should_clean = True - else: - self.sess = tf.Session(self.graph) self._should_clean = False + else: + self.graph = tf.Graph() + self.sess = tf.Session(graph=self.graph) + self._should_clean = True def _build_impl(self, feed_names, fetch_names): # pylint: disable=protected-access,attribute-defined-outside-init gin = TFInputGraph._new_obj_internal() assert (feed_names is None) == (fetch_names is None) must_have_sig_def = fetch_names is None - with self.sess.as_default(): + print('builder-session', repr(self.sess)) + # NOTE(phi-dbq): both have to be set to default + with self.sess.as_default(), self.graph.as_default(): _ginfo = self.import_graph_fn(self.sess) - # TODO: extract signature mappings if must_have_sig_def: - raise NotImplementedError("cannot extract mappings from sig_def at the moment") + _ginfo.extract_signatures() + feed_names = _ginfo.feed_names + fetch_names = _ginfo.fetch_names + gin.input_tensor_name_from_signature = _ginfo.feed_mapping + gin.output_tensor_name_from_signature = _ginfo.fetch_mapping + for tnsr_name in feed_names: assert tfx.get_op(self.graph, tnsr_name) fetches = [tfx.get_tensor(self.graph, tnsr_name) for tnsr_name in fetch_names] diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index d9b8e512..ae49cc46 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -13,17 +13,12 @@ # limitations under the License. # -from functools import wraps -import six - -import keras import tensorflow as tf from pyspark.ml.param import TypeConverters -from sparkdl.graph.builder import GraphFunction, IsolatedSession import sparkdl.graph.utils as tfx -from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder +from sparkdl.graph.input import TFInputGraph import sparkdl.utils.keras_model as kmutil class SparkDLTypeConverters(object): diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index 64d4e6f1..11330270 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -29,7 +29,7 @@ from sparkdl.graph.builder import GraphFunction, IsolatedSession import sparkdl.graph.utils as tfx -from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder +from sparkdl.graph.input import TFInputGraph from sparkdl.param.converters import SparkDLTypeConverters import sparkdl.utils.keras_model as kmutil diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 027c6043..0b4d5c2e 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -21,7 +21,7 @@ from pyspark.ml import Transformer import sparkdl.graph.utils as tfx -from sparkdl.graph.input import TFInputGraph, TFInputGraphBuilder +from sparkdl.graph.input import TFInputGraph from sparkdl.param import (keyword_only, SparkDLTypeConverters, HasInputMapping, HasOutputMapping, HasTFInputGraph, HasTFHParams) @@ -57,31 +57,6 @@ def setParams(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tf """ super(TFTransformer, self).__init__() kwargs = self._input_kwargs - # The set of parameters either come from some helper functions, - # in which case type(_maybe_gin) is already TFInputGraph. - _maybe_gin = tfInputGraph - if isinstance(_maybe_gin, TFInputGraph): - return self._set(**kwargs) - - # Otherwise, `_maybe_gin` needs to be converted to TFInputGraph - # We put all the conversion logic here rather than in SparkDLTypeConverters - if isinstance(_maybe_gin, TFInputGraphBuilder): - gin = _maybe_gin - elif isinstance(_maybe_gin, tf.Graph): - gin = TFInputGraphBuilder.fromGraph(_maybe_gin) - elif isinstance(_maybe_gin, tf.GraphDef): - gin = TFInputGraphBuilder.fromGraphDef(_maybe_gin) - else: - raise TypeError("TFTransformer expect tfInputGraph convertible to TFInputGraph, " + \ - "but the given type {} cannot be converted, ".format(type(tfInputGraph)) + \ - "please provide `tf.Graph`, `tf.GraphDef` or use one of the " + \ - "`get_params_from_*` helper functions to build parameters") - - gin, input_mapping, output_mapping = gin.build(inputMapping, outputMapping) - kwargs['tfInputGraph'] = gin - kwargs['inputMapping'] = input_mapping - kwargs['outputMapping'] = output_mapping - # Further conanonicalization, e.g. converting dict to sorted str pairs happens here return self._set(**kwargs) @@ -94,11 +69,11 @@ def _transform(self, dataset): with tf.Session(graph=graph): analyzed_df = tfs.analyze(dataset) - out_tnsr_op_names = [tfx.as_op_name(tnsr_op_name) for tnsr_op_name, _ in output_mapping] + out_tnsr_op_names = [tfx.as_op_name(tnsr_name) for tnsr_name, _ in output_mapping] tf.import_graph_def(graph_def=gin.graph_def, name='', return_elements=out_tnsr_op_names) - feed_dict = dict((tfx.op_name(graph, tnsr_op_name), col_name) - for col_name, tnsr_op_name in input_mapping) + feed_dict = dict((tfx.op_name(graph, tnsr_name), col_name) + for col_name, tnsr_name in input_mapping) fetches = [tfx.get_tensor(graph, tnsr_op_name) for tnsr_op_name in out_tnsr_op_names] out_df = tfs.map_blocks(fetches, analyzed_df, feed_dict=feed_dict) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index d6fe9296..8cc444da 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -27,7 +27,6 @@ from pyspark.sql.types import Row -from sparkdl.graph.builder import IsolatedSession from sparkdl.graph.input import * import sparkdl.graph.utils as tfx from sparkdl.transformers.tf_tensor import TFTransformer @@ -46,6 +45,8 @@ def setUp(self): self.output_col = 'outputCol' self.output_op_name = 'tnsrOpOut' + self.feed_names = [] + self.fetch_names = [] self.input_mapping = {} self.output_mapping = {} @@ -57,36 +58,39 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.model_output_root, ignore_errors=True) - def build_standard_transformers(self, sess, gin_builder_convertible): + def build_standard_transformers(self, sess, tf_input_graph): def _add_transformer(imap, omap): trnsfmr = TFTransformer( - tfInputGraph=gin_builder_convertible, inputMapping=imap, outputMapping=omap) + tfInputGraph=tf_input_graph, inputMapping=imap, outputMapping=omap) self.transformers.append(trnsfmr) - _add_transformer(self.input_mapping, self.output_mapping) - - imap = [(col, tfx.get_tensor(sess.graph, op_name)) - for col, op_name in self.input_mapping.items()] - omap = [(tfx.get_tensor(sess.graph, op_name), col) - for op_name, col in self.output_mapping.items()] + imap = dict((col, tfx.tensor_name(sess.graph, op_name)) + for col, op_name in self.input_mapping.items()) + omap = dict((tfx.tensor_name(sess.graph, op_name), col) + for op_name, col in self.output_mapping.items()) _add_transformer(imap, omap) - @contextmanager - def run_test_in_tf_session(self, replica=1): - """ [THIS IS NOT A TEST]: encapsulate general test workflow """ - + def setup_iomap(self, replica=1): if replica > 1: for i in range(replica): colname = '{}_replica{:03d}'.format(self.input_col, i) tnsr_op_name = '{}_replica{:03d}'.format(self.input_op_name, i) self.input_mapping[colname] = tnsr_op_name + self.feed_names.append(tnsr_op_name + ':0') colname = '{}_replica{:03d}'.format(self.output_col, i) tnsr_op_name = '{}_replica{:03d}'.format(self.output_op_name, i) self.output_mapping[tnsr_op_name] = colname + self.fetch_names.append(tnsr_op_name + ':0') else: self.input_mapping = {self.input_col: self.input_op_name} + self.feed_names = [self.input_op_name + ':0'] self.output_mapping = {self.output_op_name: self.output_col} + self.fetch_names = [self.output_op_name + ':0'] + + @contextmanager + def _run_test_in_tf_session(self): + """ [THIS IS NOT A TEST]: encapsulate general test workflow """ # Build local features and DataFrame from it local_features = [] @@ -141,174 +145,175 @@ def run_test_in_tf_session(self, replica=1): def test_build_from_tf_graph(self): - with self.run_test_in_tf_session() as sess: + self.setup_iomap(replica=1) + with self._run_test_in_tf_session() as sess: # Begin building graph x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) # End building graph # Begin building transformers - self.build_standard_transformers(sess, sess.graph) - self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) + self.build_standard_transformers( + sess, TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names)) gdef = sess.graph.as_graph_def() - self.build_standard_transformers(sess, gdef) - self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraphDef(gdef)) + self.build_standard_transformers( + sess, TFInputGraph.fromGraphDef(gdef, self.feed_names, self.fetch_names)) # End building transformers - def test_build_from_saved_model(self): - # Setup saved model export directory - saved_model_root = self.model_output_root - saved_model_dir = os.path.join(saved_model_root, 'saved_model') - serving_tag = "serving_tag" - serving_sigdef_key = 'prediction_signature' - builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - - with self.run_test_in_tf_session() as sess: - # Model definition: begin - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), - dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - # Model definition ends - - sess.run(w.initializer) - - sig_inputs = { - 'input_sig': tf.saved_model.utils.build_tensor_info(x)} - sig_outputs = { - 'output_sig': tf.saved_model.utils.build_tensor_info(z)} - - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs=sig_inputs, - outputs=sig_outputs) - - builder.add_meta_graph_and_variables(sess, - [serving_tag], - signature_def_map={ - serving_sigdef_key: serving_sigdef}) - builder.save() - - # Build the transformer from exported serving model - # We are using signaures, thus must provide the keys - tfInputGraph, inputMapping, outputMapping = get_params_from_saved_model( - saved_model_dir, serving_tag, serving_sigdef_key, - input_mapping={ - self.input_col: 'input_sig'}, - output_mapping={ - 'output_sig': self.output_col}) - trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, - inputMapping=inputMapping, - outputMapping=outputMapping) - self.transformers.append(trans_with_sig) - - # Build the transformer from exported serving model - # We are not using signatures, thus must provide tensor/operation names - gin_builder = TFInputGraphBuilder.fromSavedModel( - saved_model_dir, tag_set=serving_tag, signature_def_key=None) - self.build_standard_transformers(sess, gin_builder) - - - def test_build_from_checkpoint(self): - """ - Test constructing a Transformer from a TensorFlow training checkpoint - """ - # Build the TensorFlow graph - model_ckpt_dir = self.model_output_root - ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - serving_sigdef_key = 'prediction_signature' - # Warning: please use a new graph for each test cases - # or the tests could affect one another - with self.run_test_in_tf_session() as sess: - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), - dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - sess.run(w.initializer) - saver = tf.train.Saver(var_list=[w]) - _ = saver.save(sess, ckpt_path_prefix, global_step=2702) - - # Prepare the signature_def - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs={ - 'input_sig': tf.saved_model.utils.build_tensor_info(x) - }, - outputs={ - 'output_sig': tf.saved_model.utils.build_tensor_info(z) - }) - - # A rather contrived way to add signature def to a meta_graph - meta_graph_def = tf.train.export_meta_graph() - - # Find the meta_graph file (there should be only one) - _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) - self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) - ckpt_meta_fpath = _ckpt_meta_fpaths[0] - - # Add signature_def to the meta_graph and serialize it - # This will overwrite the existing meta_graph_def file - meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) - with open(ckpt_meta_fpath, mode='wb') as fout: - fout.write(meta_graph_def.SerializeToString()) - - tfInputGraph, inputMapping, outputMapping = get_params_from_checkpoint( - model_ckpt_dir, serving_sigdef_key, - input_mapping={ - self.input_col: 'input_sig'}, - output_mapping={ - 'output_sig': self.output_col}) - trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, - inputMapping=inputMapping, - outputMapping=outputMapping) - self.transformers.append(trans_with_sig) - - gin_builder = TFInputGraphBuilder.fromCheckpoint(model_ckpt_dir) - self.build_standard_transformers(sess, gin_builder) - - - def test_multi_io(self): - # Build the TensorFlow graph - with self.run_test_in_tf_session(replica=2) as sess: - xs = [] - for tnsr_op_name in self.input_mapping.values(): - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) - xs.append(x) - - zs = [] - for i, tnsr_op_name in enumerate(self.output_mapping.keys()): - z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) - zs.append(z) - - self.build_standard_transformers(sess, sess.graph) - self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) - - def test_mixed_keras_graph(self): - # Build the graph: the output should have the same leading/batch dimension - with IsolatedSession(using_keras=True) as issn: - tnsr_in = tf.placeholder( - tf.double, shape=[None, self.vec_size], name=self.input_op_name) - inp = tf.expand_dims(tnsr_in, axis=2) - # Keras layers does not take tf.double - inp = tf.cast(inp, tf.float32) - conv = Conv1D(filters=4, kernel_size=2)(inp) - pool = MaxPool1D(pool_size=2)(conv) - flat = Flatten()(pool) - dense = Dense(1)(flat) - # We must keep the leading dimension of the output - redsum = tf.reduce_sum(dense, axis=1) - tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) - - # Initialize the variables - init_op = tf.global_variables_initializer() - issn.run(init_op) - # We could train the model ... but skip it here - gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) - - with self.run_test_in_tf_session() as sess: - tf.import_graph_def(gfn.graph_def, name='') - - self.build_standard_transformers(sess, sess.graph) - self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) - self.build_standard_transformers(sess, gfn.graph_def) - self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraphDef(gfn.graph_def)) + # def test_build_from_saved_model(self): + # # Setup saved model export directory + # saved_model_root = self.model_output_root + # saved_model_dir = os.path.join(saved_model_root, 'saved_model') + # serving_tag = "serving_tag" + # serving_sigdef_key = 'prediction_signature' + # builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) + + # with self.run_test_in_tf_session() as sess: + # # Model definition: begin + # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + # w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), + # dtype=tf.float64, name='varW') + # z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + # # Model definition ends + + # sess.run(w.initializer) + + # sig_inputs = { + # 'input_sig': tf.saved_model.utils.build_tensor_info(x)} + # sig_outputs = { + # 'output_sig': tf.saved_model.utils.build_tensor_info(z)} + + # serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + # inputs=sig_inputs, + # outputs=sig_outputs) + + # builder.add_meta_graph_and_variables(sess, + # [serving_tag], + # signature_def_map={ + # serving_sigdef_key: serving_sigdef}) + # builder.save() + + # # Build the transformer from exported serving model + # # We are using signaures, thus must provide the keys + # tfInputGraph, inputMapping, outputMapping = get_params_from_saved_model( + # saved_model_dir, serving_tag, serving_sigdef_key, + # input_mapping={ + # self.input_col: 'input_sig'}, + # output_mapping={ + # 'output_sig': self.output_col}) + # trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, + # inputMapping=inputMapping, + # outputMapping=outputMapping) + # self.transformers.append(trans_with_sig) + + # # Build the transformer from exported serving model + # # We are not using signatures, thus must provide tensor/operation names + # gin_builder = TFInputGraphBuilder.fromSavedModel( + # saved_model_dir, tag_set=serving_tag, signature_def_key=None) + # self.build_standard_transformers(sess, gin_builder) + + + # def test_build_from_checkpoint(self): + # """ + # Test constructing a Transformer from a TensorFlow training checkpoint + # """ + # # Build the TensorFlow graph + # model_ckpt_dir = self.model_output_root + # ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') + # serving_sigdef_key = 'prediction_signature' + # # Warning: please use a new graph for each test cases + # # or the tests could affect one another + # with self.run_test_in_tf_session() as sess: + # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + # #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) + # w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), + # dtype=tf.float64, name='varW') + # z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + # sess.run(w.initializer) + # saver = tf.train.Saver(var_list=[w]) + # _ = saver.save(sess, ckpt_path_prefix, global_step=2702) + + # # Prepare the signature_def + # serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + # inputs={ + # 'input_sig': tf.saved_model.utils.build_tensor_info(x) + # }, + # outputs={ + # 'output_sig': tf.saved_model.utils.build_tensor_info(z) + # }) + + # # A rather contrived way to add signature def to a meta_graph + # meta_graph_def = tf.train.export_meta_graph() + + # # Find the meta_graph file (there should be only one) + # _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) + # self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) + # ckpt_meta_fpath = _ckpt_meta_fpaths[0] + + # # Add signature_def to the meta_graph and serialize it + # # This will overwrite the existing meta_graph_def file + # meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) + # with open(ckpt_meta_fpath, mode='wb') as fout: + # fout.write(meta_graph_def.SerializeToString()) + + # tfInputGraph, inputMapping, outputMapping = get_params_from_checkpoint( + # model_ckpt_dir, serving_sigdef_key, + # input_mapping={ + # self.input_col: 'input_sig'}, + # output_mapping={ + # 'output_sig': self.output_col}) + # trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, + # inputMapping=inputMapping, + # outputMapping=outputMapping) + # self.transformers.append(trans_with_sig) + + # gin_builder = TFInputGraphBuilder.fromCheckpoint(model_ckpt_dir) + # self.build_standard_transformers(sess, gin_builder) + + + # def test_multi_io(self): + # # Build the TensorFlow graph + # with self.run_test_in_tf_session(replica=2) as sess: + # xs = [] + # for tnsr_op_name in self.input_mapping.values(): + # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) + # xs.append(x) + + # zs = [] + # for i, tnsr_op_name in enumerate(self.output_mapping.keys()): + # z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) + # zs.append(z) + + # self.build_standard_transformers(sess, sess.graph) + # self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) + + # def test_mixed_keras_graph(self): + # # Build the graph: the output should have the same leading/batch dimension + # with IsolatedSession(using_keras=True) as issn: + # tnsr_in = tf.placeholder( + # tf.double, shape=[None, self.vec_size], name=self.input_op_name) + # inp = tf.expand_dims(tnsr_in, axis=2) + # # Keras layers does not take tf.double + # inp = tf.cast(inp, tf.float32) + # conv = Conv1D(filters=4, kernel_size=2)(inp) + # pool = MaxPool1D(pool_size=2)(conv) + # flat = Flatten()(pool) + # dense = Dense(1)(flat) + # # We must keep the leading dimension of the output + # redsum = tf.reduce_sum(dense, axis=1) + # tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) + + # # Initialize the variables + # init_op = tf.global_variables_initializer() + # issn.run(init_op) + # # We could train the model ... but skip it here + # gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) + + # with self.run_test_in_tf_session() as sess: + # tf.import_graph_def(gfn.graph_def, name='') + + # self.build_standard_transformers(sess, sess.graph) + # self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) + # self.build_standard_transformers(sess, gfn.graph_def) + # self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraphDef(gfn.graph_def)) From 86cd6d9310433751232579744843030f8e29781d Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 16 Sep 2017 18:08:57 -0700 Subject: [PATCH 17/80] (wip) enable saved_model tests --- python/sparkdl/graph/input.py | 69 ++++++++++--- python/tests/transformers/tf_tensor_test.py | 104 ++++++++++---------- 2 files changed, 110 insertions(+), 63 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 54375f6c..285dd79a 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -41,6 +41,28 @@ def _new_obj_internal(cls): obj.output_tensor_name_from_signature = None return obj + def translateInputMapping(self, input_mapping): + assert self.input_tensor_name_from_signature is not None + _input_mapping = {} + if isinstance(input_mapping, dict): + input_mapping = list(input_mapping.items()) + assert isinstance(input_mapping, list) + for col_name, sig_key in input_mapping: + tnsr_name = self.input_tensor_name_from_signature[sig_key] + _input_mapping[col_name] = tnsr_name + return _input_mapping + + def translateOutputMapping(self, output_mapping): + assert self.output_tensor_name_from_signature is not None + _output_mapping = {} + if isinstance(output_mapping, dict): + output_mapping = list(output_mapping.items()) + assert isinstance(output_mapping, list) + for sig_key, col_name in output_mapping: + tnsr_name = self.output_tensor_name_from_signature[sig_key] + _output_mapping[tnsr_name] = col_name + return _output_mapping + @classmethod def fromGraph(cls, graph, sess, feed_names, fetch_names): """ @@ -71,28 +93,43 @@ def import_graph_fn(sess): return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) @classmethod - def fromCheckpoint(cls, checkpoint_dir): - return cls._from_checkpoint_impl(checkpoint_dir, signature_def_key=None) + def fromCheckpoint(cls, checkpoint_dir, feed_names, fetch_names): + return cls._from_checkpoint_impl(checkpoint_dir, + signature_def_key=None, + feed_names=feed_names, fetch_names=fetch_names) @classmethod def fromCheckpointWithSignature(cls, checkpoint_dir, signature_def_key): assert signature_def_key is not None - return cls._from_checkpoint_impl(checkpoint_dir, signature_def_key) + return cls._from_checkpoint_impl(checkpoint_dir, + signature_def_key, + feed_names=None, fetch_names=None) @classmethod - def fromSavedModel(cls, saved_model_dir, tag_set): - return cls._from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None) + def fromSavedModel(cls, saved_model_dir, tag_set, feed_names, fetch_names): + return cls._from_saved_model_impl(saved_model_dir, tag_set, + signature_def_key=None, + feed_names=feed_names, fetch_names=fetch_names) @classmethod def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key): assert signature_def_key is not None - return cls._from_saved_model_impl(saved_model_dir, tag_set, signature_def_key) + return cls._from_saved_model_impl(saved_model_dir, tag_set, + signature_def_key=signature_def_key, + feed_names=None, fetch_names=None) @classmethod - def _from_checkpoint_impl(cls, checkpoint_dir, signature_def_key=None): + def _from_checkpoint_impl(cls, + checkpoint_dir, + signature_def_key=None, + feed_names=None, + fetch_names=None): """ Construct a TFInputGraphBuilder from a model checkpoint """ + assert (feed_names is None) == (fetch_names is None) + assert (feed_names is None) or (signature_def_key is None) + def import_graph_fn(sess): # Load checkpoint and import the graph with sess.as_default(): @@ -117,13 +154,19 @@ def import_graph_fn(sess): return _GinBuilderInfo(sig_def=sig_def) - return _GinBuilder(import_graph_fn).build() + return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) @classmethod - def _from_saved_model_impl(cls, saved_model_dir, tag_set, signature_def_key=None): + def _from_saved_model_impl(cls, saved_model_dir, tag_set, + signature_def_key=None, + feed_names=None, + fetch_names=None): """ Construct a TFInputGraphBuilder from a SavedModel """ + assert (feed_names is None) == (fetch_names is None) + assert (feed_names is None) or (signature_def_key is None) + def import_graph_fn(sess): tag_sets = tag_set.split(',') meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) @@ -135,7 +178,7 @@ def import_graph_fn(sess): return _GinBuilderInfo(sig_def=sig_def) - return _GinBuilder(import_graph_fn).build() + return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) class _GinBuilderInfo(object): @@ -152,16 +195,16 @@ def extract_signatures(self): self.feed_mapping = {} self.feed_names = [] - for sigdef_key, tnsr_info in self.sig_def.inputs: + for sigdef_key, tnsr_info in self.sig_def.inputs.items(): tnsr_name = tnsr_info.name self.feed_mapping[sigdef_key] = tnsr_name self.feed_names.append(tnsr_name) self.fetch_mapping = {} self.fetch_names = [] - for sigdef_key, tnsr_info in self.sig_def.outputs: + for sigdef_key, tnsr_info in self.sig_def.outputs.items(): tnsr_name = tnsr_info.name - self.feed_mapping[sigdef_key] = tnsr_name + self.fetch_mapping[sigdef_key] = tnsr_name self.fetch_names.append(tnsr_name) class _GinBuilder(object): diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 8cc444da..45622894 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -161,57 +161,61 @@ def test_build_from_tf_graph(self): # End building transformers - # def test_build_from_saved_model(self): - # # Setup saved model export directory - # saved_model_root = self.model_output_root - # saved_model_dir = os.path.join(saved_model_root, 'saved_model') - # serving_tag = "serving_tag" - # serving_sigdef_key = 'prediction_signature' - # builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - - # with self.run_test_in_tf_session() as sess: - # # Model definition: begin - # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - # w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), - # dtype=tf.float64, name='varW') - # z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - # # Model definition ends - - # sess.run(w.initializer) - - # sig_inputs = { - # 'input_sig': tf.saved_model.utils.build_tensor_info(x)} - # sig_outputs = { - # 'output_sig': tf.saved_model.utils.build_tensor_info(z)} - - # serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - # inputs=sig_inputs, - # outputs=sig_outputs) - - # builder.add_meta_graph_and_variables(sess, - # [serving_tag], - # signature_def_map={ - # serving_sigdef_key: serving_sigdef}) - # builder.save() - - # # Build the transformer from exported serving model - # # We are using signaures, thus must provide the keys - # tfInputGraph, inputMapping, outputMapping = get_params_from_saved_model( - # saved_model_dir, serving_tag, serving_sigdef_key, - # input_mapping={ - # self.input_col: 'input_sig'}, - # output_mapping={ - # 'output_sig': self.output_col}) - # trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, - # inputMapping=inputMapping, - # outputMapping=outputMapping) - # self.transformers.append(trans_with_sig) + def test_build_from_saved_model(self): + self.setup_iomap(replica=1) + # Setup saved model export directory + saved_model_root = self.model_output_root + saved_model_dir = os.path.join(saved_model_root, 'saved_model') + serving_tag = "serving_tag" + serving_sigdef_key = 'prediction_signature' + builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - # # Build the transformer from exported serving model - # # We are not using signatures, thus must provide tensor/operation names - # gin_builder = TFInputGraphBuilder.fromSavedModel( - # saved_model_dir, tag_set=serving_tag, signature_def_key=None) - # self.build_standard_transformers(sess, gin_builder) + with self._run_test_in_tf_session() as sess: + # Model definition: begin + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), + dtype=tf.float64, name='varW') + z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + # Model definition ends + + sess.run(w.initializer) + + sig_inputs = { + 'input_sig': tf.saved_model.utils.build_tensor_info(x)} + sig_outputs = { + 'output_sig': tf.saved_model.utils.build_tensor_info(z)} + + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs=sig_inputs, + outputs=sig_outputs) + + builder.add_meta_graph_and_variables(sess, + [serving_tag], + signature_def_map={ + serving_sigdef_key: serving_sigdef}) + builder.save() + + # Build the transformer from exported serving model + # We are using signaures, thus must provide the keys + tfInputGraph = TFInputGraph.fromSavedModelWithSignature( + saved_model_dir, serving_tag, serving_sigdef_key) + + inputMapping = tfInputGraph.translateInputMapping({ + self.input_col: 'input_sig' + }) + outputMapping = tfInputGraph.translateOutputMapping({ + 'output_sig': self.output_col + }) + trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, + inputMapping=inputMapping, + outputMapping=outputMapping) + self.transformers.append(trans_with_sig) + + # Build the transformer from exported serving model + # We are not using signatures, thus must provide tensor/operation names + gin_builder = TFInputGraph.fromSavedModel( + saved_model_dir, serving_tag, self.feed_names, self.fetch_names) + self.build_standard_transformers(sess, gin_builder) # def test_build_from_checkpoint(self): From ac091823b37533756f325ece95c93119056304ba Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 16 Sep 2017 19:55:30 -0700 Subject: [PATCH 18/80] (wip) enable checkpoint test --- python/tests/transformers/tf_tensor_test.py | 130 +++++++++++--------- 1 file changed, 70 insertions(+), 60 deletions(-) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 45622894..d968f797 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -49,12 +49,14 @@ def setUp(self): self.fetch_names = [] self.input_mapping = {} self.output_mapping = {} + self.setup_iomap(replica=1) self.transformers = [] self.test_case_results = [] # Build a temporary directory, which might or might not be used by the test self.model_output_root = tempfile.mkdtemp() + def tearDown(self): shutil.rmtree(self.model_output_root, ignore_errors=True) @@ -71,6 +73,11 @@ def _add_transformer(imap, omap): _add_transformer(imap, omap) def setup_iomap(self, replica=1): + self.input_mapping = {} + self.feed_names = [] + self.output_mapping = {} + self.fetch_names = [] + if replica > 1: for i in range(replica): colname = '{}_replica{:03d}'.format(self.input_col, i) @@ -145,7 +152,7 @@ def _run_test_in_tf_session(self): def test_build_from_tf_graph(self): - self.setup_iomap(replica=1) + """ Build TFTransformer from tf.Graph """ with self._run_test_in_tf_session() as sess: # Begin building graph x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) @@ -162,7 +169,7 @@ def test_build_from_tf_graph(self): def test_build_from_saved_model(self): - self.setup_iomap(replica=1) + """ Build TFTransformer from saved model """ # Setup saved model export directory saved_model_root = self.model_output_root saved_model_dir = os.path.join(saved_model_root, 'saved_model') @@ -213,67 +220,70 @@ def test_build_from_saved_model(self): # Build the transformer from exported serving model # We are not using signatures, thus must provide tensor/operation names - gin_builder = TFInputGraph.fromSavedModel( + gin = TFInputGraph.fromSavedModel( saved_model_dir, serving_tag, self.feed_names, self.fetch_names) - self.build_standard_transformers(sess, gin_builder) + self.build_standard_transformers(sess, gin) - # def test_build_from_checkpoint(self): - # """ - # Test constructing a Transformer from a TensorFlow training checkpoint - # """ - # # Build the TensorFlow graph - # model_ckpt_dir = self.model_output_root - # ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - # serving_sigdef_key = 'prediction_signature' - # # Warning: please use a new graph for each test cases - # # or the tests could affect one another - # with self.run_test_in_tf_session() as sess: - # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - # #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - # w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), - # dtype=tf.float64, name='varW') - # z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - # sess.run(w.initializer) - # saver = tf.train.Saver(var_list=[w]) - # _ = saver.save(sess, ckpt_path_prefix, global_step=2702) - - # # Prepare the signature_def - # serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - # inputs={ - # 'input_sig': tf.saved_model.utils.build_tensor_info(x) - # }, - # outputs={ - # 'output_sig': tf.saved_model.utils.build_tensor_info(z) - # }) - - # # A rather contrived way to add signature def to a meta_graph - # meta_graph_def = tf.train.export_meta_graph() - - # # Find the meta_graph file (there should be only one) - # _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) - # self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) - # ckpt_meta_fpath = _ckpt_meta_fpaths[0] - - # # Add signature_def to the meta_graph and serialize it - # # This will overwrite the existing meta_graph_def file - # meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) - # with open(ckpt_meta_fpath, mode='wb') as fout: - # fout.write(meta_graph_def.SerializeToString()) - - # tfInputGraph, inputMapping, outputMapping = get_params_from_checkpoint( - # model_ckpt_dir, serving_sigdef_key, - # input_mapping={ - # self.input_col: 'input_sig'}, - # output_mapping={ - # 'output_sig': self.output_col}) - # trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, - # inputMapping=inputMapping, - # outputMapping=outputMapping) - # self.transformers.append(trans_with_sig) - - # gin_builder = TFInputGraphBuilder.fromCheckpoint(model_ckpt_dir) - # self.build_standard_transformers(sess, gin_builder) + def test_build_from_checkpoint(self): + """ Build TFTransformer from a model checkpoint """ + # Build the TensorFlow graph + model_ckpt_dir = self.model_output_root + ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') + serving_sigdef_key = 'prediction_signature' + + with self._run_test_in_tf_session() as sess: + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) + w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), + dtype=tf.float64, name='varW') + z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + sess.run(w.initializer) + saver = tf.train.Saver(var_list=[w]) + _ = saver.save(sess, ckpt_path_prefix, global_step=2702) + + # Prepare the signature_def + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs={ + 'input_sig': tf.saved_model.utils.build_tensor_info(x) + }, + outputs={ + 'output_sig': tf.saved_model.utils.build_tensor_info(z) + }) + + # A rather contrived way to add signature def to a meta_graph + meta_graph_def = tf.train.export_meta_graph() + + # Find the meta_graph file (there should be only one) + _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) + self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) + ckpt_meta_fpath = _ckpt_meta_fpaths[0] + + # Add signature_def to the meta_graph and serialize it + # This will overwrite the existing meta_graph_def file + meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) + with open(ckpt_meta_fpath, mode='wb') as fout: + fout.write(meta_graph_def.SerializeToString()) + + # Build the transformer from exported serving model + # We are using signaures, thus must provide the keys + tfInputGraph = TFInputGraph.fromCheckpointWithSignature( + model_ckpt_dir, serving_sigdef_key) + + inputMapping = tfInputGraph.translateInputMapping({ + self.input_col: 'input_sig' + }) + outputMapping = tfInputGraph.translateOutputMapping({ + 'output_sig': self.output_col + }) + trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, + inputMapping=inputMapping, + outputMapping=outputMapping) + self.transformers.append(trans_with_sig) + + # Transformer without using signature_def + gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) + self.build_standard_transformers(sess, gin) # def test_multi_io(self): From 6b22eed89353e32d84d7b8e5cd4197084f70da82 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 16 Sep 2017 19:59:04 -0700 Subject: [PATCH 19/80] (wip) enable multiple tensor tests --- python/tests/transformers/tf_tensor_test.py | 35 ++++++++++++--------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index d968f797..545f374f 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -286,21 +286,28 @@ def test_build_from_checkpoint(self): self.build_standard_transformers(sess, gin) - # def test_multi_io(self): - # # Build the TensorFlow graph - # with self.run_test_in_tf_session(replica=2) as sess: - # xs = [] - # for tnsr_op_name in self.input_mapping.values(): - # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) - # xs.append(x) - - # zs = [] - # for i, tnsr_op_name in enumerate(self.output_mapping.keys()): - # z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) - # zs.append(z) + def test_multi_io(self): + """ Build TFTransformer with multiple I/O tensors """ + self.setup_iomap(replica=3) + with self._run_test_in_tf_session() as sess: + xs = [] + for tnsr_op_name in self.input_mapping.values(): + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) + xs.append(x) + + zs = [] + for i, tnsr_op_name in enumerate(self.output_mapping.keys()): + z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) + zs.append(z) + + gin = TFInputGraph.fromGraph( + sess.graph, sess, self.feed_names, self.fetch_names) + self.build_standard_transformers(sess, gin) + + gin = TFInputGraph.fromGraphDef( + sess.graph.as_graph_def(), self.feed_names, self.fetch_names) + self.build_standard_transformers(sess, gin) - # self.build_standard_transformers(sess, sess.graph) - # self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) # def test_mixed_keras_graph(self): # # Build the graph: the output should have the same leading/batch dimension From a3517d6f5e3c5f2a30ba5588ae513a132f374224 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 16 Sep 2017 20:14:41 -0700 Subject: [PATCH 20/80] enable all tests --- python/sparkdl/graph/input.py | 57 ++----------- python/tests/transformers/tf_tensor_test.py | 94 +++++++++++---------- 2 files changed, 56 insertions(+), 95 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 285dd79a..53b923c7 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -127,8 +127,10 @@ def _from_checkpoint_impl(cls, """ Construct a TFInputGraphBuilder from a model checkpoint """ - assert (feed_names is None) == (fetch_names is None) - assert (feed_names is None) or (signature_def_key is None) + assert (feed_names is None) == (fetch_names is None), \ + 'feed_names and fetch_names, if provided must appear together' + assert (feed_names is None) != (signature_def_key is None), \ + 'must either provide feed_names or singnature_def_key' def import_graph_fn(sess): # Load checkpoint and import the graph @@ -164,8 +166,10 @@ def _from_saved_model_impl(cls, saved_model_dir, tag_set, """ Construct a TFInputGraphBuilder from a SavedModel """ - assert (feed_names is None) == (fetch_names is None) - assert (feed_names is None) or (signature_def_key is None) + assert (feed_names is None) == (fetch_names is None), \ + 'feed_names and fetch_names, if provided must appear together' + assert (feed_names is None) != (signature_def_key is None), \ + 'must either provide feed_names or singnature_def_key' def import_graph_fn(sess): tag_sets = tag_set.split(',') @@ -225,7 +229,6 @@ def _build_impl(self, feed_names, fetch_names): gin = TFInputGraph._new_obj_internal() assert (feed_names is None) == (fetch_names is None) must_have_sig_def = fetch_names is None - print('builder-session', repr(self.sess)) # NOTE(phi-dbq): both have to be set to default with self.sess.as_default(), self.graph.as_default(): _ginfo = self.import_graph_fn(self.sess) @@ -249,47 +252,3 @@ def build(self, feed_names=None, fetch_names=None): if self._should_clean: self.sess.close() return gin - -# def the_rest(input_mapping, output_mapping): -# graph = tf.Graph() -# with tf.Session(graph=graph) as sess: -# # Append feeds and input mapping -# _input_mapping = {} -# if isinstance(input_mapping, dict): -# input_mapping = input_mapping.items() -# for input_colname, tnsr_or_sig in input_mapping: -# if sig_def: -# tnsr = sig_def.inputs[tnsr_or_sig].name -# else: -# tnsr = tnsr_or_sig -# _input_mapping[input_colname] = tfx.op_name(graph, tnsr) -# input_mapping = _input_mapping - -# # Append fetches and output mapping -# fetches = [] -# _output_mapping = {} -# # By default the output columns will have the name of their -# # corresponding `tf.Graph` operation names. -# # We have to convert them to the user specified output names -# if isinstance(output_mapping, dict): -# output_mapping = output_mapping.items() -# for tnsr_or_sig, requested_colname in output_mapping: -# if sig_def: -# tnsr = sig_def.outputs[tnsr_or_sig].name -# else: -# tnsr = tnsr_or_sig -# fetches.append(tfx.get_tensor(graph, tnsr)) -# tf_output_colname = tfx.op_name(graph, tnsr) -# # NOTE(phi-dbq): put the check here as it will be the entry point to construct -# # a `TFInputGraph` object. -# assert tf_output_colname not in _output_mapping, \ -# "operation {} has multiple output tensors and ".format(tf_output_colname) + \ -# "at least two of them are used in the output DataFrame. " + \ -# "Operation names are used to name columns which leads to conflicts. " + \ -# "You can apply `tf.identity` ops to each to avoid name conflicts." -# _output_mapping[tf_output_colname] = requested_colname -# output_mapping = _output_mapping - -# gdef = tfx.strip_and_freeze_until(fetches, graph, sess) - -# return TFInputGraph(gdef), input_mapping, output_mapping diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 545f374f..fdeb42ea 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -27,6 +27,7 @@ from pyspark.sql.types import Row +from sparkdl.graph.builder import IsolatedSession from sparkdl.graph.input import * import sparkdl.graph.utils as tfx from sparkdl.transformers.tf_tensor import TFTransformer @@ -60,6 +61,15 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.model_output_root, ignore_errors=True) + def _build_default_session_tests(self, sess): + gin = TFInputGraph.fromGraph( + sess.graph, sess, self.feed_names, self.fetch_names) + self.build_standard_transformers(sess, gin) + + gin = TFInputGraph.fromGraphDef( + sess.graph.as_graph_def(), self.feed_names, self.fetch_names) + self.build_standard_transformers(sess, gin) + def build_standard_transformers(self, sess, tf_input_graph): def _add_transformer(imap, omap): trnsfmr = TFTransformer( @@ -113,7 +123,7 @@ def _run_test_in_tf_session(self): # Build the TensorFlow graph graph = tf.Graph() - with tf.Session(graph=graph) as sess: + with tf.Session(graph=graph) as sess, graph.as_default(): # Build test graph and transformers from here yield sess @@ -148,7 +158,7 @@ def _run_test_in_tf_session(self): out_tgt = np.hstack(_results) self.assertTrue(np.allclose(out_ref, out_tgt), - msg=repr(transfomer)) + msg='not close => {} != {}'.format(out_ref.shape, out_tgt.shape)) def test_build_from_tf_graph(self): @@ -159,13 +169,7 @@ def test_build_from_tf_graph(self): _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) # End building graph - # Begin building transformers - self.build_standard_transformers( - sess, TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names)) - gdef = sess.graph.as_graph_def() - self.build_standard_transformers( - sess, TFInputGraph.fromGraphDef(gdef, self.feed_names, self.fetch_names)) - # End building transformers + self._build_default_session_tests(sess) def test_build_from_saved_model(self): @@ -224,6 +228,10 @@ def test_build_from_saved_model(self): saved_model_dir, serving_tag, self.feed_names, self.fetch_names) self.build_standard_transformers(sess, gin) + gin = TFInputGraph.fromGraph( + sess.graph, sess, self.feed_names, self.fetch_names) + self.build_standard_transformers(sess, gin) + def test_build_from_checkpoint(self): """ Build TFTransformer from a model checkpoint """ @@ -285,6 +293,10 @@ def test_build_from_checkpoint(self): gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) self.build_standard_transformers(sess, gin) + gin = TFInputGraph.fromGraph( + sess.graph, sess, self.feed_names, self.fetch_names) + self.build_standard_transformers(sess, gin) + def test_multi_io(self): """ Build TFTransformer with multiple I/O tensors """ @@ -300,41 +312,31 @@ def test_multi_io(self): z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) zs.append(z) - gin = TFInputGraph.fromGraph( - sess.graph, sess, self.feed_names, self.fetch_names) - self.build_standard_transformers(sess, gin) - - gin = TFInputGraph.fromGraphDef( - sess.graph.as_graph_def(), self.feed_names, self.fetch_names) - self.build_standard_transformers(sess, gin) - + self._build_default_session_tests(sess) + + + def test_mixed_keras_graph(self): + """ Build mixed keras graph """ + with IsolatedSession(using_keras=True) as issn: + tnsr_in = tf.placeholder( + tf.double, shape=[None, self.vec_size], name=self.input_op_name) + inp = tf.expand_dims(tnsr_in, axis=2) + # Keras layers does not take tf.double + inp = tf.cast(inp, tf.float32) + conv = Conv1D(filters=4, kernel_size=2)(inp) + pool = MaxPool1D(pool_size=2)(conv) + flat = Flatten()(pool) + dense = Dense(1)(flat) + # We must keep the leading dimension of the output + redsum = tf.reduce_logsumexp(dense, axis=1) + tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) + + # Initialize the variables + init_op = tf.global_variables_initializer() + issn.run(init_op) + # We could train the model ... but skip it here + gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) - # def test_mixed_keras_graph(self): - # # Build the graph: the output should have the same leading/batch dimension - # with IsolatedSession(using_keras=True) as issn: - # tnsr_in = tf.placeholder( - # tf.double, shape=[None, self.vec_size], name=self.input_op_name) - # inp = tf.expand_dims(tnsr_in, axis=2) - # # Keras layers does not take tf.double - # inp = tf.cast(inp, tf.float32) - # conv = Conv1D(filters=4, kernel_size=2)(inp) - # pool = MaxPool1D(pool_size=2)(conv) - # flat = Flatten()(pool) - # dense = Dense(1)(flat) - # # We must keep the leading dimension of the output - # redsum = tf.reduce_sum(dense, axis=1) - # tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) - - # # Initialize the variables - # init_op = tf.global_variables_initializer() - # issn.run(init_op) - # # We could train the model ... but skip it here - # gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) - - # with self.run_test_in_tf_session() as sess: - # tf.import_graph_def(gfn.graph_def, name='') - - # self.build_standard_transformers(sess, sess.graph) - # self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph)) - # self.build_standard_transformers(sess, gfn.graph_def) - # self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraphDef(gfn.graph_def)) + with self._run_test_in_tf_session() as sess: + tf.import_graph_def(gfn.graph_def, name='') + self._build_default_session_tests(sess) From 457a4c28d001cccc68f4964afc67d06ca107a870 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 18 Sep 2017 14:06:56 -0700 Subject: [PATCH 21/80] params and converters --- python/sparkdl/param/__init__.py | 8 +- python/sparkdl/param/converters.py | 86 +++++++++++++++++ python/sparkdl/param/shared_params.py | 130 +++++++++++++++----------- 3 files changed, 165 insertions(+), 59 deletions(-) create mode 100644 python/sparkdl/param/converters.py diff --git a/python/sparkdl/param/__init__.py b/python/sparkdl/param/__init__.py index 98a8f7dd..a291a7d4 100644 --- a/python/sparkdl/param/__init__.py +++ b/python/sparkdl/param/__init__.py @@ -14,7 +14,11 @@ # from sparkdl.param.shared_params import ( - keyword_only, HasInputCol, HasOutputCol, HasLabelCol, HasKerasModel, - HasKerasLoss, HasKerasOptimizer, HasOutputNodeName, SparkDLTypeConverters) + keyword_only, HasInputCol, HasOutputCol, HasLabelCol, + # TFTransformer Params + HasInputMapping, HasOutputMapping, HasTFHParams, + # Keras Estimator Params + HasKerasModel, HasKerasLoss, HasKerasOptimizer, HasOutputNodeName) +from sparkdl.param.converters import SparkDLTypeConverters from sparkdl.param.image_params import ( CanLoadImage, HasInputImageNodeName, HasOutputMode, OUTPUT_MODES) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py new file mode 100644 index 00000000..52f76fb9 --- /dev/null +++ b/python/sparkdl/param/converters.py @@ -0,0 +1,86 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import tensorflow as tf + +from pyspark.ml.param import TypeConverters + +import sparkdl.graph.utils as tfx +import sparkdl.utils.keras_model as kmutil + +__all__ = ['SparkDLTypeConverters'] + +class SparkDLTypeConverters(object): + @staticmethod + def toTFGraph(value): + if isinstance(value, tf.Graph): + return value + else: + raise TypeError("Could not convert %s to TensorFlow Graph" % type(value)) + + @staticmethod + def asColumnToTensorMap(value): + if isinstance(value, dict): + strs_pair_seq = [(k, tfx.as_op_name(v)) for k, v in value.items()] + return sorted(strs_pair_seq) + raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) + + @staticmethod + def asTensorToColumnMap(value): + if isinstance(value, dict): + strs_pair_seq = [(tfx.as_op_name(k), v) for k, v in value.items()] + return sorted(strs_pair_seq) + raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) + + @staticmethod + def toTFHParams(value): + if isinstance(value, tf.contrib.training.HParams): + return value + else: + raise TypeError("Could not convert %s to TensorFlow HParams" % type(value)) + + @staticmethod + def toStringOrTFTensor(value): + if isinstance(value, tf.Tensor): + return value + else: + try: + return TypeConverters.toString(value) + except TypeError: + raise TypeError("Could not convert %s to tensorflow.Tensor or str" % type(value)) + + @staticmethod + def supportedNameConverter(supportedList): + def converter(value): + if value in supportedList: + return value + else: + raise TypeError("%s %s is not in the supported list." % type(value), str(value)) + + return converter + + @staticmethod + def toKerasLoss(value): + if kmutil.is_valid_loss_function(value): + return value + raise ValueError( + "Named loss not supported in Keras: {} type({})".format(value, type(value))) + + @staticmethod + def toKerasOptimizer(value): + if kmutil.is_valid_optimizer(value): + return value + raise TypeError( + "Named optimizer not supported in Keras: {} type({})".format(value, type(value))) diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index e169e891..83883235 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -21,13 +21,15 @@ from functools import wraps -import tensorflow as tf - from pyspark.ml.param import Param, Params, TypeConverters -import sparkdl.utils.keras_model as kmutil +from sparkdl.param.converters import SparkDLTypeConverters + -# From pyspark +######################################################## +# Copied from PySpark for backward compatibility. +# They first appeared in Apache Spark version 2.1.1. +######################################################## def keyword_only(func): """ @@ -36,12 +38,14 @@ def keyword_only(func): .. note:: Should only be used to wrap a method where first arg is `self` """ + @wraps(func) def wrapper(self, *args, **kwargs): if len(args) > 0: raise TypeError("Method %s forces keyword arguments." % func.__name__) self._input_kwargs = kwargs return func(self, **kwargs) + return wrapper @@ -50,10 +54,8 @@ class HasInputCol(Params): Mixin for param inputCol: input column name. """ - inputCol = Param(Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) - - def __init__(self): - super(HasInputCol, self).__init__() + inputCol = Param( + Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) def setInputCol(self, value): """ @@ -73,8 +75,8 @@ class HasOutputCol(Params): Mixin for param outputCol: output column name. """ - outputCol = Param(Params._dummy(), - "outputCol", "output column name.", typeConverter=TypeConverters.toString) + outputCol = Param( + Params._dummy(), "outputCol", "output column name.", typeConverter=TypeConverters.toString) def __init__(self): super(HasOutputCol, self).__init__() @@ -92,54 +94,9 @@ def getOutputCol(self): """ return self.getOrDefault(self.outputCol) -############################################ +######################################################## # New in sparkdl -############################################ - -class SparkDLTypeConverters(object): - - @staticmethod - def toStringOrTFTensor(value): - if isinstance(value, tf.Tensor): - return value - else: - try: - return TypeConverters.toString(value) - except TypeError: - raise TypeError("Could not convert %s to tensorflow.Tensor or str" % type(value)) - - @staticmethod - def toTFGraph(value): - # TODO: we may want to support tf.GraphDef in the future instead of tf.Graph since user - # is less likely to mess up using GraphDef vs Graph (e.g. constants vs variables). - if isinstance(value, tf.Graph): - return value - else: - raise TypeError("Could not convert %s to tensorflow.Graph type" % type(value)) - - @staticmethod - def supportedNameConverter(supportedList): - def converter(value): - if value in supportedList: - return value - else: - raise TypeError("%s %s is not in the supported list." % type(value), str(value)) - - return converter - - @staticmethod - def toKerasLoss(value): - if kmutil.is_valid_loss_function(value): - return value - raise ValueError( - "Named loss not supported in Keras: {} type({})".format(value, type(value))) - - @staticmethod - def toKerasOptimizer(value): - if kmutil.is_valid_optimizer(value): - return value - raise TypeError( - "Named optimizer not supported in Keras: {} type({})".format(value, type(value))) +######################################################## class HasOutputNodeName(Params): @@ -233,3 +190,62 @@ def seKerasLoss(self, value): def getKerasLoss(self): return self.getOrDefault(self.kerasLoss) + + +class HasOutputMapping(Params): + """ + Mixin for param outputMapping: ordered list of ('outputTensorOpName', 'outputColName') pairs + """ + outputMapping = Param( + Params._dummy(), + "outputMapping", + "Mapping output :class:`tf.Operation` names to DataFrame column names", + typeConverter=SparkDLTypeConverters.asTensorToColumnMap) + + def setOutputMapping(self, value): + # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the + # serializable TFInputGraph object once the inputMapping and outputMapping + # parameters are provided. + raise NotImplementedError( + "Please use the Transformer's constructor to assign `outputMapping` field.") + + def getOutputMapping(self): + return self.getOrDefault(self.outputMapping) + + +class HasInputMapping(Params): + """ + Mixin for param inputMapping: ordered list of ('inputColName', 'inputTensorOpName') pairs + """ + inputMapping = Param( + Params._dummy(), + "inputMapping", + "Mapping input DataFrame column names to :class:`tf.Operation` names", + typeConverter=SparkDLTypeConverters.asColumnToTensorMap) + + def setInputMapping(self, value): + # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the + # serializable TFInputGraph object once the inputMapping and outputMapping + # parameters are provided. + raise NotImplementedError( + "Please use the Transformer's constructor to assigne `inputMapping` field.") + + def getInputMapping(self): + return self.getOrDefault(self.inputMapping) + + +class HasTFHParams(Params): + """ + Mixin for TensorFlow model hyper-parameters + """ + tfHParams = Param( + Params._dummy(), + "hparams", + "instance of :class:`tf.contrib.training.HParams`, a key-value map-like object", + typeConverter=SparkDLTypeConverters.toTFHParams) + + def setTFHParams(self, value): + return self._set(tfHParam=value) + + def getTFHParams(self): + return self.getOrDefault(self.tfHParams) From 323939af11b94554d3b377758a15e9a9257b2a6d Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 18 Sep 2017 14:58:58 -0700 Subject: [PATCH 22/80] tests --- python/sparkdl/param/converters.py | 45 +++++++++++++---- python/sparkdl/param/shared_params.py | 27 +++++------ python/tests/param/__init__.py | 15 ++++++ python/tests/param/params_test.py | 69 +++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 25 deletions(-) create mode 100644 python/tests/param/__init__.py create mode 100644 python/tests/param/params_test.py diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index 52f76fb9..1a65915a 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -13,6 +13,8 @@ # limitations under the License. # +import six + import tensorflow as tf from pyspark.ml.param import TypeConverters @@ -22,6 +24,35 @@ __all__ = ['SparkDLTypeConverters'] +def _try_convert_tf_tensor_mapping(value, is_key_tf_tensor=True): + if isinstance(value, dict): + strs_pair_seq = [] + for k, v in value.items(): + try: + if is_key_tf_tensor: + _pair = (tfx.as_tensor_name(k), v) + else: + _pair = (k, tfx.as_tensor_name(v)) + except: + err_msg = "Can NOT convert {} (type {}) to tf.Tensor name" + _not_tf_op = k if is_key_tf_tensor else v + raise TypeError(err_msg.format(_not_tf_op, type(_not_tf_op))) + + str_val = v if is_key_tf_tensor else k + if not isinstance(str_val, six.string_types): + err_msg = 'expect string type for {}, but got {}' + raise TypeError(err_msg.format(str_val, type(str_val))) + + strs_pair_seq.append(_pair) + + return sorted(strs_pair_seq) + + if is_key_tf_tensor: + raise TypeError("Could not convert %s to tf.Tensor name to str mapping" % type(value)) + else: + raise TypeError("Could not convert %s to str to tf.Tensor name mapping" % type(value)) + + class SparkDLTypeConverters(object): @staticmethod def toTFGraph(value): @@ -31,18 +62,12 @@ def toTFGraph(value): raise TypeError("Could not convert %s to TensorFlow Graph" % type(value)) @staticmethod - def asColumnToTensorMap(value): - if isinstance(value, dict): - strs_pair_seq = [(k, tfx.as_op_name(v)) for k, v in value.items()] - return sorted(strs_pair_seq) - raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) + def asColumnToTensorNameMap(value): + return _try_convert_tf_tensor_mapping(value, is_key_tf_tensor=False) @staticmethod - def asTensorToColumnMap(value): - if isinstance(value, dict): - strs_pair_seq = [(tfx.as_op_name(k), v) for k, v in value.items()] - return sorted(strs_pair_seq) - raise TypeError("Could not convert %s to TensorFlow Tensor" % type(value)) + def asTensorNameToColumnMap(value): + return _try_convert_tf_tensor_mapping(value, is_key_tf_tensor=True) @staticmethod def toTFHParams(value): diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index 83883235..890dc0b3 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -196,11 +196,10 @@ class HasOutputMapping(Params): """ Mixin for param outputMapping: ordered list of ('outputTensorOpName', 'outputColName') pairs """ - outputMapping = Param( - Params._dummy(), - "outputMapping", - "Mapping output :class:`tf.Operation` names to DataFrame column names", - typeConverter=SparkDLTypeConverters.asTensorToColumnMap) + outputMapping = Param(Params._dummy(), + "outputMapping", + "Mapping output :class:`tf.Operation` names to DataFrame column names", + typeConverter=SparkDLTypeConverters.asTensorNameToColumnMap) def setOutputMapping(self, value): # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the @@ -217,11 +216,10 @@ class HasInputMapping(Params): """ Mixin for param inputMapping: ordered list of ('inputColName', 'inputTensorOpName') pairs """ - inputMapping = Param( - Params._dummy(), - "inputMapping", - "Mapping input DataFrame column names to :class:`tf.Operation` names", - typeConverter=SparkDLTypeConverters.asColumnToTensorMap) + inputMapping = Param(Params._dummy(), + "inputMapping", + "Mapping input DataFrame column names to :class:`tf.Operation` names", + typeConverter=SparkDLTypeConverters.asColumnToTensorNameMap) def setInputMapping(self, value): # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the @@ -238,11 +236,10 @@ class HasTFHParams(Params): """ Mixin for TensorFlow model hyper-parameters """ - tfHParams = Param( - Params._dummy(), - "hparams", - "instance of :class:`tf.contrib.training.HParams`, a key-value map-like object", - typeConverter=SparkDLTypeConverters.toTFHParams) + tfHParams = Param(Params._dummy(), + "hparams", + "instance of :class:`tf.contrib.training.HParams`, a key-value map-like object", + typeConverter=SparkDLTypeConverters.toTFHParams) def setTFHParams(self, value): return self._set(tfHParam=value) diff --git a/python/tests/param/__init__.py b/python/tests/param/__init__.py new file mode 100644 index 00000000..7084f22b --- /dev/null +++ b/python/tests/param/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/tests/param/params_test.py b/python/tests/param/params_test.py new file mode 100644 index 00000000..0c10411a --- /dev/null +++ b/python/tests/param/params_test.py @@ -0,0 +1,69 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import sys + +if sys.version_info[:2] <= (2, 6): + try: + import unittest2 as unittest + except ImportError: + sys.stderr.write('Please install unittest2 to test with Python 2.6 or earlier') + sys.exit(1) +else: + import unittest + +from sparkdl.param.converters import SparkDLTypeConverters as conv + +class ParamsConverterTest(unittest.TestCase): + # pylint: disable=protected-access + + def test_tf_input_mapping_converter(self): + valid_tnsr_input = {'colA': 'tnsrOpA:0', + 'colB': 'tnsrOpB:0'} + valid_op_input = {'colA': 'tnsrOpA', + 'colB': 'tnsrOpB'} + valid_input_mapping_result = [('colA', 'tnsrOpA:0'), + ('colB', 'tnsrOpB:0')] + + for valid_input_mapping in [valid_op_input, valid_tnsr_input]: + res = conv.asColumnToTensorNameMap(valid_input_mapping) + self.assertEqual(valid_input_mapping_result, res) + + def test_tf_output_mapping_converter(self): + valid_tnsr_output = {'tnsrOpA:0': 'colA', + 'tnsrOpB:0': 'colB'} + valid_op_output = {'tnsrOpA': 'colA', + 'tnsrOpB': 'colB'} + valid_output_mapping_result = [('tnsrOpA:0', 'colA'), + ('tnsrOpB:0', 'colB')] + + for valid_output_mapping in [valid_tnsr_output, valid_op_output]: + res = conv.asTensorNameToColumnMap(valid_output_mapping) + self.assertEqual(valid_output_mapping_result, res) + + + def test_invalid_input_mapping(self): + for invalid in [['a1', 'b2'], ('c3', 'd4'), [('a', 1), ('b', 2)]]: + with self.assertRaises(TypeError): + conv.asColumnToTensorNameMap(invalid) + conv.asTensorNameToColumnMap(invalid) + + with self.assertRaises(TypeError): + # Wrong value type: must be string + conv.asTensorNameToColumnMap({1: 'a', 2.0: 'b'}) + conv.asColumnToTensorNameMap({'a': 1, 'b': 2.0}) + + # Wrong containter type: only accept dict + conv.asColumnToTensorNameMap([('colA', 'tnsrA:0'), ('colB', 'tnsrB:0')]) + conv.asTensorNameToColumnMap([('tnsrA:0', 'colA'), ('tnsrB:0', 'colB')]) From b232b3c9f7a62467a84bd5a1f949bc951e7da1e9 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 18 Sep 2017 16:59:31 -0700 Subject: [PATCH 23/80] optimize graph for inference --- python/sparkdl/graph/builder.py | 8 ++++---- python/sparkdl/transformers/tf_tensor.py | 20 ++++++++++++++++++-- python/tests/transformers/tf_tensor_test.py | 4 +++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/python/sparkdl/graph/builder.py b/python/sparkdl/graph/builder.py index 86c3b3ce..67510fc7 100644 --- a/python/sparkdl/graph/builder.py +++ b/python/sparkdl/graph/builder.py @@ -47,19 +47,20 @@ def __init__(self, graph=None, using_keras=False): self.graph = graph or tf.Graph() self.sess = tf.Session(graph=self.graph) if using_keras: + self.using_keras = True self.keras_prev_sess = K.get_session() else: + self.using_keras = False self.keras_prev_sess = None def __enter__(self): - self.sess.as_default() self.sess.__enter__() - if self.keras_prev_sess is not None: + if self.using_keras: K.set_session(self.sess) return self def __exit__(self, *args): - if self.keras_prev_sess is not None: + if self.using_keras: K.set_session(self.keras_prev_sess) self.sess.__exit__(*args) @@ -268,4 +269,3 @@ def fromList(cls, functions): gfn = issn.asGraphFunction(first_inputs, last_outputs) return gfn - diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 0b4d5c2e..8b11d9af 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -16,6 +16,7 @@ import logging import tensorflow as tf +from tensorflow.python.tools import optimize_for_inference_lib as infr_opt import tensorframes as tfs from pyspark.ml import Transformer @@ -60,17 +61,32 @@ def setParams(self, tfInputGraph=None, inputMapping=None, outputMapping=None, tf # Further conanonicalization, e.g. converting dict to sorted str pairs happens here return self._set(**kwargs) - def _transform(self, dataset): + def _optimize_for_inference(self): + """ Optimize the graph for inference """ gin = self.getTFInputGraph() input_mapping = self.getInputMapping() output_mapping = self.getOutputMapping() + input_node_names = [tfx.as_op_name(tnsr_name) for _, tnsr_name in input_mapping] + output_node_names = [tfx.as_op_name(tnsr_name) for tnsr_name, _ in output_mapping] + + # NOTE(phi-dbq): Spark DataFrame assumes float64 as default floating point type + opt_gdef = infr_opt.optimize_for_inference(gin.graph_def, + input_node_names, + output_node_names, + tf.float64.as_datatype_enum) + return opt_gdef + + def _transform(self, dataset): + graph_def = self._optimize_for_inference() + input_mapping = self.getInputMapping() + output_mapping = self.getOutputMapping() graph = tf.Graph() with tf.Session(graph=graph): analyzed_df = tfs.analyze(dataset) out_tnsr_op_names = [tfx.as_op_name(tnsr_name) for tnsr_name, _ in output_mapping] - tf.import_graph_def(graph_def=gin.graph_def, name='', return_elements=out_tnsr_op_names) + tf.import_graph_def(graph_def=graph_def, name='', return_elements=out_tnsr_op_names) feed_dict = dict((tfx.op_name(graph, tnsr_name), col_name) for col_name, tnsr_name in input_mapping) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index fdeb42ea..c20a8e72 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -157,8 +157,10 @@ def _run_test_in_tf_session(self): _results.append(np.ravel(curr_res)) out_tgt = np.hstack(_results) + err_msg = 'not close => {} != {}, max_diff {}' self.assertTrue(np.allclose(out_ref, out_tgt), - msg='not close => {} != {}'.format(out_ref.shape, out_tgt.shape)) + msg=err_msg.format(out_ref.shape, out_tgt.shape, + np.max(np.abs(out_ref - out_tgt)))) def test_build_from_tf_graph(self): From d9213667f7fdc30bfb828a3643b6eb917ef09ce2 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 19 Sep 2017 11:19:15 -0700 Subject: [PATCH 24/80] more tests --- python/sparkdl/param/shared_params.py | 12 ++---------- python/tests/param/params_test.py | 9 ++++----- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index 890dc0b3..e606abce 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -202,11 +202,7 @@ class HasOutputMapping(Params): typeConverter=SparkDLTypeConverters.asTensorNameToColumnMap) def setOutputMapping(self, value): - # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the - # serializable TFInputGraph object once the inputMapping and outputMapping - # parameters are provided. - raise NotImplementedError( - "Please use the Transformer's constructor to assign `outputMapping` field.") + return self._set(outputMapping=value) def getOutputMapping(self): return self.getOrDefault(self.outputMapping) @@ -222,11 +218,7 @@ class HasInputMapping(Params): typeConverter=SparkDLTypeConverters.asColumnToTensorNameMap) def setInputMapping(self, value): - # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the - # serializable TFInputGraph object once the inputMapping and outputMapping - # parameters are provided. - raise NotImplementedError( - "Please use the Transformer's constructor to assigne `inputMapping` field.") + return self._set(inputMapping=value) def getInputMapping(self): return self.getOrDefault(self.inputMapping) diff --git a/python/tests/param/params_test.py b/python/tests/param/params_test.py index 0c10411a..74f09755 100644 --- a/python/tests/param/params_test.py +++ b/python/tests/param/params_test.py @@ -54,16 +54,15 @@ def test_tf_output_mapping_converter(self): def test_invalid_input_mapping(self): - for invalid in [['a1', 'b2'], ('c3', 'd4'), [('a', 1), ('b', 2)]]: + for invalid in [['a1', 'b2'], ('c3', 'd4'), [('a', 1), ('b', 2)], + {1: 'a', 2.0: 'b'}, {'a': 1, 'b': 2.0}]: with self.assertRaises(TypeError): conv.asColumnToTensorNameMap(invalid) conv.asTensorNameToColumnMap(invalid) with self.assertRaises(TypeError): - # Wrong value type: must be string - conv.asTensorNameToColumnMap({1: 'a', 2.0: 'b'}) - conv.asColumnToTensorNameMap({'a': 1, 'b': 2.0}) - # Wrong containter type: only accept dict conv.asColumnToTensorNameMap([('colA', 'tnsrA:0'), ('colB', 'tnsrB:0')]) + conv.asTensorNameToColumnMap([('colA', 'tnsrA:0'), ('colB', 'tnsrB:0')]) + conv.asColumnToTensorNameMap([('tnsrA:0', 'colA'), ('tnsrB:0', 'colB')]) conv.asTensorNameToColumnMap([('tnsrA:0', 'colA'), ('tnsrB:0', 'colB')]) From 0c8c21925af3b4ae0cc9bc1682fda8b41724152c Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 19 Sep 2017 13:55:15 -0700 Subject: [PATCH 25/80] update utils --- python/sparkdl/graph/utils.py | 54 +++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/python/sparkdl/graph/utils.py b/python/sparkdl/graph/utils.py index 45d8b065..8b10c307 100644 --- a/python/sparkdl/graph/utils.py +++ b/python/sparkdl/graph/utils.py @@ -16,8 +16,6 @@ import logging import six -import webbrowser -from tempfile import NamedTemporaryFile import tensorflow as tf @@ -95,31 +93,49 @@ def get_tensor(graph, tfobj_or_name): 'cannot locate tensor {} in current graph'.format(_tensor_name) return tnsr -def as_tensor_name(name): +def as_tensor_name(tfobj_or_name): """ Derive tf.Tensor name from an op/tensor name. - We do not check if the tensor exist (as no graph parameter is passed in). + If the input is a name, we do not check if the tensor exist + (as no graph parameter is passed in). - :param name: op name or tensor name + :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ - assert isinstance(name, six.string_types) - name_parts = name.split(":") - assert len(name_parts) <= 2, name_parts - if len(name_parts) < 2: - name += ":0" - return name + if isinstance(tfobj_or_name, six.string_types): + # If input is a string, assume it is a name and infer the corresponding tensor name. + # WARNING: this depends on TensorFlow's tensor naming convention + name = tfobj_or_name + name_parts = name.split(":") + assert len(name_parts) <= 2, name_parts + if len(name_parts) < 2: + name += ":0" + return name + elif hasattr(tfobj_or_name, 'graph'): + tfobj = tfobj_or_name + return get_tensor(tfobj.graph, tfobj).name + else: + raise TypeError('invalid tf.Tensor name query type {}'.format(type(tfobj_or_name))) -def as_op_name(name): +def as_op_name(tfobj_or_name): """ - Derive tf.Operation name from an op/tensor name - We do not check if the operation exist (as no graph parameter is passed in). + Derive tf.Operation name from an op/tensor name. + If the input is a name, we do not check if the operation exist + (as no graph parameter is passed in). - :param name: op name or tensor name + :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ - assert isinstance(name, six.string_types) - name_parts = name.split(":") - assert len(name_parts) <= 2, name_parts - return name_parts[0] + if isinstance(tfobj_or_name, six.string_types): + # If input is a string, assume it is a name and infer the corresponding operation name. + # WARNING: this depends on TensorFlow's operation naming convention + name = tfobj_or_name + name_parts = name.split(":") + assert len(name_parts) <= 2, name_parts + return name_parts[0] + elif hasattr(tfobj_or_name, 'graph'): + tfobj = tfobj_or_name + return get_op(tfobj.graph, tfobj).name + else: + raise TypeError('invalid tf.Operation name query type {}'.format(type(tfobj_or_name))) def op_name(graph, tfobj_or_name): """ From 522279ac739ede50d29ca7e5e9deb5681b77040f Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 19 Sep 2017 14:19:06 -0700 Subject: [PATCH 26/80] tests --- python/tests/graph/test_utils.py | 49 ++++++++++++++++++++++++++++++++ python/tests/tests.py | 3 ++ 2 files changed, 52 insertions(+) create mode 100644 python/tests/graph/test_utils.py diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py new file mode 100644 index 00000000..de75036f --- /dev/null +++ b/python/tests/graph/test_utils.py @@ -0,0 +1,49 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import tensorflow as tf + +import sparkdl.graph.utils as tfx + +from ..tests import PythonUnitTestCase + +class TFeXtensionGraphUtilsTest(PythonUnitTestCase): + + def test_infer_graph_element_names(self): + for tnsr_idx in range(17): + op_name = 'someOp' + tnsr_name = '{}:{}'.format(op_name, tnsr_idx) + self.assertEqual(op_name, tfx.as_op_name(tnsr_name)) + self.assertEqual(tnsr_name, tfx.as_tensor_name(tnsr_name)) + + with self.assertRaises(TypeError): + for wrong_value in [7, 1.2, tf.Graph()]: + tfx.as_op_name(wrong_value) + tfx.as_tensor_name(wrong_value) + + def test_get_graph_elements(self): + op_name = 'someConstOp' + tnsr_name = '{}:0'.format(op_name) + tnsr = tf.constant(1427.08, name=op_name) + graph = tnsr.graph + + self.assertEqual(op_name, tfx.as_op_name(tnsr)) + self.assertEqual(op_name, tfx.op_name(graph, tnsr)) + self.assertEqual(tnsr_name, tfx.as_tensor_name(tnsr)) + self.assertEqual(tnsr_name, tfx.tensor_name(graph, tnsr)) + self.assertEqual(tnsr, tfx.get_tensor(graph, tnsr)) + self.assertEqual(tnsr.op, tfx.get_op(graph, tnsr)) + self.assertEqual(graph, tfx.get_op(graph, tnsr).graph) + self.assertEqual(graph, tfx.get_tensor(graph, tnsr).graph) diff --git a/python/tests/tests.py b/python/tests/tests.py index d93b31a8..ae7cec3e 100644 --- a/python/tests/tests.py +++ b/python/tests/tests.py @@ -29,6 +29,9 @@ from pyspark.sql import SQLContext from pyspark.sql import SparkSession +class PythonUnitTestCase(unittest.TestCase): + # Just the plain test unittest.TestCase, but won't have to do import check + pass class SparkDLTestCase(unittest.TestCase): From f4d938cd3345024750329925238923e33f40666c Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 19 Sep 2017 14:22:17 -0700 Subject: [PATCH 27/80] intro: TFInputGraph --- python/sparkdl/graph/input.py | 254 ++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 python/sparkdl/graph/input.py diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py new file mode 100644 index 00000000..53b923c7 --- /dev/null +++ b/python/sparkdl/graph/input.py @@ -0,0 +1,254 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import absolute_import, division, print_function + +import tensorflow as tf +from tensorflow.core.protobuf import meta_graph_pb2 + +import sparkdl.graph.utils as tfx + +__all__ = ["TFInputGraph"] + +class TFInputGraph(object): + """ + An opaque serializable object containing TensorFlow graph. + + [WARNING] This class should not be called by any user code. + """ + def __init__(self): + raise NotImplementedError( + "Please do NOT construct TFInputGraph directly. Instead, use one of the helper functions") + + @classmethod + def _new_obj_internal(cls): + # pylint: disable=attribute-defined-outside-init + obj = object.__new__(cls) + # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. + obj.graph_def = None + obj.input_tensor_name_from_signature = None + obj.output_tensor_name_from_signature = None + return obj + + def translateInputMapping(self, input_mapping): + assert self.input_tensor_name_from_signature is not None + _input_mapping = {} + if isinstance(input_mapping, dict): + input_mapping = list(input_mapping.items()) + assert isinstance(input_mapping, list) + for col_name, sig_key in input_mapping: + tnsr_name = self.input_tensor_name_from_signature[sig_key] + _input_mapping[col_name] = tnsr_name + return _input_mapping + + def translateOutputMapping(self, output_mapping): + assert self.output_tensor_name_from_signature is not None + _output_mapping = {} + if isinstance(output_mapping, dict): + output_mapping = list(output_mapping.items()) + assert isinstance(output_mapping, list) + for sig_key, col_name in output_mapping: + tnsr_name = self.output_tensor_name_from_signature[sig_key] + _output_mapping[tnsr_name] = col_name + return _output_mapping + + @classmethod + def fromGraph(cls, graph, sess, feed_names, fetch_names): + """ + Construct a TFInputGraphBuilder from a in memory tf.Graph object + """ + assert isinstance(graph, tf.Graph), \ + ('expect tf.Graph type but got', type(graph)) + + def import_graph_fn(_sess): + assert _sess == sess, 'must have the same session' + return _GinBuilderInfo() + + return _GinBuilder(import_graph_fn, sess, graph).build(feed_names, fetch_names) + + @classmethod + def fromGraphDef(cls, graph_def, feed_names, fetch_names): + """ + Construct a TFInputGraphBuilder from a tf.GraphDef object + """ + assert isinstance(graph_def, tf.GraphDef), \ + ('expect tf.GraphDef type but got', type(graph_def)) + + def import_graph_fn(sess): + with sess.as_default(): + tf.import_graph_def(graph_def, name='') + return _GinBuilderInfo() + + return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) + + @classmethod + def fromCheckpoint(cls, checkpoint_dir, feed_names, fetch_names): + return cls._from_checkpoint_impl(checkpoint_dir, + signature_def_key=None, + feed_names=feed_names, fetch_names=fetch_names) + + @classmethod + def fromCheckpointWithSignature(cls, checkpoint_dir, signature_def_key): + assert signature_def_key is not None + return cls._from_checkpoint_impl(checkpoint_dir, + signature_def_key, + feed_names=None, fetch_names=None) + + @classmethod + def fromSavedModel(cls, saved_model_dir, tag_set, feed_names, fetch_names): + return cls._from_saved_model_impl(saved_model_dir, tag_set, + signature_def_key=None, + feed_names=feed_names, fetch_names=fetch_names) + + @classmethod + def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key): + assert signature_def_key is not None + return cls._from_saved_model_impl(saved_model_dir, tag_set, + signature_def_key=signature_def_key, + feed_names=None, fetch_names=None) + + @classmethod + def _from_checkpoint_impl(cls, + checkpoint_dir, + signature_def_key=None, + feed_names=None, + fetch_names=None): + """ + Construct a TFInputGraphBuilder from a model checkpoint + """ + assert (feed_names is None) == (fetch_names is None), \ + 'feed_names and fetch_names, if provided must appear together' + assert (feed_names is None) != (signature_def_key is None), \ + 'must either provide feed_names or singnature_def_key' + + def import_graph_fn(sess): + # Load checkpoint and import the graph + with sess.as_default(): + ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) + + # NOTE(phi-dbq): we must manually load meta_graph_def to get the signature_def + # the current `import_graph_def` function seems to ignore + # any signature_def fields in a checkpoint's meta_graph_def. + meta_graph_def = meta_graph_pb2.MetaGraphDef() + with open("{}.meta".format(ckpt_path), 'rb') as fin: + meta_graph_def.ParseFromString(fin.read()) + + saver = tf.train.import_meta_graph(meta_graph_def, clear_devices=True) + saver.restore(sess, ckpt_path) + + sig_def = None + if signature_def_key is not None: + sig_def = meta_graph_def.signature_def[signature_def_key] + assert sig_def, 'singnature_def_key {} provided, '.format(signature_def_key) + \ + 'but failed to find it from the meta_graph_def ' + \ + 'from checkpoint {}'.format(checkpoint_dir) + + return _GinBuilderInfo(sig_def=sig_def) + + return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) + + @classmethod + def _from_saved_model_impl(cls, saved_model_dir, tag_set, + signature_def_key=None, + feed_names=None, + fetch_names=None): + """ + Construct a TFInputGraphBuilder from a SavedModel + """ + assert (feed_names is None) == (fetch_names is None), \ + 'feed_names and fetch_names, if provided must appear together' + assert (feed_names is None) != (signature_def_key is None), \ + 'must either provide feed_names or singnature_def_key' + + def import_graph_fn(sess): + tag_sets = tag_set.split(',') + meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) + + sig_def = None + if signature_def_key is not None: + sig_def = tf.contrib.saved_model.get_signature_def_by_key( + meta_graph_def, signature_def_key) + + return _GinBuilderInfo(sig_def=sig_def) + + return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) + + +class _GinBuilderInfo(object): + def __init__(self, sig_def=None): + self.sig_def = sig_def + self.feed_names = None + self.feed_mapping = None + self.fetch_names = None + self.fetch_mapping = None + + def extract_signatures(self): + assert self.sig_def is not None, \ + "ask to find sigdef mapping, but not found any" + + self.feed_mapping = {} + self.feed_names = [] + for sigdef_key, tnsr_info in self.sig_def.inputs.items(): + tnsr_name = tnsr_info.name + self.feed_mapping[sigdef_key] = tnsr_name + self.feed_names.append(tnsr_name) + + self.fetch_mapping = {} + self.fetch_names = [] + for sigdef_key, tnsr_info in self.sig_def.outputs.items(): + tnsr_name = tnsr_info.name + self.fetch_mapping[sigdef_key] = tnsr_name + self.fetch_names.append(tnsr_name) + +class _GinBuilder(object): + def __init__(self, import_graph_fn, sess=None, graph=None): + self.import_graph_fn = import_graph_fn + assert (sess is None) == (graph is None) + if sess is not None: + self.graph = graph + self.sess = sess + self._should_clean = False + else: + self.graph = tf.Graph() + self.sess = tf.Session(graph=self.graph) + self._should_clean = True + + def _build_impl(self, feed_names, fetch_names): + # pylint: disable=protected-access,attribute-defined-outside-init + gin = TFInputGraph._new_obj_internal() + assert (feed_names is None) == (fetch_names is None) + must_have_sig_def = fetch_names is None + # NOTE(phi-dbq): both have to be set to default + with self.sess.as_default(), self.graph.as_default(): + _ginfo = self.import_graph_fn(self.sess) + if must_have_sig_def: + _ginfo.extract_signatures() + feed_names = _ginfo.feed_names + fetch_names = _ginfo.fetch_names + gin.input_tensor_name_from_signature = _ginfo.feed_mapping + gin.output_tensor_name_from_signature = _ginfo.fetch_mapping + + for tnsr_name in feed_names: + assert tfx.get_op(self.graph, tnsr_name) + fetches = [tfx.get_tensor(self.graph, tnsr_name) for tnsr_name in fetch_names] + gin.graph_def = tfx.strip_and_freeze_until(fetches, self.graph, self.sess) + return gin + + def build(self, feed_names=None, fetch_names=None): + try: + gin = self._build_impl(feed_names, fetch_names) + finally: + if self._should_clean: + self.sess.close() + return gin From cd3aa8def8daea75ec0872069f75fa501335eb50 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 19 Sep 2017 15:17:28 -0700 Subject: [PATCH 28/80] tests --- python/sparkdl/graph/input.py | 22 --- python/tests/graph/test_input_graph.py | 223 +++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 22 deletions(-) create mode 100644 python/tests/graph/test_input_graph.py diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 53b923c7..51fd017c 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -41,28 +41,6 @@ def _new_obj_internal(cls): obj.output_tensor_name_from_signature = None return obj - def translateInputMapping(self, input_mapping): - assert self.input_tensor_name_from_signature is not None - _input_mapping = {} - if isinstance(input_mapping, dict): - input_mapping = list(input_mapping.items()) - assert isinstance(input_mapping, list) - for col_name, sig_key in input_mapping: - tnsr_name = self.input_tensor_name_from_signature[sig_key] - _input_mapping[col_name] = tnsr_name - return _input_mapping - - def translateOutputMapping(self, output_mapping): - assert self.output_tensor_name_from_signature is not None - _output_mapping = {} - if isinstance(output_mapping, dict): - output_mapping = list(output_mapping.items()) - assert isinstance(output_mapping, list) - for sig_key, col_name in output_mapping: - tnsr_name = self.output_tensor_name_from_signature[sig_key] - _output_mapping[tnsr_name] = col_name - return _output_mapping - @classmethod def fromGraph(cls, graph, sess, feed_names, fetch_names): """ diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py new file mode 100644 index 00000000..d16d62eb --- /dev/null +++ b/python/tests/graph/test_input_graph.py @@ -0,0 +1,223 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import absolute_import, division, print_function + +from contextlib import contextmanager +from glob import glob +import os +import shutil +import tempfile + +import numpy as np +import tensorflow as tf + +from sparkdl.graph.input import * +import sparkdl.graph.utils as tfx + +from ..tests import PythonUnitTestCase + + +class TFInputGraphTest(PythonUnitTestCase): + + def setUp(self): + self.vec_size = 23 + self.num_samples = 107 + + self.input_col = 'dfInputCol' + self.input_op_name = 'tnsrOpIn' + self.output_col = 'dfOutputCol' + self.output_op_name = 'tnsrOpOut' + + self.feed_names = [] + self.fetch_names = [] + self.input_mapping = {} + self.output_mapping = {} + self.setup_iomap(replica=1) + + self.input_graphs = [] + self.test_case_results = [] + # Build a temporary directory, which might or might not be used by the test + self.model_output_root = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.model_output_root, ignore_errors=True) + + def setup_iomap(self, replica=1): + self.input_mapping = {} + self.feed_names = [] + self.output_mapping = {} + self.fetch_names = [] + + if replica > 1: + for i in range(replica): + colname = '{}_replica{:03d}'.format(self.input_col, i) + tnsr_op_name = '{}_replica{:03d}'.format(self.input_op_name, i) + self.input_mapping[colname] = tnsr_op_name + self.feed_names.append(tnsr_op_name + ':0') + + colname = '{}_replica{:03d}'.format(self.output_col, i) + tnsr_op_name = '{}_replica{:03d}'.format(self.output_op_name, i) + self.output_mapping[tnsr_op_name] = colname + self.fetch_names.append(tnsr_op_name + ':0') + else: + self.input_mapping = {self.input_col: self.input_op_name} + self.feed_names = [self.input_op_name + ':0'] + self.output_mapping = {self.output_op_name: self.output_col} + self.fetch_names = [self.output_op_name + ':0'] + + @contextmanager + def _run_test_in_tf_session(self): + """ [THIS IS NOT A TEST]: encapsulate general test workflow """ + + # Build the TensorFlow graph + graph = tf.Graph() + with tf.Session(graph=graph) as sess, graph.as_default(): + # Build test graph and transformers from here + yield sess + + ref_feed = tfx.get_tensor(graph, self.input_op_name) + ref_fetch = tfx.get_tensor(graph, self.output_op_name) + + def check_input_graph(tgt_gdef, test_idx): + namespace = 'TEST_TGT_NS{:03d}'.format(test_idx) + tf.import_graph_def(tgt_gdef, name=namespace) + tgt_feed = tfx.get_tensor(graph, '{}/{}'.format(namespace, self.input_op_name)) + tgt_fetch = tfx.get_tensor(graph, '{}/{}'.format(namespace, self.output_op_name)) + + for _ in range(10): + local_data = np.random.randn(31, self.vec_size) + ref_out = sess.run(ref_fetch, feed_dict={ref_feed: local_data}) + tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: local_data}) + self.assertTrue(np.allclose(ref_out, tgt_out)) + + for test_idx, input_graph in enumerate(self.input_graphs): + check_input_graph(input_graph.graph_def, test_idx) + + + def test_build_from_tf_graph(self): + """ Build TFTransformer from tf.Graph """ + with self._run_test_in_tf_session() as sess: + # Begin building graph + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) + + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + self.input_graphs.append(gin) + # End building graph + + def test_build_from_saved_model(self): + """ Build TFTransformer from saved model """ + # Setup saved model export directory + saved_model_root = self.model_output_root + saved_model_dir = os.path.join(saved_model_root, 'saved_model') + serving_tag = "serving_tag" + serving_sigdef_key = 'prediction_signature' + builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) + + with self._run_test_in_tf_session() as sess: + # Model definition: begin + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), + dtype=tf.float64, name='varW') + z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + # Model definition ends + + sess.run(w.initializer) + + sig_inputs = { + 'input_sig': tf.saved_model.utils.build_tensor_info(x)} + sig_outputs = { + 'output_sig': tf.saved_model.utils.build_tensor_info(z)} + + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs=sig_inputs, + outputs=sig_outputs) + + builder.add_meta_graph_and_variables(sess, + [serving_tag], + signature_def_map={ + serving_sigdef_key: serving_sigdef}) + builder.save() + + # Build the transformer from exported serving model + # We are using signaures, thus must provide the keys + gin = TFInputGraph.fromSavedModelWithSignature( + saved_model_dir, serving_tag, serving_sigdef_key) + self.input_graphs.append(gin) + + # Build the transformer from exported serving model + # We are not using signatures, thus must provide tensor/operation names + gin = TFInputGraph.fromSavedModel( + saved_model_dir, serving_tag, self.feed_names, self.fetch_names) + self.input_graphs.append(gin) + + gin = TFInputGraph.fromGraph( + sess.graph, sess, self.feed_names, self.fetch_names) + self.input_graphs.append(gin) + + + def test_build_from_checkpoint(self): + """ Build TFTransformer from a model checkpoint """ + # Build the TensorFlow graph + model_ckpt_dir = self.model_output_root + ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') + serving_sigdef_key = 'prediction_signature' + + with self._run_test_in_tf_session() as sess: + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) + w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), + dtype=tf.float64, name='varW') + z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + sess.run(w.initializer) + saver = tf.train.Saver(var_list=[w]) + _ = saver.save(sess, ckpt_path_prefix, global_step=2702) + + # Prepare the signature_def + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs={ + 'input_sig': tf.saved_model.utils.build_tensor_info(x) + }, + outputs={ + 'output_sig': tf.saved_model.utils.build_tensor_info(z) + }) + + # A rather contrived way to add signature def to a meta_graph + meta_graph_def = tf.train.export_meta_graph() + + # Find the meta_graph file (there should be only one) + _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) + self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) + ckpt_meta_fpath = _ckpt_meta_fpaths[0] + + # Add signature_def to the meta_graph and serialize it + # This will overwrite the existing meta_graph_def file + meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) + with open(ckpt_meta_fpath, mode='wb') as fout: + fout.write(meta_graph_def.SerializeToString()) + + # Build the transformer from exported serving model + # We are using signaures, thus must provide the keys + gin = TFInputGraph.fromCheckpointWithSignature( + model_ckpt_dir, serving_sigdef_key) + self.input_graphs.append(gin) + + # Transformer without using signature_def + gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) + self.input_graphs.append(gin) + + gin = TFInputGraph.fromGraph( + sess.graph, sess, self.feed_names, self.fetch_names) + self.input_graphs.append(gin) From 07c58e62b9c22d40ff50a0679db3f6dde248f22c Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 20 Sep 2017 17:28:51 -0700 Subject: [PATCH 29/80] allows setting TFInputGraph --- python/sparkdl/param/shared_params.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index b1ede605..00478b85 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -239,11 +239,7 @@ def __init__(self): self._setDefault(tfInputGraph=None) def setTFInputGraph(self, value): - # NOTE(phi-dbq): due to the nature of TensorFlow import modes, we can only derive the - # serializable TFInputGraph object once the inputMapping and outputMapping - # parameters are provided. - raise NotImplementedError( - "Please use the Transformer's constructor to assign `tfInputGraph` field.") + return self._set(tfInputGraph=value) def getTFInputGraph(self): return self.getOrDefault(self.tfInputGraph) From 269ad15e882951f2f480dce955ccf898d016d319 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 20 Sep 2017 17:48:34 -0700 Subject: [PATCH 30/80] utilize test_input_graph for transformer tests --- python/tests/graph/test_input_graph.py | 6 +- python/tests/transformers/tf_tensor_test.py | 285 ++++---------------- 2 files changed, 58 insertions(+), 233 deletions(-) diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index d16d62eb..950240dd 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -107,7 +107,7 @@ def check_input_graph(tgt_gdef, test_idx): def test_build_from_tf_graph(self): - """ Build TFTransformer from tf.Graph """ + """ Build TFInputGraph from tf.Graph """ with self._run_test_in_tf_session() as sess: # Begin building graph x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) @@ -118,7 +118,7 @@ def test_build_from_tf_graph(self): # End building graph def test_build_from_saved_model(self): - """ Build TFTransformer from saved model """ + """ Build TFInputGraph from saved model """ # Setup saved model export directory saved_model_root = self.model_output_root saved_model_dir = os.path.join(saved_model_root, 'saved_model') @@ -169,7 +169,7 @@ def test_build_from_saved_model(self): def test_build_from_checkpoint(self): - """ Build TFTransformer from a model checkpoint """ + """ Build TFInputGraph from a model checkpoint """ # Build the TensorFlow graph model_ckpt_dir = self.model_output_root ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index c20a8e72..57e40f71 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -33,33 +33,10 @@ from sparkdl.transformers.tf_tensor import TFTransformer from ..tests import SparkDLTestCase +from ..graph.test_input_graph import TFInputGraphTest -class TFTransformerTest(SparkDLTestCase): - - def setUp(self): - self.vec_size = 17 - self.num_vecs = 31 - - self.input_col = 'vec' - self.input_op_name = 'tnsrOpIn' - self.output_col = 'outputCol' - self.output_op_name = 'tnsrOpOut' - - self.feed_names = [] - self.fetch_names = [] - self.input_mapping = {} - self.output_mapping = {} - self.setup_iomap(replica=1) - - self.transformers = [] - self.test_case_results = [] - # Build a temporary directory, which might or might not be used by the test - self.model_output_root = tempfile.mkdtemp() - - - def tearDown(self): - shutil.rmtree(self.model_output_root, ignore_errors=True) +class TFTransformerTest(TFInputGraphTest, SparkDLTestCase): def _build_default_session_tests(self, sess): gin = TFInputGraph.fromGraph( @@ -82,36 +59,14 @@ def _add_transformer(imap, omap): for op_name, col in self.output_mapping.items()) _add_transformer(imap, omap) - def setup_iomap(self, replica=1): - self.input_mapping = {} - self.feed_names = [] - self.output_mapping = {} - self.fetch_names = [] - - if replica > 1: - for i in range(replica): - colname = '{}_replica{:03d}'.format(self.input_col, i) - tnsr_op_name = '{}_replica{:03d}'.format(self.input_op_name, i) - self.input_mapping[colname] = tnsr_op_name - self.feed_names.append(tnsr_op_name + ':0') - - colname = '{}_replica{:03d}'.format(self.output_col, i) - tnsr_op_name = '{}_replica{:03d}'.format(self.output_op_name, i) - self.output_mapping[tnsr_op_name] = colname - self.fetch_names.append(tnsr_op_name + ':0') - else: - self.input_mapping = {self.input_col: self.input_op_name} - self.feed_names = [self.input_op_name + ':0'] - self.output_mapping = {self.output_op_name: self.output_col} - self.fetch_names = [self.output_op_name + ':0'] - @contextmanager def _run_test_in_tf_session(self): """ [THIS IS NOT A TEST]: encapsulate general test workflow """ # Build local features and DataFrame from it + print("OVERRIDING default", repr(self.__class__)) local_features = [] - for idx in range(self.num_vecs): + for idx in range(self.num_samples): _dict = {'idx': idx} for colname, _ in self.input_mapping.items(): _dict[colname] = np.random.randn(self.vec_size).tolist() @@ -142,9 +97,16 @@ def _run_test_in_tf_session(self): out_ref = np.hstack(_results) + # We have sessions, now create transformers out of them + # Apply the transform - for transfomer in self.transformers: - out_df = transfomer.transform(analyzed_df) + for input_graph in self.input_graphs: + transformer = TFTransformer(tfInputGraph=input_graph, + inputMapping=self.input_mapping, + outputMapping=self.output_mapping) + print('built transformer', repr(transformer)) + + out_df = transformer.transform(analyzed_df) out_colnames = [] for old_colname, new_colname in self.output_mapping.items(): out_colnames.append(new_colname) @@ -163,182 +125,45 @@ def _run_test_in_tf_session(self): np.max(np.abs(out_ref - out_tgt)))) - def test_build_from_tf_graph(self): - """ Build TFTransformer from tf.Graph """ - with self._run_test_in_tf_session() as sess: - # Begin building graph - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) - # End building graph - - self._build_default_session_tests(sess) - - - def test_build_from_saved_model(self): - """ Build TFTransformer from saved model """ - # Setup saved model export directory - saved_model_root = self.model_output_root - saved_model_dir = os.path.join(saved_model_root, 'saved_model') - serving_tag = "serving_tag" - serving_sigdef_key = 'prediction_signature' - builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - - with self._run_test_in_tf_session() as sess: - # Model definition: begin - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), - dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - # Model definition ends - - sess.run(w.initializer) - - sig_inputs = { - 'input_sig': tf.saved_model.utils.build_tensor_info(x)} - sig_outputs = { - 'output_sig': tf.saved_model.utils.build_tensor_info(z)} - - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs=sig_inputs, - outputs=sig_outputs) - - builder.add_meta_graph_and_variables(sess, - [serving_tag], - signature_def_map={ - serving_sigdef_key: serving_sigdef}) - builder.save() - - # Build the transformer from exported serving model - # We are using signaures, thus must provide the keys - tfInputGraph = TFInputGraph.fromSavedModelWithSignature( - saved_model_dir, serving_tag, serving_sigdef_key) - - inputMapping = tfInputGraph.translateInputMapping({ - self.input_col: 'input_sig' - }) - outputMapping = tfInputGraph.translateOutputMapping({ - 'output_sig': self.output_col - }) - trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, - inputMapping=inputMapping, - outputMapping=outputMapping) - self.transformers.append(trans_with_sig) - - # Build the transformer from exported serving model - # We are not using signatures, thus must provide tensor/operation names - gin = TFInputGraph.fromSavedModel( - saved_model_dir, serving_tag, self.feed_names, self.fetch_names) - self.build_standard_transformers(sess, gin) - - gin = TFInputGraph.fromGraph( - sess.graph, sess, self.feed_names, self.fetch_names) - self.build_standard_transformers(sess, gin) - - - def test_build_from_checkpoint(self): - """ Build TFTransformer from a model checkpoint """ - # Build the TensorFlow graph - model_ckpt_dir = self.model_output_root - ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - serving_sigdef_key = 'prediction_signature' - - with self._run_test_in_tf_session() as sess: - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), - dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - sess.run(w.initializer) - saver = tf.train.Saver(var_list=[w]) - _ = saver.save(sess, ckpt_path_prefix, global_step=2702) - - # Prepare the signature_def - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs={ - 'input_sig': tf.saved_model.utils.build_tensor_info(x) - }, - outputs={ - 'output_sig': tf.saved_model.utils.build_tensor_info(z) - }) - - # A rather contrived way to add signature def to a meta_graph - meta_graph_def = tf.train.export_meta_graph() - - # Find the meta_graph file (there should be only one) - _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) - self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) - ckpt_meta_fpath = _ckpt_meta_fpaths[0] - - # Add signature_def to the meta_graph and serialize it - # This will overwrite the existing meta_graph_def file - meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) - with open(ckpt_meta_fpath, mode='wb') as fout: - fout.write(meta_graph_def.SerializeToString()) - - # Build the transformer from exported serving model - # We are using signaures, thus must provide the keys - tfInputGraph = TFInputGraph.fromCheckpointWithSignature( - model_ckpt_dir, serving_sigdef_key) - - inputMapping = tfInputGraph.translateInputMapping({ - self.input_col: 'input_sig' - }) - outputMapping = tfInputGraph.translateOutputMapping({ - 'output_sig': self.output_col - }) - trans_with_sig = TFTransformer(tfInputGraph=tfInputGraph, - inputMapping=inputMapping, - outputMapping=outputMapping) - self.transformers.append(trans_with_sig) - - # Transformer without using signature_def - gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) - self.build_standard_transformers(sess, gin) - - gin = TFInputGraph.fromGraph( - sess.graph, sess, self.feed_names, self.fetch_names) - self.build_standard_transformers(sess, gin) - - - def test_multi_io(self): - """ Build TFTransformer with multiple I/O tensors """ - self.setup_iomap(replica=3) - with self._run_test_in_tf_session() as sess: - xs = [] - for tnsr_op_name in self.input_mapping.values(): - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) - xs.append(x) - - zs = [] - for i, tnsr_op_name in enumerate(self.output_mapping.keys()): - z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) - zs.append(z) - - self._build_default_session_tests(sess) - - - def test_mixed_keras_graph(self): - """ Build mixed keras graph """ - with IsolatedSession(using_keras=True) as issn: - tnsr_in = tf.placeholder( - tf.double, shape=[None, self.vec_size], name=self.input_op_name) - inp = tf.expand_dims(tnsr_in, axis=2) - # Keras layers does not take tf.double - inp = tf.cast(inp, tf.float32) - conv = Conv1D(filters=4, kernel_size=2)(inp) - pool = MaxPool1D(pool_size=2)(conv) - flat = Flatten()(pool) - dense = Dense(1)(flat) - # We must keep the leading dimension of the output - redsum = tf.reduce_logsumexp(dense, axis=1) - tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) - - # Initialize the variables - init_op = tf.global_variables_initializer() - issn.run(init_op) - # We could train the model ... but skip it here - gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) - - with self._run_test_in_tf_session() as sess: - tf.import_graph_def(gfn.graph_def, name='') - self._build_default_session_tests(sess) + # def test_multi_io(self): + # """ Build TFTransformer with multiple I/O tensors """ + # self.setup_iomap(replica=3) + # with self._run_test_in_tf_session() as sess: + # xs = [] + # for tnsr_op_name in self.input_mapping.values(): + # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) + # xs.append(x) + + # zs = [] + # for i, tnsr_op_name in enumerate(self.output_mapping.keys()): + # z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) + # zs.append(z) + + # self._build_default_session_tests(sess) + + + # def test_mixed_keras_graph(self): + # """ Build mixed keras graph """ + # with IsolatedSession(using_keras=True) as issn: + # tnsr_in = tf.placeholder( + # tf.double, shape=[None, self.vec_size], name=self.input_op_name) + # inp = tf.expand_dims(tnsr_in, axis=2) + # # Keras layers does not take tf.double + # inp = tf.cast(inp, tf.float32) + # conv = Conv1D(filters=4, kernel_size=2)(inp) + # pool = MaxPool1D(pool_size=2)(conv) + # flat = Flatten()(pool) + # dense = Dense(1)(flat) + # # We must keep the leading dimension of the output + # redsum = tf.reduce_logsumexp(dense, axis=1) + # tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) + + # # Initialize the variables + # init_op = tf.global_variables_initializer() + # issn.run(init_op) + # # We could train the model ... but skip it here + # gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) + + # with self._run_test_in_tf_session() as sess: + # tf.import_graph_def(gfn.graph_def, name='') + # self._build_default_session_tests(sess) From 84a813823185f5500e2cb92a46b421b664aad3fc Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 20 Sep 2017 18:23:12 -0700 Subject: [PATCH 31/80] enable all tests Signed-off-by: Philip Yang --- python/tests/graph/test_input_graph.py | 42 ++++-- python/tests/transformers/tf_tensor_test.py | 151 ++++++++++---------- 2 files changed, 102 insertions(+), 91 deletions(-) diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index 950240dd..f66c4a2b 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -36,14 +36,23 @@ def setUp(self): self.num_samples = 107 self.input_col = 'dfInputCol' - self.input_op_name = 'tnsrOpIn' self.output_col = 'dfOutputCol' + + self.input_op_name = 'tnsrOpIn' self.output_op_name = 'tnsrOpOut' + self.input_graph_with_signature = set() + self.input_sigkey = 'well_known_input_sig' + self.output_sigkey = 'well_known_output_sig' + self.serving_tag = "serving_tag" + self.serving_sigdef_key = 'prediction_signature' + self.feed_names = [] self.fetch_names = [] self.input_mapping = {} self.output_mapping = {} + self.sig_input_mapping = {} + self.sig_output_mapping = {} self.setup_iomap(replica=1) self.input_graphs = [] @@ -64,17 +73,23 @@ def setup_iomap(self, replica=1): for i in range(replica): colname = '{}_replica{:03d}'.format(self.input_col, i) tnsr_op_name = '{}_replica{:03d}'.format(self.input_op_name, i) + sig_name = '{}_replica{:03d}'.format(self.input_sigkey, i) self.input_mapping[colname] = tnsr_op_name + self.sig_input_mapping[colname] = sig_name self.feed_names.append(tnsr_op_name + ':0') colname = '{}_replica{:03d}'.format(self.output_col, i) tnsr_op_name = '{}_replica{:03d}'.format(self.output_op_name, i) + sig_name = '{}_replica{:03d}'.format(self.output_sigkey, i) self.output_mapping[tnsr_op_name] = colname + self.sig_output_mapping[sig_name] = colname self.fetch_names.append(tnsr_op_name + ':0') else: self.input_mapping = {self.input_col: self.input_op_name} + self.sig_input_mapping = {self.input_col: self.input_sigkey} self.feed_names = [self.input_op_name + ':0'] self.output_mapping = {self.output_op_name: self.output_col} + self.sig_output_mapping = {self.output_sigkey: self.output_col} self.fetch_names = [self.output_op_name + ':0'] @contextmanager @@ -122,8 +137,6 @@ def test_build_from_saved_model(self): # Setup saved model export directory saved_model_root = self.model_output_root saved_model_dir = os.path.join(saved_model_root, 'saved_model') - serving_tag = "serving_tag" - serving_sigdef_key = 'prediction_signature' builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) with self._run_test_in_tf_session() as sess: @@ -137,30 +150,31 @@ def test_build_from_saved_model(self): sess.run(w.initializer) sig_inputs = { - 'input_sig': tf.saved_model.utils.build_tensor_info(x)} + self.input_sigkey: tf.saved_model.utils.build_tensor_info(x)} sig_outputs = { - 'output_sig': tf.saved_model.utils.build_tensor_info(z)} + self.output_sigkey: tf.saved_model.utils.build_tensor_info(z)} serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( inputs=sig_inputs, outputs=sig_outputs) builder.add_meta_graph_and_variables(sess, - [serving_tag], + [self.serving_tag], signature_def_map={ - serving_sigdef_key: serving_sigdef}) + self.serving_sigdef_key: serving_sigdef}) builder.save() # Build the transformer from exported serving model # We are using signaures, thus must provide the keys gin = TFInputGraph.fromSavedModelWithSignature( - saved_model_dir, serving_tag, serving_sigdef_key) + saved_model_dir, self.serving_tag, self.serving_sigdef_key) self.input_graphs.append(gin) + self.input_graph_with_signature.add(gin) # Build the transformer from exported serving model # We are not using signatures, thus must provide tensor/operation names gin = TFInputGraph.fromSavedModel( - saved_model_dir, serving_tag, self.feed_names, self.fetch_names) + saved_model_dir, self.serving_tag, self.feed_names, self.fetch_names) self.input_graphs.append(gin) gin = TFInputGraph.fromGraph( @@ -173,7 +187,6 @@ def test_build_from_checkpoint(self): # Build the TensorFlow graph model_ckpt_dir = self.model_output_root ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - serving_sigdef_key = 'prediction_signature' with self._run_test_in_tf_session() as sess: x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) @@ -188,10 +201,10 @@ def test_build_from_checkpoint(self): # Prepare the signature_def serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( inputs={ - 'input_sig': tf.saved_model.utils.build_tensor_info(x) + self.input_sigkey: tf.saved_model.utils.build_tensor_info(x) }, outputs={ - 'output_sig': tf.saved_model.utils.build_tensor_info(z) + self.output_sigkey: tf.saved_model.utils.build_tensor_info(z) }) # A rather contrived way to add signature def to a meta_graph @@ -204,15 +217,16 @@ def test_build_from_checkpoint(self): # Add signature_def to the meta_graph and serialize it # This will overwrite the existing meta_graph_def file - meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) + meta_graph_def.signature_def[self.serving_sigdef_key].CopyFrom(serving_sigdef) with open(ckpt_meta_fpath, mode='wb') as fout: fout.write(meta_graph_def.SerializeToString()) # Build the transformer from exported serving model # We are using signaures, thus must provide the keys gin = TFInputGraph.fromCheckpointWithSignature( - model_ckpt_dir, serving_sigdef_key) + model_ckpt_dir, self.serving_sigdef_key) self.input_graphs.append(gin) + self.input_graph_with_signature.add(gin) # Transformer without using signature_def gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 57e40f71..1c8e6838 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -38,33 +38,15 @@ class TFTransformerTest(TFInputGraphTest, SparkDLTestCase): - def _build_default_session_tests(self, sess): - gin = TFInputGraph.fromGraph( - sess.graph, sess, self.feed_names, self.fetch_names) - self.build_standard_transformers(sess, gin) - - gin = TFInputGraph.fromGraphDef( - sess.graph.as_graph_def(), self.feed_names, self.fetch_names) - self.build_standard_transformers(sess, gin) - - def build_standard_transformers(self, sess, tf_input_graph): - def _add_transformer(imap, omap): - trnsfmr = TFTransformer( - tfInputGraph=tf_input_graph, inputMapping=imap, outputMapping=omap) - self.transformers.append(trnsfmr) - - imap = dict((col, tfx.tensor_name(sess.graph, op_name)) - for col, op_name in self.input_mapping.items()) - omap = dict((tfx.tensor_name(sess.graph, op_name), col) - for op_name, col in self.output_mapping.items()) - _add_transformer(imap, omap) + def setUp(self): + super(TFTransformerTest, self).setUp() + self.all_close_tol = 1e-8 @contextmanager def _run_test_in_tf_session(self): """ [THIS IS NOT A TEST]: encapsulate general test workflow """ # Build local features and DataFrame from it - print("OVERRIDING default", repr(self.__class__)) local_features = [] for idx in range(self.num_samples): _dict = {'idx': idx} @@ -98,14 +80,7 @@ def _run_test_in_tf_session(self): out_ref = np.hstack(_results) # We have sessions, now create transformers out of them - - # Apply the transform - for input_graph in self.input_graphs: - transformer = TFTransformer(tfInputGraph=input_graph, - inputMapping=self.input_mapping, - outputMapping=self.output_mapping) - print('built transformer', repr(transformer)) - + def check_transformer(transformer): out_df = transformer.transform(analyzed_df) out_colnames = [] for old_colname, new_colname in self.output_mapping.items(): @@ -119,51 +94,73 @@ def _run_test_in_tf_session(self): _results.append(np.ravel(curr_res)) out_tgt = np.hstack(_results) - err_msg = 'not close => {} != {}, max_diff {}' - self.assertTrue(np.allclose(out_ref, out_tgt), - msg=err_msg.format(out_ref.shape, out_tgt.shape, - np.max(np.abs(out_ref - out_tgt)))) - - - # def test_multi_io(self): - # """ Build TFTransformer with multiple I/O tensors """ - # self.setup_iomap(replica=3) - # with self._run_test_in_tf_session() as sess: - # xs = [] - # for tnsr_op_name in self.input_mapping.values(): - # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) - # xs.append(x) - - # zs = [] - # for i, tnsr_op_name in enumerate(self.output_mapping.keys()): - # z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) - # zs.append(z) - - # self._build_default_session_tests(sess) - - - # def test_mixed_keras_graph(self): - # """ Build mixed keras graph """ - # with IsolatedSession(using_keras=True) as issn: - # tnsr_in = tf.placeholder( - # tf.double, shape=[None, self.vec_size], name=self.input_op_name) - # inp = tf.expand_dims(tnsr_in, axis=2) - # # Keras layers does not take tf.double - # inp = tf.cast(inp, tf.float32) - # conv = Conv1D(filters=4, kernel_size=2)(inp) - # pool = MaxPool1D(pool_size=2)(conv) - # flat = Flatten()(pool) - # dense = Dense(1)(flat) - # # We must keep the leading dimension of the output - # redsum = tf.reduce_logsumexp(dense, axis=1) - # tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) - - # # Initialize the variables - # init_op = tf.global_variables_initializer() - # issn.run(init_op) - # # We could train the model ... but skip it here - # gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) - - # with self._run_test_in_tf_session() as sess: - # tf.import_graph_def(gfn.graph_def, name='') - # self._build_default_session_tests(sess) + err_msg = 'not close => shape {} != {}, max_diff {} > {}' + self.assertTrue(np.allclose(out_ref, out_tgt, atol=self.all_close_tol), + msg=err_msg.format(out_ref.shape, + out_tgt.shape, + np.max(np.abs(out_ref - out_tgt)), + self.all_close_tol)) + + # Apply the transform + for input_graph in self.input_graphs: + transformer = TFTransformer(tfInputGraph=input_graph, + inputMapping=self.input_mapping, + outputMapping=self.output_mapping) + check_transformer(transformer) + + if input_graph in self.input_graph_with_signature: + _imap = input_graph.translateInputMapping(self.sig_input_mapping) + _omap = input_graph.translateOutputMapping(self.sig_output_mapping) + transformer = TFTransformer(tfInputGraph=input_graph, + inputMapping=_imap, + outputMapping=_omap) + check_transformer(transformer) + + def test_multi_io(self): + """ Build TFTransformer with multiple I/O tensors """ + self.setup_iomap(replica=3) + with self._run_test_in_tf_session() as sess: + xs = [] + for tnsr_op_name in self.input_mapping.values(): + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) + xs.append(x) + + zs = [] + for i, tnsr_op_name in enumerate(self.output_mapping.keys()): + z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) + zs.append(z) + + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + self.input_graphs.append(gin) + + + def test_mixed_keras_graph(self): + """ Build TFTransformer from mixed keras graph """ + with IsolatedSession(using_keras=True) as issn: + tnsr_in = tf.placeholder( + tf.double, shape=[None, self.vec_size], name=self.input_op_name) + inp = tf.expand_dims(tnsr_in, axis=2) + # Keras layers does not take tf.double + inp = tf.cast(inp, tf.float32) + conv = Conv1D(filters=4, kernel_size=2)(inp) + pool = MaxPool1D(pool_size=2)(conv) + flat = Flatten()(pool) + dense = Dense(1)(flat) + # We must keep the leading dimension of the output + redsum = tf.reduce_logsumexp(dense, axis=1) + tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) + + # Initialize the variables + init_op = tf.global_variables_initializer() + issn.run(init_op) + # We could train the model ... but skip it here + gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) + + self.all_close_tol = 1e-5 + gin = TFInputGraph.fromGraphDef(gfn.graph_def, self.feed_names, self.fetch_names) + self.input_graphs.append(gin) + + with self._run_test_in_tf_session() as sess: + tf.import_graph_def(gfn.graph_def, name='') + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + self.input_graphs.append(gin) From 7287ab7d1f06d669a7c02aa199b57e1312c58ae4 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 21 Sep 2017 14:47:12 -0700 Subject: [PATCH 32/80] fix style Using the following YAPF style ======================================================== based_on_style = pep8 ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=True BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=False COLUMN_LIMIT=100 SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=False SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=True SPLIT_BEFORE_FIRST_ARGUMENT=False SPLIT_BEFORE_NAMED_ASSIGNS=False SPLIT_PENALTY_AFTER_OPENING_BRACKET=30 USE_TABS=False ======================================================== --- python/sparkdl/param/shared_params.py | 32 +++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index e606abce..b54792c2 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -12,25 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # - """ Some parts are copied from pyspark.ml.param.shared and some are complementary to pyspark.ml.param. The copy is due to some useful pyspark fns/classes being private APIs. """ - +import textwrap from functools import wraps from pyspark.ml.param import Param, Params, TypeConverters from sparkdl.param.converters import SparkDLTypeConverters - ######################################################## # Copied from PySpark for backward compatibility. # They first appeared in Apache Spark version 2.1.1. ######################################################## + def keyword_only(func): """ A decorator that forces keyword arguments in the wrapped method @@ -54,8 +53,8 @@ class HasInputCol(Params): Mixin for param inputCol: input column name. """ - inputCol = Param( - Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) + inputCol = Param(Params._dummy(), "inputCol", "input column name.", + typeConverter=TypeConverters.toString) def setInputCol(self, value): """ @@ -75,8 +74,8 @@ class HasOutputCol(Params): Mixin for param outputCol: output column name. """ - outputCol = Param( - Params._dummy(), "outputCol", "output column name.", typeConverter=TypeConverters.toString) + outputCol = Param(Params._dummy(), "outputCol", "output column name.", + typeConverter=TypeConverters.toString) def __init__(self): super(HasOutputCol, self).__init__() @@ -94,6 +93,7 @@ def getOutputCol(self): """ return self.getOrDefault(self.outputCol) + ######################################################## # New in sparkdl ######################################################## @@ -196,8 +196,7 @@ class HasOutputMapping(Params): """ Mixin for param outputMapping: ordered list of ('outputTensorOpName', 'outputColName') pairs """ - outputMapping = Param(Params._dummy(), - "outputMapping", + outputMapping = Param(Params._dummy(), "outputMapping", "Mapping output :class:`tf.Operation` names to DataFrame column names", typeConverter=SparkDLTypeConverters.asTensorNameToColumnMap) @@ -212,8 +211,7 @@ class HasInputMapping(Params): """ Mixin for param inputMapping: ordered list of ('inputColName', 'inputTensorOpName') pairs """ - inputMapping = Param(Params._dummy(), - "inputMapping", + inputMapping = Param(Params._dummy(), "inputMapping", "Mapping input DataFrame column names to :class:`tf.Operation` names", typeConverter=SparkDLTypeConverters.asColumnToTensorNameMap) @@ -228,9 +226,15 @@ class HasTFHParams(Params): """ Mixin for TensorFlow model hyper-parameters """ - tfHParams = Param(Params._dummy(), - "hparams", - "instance of :class:`tf.contrib.training.HParams`, a key-value map-like object", + tfHParams = Param(Params._dummy(), "hparams", + textwrap.dedent("""\ + instance of :class:`tf.contrib.training.HParams`, a namespace-like + key-value object, storing parameters to be used to define the final + TensorFlow graph for the Transformer. + + Currently accepted values are: + - `batch_size`: number of samples provided to the inference graph + during each evaluation function call.""", typeConverter=SparkDLTypeConverters.toTFHParams) def setTFHParams(self, value): From 4f11374754dbbb7a5114c02f05b27ddc62630a53 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 21 Sep 2017 14:47:12 -0700 Subject: [PATCH 33/80] fix style Using the following YAPF style ======================================================== based_on_style = pep8 ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=True BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=False COLUMN_LIMIT=100 SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=False SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=True SPLIT_BEFORE_FIRST_ARGUMENT=False SPLIT_BEFORE_NAMED_ASSIGNS=False SPLIT_PENALTY_AFTER_OPENING_BRACKET=30 USE_TABS=False ======================================================== --- python/sparkdl/param/shared_params.py | 31 +++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index e606abce..433c591a 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -12,25 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # - """ Some parts are copied from pyspark.ml.param.shared and some are complementary to pyspark.ml.param. The copy is due to some useful pyspark fns/classes being private APIs. """ - +import textwrap from functools import wraps from pyspark.ml.param import Param, Params, TypeConverters from sparkdl.param.converters import SparkDLTypeConverters - ######################################################## # Copied from PySpark for backward compatibility. # They first appeared in Apache Spark version 2.1.1. ######################################################## + def keyword_only(func): """ A decorator that forces keyword arguments in the wrapped method @@ -54,8 +53,8 @@ class HasInputCol(Params): Mixin for param inputCol: input column name. """ - inputCol = Param( - Params._dummy(), "inputCol", "input column name.", typeConverter=TypeConverters.toString) + inputCol = Param(Params._dummy(), "inputCol", "input column name.", + typeConverter=TypeConverters.toString) def setInputCol(self, value): """ @@ -75,8 +74,8 @@ class HasOutputCol(Params): Mixin for param outputCol: output column name. """ - outputCol = Param( - Params._dummy(), "outputCol", "output column name.", typeConverter=TypeConverters.toString) + outputCol = Param(Params._dummy(), "outputCol", "output column name.", + typeConverter=TypeConverters.toString) def __init__(self): super(HasOutputCol, self).__init__() @@ -94,6 +93,7 @@ def getOutputCol(self): """ return self.getOrDefault(self.outputCol) + ######################################################## # New in sparkdl ######################################################## @@ -196,8 +196,7 @@ class HasOutputMapping(Params): """ Mixin for param outputMapping: ordered list of ('outputTensorOpName', 'outputColName') pairs """ - outputMapping = Param(Params._dummy(), - "outputMapping", + outputMapping = Param(Params._dummy(), "outputMapping", "Mapping output :class:`tf.Operation` names to DataFrame column names", typeConverter=SparkDLTypeConverters.asTensorNameToColumnMap) @@ -212,8 +211,7 @@ class HasInputMapping(Params): """ Mixin for param inputMapping: ordered list of ('inputColName', 'inputTensorOpName') pairs """ - inputMapping = Param(Params._dummy(), - "inputMapping", + inputMapping = Param(Params._dummy(), "inputMapping", "Mapping input DataFrame column names to :class:`tf.Operation` names", typeConverter=SparkDLTypeConverters.asColumnToTensorNameMap) @@ -228,9 +226,14 @@ class HasTFHParams(Params): """ Mixin for TensorFlow model hyper-parameters """ - tfHParams = Param(Params._dummy(), - "hparams", - "instance of :class:`tf.contrib.training.HParams`, a key-value map-like object", + tfHParams = Param(Params._dummy(), "hparams", + textwrap.dedent("""\ + instance of :class:`tf.contrib.training.HParams`, a namespace-like + key-value object, storing parameters to be used to define the final + TensorFlow graph for the Transformer. + + Currently accepted values are: + - `batch_size`: number of samples evaluated together in inference steps"""), typeConverter=SparkDLTypeConverters.toTFHParams) def setTFHParams(self, value): From 7b6ec3a1831500a3b35119d19b884edd31a214c7 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 21 Sep 2017 16:46:13 -0700 Subject: [PATCH 34/80] autogen test cases --- python/tests/param/params_test.py | 83 +++++++++++++++++-------------- python/tests/tests.py | 3 ++ 2 files changed, 49 insertions(+), 37 deletions(-) diff --git a/python/tests/param/params_test.py b/python/tests/param/params_test.py index 74f09755..03a567c6 100644 --- a/python/tests/param/params_test.py +++ b/python/tests/param/params_test.py @@ -12,57 +12,66 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import sys +from __future__ import absolute_import, division, print_function -if sys.version_info[:2] <= (2, 6): - try: - import unittest2 as unittest - except ImportError: - sys.stderr.write('Please install unittest2 to test with Python 2.6 or earlier') - sys.exit(1) -else: - import unittest +from six import with_metaclass from sparkdl.param.converters import SparkDLTypeConverters as conv -class ParamsConverterTest(unittest.TestCase): +from ..tests import PythonUnitTestCase + + +class TestGenInvalidMeta(type): + def __new__(cls, name, bases, attrs): + """ implement test cases here """ + test_cases = [['a1', 'b2'], ('c3', 'd4'), [('a', 1), ('b', 2)]] + + def _check_col2tnsr(case): + def impl(self): + with self.assertRaises(TypeError): + conv.asColumnToTensorNameMap(case) + return impl + + def _check_tnsr2col(case): + def impl(self): + with self.assertRaises(TypeError): + conv.asTensorNameToColumnMap(case) + return impl + + def _add_test_fn(fn_name, fn_impl): + fn_impl.__name__ = fn_name + attrs[fn_name] = fn_impl + + for idx, case in enumerate(test_cases): + _add_test_fn('test_invalid_col2tnsr_{}'.format(idx), + _check_col2tnsr(case)) + _add_test_fn('test_invalid_tnsr2col_{}'.format(idx), + _check_tnsr2col(case)) + + return super(TestGenInvalidMeta, cls).__new__(cls, name, bases, attrs) + + +class ParamsConverterTest(with_metaclass(TestGenInvalidMeta, PythonUnitTestCase)): # pylint: disable=protected-access + @classmethod + def setUpClass(cls): + print(repr(cls), cls) + def test_tf_input_mapping_converter(self): - valid_tnsr_input = {'colA': 'tnsrOpA:0', - 'colB': 'tnsrOpB:0'} - valid_op_input = {'colA': 'tnsrOpA', - 'colB': 'tnsrOpB'} - valid_input_mapping_result = [('colA', 'tnsrOpA:0'), - ('colB', 'tnsrOpB:0')] + valid_tnsr_input = {'colA': 'tnsrOpA:0', 'colB': 'tnsrOpB:0'} + valid_op_input = {'colA': 'tnsrOpA', 'colB': 'tnsrOpB'} + valid_input_mapping_result = [('colA', 'tnsrOpA:0'), ('colB', 'tnsrOpB:0')] for valid_input_mapping in [valid_op_input, valid_tnsr_input]: res = conv.asColumnToTensorNameMap(valid_input_mapping) self.assertEqual(valid_input_mapping_result, res) def test_tf_output_mapping_converter(self): - valid_tnsr_output = {'tnsrOpA:0': 'colA', - 'tnsrOpB:0': 'colB'} - valid_op_output = {'tnsrOpA': 'colA', - 'tnsrOpB': 'colB'} - valid_output_mapping_result = [('tnsrOpA:0', 'colA'), - ('tnsrOpB:0', 'colB')] + valid_tnsr_output = {'tnsrOpA:0': 'colA', 'tnsrOpB:0': 'colB'} + valid_op_output = {'tnsrOpA': 'colA', 'tnsrOpB': 'colB'} + valid_output_mapping_result = [('tnsrOpA:0', 'colA'), ('tnsrOpB:0', 'colB')] for valid_output_mapping in [valid_tnsr_output, valid_op_output]: res = conv.asTensorNameToColumnMap(valid_output_mapping) self.assertEqual(valid_output_mapping_result, res) - - - def test_invalid_input_mapping(self): - for invalid in [['a1', 'b2'], ('c3', 'd4'), [('a', 1), ('b', 2)], - {1: 'a', 2.0: 'b'}, {'a': 1, 'b': 2.0}]: - with self.assertRaises(TypeError): - conv.asColumnToTensorNameMap(invalid) - conv.asTensorNameToColumnMap(invalid) - - with self.assertRaises(TypeError): - # Wrong containter type: only accept dict - conv.asColumnToTensorNameMap([('colA', 'tnsrA:0'), ('colB', 'tnsrB:0')]) - conv.asTensorNameToColumnMap([('colA', 'tnsrA:0'), ('colB', 'tnsrB:0')]) - conv.asColumnToTensorNameMap([('tnsrA:0', 'colA'), ('tnsrB:0', 'colB')]) - conv.asTensorNameToColumnMap([('tnsrA:0', 'colA'), ('tnsrB:0', 'colB')]) diff --git a/python/tests/tests.py b/python/tests/tests.py index d93b31a8..ae7cec3e 100644 --- a/python/tests/tests.py +++ b/python/tests/tests.py @@ -29,6 +29,9 @@ from pyspark.sql import SQLContext from pyspark.sql import SparkSession +class PythonUnitTestCase(unittest.TestCase): + # Just the plain test unittest.TestCase, but won't have to do import check + pass class SparkDLTestCase(unittest.TestCase): From 561f8e747592420257daee1f95d1b3ae8df5c90d Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 22 Sep 2017 16:56:08 -0700 Subject: [PATCH 35/80] test refactoring With better subtest cases --- python/tests/param/params_test.py | 81 +++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/python/tests/param/params_test.py b/python/tests/param/params_test.py index 03a567c6..fcb882a0 100644 --- a/python/tests/param/params_test.py +++ b/python/tests/param/params_test.py @@ -14,44 +14,73 @@ # from __future__ import absolute_import, division, print_function +from collections import namedtuple from six import with_metaclass -from sparkdl.param.converters import SparkDLTypeConverters as conv +from sparkdl.param.converters import SparkDLTypeConverters from ..tests import PythonUnitTestCase class TestGenInvalidMeta(type): - def __new__(cls, name, bases, attrs): - """ implement test cases here """ - test_cases = [['a1', 'b2'], ('c3', 'd4'), [('a', 1), ('b', 2)]] + """ + Generate one test function for each test case + """ - def _check_col2tnsr(case): - def impl(self): - with self.assertRaises(TypeError): - conv.asColumnToTensorNameMap(case) - return impl + def __new__(mcs, name, bases, attrs): + _add_invalid_col2tnsr_mapping_tests() + attrs.update(_TEST_FUNCTIONS_REGISTRY) + return super(TestGenInvalidMeta, mcs).__new__(mcs, name, bases, attrs) - def _check_tnsr2col(case): - def impl(self): - with self.assertRaises(TypeError): - conv.asTensorNameToColumnMap(case) - return impl - def _add_test_fn(fn_name, fn_impl): - fn_impl.__name__ = fn_name - attrs[fn_name] = fn_impl +_TEST_FUNCTIONS_REGISTRY = {} - for idx, case in enumerate(test_cases): - _add_test_fn('test_invalid_col2tnsr_{}'.format(idx), - _check_col2tnsr(case)) - _add_test_fn('test_invalid_tnsr2col_{}'.format(idx), - _check_tnsr2col(case)) +TestCase = namedtuple('TestCase', ['data', 'reason']) - return super(TestGenInvalidMeta, cls).__new__(cls, name, bases, attrs) + +def _assemble_and_register_test(fn_name, fn_impl, description): + fn_impl.__name__ = fn_name + fn_impl.__doc__ = 'Auto Test: {}'.format(description) + _TEST_FUNCTIONS_REGISTRY[fn_name] = fn_impl + + +def _add_invalid_col2tnsr_mapping_tests(): + """ implement test cases here """ + test_cases = [ + TestCase(data=['a1', 'b2'], reason='required pair but get single element'), + TestCase(data=('c3', 'd4'), reason='required pair but get single element'), + TestCase(data=[('a', 1), ('b', 2)], reason='only accept dict, but get list'), + ] + + # Add tests for `asColumnToTensorNameMap` + for idx, test_case in enumerate(test_cases): + + def test_fn_impl(self): + with self.assertRaises(TypeError, msg=test_case.reason): + SparkDLTypeConverters.asColumnToTensorNameMap(test_case.data) + + test_fn_name = 'test_invalid_col2tnsr_{}'.format(idx) + test_fn_impl.__name__ = test_fn_name + _desc = 'Test invalid column => tensor name mapping: {}' + test_fn_impl.__doc__ = _desc.format(test_case.reason) + _TEST_FUNCTIONS_REGISTRY[test_fn_name] = test_fn_impl + + # Add tests for `asTensorNameToColumnMap` + for idx, test_case in enumerate(test_cases): + + def test_fn_impl(self): # pylint: disable=function-redefined + with self.assertRaises(TypeError, msg=test_case.reason): + SparkDLTypeConverters.asTensorNameToColumnMap(test_case.data) + + test_fn_name = 'test_invalid_tnsr2col_{}'.format(idx) + test_fn_impl.__name__ = test_fn_name + _desc = 'Test invalid tensor name => column mapping: {}' + test_fn_impl.__doc__ = _desc.format(test_case.reason) + _TEST_FUNCTIONS_REGISTRY[test_fn_name] = test_fn_impl class ParamsConverterTest(with_metaclass(TestGenInvalidMeta, PythonUnitTestCase)): + """ Test MLlib Params introduced in Spark Deep Learning Pipeline """ # pylint: disable=protected-access @classmethod @@ -59,19 +88,21 @@ def setUpClass(cls): print(repr(cls), cls) def test_tf_input_mapping_converter(self): + """ Test valid input mapping conversion """ valid_tnsr_input = {'colA': 'tnsrOpA:0', 'colB': 'tnsrOpB:0'} valid_op_input = {'colA': 'tnsrOpA', 'colB': 'tnsrOpB'} valid_input_mapping_result = [('colA', 'tnsrOpA:0'), ('colB', 'tnsrOpB:0')] for valid_input_mapping in [valid_op_input, valid_tnsr_input]: - res = conv.asColumnToTensorNameMap(valid_input_mapping) + res = SparkDLTypeConverters.asColumnToTensorNameMap(valid_input_mapping) self.assertEqual(valid_input_mapping_result, res) def test_tf_output_mapping_converter(self): + """ Test valid output mapping conversion """ valid_tnsr_output = {'tnsrOpA:0': 'colA', 'tnsrOpB:0': 'colB'} valid_op_output = {'tnsrOpA': 'colA', 'tnsrOpB': 'colB'} valid_output_mapping_result = [('tnsrOpA:0', 'colA'), ('tnsrOpB:0', 'colB')] for valid_output_mapping in [valid_tnsr_output, valid_op_output]: - res = conv.asTensorNameToColumnMap(valid_output_mapping) + res = SparkDLTypeConverters.asTensorNameToColumnMap(valid_output_mapping) self.assertEqual(valid_output_mapping_result, res) From 72485175acc88a4ad83054b3248af10149d1e667 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 22 Sep 2017 17:08:01 -0700 Subject: [PATCH 36/80] further refactor --- python/tests/param/params_test.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/python/tests/param/params_test.py b/python/tests/param/params_test.py index fcb882a0..6791607e 100644 --- a/python/tests/param/params_test.py +++ b/python/tests/param/params_test.py @@ -22,35 +22,27 @@ from ..tests import PythonUnitTestCase -class TestGenInvalidMeta(type): +class TestGenMeta(type): """ - Generate one test function for each test case + This meta-class add test cases to the main unit-test class. """ def __new__(mcs, name, bases, attrs): _add_invalid_col2tnsr_mapping_tests() attrs.update(_TEST_FUNCTIONS_REGISTRY) - return super(TestGenInvalidMeta, mcs).__new__(mcs, name, bases, attrs) - + return super(TestGenMeta, mcs).__new__(mcs, name, bases, attrs) +# Stores test function name mapped to implementation body _TEST_FUNCTIONS_REGISTRY = {} TestCase = namedtuple('TestCase', ['data', 'reason']) -def _assemble_and_register_test(fn_name, fn_impl, description): - fn_impl.__name__ = fn_name - fn_impl.__doc__ = 'Auto Test: {}'.format(description) - _TEST_FUNCTIONS_REGISTRY[fn_name] = fn_impl - - def _add_invalid_col2tnsr_mapping_tests(): - """ implement test cases here """ - test_cases = [ - TestCase(data=['a1', 'b2'], reason='required pair but get single element'), - TestCase(data=('c3', 'd4'), reason='required pair but get single element'), - TestCase(data=[('a', 1), ('b', 2)], reason='only accept dict, but get list'), - ] + """ Create a list of test cases and construct individual test functions for each case """ + test_cases = [TestCase(data=['a1', 'b2'], reason='required pair but get single element'), + TestCase(data=('c3', 'd4'), reason='required pair but get single element'), + TestCase(data=[('a', 1), ('b', 2)], reason='only accept dict, but get list'),] # Add tests for `asColumnToTensorNameMap` for idx, test_case in enumerate(test_cases): @@ -79,7 +71,7 @@ def test_fn_impl(self): # pylint: disable=function-redefined _TEST_FUNCTIONS_REGISTRY[test_fn_name] = test_fn_impl -class ParamsConverterTest(with_metaclass(TestGenInvalidMeta, PythonUnitTestCase)): +class ParamsConverterTest(with_metaclass(TestGenMeta, PythonUnitTestCase)): """ Test MLlib Params introduced in Spark Deep Learning Pipeline """ # pylint: disable=protected-access From e09027fd78df5a4044eaf778a144ed9bd4d96624 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 22 Sep 2017 18:44:30 -0700 Subject: [PATCH 37/80] Merge branch 'tf-transformer-part1' into tf-transformer-part3 --- python/sparkdl/graph/input.py | 131 ++++++++++++++++------------------ 1 file changed, 63 insertions(+), 68 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 51fd017c..f85414ff 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -21,15 +21,18 @@ __all__ = ["TFInputGraph"] + class TFInputGraph(object): """ An opaque serializable object containing TensorFlow graph. [WARNING] This class should not be called by any user code. """ + def __init__(self): raise NotImplementedError( - "Please do NOT construct TFInputGraph directly. Instead, use one of the helper functions") + "Please do NOT construct TFInputGraph directly. Instead, use one of the helper functions" + ) @classmethod def _new_obj_internal(cls): @@ -72,95 +75,86 @@ def import_graph_fn(sess): @classmethod def fromCheckpoint(cls, checkpoint_dir, feed_names, fetch_names): - return cls._from_checkpoint_impl(checkpoint_dir, - signature_def_key=None, - feed_names=feed_names, fetch_names=fetch_names) + return _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=feed_names, + fetch_names=fetch_names) @classmethod def fromCheckpointWithSignature(cls, checkpoint_dir, signature_def_key): assert signature_def_key is not None - return cls._from_checkpoint_impl(checkpoint_dir, - signature_def_key, - feed_names=None, fetch_names=None) + return _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names=None, + fetch_names=None) @classmethod def fromSavedModel(cls, saved_model_dir, tag_set, feed_names, fetch_names): - return cls._from_saved_model_impl(saved_model_dir, tag_set, - signature_def_key=None, - feed_names=feed_names, fetch_names=fetch_names) + return _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, + feed_names=feed_names, fetch_names=fetch_names) @classmethod def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key): assert signature_def_key is not None - return cls._from_saved_model_impl(saved_model_dir, tag_set, - signature_def_key=signature_def_key, - feed_names=None, fetch_names=None) + return _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=signature_def_key, + feed_names=None, fetch_names=None) - @classmethod - def _from_checkpoint_impl(cls, - checkpoint_dir, - signature_def_key=None, - feed_names=None, - fetch_names=None): - """ - Construct a TFInputGraphBuilder from a model checkpoint - """ - assert (feed_names is None) == (fetch_names is None), \ - 'feed_names and fetch_names, if provided must appear together' - assert (feed_names is None) != (signature_def_key is None), \ - 'must either provide feed_names or singnature_def_key' - def import_graph_fn(sess): - # Load checkpoint and import the graph - with sess.as_default(): - ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) +def _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=None, + fetch_names=None): + """ + Construct a TFInputGraphBuilder from a model checkpoint + """ + assert (feed_names is None) == (fetch_names is None), \ + 'feed_names and fetch_names, if provided must appear together' + assert (feed_names is None) != (signature_def_key is None), \ + 'must either provide feed_names or singnature_def_key' + + def import_graph_fn(sess): + # Load checkpoint and import the graph + with sess.as_default(): + ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) + + # NOTE(phi-dbq): we must manually load meta_graph_def to get the signature_def + # the current `import_graph_def` function seems to ignore + # any signature_def fields in a checkpoint's meta_graph_def. + meta_graph_def = meta_graph_pb2.MetaGraphDef() + with open("{}.meta".format(ckpt_path), 'rb') as fin: + meta_graph_def.ParseFromString(fin.read()) + + saver = tf.train.import_meta_graph(meta_graph_def, clear_devices=True) + saver.restore(sess, ckpt_path) - # NOTE(phi-dbq): we must manually load meta_graph_def to get the signature_def - # the current `import_graph_def` function seems to ignore - # any signature_def fields in a checkpoint's meta_graph_def. - meta_graph_def = meta_graph_pb2.MetaGraphDef() - with open("{}.meta".format(ckpt_path), 'rb') as fin: - meta_graph_def.ParseFromString(fin.read()) + sig_def = None + if signature_def_key is not None: + sig_def = meta_graph_def.signature_def[signature_def_key] + assert sig_def, 'singnature_def_key {} provided, '.format(signature_def_key) + \ + 'but failed to find it from the meta_graph_def ' + \ + 'from checkpoint {}'.format(checkpoint_dir) - saver = tf.train.import_meta_graph(meta_graph_def, clear_devices=True) - saver.restore(sess, ckpt_path) + return _GinBuilderInfo(sig_def=sig_def) - sig_def = None - if signature_def_key is not None: - sig_def = meta_graph_def.signature_def[signature_def_key] - assert sig_def, 'singnature_def_key {} provided, '.format(signature_def_key) + \ - 'but failed to find it from the meta_graph_def ' + \ - 'from checkpoint {}'.format(checkpoint_dir) + return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) - return _GinBuilderInfo(sig_def=sig_def) - return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) +def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, feed_names=None, + fetch_names=None): + """ + Construct a TFInputGraphBuilder from a SavedModel + """ + assert (feed_names is None) == (fetch_names is None), \ + 'feed_names and fetch_names, if provided must appear together' + assert (feed_names is None) != (signature_def_key is None), \ + 'must either provide feed_names or singnature_def_key' - @classmethod - def _from_saved_model_impl(cls, saved_model_dir, tag_set, - signature_def_key=None, - feed_names=None, - fetch_names=None): - """ - Construct a TFInputGraphBuilder from a SavedModel - """ - assert (feed_names is None) == (fetch_names is None), \ - 'feed_names and fetch_names, if provided must appear together' - assert (feed_names is None) != (signature_def_key is None), \ - 'must either provide feed_names or singnature_def_key' + def import_graph_fn(sess): + tag_sets = tag_set.split(',') + meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) - def import_graph_fn(sess): - tag_sets = tag_set.split(',') - meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) + sig_def = None + if signature_def_key is not None: + sig_def = tf.contrib.saved_model.get_signature_def_by_key(meta_graph_def, + signature_def_key) - sig_def = None - if signature_def_key is not None: - sig_def = tf.contrib.saved_model.get_signature_def_by_key( - meta_graph_def, signature_def_key) + return _GinBuilderInfo(sig_def=sig_def) - return _GinBuilderInfo(sig_def=sig_def) - - return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) + return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) class _GinBuilderInfo(object): @@ -189,6 +183,7 @@ def extract_signatures(self): self.fetch_mapping[sigdef_key] = tnsr_name self.fetch_names.append(tnsr_name) + class _GinBuilder(object): def __init__(self, import_graph_fn, sess=None, graph=None): self.import_graph_fn = import_graph_fn From 40caaceeffe1a72cccbe0c70bc834e67a96c9a41 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 22 Sep 2017 19:34:13 -0700 Subject: [PATCH 38/80] and so there is no helper classes --- python/sparkdl/graph/input.py | 200 +++++++++++++++------------------- 1 file changed, 85 insertions(+), 115 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index f85414ff..e69a58e9 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -15,7 +15,7 @@ from __future__ import absolute_import, division, print_function import tensorflow as tf -from tensorflow.core.protobuf import meta_graph_pb2 +from tensorflow.core.protobuf import meta_graph_pb2 # pylint: disable=no-name-in-module import sparkdl.graph.utils as tfx @@ -24,54 +24,50 @@ class TFInputGraph(object): """ - An opaque serializable object containing TensorFlow graph. + An opaque object containing TensorFlow graph. + This object can be serialized. [WARNING] This class should not be called by any user code. """ def __init__(self): raise NotImplementedError( - "Please do NOT construct TFInputGraph directly. Instead, use one of the helper functions" - ) + "Please do NOT build TFInputGraph directly. Instead, use one of the helper functions") @classmethod def _new_obj_internal(cls): # pylint: disable=attribute-defined-outside-init obj = object.__new__(cls) # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. + ##============================================================ obj.graph_def = None obj.input_tensor_name_from_signature = None obj.output_tensor_name_from_signature = None + ##============================================================ return obj @classmethod def fromGraph(cls, graph, sess, feed_names, fetch_names): """ - Construct a TFInputGraphBuilder from a in memory tf.Graph object + Construct a TFInputGraph from a in memory `tf.Graph` object """ - assert isinstance(graph, tf.Graph), \ - ('expect tf.Graph type but got', type(graph)) - - def import_graph_fn(_sess): - assert _sess == sess, 'must have the same session' - return _GinBuilderInfo() - - return _GinBuilder(import_graph_fn, sess, graph).build(feed_names, fetch_names) + return _build_impl(sess=sess, graph=graph, sig_def=None, + feed_names=feed_names, fetch_names=fetch_names) @classmethod def fromGraphDef(cls, graph_def, feed_names, fetch_names): """ - Construct a TFInputGraphBuilder from a tf.GraphDef object + Construct a TFInputGraph from a tf.GraphDef object """ assert isinstance(graph_def, tf.GraphDef), \ ('expect tf.GraphDef type but got', type(graph_def)) - def import_graph_fn(sess): - with sess.as_default(): - tf.import_graph_def(graph_def, name='') - return _GinBuilderInfo() - - return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) + graph = tf.Graph() + with tf.Session(graph=graph) as sess: + tf.import_graph_def(graph_def, name='') + gin = _build_impl(sess=sess, graph=graph, sig_def=None, + feed_names=feed_names, fetch_names=fetch_names) + return gin @classmethod def fromCheckpoint(cls, checkpoint_dir, feed_names, fetch_names): @@ -99,51 +95,52 @@ def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key def _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=None, fetch_names=None): """ - Construct a TFInputGraphBuilder from a model checkpoint + Construct a TFInputGraph from a model checkpoint """ assert (feed_names is None) == (fetch_names is None), \ 'feed_names and fetch_names, if provided must appear together' assert (feed_names is None) != (signature_def_key is None), \ 'must either provide feed_names or singnature_def_key' - def import_graph_fn(sess): + graph = tf.Graph() + with tf.Session(graph=graph) as sess: # Load checkpoint and import the graph - with sess.as_default(): - ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) + ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) - # NOTE(phi-dbq): we must manually load meta_graph_def to get the signature_def - # the current `import_graph_def` function seems to ignore - # any signature_def fields in a checkpoint's meta_graph_def. - meta_graph_def = meta_graph_pb2.MetaGraphDef() - with open("{}.meta".format(ckpt_path), 'rb') as fin: - meta_graph_def.ParseFromString(fin.read()) + # NOTE(phi-dbq): we must manually load meta_graph_def to get the signature_def + # the current `import_graph_def` function seems to ignore + # any signature_def fields in a checkpoint's meta_graph_def. + meta_graph_def = meta_graph_pb2.MetaGraphDef() + with open("{}.meta".format(ckpt_path), 'rb') as fin: + meta_graph_def.ParseFromString(fin.read()) - saver = tf.train.import_meta_graph(meta_graph_def, clear_devices=True) - saver.restore(sess, ckpt_path) + saver = tf.train.import_meta_graph(meta_graph_def, clear_devices=True) + saver.restore(sess, ckpt_path) - sig_def = None - if signature_def_key is not None: - sig_def = meta_graph_def.signature_def[signature_def_key] - assert sig_def, 'singnature_def_key {} provided, '.format(signature_def_key) + \ - 'but failed to find it from the meta_graph_def ' + \ - 'from checkpoint {}'.format(checkpoint_dir) - - return _GinBuilderInfo(sig_def=sig_def) + sig_def = None + if signature_def_key is not None: + sig_def = meta_graph_def.signature_def[signature_def_key] + assert sig_def, 'singnature_def_key {} provided, '.format(signature_def_key) + \ + 'but failed to find it from the meta_graph_def ' + \ + 'from checkpoint {}'.format(checkpoint_dir) - return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) + gin = _build_impl(sess=sess, graph=graph, sig_def=sig_def, + feed_names=feed_names, fetch_names=fetch_names) + return gin def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, feed_names=None, fetch_names=None): """ - Construct a TFInputGraphBuilder from a SavedModel + Construct a TFInputGraph from a SavedModel """ assert (feed_names is None) == (fetch_names is None), \ 'feed_names and fetch_names, if provided must appear together' assert (feed_names is None) != (signature_def_key is None), \ 'must either provide feed_names or singnature_def_key' - def import_graph_fn(sess): + graph = tf.Graph() + with tf.Session(graph=graph) as sess: tag_sets = tag_set.split(',') meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) @@ -152,76 +149,49 @@ def import_graph_fn(sess): sig_def = tf.contrib.saved_model.get_signature_def_by_key(meta_graph_def, signature_def_key) - return _GinBuilderInfo(sig_def=sig_def) - - return _GinBuilder(import_graph_fn).build(feed_names, fetch_names) - - -class _GinBuilderInfo(object): - def __init__(self, sig_def=None): - self.sig_def = sig_def - self.feed_names = None - self.feed_mapping = None - self.fetch_names = None - self.fetch_mapping = None - - def extract_signatures(self): - assert self.sig_def is not None, \ - "ask to find sigdef mapping, but not found any" - - self.feed_mapping = {} - self.feed_names = [] - for sigdef_key, tnsr_info in self.sig_def.inputs.items(): - tnsr_name = tnsr_info.name - self.feed_mapping[sigdef_key] = tnsr_name - self.feed_names.append(tnsr_name) - - self.fetch_mapping = {} - self.fetch_names = [] - for sigdef_key, tnsr_info in self.sig_def.outputs.items(): - tnsr_name = tnsr_info.name - self.fetch_mapping[sigdef_key] = tnsr_name - self.fetch_names.append(tnsr_name) - - -class _GinBuilder(object): - def __init__(self, import_graph_fn, sess=None, graph=None): - self.import_graph_fn = import_graph_fn - assert (sess is None) == (graph is None) - if sess is not None: - self.graph = graph - self.sess = sess - self._should_clean = False - else: - self.graph = tf.Graph() - self.sess = tf.Session(graph=self.graph) - self._should_clean = True - - def _build_impl(self, feed_names, fetch_names): - # pylint: disable=protected-access,attribute-defined-outside-init - gin = TFInputGraph._new_obj_internal() - assert (feed_names is None) == (fetch_names is None) - must_have_sig_def = fetch_names is None - # NOTE(phi-dbq): both have to be set to default - with self.sess.as_default(), self.graph.as_default(): - _ginfo = self.import_graph_fn(self.sess) - if must_have_sig_def: - _ginfo.extract_signatures() - feed_names = _ginfo.feed_names - fetch_names = _ginfo.fetch_names - gin.input_tensor_name_from_signature = _ginfo.feed_mapping - gin.output_tensor_name_from_signature = _ginfo.fetch_mapping - - for tnsr_name in feed_names: - assert tfx.get_op(self.graph, tnsr_name) - fetches = [tfx.get_tensor(self.graph, tnsr_name) for tnsr_name in fetch_names] - gin.graph_def = tfx.strip_and_freeze_until(fetches, self.graph, self.sess) - return gin + gin = _build_impl(sess=sess, graph=graph, sig_def=sig_def, + feed_names=feed_names, fetch_names=fetch_names) + return gin - def build(self, feed_names=None, fetch_names=None): - try: - gin = self._build_impl(feed_names, fetch_names) - finally: - if self._should_clean: - self.sess.close() - return gin + +def _build_impl(sess, graph, sig_def, feed_names, fetch_names): + # pylint: disable=protected-access,attribute-defined-outside-init + assert (feed_names is None) == (fetch_names is None), \ + "if provided, feed_names {} and fetch_names {} ".format(feed_names, fetch_names) + \ + "must be provided together" + # NOTE(phi-dbq): both have to be set to default + with sess.as_default(), graph.as_default(): + #_ginfo = import_graph_fn(sess) + # If `feed_names` nor `fetch_names` is not provided, must infer them from signature + if feed_names is None and fetch_names is None: + assert sig_def is not None, \ + "require graph info to figure out the signature mapping" + + feed_mapping = {} + feed_names = [] + for sigdef_key, tnsr_info in sig_def.inputs.items(): + tnsr_name = tnsr_info.name + feed_mapping[sigdef_key] = tnsr_name + feed_names.append(tnsr_name) + + fetch_mapping = {} + fetch_names = [] + for sigdef_key, tnsr_info in sig_def.outputs.items(): + tnsr_name = tnsr_info.name + fetch_mapping[sigdef_key] = tnsr_name + fetch_names.append(tnsr_name) + else: + feed_mapping = None + fetch_mapping = None + + for tnsr_name in feed_names: + assert tfx.get_op(graph, tnsr_name), \ + 'requested tensor {} but found none in graph {}'.format(tnsr_name, graph) + fetches = [tfx.get_tensor(graph, tnsr_name) for tnsr_name in fetch_names] + graph_def = tfx.strip_and_freeze_until(fetches, graph, sess) + + gin = TFInputGraph._new_obj_internal() + gin.input_tensor_name_from_signature = feed_mapping + gin.output_tensor_name_from_signature = fetch_mapping + gin.graph_def = graph_def + return gin From 883321e410aa91da0f54d3003fb0e4dc010007ef Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 22 Sep 2017 19:55:14 -0700 Subject: [PATCH 39/80] input graph --- python/sparkdl/graph/input.py | 97 +++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 577f7fa9..5abc981a 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -73,8 +73,8 @@ def fromGraph(cls, graph, sess, feed_names, fetch_names): """ Construct a TFInputGraph from a in memory `tf.Graph` object """ - return _build_impl(sess=sess, graph=graph, sig_def=None, - feed_names=feed_names, fetch_names=fetch_names) + return _build_with_feeds_fetches(sess=sess, graph=graph, + feed_names=feed_names, fetch_names=fetch_names) @classmethod def fromGraphDef(cls, graph_def, feed_names, fetch_names): @@ -87,8 +87,8 @@ def fromGraphDef(cls, graph_def, feed_names, fetch_names): graph = tf.Graph() with tf.Session(graph=graph) as sess: tf.import_graph_def(graph_def, name='') - gin = _build_impl(sess=sess, graph=graph, sig_def=None, - feed_names=feed_names, fetch_names=fetch_names) + gin = _build_with_feeds_fetches(sess=sess, graph=graph, + feed_names=feed_names, fetch_names=fetch_names) return gin @classmethod @@ -114,8 +114,7 @@ def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key feed_names=None, fetch_names=None) -def _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=None, - fetch_names=None): +def _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names, fetch_names): """ Construct a TFInputGraph from a model checkpoint """ @@ -139,20 +138,16 @@ def _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=Non saver = tf.train.import_meta_graph(meta_graph_def, clear_devices=True) saver.restore(sess, ckpt_path) - sig_def = None if signature_def_key is not None: sig_def = meta_graph_def.signature_def[signature_def_key] - assert sig_def, 'singnature_def_key {} provided, '.format(signature_def_key) + \ - 'but failed to find it from the meta_graph_def ' + \ - 'from checkpoint {}'.format(checkpoint_dir) - - gin = _build_impl(sess=sess, graph=graph, sig_def=sig_def, - feed_names=feed_names, fetch_names=fetch_names) + gin = _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) + else: + gin = _build_with_feeds_fetches(sess=sess, graph=graph, + feed_names=feed_names, fetch_names=fetch_names) return gin -def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, feed_names=None, - fetch_names=None): +def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key, feed_names, fetch_names): """ Construct a TFInputGraph from a SavedModel """ @@ -166,46 +161,58 @@ def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, fee tag_sets = tag_set.split(',') meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) - sig_def = None if signature_def_key is not None: sig_def = tf.contrib.saved_model.get_signature_def_by_key(meta_graph_def, signature_def_key) + gin = _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) + else: + gin = _build_with_feeds_fetches(sess=sess, graph=graph, + feed_names=feed_names, fetch_names=fetch_names) + return gin + + +def _build_with_sig_def(sess, graph, sig_def): + # pylint: disable=protected-access,attribute-defined-outside-init + assert sig_def, \ + 'signature_def {} provided, '.format(sig_def) + \ + 'but failed to find it from the meta_graph_def' + + with sess.as_default(), graph.as_default(): + feed_mapping = {} + feed_names = [] + for sigdef_key, tnsr_info in sig_def.inputs.items(): + tnsr_name = tnsr_info.name + feed_mapping[sigdef_key] = tnsr_name + feed_names.append(tnsr_name) + + fetch_mapping = {} + fetch_names = [] + for sigdef_key, tnsr_info in sig_def.outputs.items(): + tnsr_name = tnsr_info.name + fetch_mapping[sigdef_key] = tnsr_name + fetch_names.append(tnsr_name) + + for tnsr_name in feed_names: + assert tfx.get_op(graph, tnsr_name), \ + 'requested tensor {} but found none in graph {}'.format(tnsr_name, graph) + fetches = [tfx.get_tensor(graph, tnsr_name) for tnsr_name in fetch_names] + graph_def = tfx.strip_and_freeze_until(fetches, graph, sess) - gin = _build_impl(sess=sess, graph=graph, sig_def=sig_def, - feed_names=feed_names, fetch_names=fetch_names) + gin = TFInputGraph._new_obj_internal() + gin.input_tensor_name_from_signature = feed_mapping + gin.output_tensor_name_from_signature = fetch_mapping + gin.graph_def = graph_def return gin -def _build_impl(sess, graph, sig_def, feed_names, fetch_names): +def _build_with_feeds_fetches(sess, graph, feed_names, fetch_names): # pylint: disable=protected-access,attribute-defined-outside-init - assert (feed_names is None) == (fetch_names is None), \ - "if provided, feed_names {} and fetch_names {} ".format(feed_names, fetch_names) + \ - "must be provided together" + assert (feed_names is not None) and (fetch_names is not None), \ + "must provide feed_names {} and fetch_names {}".format(feed_names, fetch_names) # NOTE(phi-dbq): both have to be set to default with sess.as_default(), graph.as_default(): #_ginfo = import_graph_fn(sess) # If `feed_names` nor `fetch_names` is not provided, must infer them from signature - if feed_names is None and fetch_names is None: - assert sig_def is not None, \ - "require graph info to figure out the signature mapping" - - feed_mapping = {} - feed_names = [] - for sigdef_key, tnsr_info in sig_def.inputs.items(): - tnsr_name = tnsr_info.name - feed_mapping[sigdef_key] = tnsr_name - feed_names.append(tnsr_name) - - fetch_mapping = {} - fetch_names = [] - for sigdef_key, tnsr_info in sig_def.outputs.items(): - tnsr_name = tnsr_info.name - fetch_mapping[sigdef_key] = tnsr_name - fetch_names.append(tnsr_name) - else: - feed_mapping = None - fetch_mapping = None - for tnsr_name in feed_names: assert tfx.get_op(graph, tnsr_name), \ 'requested tensor {} but found none in graph {}'.format(tnsr_name, graph) @@ -213,7 +220,7 @@ def _build_impl(sess, graph, sig_def, feed_names, fetch_names): graph_def = tfx.strip_and_freeze_until(fetches, graph, sess) gin = TFInputGraph._new_obj_internal() - gin.input_tensor_name_from_signature = feed_mapping - gin.output_tensor_name_from_signature = fetch_mapping + gin.input_tensor_name_from_signature = None + gin.output_tensor_name_from_signature = None gin.graph_def = graph_def return gin From c72444b7fe0b4188172bf71277cb30d2842d1ffc Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 22 Sep 2017 21:01:17 -0700 Subject: [PATCH 40/80] docs --- python/sparkdl/graph/input.py | 111 +++++++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 16 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 5abc981a..10f5bbbe 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -21,13 +21,14 @@ __all__ = ["TFInputGraph"] +# pylint: disable=invalid-name,wrong-spelling-in-docstring,wrong-spelling-in-comment class TFInputGraph(object): """ An opaque object containing TensorFlow graph. This object can be serialized. - [WARNING] This class should not be called by any user code. + .. warning: This class should not be called by any user code. """ def __init__(self): @@ -47,6 +48,17 @@ def _new_obj_internal(cls): return obj def translateInputMapping(self, input_mapping): + """ + When the meta_graph contains signature_def, we expect users to provide + input and output mapping with respect to the tensor reference keys + embedded in the `signature_def`. + + This function translates the input_mapping into the canonical format, + which maps input DataFrame column names to tensor names. + + :param input_mapping: dict, DataFrame column name to tensor reference names + defined in the signature_def key. + """ assert self.input_tensor_name_from_signature is not None _input_mapping = {} if isinstance(input_mapping, dict): @@ -58,6 +70,17 @@ def translateInputMapping(self, input_mapping): return _input_mapping def translateOutputMapping(self, output_mapping): + """ + When the meta_graph contains signature_def, we expect users to provide + input and output mapping with respect to the tensor reference keys + embedded in the `signature_def`. + + This function translates the output_mapping into the canonical format, + which maps tensor names into input DataFrame column names. + + :param output_mapping: dict, tensor reference names defined in the signature_def keys + into the output DataFrame column names. + """ assert self.output_tensor_name_from_signature is not None _output_mapping = {} if isinstance(output_mapping, dict): @@ -71,15 +94,23 @@ def translateOutputMapping(self, output_mapping): @classmethod def fromGraph(cls, graph, sess, feed_names, fetch_names): """ - Construct a TFInputGraph from a in memory `tf.Graph` object + Construct a TFInputGraph from a in memory `tf.Graph` object. + The graph might contain variables that are maintained in the provided session. + :param graph: `tf.Graph` + :param feed_names: list, names of the input tensors. + :param fetch_names: list, names of the output tensors. """ - return _build_with_feeds_fetches(sess=sess, graph=graph, - feed_names=feed_names, fetch_names=fetch_names) + return _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) @classmethod def fromGraphDef(cls, graph_def, feed_names, fetch_names): """ - Construct a TFInputGraph from a tf.GraphDef object + Construct a TFInputGraph from a tf.GraphDef object. + :param graph_def: `tf.GraphDef`, a serializable object containing the topology and + computation units of the data-flow graph. + :param feed_names: list, names of the input tensors. + :param fetch_names: list, names of the output tensors. """ assert isinstance(graph_def, tf.GraphDef), \ ('expect tf.GraphDef type but got', type(graph_def)) @@ -87,28 +118,66 @@ def fromGraphDef(cls, graph_def, feed_names, fetch_names): graph = tf.Graph() with tf.Session(graph=graph) as sess: tf.import_graph_def(graph_def, name='') - gin = _build_with_feeds_fetches(sess=sess, graph=graph, - feed_names=feed_names, fetch_names=fetch_names) + gin = _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) return gin @classmethod def fromCheckpoint(cls, checkpoint_dir, feed_names, fetch_names): + """ + Construct a TFInputGraph object from a checkpoint, ignore the embedded + signature_def, if there is any. + :param checkpoint_dir: str, name of the directory containing the TensorFlow graph + training checkpoint. + :feed_names: list, names of the input tensors. + :fetch_names: list, names of the output tensors. + """ return _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=feed_names, fetch_names=fetch_names) @classmethod def fromCheckpointWithSignature(cls, checkpoint_dir, signature_def_key): + """ + Construct a TFInputGraph object from a checkpoint, using the embedded + signature_def. Throw error if we cannot find an entry with the `signature_def_key` + inside the `signature_def`. + :param checkpoint_dir: str, name of the directory containing the TensorFlow graph + training checkpoint. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` + from which we retrieve the signature key to tensor names mapping. + """ assert signature_def_key is not None return _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names=None, fetch_names=None) @classmethod def fromSavedModel(cls, saved_model_dir, tag_set, feed_names, fetch_names): + """ + Construct a TFInputGraph object from a saved model (`tf.SavedModel`) directory. + Ignore the the embedded signature_def, if there is any. + :param saved_model_dir: str, name of the directory containing the TensorFlow graph + training checkpoint. + :param tag_set: str, name of the graph stored in this meta_graph of the saved model + that we are interested in using. + :feed_names: list, names of the input tensors. + :fetch_names: list, names of the output tensors. + """ return _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, feed_names=feed_names, fetch_names=fetch_names) @classmethod def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key): + """ + Construct a TFInputGraph object from a saved model (`tf.SavedModel`) directory, + using the embedded signature_def. Throw error if we cannot find an entry with + the `signature_def_key` inside the `signature_def`. + :param saved_model_dir: str, name of the directory containing the TensorFlow graph + training checkpoint. + :param tag_set: str, name of the graph stored in this meta_graph of the saved model + that we are interested in using. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` + from which we retrieve the signature key to tensor names mapping. + """ assert signature_def_key is not None return _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=signature_def_key, feed_names=None, fetch_names=None) @@ -116,7 +185,13 @@ def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key def _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names, fetch_names): """ - Construct a TFInputGraph from a model checkpoint + Construct a TFInputGraph from a model checkpoint. + Notice that one should either provide the `signature_def_key` or provide both + `feed_names` and `fetch_names`. Please set the unprovided values to None. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` + from which we retrieve the signature key to tensor names mapping. + :feed_names: list, names of the input tensors. + :fetch_names: list, names of the output tensors. """ assert (feed_names is None) == (fetch_names is None), \ 'feed_names and fetch_names, if provided must appear together' @@ -142,14 +217,20 @@ def _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names, fetch_n sig_def = meta_graph_def.signature_def[signature_def_key] gin = _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) else: - gin = _build_with_feeds_fetches(sess=sess, graph=graph, - feed_names=feed_names, fetch_names=fetch_names) + gin = _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) return gin def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key, feed_names, fetch_names): """ - Construct a TFInputGraph from a SavedModel + Construct a TFInputGraph from a SavedModel. + Notice that one should either provide the `signature_def_key` or provide both + `feed_names` and `fetch_names`. Please set the unprovided values to None. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` + from which we retrieve the signature key to tensor names mapping. + :feed_names: list, names of the input tensors. + :fetch_names: list, names of the output tensors. """ assert (feed_names is None) == (fetch_names is None), \ 'feed_names and fetch_names, if provided must appear together' @@ -166,8 +247,8 @@ def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key, feed_nam signature_def_key) gin = _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) else: - gin = _build_with_feeds_fetches(sess=sess, graph=graph, - feed_names=feed_names, fetch_names=fetch_names) + gin = _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) return gin @@ -209,10 +290,8 @@ def _build_with_feeds_fetches(sess, graph, feed_names, fetch_names): # pylint: disable=protected-access,attribute-defined-outside-init assert (feed_names is not None) and (fetch_names is not None), \ "must provide feed_names {} and fetch_names {}".format(feed_names, fetch_names) - # NOTE(phi-dbq): both have to be set to default + with sess.as_default(), graph.as_default(): - #_ginfo = import_graph_fn(sess) - # If `feed_names` nor `fetch_names` is not provided, must infer them from signature for tnsr_name in feed_names: assert tfx.get_op(graph, tnsr_name), \ 'requested tensor {} but found none in graph {}'.format(tnsr_name, graph) From e963d11d12b6b5bd30a9b39af63ab192afd29000 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 22 Sep 2017 19:56:33 -0700 Subject: [PATCH 41/80] and into more pieces --- python/sparkdl/graph/input.py | 167 +++++++++++++++++++++++----------- 1 file changed, 116 insertions(+), 51 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index e69a58e9..ac66adba 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -21,13 +21,15 @@ __all__ = ["TFInputGraph"] +# pylint: disable=invalid-name,wrong-spelling-in-docstring,wrong-spelling-in-comment + class TFInputGraph(object): """ An opaque object containing TensorFlow graph. This object can be serialized. - [WARNING] This class should not be called by any user code. + .. warning: This class should not be called by any user code. """ def __init__(self): @@ -49,15 +51,23 @@ def _new_obj_internal(cls): @classmethod def fromGraph(cls, graph, sess, feed_names, fetch_names): """ - Construct a TFInputGraph from a in memory `tf.Graph` object + Construct a TFInputGraph from a in memory `tf.Graph` object. + The graph might contain variables that are maintained in the provided session. + :param graph: `tf.Graph` + :param feed_names: list, names of the input tensors. + :param fetch_names: list, names of the output tensors. """ - return _build_impl(sess=sess, graph=graph, sig_def=None, - feed_names=feed_names, fetch_names=fetch_names) + return _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) @classmethod def fromGraphDef(cls, graph_def, feed_names, fetch_names): """ - Construct a TFInputGraph from a tf.GraphDef object + Construct a TFInputGraph from a tf.GraphDef object. + :param graph_def: `tf.GraphDef`, a serializable object containing the topology and + computation units of the data-flow graph. + :param feed_names: list, names of the input tensors. + :param fetch_names: list, names of the output tensors. """ assert isinstance(graph_def, tf.GraphDef), \ ('expect tf.GraphDef type but got', type(graph_def)) @@ -65,37 +75,80 @@ def fromGraphDef(cls, graph_def, feed_names, fetch_names): graph = tf.Graph() with tf.Session(graph=graph) as sess: tf.import_graph_def(graph_def, name='') - gin = _build_impl(sess=sess, graph=graph, sig_def=None, - feed_names=feed_names, fetch_names=fetch_names) + gin = _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) return gin @classmethod def fromCheckpoint(cls, checkpoint_dir, feed_names, fetch_names): + """ + Construct a TFInputGraph object from a checkpoint, ignore the embedded + signature_def, if there is any. + :param checkpoint_dir: str, name of the directory containing the TensorFlow graph + training checkpoint. + :feed_names: list, names of the input tensors. + :fetch_names: list, names of the output tensors. + """ return _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=feed_names, fetch_names=fetch_names) @classmethod def fromCheckpointWithSignature(cls, checkpoint_dir, signature_def_key): + """ + Construct a TFInputGraph object from a checkpoint, using the embedded + signature_def. Throw error if we cannot find an entry with the `signature_def_key` + inside the `signature_def`. + :param checkpoint_dir: str, name of the directory containing the TensorFlow graph + training checkpoint. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` + from which we retrieve the signature key to tensor names mapping. + """ assert signature_def_key is not None return _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names=None, fetch_names=None) @classmethod def fromSavedModel(cls, saved_model_dir, tag_set, feed_names, fetch_names): + """ + Construct a TFInputGraph object from a saved model (`tf.SavedModel`) directory. + Ignore the the embedded signature_def, if there is any. + :param saved_model_dir: str, name of the directory containing the TensorFlow graph + training checkpoint. + :param tag_set: str, name of the graph stored in this meta_graph of the saved model + that we are interested in using. + :feed_names: list, names of the input tensors. + :fetch_names: list, names of the output tensors. + """ return _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, feed_names=feed_names, fetch_names=fetch_names) @classmethod def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key): + """ + Construct a TFInputGraph object from a saved model (`tf.SavedModel`) directory, + using the embedded signature_def. Throw error if we cannot find an entry with + the `signature_def_key` inside the `signature_def`. + :param saved_model_dir: str, name of the directory containing the TensorFlow graph + training checkpoint. + :param tag_set: str, name of the graph stored in this meta_graph of the saved model + that we are interested in using. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` + from which we retrieve the signature key to tensor names mapping. + """ assert signature_def_key is not None return _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=signature_def_key, feed_names=None, fetch_names=None) -def _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=None, - fetch_names=None): +def _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names, fetch_names): """ - Construct a TFInputGraph from a model checkpoint + Construct a TFInputGraph from a model checkpoint. + Notice that one should either provide the `signature_def_key` or provide both + `feed_names` and `fetch_names`. Please set the unprovided values to None. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` + from which we retrieve the signature key to tensor names mapping. + :feed_names: list, names of the input tensors. + :fetch_names: list, names of the output tensors. """ assert (feed_names is None) == (fetch_names is None), \ 'feed_names and fetch_names, if provided must appear together' @@ -117,22 +170,24 @@ def _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=Non saver = tf.train.import_meta_graph(meta_graph_def, clear_devices=True) saver.restore(sess, ckpt_path) - sig_def = None if signature_def_key is not None: sig_def = meta_graph_def.signature_def[signature_def_key] - assert sig_def, 'singnature_def_key {} provided, '.format(signature_def_key) + \ - 'but failed to find it from the meta_graph_def ' + \ - 'from checkpoint {}'.format(checkpoint_dir) - - gin = _build_impl(sess=sess, graph=graph, sig_def=sig_def, - feed_names=feed_names, fetch_names=fetch_names) + gin = _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) + else: + gin = _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) return gin -def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, feed_names=None, - fetch_names=None): +def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key, feed_names, fetch_names): """ - Construct a TFInputGraph from a SavedModel + Construct a TFInputGraph from a SavedModel. + Notice that one should either provide the `signature_def_key` or provide both + `feed_names` and `fetch_names`. Please set the unprovided values to None. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` + from which we retrieve the signature key to tensor names mapping. + :feed_names: list, names of the input tensors. + :fetch_names: list, names of the output tensors. """ assert (feed_names is None) == (fetch_names is None), \ 'feed_names and fetch_names, if provided must appear together' @@ -144,45 +199,36 @@ def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, fee tag_sets = tag_set.split(',') meta_graph_def = tf.saved_model.loader.load(sess, tag_sets, saved_model_dir) - sig_def = None if signature_def_key is not None: sig_def = tf.contrib.saved_model.get_signature_def_by_key(meta_graph_def, signature_def_key) - - gin = _build_impl(sess=sess, graph=graph, sig_def=sig_def, - feed_names=feed_names, fetch_names=fetch_names) + gin = _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) + else: + gin = _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) return gin -def _build_impl(sess, graph, sig_def, feed_names, fetch_names): +def _build_with_sig_def(sess, graph, sig_def): # pylint: disable=protected-access,attribute-defined-outside-init - assert (feed_names is None) == (fetch_names is None), \ - "if provided, feed_names {} and fetch_names {} ".format(feed_names, fetch_names) + \ - "must be provided together" - # NOTE(phi-dbq): both have to be set to default + assert sig_def, \ + 'signature_def {} provided, '.format(sig_def) + \ + 'but failed to find it from the meta_graph_def' + with sess.as_default(), graph.as_default(): - #_ginfo = import_graph_fn(sess) - # If `feed_names` nor `fetch_names` is not provided, must infer them from signature - if feed_names is None and fetch_names is None: - assert sig_def is not None, \ - "require graph info to figure out the signature mapping" - - feed_mapping = {} - feed_names = [] - for sigdef_key, tnsr_info in sig_def.inputs.items(): - tnsr_name = tnsr_info.name - feed_mapping[sigdef_key] = tnsr_name - feed_names.append(tnsr_name) - - fetch_mapping = {} - fetch_names = [] - for sigdef_key, tnsr_info in sig_def.outputs.items(): - tnsr_name = tnsr_info.name - fetch_mapping[sigdef_key] = tnsr_name - fetch_names.append(tnsr_name) - else: - feed_mapping = None - fetch_mapping = None + feed_mapping = {} + feed_names = [] + for sigdef_key, tnsr_info in sig_def.inputs.items(): + tnsr_name = tnsr_info.name + feed_mapping[sigdef_key] = tnsr_name + feed_names.append(tnsr_name) + + fetch_mapping = {} + fetch_names = [] + for sigdef_key, tnsr_info in sig_def.outputs.items(): + tnsr_name = tnsr_info.name + fetch_mapping[sigdef_key] = tnsr_name + fetch_names.append(tnsr_name) for tnsr_name in feed_names: assert tfx.get_op(graph, tnsr_name), \ @@ -195,3 +241,22 @@ def _build_impl(sess, graph, sig_def, feed_names, fetch_names): gin.output_tensor_name_from_signature = fetch_mapping gin.graph_def = graph_def return gin + + +def _build_with_feeds_fetches(sess, graph, feed_names, fetch_names): + # pylint: disable=protected-access,attribute-defined-outside-init + assert (feed_names is not None) and (fetch_names is not None), \ + "must provide feed_names {} and fetch_names {}".format(feed_names, fetch_names) + + with sess.as_default(), graph.as_default(): + for tnsr_name in feed_names: + assert tfx.get_op(graph, tnsr_name), \ + 'requested tensor {} but found none in graph {}'.format(tnsr_name, graph) + fetches = [tfx.get_tensor(graph, tnsr_name) for tnsr_name in fetch_names] + graph_def = tfx.strip_and_freeze_until(fetches, graph, sess) + + gin = TFInputGraph._new_obj_internal() + gin.input_tensor_name_from_signature = None + gin.output_tensor_name_from_signature = None + gin.graph_def = graph_def + return gin From f7a7d382c5fd1b8e4aefbd7edc580ca374f106fa Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 25 Sep 2017 11:03:45 -0700 Subject: [PATCH 42/80] update converter and test cases --- python/sparkdl/param/converters.py | 29 ++++++---- python/tests/param/params_test.py | 88 +++++++++++++++++++----------- 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index 1a65915a..a951e468 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -24,24 +24,33 @@ __all__ = ['SparkDLTypeConverters'] +def _get_strict_tensor_name(_maybe_tnsr_name): + assert isinstance(_maybe_tnsr_name, six.string_types), \ + "must provide a strict tensor name as input, but got {}".format(type(_maybe_tnsr_name)) + assert tfx.as_tensor_name(_maybe_tnsr_name) == _maybe_tnsr_name, \ + "input {} must be a valid tensor name".format(_maybe_tnsr_name) + return _maybe_tnsr_name + def _try_convert_tf_tensor_mapping(value, is_key_tf_tensor=True): if isinstance(value, dict): strs_pair_seq = [] for k, v in value.items(): + # Check if the non-tensor value is of string type + _non_tnsr_str_val = v if is_key_tf_tensor else k + if not isinstance(_non_tnsr_str_val, six.string_types): + err_msg = 'expect string type for {}, but got {}' + raise TypeError(err_msg.format(_non_tnsr_str_val, type(_non_tnsr_str_val))) + + # Check if the tensor name is actually valid try: if is_key_tf_tensor: - _pair = (tfx.as_tensor_name(k), v) + _pair = (_get_strict_tensor_name(k), v) else: - _pair = (k, tfx.as_tensor_name(v)) - except: - err_msg = "Can NOT convert {} (type {}) to tf.Tensor name" + _pair = (k, _get_strict_tensor_name(v)) + except Exception as exc: + err_msg = "Can NOT convert {} (type {}) to tf.Tensor name: {}" _not_tf_op = k if is_key_tf_tensor else v - raise TypeError(err_msg.format(_not_tf_op, type(_not_tf_op))) - - str_val = v if is_key_tf_tensor else k - if not isinstance(str_val, six.string_types): - err_msg = 'expect string type for {}, but got {}' - raise TypeError(err_msg.format(str_val, type(str_val))) + raise TypeError(err_msg.format(_not_tf_op, type(_not_tf_op), exc)) strs_pair_seq.append(_pair) diff --git a/python/tests/param/params_test.py b/python/tests/param/params_test.py index 6791607e..f1f60c1f 100644 --- a/python/tests/param/params_test.py +++ b/python/tests/param/params_test.py @@ -25,54 +25,82 @@ class TestGenMeta(type): """ This meta-class add test cases to the main unit-test class. + To add test cases, implement the test logic in a function + >>> `def _my_test_impl(): ...` + then call the following function + >>> `_register_test_case(fn_impl=_my_test_impl, name=..., doc=...)` """ - def __new__(mcs, name, bases, attrs): _add_invalid_col2tnsr_mapping_tests() attrs.update(_TEST_FUNCTIONS_REGISTRY) return super(TestGenMeta, mcs).__new__(mcs, name, bases, attrs) + # Stores test function name mapped to implementation body _TEST_FUNCTIONS_REGISTRY = {} -TestCase = namedtuple('TestCase', ['data', 'reason']) +TestCase = namedtuple('TestCase', ['data', 'description']) +def _register_test_case(fn_impl, name, doc): + """ Add an individual test case """ + fn_impl.__name__ = name + fn_impl.__doc__ = doc + _TEST_FUNCTIONS_REGISTRY[name] = fn_impl def _add_invalid_col2tnsr_mapping_tests(): """ Create a list of test cases and construct individual test functions for each case """ - test_cases = [TestCase(data=['a1', 'b2'], reason='required pair but get single element'), - TestCase(data=('c3', 'd4'), reason='required pair but get single element'), - TestCase(data=[('a', 1), ('b', 2)], reason='only accept dict, but get list'),] - - # Add tests for `asColumnToTensorNameMap` - for idx, test_case in enumerate(test_cases): - + shared_test_cases = [ + TestCase(data=['a1', 'b2'], description='required pair but get single element'), + TestCase(data=('c3', 'd4'), description='required pair but get single element'), + TestCase(data=[('a', 1), ('b', 2)], description='only accept dict, but get list'), + TestCase(data={1: 'a', 2.0: 'b'}, description='wrong mapping type'), + TestCase(data={'a': 1.0, 'b': 2}, description='wrong mapping type'), + ] + + # Specify test cases for `asColumnToTensorNameMap` + # Add additional test cases specific to this one + col2tnsr_test_cases = shared_test_cases + [ + TestCase(data={'colA': 'tnsrOpA', 'colB': 'tnsrOpB'}, + description='strict tensor name required'), + ] + _fn_name_template = 'test_invalid_col2tnsr_{idx}' + _fn_doc_template = 'Test invalid column => tensor name mapping: {description}' + + for idx, test_case in enumerate(col2tnsr_test_cases): + # Add the actual test logic here def test_fn_impl(self): - with self.assertRaises(TypeError, msg=test_case.reason): + with self.assertRaises(TypeError, msg=test_case.description): SparkDLTypeConverters.asColumnToTensorNameMap(test_case.data) - test_fn_name = 'test_invalid_col2tnsr_{}'.format(idx) - test_fn_impl.__name__ = test_fn_name - _desc = 'Test invalid column => tensor name mapping: {}' - test_fn_impl.__doc__ = _desc.format(test_case.reason) - _TEST_FUNCTIONS_REGISTRY[test_fn_name] = test_fn_impl + _name = _fn_name_template.format(idx=idx) + _doc = _fn_doc_template.format(description=test_case.description) + _register_test_case(fn_impl=test_fn_impl, name=_name, doc=_doc) - # Add tests for `asTensorNameToColumnMap` - for idx, test_case in enumerate(test_cases): + # Specify tests for `asTensorNameToColumnMap` + tnsr2col_test_cases = shared_test_cases + [ + TestCase(data={'tnsrOpA': 'colA', 'tnsrOpB': 'colB'}, + description='strict tensor name required'), + ] + _fn_name_template = 'test_invalid_tnsr2col_{idx}' + _fn_doc_template = 'Test invalid tensor name => column mapping: {description}' + + for idx, test_case in enumerate(tnsr2col_test_cases): + # Add the actual test logic here def test_fn_impl(self): # pylint: disable=function-redefined - with self.assertRaises(TypeError, msg=test_case.reason): + with self.assertRaises(TypeError, msg=test_case.description): SparkDLTypeConverters.asTensorNameToColumnMap(test_case.data) - test_fn_name = 'test_invalid_tnsr2col_{}'.format(idx) - test_fn_impl.__name__ = test_fn_name - _desc = 'Test invalid tensor name => column mapping: {}' - test_fn_impl.__doc__ = _desc.format(test_case.reason) - _TEST_FUNCTIONS_REGISTRY[test_fn_name] = test_fn_impl + _name = _fn_name_template.format(idx=idx) + _doc = _fn_doc_template.format(description=test_case.description) + _register_test_case(fn_impl=test_fn_impl, name=_name, doc=_doc) class ParamsConverterTest(with_metaclass(TestGenMeta, PythonUnitTestCase)): - """ Test MLlib Params introduced in Spark Deep Learning Pipeline """ + """ + Test MLlib Params introduced in Spark Deep Learning Pipeline + Additional test cases are attached via the meta class `TestGenMeta`. + """ # pylint: disable=protected-access @classmethod @@ -82,19 +110,15 @@ def setUpClass(cls): def test_tf_input_mapping_converter(self): """ Test valid input mapping conversion """ valid_tnsr_input = {'colA': 'tnsrOpA:0', 'colB': 'tnsrOpB:0'} - valid_op_input = {'colA': 'tnsrOpA', 'colB': 'tnsrOpB'} valid_input_mapping_result = [('colA', 'tnsrOpA:0'), ('colB', 'tnsrOpB:0')] - for valid_input_mapping in [valid_op_input, valid_tnsr_input]: - res = SparkDLTypeConverters.asColumnToTensorNameMap(valid_input_mapping) - self.assertEqual(valid_input_mapping_result, res) + res = SparkDLTypeConverters.asColumnToTensorNameMap(valid_tnsr_input) + self.assertEqual(valid_input_mapping_result, res) def test_tf_output_mapping_converter(self): """ Test valid output mapping conversion """ valid_tnsr_output = {'tnsrOpA:0': 'colA', 'tnsrOpB:0': 'colB'} - valid_op_output = {'tnsrOpA': 'colA', 'tnsrOpB': 'colB'} valid_output_mapping_result = [('tnsrOpA:0', 'colA'), ('tnsrOpB:0', 'colB')] - for valid_output_mapping in [valid_tnsr_output, valid_op_output]: - res = SparkDLTypeConverters.asTensorNameToColumnMap(valid_output_mapping) - self.assertEqual(valid_output_mapping_result, res) + res = SparkDLTypeConverters.asTensorNameToColumnMap(valid_tnsr_output) + self.assertEqual(valid_output_mapping_result, res) From ce606296551b11b2cf9c84e6eae3266850487b88 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 25 Sep 2017 14:22:21 -0700 Subject: [PATCH 43/80] class & docs --- python/sparkdl/graph/input.py | 128 ++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 35 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index ac66adba..433def66 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -21,7 +21,7 @@ __all__ = ["TFInputGraph"] -# pylint: disable=invalid-name,wrong-spelling-in-docstring,wrong-spelling-in-comment +# pylint: disable=invalid-name,wrong-spelling-in-comment,wrong-spelling-in-docstring class TFInputGraph(object): @@ -29,30 +29,87 @@ class TFInputGraph(object): An opaque object containing TensorFlow graph. This object can be serialized. - .. warning: This class should not be called by any user code. + When the graph contains serving signatures in which a set of well-known names are associtated + with their corresponding raw tensor names in the graph, we extract and store them here. + For example, the TensorFlow saved model may contain the following structure, + so that end users can retrieve the the input tensor via `well_known_input_sig` and + the output tensor via `well_known_output_sig` without knowing the actual tensor names a priori. + + .. code-block:: python + + sigdef: {'well_known_prediction_signature': + inputs { key: "well_known_input_sig" + value { + name: "tnsrIn:0" + dtype: DT_DOUBLE + tensor_shape { dim { size: -1 } dim { size: 17 } } + } + } + outputs { key: "well_known_output_sig" + value { + name: "tnsrOut:0" + dtype: DT_DOUBLE + tensor_shape { dim { size: -1 } } + } + }} + + + In this case, we will store the following mappings in the :py:obj:`TFInputGraph` object. + + .. code-block:: python + + self.input_tensor_name_from_signature = {'well_known_input_sig': 'tnsrIn:0'} + self.output_tensor_name_from_signature = {'well_known_output_sig': 'tnsrOut:0'} + + + .. warning:: The structure of the object is subject to change. + + + .. note:: We recommend constructing this object using one of the class constructor methods. + + - :py:meth:`fromGraph` + - :py:meth:`fromGraphDef` + - :py:meth:`fromCheckpoint` + - :py:meth:`fromCheckpointWithSignature` + - :py:meth:`fromSavedModel` + - :py:meth:`fromSavedModelWithSignature` + + + :param graph_def: :py:obj:`tf.GraphDef`, a serializable object containing the topology and + computation units of the TensorFlow graph. The graph object is prepared for + inference, i.e. the variables are converted to constants and operations like + BatchNormalization_ are converted to be independent of input batch. + + .. _BatchNormalization: https://www.tensorflow.org/api_docs/python/tf/layers/batch_normalization + + :param input_tensor_name_from_signature: dict, signature key names mapped to tensor names. + Please see the example above. + :param output_tensor_name_from_signature: dict, signature key names mapped to tensor names + Please see the example above. """ - def __init__(self): - raise NotImplementedError( - "Please do NOT build TFInputGraph directly. Instead, use one of the helper functions") - @classmethod - def _new_obj_internal(cls): - # pylint: disable=attribute-defined-outside-init - obj = object.__new__(cls) - # TODO: for (de-)serialization, the class should correspond to a ProtocolBuffer definition. - ##============================================================ - obj.graph_def = None - obj.input_tensor_name_from_signature = None - obj.output_tensor_name_from_signature = None - ##============================================================ - return obj + def __init__(self, graph_def, input_tensor_name_from_signature, + output_tensor_name_from_signature): + self.graph_def = graph_def + self.input_tensor_name_from_signature = input_tensor_name_from_signature + self.output_tensor_name_from_signature = output_tensor_name_from_signature @classmethod def fromGraph(cls, graph, sess, feed_names, fetch_names): """ Construct a TFInputGraph from a in memory `tf.Graph` object. The graph might contain variables that are maintained in the provided session. + Thus we need an active session in which the graph's variables are initialized or + restored. We do not close the session. As a result, this constructor can be used + inside a standard TensorFlow session context. + + .. code-block:: python + + with tf.Session() as sess: + graph = import_my_tensorflow_graph(...) + TFInputGraph.fromGraph(graph, sess, ...) + :param graph: `tf.Graph` :param feed_names: list, names of the input tensors. :param fetch_names: list, names of the output tensors. @@ -64,8 +121,9 @@ def fromGraph(cls, graph, sess, feed_names, fetch_names): def fromGraphDef(cls, graph_def, feed_names, fetch_names): """ Construct a TFInputGraph from a tf.GraphDef object. + :param graph_def: `tf.GraphDef`, a serializable object containing the topology and - computation units of the data-flow graph. + computation units of the TensorFlow graph. :param feed_names: list, names of the input tensors. :param fetch_names: list, names of the output tensors. """ @@ -84,10 +142,11 @@ def fromCheckpoint(cls, checkpoint_dir, feed_names, fetch_names): """ Construct a TFInputGraph object from a checkpoint, ignore the embedded signature_def, if there is any. + :param checkpoint_dir: str, name of the directory containing the TensorFlow graph training checkpoint. - :feed_names: list, names of the input tensors. - :fetch_names: list, names of the output tensors. + :param feed_names: list, names of the input tensors. + :param fetch_names: list, names of the output tensors. """ return _from_checkpoint_impl(checkpoint_dir, signature_def_key=None, feed_names=feed_names, fetch_names=fetch_names) @@ -98,6 +157,7 @@ def fromCheckpointWithSignature(cls, checkpoint_dir, signature_def_key): Construct a TFInputGraph object from a checkpoint, using the embedded signature_def. Throw error if we cannot find an entry with the `signature_def_key` inside the `signature_def`. + :param checkpoint_dir: str, name of the directory containing the TensorFlow graph training checkpoint. :param signature_def_key: str, name of the mapping contained inside the `signature_def` @@ -112,12 +172,13 @@ def fromSavedModel(cls, saved_model_dir, tag_set, feed_names, fetch_names): """ Construct a TFInputGraph object from a saved model (`tf.SavedModel`) directory. Ignore the the embedded signature_def, if there is any. + :param saved_model_dir: str, name of the directory containing the TensorFlow graph training checkpoint. :param tag_set: str, name of the graph stored in this meta_graph of the saved model that we are interested in using. - :feed_names: list, names of the input tensors. - :fetch_names: list, names of the output tensors. + :param feed_names: list, names of the input tensors. + :param fetch_names: list, names of the output tensors. """ return _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=None, feed_names=feed_names, fetch_names=fetch_names) @@ -128,6 +189,7 @@ def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key Construct a TFInputGraph object from a saved model (`tf.SavedModel`) directory, using the embedded signature_def. Throw error if we cannot find an entry with the `signature_def_key` inside the `signature_def`. + :param saved_model_dir: str, name of the directory containing the TensorFlow graph training checkpoint. :param tag_set: str, name of the graph stored in this meta_graph of the saved model @@ -145,10 +207,11 @@ def _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names, fetch_n Construct a TFInputGraph from a model checkpoint. Notice that one should either provide the `signature_def_key` or provide both `feed_names` and `fetch_names`. Please set the unprovided values to None. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` from which we retrieve the signature key to tensor names mapping. - :feed_names: list, names of the input tensors. - :fetch_names: list, names of the output tensors. + :param feed_names: list, names of the input tensors. + :param fetch_names: list, names of the output tensors. """ assert (feed_names is None) == (fetch_names is None), \ 'feed_names and fetch_names, if provided must appear together' @@ -184,10 +247,11 @@ def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key, feed_nam Construct a TFInputGraph from a SavedModel. Notice that one should either provide the `signature_def_key` or provide both `feed_names` and `fetch_names`. Please set the unprovided values to None. + :param signature_def_key: str, name of the mapping contained inside the `signature_def` from which we retrieve the signature key to tensor names mapping. - :feed_names: list, names of the input tensors. - :fetch_names: list, names of the output tensors. + :param feed_names: list, names of the input tensors. + :param fetch_names: list, names of the output tensors. """ assert (feed_names is None) == (fetch_names is None), \ 'feed_names and fetch_names, if provided must appear together' @@ -236,11 +300,8 @@ def _build_with_sig_def(sess, graph, sig_def): fetches = [tfx.get_tensor(graph, tnsr_name) for tnsr_name in fetch_names] graph_def = tfx.strip_and_freeze_until(fetches, graph, sess) - gin = TFInputGraph._new_obj_internal() - gin.input_tensor_name_from_signature = feed_mapping - gin.output_tensor_name_from_signature = fetch_mapping - gin.graph_def = graph_def - return gin + return TFInputGraph(graph_def=graph_def, input_tensor_name_from_signature=feed_mapping, + output_tensor_name_from_signature=fetch_mapping) def _build_with_feeds_fetches(sess, graph, feed_names, fetch_names): @@ -255,8 +316,5 @@ def _build_with_feeds_fetches(sess, graph, feed_names, fetch_names): fetches = [tfx.get_tensor(graph, tnsr_name) for tnsr_name in fetch_names] graph_def = tfx.strip_and_freeze_until(fetches, graph, sess) - gin = TFInputGraph._new_obj_internal() - gin.input_tensor_name_from_signature = None - gin.output_tensor_name_from_signature = None - gin.graph_def = graph_def - return gin + return TFInputGraph(graph_def=graph_def, input_tensor_name_from_signature=None, + output_tensor_name_from_signature=None) From e0cf2ffcb59af25596138b3a7eb717a94542fb3f Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 25 Sep 2017 15:57:06 -0700 Subject: [PATCH 44/80] update docs --- python/sparkdl/graph/input.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 433def66..d1be0e0c 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -29,6 +29,16 @@ class TFInputGraph(object): An opaque object containing TensorFlow graph. This object can be serialized. + .. note:: We recommend constructing this object using one of the class constructor methods. + + - :py:meth:`fromGraph` + - :py:meth:`fromGraphDef` + - :py:meth:`fromCheckpoint` + - :py:meth:`fromCheckpointWithSignature` + - :py:meth:`fromSavedModel` + - :py:meth:`fromSavedModelWithSignature` + + When the graph contains serving signatures in which a set of well-known names are associtated with their corresponding raw tensor names in the graph, we extract and store them here. For example, the TensorFlow saved model may contain the following structure, @@ -54,25 +64,12 @@ class TFInputGraph(object): }} - In this case, we will store the following mappings in the :py:obj:`TFInputGraph` object. + In this case, the class will internally store the mapping from signature names to tensor names. .. code-block:: python - self.input_tensor_name_from_signature = {'well_known_input_sig': 'tnsrIn:0'} - self.output_tensor_name_from_signature = {'well_known_output_sig': 'tnsrOut:0'} - - - .. warning:: The structure of the object is subject to change. - - - .. note:: We recommend constructing this object using one of the class constructor methods. - - - :py:meth:`fromGraph` - - :py:meth:`fromGraphDef` - - :py:meth:`fromCheckpoint` - - :py:meth:`fromCheckpointWithSignature` - - :py:meth:`fromSavedModel` - - :py:meth:`fromSavedModelWithSignature` + {'well_known_input_sig': 'tnsrIn:0'} + {'well_known_output_sig': 'tnsrOut:0'} :param graph_def: :py:obj:`tf.GraphDef`, a serializable object containing the topology and From fcabcb603843e9fa45186d67827d0986f0f36a16 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 25 Sep 2017 16:28:09 -0700 Subject: [PATCH 45/80] using `parameterized` to simplify testing logic https://github.com/wolever/parameterized --- python/requirements.txt | 1 + python/tests/param/params_test.py | 109 ++++++++---------------------- 2 files changed, 30 insertions(+), 80 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index a98a4d17..9d2133fa 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -4,6 +4,7 @@ h5py>=2.7.0 keras==2.0.4 # NOTE: this package has only been tested with keras 2.0.4 and may not work with other releases nose>=1.3.7 # for testing numpy>=1.11.2 +parameterized>=0.6.1 # for testing pillow>=4.1.1,<4.2 pygments>=2.2.0 tensorflow==1.3.0 diff --git a/python/tests/param/params_test.py b/python/tests/param/params_test.py index f1f60c1f..dffa4dce 100644 --- a/python/tests/param/params_test.py +++ b/python/tests/param/params_test.py @@ -15,97 +15,36 @@ from __future__ import absolute_import, division, print_function from collections import namedtuple -from six import with_metaclass +# Use this to create parameterized test cases +from parameterized import parameterized from sparkdl.param.converters import SparkDLTypeConverters from ..tests import PythonUnitTestCase - -class TestGenMeta(type): - """ - This meta-class add test cases to the main unit-test class. - To add test cases, implement the test logic in a function - >>> `def _my_test_impl(): ...` - then call the following function - >>> `_register_test_case(fn_impl=_my_test_impl, name=..., doc=...)` - """ - def __new__(mcs, name, bases, attrs): - _add_invalid_col2tnsr_mapping_tests() - attrs.update(_TEST_FUNCTIONS_REGISTRY) - return super(TestGenMeta, mcs).__new__(mcs, name, bases, attrs) - - -# Stores test function name mapped to implementation body -_TEST_FUNCTIONS_REGISTRY = {} - TestCase = namedtuple('TestCase', ['data', 'description']) -def _register_test_case(fn_impl, name, doc): - """ Add an individual test case """ - fn_impl.__name__ = name - fn_impl.__doc__ = doc - _TEST_FUNCTIONS_REGISTRY[name] = fn_impl - -def _add_invalid_col2tnsr_mapping_tests(): - """ Create a list of test cases and construct individual test functions for each case """ - shared_test_cases = [ - TestCase(data=['a1', 'b2'], description='required pair but get single element'), - TestCase(data=('c3', 'd4'), description='required pair but get single element'), - TestCase(data=[('a', 1), ('b', 2)], description='only accept dict, but get list'), - TestCase(data={1: 'a', 2.0: 'b'}, description='wrong mapping type'), - TestCase(data={'a': 1.0, 'b': 2}, description='wrong mapping type'), - ] - - # Specify test cases for `asColumnToTensorNameMap` - # Add additional test cases specific to this one - col2tnsr_test_cases = shared_test_cases + [ - TestCase(data={'colA': 'tnsrOpA', 'colB': 'tnsrOpB'}, - description='strict tensor name required'), - ] - _fn_name_template = 'test_invalid_col2tnsr_{idx}' - _fn_doc_template = 'Test invalid column => tensor name mapping: {description}' - - for idx, test_case in enumerate(col2tnsr_test_cases): - # Add the actual test logic here - def test_fn_impl(self): - with self.assertRaises(TypeError, msg=test_case.description): - SparkDLTypeConverters.asColumnToTensorNameMap(test_case.data) - - _name = _fn_name_template.format(idx=idx) - _doc = _fn_doc_template.format(description=test_case.description) - _register_test_case(fn_impl=test_fn_impl, name=_name, doc=_doc) - - - # Specify tests for `asTensorNameToColumnMap` - tnsr2col_test_cases = shared_test_cases + [ - TestCase(data={'tnsrOpA': 'colA', 'tnsrOpB': 'colB'}, - description='strict tensor name required'), - ] - _fn_name_template = 'test_invalid_tnsr2col_{idx}' - _fn_doc_template = 'Test invalid tensor name => column mapping: {description}' - - for idx, test_case in enumerate(tnsr2col_test_cases): - # Add the actual test logic here - def test_fn_impl(self): # pylint: disable=function-redefined - with self.assertRaises(TypeError, msg=test_case.description): - SparkDLTypeConverters.asTensorNameToColumnMap(test_case.data) - - _name = _fn_name_template.format(idx=idx) - _doc = _fn_doc_template.format(description=test_case.description) - _register_test_case(fn_impl=test_fn_impl, name=_name, doc=_doc) - - -class ParamsConverterTest(with_metaclass(TestGenMeta, PythonUnitTestCase)): +_shared_invalid_test_cases = [ + TestCase(data=['a1', 'b2'], description='required pair but get single element'), + TestCase(data=('c3', 'd4'), description='required pair but get single element'), + TestCase(data=[('a', 1), ('b', 2)], description='only accept dict, but get list'), + TestCase(data={1: 'a', 2.0: 'b'}, description='wrong mapping type'), + TestCase(data={'a': 1.0, 'b': 2}, description='wrong mapping type'), +] +_col2tnsr_test_cases = _shared_invalid_test_cases + [ + TestCase(data={'colA': 'tnsrOpA', 'colB': 'tnsrOpB'}, + description='strict tensor name required'), +] +_tnsr2col_test_cases = _shared_invalid_test_cases + [ + TestCase(data={'tnsrOpA': 'colA', 'tnsrOpB': 'colB'}, + description='strict tensor name required'), +] + +class ParamsConverterTest(PythonUnitTestCase): """ Test MLlib Params introduced in Spark Deep Learning Pipeline Additional test cases are attached via the meta class `TestGenMeta`. """ - # pylint: disable=protected-access - - @classmethod - def setUpClass(cls): - print(repr(cls), cls) def test_tf_input_mapping_converter(self): """ Test valid input mapping conversion """ @@ -122,3 +61,13 @@ def test_tf_output_mapping_converter(self): res = SparkDLTypeConverters.asTensorNameToColumnMap(valid_tnsr_output) self.assertEqual(valid_output_mapping_result, res) + + @parameterized.expand(_col2tnsr_test_cases) + def test_invalid_input_mapping(self, data, description): + with self.assertRaises(TypeError, msg=description): + SparkDLTypeConverters.asColumnToTensorNameMap(data) + + @parameterized.expand(_tnsr2col_test_cases) + def test_invalid_output_mapping(self, data, description): + with self.assertRaises(TypeError, msg=description): + SparkDLTypeConverters.asTensorNameToColumnMap(data) From 77b39068a4411014908dde271257052228ec06d5 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 26 Sep 2017 10:43:59 -0700 Subject: [PATCH 46/80] converter changes --- python/sparkdl/param/converters.py | 135 ++++++++++++++++++----------- python/tests/param/params_test.py | 12 +-- 2 files changed, 92 insertions(+), 55 deletions(-) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index a951e468..eeb2dfb0 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -24,45 +24,15 @@ __all__ = ['SparkDLTypeConverters'] -def _get_strict_tensor_name(_maybe_tnsr_name): - assert isinstance(_maybe_tnsr_name, six.string_types), \ - "must provide a strict tensor name as input, but got {}".format(type(_maybe_tnsr_name)) - assert tfx.as_tensor_name(_maybe_tnsr_name) == _maybe_tnsr_name, \ - "input {} must be a valid tensor name".format(_maybe_tnsr_name) - return _maybe_tnsr_name - -def _try_convert_tf_tensor_mapping(value, is_key_tf_tensor=True): - if isinstance(value, dict): - strs_pair_seq = [] - for k, v in value.items(): - # Check if the non-tensor value is of string type - _non_tnsr_str_val = v if is_key_tf_tensor else k - if not isinstance(_non_tnsr_str_val, six.string_types): - err_msg = 'expect string type for {}, but got {}' - raise TypeError(err_msg.format(_non_tnsr_str_val, type(_non_tnsr_str_val))) - - # Check if the tensor name is actually valid - try: - if is_key_tf_tensor: - _pair = (_get_strict_tensor_name(k), v) - else: - _pair = (k, _get_strict_tensor_name(v)) - except Exception as exc: - err_msg = "Can NOT convert {} (type {}) to tf.Tensor name: {}" - _not_tf_op = k if is_key_tf_tensor else v - raise TypeError(err_msg.format(_not_tf_op, type(_not_tf_op), exc)) - - strs_pair_seq.append(_pair) - - return sorted(strs_pair_seq) - - if is_key_tf_tensor: - raise TypeError("Could not convert %s to tf.Tensor name to str mapping" % type(value)) - else: - raise TypeError("Could not convert %s to str to tf.Tensor name mapping" % type(value)) - class SparkDLTypeConverters(object): + """ + .. note:: DeveloperApi + + Factory methods for common type conversion functions for :py:func:`Param.typeConverter`. + These methods are similar to :py:class:`spark.ml.param.TypeConverters`. + They provide support for the `Params` types introduced in Spark Deep Learning Pipelines. + """ @staticmethod def toTFGraph(value): if isinstance(value, tf.Graph): @@ -72,14 +42,47 @@ def toTFGraph(value): @staticmethod def asColumnToTensorNameMap(value): - return _try_convert_tf_tensor_mapping(value, is_key_tf_tensor=False) + """ + Convert a value to a column name to :py:obj:`tf.Tensor` name mapping + as a sorted list of string pairs, if possible. + """ + if isinstance(value, dict): + strs_pair_seq = [] + for _maybe_col_name, _maybe_tnsr_name in value.items(): + # Check if the non-tensor value is of string type + _col_name = _get_strict_col_name(_maybe_col_name) + # Check if the tensor name is actually valid + _tnsr_name = _get_strict_tensor_name(_maybe_tnsr_name) + strs_pair_seq.append((_col_name, _tnsr_name)) + + return sorted(strs_pair_seq) + + err_msg = "Could not convert [type {}] {} to column name to tf.Tensor name mapping" + raise TypeError(err_msg.format(type(value), value)) @staticmethod def asTensorNameToColumnMap(value): - return _try_convert_tf_tensor_mapping(value, is_key_tf_tensor=True) + """ + Convert a value to a :py:obj:`tf.Tensor` name to column name mapping + as a sorted list of string pairs, if possible. + """ + if isinstance(value, dict): + strs_pair_seq = [] + for _maybe_tnsr_name, _maybe_col_name in value.items(): + # Check if the non-tensor value is of string type + _col_name = _get_strict_col_name(_maybe_col_name) + # Check if the tensor name is actually valid + _tnsr_name = _get_strict_tensor_name(_maybe_tnsr_name) + strs_pair_seq.append((_tnsr_name, _col_name)) + + return sorted(strs_pair_seq) + + err_msg = "Could not convert [type {}] {} to tf.Tensor name to column name mapping" + raise TypeError(err_msg.format(type(value), value)) @staticmethod def toTFHParams(value): + """ Convert a value to a :py:obj:`tf.contrib.training.HParams` object, if possible. """ if isinstance(value, tf.contrib.training.HParams): return value else: @@ -87,34 +90,66 @@ def toTFHParams(value): @staticmethod def toStringOrTFTensor(value): + """ Convert a value to a str or a :py:obj:`tf.Tensor` object, if possible. """ if isinstance(value, tf.Tensor): return value - else: - try: - return TypeConverters.toString(value) - except TypeError: - raise TypeError("Could not convert %s to tensorflow.Tensor or str" % type(value)) + try: + return TypeConverters.toString(value) + except Exception as exc: + err_msg = "Could not convert [type {}] {} to tf.Tensor or str. {}" + raise TypeError(err_msg.format(type(value), value, exc)) @staticmethod def supportedNameConverter(supportedList): + """ + Create a converter that try to check if a value is part of the supported list. + + :param supportedList: list, containing supported objects. + :return: a converter that try to convert a value if it is part of the `supportedList`. + """ def converter(value): if value in supportedList: return value - else: - raise TypeError("%s %s is not in the supported list." % type(value), str(value)) + err_msg = "[type {}] {} is not in the supported list: {}" + raise TypeError(err_msg.format(type(value), str(value), supportedList)) return converter @staticmethod def toKerasLoss(value): + """ Convert a value to a name of Keras loss function, if possible """ if kmutil.is_valid_loss_function(value): return value - raise ValueError( - "Named loss not supported in Keras: {} type({})".format(value, type(value))) + err_msg = "Named loss not supported in Keras: [type {}] {}" + raise ValueError(err_msg.format(type(value), value)) @staticmethod def toKerasOptimizer(value): + """ Convert a value to a name of Keras optimizer, if possible """ if kmutil.is_valid_optimizer(value): return value - raise TypeError( - "Named optimizer not supported in Keras: {} type({})".format(value, type(value))) + err_msg = "Named optimizer not supported in Keras: [type {}] {}" + raise TypeError(err_msg.format(type(value), value)) + + +def _get_strict_tensor_name(_maybe_tnsr_name): + """ Check if the input is a valid tensor name """ + try: + assert isinstance(_maybe_tnsr_name, six.string_types), \ + "must provide a strict tensor name as input, but got {}".format(type(_maybe_tnsr_name)) + assert tfx.as_tensor_name(_maybe_tnsr_name) == _maybe_tnsr_name, \ + "input {} must be a valid tensor name".format(_maybe_tnsr_name) + except Exception as exc: + err_msg = "Can NOT convert [type {}] {} to tf.Tensor name: {}" + raise TypeError(err_msg.format(type(_maybe_tnsr_name), _maybe_tnsr_name, exc)) + else: + return _maybe_tnsr_name + + +def _get_strict_col_name(_maybe_col_name): + """ Check if the given colunm name is a valid column name """ + # We only check if the column name candidate is a string type + if not isinstance(_maybe_col_name, six.string_types): + err_msg = 'expect string type but got type {} for {}' + raise TypeError(err_msg.format(type(_maybe_col_name), _maybe_col_name)) + return _maybe_col_name diff --git a/python/tests/param/params_test.py b/python/tests/param/params_test.py index dffa4dce..f7479385 100644 --- a/python/tests/param/params_test.py +++ b/python/tests/param/params_test.py @@ -25,19 +25,19 @@ TestCase = namedtuple('TestCase', ['data', 'description']) _shared_invalid_test_cases = [ - TestCase(data=['a1', 'b2'], description='required pair but get single element'), - TestCase(data=('c3', 'd4'), description='required pair but get single element'), - TestCase(data=[('a', 1), ('b', 2)], description='only accept dict, but get list'), + TestCase(data=['a1', 'b2'], description='required pair but got single element'), + TestCase(data=('c3', 'd4'), description='required pair but got single element'), + TestCase(data=[('a', 1), ('b', 2)], description='only accept dict, but got list'), TestCase(data={1: 'a', 2.0: 'b'}, description='wrong mapping type'), TestCase(data={'a': 1.0, 'b': 2}, description='wrong mapping type'), ] _col2tnsr_test_cases = _shared_invalid_test_cases + [ TestCase(data={'colA': 'tnsrOpA', 'colB': 'tnsrOpB'}, - description='strict tensor name required'), + description='tensor name required'), ] _tnsr2col_test_cases = _shared_invalid_test_cases + [ TestCase(data={'tnsrOpA': 'colA', 'tnsrOpB': 'colB'}, - description='strict tensor name required'), + description='tensor name required'), ] class ParamsConverterTest(PythonUnitTestCase): @@ -64,10 +64,12 @@ def test_tf_output_mapping_converter(self): @parameterized.expand(_col2tnsr_test_cases) def test_invalid_input_mapping(self, data, description): + """ Test invalid column name to tensor name mapping """ with self.assertRaises(TypeError, msg=description): SparkDLTypeConverters.asColumnToTensorNameMap(data) @parameterized.expand(_tnsr2col_test_cases) def test_invalid_output_mapping(self, data, description): + """ Test invalid tensor name to column name mapping """ with self.assertRaises(TypeError, msg=description): SparkDLTypeConverters.asTensorNameToColumnMap(data) From 76e9fb92eef7d1bf47fce7d1e48bb2c5f5dc40bb Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 28 Sep 2017 14:46:27 -0700 Subject: [PATCH 47/80] PR comments converters simpiliciation --- python/sparkdl/param/converters.py | 107 ++++++++++++--------- python/sparkdl/param/image_params.py | 2 +- python/sparkdl/transformers/named_image.py | 6 +- python/sparkdl/transformers/tf_image.py | 18 ++-- 4 files changed, 69 insertions(+), 64 deletions(-) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index eeb2dfb0..4a0fe21d 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -33,12 +33,12 @@ class SparkDLTypeConverters(object): These methods are similar to :py:class:`spark.ml.param.TypeConverters`. They provide support for the `Params` types introduced in Spark Deep Learning Pipelines. """ + @staticmethod def toTFGraph(value): - if isinstance(value, tf.Graph): - return value - else: - raise TypeError("Could not convert %s to TensorFlow Graph" % type(value)) + if not isinstance(value, tf.Graph): + raise TypeError("Could not convert %s to tf.Graph" % type(value)) + return value @staticmethod def asColumnToTensorNameMap(value): @@ -46,19 +46,20 @@ def asColumnToTensorNameMap(value): Convert a value to a column name to :py:obj:`tf.Tensor` name mapping as a sorted list of string pairs, if possible. """ - if isinstance(value, dict): - strs_pair_seq = [] - for _maybe_col_name, _maybe_tnsr_name in value.items(): - # Check if the non-tensor value is of string type - _col_name = _get_strict_col_name(_maybe_col_name) - # Check if the tensor name is actually valid - _tnsr_name = _get_strict_tensor_name(_maybe_tnsr_name) - strs_pair_seq.append((_col_name, _tnsr_name)) + if not isinstance(value, dict): + err_msg = "Could not convert [type {}] {} to column name to tf.Tensor name mapping" + raise TypeError(err_msg.format(type(value), value)) - return sorted(strs_pair_seq) + # Convertion logic after quick type check + strs_pair_seq = [] + for _maybe_col_name, _maybe_tnsr_name in value.items(): + # Check if the non-tensor value is of string type + _check_is_str(_maybe_col_name) + # Check if the tensor name looks like a tensor name + _check_is_tensor_name(_maybe_tnsr_name) + strs_pair_seq.append((_maybe_col_name, _maybe_tnsr_name)) - err_msg = "Could not convert [type {}] {} to column name to tf.Tensor name mapping" - raise TypeError(err_msg.format(type(value), value)) + return sorted(strs_pair_seq) @staticmethod def asTensorNameToColumnMap(value): @@ -66,79 +67,89 @@ def asTensorNameToColumnMap(value): Convert a value to a :py:obj:`tf.Tensor` name to column name mapping as a sorted list of string pairs, if possible. """ - if isinstance(value, dict): - strs_pair_seq = [] - for _maybe_tnsr_name, _maybe_col_name in value.items(): - # Check if the non-tensor value is of string type - _col_name = _get_strict_col_name(_maybe_col_name) - # Check if the tensor name is actually valid - _tnsr_name = _get_strict_tensor_name(_maybe_tnsr_name) - strs_pair_seq.append((_tnsr_name, _col_name)) + if not isinstance(value, dict): + err_msg = "Could not convert [type {}] {} to tf.Tensor name to column name mapping" + raise TypeError(err_msg.format(type(value), value)) - return sorted(strs_pair_seq) + # Convertion logic after quick type check + strs_pair_seq = [] + for _maybe_tnsr_name, _maybe_col_name in value.items(): + # Check if the non-tensor value is of string type + _check_is_str(_maybe_col_name) + # Check if the tensor name looks like a tensor name + _check_is_tensor_name(_maybe_tnsr_name) + strs_pair_seq.append((_maybe_tnsr_name, _maybe_col_name)) - err_msg = "Could not convert [type {}] {} to tf.Tensor name to column name mapping" - raise TypeError(err_msg.format(type(value), value)) + return sorted(strs_pair_seq) @staticmethod def toTFHParams(value): """ Convert a value to a :py:obj:`tf.contrib.training.HParams` object, if possible. """ - if isinstance(value, tf.contrib.training.HParams): - return value - else: + if not isinstance(value, tf.contrib.training.HParams): raise TypeError("Could not convert %s to TensorFlow HParams" % type(value)) + return value + @staticmethod - def toStringOrTFTensor(value): + def toTFTensorName(value): """ Convert a value to a str or a :py:obj:`tf.Tensor` object, if possible. """ if isinstance(value, tf.Tensor): - return value + return value.name try: + _check_is_tensor_name(value) return TypeConverters.toString(value) except Exception as exc: err_msg = "Could not convert [type {}] {} to tf.Tensor or str. {}" raise TypeError(err_msg.format(type(value), value, exc)) @staticmethod - def supportedNameConverter(supportedList): + def buildCheckList(supportedList): """ Create a converter that try to check if a value is part of the supported list. :param supportedList: list, containing supported objects. :return: a converter that try to convert a value if it is part of the `supportedList`. """ + def converter(value): - if value in supportedList: - return value - err_msg = "[type {}] {} is not in the supported list: {}" - raise TypeError(err_msg.format(type(value), str(value), supportedList)) + if value not in supportedList: + err_msg = "[type {}] {} is not in the supported list: {}" + raise TypeError(err_msg.format(type(value), str(value), supportedList)) + + return value return converter @staticmethod def toKerasLoss(value): """ Convert a value to a name of Keras loss function, if possible """ - if kmutil.is_valid_loss_function(value): - return value - err_msg = "Named loss not supported in Keras: [type {}] {}" - raise ValueError(err_msg.format(type(value), value)) + # return early in for clarify as well as less indentation + if not kmutil.is_valid_loss_function(value): + err_msg = "Named loss not supported in Keras: [type {}] {}" + raise ValueError(err_msg.format(type(value), value)) + + return value @staticmethod def toKerasOptimizer(value): """ Convert a value to a name of Keras optimizer, if possible """ - if kmutil.is_valid_optimizer(value): - return value - err_msg = "Named optimizer not supported in Keras: [type {}] {}" - raise TypeError(err_msg.format(type(value), value)) + if not kmutil.is_valid_optimizer(value): + err_msg = "Named optimizer not supported in Keras: [type {}] {}" + raise TypeError(err_msg.format(type(value), value)) + + return value -def _get_strict_tensor_name(_maybe_tnsr_name): +def _check_is_tensor_name(_maybe_tnsr_name): """ Check if the input is a valid tensor name """ try: assert isinstance(_maybe_tnsr_name, six.string_types), \ "must provide a strict tensor name as input, but got {}".format(type(_maybe_tnsr_name)) - assert tfx.as_tensor_name(_maybe_tnsr_name) == _maybe_tnsr_name, \ - "input {} must be a valid tensor name".format(_maybe_tnsr_name) + + # The check is taken from TensorFlow's NodeDef protocol buffer. + # https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/core/framework/node_def.proto#L21-L25 + _, src_idx = _maybe_tnsr_name.split(":") + _ = int(src_idx) except Exception as exc: err_msg = "Can NOT convert [type {}] {} to tf.Tensor name: {}" raise TypeError(err_msg.format(type(_maybe_tnsr_name), _maybe_tnsr_name, exc)) @@ -146,7 +157,7 @@ def _get_strict_tensor_name(_maybe_tnsr_name): return _maybe_tnsr_name -def _get_strict_col_name(_maybe_col_name): +def _check_is_str(_maybe_col_name): """ Check if the given colunm name is a valid column name """ # We only check if the column name candidate is a string type if not isinstance(_maybe_col_name, six.string_types): diff --git a/python/sparkdl/param/image_params.py b/python/sparkdl/param/image_params.py index 6807ce2a..a423adae 100644 --- a/python/sparkdl/param/image_params.py +++ b/python/sparkdl/param/image_params.py @@ -107,7 +107,7 @@ class HasOutputMode(Params): "How the output column should be formatted. 'vector' for a 1-d MLlib " + "Vector of floats. 'image' to format the output to work with the image " + "tools in this package.", - typeConverter=SparkDLTypeConverters.supportedNameConverter(OUTPUT_MODES)) + typeConverter=SparkDLTypeConverters.buildCheckList(OUTPUT_MODES)) def setOutputMode(self, value): return self._set(outputMode=value) diff --git a/python/sparkdl/transformers/named_image.py b/python/sparkdl/transformers/named_image.py index 156c4e1e..76fce766 100644 --- a/python/sparkdl/transformers/named_image.py +++ b/python/sparkdl/transformers/named_image.py @@ -40,7 +40,7 @@ class DeepImagePredictor(Transformer, HasInputCol, HasOutputCol): """ modelName = Param(Params._dummy(), "modelName", "A deep learning model name", - typeConverter=SparkDLTypeConverters.supportedNameConverter(SUPPORTED_MODELS)) + typeConverter=SparkDLTypeConverters.buildCheckList(SUPPORTED_MODELS)) decodePredictions = Param(Params._dummy(), "decodePredictions", "If true, output predictions in the (class, description, probability) format", typeConverter=TypeConverters.toBoolean) @@ -125,7 +125,7 @@ class DeepImageFeaturizer(Transformer, HasInputCol, HasOutputCol): """ modelName = Param(Params._dummy(), "modelName", "A deep learning model name", - typeConverter=SparkDLTypeConverters.supportedNameConverter(SUPPORTED_MODELS)) + typeConverter=SparkDLTypeConverters.buildCheckList(SUPPORTED_MODELS)) @keyword_only def __init__(self, inputCol=None, outputCol=None, modelName=None): @@ -169,7 +169,7 @@ class _NamedImageTransformer(Transformer, HasInputCol, HasOutputCol): """ modelName = Param(Params._dummy(), "modelName", "A deep learning model name", - typeConverter=SparkDLTypeConverters.supportedNameConverter(SUPPORTED_MODELS)) + typeConverter=SparkDLTypeConverters.buildCheckList(SUPPORTED_MODELS)) featurize = Param(Params._dummy(), "featurize", "If true, output features. If false, output predictions. Either way the output is a vector.", typeConverter=TypeConverters.toBoolean) diff --git a/python/sparkdl/transformers/tf_image.py b/python/sparkdl/transformers/tf_image.py index da37fcad..13e1b28c 100644 --- a/python/sparkdl/transformers/tf_image.py +++ b/python/sparkdl/transformers/tf_image.py @@ -54,10 +54,10 @@ class TFImageTransformer(Transformer, HasInputCol, HasOutputCol, HasOutputMode): typeConverter=SparkDLTypeConverters.toTFGraph) inputTensor = Param(Params._dummy(), "inputTensor", "A TensorFlow tensor object or name representing the input image", - typeConverter=SparkDLTypeConverters.toStringOrTFTensor) + typeConverter=SparkDLTypeConverters.toTFTensorName) outputTensor = Param(Params._dummy(), "outputTensor", "A TensorFlow tensor object or name representing the output", - typeConverter=SparkDLTypeConverters.toStringOrTFTensor) + typeConverter=SparkDLTypeConverters.toTFTensorName) @keyword_only def __init__(self, inputCol=None, outputCol=None, graph=None, @@ -99,18 +99,12 @@ def getGraph(self): return self.getOrDefault(self.graph) def getInputTensor(self): - tensor_or_name = self.getOrDefault(self.inputTensor) - if isinstance(tensor_or_name, tf.Tensor): - return tensor_or_name - else: - return self.getGraph().get_tensor_by_name(tensor_or_name) + tensor_name = self.getOrDefault(self.inputTensor) + return self.getGraph().get_tensor_by_name(tensor_name) def getOutputTensor(self): - tensor_or_name = self.getOrDefault(self.outputTensor) - if isinstance(tensor_or_name, tf.Tensor): - return tensor_or_name - else: - return self.getGraph().get_tensor_by_name(tensor_or_name) + tensor_name = self.getOrDefault(self.outputTensor) + return self.getGraph().get_tensor_by_name(tensor_name) def _transform(self, dataset): graph = self.getGraph() From 66507f4347fa5900b2de48b363a63d34f28e8b49 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 28 Sep 2017 18:29:19 -0700 Subject: [PATCH 48/80] tf_image inputTensor default setter bug-fix --- python/sparkdl/param/converters.py | 29 ++++++++++--------- .../transformers/keras_applications.py | 1 - python/sparkdl/transformers/tf_image.py | 27 +++++++++-------- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index 4a0fe21d..b23490ff 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -92,14 +92,15 @@ def toTFHParams(value): @staticmethod def toTFTensorName(value): - """ Convert a value to a str or a :py:obj:`tf.Tensor` object, if possible. """ + """ Convert a value to a :py:obj:`tf.Tensor` name, if possible. """ if isinstance(value, tf.Tensor): return value.name try: - _check_is_tensor_name(value) - return TypeConverters.toString(value) + _maybe_tnsr_name = TypeConverters.toString(value) + _check_is_tensor_name(_maybe_tnsr_name) + return _maybe_tnsr_name except Exception as exc: - err_msg = "Could not convert [type {}] {} to tf.Tensor or str. {}" + err_msg = "Could not convert [type {}] {} to tf.Tensor name. {}" raise TypeError(err_msg.format(type(value), value, exc)) @staticmethod @@ -142,19 +143,19 @@ def toKerasOptimizer(value): def _check_is_tensor_name(_maybe_tnsr_name): """ Check if the input is a valid tensor name """ - try: - assert isinstance(_maybe_tnsr_name, six.string_types), \ - "must provide a strict tensor name as input, but got {}".format(type(_maybe_tnsr_name)) + assert isinstance(_maybe_tnsr_name, six.string_types), \ + "expect tensor name to be of string type, but got [type {}]".format(type(_maybe_tnsr_name)) - # The check is taken from TensorFlow's NodeDef protocol buffer. - # https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/core/framework/node_def.proto#L21-L25 + # The check is taken from TensorFlow's NodeDef protocol buffer. + # https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/core/framework/node_def.proto#L21-L25 + try: _, src_idx = _maybe_tnsr_name.split(":") _ = int(src_idx) - except Exception as exc: - err_msg = "Can NOT convert [type {}] {} to tf.Tensor name: {}" - raise TypeError(err_msg.format(type(_maybe_tnsr_name), _maybe_tnsr_name, exc)) - else: - return _maybe_tnsr_name + except Exception: + err_msg = "Tensor name must be of type :, but got {}" + raise TypeError(err_msg.format(_maybe_tnsr_name)) + + return _maybe_tnsr_name def _check_is_str(_maybe_col_name): diff --git a/python/sparkdl/transformers/keras_applications.py b/python/sparkdl/transformers/keras_applications.py index 733ac654..50c30d4f 100644 --- a/python/sparkdl/transformers/keras_applications.py +++ b/python/sparkdl/transformers/keras_applications.py @@ -109,4 +109,3 @@ def _testKerasModel(self, include_top): "InceptionV3": InceptionV3Model, "Xception": XceptionModel } - diff --git a/python/sparkdl/transformers/tf_image.py b/python/sparkdl/transformers/tf_image.py index 13e1b28c..943af6e8 100644 --- a/python/sparkdl/transformers/tf_image.py +++ b/python/sparkdl/transformers/tf_image.py @@ -28,6 +28,12 @@ import sparkdl.utils.jvmapi as JVMAPI import sparkdl.graph.utils as tfx +__all__ = ['TFImageTransformer'] + +IMAGE_INPUT_TENSOR_NAME = tfx.as_tensor_name(utils.IMAGE_INPUT_PLACEHOLDER_NAME) +USER_GRAPH_NAMESPACE = 'given' +NEW_OUTPUT_PREFIX = 'sdl_flattened' + class TFImageTransformer(Transformer, HasInputCol, HasOutputCol, HasOutputMode): """ Applies the Tensorflow graph to the image column in DataFrame. @@ -47,9 +53,6 @@ class TFImageTransformer(Transformer, HasInputCol, HasOutputCol, HasOutputMode): since a new session is created inside this transformer. """ - USER_GRAPH_NAMESPACE = 'given' - NEW_OUTPUT_PREFIX = 'sdl_flattened' - graph = Param(Params._dummy(), "graph", "A TensorFlow computation graph", typeConverter=SparkDLTypeConverters.toTFGraph) inputTensor = Param(Params._dummy(), "inputTensor", @@ -61,28 +64,28 @@ class TFImageTransformer(Transformer, HasInputCol, HasOutputCol, HasOutputMode): @keyword_only def __init__(self, inputCol=None, outputCol=None, graph=None, - inputTensor=utils.IMAGE_INPUT_PLACEHOLDER_NAME, outputTensor=None, + inputTensor=IMAGE_INPUT_TENSOR_NAME, outputTensor=None, outputMode="vector"): """ __init__(self, inputCol=None, outputCol=None, graph=None, - inputTensor=utils.IMAGE_INPUT_PLACEHOLDER_NAME, outputTensor=None, + inputTensor=IMAGE_INPUT_TENSOR_NAME, outputTensor=None, outputMode="vector") """ super(TFImageTransformer, self).__init__() - self._setDefault(inputTensor=utils.IMAGE_INPUT_PLACEHOLDER_NAME) - self._setDefault(outputMode="vector") kwargs = self._input_kwargs self.setParams(**kwargs) @keyword_only def setParams(self, inputCol=None, outputCol=None, graph=None, - inputTensor=utils.IMAGE_INPUT_PLACEHOLDER_NAME, outputTensor=None, + inputTensor=IMAGE_INPUT_TENSOR_NAME, outputTensor=None, outputMode="vector"): """ setParams(self, inputCol=None, outputCol=None, graph=None, - inputTensor=utils.IMAGE_INPUT_PLACEHOLDER_NAME, outputTensor=None, + inputTensor=IMAGE_INPUT_TENSOR_NAME, outputTensor=None, outputMode="vector") """ + self._setDefault(inputTensor=IMAGE_INPUT_TENSOR_NAME) + self._setDefault(outputMode="vector") kwargs = self._input_kwargs return self._set(**kwargs) @@ -179,7 +182,7 @@ def _addReshapeLayers(self, tf_graph, dtype="uint8"): # Add on the original graph tf.import_graph_def(gdef, input_map={input_tensor_name: image_reshaped_expanded}, return_elements=[self.getOutputTensor().name], - name=self.USER_GRAPH_NAMESPACE) + name=USER_GRAPH_NAMESPACE) # Flatten the output for tensorframes output_node = g.get_tensor_by_name(self._getOriginalOutputTensorName()) @@ -198,10 +201,10 @@ def _stripGraph(self, tf_graph): return g def _getOriginalOutputTensorName(self): - return self.USER_GRAPH_NAMESPACE + '/' + self.getOutputTensor().name + return USER_GRAPH_NAMESPACE + '/' + self.getOutputTensor().name def _getFinalOutputTensorName(self): - return self.NEW_OUTPUT_PREFIX + '_' + self.getOutputTensor().name + return NEW_OUTPUT_PREFIX + '_' + self.getOutputTensor().name def _getFinalOutputOpName(self): return tfx.as_op_name(self._getFinalOutputTensorName()) From d239a5ad1bee9617e564e62bc6d144d9cc8cd8bd Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 28 Sep 2017 21:57:34 -0700 Subject: [PATCH 49/80] use type error, always --- python/sparkdl/param/converters.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index b23490ff..b8af76ff 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -143,8 +143,9 @@ def toKerasOptimizer(value): def _check_is_tensor_name(_maybe_tnsr_name): """ Check if the input is a valid tensor name """ - assert isinstance(_maybe_tnsr_name, six.string_types), \ - "expect tensor name to be of string type, but got [type {}]".format(type(_maybe_tnsr_name)) + if not isinstance(_maybe_tnsr_name, six.string_types): + err_msg = "expect tensor name to be of string type, but got [type {}]" + raise TypeError(err_msg.format(type(_maybe_tnsr_name))) # The check is taken from TensorFlow's NodeDef protocol buffer. # https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/core/framework/node_def.proto#L21-L25 From 5947c9cdc2a85a8264611a8e08ddd6bea8215833 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 28 Sep 2017 22:37:16 -0700 Subject: [PATCH 50/80] doc updates --- python/sparkdl/param/converters.py | 18 +++++++++++++----- python/tests/tests.py | 3 ++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index b8af76ff..758fb81b 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -13,29 +13,36 @@ # limitations under the License. # +# pylint: disable=wrong-spelling-in-docstring,invalid-name,import-error + +""" SparkDLTypeConverters +Type conversion utilities for definition Spark Deep Learning related MLlib `Params`. +""" + import six import tensorflow as tf from pyspark.ml.param import TypeConverters -import sparkdl.graph.utils as tfx import sparkdl.utils.keras_model as kmutil __all__ = ['SparkDLTypeConverters'] - class SparkDLTypeConverters(object): """ .. note:: DeveloperApi - Factory methods for common type conversion functions for :py:func:`Param.typeConverter`. + Factory methods for type conversion functions for :py:func:`Param.typeConverter`. These methods are similar to :py:class:`spark.ml.param.TypeConverters`. They provide support for the `Params` types introduced in Spark Deep Learning Pipelines. """ @staticmethod def toTFGraph(value): + """ + Convert a value to a :py:obj:`tf.Graph` object, if possible. + """ if not isinstance(value, tf.Graph): raise TypeError("Could not convert %s to tf.Graph" % type(value)) return value @@ -50,7 +57,7 @@ def asColumnToTensorNameMap(value): err_msg = "Could not convert [type {}] {} to column name to tf.Tensor name mapping" raise TypeError(err_msg.format(type(value), value)) - # Convertion logic after quick type check + # Conversion logic after quick type check strs_pair_seq = [] for _maybe_col_name, _maybe_tnsr_name in value.items(): # Check if the non-tensor value is of string type @@ -71,7 +78,7 @@ def asTensorNameToColumnMap(value): err_msg = "Could not convert [type {}] {} to tf.Tensor name to column name mapping" raise TypeError(err_msg.format(type(value), value)) - # Convertion logic after quick type check + # Conversion logic after quick type check strs_pair_seq = [] for _maybe_tnsr_name, _maybe_col_name in value.items(): # Check if the non-tensor value is of string type @@ -113,6 +120,7 @@ def buildCheckList(supportedList): """ def converter(value): + """ Implementing the conversion logic """ if value not in supportedList: err_msg = "[type {}] {} is not in the supported list: {}" raise TypeError(err_msg.format(type(value), str(value), supportedList)) diff --git a/python/tests/tests.py b/python/tests/tests.py index ae7cec3e..9492a07b 100644 --- a/python/tests/tests.py +++ b/python/tests/tests.py @@ -30,7 +30,8 @@ from pyspark.sql import SparkSession class PythonUnitTestCase(unittest.TestCase): - # Just the plain test unittest.TestCase, but won't have to do import check + # We try to use unittest2 for python 2.6 or earlier + # This class is created to avoid replicating this logic in various places. pass class SparkDLTestCase(unittest.TestCase): From 202e7eab212e6641ca6936ecaf1c80339f0e7cc1 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 28 Sep 2017 23:16:46 -0700 Subject: [PATCH 51/80] refactoring tfx API --- python/sparkdl/graph/builder.py | 11 ++-- python/sparkdl/graph/tensorframes_udf.py | 14 ++--- python/sparkdl/graph/utils.py | 62 +++++++++------------- python/sparkdl/transformers/keras_image.py | 8 +-- python/sparkdl/transformers/tf_image.py | 6 +-- python/tests/graph/test_builder.py | 18 +++---- python/tests/graph/test_pieces.py | 4 +- python/tests/graph/test_utils.py | 24 ++++----- python/tests/udf/keras_sql_udf_test.py | 3 +- 9 files changed, 67 insertions(+), 83 deletions(-) diff --git a/python/sparkdl/graph/builder.py b/python/sparkdl/graph/builder.py index 86c3b3ce..54516a78 100644 --- a/python/sparkdl/graph/builder.py +++ b/python/sparkdl/graph/builder.py @@ -87,8 +87,8 @@ def asGraphFunction(self, inputs, outputs, strip_and_freeze=True): else: gdef = self.graph.as_graph_def(add_shapes=True) return GraphFunction(graph_def=gdef, - input_names=[tfx.validated_input(self.graph, elem) for elem in inputs], - output_names=[tfx.validated_output(self.graph, elem) for elem in outputs]) + input_names=[tfx.validated_input(elem, self.graph) for elem in inputs], + output_names=[tfx.validated_output(elem, self.graph) for elem in outputs]) def importGraphFunction(self, gfn, input_map=None, prefix="GFN-IMPORT", **gdef_kargs): """ @@ -130,8 +130,8 @@ def importGraphFunction(self, gfn, input_map=None, prefix="GFN-IMPORT", **gdef_k return_elements=gfn.output_names, name=scope_name, **gdef_kargs) - feeds = [tfx.get_tensor(self.graph, name) for name in input_names] - fetches = [tfx.get_tensor(self.graph, name) for name in output_names] + feeds = [tfx.get_tensor(name, self.graph) for name in input_names] + fetches = [tfx.get_tensor(name, self.graph) for name in output_names] return (feeds, fetches) @@ -233,7 +233,7 @@ def fromList(cls, functions): _, first_gfn = functions[0] feeds, _ = issn.importGraphFunction(first_gfn, prefix='') for tnsr in feeds: - name = tfx.op_name(issn.graph, tnsr) + name = tfx.op_name(tnsr, issn.graph) first_input_info.append((tnsr.dtype, tnsr.shape, name)) # TODO: make sure that this graph is not reused to prevent name conflict # Report error if the graph is not manipulated by anyone else @@ -268,4 +268,3 @@ def fromList(cls, functions): gfn = issn.asGraphFunction(first_inputs, last_outputs) return gfn - diff --git a/python/sparkdl/graph/tensorframes_udf.py b/python/sparkdl/graph/tensorframes_udf.py index 54027b8d..aa1531b4 100644 --- a/python/sparkdl/graph/tensorframes_udf.py +++ b/python/sparkdl/graph/tensorframes_udf.py @@ -33,7 +33,7 @@ def makeGraphUDF(graph, udf_name, fetches, feeds_to_fields_map=None, blocked=Fal .. code-block:: python from sparkdl.graph.tensorframes_udf import makeUDF - + with IsolatedSession() as issn: x = tf.placeholder(tf.double, shape=[], name="input_x") z = tf.add(x, 3, name='z') @@ -45,7 +45,7 @@ def makeGraphUDF(graph, udf_name, fetches, feeds_to_fields_map=None, blocked=Fal df = spark.createDataFrame([Row(xCol=float(x)) for x in range(100)]) df.createOrReplaceTempView("my_float_table") - spark.sql("select my_tensorflow_udf(xCol) as zCol from my_float_table").show() + spark.sql("select my_tensorflow_udf(xCol) as zCol from my_float_table").show() :param graph: :py:class:`tf.Graph`, a TensorFlow Graph :param udf_name: str, name of the SQL UDF @@ -77,18 +77,18 @@ def makeGraphUDF(graph, udf_name, fetches, feeds_to_fields_map=None, blocked=Fal tfs.core._add_graph(graph, jvm_builder) # Obtain the fetches and their shapes - fetch_names = [tfx.tensor_name(graph, fetch) for fetch in fetches] - fetch_shapes = [tfx.get_shape(graph, fetch) for fetch in fetches] + fetch_names = [tfx.tensor_name(fetch, graph) for fetch in fetches] + fetch_shapes = [tfx.get_shape(fetch, graph) for fetch in fetches] # Traverse the graph nodes and obtain all the placeholders and their shapes placeholder_names = [] placeholder_shapes = [] for node in graph.as_graph_def(add_shapes=True).node: if len(node.input) == 0 and str(node.op) == 'Placeholder': - tnsr_name = tfx.tensor_name(graph, node.name) + tnsr_name = tfx.tensor_name(node.name, graph) tnsr = graph.get_tensor_by_name(tnsr_name) try: - tnsr_shape = tfx.get_shape(graph, tnsr) + tnsr_shape = tfx.get_shape(tnsr, graph) placeholder_names.append(tnsr_name) placeholder_shapes.append(tnsr_shape) except ValueError: @@ -98,7 +98,7 @@ def makeGraphUDF(graph, udf_name, fetches, feeds_to_fields_map=None, blocked=Fal jvm_builder.shape(fetch_names + placeholder_names, fetch_shapes + placeholder_shapes) jvm_builder.fetches(fetch_names) # Passing feeds to TensorFrames - placeholder_op_names = [tfx.op_name(graph, name) for name in placeholder_names] + placeholder_op_names = [tfx.op_name(name, graph) for name in placeholder_names] # Passing the graph input to DataFrame column mapping and additional placeholder names tfs.core._add_inputs(jvm_builder, feeds_to_fields_map, placeholder_op_names) diff --git a/python/sparkdl/graph/utils.py b/python/sparkdl/graph/utils.py index 8b10c307..b3f56836 100644 --- a/python/sparkdl/graph/utils.py +++ b/python/sparkdl/graph/utils.py @@ -40,7 +40,7 @@ def validated_graph(graph): assert isinstance(graph, tf.Graph), 'must provide tf.Graph, but get {}'.format(type(graph)) return graph -def get_shape(graph, tfobj_or_name): +def get_shape(tfobj_or_name, graph): """ Return the shape of the tensor as a list @@ -48,10 +48,10 @@ def get_shape(graph, tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ graph = validated_graph(graph) - _shape = get_tensor(graph, tfobj_or_name).get_shape().as_list() + _shape = get_tensor(tfobj_or_name, graph).get_shape().as_list() return [-1 if x is None else x for x in _shape] -def get_op(graph, tfobj_or_name): +def get_op(tfobj_or_name, graph): """ Get a tf.Operation object @@ -65,14 +65,14 @@ def get_op(graph, tfobj_or_name): if isinstance(tfobj_or_name, tf.Tensor): name = tfobj_or_name.name if not isinstance(name, six.string_types): - raise TypeError('invalid op request for {} of {}'.format(name, type(name))) - _op_name = as_op_name(name) + raise TypeError('invalid op request for [type {}] {}'.format(type(name), name)) + _op_name = op_name(name, graph=None) op = graph.get_operation_by_name(_op_name) assert op is not None, \ 'cannot locate op {} in current graph'.format(_op_name) return op -def get_tensor(graph, tfobj_or_name): +def get_tensor(tfobj_or_name, graph): """ Get a tf.Tensor object @@ -87,13 +87,13 @@ def get_tensor(graph, tfobj_or_name): name = tfobj_or_name.name if not isinstance(name, six.string_types): raise TypeError('invalid tensor request for {} of {}'.format(name, type(name))) - _tensor_name = as_tensor_name(name) + _tensor_name = tensor_name(name, graph=None) tnsr = graph.get_tensor_by_name(_tensor_name) assert tnsr is not None, \ 'cannot locate tensor {} in current graph'.format(_tensor_name) return tnsr -def as_tensor_name(tfobj_or_name): +def tensor_name(tfobj_or_name, graph=None): """ Derive tf.Tensor name from an op/tensor name. If the input is a name, we do not check if the tensor exist @@ -101,6 +101,10 @@ def as_tensor_name(tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ + # If `graph` is provided, directly get the graph operation + if graph is not None: + return get_tensor(tfobj_or_name, graph).name + # If `graph` is absent, check if other cases if isinstance(tfobj_or_name, six.string_types): # If input is a string, assume it is a name and infer the corresponding tensor name. # WARNING: this depends on TensorFlow's tensor naming convention @@ -111,12 +115,11 @@ def as_tensor_name(tfobj_or_name): name += ":0" return name elif hasattr(tfobj_or_name, 'graph'): - tfobj = tfobj_or_name - return get_tensor(tfobj.graph, tfobj).name + return get_tensor(tfobj_or_name, tfobj_or_name.graph).name else: raise TypeError('invalid tf.Tensor name query type {}'.format(type(tfobj_or_name))) -def as_op_name(tfobj_or_name): +def op_name(tfobj_or_name, graph=None): """ Derive tf.Operation name from an op/tensor name. If the input is a name, we do not check if the operation exist @@ -124,6 +127,10 @@ def as_op_name(tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ + # If `graph` is provided, directly get the graph operation + if graph is not None: + return get_op(tfobj_or_name, graph).name + # If `graph` is absent, check if other cases if isinstance(tfobj_or_name, six.string_types): # If input is a string, assume it is a name and infer the corresponding operation name. # WARNING: this depends on TensorFlow's operation naming convention @@ -132,32 +139,11 @@ def as_op_name(tfobj_or_name): assert len(name_parts) <= 2, name_parts return name_parts[0] elif hasattr(tfobj_or_name, 'graph'): - tfobj = tfobj_or_name - return get_op(tfobj.graph, tfobj).name + return get_op(tfobj_or_name, tfobj_or_name.graph).name else: raise TypeError('invalid tf.Operation name query type {}'.format(type(tfobj_or_name))) -def op_name(graph, tfobj_or_name): - """ - Get the name of a tf.Operation - - :param graph: tf.Graph, a TensorFlow Graph object - :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either - """ - graph = validated_graph(graph) - return get_op(graph, tfobj_or_name).name - -def tensor_name(graph, tfobj_or_name): - """ - Get the name of a tf.Tensor - - :param graph: tf.Graph, a TensorFlow Graph object - :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either - """ - graph = validated_graph(graph) - return get_tensor(graph, tfobj_or_name).name - -def validated_output(graph, tfobj_or_name): +def validated_output(tfobj_or_name, graph): """ Validate and return the output names useable GraphFunction @@ -165,9 +151,9 @@ def validated_output(graph, tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ graph = validated_graph(graph) - return op_name(graph, tfobj_or_name) + return op_name(tfobj_or_name, graph) -def validated_input(graph, tfobj_or_name): +def validated_input(tfobj_or_name, graph): """ Validate and return the input names useable GraphFunction @@ -175,7 +161,7 @@ def validated_input(graph, tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ graph = validated_graph(graph) - name = op_name(graph, tfobj_or_name) + name = op_name(tfobj_or_name, graph) op = graph.get_operation_by_name(name) assert 'Placeholder' == op.type, \ ('input must be Placeholder, but get', op.type) @@ -202,7 +188,7 @@ def strip_and_freeze_until(fetches, graph, sess=None, return_graph=False): gdef_frozen = tf.graph_util.convert_variables_to_constants( sess, graph.as_graph_def(add_shapes=True), - [op_name(graph, tnsr) for tnsr in fetches]) + [op_name(tnsr, graph) for tnsr in fetches]) if should_close_session: sess.close() diff --git a/python/sparkdl/transformers/keras_image.py b/python/sparkdl/transformers/keras_image.py index de10fc87..3c2762d9 100644 --- a/python/sparkdl/transformers/keras_image.py +++ b/python/sparkdl/transformers/keras_image.py @@ -76,14 +76,14 @@ def _transform(self, dataset): return transformer.transform(image_df).drop(self._loadedImageCol()) def _loadTFGraph(self): - with KSessionWrap() as (sess, g): + with KSessionWrap() as (sess, graph): assert K.backend() == "tensorflow", \ "Keras backend is not tensorflow but KerasImageTransformer only supports " + \ "tensorflow-backed Keras models." - with g.as_default(): + with graph.as_default(): K.set_learning_phase(0) # Testing phase model = load_model(self.getModelFile()) - out_op_name = tfx.op_name(g, model.output) + out_op_name = tfx.op_name(model.output, graph) self._inputTensor = model.input.name self._outputTensor = model.output.name - return tfx.strip_and_freeze_until([out_op_name], g, sess, return_graph=True) + return tfx.strip_and_freeze_until([out_op_name], graph, sess, return_graph=True) diff --git a/python/sparkdl/transformers/tf_image.py b/python/sparkdl/transformers/tf_image.py index 943af6e8..152a7fea 100644 --- a/python/sparkdl/transformers/tf_image.py +++ b/python/sparkdl/transformers/tf_image.py @@ -30,7 +30,7 @@ __all__ = ['TFImageTransformer'] -IMAGE_INPUT_TENSOR_NAME = tfx.as_tensor_name(utils.IMAGE_INPUT_PLACEHOLDER_NAME) +IMAGE_INPUT_TENSOR_NAME = tfx.tensor_name(utils.IMAGE_INPUT_PLACEHOLDER_NAME) USER_GRAPH_NAMESPACE = 'given' NEW_OUTPUT_PREFIX = 'sdl_flattened' @@ -136,7 +136,7 @@ def _transform(self, dataset): "__sdl_image_data") ) - tfs_output_name = tfx.op_name(final_graph, output_tensor) + tfs_output_name = tfx.op_name(output_tensor, final_graph) original_output_name = self._getOriginalOutputTensorName() output_shape = final_graph.get_tensor_by_name(original_output_name).shape output_mode = self.getOrDefault(self.outputMode) @@ -207,7 +207,7 @@ def _getFinalOutputTensorName(self): return NEW_OUTPUT_PREFIX + '_' + self.getOutputTensor().name def _getFinalOutputOpName(self): - return tfx.as_op_name(self._getFinalOutputTensorName()) + return tfx.op_name(self._getFinalOutputTensorName()) def _convertOutputToImage(self, df, tfs_output_col, output_shape): assert len(output_shape) == 4, str(output_shape) + " does not have 4 dimensions" diff --git a/python/tests/graph/test_builder.py b/python/tests/graph/test_builder.py index b0736896..93b3c9f5 100644 --- a/python/tests/graph/test_builder.py +++ b/python/tests/graph/test_builder.py @@ -78,15 +78,15 @@ def test_get_graph_elements(self): z = tf.add(x, 3, name='z') g = issn.graph - self.assertEqual(tfx.get_tensor(g, z), z) - self.assertEqual(tfx.get_tensor(g, x), x) - self.assertEqual(g.get_tensor_by_name("x:0"), tfx.get_tensor(g, x)) - self.assertEqual("x:0", tfx.tensor_name(g, x)) - self.assertEqual(g.get_operation_by_name("x"), tfx.get_op(g, x)) - self.assertEqual("x", tfx.op_name(g, x)) - self.assertEqual("z", tfx.op_name(g, z)) - self.assertEqual(tfx.tensor_name(g, z), "z:0") - self.assertEqual(tfx.tensor_name(g, x), "x:0") + self.assertEqual(tfx.get_tensor(z, g), z) + self.assertEqual(tfx.get_tensor(x, g), x) + self.assertEqual(g.get_tensor_by_name("x:0"), tfx.get_tensor(x, g)) + self.assertEqual("x:0", tfx.tensor_name(x, g)) + self.assertEqual(g.get_operation_by_name("x"), tfx.get_op(x, g)) + self.assertEqual("x", tfx.op_name(x, g)) + self.assertEqual("z", tfx.op_name(z, g)) + self.assertEqual(tfx.tensor_name(z, g), "z:0") + self.assertEqual(tfx.tensor_name(x, g), "x:0") def test_import_export_graph_function(self): """ Function import and export must be consistent """ diff --git a/python/tests/graph/test_pieces.py b/python/tests/graph/test_pieces.py index 1497d137..9d659265 100644 --- a/python/tests/graph/test_pieces.py +++ b/python/tests/graph/test_pieces.py @@ -55,7 +55,7 @@ def exec_gfn_spimg_decode(spimg_dict, img_dtype): gfn = gfac.buildSpImageConverter(img_dtype) with IsolatedSession() as issn: feeds, fetches = issn.importGraphFunction(gfn, prefix="") - feed_dict = dict((tnsr, spimg_dict[tfx.op_name(issn.graph, tnsr)]) for tnsr in feeds) + feed_dict = dict((tnsr, spimg_dict[tfx.op_name(tnsr, issn.graph)]) for tnsr in feeds) img_out = issn.run(fetches[0], feed_dict=feed_dict) return img_out @@ -159,7 +159,7 @@ def test_pipeline(self): with IsolatedSession() as issn: # Need blank import scope name so that spimg fields match the input names feeds, fetches = issn.importGraphFunction(piped_model, prefix="") - feed_dict = dict((tnsr, spimg_input_dict[tfx.op_name(issn.graph, tnsr)]) for tnsr in feeds) + feed_dict = dict((tnsr, spimg_input_dict[tfx.op_name(tnsr, issn.graph)]) for tnsr in feeds) preds_tgt = issn.run(fetches[0], feed_dict=feed_dict) # Uncomment the line below to see the graph # tfx.write_visualization_html(issn.graph, diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py index de75036f..94c613cc 100644 --- a/python/tests/graph/test_utils.py +++ b/python/tests/graph/test_utils.py @@ -25,13 +25,13 @@ def test_infer_graph_element_names(self): for tnsr_idx in range(17): op_name = 'someOp' tnsr_name = '{}:{}'.format(op_name, tnsr_idx) - self.assertEqual(op_name, tfx.as_op_name(tnsr_name)) - self.assertEqual(tnsr_name, tfx.as_tensor_name(tnsr_name)) + self.assertEqual(op_name, tfx.op_name(tnsr_name)) + self.assertEqual(tnsr_name, tfx.tensor_name(tnsr_name)) with self.assertRaises(TypeError): for wrong_value in [7, 1.2, tf.Graph()]: - tfx.as_op_name(wrong_value) - tfx.as_tensor_name(wrong_value) + tfx.op_name(wrong_value) + tfx.tensor_name(wrong_value) def test_get_graph_elements(self): op_name = 'someConstOp' @@ -39,11 +39,11 @@ def test_get_graph_elements(self): tnsr = tf.constant(1427.08, name=op_name) graph = tnsr.graph - self.assertEqual(op_name, tfx.as_op_name(tnsr)) - self.assertEqual(op_name, tfx.op_name(graph, tnsr)) - self.assertEqual(tnsr_name, tfx.as_tensor_name(tnsr)) - self.assertEqual(tnsr_name, tfx.tensor_name(graph, tnsr)) - self.assertEqual(tnsr, tfx.get_tensor(graph, tnsr)) - self.assertEqual(tnsr.op, tfx.get_op(graph, tnsr)) - self.assertEqual(graph, tfx.get_op(graph, tnsr).graph) - self.assertEqual(graph, tfx.get_tensor(graph, tnsr).graph) + self.assertEqual(op_name, tfx.op_name(tnsr)) + self.assertEqual(op_name, tfx.op_name(tnsr, graph)) + self.assertEqual(tnsr_name, tfx.tensor_name(tnsr)) + self.assertEqual(tnsr_name, tfx.tensor_name(tnsr, graph)) + self.assertEqual(tnsr, tfx.get_tensor(tnsr, graph)) + self.assertEqual(tnsr.op, tfx.get_op(tnsr, graph)) + self.assertEqual(graph, tfx.get_op(tnsr, graph).graph) + self.assertEqual(graph, tfx.get_tensor(tnsr, graph).graph) diff --git a/python/tests/udf/keras_sql_udf_test.py b/python/tests/udf/keras_sql_udf_test.py index d1473b3c..5c67c854 100644 --- a/python/tests/udf/keras_sql_udf_test.py +++ b/python/tests/udf/keras_sql_udf_test.py @@ -66,7 +66,7 @@ def test_simple_keras_udf(self): makeGraphUDF(issn.graph, 'my_keras_model_udf', model.outputs, - {tfx.op_name(issn.graph, model.inputs[0]): 'image_col'}) + {tfx.op_name(model.inputs[0], issn.graph): 'image_col'}) # Run the training procedure # Export the graph in this IsolatedSession as a GraphFunction # gfn = issn.asGraphFunction(model.inputs, model.outputs) @@ -168,4 +168,3 @@ def test_map_blocks_sql_1(self): data2 = df2.collect() assert len(data2) == 5, data2 assert data2[0].z == 3.0, data2 - From aef1661331b0188e532b00aced94991379a5cdce Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 28 Sep 2017 23:16:46 -0700 Subject: [PATCH 52/80] refactoring tfx API --- python/sparkdl/graph/builder.py | 11 ++-- python/sparkdl/graph/tensorframes_udf.py | 14 ++--- python/sparkdl/graph/utils.py | 62 +++++++++------------- python/sparkdl/transformers/keras_image.py | 8 +-- python/sparkdl/transformers/tf_image.py | 6 +-- python/tests/graph/test_builder.py | 18 +++---- python/tests/graph/test_pieces.py | 4 +- python/tests/graph/test_utils.py | 24 ++++----- python/tests/udf/keras_sql_udf_test.py | 3 +- 9 files changed, 67 insertions(+), 83 deletions(-) diff --git a/python/sparkdl/graph/builder.py b/python/sparkdl/graph/builder.py index 86c3b3ce..54516a78 100644 --- a/python/sparkdl/graph/builder.py +++ b/python/sparkdl/graph/builder.py @@ -87,8 +87,8 @@ def asGraphFunction(self, inputs, outputs, strip_and_freeze=True): else: gdef = self.graph.as_graph_def(add_shapes=True) return GraphFunction(graph_def=gdef, - input_names=[tfx.validated_input(self.graph, elem) for elem in inputs], - output_names=[tfx.validated_output(self.graph, elem) for elem in outputs]) + input_names=[tfx.validated_input(elem, self.graph) for elem in inputs], + output_names=[tfx.validated_output(elem, self.graph) for elem in outputs]) def importGraphFunction(self, gfn, input_map=None, prefix="GFN-IMPORT", **gdef_kargs): """ @@ -130,8 +130,8 @@ def importGraphFunction(self, gfn, input_map=None, prefix="GFN-IMPORT", **gdef_k return_elements=gfn.output_names, name=scope_name, **gdef_kargs) - feeds = [tfx.get_tensor(self.graph, name) for name in input_names] - fetches = [tfx.get_tensor(self.graph, name) for name in output_names] + feeds = [tfx.get_tensor(name, self.graph) for name in input_names] + fetches = [tfx.get_tensor(name, self.graph) for name in output_names] return (feeds, fetches) @@ -233,7 +233,7 @@ def fromList(cls, functions): _, first_gfn = functions[0] feeds, _ = issn.importGraphFunction(first_gfn, prefix='') for tnsr in feeds: - name = tfx.op_name(issn.graph, tnsr) + name = tfx.op_name(tnsr, issn.graph) first_input_info.append((tnsr.dtype, tnsr.shape, name)) # TODO: make sure that this graph is not reused to prevent name conflict # Report error if the graph is not manipulated by anyone else @@ -268,4 +268,3 @@ def fromList(cls, functions): gfn = issn.asGraphFunction(first_inputs, last_outputs) return gfn - diff --git a/python/sparkdl/graph/tensorframes_udf.py b/python/sparkdl/graph/tensorframes_udf.py index 54027b8d..aa1531b4 100644 --- a/python/sparkdl/graph/tensorframes_udf.py +++ b/python/sparkdl/graph/tensorframes_udf.py @@ -33,7 +33,7 @@ def makeGraphUDF(graph, udf_name, fetches, feeds_to_fields_map=None, blocked=Fal .. code-block:: python from sparkdl.graph.tensorframes_udf import makeUDF - + with IsolatedSession() as issn: x = tf.placeholder(tf.double, shape=[], name="input_x") z = tf.add(x, 3, name='z') @@ -45,7 +45,7 @@ def makeGraphUDF(graph, udf_name, fetches, feeds_to_fields_map=None, blocked=Fal df = spark.createDataFrame([Row(xCol=float(x)) for x in range(100)]) df.createOrReplaceTempView("my_float_table") - spark.sql("select my_tensorflow_udf(xCol) as zCol from my_float_table").show() + spark.sql("select my_tensorflow_udf(xCol) as zCol from my_float_table").show() :param graph: :py:class:`tf.Graph`, a TensorFlow Graph :param udf_name: str, name of the SQL UDF @@ -77,18 +77,18 @@ def makeGraphUDF(graph, udf_name, fetches, feeds_to_fields_map=None, blocked=Fal tfs.core._add_graph(graph, jvm_builder) # Obtain the fetches and their shapes - fetch_names = [tfx.tensor_name(graph, fetch) for fetch in fetches] - fetch_shapes = [tfx.get_shape(graph, fetch) for fetch in fetches] + fetch_names = [tfx.tensor_name(fetch, graph) for fetch in fetches] + fetch_shapes = [tfx.get_shape(fetch, graph) for fetch in fetches] # Traverse the graph nodes and obtain all the placeholders and their shapes placeholder_names = [] placeholder_shapes = [] for node in graph.as_graph_def(add_shapes=True).node: if len(node.input) == 0 and str(node.op) == 'Placeholder': - tnsr_name = tfx.tensor_name(graph, node.name) + tnsr_name = tfx.tensor_name(node.name, graph) tnsr = graph.get_tensor_by_name(tnsr_name) try: - tnsr_shape = tfx.get_shape(graph, tnsr) + tnsr_shape = tfx.get_shape(tnsr, graph) placeholder_names.append(tnsr_name) placeholder_shapes.append(tnsr_shape) except ValueError: @@ -98,7 +98,7 @@ def makeGraphUDF(graph, udf_name, fetches, feeds_to_fields_map=None, blocked=Fal jvm_builder.shape(fetch_names + placeholder_names, fetch_shapes + placeholder_shapes) jvm_builder.fetches(fetch_names) # Passing feeds to TensorFrames - placeholder_op_names = [tfx.op_name(graph, name) for name in placeholder_names] + placeholder_op_names = [tfx.op_name(name, graph) for name in placeholder_names] # Passing the graph input to DataFrame column mapping and additional placeholder names tfs.core._add_inputs(jvm_builder, feeds_to_fields_map, placeholder_op_names) diff --git a/python/sparkdl/graph/utils.py b/python/sparkdl/graph/utils.py index 8b10c307..b3f56836 100644 --- a/python/sparkdl/graph/utils.py +++ b/python/sparkdl/graph/utils.py @@ -40,7 +40,7 @@ def validated_graph(graph): assert isinstance(graph, tf.Graph), 'must provide tf.Graph, but get {}'.format(type(graph)) return graph -def get_shape(graph, tfobj_or_name): +def get_shape(tfobj_or_name, graph): """ Return the shape of the tensor as a list @@ -48,10 +48,10 @@ def get_shape(graph, tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ graph = validated_graph(graph) - _shape = get_tensor(graph, tfobj_or_name).get_shape().as_list() + _shape = get_tensor(tfobj_or_name, graph).get_shape().as_list() return [-1 if x is None else x for x in _shape] -def get_op(graph, tfobj_or_name): +def get_op(tfobj_or_name, graph): """ Get a tf.Operation object @@ -65,14 +65,14 @@ def get_op(graph, tfobj_or_name): if isinstance(tfobj_or_name, tf.Tensor): name = tfobj_or_name.name if not isinstance(name, six.string_types): - raise TypeError('invalid op request for {} of {}'.format(name, type(name))) - _op_name = as_op_name(name) + raise TypeError('invalid op request for [type {}] {}'.format(type(name), name)) + _op_name = op_name(name, graph=None) op = graph.get_operation_by_name(_op_name) assert op is not None, \ 'cannot locate op {} in current graph'.format(_op_name) return op -def get_tensor(graph, tfobj_or_name): +def get_tensor(tfobj_or_name, graph): """ Get a tf.Tensor object @@ -87,13 +87,13 @@ def get_tensor(graph, tfobj_or_name): name = tfobj_or_name.name if not isinstance(name, six.string_types): raise TypeError('invalid tensor request for {} of {}'.format(name, type(name))) - _tensor_name = as_tensor_name(name) + _tensor_name = tensor_name(name, graph=None) tnsr = graph.get_tensor_by_name(_tensor_name) assert tnsr is not None, \ 'cannot locate tensor {} in current graph'.format(_tensor_name) return tnsr -def as_tensor_name(tfobj_or_name): +def tensor_name(tfobj_or_name, graph=None): """ Derive tf.Tensor name from an op/tensor name. If the input is a name, we do not check if the tensor exist @@ -101,6 +101,10 @@ def as_tensor_name(tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ + # If `graph` is provided, directly get the graph operation + if graph is not None: + return get_tensor(tfobj_or_name, graph).name + # If `graph` is absent, check if other cases if isinstance(tfobj_or_name, six.string_types): # If input is a string, assume it is a name and infer the corresponding tensor name. # WARNING: this depends on TensorFlow's tensor naming convention @@ -111,12 +115,11 @@ def as_tensor_name(tfobj_or_name): name += ":0" return name elif hasattr(tfobj_or_name, 'graph'): - tfobj = tfobj_or_name - return get_tensor(tfobj.graph, tfobj).name + return get_tensor(tfobj_or_name, tfobj_or_name.graph).name else: raise TypeError('invalid tf.Tensor name query type {}'.format(type(tfobj_or_name))) -def as_op_name(tfobj_or_name): +def op_name(tfobj_or_name, graph=None): """ Derive tf.Operation name from an op/tensor name. If the input is a name, we do not check if the operation exist @@ -124,6 +127,10 @@ def as_op_name(tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ + # If `graph` is provided, directly get the graph operation + if graph is not None: + return get_op(tfobj_or_name, graph).name + # If `graph` is absent, check if other cases if isinstance(tfobj_or_name, six.string_types): # If input is a string, assume it is a name and infer the corresponding operation name. # WARNING: this depends on TensorFlow's operation naming convention @@ -132,32 +139,11 @@ def as_op_name(tfobj_or_name): assert len(name_parts) <= 2, name_parts return name_parts[0] elif hasattr(tfobj_or_name, 'graph'): - tfobj = tfobj_or_name - return get_op(tfobj.graph, tfobj).name + return get_op(tfobj_or_name, tfobj_or_name.graph).name else: raise TypeError('invalid tf.Operation name query type {}'.format(type(tfobj_or_name))) -def op_name(graph, tfobj_or_name): - """ - Get the name of a tf.Operation - - :param graph: tf.Graph, a TensorFlow Graph object - :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either - """ - graph = validated_graph(graph) - return get_op(graph, tfobj_or_name).name - -def tensor_name(graph, tfobj_or_name): - """ - Get the name of a tf.Tensor - - :param graph: tf.Graph, a TensorFlow Graph object - :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either - """ - graph = validated_graph(graph) - return get_tensor(graph, tfobj_or_name).name - -def validated_output(graph, tfobj_or_name): +def validated_output(tfobj_or_name, graph): """ Validate and return the output names useable GraphFunction @@ -165,9 +151,9 @@ def validated_output(graph, tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ graph = validated_graph(graph) - return op_name(graph, tfobj_or_name) + return op_name(tfobj_or_name, graph) -def validated_input(graph, tfobj_or_name): +def validated_input(tfobj_or_name, graph): """ Validate and return the input names useable GraphFunction @@ -175,7 +161,7 @@ def validated_input(graph, tfobj_or_name): :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either """ graph = validated_graph(graph) - name = op_name(graph, tfobj_or_name) + name = op_name(tfobj_or_name, graph) op = graph.get_operation_by_name(name) assert 'Placeholder' == op.type, \ ('input must be Placeholder, but get', op.type) @@ -202,7 +188,7 @@ def strip_and_freeze_until(fetches, graph, sess=None, return_graph=False): gdef_frozen = tf.graph_util.convert_variables_to_constants( sess, graph.as_graph_def(add_shapes=True), - [op_name(graph, tnsr) for tnsr in fetches]) + [op_name(tnsr, graph) for tnsr in fetches]) if should_close_session: sess.close() diff --git a/python/sparkdl/transformers/keras_image.py b/python/sparkdl/transformers/keras_image.py index de10fc87..3c2762d9 100644 --- a/python/sparkdl/transformers/keras_image.py +++ b/python/sparkdl/transformers/keras_image.py @@ -76,14 +76,14 @@ def _transform(self, dataset): return transformer.transform(image_df).drop(self._loadedImageCol()) def _loadTFGraph(self): - with KSessionWrap() as (sess, g): + with KSessionWrap() as (sess, graph): assert K.backend() == "tensorflow", \ "Keras backend is not tensorflow but KerasImageTransformer only supports " + \ "tensorflow-backed Keras models." - with g.as_default(): + with graph.as_default(): K.set_learning_phase(0) # Testing phase model = load_model(self.getModelFile()) - out_op_name = tfx.op_name(g, model.output) + out_op_name = tfx.op_name(model.output, graph) self._inputTensor = model.input.name self._outputTensor = model.output.name - return tfx.strip_and_freeze_until([out_op_name], g, sess, return_graph=True) + return tfx.strip_and_freeze_until([out_op_name], graph, sess, return_graph=True) diff --git a/python/sparkdl/transformers/tf_image.py b/python/sparkdl/transformers/tf_image.py index 943af6e8..152a7fea 100644 --- a/python/sparkdl/transformers/tf_image.py +++ b/python/sparkdl/transformers/tf_image.py @@ -30,7 +30,7 @@ __all__ = ['TFImageTransformer'] -IMAGE_INPUT_TENSOR_NAME = tfx.as_tensor_name(utils.IMAGE_INPUT_PLACEHOLDER_NAME) +IMAGE_INPUT_TENSOR_NAME = tfx.tensor_name(utils.IMAGE_INPUT_PLACEHOLDER_NAME) USER_GRAPH_NAMESPACE = 'given' NEW_OUTPUT_PREFIX = 'sdl_flattened' @@ -136,7 +136,7 @@ def _transform(self, dataset): "__sdl_image_data") ) - tfs_output_name = tfx.op_name(final_graph, output_tensor) + tfs_output_name = tfx.op_name(output_tensor, final_graph) original_output_name = self._getOriginalOutputTensorName() output_shape = final_graph.get_tensor_by_name(original_output_name).shape output_mode = self.getOrDefault(self.outputMode) @@ -207,7 +207,7 @@ def _getFinalOutputTensorName(self): return NEW_OUTPUT_PREFIX + '_' + self.getOutputTensor().name def _getFinalOutputOpName(self): - return tfx.as_op_name(self._getFinalOutputTensorName()) + return tfx.op_name(self._getFinalOutputTensorName()) def _convertOutputToImage(self, df, tfs_output_col, output_shape): assert len(output_shape) == 4, str(output_shape) + " does not have 4 dimensions" diff --git a/python/tests/graph/test_builder.py b/python/tests/graph/test_builder.py index b0736896..93b3c9f5 100644 --- a/python/tests/graph/test_builder.py +++ b/python/tests/graph/test_builder.py @@ -78,15 +78,15 @@ def test_get_graph_elements(self): z = tf.add(x, 3, name='z') g = issn.graph - self.assertEqual(tfx.get_tensor(g, z), z) - self.assertEqual(tfx.get_tensor(g, x), x) - self.assertEqual(g.get_tensor_by_name("x:0"), tfx.get_tensor(g, x)) - self.assertEqual("x:0", tfx.tensor_name(g, x)) - self.assertEqual(g.get_operation_by_name("x"), tfx.get_op(g, x)) - self.assertEqual("x", tfx.op_name(g, x)) - self.assertEqual("z", tfx.op_name(g, z)) - self.assertEqual(tfx.tensor_name(g, z), "z:0") - self.assertEqual(tfx.tensor_name(g, x), "x:0") + self.assertEqual(tfx.get_tensor(z, g), z) + self.assertEqual(tfx.get_tensor(x, g), x) + self.assertEqual(g.get_tensor_by_name("x:0"), tfx.get_tensor(x, g)) + self.assertEqual("x:0", tfx.tensor_name(x, g)) + self.assertEqual(g.get_operation_by_name("x"), tfx.get_op(x, g)) + self.assertEqual("x", tfx.op_name(x, g)) + self.assertEqual("z", tfx.op_name(z, g)) + self.assertEqual(tfx.tensor_name(z, g), "z:0") + self.assertEqual(tfx.tensor_name(x, g), "x:0") def test_import_export_graph_function(self): """ Function import and export must be consistent """ diff --git a/python/tests/graph/test_pieces.py b/python/tests/graph/test_pieces.py index 1497d137..9d659265 100644 --- a/python/tests/graph/test_pieces.py +++ b/python/tests/graph/test_pieces.py @@ -55,7 +55,7 @@ def exec_gfn_spimg_decode(spimg_dict, img_dtype): gfn = gfac.buildSpImageConverter(img_dtype) with IsolatedSession() as issn: feeds, fetches = issn.importGraphFunction(gfn, prefix="") - feed_dict = dict((tnsr, spimg_dict[tfx.op_name(issn.graph, tnsr)]) for tnsr in feeds) + feed_dict = dict((tnsr, spimg_dict[tfx.op_name(tnsr, issn.graph)]) for tnsr in feeds) img_out = issn.run(fetches[0], feed_dict=feed_dict) return img_out @@ -159,7 +159,7 @@ def test_pipeline(self): with IsolatedSession() as issn: # Need blank import scope name so that spimg fields match the input names feeds, fetches = issn.importGraphFunction(piped_model, prefix="") - feed_dict = dict((tnsr, spimg_input_dict[tfx.op_name(issn.graph, tnsr)]) for tnsr in feeds) + feed_dict = dict((tnsr, spimg_input_dict[tfx.op_name(tnsr, issn.graph)]) for tnsr in feeds) preds_tgt = issn.run(fetches[0], feed_dict=feed_dict) # Uncomment the line below to see the graph # tfx.write_visualization_html(issn.graph, diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py index de75036f..94c613cc 100644 --- a/python/tests/graph/test_utils.py +++ b/python/tests/graph/test_utils.py @@ -25,13 +25,13 @@ def test_infer_graph_element_names(self): for tnsr_idx in range(17): op_name = 'someOp' tnsr_name = '{}:{}'.format(op_name, tnsr_idx) - self.assertEqual(op_name, tfx.as_op_name(tnsr_name)) - self.assertEqual(tnsr_name, tfx.as_tensor_name(tnsr_name)) + self.assertEqual(op_name, tfx.op_name(tnsr_name)) + self.assertEqual(tnsr_name, tfx.tensor_name(tnsr_name)) with self.assertRaises(TypeError): for wrong_value in [7, 1.2, tf.Graph()]: - tfx.as_op_name(wrong_value) - tfx.as_tensor_name(wrong_value) + tfx.op_name(wrong_value) + tfx.tensor_name(wrong_value) def test_get_graph_elements(self): op_name = 'someConstOp' @@ -39,11 +39,11 @@ def test_get_graph_elements(self): tnsr = tf.constant(1427.08, name=op_name) graph = tnsr.graph - self.assertEqual(op_name, tfx.as_op_name(tnsr)) - self.assertEqual(op_name, tfx.op_name(graph, tnsr)) - self.assertEqual(tnsr_name, tfx.as_tensor_name(tnsr)) - self.assertEqual(tnsr_name, tfx.tensor_name(graph, tnsr)) - self.assertEqual(tnsr, tfx.get_tensor(graph, tnsr)) - self.assertEqual(tnsr.op, tfx.get_op(graph, tnsr)) - self.assertEqual(graph, tfx.get_op(graph, tnsr).graph) - self.assertEqual(graph, tfx.get_tensor(graph, tnsr).graph) + self.assertEqual(op_name, tfx.op_name(tnsr)) + self.assertEqual(op_name, tfx.op_name(tnsr, graph)) + self.assertEqual(tnsr_name, tfx.tensor_name(tnsr)) + self.assertEqual(tnsr_name, tfx.tensor_name(tnsr, graph)) + self.assertEqual(tnsr, tfx.get_tensor(tnsr, graph)) + self.assertEqual(tnsr.op, tfx.get_op(tnsr, graph)) + self.assertEqual(graph, tfx.get_op(tnsr, graph).graph) + self.assertEqual(graph, tfx.get_tensor(tnsr, graph).graph) diff --git a/python/tests/udf/keras_sql_udf_test.py b/python/tests/udf/keras_sql_udf_test.py index d1473b3c..5c67c854 100644 --- a/python/tests/udf/keras_sql_udf_test.py +++ b/python/tests/udf/keras_sql_udf_test.py @@ -66,7 +66,7 @@ def test_simple_keras_udf(self): makeGraphUDF(issn.graph, 'my_keras_model_udf', model.outputs, - {tfx.op_name(issn.graph, model.inputs[0]): 'image_col'}) + {tfx.op_name(model.inputs[0], issn.graph): 'image_col'}) # Run the training procedure # Export the graph in this IsolatedSession as a GraphFunction # gfn = issn.asGraphFunction(model.inputs, model.outputs) @@ -168,4 +168,3 @@ def test_map_blocks_sql_1(self): data2 = df2.collect() assert len(data2) == 5, data2 assert data2[0].z == 3.0, data2 - From ead1ed62eaaab9a817dafe5b86c409c07b3d81dd Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 29 Sep 2017 09:02:41 -0700 Subject: [PATCH 53/80] test refactoring --- python/tests/graph/test_utils.py | 91 +++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 24 deletions(-) diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py index 94c613cc..ba7c8a08 100644 --- a/python/tests/graph/test_utils.py +++ b/python/tests/graph/test_utils.py @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import absolute_import, division, print_function + +from collections import namedtuple +# Use this to create parameterized test cases +from parameterized import parameterized import tensorflow as tf @@ -19,31 +24,69 @@ from ..tests import PythonUnitTestCase +TestCase = namedtuple('TestCase', ['data', 'description']) + + +def _gen_graph_elems_names(): + op_name = 'someOp' + for tnsr_idx in range(17): + tnsr_name = '{}:{}'.format(op_name, tnsr_idx) + yield TestCase(data=(op_name, tfx.op_name(tnsr_name)), + description='must get the same op name from its tensor') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr_name)), + description='must get the tensor name from its operation') + + +def _gen_wrong_graph_elems_types(): + for wrong_val in [7, 1.2, tf.Graph()]: + yield TestCase(data=wrong_val, description='wrong type {}'.format(type(wrong_val))) + + +def _gen_graph_elems(): + op_name = 'someConstOp' + tnsr_name = '{}:0'.format(op_name) + tnsr = tf.constant(1427.08, name=op_name) + graph = tnsr.graph + + yield TestCase(data=(op_name, tfx.op_name(tnsr)), + description='get op name from tensor (no graph)') + yield TestCase(data=(op_name, tfx.op_name(tnsr, graph)), + description='get op name from tensor (with graph)') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr)), + description='get tensor name from tensor (no graph)') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr, graph)), + description='get tensor name from tensor (with graph)') + yield TestCase(data=(tnsr, tfx.get_tensor(tnsr, graph)), + description='get tensor from the same tensor (with graph)') + yield TestCase(data=(tnsr.op, tfx.get_op(tnsr, graph)), + description='get op from tensor (with graph)') + yield TestCase(data=(graph, tfx.get_op(tnsr, graph).graph), + description='get graph from retrieved op (with graph)') + yield TestCase(data=(graph, tfx.get_tensor(tnsr, graph).graph), + description='get graph from retrieved tensor (with graph)') + + class TFeXtensionGraphUtilsTest(PythonUnitTestCase): + @parameterized.expand(_gen_graph_elems_names) + def test_valid_graph_element_names(self, data, description): + """ Must get correct names from valid graph element names """ + name_a, name_b = data + self.assertEqual(name_a, name_b, msg=description) - def test_infer_graph_element_names(self): - for tnsr_idx in range(17): - op_name = 'someOp' - tnsr_name = '{}:{}'.format(op_name, tnsr_idx) - self.assertEqual(op_name, tfx.op_name(tnsr_name)) - self.assertEqual(tnsr_name, tfx.tensor_name(tnsr_name)) + @parameterized.expand(_gen_wrong_graph_elems_types) + def test_wrong_op_types(self, data, description): + """ Must fail when provided wrong types """ + with self.assertRaises(TypeError): + tfx.op_name(data, msg=description) + @parameterized.expand(_gen_wrong_graph_elems_types) + def test_wrong_op_types(self, data, description): + """ Must fail when provided wrong types """ with self.assertRaises(TypeError): - for wrong_value in [7, 1.2, tf.Graph()]: - tfx.op_name(wrong_value) - tfx.tensor_name(wrong_value) - - def test_get_graph_elements(self): - op_name = 'someConstOp' - tnsr_name = '{}:0'.format(op_name) - tnsr = tf.constant(1427.08, name=op_name) - graph = tnsr.graph - - self.assertEqual(op_name, tfx.op_name(tnsr)) - self.assertEqual(op_name, tfx.op_name(tnsr, graph)) - self.assertEqual(tnsr_name, tfx.tensor_name(tnsr)) - self.assertEqual(tnsr_name, tfx.tensor_name(tnsr, graph)) - self.assertEqual(tnsr, tfx.get_tensor(tnsr, graph)) - self.assertEqual(tnsr.op, tfx.get_op(tnsr, graph)) - self.assertEqual(graph, tfx.get_op(tnsr, graph).graph) - self.assertEqual(graph, tfx.get_tensor(tnsr, graph).graph) + tfx.op_name(data, msg=description) + + @parameterized.expand(_gen_graph_elems) + def test_get_graph_elements(self, data, description): + """ Must get correct graph elements from valid graph elements or their names """ + tfobj_or_name_a, tfobj_or_name_b = data + self.assertEqual(tfobj_or_name_a, tfobj_or_name_b, msg=description) From 20e2dbc4207b301a8df45b25b5ef1da9393fd98b Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 29 Sep 2017 13:45:51 -0700 Subject: [PATCH 54/80] update tfx utils usage --- python/sparkdl/graph/input.py | 8 ++++---- python/tests/graph/test_input_graph.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index d1be0e0c..7f508b56 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -292,9 +292,9 @@ def _build_with_sig_def(sess, graph, sig_def): fetch_names.append(tnsr_name) for tnsr_name in feed_names: - assert tfx.get_op(graph, tnsr_name), \ + assert tfx.get_op(tnsr_name, graph), \ 'requested tensor {} but found none in graph {}'.format(tnsr_name, graph) - fetches = [tfx.get_tensor(graph, tnsr_name) for tnsr_name in fetch_names] + fetches = [tfx.get_tensor(tnsr_name, graph) for tnsr_name in fetch_names] graph_def = tfx.strip_and_freeze_until(fetches, graph, sess) return TFInputGraph(graph_def=graph_def, input_tensor_name_from_signature=feed_mapping, @@ -308,9 +308,9 @@ def _build_with_feeds_fetches(sess, graph, feed_names, fetch_names): with sess.as_default(), graph.as_default(): for tnsr_name in feed_names: - assert tfx.get_op(graph, tnsr_name), \ + assert tfx.get_op(tnsr_name, graph), \ 'requested tensor {} but found none in graph {}'.format(tnsr_name, graph) - fetches = [tfx.get_tensor(graph, tnsr_name) for tnsr_name in fetch_names] + fetches = [tfx.get_tensor(tnsr_name, graph) for tnsr_name in fetch_names] graph_def = tfx.strip_and_freeze_until(fetches, graph, sess) return TFInputGraph(graph_def=graph_def, input_tensor_name_from_signature=None, diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index d16d62eb..9ceeca9d 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -87,14 +87,14 @@ def _run_test_in_tf_session(self): # Build test graph and transformers from here yield sess - ref_feed = tfx.get_tensor(graph, self.input_op_name) - ref_fetch = tfx.get_tensor(graph, self.output_op_name) + ref_feed = tfx.get_tensor(self.input_op_name, graph) + ref_fetch = tfx.get_tensor(self.output_op_name, graph) def check_input_graph(tgt_gdef, test_idx): namespace = 'TEST_TGT_NS{:03d}'.format(test_idx) tf.import_graph_def(tgt_gdef, name=namespace) - tgt_feed = tfx.get_tensor(graph, '{}/{}'.format(namespace, self.input_op_name)) - tgt_fetch = tfx.get_tensor(graph, '{}/{}'.format(namespace, self.output_op_name)) + tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, self.input_op_name), graph) + tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) for _ in range(10): local_data = np.random.randn(31, self.vec_size) From 20a534612af3e8ba150ef64aafd9ae7d1b7b7fd0 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 29 Sep 2017 16:18:54 -0700 Subject: [PATCH 55/80] one way to build these tests --- python/tests/graph/test_input_graph.py | 235 +++++++++++++++---------- 1 file changed, 138 insertions(+), 97 deletions(-) diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index 9ceeca9d..a237cfd8 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -14,13 +14,17 @@ # from __future__ import absolute_import, division, print_function +from collections import namedtuple from contextlib import contextmanager + from glob import glob import os import shutil import tempfile import numpy as np +# Use this to create parameterized test cases +from parameterized import parameterized import tensorflow as tf from sparkdl.graph.input import * @@ -29,11 +33,11 @@ from ..tests import PythonUnitTestCase -class TFInputGraphTest(PythonUnitTestCase): - - def setUp(self): - self.vec_size = 23 - self.num_samples = 107 +class TestGenBase(object): + def __init__(self, vec_size, test_batch_size): + # Testing data spec + self.vec_size = vec_size + self.test_batch_size = test_batch_size self.input_col = 'dfInputCol' self.input_op_name = 'tnsrOpIn' @@ -46,12 +50,12 @@ def setUp(self): self.output_mapping = {} self.setup_iomap(replica=1) + self.test_cases = [] self.input_graphs = [] - self.test_case_results = [] # Build a temporary directory, which might or might not be used by the test self.model_output_root = tempfile.mkdtemp() - def tearDown(self): + def tear_down_env(self): shutil.rmtree(self.model_output_root, ignore_errors=True) def setup_iomap(self, replica=1): @@ -78,46 +82,66 @@ def setup_iomap(self, replica=1): self.fetch_names = [self.output_op_name + ':0'] @contextmanager - def _run_test_in_tf_session(self): - """ [THIS IS NOT A TEST]: encapsulate general test workflow """ + def prep_tf_session(self): + """ Create a session to let build testing graphs """ # Build the TensorFlow graph graph = tf.Graph() with tf.Session(graph=graph) as sess, graph.as_default(): # Build test graph and transformers from here + # Notice that any TensorFlow graph could potentially be constructed in this session. + # The graph might contain variables only usable in this session. yield sess + # Find the input/output tensors. We expect them to use canonical names. ref_feed = tfx.get_tensor(self.input_op_name, graph) ref_fetch = tfx.get_tensor(self.output_op_name, graph) - def check_input_graph(tgt_gdef, test_idx): + def create_test_result(tgt_gdef, test_idx): namespace = 'TEST_TGT_NS{:03d}'.format(test_idx) tf.import_graph_def(tgt_gdef, name=namespace) tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, self.input_op_name), graph) tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) - for _ in range(10): - local_data = np.random.randn(31, self.vec_size) - ref_out = sess.run(ref_fetch, feed_dict={ref_feed: local_data}) - tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: local_data}) - self.assertTrue(np.allclose(ref_out, tgt_out)) + local_data = np.random.randn(self.test_batch_size, self.vec_size) + tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: local_data}) + ref_out = sess.run(ref_fetch, feed_dict={ref_feed: local_data}) + + return ref_out, tgt_out for test_idx, input_graph in enumerate(self.input_graphs): - check_input_graph(input_graph.graph_def, test_idx) + res = create_test_result(input_graph.graph_def, test_idx) + self.test_cases.append(res) + + def register(self, obj_for_test): + self.input_graphs.append(obj_for_test) + + def build_input_graphs(self): + raise NotImplementedError("build your graph and test cases here") + + def generate(self): + self.build_input_graphs() + return self - def test_build_from_tf_graph(self): +class GenTestFromGraph(TestGenBase): + def build_input_graphs(self): """ Build TFTransformer from tf.Graph """ - with self._run_test_in_tf_session() as sess: + with self.prep_tf_session() as sess: # Begin building graph x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.input_graphs.append(gin) # End building graph - def test_build_from_saved_model(self): + self.register( + TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names)) + self.register( + TFInputGraph.fromGraphDef(sess.graph.as_graph_def(), self.feed_names, + self.fetch_names)) + + +class GenTestFromSavedModel(TestGenBase): + def build_input_graphs(self): """ Build TFTransformer from saved model """ # Setup saved model export directory saved_model_root = self.model_output_root @@ -126,98 +150,115 @@ def test_build_from_saved_model(self): serving_sigdef_key = 'prediction_signature' builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - with self._run_test_in_tf_session() as sess: + with self.prep_tf_session() as sess: # Model definition: begin x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), - dtype=tf.float64, name='varW') + w = tf.Variable( + tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) # Model definition ends sess.run(w.initializer) - sig_inputs = { - 'input_sig': tf.saved_model.utils.build_tensor_info(x)} - sig_outputs = { - 'output_sig': tf.saved_model.utils.build_tensor_info(z)} + sig_inputs = {'input_sig': tf.saved_model.utils.build_tensor_info(x)} + sig_outputs = {'output_sig': tf.saved_model.utils.build_tensor_info(z)} serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs=sig_inputs, - outputs=sig_outputs) + inputs=sig_inputs, outputs=sig_outputs) - builder.add_meta_graph_and_variables(sess, - [serving_tag], - signature_def_map={ - serving_sigdef_key: serving_sigdef}) + builder.add_meta_graph_and_variables( + sess, + [serving_tag], signature_def_map={serving_sigdef_key: serving_sigdef}) builder.save() # Build the transformer from exported serving model # We are using signaures, thus must provide the keys - gin = TFInputGraph.fromSavedModelWithSignature( - saved_model_dir, serving_tag, serving_sigdef_key) - self.input_graphs.append(gin) + gin = TFInputGraph.fromSavedModelWithSignature(saved_model_dir, serving_tag, + serving_sigdef_key) + self.register(gin) # Build the transformer from exported serving model # We are not using signatures, thus must provide tensor/operation names - gin = TFInputGraph.fromSavedModel( - saved_model_dir, serving_tag, self.feed_names, self.fetch_names) - self.input_graphs.append(gin) - - gin = TFInputGraph.fromGraph( - sess.graph, sess, self.feed_names, self.fetch_names) - self.input_graphs.append(gin) - + gin = TFInputGraph.fromSavedModel(saved_model_dir, serving_tag, self.feed_names, + self.fetch_names) + self.register(gin) - def test_build_from_checkpoint(self): - """ Build TFTransformer from a model checkpoint """ - # Build the TensorFlow graph - model_ckpt_dir = self.model_output_root - ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - serving_sigdef_key = 'prediction_signature' - - with self._run_test_in_tf_session() as sess: - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), - dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - sess.run(w.initializer) - saver = tf.train.Saver(var_list=[w]) - _ = saver.save(sess, ckpt_path_prefix, global_step=2702) - - # Prepare the signature_def - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs={ - 'input_sig': tf.saved_model.utils.build_tensor_info(x) - }, - outputs={ - 'output_sig': tf.saved_model.utils.build_tensor_info(z) - }) - - # A rather contrived way to add signature def to a meta_graph - meta_graph_def = tf.train.export_meta_graph() - - # Find the meta_graph file (there should be only one) - _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) - self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) - ckpt_meta_fpath = _ckpt_meta_fpaths[0] - - # Add signature_def to the meta_graph and serialize it - # This will overwrite the existing meta_graph_def file - meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) - with open(ckpt_meta_fpath, mode='wb') as fout: - fout.write(meta_graph_def.SerializeToString()) - - # Build the transformer from exported serving model - # We are using signaures, thus must provide the keys - gin = TFInputGraph.fromCheckpointWithSignature( - model_ckpt_dir, serving_sigdef_key) - self.input_graphs.append(gin) + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + self.register(gin) + + # def test_build_from_checkpoint(self): + # """ Build TFTransformer from a model checkpoint """ + # # Build the TensorFlow graph + # model_ckpt_dir = self.model_output_root + # ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') + # serving_sigdef_key = 'prediction_signature' + + # with self._run_test_in_tf_session() as sess: + # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + # #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) + # w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), + # dtype=tf.float64, name='varW') + # z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + # sess.run(w.initializer) + # saver = tf.train.Saver(var_list=[w]) + # _ = saver.save(sess, ckpt_path_prefix, global_step=2702) + + # # Prepare the signature_def + # serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + # inputs={ + # 'input_sig': tf.saved_model.utils.build_tensor_info(x) + # }, + # outputs={ + # 'output_sig': tf.saved_model.utils.build_tensor_info(z) + # }) + + # # A rather contrived way to add signature def to a meta_graph + # meta_graph_def = tf.train.export_meta_graph() + + # # Find the meta_graph file (there should be only one) + # _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) + # self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) + # ckpt_meta_fpath = _ckpt_meta_fpaths[0] + + # # Add signature_def to the meta_graph and serialize it + # # This will overwrite the existing meta_graph_def file + # meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) + # with open(ckpt_meta_fpath, mode='wb') as fout: + # fout.write(meta_graph_def.SerializeToString()) + + # # Build the transformer from exported serving model + # # We are using signaures, thus must provide the keys + # gin = TFInputGraph.fromCheckpointWithSignature( + # model_ckpt_dir, serving_sigdef_key) + # self.input_graphs.append(gin) + + # # Transformer without using signature_def + # gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) + # self.input_graphs.append(gin) + + # gin = TFInputGraph.fromGraph( + # sess.graph, sess, self.feed_names, self.fetch_names) + # self.input_graphs.append(gin) + + +_GEN_TEST_CASES = [] + +_GEN_TEST_CASES.append(GenTestFromGraph(vec_size=23, test_batch_size=71).generate()) +_GEN_TEST_CASES.append(GenTestFromGraph(vec_size=3, test_batch_size=17).generate()) +_GEN_TEST_CASES.append(GenTestFromSavedModel(vec_size=13, test_batch_size=39).generate()) + +_TEST_CASES = [] +for obj in _GEN_TEST_CASES: + _TEST_CASES += obj.test_cases - # Transformer without using signature_def - gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) - self.input_graphs.append(gin) - gin = TFInputGraph.fromGraph( - sess.graph, sess, self.feed_names, self.fetch_names) - self.input_graphs.append(gin) +class TFInputGraphTest(PythonUnitTestCase): + @classmethod + def tearDownClass(cls): + for obj in _GEN_TEST_CASES: + obj.tear_down_env() + + @parameterized.expand(_TEST_CASES) + def test_from_tf_graph(self, ref_out, tgt_out): + """ ABC """ + self.assertTrue(np.allclose(ref_out, tgt_out)) From cf64708a8d731d9e8f0f6b16875cd5599a84f4e2 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 29 Sep 2017 17:29:41 -0700 Subject: [PATCH 56/80] tests refactored --- python/tests/graph/test_input_graph.py | 181 +++++++++++++------------ 1 file changed, 94 insertions(+), 87 deletions(-) diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index a237cfd8..b80928f1 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -34,7 +34,7 @@ class TestGenBase(object): - def __init__(self, vec_size, test_batch_size): + def __init__(self, vec_size=17, test_batch_size=231): # Testing data spec self.vec_size = vec_size self.test_batch_size = test_batch_size @@ -48,7 +48,7 @@ def __init__(self, vec_size, test_batch_size): self.fetch_names = [] self.input_mapping = {} self.output_mapping = {} - self.setup_iomap(replica=1) + self.reset_iomap(replica=1) self.test_cases = [] self.input_graphs = [] @@ -58,7 +58,7 @@ def __init__(self, vec_size, test_batch_size): def tear_down_env(self): shutil.rmtree(self.model_output_root, ignore_errors=True) - def setup_iomap(self, replica=1): + def reset_iomap(self, replica=1): self.input_mapping = {} self.feed_names = [] self.output_mapping = {} @@ -83,7 +83,11 @@ def setup_iomap(self, replica=1): @contextmanager def prep_tf_session(self): - """ Create a session to let build testing graphs """ + """ Create a session to let build testing graphs + + Downstream classes could also choose to override this function to + build custom testing behaviors + """ # Build the TensorFlow graph graph = tf.Graph() @@ -104,8 +108,9 @@ def create_test_result(tgt_gdef, test_idx): tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) local_data = np.random.randn(self.test_batch_size, self.vec_size) - tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: local_data}) ref_out = sess.run(ref_fetch, feed_dict={ref_feed: local_data}) + # Run on the testing target + tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: local_data}) return ref_out, tgt_out @@ -113,16 +118,12 @@ def create_test_result(tgt_gdef, test_idx): res = create_test_result(input_graph.graph_def, test_idx) self.test_cases.append(res) - def register(self, obj_for_test): - self.input_graphs.append(obj_for_test) + def register(self, tf_input_graph): + self.input_graphs.append(tf_input_graph) def build_input_graphs(self): raise NotImplementedError("build your graph and test cases here") - def generate(self): - self.build_input_graphs() - return self - class GenTestFromGraph(TestGenBase): def build_input_graphs(self): @@ -144,8 +145,7 @@ class GenTestFromSavedModel(TestGenBase): def build_input_graphs(self): """ Build TFTransformer from saved model """ # Setup saved model export directory - saved_model_root = self.model_output_root - saved_model_dir = os.path.join(saved_model_root, 'saved_model') + saved_model_dir = os.path.join(self.model_output_root, 'saved_model') serving_tag = "serving_tag" serving_sigdef_key = 'prediction_signature' builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) @@ -172,7 +172,7 @@ def build_input_graphs(self): builder.save() # Build the transformer from exported serving model - # We are using signaures, thus must provide the keys + # We are using signatures, thus must provide the keys gin = TFInputGraph.fromSavedModelWithSignature(saved_model_dir, serving_tag, serving_sigdef_key) self.register(gin) @@ -186,79 +186,86 @@ def build_input_graphs(self): gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) self.register(gin) - # def test_build_from_checkpoint(self): - # """ Build TFTransformer from a model checkpoint """ - # # Build the TensorFlow graph - # model_ckpt_dir = self.model_output_root - # ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - # serving_sigdef_key = 'prediction_signature' - - # with self._run_test_in_tf_session() as sess: - # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - # #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - # w = tf.Variable(tf.random_normal([self.vec_size], dtype=tf.float64), - # dtype=tf.float64, name='varW') - # z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - # sess.run(w.initializer) - # saver = tf.train.Saver(var_list=[w]) - # _ = saver.save(sess, ckpt_path_prefix, global_step=2702) - - # # Prepare the signature_def - # serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - # inputs={ - # 'input_sig': tf.saved_model.utils.build_tensor_info(x) - # }, - # outputs={ - # 'output_sig': tf.saved_model.utils.build_tensor_info(z) - # }) - - # # A rather contrived way to add signature def to a meta_graph - # meta_graph_def = tf.train.export_meta_graph() - - # # Find the meta_graph file (there should be only one) - # _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) - # self.assertEqual(len(_ckpt_meta_fpaths), 1, msg=','.join(_ckpt_meta_fpaths)) - # ckpt_meta_fpath = _ckpt_meta_fpaths[0] - - # # Add signature_def to the meta_graph and serialize it - # # This will overwrite the existing meta_graph_def file - # meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) - # with open(ckpt_meta_fpath, mode='wb') as fout: - # fout.write(meta_graph_def.SerializeToString()) - - # # Build the transformer from exported serving model - # # We are using signaures, thus must provide the keys - # gin = TFInputGraph.fromCheckpointWithSignature( - # model_ckpt_dir, serving_sigdef_key) - # self.input_graphs.append(gin) - - # # Transformer without using signature_def - # gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) - # self.input_graphs.append(gin) - - # gin = TFInputGraph.fromGraph( - # sess.graph, sess, self.feed_names, self.fetch_names) - # self.input_graphs.append(gin) - - -_GEN_TEST_CASES = [] - -_GEN_TEST_CASES.append(GenTestFromGraph(vec_size=23, test_batch_size=71).generate()) -_GEN_TEST_CASES.append(GenTestFromGraph(vec_size=3, test_batch_size=17).generate()) -_GEN_TEST_CASES.append(GenTestFromSavedModel(vec_size=13, test_batch_size=39).generate()) - -_TEST_CASES = [] -for obj in _GEN_TEST_CASES: - _TEST_CASES += obj.test_cases + +class GenTestFromCheckpoint(TestGenBase): + def build_input_graphs(self): + """ Build TFTransformer from a model checkpoint """ + # Build the TensorFlow graph + model_ckpt_dir = self.model_output_root + ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') + serving_sigdef_key = 'prediction_signature' + + with self.prep_tf_session() as sess: + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) + w = tf.Variable( + tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') + z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + sess.run(w.initializer) + saver = tf.train.Saver(var_list=[w]) + _ = saver.save(sess, ckpt_path_prefix, global_step=2702) + + # Prepare the signature_def + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs={'input_sig': tf.saved_model.utils.build_tensor_info(x)}, + outputs={'output_sig': tf.saved_model.utils.build_tensor_info(z)}) + + # A rather contrived way to add signature def to a meta_graph + meta_graph_def = tf.train.export_meta_graph() + + # Find the meta_graph file (there should be only one) + _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) + assert len(_ckpt_meta_fpaths) == 1, \ + 'expected only one meta graph, but got {}'.format(','.join(_ckpt_meta_fpaths)) + ckpt_meta_fpath = _ckpt_meta_fpaths[0] + + # Add signature_def to the meta_graph and serialize it + # This will overwrite the existing meta_graph_def file + meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) + with open(ckpt_meta_fpath, mode='wb') as fout: + fout.write(meta_graph_def.SerializeToString()) + + # Build the transformer from exported serving model + # We are using signaures, thus must provide the keys + gin = TFInputGraph.fromCheckpointWithSignature(model_ckpt_dir, serving_sigdef_key) + self.register(gin) + + # Transformer without using signature_def + gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) + self.register(gin) + + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + self.register(gin) + + +TestCase = namedtuple('TestCase', ['ref_out', 'tgt_out', 'description']) + +_TEST_CASES_GENERATORS = [] + + +def register(obj): + _TEST_CASES_GENERATORS.append(obj) + + +#======================================================================== +# Register all test objects here +register(GenTestFromGraph(vec_size=23, test_batch_size=71)) +register(GenTestFromGraph(vec_size=3, test_batch_size=17)) +register(GenTestFromSavedModel(vec_size=13, test_batch_size=39)) +register(GenTestFromCheckpoint(vec_size=13, test_batch_size=39)) +#======================================================================== + +_ALL_TEST_CASES = [] +for obj in _TEST_CASES_GENERATORS: + obj.build_input_graphs() + for ref_out, tgt_out in obj.test_cases: + test_case = TestCase(ref_out=ref_out, tgt_out=tgt_out, description=type(obj)) + _ALL_TEST_CASES.append(test_case) + obj.tear_down_env() class TFInputGraphTest(PythonUnitTestCase): - @classmethod - def tearDownClass(cls): - for obj in _GEN_TEST_CASES: - obj.tear_down_env() - - @parameterized.expand(_TEST_CASES) - def test_from_tf_graph(self, ref_out, tgt_out): - """ ABC """ - self.assertTrue(np.allclose(ref_out, tgt_out)) + @parameterized.expand(_ALL_TEST_CASES) + def test_tf_input_graph(self, ref_out, tgt_out, description): + """ Test build TFInputGraph from various methods """ + self.assertTrue(np.allclose(ref_out, tgt_out), msg=description) From c3b3a8601d9dc258fd176cc95219d53f2deaa71b Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 29 Sep 2017 18:58:34 -0700 Subject: [PATCH 57/80] test cases in a single class THis will make things easier when we want to extend other base class functions. --- python/tests/graph/test_input_graph.py | 35 +++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index b80928f1..6f4ce2ec 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -53,10 +53,12 @@ def __init__(self, vec_size=17, test_batch_size=231): self.test_cases = [] self.input_graphs = [] # Build a temporary directory, which might or might not be used by the test - self.model_output_root = tempfile.mkdtemp() + self.saved_model_root = tempfile.mkdtemp() + self.checkpoint_root = tempfile.mkdtemp() def tear_down_env(self): - shutil.rmtree(self.model_output_root, ignore_errors=True) + shutil.rmtree(self.saved_model_root, ignore_errors=True) + shutil.rmtree(self.checkpoint_root, ignore_errors=True) def reset_iomap(self, replica=1): self.input_mapping = {} @@ -118,6 +120,9 @@ def create_test_result(tgt_gdef, test_idx): res = create_test_result(input_graph.graph_def, test_idx) self.test_cases.append(res) + # Cleanup the result for next rounds + self.input_graphs = [] + def register(self, tf_input_graph): self.input_graphs.append(tf_input_graph) @@ -125,8 +130,14 @@ def build_input_graphs(self): raise NotImplementedError("build your graph and test cases here") -class GenTestFromGraph(TestGenBase): +class GenTestCases(TestGenBase): + def build_input_graphs(self): + self.build_from_checkpoint() + self.build_from_graph() + self.build_from_saved_model() + + def build_from_graph(self): """ Build TFTransformer from tf.Graph """ with self.prep_tf_session() as sess: # Begin building graph @@ -140,12 +151,10 @@ def build_input_graphs(self): TFInputGraph.fromGraphDef(sess.graph.as_graph_def(), self.feed_names, self.fetch_names)) - -class GenTestFromSavedModel(TestGenBase): - def build_input_graphs(self): + def build_from_saved_model(self): """ Build TFTransformer from saved model """ # Setup saved model export directory - saved_model_dir = os.path.join(self.model_output_root, 'saved_model') + saved_model_dir = os.path.join(self.saved_model_root, 'saved_model') serving_tag = "serving_tag" serving_sigdef_key = 'prediction_signature' builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) @@ -187,11 +196,10 @@ def build_input_graphs(self): self.register(gin) -class GenTestFromCheckpoint(TestGenBase): - def build_input_graphs(self): + def build_from_checkpoint(self): """ Build TFTransformer from a model checkpoint """ # Build the TensorFlow graph - model_ckpt_dir = self.model_output_root + model_ckpt_dir = self.checkpoint_root ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') serving_sigdef_key = 'prediction_signature' @@ -249,10 +257,9 @@ def register(obj): #======================================================================== # Register all test objects here -register(GenTestFromGraph(vec_size=23, test_batch_size=71)) -register(GenTestFromGraph(vec_size=3, test_batch_size=17)) -register(GenTestFromSavedModel(vec_size=13, test_batch_size=39)) -register(GenTestFromCheckpoint(vec_size=13, test_batch_size=39)) +register(GenTestCases(vec_size=23, test_batch_size=71)) +register(GenTestCases(vec_size=13, test_batch_size=71)) +register(GenTestCases(vec_size=5, test_batch_size=71)) #======================================================================== _ALL_TEST_CASES = [] From e47060fedba5fe8105e40ce97b2cdfd20481b127 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 29 Sep 2017 20:09:12 -0700 Subject: [PATCH 58/80] shuffle things around Signed-off-by: Philip Yang --- python/tests/graph/base_utils.py | 250 +++++++++++++++++++++++++ python/tests/graph/test_input_graph.py | 240 +----------------------- 2 files changed, 258 insertions(+), 232 deletions(-) create mode 100644 python/tests/graph/base_utils.py diff --git a/python/tests/graph/base_utils.py b/python/tests/graph/base_utils.py new file mode 100644 index 00000000..1cabce26 --- /dev/null +++ b/python/tests/graph/base_utils.py @@ -0,0 +1,250 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import absolute_import, division, print_function + +from collections import namedtuple +from contextlib import contextmanager +from glob import glob +import os +import shutil +import tempfile + +import numpy as np +import tensorflow as tf + +from sparkdl.graph.input import * +import sparkdl.graph.utils as tfx + + +TestCase = namedtuple('TestCase', ['ref_out', 'tgt_out', 'description']) +_GinInfo = namedtuple('_GinInfo', ['gin', 'description']) + +class TestGenBase(object): + def __init__(self, vec_size=17, test_batch_size=231): + # Testing data spec + self.vec_size = vec_size + self.test_batch_size = test_batch_size + + self.input_col = 'dfInputCol' + self.input_op_name = 'tnsrOpIn' + self.output_col = 'dfOutputCol' + self.output_op_name = 'tnsrOpOut' + + self.feed_names = [] + self.fetch_names = [] + self.input_mapping = {} + self.output_mapping = {} + self.reset_iomap(replica=1) + + self.test_cases = [] + self.input_graphs = [] + # Build a temporary directory, which might or might not be used by the test + self.saved_model_root = tempfile.mkdtemp() + self.checkpoint_root = tempfile.mkdtemp() + + def tear_down_env(self): + shutil.rmtree(self.saved_model_root, ignore_errors=True) + shutil.rmtree(self.checkpoint_root, ignore_errors=True) + + def reset_iomap(self, replica=1): + self.input_mapping = {} + self.feed_names = [] + self.output_mapping = {} + self.fetch_names = [] + + if replica > 1: + for i in range(replica): + colname = '{}_replica{:03d}'.format(self.input_col, i) + tnsr_op_name = '{}_replica{:03d}'.format(self.input_op_name, i) + self.input_mapping[colname] = tnsr_op_name + self.feed_names.append(tnsr_op_name + ':0') + + colname = '{}_replica{:03d}'.format(self.output_col, i) + tnsr_op_name = '{}_replica{:03d}'.format(self.output_op_name, i) + self.output_mapping[tnsr_op_name] = colname + self.fetch_names.append(tnsr_op_name + ':0') + else: + self.input_mapping = {self.input_col: self.input_op_name} + self.feed_names = [self.input_op_name + ':0'] + self.output_mapping = {self.output_op_name: self.output_col} + self.fetch_names = [self.output_op_name + ':0'] + + @contextmanager + def prep_tf_session(self): + """ Create a session to let build testing graphs + + Downstream classes could also choose to override this function to + build custom testing behaviors + """ + # Reset states + self.input_graphs = [] + + # Build the TensorFlow graph + graph = tf.Graph() + with tf.Session(graph=graph) as sess, graph.as_default(): + # Build test graph and transformers from here + # Notice that any TensorFlow graph could potentially be constructed in this session. + # The graph might contain variables only usable in this session. + yield sess + + # Find the input/output tensors. We expect them to use canonical names. + ref_feed = tfx.get_tensor(self.input_op_name, graph) + ref_fetch = tfx.get_tensor(self.output_op_name, graph) + + def create_test_result(tgt_gdef, test_idx): + namespace = 'TEST_TGT_NS{:03d}'.format(test_idx) + tf.import_graph_def(tgt_gdef, name=namespace) + tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, self.input_op_name), graph) + tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) + + local_data = np.random.randn(self.test_batch_size, self.vec_size) + ref_out = sess.run(ref_fetch, feed_dict={ref_feed: local_data}) + # Run on the testing target + tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: local_data}) + + return ref_out, tgt_out + + for test_idx, gin_info in enumerate(self.input_graphs): + gdef = gin_info.gin.graph_def + ref_out, tgt_out = create_test_result(gdef, test_idx) + self.test_cases.append(TestCase(ref_out=ref_out, tgt_out=tgt_out, + description=gin_info.description)) + + def register(self, gin, description): + self.input_graphs.append(_GinInfo(gin=gin, description=description)) + + def build_input_graphs(self): + raise NotImplementedError("build your graph and test cases here") + + +class GenTestCases(TestGenBase): + """ Define various test graphs + + Please define all the graphs to be built for test cases in this class. + This is useful for other classes to inherent and change the definition of + the actual testing method. That is, by overriding :py:meth:`prep_tf_session`, + the subclass can change the evaluation behavior. + """ + def build_input_graphs(self): + self.build_from_checkpoint() + self.build_from_graph() + self.build_from_saved_model() + + def build_from_graph(self): + """ Build TFTransformer from tf.Graph """ + with self.prep_tf_session() as sess: + # Begin building graph + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) + # End building graph + + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + self.register(gin=gin, description='from graph with graph') + gin = TFInputGraph.fromGraphDef(sess.graph.as_graph_def(), self.feed_names, + self.fetch_names) + self.register(gin=gin, description='from graph with graph_def') + + def build_from_saved_model(self): + """ Build TFTransformer from saved model """ + # Setup saved model export directory + saved_model_dir = os.path.join(self.saved_model_root, 'saved_model') + serving_tag = "serving_tag" + serving_sigdef_key = 'prediction_signature' + builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) + + with self.prep_tf_session() as sess: + # Model definition: begin + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + w = tf.Variable( + tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') + z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + # Model definition ends + + sess.run(w.initializer) + + sig_inputs = {'input_sig': tf.saved_model.utils.build_tensor_info(x)} + sig_outputs = {'output_sig': tf.saved_model.utils.build_tensor_info(z)} + + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs=sig_inputs, outputs=sig_outputs) + + builder.add_meta_graph_and_variables( + sess, + [serving_tag], signature_def_map={serving_sigdef_key: serving_sigdef}) + builder.save() + + # Build the transformer from exported serving model + # We are using signatures, thus must provide the keys + gin = TFInputGraph.fromSavedModelWithSignature(saved_model_dir, serving_tag, + serving_sigdef_key) + self.register(gin=gin, description='saved model with signature') + + # Build the transformer from exported serving model + # We are not using signatures, thus must provide tensor/operation names + gin = TFInputGraph.fromSavedModel(saved_model_dir, serving_tag, self.feed_names, + self.fetch_names) + self.register(gin=gin, description='saved model no signature') + + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + self.register(gin=gin, description='saved model with graph') + + def build_from_checkpoint(self): + """ Build TFTransformer from a model checkpoint """ + # Build the TensorFlow graph + model_ckpt_dir = self.checkpoint_root + ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') + serving_sigdef_key = 'prediction_signature' + + with self.prep_tf_session() as sess: + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) + #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) + w = tf.Variable( + tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') + z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) + sess.run(w.initializer) + saver = tf.train.Saver(var_list=[w]) + _ = saver.save(sess, ckpt_path_prefix, global_step=2702) + + # Prepare the signature_def + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs={'input_sig': tf.saved_model.utils.build_tensor_info(x)}, + outputs={'output_sig': tf.saved_model.utils.build_tensor_info(z)}) + + # A rather contrived way to add signature def to a meta_graph + meta_graph_def = tf.train.export_meta_graph() + + # Find the meta_graph file (there should be only one) + _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) + assert len(_ckpt_meta_fpaths) == 1, \ + 'expected only one meta graph, but got {}'.format(','.join(_ckpt_meta_fpaths)) + ckpt_meta_fpath = _ckpt_meta_fpaths[0] + + # Add signature_def to the meta_graph and serialize it + # This will overwrite the existing meta_graph_def file + meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) + with open(ckpt_meta_fpath, mode='wb') as fout: + fout.write(meta_graph_def.SerializeToString()) + + # Build the transformer from exported serving model + # We are using signaures, thus must provide the keys + gin = TFInputGraph.fromCheckpointWithSignature(model_ckpt_dir, serving_sigdef_key) + self.register(gin=gin, description='checkpoint with signature') + + # Transformer without using signature_def + gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) + self.register(gin=gin, description='checkpoint no signature') + + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + self.register(gin=gin, description='checkpoing, graph') diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index 6f4ce2ec..f0e57bf9 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -15,259 +15,35 @@ from __future__ import absolute_import, division, print_function from collections import namedtuple -from contextlib import contextmanager - -from glob import glob -import os -import shutil -import tempfile import numpy as np # Use this to create parameterized test cases from parameterized import parameterized -import tensorflow as tf - -from sparkdl.graph.input import * -import sparkdl.graph.utils as tfx from ..tests import PythonUnitTestCase +from .base_utils import GenTestCases - -class TestGenBase(object): - def __init__(self, vec_size=17, test_batch_size=231): - # Testing data spec - self.vec_size = vec_size - self.test_batch_size = test_batch_size - - self.input_col = 'dfInputCol' - self.input_op_name = 'tnsrOpIn' - self.output_col = 'dfOutputCol' - self.output_op_name = 'tnsrOpOut' - - self.feed_names = [] - self.fetch_names = [] - self.input_mapping = {} - self.output_mapping = {} - self.reset_iomap(replica=1) - - self.test_cases = [] - self.input_graphs = [] - # Build a temporary directory, which might or might not be used by the test - self.saved_model_root = tempfile.mkdtemp() - self.checkpoint_root = tempfile.mkdtemp() - - def tear_down_env(self): - shutil.rmtree(self.saved_model_root, ignore_errors=True) - shutil.rmtree(self.checkpoint_root, ignore_errors=True) - - def reset_iomap(self, replica=1): - self.input_mapping = {} - self.feed_names = [] - self.output_mapping = {} - self.fetch_names = [] - - if replica > 1: - for i in range(replica): - colname = '{}_replica{:03d}'.format(self.input_col, i) - tnsr_op_name = '{}_replica{:03d}'.format(self.input_op_name, i) - self.input_mapping[colname] = tnsr_op_name - self.feed_names.append(tnsr_op_name + ':0') - - colname = '{}_replica{:03d}'.format(self.output_col, i) - tnsr_op_name = '{}_replica{:03d}'.format(self.output_op_name, i) - self.output_mapping[tnsr_op_name] = colname - self.fetch_names.append(tnsr_op_name + ':0') - else: - self.input_mapping = {self.input_col: self.input_op_name} - self.feed_names = [self.input_op_name + ':0'] - self.output_mapping = {self.output_op_name: self.output_col} - self.fetch_names = [self.output_op_name + ':0'] - - @contextmanager - def prep_tf_session(self): - """ Create a session to let build testing graphs - - Downstream classes could also choose to override this function to - build custom testing behaviors - """ - - # Build the TensorFlow graph - graph = tf.Graph() - with tf.Session(graph=graph) as sess, graph.as_default(): - # Build test graph and transformers from here - # Notice that any TensorFlow graph could potentially be constructed in this session. - # The graph might contain variables only usable in this session. - yield sess - - # Find the input/output tensors. We expect them to use canonical names. - ref_feed = tfx.get_tensor(self.input_op_name, graph) - ref_fetch = tfx.get_tensor(self.output_op_name, graph) - - def create_test_result(tgt_gdef, test_idx): - namespace = 'TEST_TGT_NS{:03d}'.format(test_idx) - tf.import_graph_def(tgt_gdef, name=namespace) - tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, self.input_op_name), graph) - tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) - - local_data = np.random.randn(self.test_batch_size, self.vec_size) - ref_out = sess.run(ref_fetch, feed_dict={ref_feed: local_data}) - # Run on the testing target - tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: local_data}) - - return ref_out, tgt_out - - for test_idx, input_graph in enumerate(self.input_graphs): - res = create_test_result(input_graph.graph_def, test_idx) - self.test_cases.append(res) - - # Cleanup the result for next rounds - self.input_graphs = [] - - def register(self, tf_input_graph): - self.input_graphs.append(tf_input_graph) - - def build_input_graphs(self): - raise NotImplementedError("build your graph and test cases here") - - -class GenTestCases(TestGenBase): - - def build_input_graphs(self): - self.build_from_checkpoint() - self.build_from_graph() - self.build_from_saved_model() - - def build_from_graph(self): - """ Build TFTransformer from tf.Graph """ - with self.prep_tf_session() as sess: - # Begin building graph - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) - # End building graph - - self.register( - TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names)) - self.register( - TFInputGraph.fromGraphDef(sess.graph.as_graph_def(), self.feed_names, - self.fetch_names)) - - def build_from_saved_model(self): - """ Build TFTransformer from saved model """ - # Setup saved model export directory - saved_model_dir = os.path.join(self.saved_model_root, 'saved_model') - serving_tag = "serving_tag" - serving_sigdef_key = 'prediction_signature' - builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - - with self.prep_tf_session() as sess: - # Model definition: begin - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - w = tf.Variable( - tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - # Model definition ends - - sess.run(w.initializer) - - sig_inputs = {'input_sig': tf.saved_model.utils.build_tensor_info(x)} - sig_outputs = {'output_sig': tf.saved_model.utils.build_tensor_info(z)} - - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs=sig_inputs, outputs=sig_outputs) - - builder.add_meta_graph_and_variables( - sess, - [serving_tag], signature_def_map={serving_sigdef_key: serving_sigdef}) - builder.save() - - # Build the transformer from exported serving model - # We are using signatures, thus must provide the keys - gin = TFInputGraph.fromSavedModelWithSignature(saved_model_dir, serving_tag, - serving_sigdef_key) - self.register(gin) - - # Build the transformer from exported serving model - # We are not using signatures, thus must provide tensor/operation names - gin = TFInputGraph.fromSavedModel(saved_model_dir, serving_tag, self.feed_names, - self.fetch_names) - self.register(gin) - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.register(gin) - - - def build_from_checkpoint(self): - """ Build TFTransformer from a model checkpoint """ - # Build the TensorFlow graph - model_ckpt_dir = self.checkpoint_root - ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - serving_sigdef_key = 'prediction_signature' - - with self.prep_tf_session() as sess: - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - w = tf.Variable( - tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - sess.run(w.initializer) - saver = tf.train.Saver(var_list=[w]) - _ = saver.save(sess, ckpt_path_prefix, global_step=2702) - - # Prepare the signature_def - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs={'input_sig': tf.saved_model.utils.build_tensor_info(x)}, - outputs={'output_sig': tf.saved_model.utils.build_tensor_info(z)}) - - # A rather contrived way to add signature def to a meta_graph - meta_graph_def = tf.train.export_meta_graph() - - # Find the meta_graph file (there should be only one) - _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) - assert len(_ckpt_meta_fpaths) == 1, \ - 'expected only one meta graph, but got {}'.format(','.join(_ckpt_meta_fpaths)) - ckpt_meta_fpath = _ckpt_meta_fpaths[0] - - # Add signature_def to the meta_graph and serialize it - # This will overwrite the existing meta_graph_def file - meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) - with open(ckpt_meta_fpath, mode='wb') as fout: - fout.write(meta_graph_def.SerializeToString()) - - # Build the transformer from exported serving model - # We are using signaures, thus must provide the keys - gin = TFInputGraph.fromCheckpointWithSignature(model_ckpt_dir, serving_sigdef_key) - self.register(gin) - - # Transformer without using signature_def - gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) - self.register(gin) - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.register(gin) - - -TestCase = namedtuple('TestCase', ['ref_out', 'tgt_out', 'description']) +#======================================================================== +# Don't have to modify the content below _TEST_CASES_GENERATORS = [] -def register(obj): +def _REGISTER_(obj): _TEST_CASES_GENERATORS.append(obj) #======================================================================== # Register all test objects here -register(GenTestCases(vec_size=23, test_batch_size=71)) -register(GenTestCases(vec_size=13, test_batch_size=71)) -register(GenTestCases(vec_size=5, test_batch_size=71)) +_REGISTER_(GenTestCases(vec_size=23, test_batch_size=71)) +_REGISTER_(GenTestCases(vec_size=13, test_batch_size=23)) +_REGISTER_(GenTestCases(vec_size=5, test_batch_size=17)) #======================================================================== _ALL_TEST_CASES = [] for obj in _TEST_CASES_GENERATORS: obj.build_input_graphs() - for ref_out, tgt_out in obj.test_cases: - test_case = TestCase(ref_out=ref_out, tgt_out=tgt_out, description=type(obj)) - _ALL_TEST_CASES.append(test_case) + _ALL_TEST_CASES += obj.test_cases obj.tear_down_env() From 4e8f4e3863a4deae25ab73c9b18fb546f01cb615 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 30 Sep 2017 13:25:40 -0700 Subject: [PATCH 59/80] docs mostly --- python/docs/sparkdl.rst | 2 ++ python/sparkdl/param/converters.py | 9 ++++++++- python/tests/graph/base_utils.py | 23 ++++++++++++++++++----- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/python/docs/sparkdl.rst b/python/docs/sparkdl.rst index c92e60cc..bf0c86f8 100644 --- a/python/docs/sparkdl.rst +++ b/python/docs/sparkdl.rst @@ -6,8 +6,10 @@ Subpackages .. toctree:: + sparkdl.estimators sparkdl.graph sparkdl.image + sparkdl.param sparkdl.transformers sparkdl.udf sparkdl.utils diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index 758fb81b..ef75228d 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -156,7 +156,14 @@ def _check_is_tensor_name(_maybe_tnsr_name): raise TypeError(err_msg.format(type(_maybe_tnsr_name))) # The check is taken from TensorFlow's NodeDef protocol buffer. - # https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/core/framework/node_def.proto#L21-L25 + # Each input is "node:src_output" with "node" being a string name and + # "src_output" indicating which output tensor to use from "node". If + # "src_output" is 0 the ":0" suffix can be omitted. Regular inputs + # may optionally be followed by control inputs that have the format + # "^node". + # Reference: + # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/node_def.proto + # https://stackoverflow.com/questions/36150834/how-does-tensorflow-name-tensors try: _, src_idx = _maybe_tnsr_name.split(":") _ = int(src_idx) diff --git a/python/tests/graph/base_utils.py b/python/tests/graph/base_utils.py index 1cabce26..b43a1b63 100644 --- a/python/tests/graph/base_utils.py +++ b/python/tests/graph/base_utils.py @@ -37,19 +37,32 @@ def __init__(self, vec_size=17, test_batch_size=231): self.vec_size = vec_size self.test_batch_size = test_batch_size - self.input_col = 'dfInputCol' + # TensorFlow graph element names self.input_op_name = 'tnsrOpIn' - self.output_col = 'dfOutputCol' - self.output_op_name = 'tnsrOpOut' - self.feed_names = [] + self.output_op_name = 'tnsrOpOut' self.fetch_names = [] + + # DataFrame column names + self.input_col = 'dfInputCol' + self.output_col = 'dfOutputCol' + + # Connecting data from Spark to TensorFlow self.input_mapping = {} self.output_mapping = {} + + # When testing against multiple graph inputs, + # derive new names for the DataFrame columns and TensorFlow graph elements. self.reset_iomap(replica=1) - self.test_cases = [] + # The basic stage contains the opaque :py:obj:`TFInputGraph` objects + # Any derived that override the :py:obj:`build_input_graphs` method will + # populate this field. self.input_graphs = [] + + # Construct final test cases, which will be passed to final test cases + self.test_cases = [] + # Build a temporary directory, which might or might not be used by the test self.saved_model_root = tempfile.mkdtemp() self.checkpoint_root = tempfile.mkdtemp() From eaa5fa0628afe9f71d43e0ec69ba1c53eaa4bba6 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 30 Sep 2017 13:39:50 -0700 Subject: [PATCH 60/80] yapf'd --- python/tests/graph/base_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/tests/graph/base_utils.py b/python/tests/graph/base_utils.py index b43a1b63..010f8147 100644 --- a/python/tests/graph/base_utils.py +++ b/python/tests/graph/base_utils.py @@ -27,10 +27,10 @@ from sparkdl.graph.input import * import sparkdl.graph.utils as tfx - TestCase = namedtuple('TestCase', ['ref_out', 'tgt_out', 'description']) _GinInfo = namedtuple('_GinInfo', ['gin', 'description']) + class TestGenBase(object): def __init__(self, vec_size=17, test_batch_size=231): # Testing data spec @@ -132,8 +132,8 @@ def create_test_result(tgt_gdef, test_idx): for test_idx, gin_info in enumerate(self.input_graphs): gdef = gin_info.gin.graph_def ref_out, tgt_out = create_test_result(gdef, test_idx) - self.test_cases.append(TestCase(ref_out=ref_out, tgt_out=tgt_out, - description=gin_info.description)) + self.test_cases.append( + TestCase(ref_out=ref_out, tgt_out=tgt_out, description=gin_info.description)) def register(self, gin, description): self.input_graphs.append(_GinInfo(gin=gin, description=description)) @@ -150,6 +150,7 @@ class GenTestCases(TestGenBase): the actual testing method. That is, by overriding :py:meth:`prep_tf_session`, the subclass can change the evaluation behavior. """ + def build_input_graphs(self): self.build_from_checkpoint() self.build_from_graph() @@ -194,8 +195,7 @@ def build_from_saved_model(self): inputs=sig_inputs, outputs=sig_outputs) builder.add_meta_graph_and_variables( - sess, - [serving_tag], signature_def_map={serving_sigdef_key: serving_sigdef}) + sess, [serving_tag], signature_def_map={serving_sigdef_key: serving_sigdef}) builder.save() # Build the transformer from exported serving model From 43d6583c16184fabc98cd6e1bc8cbb21624531cb Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 2 Oct 2017 10:25:19 -0700 Subject: [PATCH 61/80] consolidate tempdir creation --- python/tests/graph/base_utils.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/tests/graph/base_utils.py b/python/tests/graph/base_utils.py index 010f8147..b788cd4f 100644 --- a/python/tests/graph/base_utils.py +++ b/python/tests/graph/base_utils.py @@ -64,12 +64,16 @@ def __init__(self, vec_size=17, test_batch_size=231): self.test_cases = [] # Build a temporary directory, which might or might not be used by the test - self.saved_model_root = tempfile.mkdtemp() - self.checkpoint_root = tempfile.mkdtemp() + self._temp_dirs = [] + + def make_tempdir(self): + tmp_dir = tempfile.mkdtemp() + self._temp_dirs.append(tmp_dir) + return tmp_dir def tear_down_env(self): - shutil.rmtree(self.saved_model_root, ignore_errors=True) - shutil.rmtree(self.checkpoint_root, ignore_errors=True) + for tmp_dir in self._temp_dirs: + shutil.rmtree(tmp_dir, ignore_errors=True) def reset_iomap(self, replica=1): self.input_mapping = {} @@ -102,6 +106,9 @@ def prep_tf_session(self): build custom testing behaviors """ # Reset states + # In each `prep_tf_session`, the implementation is expected to define ONE graph and + # pass all test cases derived from it. We execute the graph and compare the result + # with each test case, and return the numerical results. self.input_graphs = [] # Build the TensorFlow graph @@ -173,7 +180,7 @@ def build_from_graph(self): def build_from_saved_model(self): """ Build TFTransformer from saved model """ # Setup saved model export directory - saved_model_dir = os.path.join(self.saved_model_root, 'saved_model') + saved_model_dir = os.path.join(self.make_tempdir(), 'saved_model') serving_tag = "serving_tag" serving_sigdef_key = 'prediction_signature' builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) @@ -216,7 +223,7 @@ def build_from_saved_model(self): def build_from_checkpoint(self): """ Build TFTransformer from a model checkpoint """ # Build the TensorFlow graph - model_ckpt_dir = self.checkpoint_root + model_ckpt_dir = self.make_tempdir() ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') serving_sigdef_key = 'prediction_signature' From 8b75d44828f7fc7e30114faf19b13353a403f279 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 2 Oct 2017 11:07:10 -0700 Subject: [PATCH 62/80] Address PR comments --- python/sparkdl/param/converters.py | 68 +++++++++++++++++------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index 758fb81b..246d46cf 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -13,10 +13,15 @@ # limitations under the License. # -# pylint: disable=wrong-spelling-in-docstring,invalid-name,import-error +# pylint: disable=invalid-name,import-error """ SparkDLTypeConverters -Type conversion utilities for definition Spark Deep Learning related MLlib `Params`. + +Type conversion utilities for defining MLlib `Params` used in Spark Deep Learning Pipelines. + +.. note:: We follow the convention of MLlib to name these utilities "converters", + but most of them act as type checkers that return the argument if it is + the desired type and raise `TypeError` otherwise. """ import six @@ -33,7 +38,7 @@ class SparkDLTypeConverters(object): """ .. note:: DeveloperApi - Factory methods for type conversion functions for :py:func:`Param.typeConverter`. + Methods for type conversion functions for :py:func:`Param.typeConverter`. These methods are similar to :py:class:`spark.ml.param.TypeConverters`. They provide support for the `Params` types introduced in Spark Deep Learning Pipelines. """ @@ -50,19 +55,16 @@ def toTFGraph(value): @staticmethod def asColumnToTensorNameMap(value): """ - Convert a value to a column name to :py:obj:`tf.Tensor` name mapping - as a sorted list of string pairs, if possible. + Convert a value to a column name to :py:class:`tf.Tensor` name mapping + as a sorted list (in lexicographical order) of string pairs, if possible. """ if not isinstance(value, dict): err_msg = "Could not convert [type {}] {} to column name to tf.Tensor name mapping" raise TypeError(err_msg.format(type(value), value)) - # Conversion logic after quick type check strs_pair_seq = [] for _maybe_col_name, _maybe_tnsr_name in value.items(): - # Check if the non-tensor value is of string type _check_is_str(_maybe_col_name) - # Check if the tensor name looks like a tensor name _check_is_tensor_name(_maybe_tnsr_name) strs_pair_seq.append((_maybe_col_name, _maybe_tnsr_name)) @@ -71,19 +73,16 @@ def asColumnToTensorNameMap(value): @staticmethod def asTensorNameToColumnMap(value): """ - Convert a value to a :py:obj:`tf.Tensor` name to column name mapping - as a sorted list of string pairs, if possible. + Convert a value to a :py:class:`tf.Tensor` name to column name mapping + as a sorted list (in lexicographical order) of string pairs, if possible. """ if not isinstance(value, dict): err_msg = "Could not convert [type {}] {} to tf.Tensor name to column name mapping" raise TypeError(err_msg.format(type(value), value)) - # Conversion logic after quick type check strs_pair_seq = [] for _maybe_tnsr_name, _maybe_col_name in value.items(): - # Check if the non-tensor value is of string type _check_is_str(_maybe_col_name) - # Check if the tensor name looks like a tensor name _check_is_tensor_name(_maybe_tnsr_name) strs_pair_seq.append((_maybe_tnsr_name, _maybe_col_name)) @@ -91,7 +90,10 @@ def asTensorNameToColumnMap(value): @staticmethod def toTFHParams(value): - """ Convert a value to a :py:obj:`tf.contrib.training.HParams` object, if possible. """ + """ + Check that the given value is a :py:class:`tf.contrib.training.HParams` object, + and return it. Raise an error otherwise. + """ if not isinstance(value, tf.contrib.training.HParams): raise TypeError("Could not convert %s to TensorFlow HParams" % type(value)) @@ -99,13 +101,15 @@ def toTFHParams(value): @staticmethod def toTFTensorName(value): - """ Convert a value to a :py:obj:`tf.Tensor` name, if possible. """ + """ + Check if a value is a valid :py:class:`tf.Tensor` name and return it. + Raise an error otherwise. + """ if isinstance(value, tf.Tensor): return value.name try: - _maybe_tnsr_name = TypeConverters.toString(value) - _check_is_tensor_name(_maybe_tnsr_name) - return _maybe_tnsr_name + _check_is_tensor_name(value) + return value except Exception as exc: err_msg = "Could not convert [type {}] {} to tf.Tensor name. {}" raise TypeError(err_msg.format(type(value), value, exc)) @@ -113,10 +117,11 @@ def toTFTensorName(value): @staticmethod def buildCheckList(supportedList): """ - Create a converter that try to check if a value is part of the supported list. + Create a "converter" that try to check if a value is part of the supported list of values. :param supportedList: list, containing supported objects. - :return: a converter that try to convert a value if it is part of the `supportedList`. + :return: a converter that try to check if a value is part of the `supportedList` and return it. + Raise an error otherwise. """ def converter(value): @@ -131,7 +136,10 @@ def converter(value): @staticmethod def toKerasLoss(value): - """ Convert a value to a name of Keras loss function, if possible """ + """ + Check if a value is a valid Keras loss function name and return it. + Otherwise raise an error. + """ # return early in for clarify as well as less indentation if not kmutil.is_valid_loss_function(value): err_msg = "Named loss not supported in Keras: [type {}] {}" @@ -141,7 +149,10 @@ def toKerasLoss(value): @staticmethod def toKerasOptimizer(value): - """ Convert a value to a name of Keras optimizer, if possible """ + """ + Check if a value is a valid name of Keras optimizer and return it. + Otherwise raise an error. + """ if not kmutil.is_valid_optimizer(value): err_msg = "Named optimizer not supported in Keras: [type {}] {}" raise TypeError(err_msg.format(type(value), value)) @@ -150,7 +161,7 @@ def toKerasOptimizer(value): def _check_is_tensor_name(_maybe_tnsr_name): - """ Check if the input is a valid tensor name """ + """ Check if the input is a valid tensor name or raise a `TypeError` otherwise. """ if not isinstance(_maybe_tnsr_name, six.string_types): err_msg = "expect tensor name to be of string type, but got [type {}]" raise TypeError(err_msg.format(type(_maybe_tnsr_name))) @@ -164,13 +175,10 @@ def _check_is_tensor_name(_maybe_tnsr_name): err_msg = "Tensor name must be of type :, but got {}" raise TypeError(err_msg.format(_maybe_tnsr_name)) - return _maybe_tnsr_name - -def _check_is_str(_maybe_col_name): - """ Check if the given colunm name is a valid column name """ +def _check_is_str(_maybe_str): + """ Check if the value is a valid string type or raise a `TypeError` otherwise. """ # We only check if the column name candidate is a string type - if not isinstance(_maybe_col_name, six.string_types): + if not isinstance(_maybe_str, six.string_types): err_msg = 'expect string type but got type {} for {}' - raise TypeError(err_msg.format(type(_maybe_col_name), _maybe_col_name)) - return _maybe_col_name + raise TypeError(err_msg.format(type(_maybe_str), _maybe_str)) From 055ce14ad9290cc679a12d332a525b19c657dd1c Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 2 Oct 2017 13:48:41 -0700 Subject: [PATCH 63/80] PR comments 1. docs in graph/utils.py --- python/sparkdl/graph/utils.py | 51 ++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/python/sparkdl/graph/utils.py b/python/sparkdl/graph/utils.py index b3f56836..3bcbef0e 100644 --- a/python/sparkdl/graph/utils.py +++ b/python/sparkdl/graph/utils.py @@ -33,9 +33,10 @@ def validated_graph(graph): """ - Check if the input is a valid tf.Graph + Check if the input is a valid :py:class:`tf.Graph` and return it. + Raise an error otherwise. - :param graph: tf.Graph, a TensorFlow Graph object + :param graph: :py:class:`tf.Graph`, a TensorFlow Graph object """ assert isinstance(graph, tf.Graph), 'must provide tf.Graph, but get {}'.format(type(graph)) return graph @@ -53,10 +54,12 @@ def get_shape(tfobj_or_name, graph): def get_op(tfobj_or_name, graph): """ - Get a tf.Operation object + Get a :py:class:`tf.Operation` object. - :param graph: tf.Graph, a TensorFlow Graph object - :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either + :param tfobj_or_name: either a :py:class:`tf.Tensor`, :py:class:`tf.Operation` or + a name to either. + :param graph: a :py:class:`tf.Graph` object containing the operation. + By default the graph we don't require this argument to be provided. """ graph = validated_graph(graph) if isinstance(tfobj_or_name, tf.Operation): @@ -76,8 +79,10 @@ def get_tensor(tfobj_or_name, graph): """ Get a tf.Tensor object - :param graph: tf.Graph, a TensorFlow Graph object - :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either + :param tfobj_or_name: either a :py:class:`tf.Tensor`, :py:class:`tf.Operation` or + a name to either. + :param graph: a :py:class:`tf.Graph` object containing the tensor. + By default the graph we don't require this argument to be provided. """ graph = validated_graph(graph) if isinstance(tfobj_or_name, tf.Tensor): @@ -95,16 +100,20 @@ def get_tensor(tfobj_or_name, graph): def tensor_name(tfobj_or_name, graph=None): """ - Derive tf.Tensor name from an op/tensor name. - If the input is a name, we do not check if the tensor exist - (as no graph parameter is passed in). + Derive the :py:class:`tf.Tensor` name from a :py:class:`tf.Operation` or :py:class:`tf.Tensor` + object, or its name. + If a name is provided and the graph is not, we will derive the tensor name based on + TensorFlow's naming convention. + If the input is a TensorFlow object, or the graph is given, we also check that + the tensor exists in the associated graph. - :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either + :param tfobj_or_name: either a :py:class:`tf.Tensor`, :py:class:`tf.Operation` or + a name to either. + :param graph: a :py:class:`tf.Graph` object containing the tensor. + By default the graph we don't require this argument to be provided. """ - # If `graph` is provided, directly get the graph operation if graph is not None: return get_tensor(tfobj_or_name, graph).name - # If `graph` is absent, check if other cases if isinstance(tfobj_or_name, six.string_types): # If input is a string, assume it is a name and infer the corresponding tensor name. # WARNING: this depends on TensorFlow's tensor naming convention @@ -121,16 +130,20 @@ def tensor_name(tfobj_or_name, graph=None): def op_name(tfobj_or_name, graph=None): """ - Derive tf.Operation name from an op/tensor name. - If the input is a name, we do not check if the operation exist - (as no graph parameter is passed in). + Derive the :py:class:`tf.Operation` name from a :py:class:`tf.Operation` or + :py:class:`tf.Tensor` object, or its name. + If a name is provided and the graph is not, we will derive the operation name based on + TensorFlow's naming convention. + If the input is a TensorFlow object, or the graph is given, we also check that + the operation exists in the associated graph. - :param tfobj_or_name: either a tf.Tensor, tf.Operation or a name to either + :param tfobj_or_name: either a :py:class:`tf.Tensor`, :py:class:`tf.Operation` or + a name to either. + :param graph: a :py:class:`tf.Graph` object containing the operation. + By default the graph we don't require this argument to be provided. """ - # If `graph` is provided, directly get the graph operation if graph is not None: return get_op(tfobj_or_name, graph).name - # If `graph` is absent, check if other cases if isinstance(tfobj_or_name, six.string_types): # If input is a string, assume it is a name and infer the corresponding operation name. # WARNING: this depends on TensorFlow's operation naming convention From 2d48b3277da8ece68e4c31b3964d0ff1d3654474 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 2 Oct 2017 13:56:35 -0700 Subject: [PATCH 64/80] (wip) utils test --- python/tests/graph/test_utils.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py index ba7c8a08..c8a7776d 100644 --- a/python/tests/graph/test_utils.py +++ b/python/tests/graph/test_utils.py @@ -27,7 +27,7 @@ TestCase = namedtuple('TestCase', ['data', 'description']) -def _gen_graph_elems_names(): +def _gen_tensor_op_string_input_tests(): op_name = 'someOp' for tnsr_idx in range(17): tnsr_name = '{}:{}'.format(op_name, tnsr_idx) @@ -37,12 +37,12 @@ def _gen_graph_elems_names(): description='must get the tensor name from its operation') -def _gen_wrong_graph_elems_types(): +def _gen_invalid_tensor_op_input_with_wrong_types(): for wrong_val in [7, 1.2, tf.Graph()]: yield TestCase(data=wrong_val, description='wrong type {}'.format(type(wrong_val))) -def _gen_graph_elems(): +def _gen_valid_tensor_op_objects(): op_name = 'someConstOp' tnsr_name = '{}:0'.format(op_name) tnsr = tf.constant(1427.08, name=op_name) @@ -67,25 +67,25 @@ def _gen_graph_elems(): class TFeXtensionGraphUtilsTest(PythonUnitTestCase): - @parameterized.expand(_gen_graph_elems_names) + @parameterized.expand(_gen_tensor_op_string_input_tests) def test_valid_graph_element_names(self, data, description): """ Must get correct names from valid graph element names """ name_a, name_b = data self.assertEqual(name_a, name_b, msg=description) - @parameterized.expand(_gen_wrong_graph_elems_types) - def test_wrong_op_types(self, data, description): + @parameterized.expand(_gen_invalid_tensor_op_input_with_wrong_types) + def test_wrong_tensor_types(self, data, description): """ Must fail when provided wrong types """ with self.assertRaises(TypeError): - tfx.op_name(data, msg=description) + tfx.tensor_name(data, msg=description) - @parameterized.expand(_gen_wrong_graph_elems_types) + @parameterized.expand(_gen_invalid_tensor_op_input_with_wrong_types) def test_wrong_op_types(self, data, description): """ Must fail when provided wrong types """ with self.assertRaises(TypeError): tfx.op_name(data, msg=description) - @parameterized.expand(_gen_graph_elems) + @parameterized.expand(_gen_valid_tensor_op_objects) def test_get_graph_elements(self, data, description): """ Must get correct graph elements from valid graph elements or their names """ tfobj_or_name_a, tfobj_or_name_b = data From 742cdaf5608b945257d3866ed6b82bd27031cfa4 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 2 Oct 2017 15:26:24 -0700 Subject: [PATCH 65/80] a few more tests for utils --- python/tests/graph/test_utils.py | 42 +++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py index c8a7776d..f1843cf4 100644 --- a/python/tests/graph/test_utils.py +++ b/python/tests/graph/test_utils.py @@ -52,14 +52,54 @@ def _gen_valid_tensor_op_objects(): description='get op name from tensor (no graph)') yield TestCase(data=(op_name, tfx.op_name(tnsr, graph)), description='get op name from tensor (with graph)') + yield TestCase(data=(op_name, tfx.op_name(tnsr_name)), + description='get op name from tensor name (no graph)') + yield TestCase(data=(op_name, tfx.op_name(tnsr_name, graph)), + description='get op name from tensor name (with graph)') + yield TestCase(data=(op_name, tfx.op_name(tnsr.op)), + description='get op name from op (no graph)') + yield TestCase(data=(op_name, tfx.op_name(tnsr.op, graph)), + description='get op name from op (with graph)') + yield TestCase(data=(op_name, tfx.op_name(op_name)), + description='get op name from op name (no graph)') + yield TestCase(data=(op_name, tfx.op_name(op_name, graph)), + description='get op name from op name (with graph)') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr)), description='get tensor name from tensor (no graph)') yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr, graph)), description='get tensor name from tensor (with graph)') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr_name)), + description='get tensor name from tensor name (no graph)') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr_name, graph)), + description='get tensor name from tensor name (with graph)') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr.op)), + description='get tensor name from op (no graph)') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr.op, graph)), + description='get tensor name from op (with graph)') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr_name)), + description='get tensor name from op name (no graph)') + yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr_name, graph)), + description='get tensor name from op name (with graph)') + yield TestCase(data=(tnsr, tfx.get_tensor(tnsr, graph)), - description='get tensor from the same tensor (with graph)') + description='get tensor from tensor (with graph)') + yield TestCase(data=(tnsr, tfx.get_tensor(tnsr_name, graph)), + description='get tensor from tensor name (with graph)') + yield TestCase(data=(tnsr, tfx.get_tensor(tnsr.op, graph)), + description='get tensor from op (with graph)') + yield TestCase(data=(tnsr, tfx.get_tensor(op_name, graph)), + description='get tensor from op name (with graph)') + yield TestCase(data=(tnsr.op, tfx.get_op(tnsr, graph)), description='get op from tensor (with graph)') + yield TestCase(data=(tnsr.op, tfx.get_op(tnsr_name, graph)), + description='get op from tensor name (with graph)') + yield TestCase(data=(tnsr.op, tfx.get_op(tnsr.op, graph)), + description='get op from op (with graph)') + yield TestCase(data=(tnsr.op, tfx.get_op(op_name, graph)), + description='get op from op name (with graph)') + yield TestCase(data=(graph, tfx.get_op(tnsr, graph).graph), description='get graph from retrieved op (with graph)') yield TestCase(data=(graph, tfx.get_tensor(tnsr, graph).graph), From f0912fb177fa38cab5e6d21be769e123e568f6e5 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 2 Oct 2017 16:27:50 -0700 Subject: [PATCH 66/80] test update cont'd --- python/tests/graph/test_utils.py | 77 ++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py index f1843cf4..b973c669 100644 --- a/python/tests/graph/test_utils.py +++ b/python/tests/graph/test_utils.py @@ -29,15 +29,15 @@ def _gen_tensor_op_string_input_tests(): op_name = 'someOp' - for tnsr_idx in range(17): + for tnsr_idx in [0, 1, 2, 3, 5, 8, 15, 17]: tnsr_name = '{}:{}'.format(op_name, tnsr_idx) yield TestCase(data=(op_name, tfx.op_name(tnsr_name)), - description='must get the same op name from its tensor') + description='test tensor name to op name') yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr_name)), - description='must get the tensor name from its operation') + description='test tensor name to tensor name') -def _gen_invalid_tensor_op_input_with_wrong_types(): +def _gen_invalid_tensor_or_op_input_with_wrong_types(): for wrong_val in [7, 1.2, tf.Graph()]: yield TestCase(data=wrong_val, description='wrong type {}'.format(type(wrong_val))) @@ -48,6 +48,7 @@ def _gen_valid_tensor_op_objects(): tnsr = tf.constant(1427.08, name=op_name) graph = tnsr.graph + # Test for op_name yield TestCase(data=(op_name, tfx.op_name(tnsr)), description='get op name from tensor (no graph)') yield TestCase(data=(op_name, tfx.op_name(tnsr, graph)), @@ -65,6 +66,7 @@ def _gen_valid_tensor_op_objects(): yield TestCase(data=(op_name, tfx.op_name(op_name, graph)), description='get op name from op name (with graph)') + # Test for tensor_name yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr)), description='get tensor name from tensor (no graph)') yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr, graph)), @@ -82,51 +84,78 @@ def _gen_valid_tensor_op_objects(): yield TestCase(data=(tnsr_name, tfx.tensor_name(tnsr_name, graph)), description='get tensor name from op name (with graph)') + # Test for get_tensor yield TestCase(data=(tnsr, tfx.get_tensor(tnsr, graph)), - description='get tensor from tensor (with graph)') + description='get tensor from tensor') yield TestCase(data=(tnsr, tfx.get_tensor(tnsr_name, graph)), - description='get tensor from tensor name (with graph)') + description='get tensor from tensor name') yield TestCase(data=(tnsr, tfx.get_tensor(tnsr.op, graph)), - description='get tensor from op (with graph)') + description='get tensor from op') yield TestCase(data=(tnsr, tfx.get_tensor(op_name, graph)), - description='get tensor from op name (with graph)') + description='get tensor from op name') + # Test for get_op yield TestCase(data=(tnsr.op, tfx.get_op(tnsr, graph)), - description='get op from tensor (with graph)') + description='get op from tensor') yield TestCase(data=(tnsr.op, tfx.get_op(tnsr_name, graph)), - description='get op from tensor name (with graph)') + description='get op from tensor name') yield TestCase(data=(tnsr.op, tfx.get_op(tnsr.op, graph)), - description='get op from op (with graph)') + description='get op from op') yield TestCase(data=(tnsr.op, tfx.get_op(op_name, graph)), - description='get op from op name (with graph)') + description='test op from op name') + # Test get_tensor and get_op returns tensor or op contained in the same graph yield TestCase(data=(graph, tfx.get_op(tnsr, graph).graph), - description='get graph from retrieved op (with graph)') + description='test graph from getting op fron tensor') yield TestCase(data=(graph, tfx.get_tensor(tnsr, graph).graph), - description='get graph from retrieved tensor (with graph)') + description='test graph from getting tensor from tensor') + yield TestCase(data=(graph, tfx.get_op(tnsr_name, graph).graph), + description='test graph from getting op fron tensor name') + yield TestCase(data=(graph, tfx.get_tensor(tnsr_name, graph).graph), + description='test graph from getting tensor from tensor name') + yield TestCase(data=(graph, tfx.get_op(tnsr.op, graph).graph), + description='test graph from getting op from op') + yield TestCase(data=(graph, tfx.get_tensor(tnsr.op, graph).graph), + description='test graph from getting tensor from op') + yield TestCase(data=(graph, tfx.get_op(op_name, graph).graph), + description='test graph from getting op from op name') + yield TestCase(data=(graph, tfx.get_tensor(op_name, graph).graph), + description='test graph from getting tensor from op name') class TFeXtensionGraphUtilsTest(PythonUnitTestCase): @parameterized.expand(_gen_tensor_op_string_input_tests) - def test_valid_graph_element_names(self, data, description): + def test_valid_tensor_op_name_inputs(self, data, description): """ Must get correct names from valid graph element names """ name_a, name_b = data self.assertEqual(name_a, name_b, msg=description) - @parameterized.expand(_gen_invalid_tensor_op_input_with_wrong_types) - def test_wrong_tensor_types(self, data, description): + @parameterized.expand(_gen_invalid_tensor_or_op_input_with_wrong_types) + def test_invalid_tensor_name_inputs_with_wrong_types(self, data, description): """ Must fail when provided wrong types """ - with self.assertRaises(TypeError): - tfx.tensor_name(data, msg=description) + with self.assertRaises(TypeError, msg=description): + tfx.tensor_name(data) - @parameterized.expand(_gen_invalid_tensor_op_input_with_wrong_types) - def test_wrong_op_types(self, data, description): + @parameterized.expand(_gen_invalid_tensor_or_op_input_with_wrong_types) + def test_invalid_op_name_inputs_with_wrong_types(self, data, description): """ Must fail when provided wrong types """ - with self.assertRaises(TypeError): - tfx.op_name(data, msg=description) + with self.assertRaises(TypeError, msg=description): + tfx.op_name(data) + + @parameterized.expand(_gen_invalid_tensor_or_op_input_with_wrong_types) + def test_invalid_op_inputs_with_wrong_types(self, data, description): + """ Must fail when provided wrong types """ + with self.assertRaises(TypeError, msg=description): + tfx.get_op(data, tf.Graph()) + + @parameterized.expand(_gen_invalid_tensor_or_op_input_with_wrong_types) + def test_invalid_tensor_inputs_with_wrong_types(self, data, description): + """ Must fail when provided wrong types """ + with self.assertRaises(TypeError, msg=description): + tfx.get_tensor(data, tf.Graph()) @parameterized.expand(_gen_valid_tensor_op_objects) - def test_get_graph_elements(self, data, description): + def test_valid_tensor_op_object_inputs(self, data, description): """ Must get correct graph elements from valid graph elements or their names """ tfobj_or_name_a, tfobj_or_name_b = data self.assertEqual(tfobj_or_name_a, tfobj_or_name_b, msg=description) From f5107adc451946769ac46e03b61768736b93a5bb Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 2 Oct 2017 19:58:57 -0700 Subject: [PATCH 67/80] (wip) PR comments --- python/sparkdl/graph/input.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 7f508b56..9ac76a91 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -104,10 +104,11 @@ def fromGraph(cls, graph, sess, feed_names, fetch_names): .. code-block:: python with tf.Session() as sess: - graph = import_my_tensorflow_graph(...) - TFInputGraph.fromGraph(graph, sess, ...) + graph = import_my_tensorflow_graph(...) + input = TFInputGraph.fromGraph(graph, sess, ...) - :param graph: `tf.Graph` + :param graph: a :py:class:`tf.Graph` object containing the topology and computation units of + the TensorFlow graph. :param feed_names: list, names of the input tensors. :param fetch_names: list, names of the output tensors. """ @@ -119,7 +120,7 @@ def fromGraphDef(cls, graph_def, feed_names, fetch_names): """ Construct a TFInputGraph from a tf.GraphDef object. - :param graph_def: `tf.GraphDef`, a serializable object containing the topology and + :param graph_def: :py:class:`tf.GraphDef`, a serializable object containing the topology and computation units of the TensorFlow graph. :param feed_names: list, names of the input tensors. :param fetch_names: list, names of the output tensors. @@ -152,13 +153,13 @@ def fromCheckpoint(cls, checkpoint_dir, feed_names, fetch_names): def fromCheckpointWithSignature(cls, checkpoint_dir, signature_def_key): """ Construct a TFInputGraph object from a checkpoint, using the embedded - signature_def. Throw error if we cannot find an entry with the `signature_def_key` + signature_def. Throw an error if we cannot find an entry with the `signature_def_key` inside the `signature_def`. :param checkpoint_dir: str, name of the directory containing the TensorFlow graph training checkpoint. - :param signature_def_key: str, name of the mapping contained inside the `signature_def` - from which we retrieve the signature key to tensor names mapping. + :param signature_def_key: str, key (name) of the signature_def to use. It should be in + the list of `signature_def` structures saved with the checkpoint. """ assert signature_def_key is not None return _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names=None, @@ -191,8 +192,9 @@ def fromSavedModelWithSignature(cls, saved_model_dir, tag_set, signature_def_key training checkpoint. :param tag_set: str, name of the graph stored in this meta_graph of the saved model that we are interested in using. - :param signature_def_key: str, name of the mapping contained inside the `signature_def` - from which we retrieve the signature key to tensor names mapping. + :param signature_def_key: str, key (name) of the signature_def to use. It should be in + the list of `signature_def` structures saved with the + TensorFlow `SavedModel`. """ assert signature_def_key is not None return _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key=signature_def_key, @@ -211,7 +213,7 @@ def _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names, fetch_n :param fetch_names: list, names of the output tensors. """ assert (feed_names is None) == (fetch_names is None), \ - 'feed_names and fetch_names, if provided must appear together' + 'feed_names and fetch_names, if provided must be both non-None.' assert (feed_names is None) != (signature_def_key is None), \ 'must either provide feed_names or singnature_def_key' @@ -272,9 +274,7 @@ def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key, feed_nam def _build_with_sig_def(sess, graph, sig_def): # pylint: disable=protected-access,attribute-defined-outside-init - assert sig_def, \ - 'signature_def {} provided, '.format(sig_def) + \ - 'but failed to find it from the meta_graph_def' + assert sig_def, 'signature_def must not be None' with sess.as_default(), graph.as_default(): feed_mapping = {} @@ -284,6 +284,7 @@ def _build_with_sig_def(sess, graph, sig_def): feed_mapping[sigdef_key] = tnsr_name feed_names.append(tnsr_name) + # TODO: IN-THIS-PR, test if these mappings are constructed correctly. fetch_mapping = {} fetch_names = [] for sigdef_key, tnsr_info in sig_def.outputs.items(): @@ -302,9 +303,8 @@ def _build_with_sig_def(sess, graph, sig_def): def _build_with_feeds_fetches(sess, graph, feed_names, fetch_names): - # pylint: disable=protected-access,attribute-defined-outside-init - assert (feed_names is not None) and (fetch_names is not None), \ - "must provide feed_names {} and fetch_names {}".format(feed_names, fetch_names) + assert feed_names is not None, "must provide feed_names" + assert fetch_names is not None, "must provide fetch names" with sess.as_default(), graph.as_default(): for tnsr_name in feed_names: From ac681b03ab2c8aac61f253c3949ba5c944a0373a Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 3 Oct 2017 12:14:44 -0700 Subject: [PATCH 68/80] more tests --- python/sparkdl/graph/input.py | 2 +- python/tests/graph/base_utils.py | 71 +++++++++++++++++++++----- python/tests/graph/test_input_graph.py | 17 ++++-- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 9ac76a91..6779bbeb 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -273,7 +273,7 @@ def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key, feed_nam def _build_with_sig_def(sess, graph, sig_def): - # pylint: disable=protected-access,attribute-defined-outside-init + # pylint: disable=protected-access assert sig_def, 'signature_def must not be None' with sess.as_default(), graph.as_default(): diff --git a/python/tests/graph/base_utils.py b/python/tests/graph/base_utils.py index b788cd4f..c09173e3 100644 --- a/python/tests/graph/base_utils.py +++ b/python/tests/graph/base_utils.py @@ -17,6 +17,7 @@ from collections import namedtuple from contextlib import contextmanager from glob import glob +import itertools import os import shutil import tempfile @@ -27,7 +28,8 @@ from sparkdl.graph.input import * import sparkdl.graph.utils as tfx -TestCase = namedtuple('TestCase', ['ref_out', 'tgt_out', 'description']) +TestCase = namedtuple('TestCase', ['bool_result', 'err_msg']) +TestFn = namedtuple('TestFn', ['test_fn', 'description']) _GinInfo = namedtuple('_GinInfo', ['gin', 'description']) @@ -67,6 +69,10 @@ def __init__(self, vec_size=17, test_batch_size=231): self._temp_dirs = [] def make_tempdir(self): + """ Create temp directories using this function. + At the end of this test object's life cycle, the temporary directories + will all be cleaned up. + """ tmp_dir = tempfile.mkdtemp() self._temp_dirs.append(tmp_dir) return tmp_dir @@ -123,24 +129,33 @@ def prep_tf_session(self): ref_feed = tfx.get_tensor(self.input_op_name, graph) ref_fetch = tfx.get_tensor(self.output_op_name, graph) - def create_test_result(tgt_gdef, test_idx): - namespace = 'TEST_TGT_NS{:03d}'.format(test_idx) - tf.import_graph_def(tgt_gdef, name=namespace) + test_data = np.random.randn(self.test_batch_size, self.vec_size) + ref_out = sess.run(ref_fetch, feed_dict={ref_feed: test_data}) + + def create_test_case(graph_def, description): + graph = tf.Graph() + with tf.Session(graph=graph) as sess: + namespace = 'TEST_TF_INPUT_GRAPH' + tf.import_graph_def(graph_def, name=namespace) tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, self.input_op_name), graph) tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) - - local_data = np.random.randn(self.test_batch_size, self.vec_size) - ref_out = sess.run(ref_fetch, feed_dict={ref_feed: local_data}) # Run on the testing target - tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: local_data}) + tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: test_data}) - return ref_out, tgt_out + # Uncomment to check if test cases work in parallel + # if np.random(1) < 0.3: + # raise RuntimeError('randomly killing tests') - for test_idx, gin_info in enumerate(self.input_graphs): - gdef = gin_info.gin.graph_def - ref_out, tgt_out = create_test_result(gdef, test_idx) - self.test_cases.append( - TestCase(ref_out=ref_out, tgt_out=tgt_out, description=gin_info.description)) + max_diff = np.max(np.abs(ref_out - tgt_out)) + err_msg = '{}: max abs diff {}'.format(description, max_diff) + return TestCase(bool_result=np.allclose(ref_out, tgt_out), err_msg=err_msg) + + for gin_info in self.input_graphs: + gdef = gin_info.gin.graph_def + description = gin_info.description + test_case = TestFn(test_fn=lambda: create_test_case(gdef, description), + description=description) + self.test_cases.append(test_case) def register(self, gin, description): self.input_graphs.append(_GinInfo(gin=gin, description=description)) @@ -211,6 +226,10 @@ def build_from_saved_model(self): serving_sigdef_key) self.register(gin=gin, description='saved model with signature') + imap_ref = {'input_sig': tfx.tensor_name(x)} + omap_ref = {'output_sig': tfx.tensor_name(z)} + self._add_signature_tensor_name_test_cases(gin, imap_ref, omap_ref) + # Build the transformer from exported serving model # We are not using signatures, thus must provide tensor/operation names gin = TFInputGraph.fromSavedModel(saved_model_dir, serving_tag, self.feed_names, @@ -262,9 +281,33 @@ def build_from_checkpoint(self): gin = TFInputGraph.fromCheckpointWithSignature(model_ckpt_dir, serving_sigdef_key) self.register(gin=gin, description='checkpoint with signature') + imap_ref = {'input_sig': tfx.tensor_name(x)} + omap_ref = {'output_sig': tfx.tensor_name(z)} + self._add_signature_tensor_name_test_cases(gin, imap_ref, omap_ref) + # Transformer without using signature_def gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) self.register(gin=gin, description='checkpoint no signature') gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) self.register(gin=gin, description='checkpoing, graph') + + def _add_signature_tensor_name_test_cases(self, gin, imap_ref, omap_ref): + """ Add tests for checking signature to tensor names mapping """ + imap_tgt = gin.input_tensor_name_from_signature + description = 'test input signature to tensor name mapping' + + def check_imap(): + err_msg = '{}: {} != {}'.format(description, imap_tgt, imap_ref) + return TestCase(bool_result=imap_ref == imap_tgt, err_msg=err_msg) + + self.test_cases.append(TestFn(test_fn=check_imap, description=description)) + + omap_tgt = gin.output_tensor_name_from_signature + description = 'test output signature to tensor name mapping' + + def check_omap(): + err_msg = '{}: {} != {}'.format(description, omap_tgt, omap_ref) + return TestCase(bool_result=omap_ref == omap_tgt, err_msg=err_msg) + + self.test_cases.append(TestFn(test_fn=check_omap, description=description)) diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index f0e57bf9..729ccc18 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -15,6 +15,7 @@ from __future__ import absolute_import, division, print_function from collections import namedtuple +import itertools import numpy as np # Use this to create parameterized test cases @@ -41,14 +42,22 @@ def _REGISTER_(obj): #======================================================================== _ALL_TEST_CASES = [] +_CLEAN_UP_TASKS = [] + for obj in _TEST_CASES_GENERATORS: obj.build_input_graphs() _ALL_TEST_CASES += obj.test_cases - obj.tear_down_env() + _CLEAN_UP_TASKS.append(obj.tear_down_env) class TFInputGraphTest(PythonUnitTestCase): + @classmethod + def tearDownClass(cls): + for clean_fn in _CLEAN_UP_TASKS: + clean_fn() + @parameterized.expand(_ALL_TEST_CASES) - def test_tf_input_graph(self, ref_out, tgt_out, description): - """ Test build TFInputGraph from various methods """ - self.assertTrue(np.allclose(ref_out, tgt_out), msg=description) + def test_tf_input_graph(self, test_fn, description): # pylint: disable=unused-argument + """ Test build TFInputGraph from various sources """ + bool_result, err_msg = test_fn() + self.assertTrue(bool_result, msg=err_msg) From 4d173c546d3db783cf5d0913f6909108b427e967 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 3 Oct 2017 12:20:32 -0700 Subject: [PATCH 69/80] change test generator module name --- ...{base_utils.py => base_test_generators.py} | 43 ++++++++++--------- python/tests/graph/test_input_graph.py | 2 +- 2 files changed, 23 insertions(+), 22 deletions(-) rename python/tests/graph/{base_utils.py => base_test_generators.py} (91%) diff --git a/python/tests/graph/base_utils.py b/python/tests/graph/base_test_generators.py similarity index 91% rename from python/tests/graph/base_utils.py rename to python/tests/graph/base_test_generators.py index c09173e3..d17b7b12 100644 --- a/python/tests/graph/base_utils.py +++ b/python/tests/graph/base_test_generators.py @@ -129,32 +129,33 @@ def prep_tf_session(self): ref_feed = tfx.get_tensor(self.input_op_name, graph) ref_fetch = tfx.get_tensor(self.output_op_name, graph) + # Build test data and reference results test_data = np.random.randn(self.test_batch_size, self.vec_size) ref_out = sess.run(ref_fetch, feed_dict={ref_feed: test_data}) - def create_test_case(graph_def, description): - graph = tf.Graph() - with tf.Session(graph=graph) as sess: - namespace = 'TEST_TF_INPUT_GRAPH' - tf.import_graph_def(graph_def, name=namespace) - tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, self.input_op_name), graph) - tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) - # Run on the testing target - tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: test_data}) - - # Uncomment to check if test cases work in parallel - # if np.random(1) < 0.3: - # raise RuntimeError('randomly killing tests') - - max_diff = np.max(np.abs(ref_out - tgt_out)) - err_msg = '{}: max abs diff {}'.format(description, max_diff) - return TestCase(bool_result=np.allclose(ref_out, tgt_out), err_msg=err_msg) - for gin_info in self.input_graphs: - gdef = gin_info.gin.graph_def + graph_def = gin_info.gin.graph_def description = gin_info.description - test_case = TestFn(test_fn=lambda: create_test_case(gdef, description), - description=description) + + def gen_input_graph_test_case(): + graph = tf.Graph() + with tf.Session(graph=graph) as sess: + namespace = 'TEST_TF_INPUT_GRAPH' + tf.import_graph_def(graph_def, name=namespace) + tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, self.input_op_name), graph) + tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) + # Run on the testing target + tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: test_data}) + + # Uncomment to check if test cases work in parallel + # if np.random(1) < 0.3: + # raise RuntimeError('randomly killing tests') + + max_diff = np.max(np.abs(ref_out - tgt_out)) + err_msg = '{}: max abs diff {}'.format(description, max_diff) + return TestCase(bool_result=np.allclose(ref_out, tgt_out), err_msg=err_msg) + + test_case = TestFn(test_fn=gen_input_graph_test_case, description=description) self.test_cases.append(test_case) def register(self, gin, description): diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index 729ccc18..085ca75b 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -22,7 +22,7 @@ from parameterized import parameterized from ..tests import PythonUnitTestCase -from .base_utils import GenTestCases +from .base_test_generators import GenTestCases #======================================================================== # Don't have to modify the content below From 22754c9f248e3cf14795977d384a0525df117035 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 3 Oct 2017 13:45:20 -0700 Subject: [PATCH 70/80] tensor tests --- python/tests/graph/base_test_generators.py | 2 + python/tests/tests.py | 17 +- python/tests/transformers/tf_tensor_test.py | 185 ++++++++++++-------- 3 files changed, 130 insertions(+), 74 deletions(-) diff --git a/python/tests/graph/base_test_generators.py b/python/tests/graph/base_test_generators.py index d17b7b12..c1c4e61c 100644 --- a/python/tests/graph/base_test_generators.py +++ b/python/tests/graph/base_test_generators.py @@ -28,6 +28,8 @@ from sparkdl.graph.input import * import sparkdl.graph.utils as tfx +__all__ = ['TestCase', 'GenTestCases', 'TestFn'] + TestCase = namedtuple('TestCase', ['bool_result', 'err_msg']) TestFn = namedtuple('TestFn', ['test_fn', 'description']) _GinInfo = namedtuple('_GinInfo', ['gin', 'description']) diff --git a/python/tests/tests.py b/python/tests/tests.py index 9492a07b..4bf9d65d 100644 --- a/python/tests/tests.py +++ b/python/tests/tests.py @@ -34,21 +34,32 @@ class PythonUnitTestCase(unittest.TestCase): # This class is created to avoid replicating this logic in various places. pass -class SparkDLTestCase(unittest.TestCase): +class TestSparkContext(object): @classmethod - def setUpClass(cls): + def setup_env(cls): cls.sc = SparkContext('local[*]', cls.__name__) cls.sql = SQLContext(cls.sc) cls.session = SparkSession.builder.getOrCreate() @classmethod - def tearDownClass(cls): + def tear_down_env(cls): cls.session.stop() cls.session = None cls.sc.stop() cls.sc = None cls.sql = None + +class SparkDLTestCase(TestSparkContext, unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.setup_env() + + @classmethod + def tearDownClass(cls): + cls.tear_down_env() + def assertDfHasCols(self, df, cols = []): map(lambda c: self.assertIn(c, df.columns), cols) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 1c8e6838..65c27d29 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -22,6 +22,7 @@ from keras.layers import Conv1D, Dense, Flatten, MaxPool1D import numpy as np +from parameterized import parameterized import tensorflow as tf import tensorframes as tfs @@ -32,32 +33,33 @@ import sparkdl.graph.utils as tfx from sparkdl.transformers.tf_tensor import TFTransformer -from ..tests import SparkDLTestCase -from ..graph.test_input_graph import TFInputGraphTest +from ..tests import SparkDLTestCase, TestSparkContext +from ..graph.base_test_generators import * -class TFTransformerTest(TFInputGraphTest, SparkDLTestCase): +class GenTFTransformerTestCases(GenTestCases): - def setUp(self): - super(TFTransformerTest, self).setUp() + def __init__(self, vec_size=17, test_batch_size=231): + super(GenTFTransformerTestCases, self).__init__(vec_size, test_batch_size) + self.input_graph_with_signature = None + self.sig_input_mapping = None + self.sig_output_mapping = None self.all_close_tol = 1e-8 @contextmanager - def _run_test_in_tf_session(self): + def prep_tf_session(self): """ [THIS IS NOT A TEST]: encapsulate general test workflow """ + self.input_graphs = [] # Build local features and DataFrame from it local_features = [] - for idx in range(self.num_samples): + for idx in range(self.test_batch_size): _dict = {'idx': idx} for colname, _ in self.input_mapping.items(): _dict[colname] = np.random.randn(self.vec_size).tolist() local_features.append(Row(**_dict)) - df = self.session.createDataFrame(local_features) - analyzed_df = tfs.analyze(df) - # Build the TensorFlow graph graph = tf.Graph() with tf.Session(graph=graph) as sess, graph.as_default(): @@ -67,11 +69,11 @@ def _run_test_in_tf_session(self): # Get the reference data _results = [] for row in local_features: - fetches = [tfx.get_tensor(graph, tnsr_op_name) - for tnsr_op_name in self.output_mapping.keys()] + fetches = [tfx.get_tensor(tnsr_op_name, graph) + for tnsr_op_name, _ in self.output_mapping.items()] feed_dict = {} for colname, tnsr_op_name in self.input_mapping.items(): - tnsr = tfx.get_tensor(graph, tnsr_op_name) + tnsr = tfx.get_tensor(tnsr_op_name, graph) feed_dict[tnsr] = np.array(row[colname])[np.newaxis, :] curr_res = sess.run(fetches, feed_dict=feed_dict) @@ -80,7 +82,9 @@ def _run_test_in_tf_session(self): out_ref = np.hstack(_results) # We have sessions, now create transformers out of them - def check_transformer(transformer): + def check_transformer(transformer, create_dataframe_fn): + df = create_dataframe_fn(local_features) + analyzed_df = tfs.analyze(df) out_df = transformer.transform(analyzed_df) out_colnames = [] for old_colname, new_colname in self.output_mapping.items(): @@ -94,19 +98,20 @@ def check_transformer(transformer): _results.append(np.ravel(curr_res)) out_tgt = np.hstack(_results) - err_msg = 'not close => shape {} != {}, max_diff {} > {}' - self.assertTrue(np.allclose(out_ref, out_tgt, atol=self.all_close_tol), - msg=err_msg.format(out_ref.shape, - out_tgt.shape, - np.max(np.abs(out_ref - out_tgt)), - self.all_close_tol)) + _err_msg = 'not close => shape {} != {}, max_diff {} > {}' + bool_result = np.allclose(out_ref, out_tgt, atol=self.all_close_tol) + max_diff = np.max(np.abs(out_ref - out_tgt)) + err_msg = _err_msg.format(out_ref.shape, out_tgt.shape, + max_diff, self.all_close_tol) + return TestCase(bool_result=bool_result, err_msg=err_msg) # Apply the transform for input_graph in self.input_graphs: transformer = TFTransformer(tfInputGraph=input_graph, inputMapping=self.input_mapping, outputMapping=self.output_mapping) - check_transformer(transformer) + self.test_cases.append(TestFn(test_fn=lambda fn: check_transformer(transformer, fn), + description='dunno')) if input_graph in self.input_graph_with_signature: _imap = input_graph.translateInputMapping(self.sig_input_mapping) @@ -114,53 +119,91 @@ def check_transformer(transformer): transformer = TFTransformer(tfInputGraph=input_graph, inputMapping=_imap, outputMapping=_omap) - check_transformer(transformer) - - def test_multi_io(self): - """ Build TFTransformer with multiple I/O tensors """ - self.setup_iomap(replica=3) - with self._run_test_in_tf_session() as sess: - xs = [] - for tnsr_op_name in self.input_mapping.values(): - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) - xs.append(x) - - zs = [] - for i, tnsr_op_name in enumerate(self.output_mapping.keys()): - z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) - zs.append(z) - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.input_graphs.append(gin) - - - def test_mixed_keras_graph(self): - """ Build TFTransformer from mixed keras graph """ - with IsolatedSession(using_keras=True) as issn: - tnsr_in = tf.placeholder( - tf.double, shape=[None, self.vec_size], name=self.input_op_name) - inp = tf.expand_dims(tnsr_in, axis=2) - # Keras layers does not take tf.double - inp = tf.cast(inp, tf.float32) - conv = Conv1D(filters=4, kernel_size=2)(inp) - pool = MaxPool1D(pool_size=2)(conv) - flat = Flatten()(pool) - dense = Dense(1)(flat) - # We must keep the leading dimension of the output - redsum = tf.reduce_logsumexp(dense, axis=1) - tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) - - # Initialize the variables - init_op = tf.global_variables_initializer() - issn.run(init_op) - # We could train the model ... but skip it here - gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) - - self.all_close_tol = 1e-5 - gin = TFInputGraph.fromGraphDef(gfn.graph_def, self.feed_names, self.fetch_names) - self.input_graphs.append(gin) - - with self._run_test_in_tf_session() as sess: - tf.import_graph_def(gfn.graph_def, name='') - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.input_graphs.append(gin) + self.test_cases.append(TestFn(test_fn=lambda fn: check_transformer(transformer, fn), + description='dunno 2')) + + # def test_multi_io(self): + # """ Build TFTransformer with multiple I/O tensors """ + # self.setup_iomap(replica=3) + # with self._run_test_in_tf_session() as sess: + # xs = [] + # for tnsr_op_name in self.input_mapping.values(): + # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) + # xs.append(x) + + # zs = [] + # for i, tnsr_op_name in enumerate(self.output_mapping.keys()): + # z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) + # zs.append(z) + + # gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + # self.input_graphs.append(gin) + + + # def test_mixed_keras_graph(self): + # """ Build TFTransformer from mixed keras graph """ + # with IsolatedSession(using_keras=True) as issn: + # tnsr_in = tf.placeholder( + # tf.double, shape=[None, self.vec_size], name=self.input_op_name) + # inp = tf.expand_dims(tnsr_in, axis=2) + # # Keras layers does not take tf.double + # inp = tf.cast(inp, tf.float32) + # conv = Conv1D(filters=4, kernel_size=2)(inp) + # pool = MaxPool1D(pool_size=2)(conv) + # flat = Flatten()(pool) + # dense = Dense(1)(flat) + # # We must keep the leading dimension of the output + # redsum = tf.reduce_logsumexp(dense, axis=1) + # tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) + + # # Initialize the variables + # init_op = tf.global_variables_initializer() + # issn.run(init_op) + # # We could train the model ... but skip it here + # gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) + + # self.all_close_tol = 1e-5 + # gin = TFInputGraph.fromGraphDef(gfn.graph_def, self.feed_names, self.fetch_names) + # self.input_graphs.append(gin) + + # with self._run_test_in_tf_session() as sess: + # tf.import_graph_def(gfn.graph_def, name='') + # gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + # self.input_graphs.append(gin) + + + +_TEST_CASES_GENERATORS = [] + + +def _REGISTER_(obj): + _TEST_CASES_GENERATORS.append(obj) + + +#======================================================================== +# Register all test objects here +_REGISTER_(GenTFTransformerTestCases(vec_size=23, test_batch_size=71)) +# _REGISTER_(GenTestCases(vec_size=13, test_batch_size=23)) +# _REGISTER_(GenTestCases(vec_size=5, test_batch_size=17)) +#======================================================================== + +_ALL_TEST_CASES = [] +_CLEAN_UP_TASKS = [] + +for obj in _TEST_CASES_GENERATORS: + obj.build_input_graphs() + _ALL_TEST_CASES += obj.test_cases + _CLEAN_UP_TASKS.append(obj.tear_down_env) + + +class TFTransformerTests(SparkDLTestCase): + @classmethod + def tearDownClass(cls): + for clean_fn in _CLEAN_UP_TASKS: + clean_fn() + + @parameterized.expand(_ALL_TEST_CASES) + def test_tf_transformers(self, test_fn, description): # pylint: disable=unused-argument + """ Test build TFInputGraph from various sources """ + bool_result, err_msg = test_fn(lambda xs: self.session.createDataFrame(xs)) + self.assertTrue(bool_result, msg=err_msg) From 0144b8c967316f247680c706826eb916bc9aac42 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 3 Oct 2017 15:10:15 -0700 Subject: [PATCH 71/80] tensor test update --- python/sparkdl/param/converters.py | 1 + python/sparkdl/transformers/tf_tensor.py | 13 +- python/tests/graph/base_test_generators.py | 119 +++++++++------ python/tests/graph/test_input_graph.py | 2 +- python/tests/transformers/tf_tensor_test.py | 158 +++++++++++--------- 5 files changed, 169 insertions(+), 124 deletions(-) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index c9b89771..620f6155 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -30,6 +30,7 @@ from pyspark.ml.param import TypeConverters +from sparkdl.graph.input import * import sparkdl.utils.keras_model as kmutil __all__ = ['SparkDLTypeConverters'] diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 8b11d9af..1edf804d 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -66,8 +66,8 @@ def _optimize_for_inference(self): gin = self.getTFInputGraph() input_mapping = self.getInputMapping() output_mapping = self.getOutputMapping() - input_node_names = [tfx.as_op_name(tnsr_name) for _, tnsr_name in input_mapping] - output_node_names = [tfx.as_op_name(tnsr_name) for tnsr_name, _ in output_mapping] + input_node_names = [tfx.op_name(tnsr_name) for _, tnsr_name in input_mapping] + output_node_names = [tfx.op_name(tnsr_name) for tnsr_name, _ in output_mapping] # NOTE(phi-dbq): Spark DataFrame assumes float64 as default floating point type opt_gdef = infr_opt.optimize_for_inference(gin.graph_def, @@ -85,17 +85,18 @@ def _transform(self, dataset): with tf.Session(graph=graph): analyzed_df = tfs.analyze(dataset) - out_tnsr_op_names = [tfx.as_op_name(tnsr_name) for tnsr_name, _ in output_mapping] + out_tnsr_op_names = [tfx.op_name(tnsr_name) for tnsr_name, _ in output_mapping] tf.import_graph_def(graph_def=graph_def, name='', return_elements=out_tnsr_op_names) - feed_dict = dict((tfx.op_name(graph, tnsr_name), col_name) + feed_dict = dict((tfx.op_name(tnsr_name, graph), col_name) for col_name, tnsr_name in input_mapping) - fetches = [tfx.get_tensor(graph, tnsr_op_name) for tnsr_op_name in out_tnsr_op_names] + fetches = [tfx.get_tensor(tnsr_op_name, graph) for tnsr_op_name in out_tnsr_op_names] out_df = tfs.map_blocks(fetches, analyzed_df, feed_dict=feed_dict) # We still have to rename output columns - for old_colname, new_colname in output_mapping: + for tnsr_name, new_colname in output_mapping: + old_colname = tfx.op_name(tnsr_name, graph) if old_colname != new_colname: out_df = out_df.withColumnRenamed(old_colname, new_colname) diff --git a/python/tests/graph/base_test_generators.py b/python/tests/graph/base_test_generators.py index c1c4e61c..efd22817 100644 --- a/python/tests/graph/base_test_generators.py +++ b/python/tests/graph/base_test_generators.py @@ -31,7 +31,7 @@ __all__ = ['TestCase', 'GenTestCases', 'TestFn'] TestCase = namedtuple('TestCase', ['bool_result', 'err_msg']) -TestFn = namedtuple('TestFn', ['test_fn', 'description']) +TestFn = namedtuple('TestFn', ['test_fn', 'description', 'metadata']) _GinInfo = namedtuple('_GinInfo', ['gin', 'description']) @@ -47,6 +47,14 @@ def __init__(self, vec_size=17, test_batch_size=231): self.output_op_name = 'tnsrOpOut' self.fetch_names = [] + # Serving signatures + self.serving_tag = "serving_tag" + self.serving_sigdef_key = 'prediction_signature' + self.input_sig_name = 'wellKnownInputSig' + self.output_sig_name = 'wellKnownOutputSig' + self.sig_input_mapping = {} + self.sig_output_mapping = {} + # DataFrame column names self.input_col = 'dfInputCol' self.output_col = 'dfOutputCol' @@ -55,14 +63,15 @@ def __init__(self, vec_size=17, test_batch_size=231): self.input_mapping = {} self.output_mapping = {} - # When testing against multiple graph inputs, - # derive new names for the DataFrame columns and TensorFlow graph elements. - self.reset_iomap(replica=1) + # # When testing against multiple graph inputs, + # # derive new names for the DataFrame columns and TensorFlow graph elements. + # self.reset_iomap(replica=1) # The basic stage contains the opaque :py:obj:`TFInputGraph` objects # Any derived that override the :py:obj:`build_input_graphs` method will # populate this field. self.input_graphs = [] + self.input_graph_with_signature = set() # Construct final test cases, which will be passed to final test cases self.test_cases = [] @@ -85,29 +94,40 @@ def tear_down_env(self): def reset_iomap(self, replica=1): self.input_mapping = {} + self.sig_input_mapping = {} self.feed_names = [] self.output_mapping = {} + self.sig_output_mapping = {} self.fetch_names = [] if replica > 1: + _template = '{}_replica{:03d}' for i in range(replica): - colname = '{}_replica{:03d}'.format(self.input_col, i) - tnsr_op_name = '{}_replica{:03d}'.format(self.input_op_name, i) - self.input_mapping[colname] = tnsr_op_name - self.feed_names.append(tnsr_op_name + ':0') - - colname = '{}_replica{:03d}'.format(self.output_col, i) - tnsr_op_name = '{}_replica{:03d}'.format(self.output_op_name, i) - self.output_mapping[tnsr_op_name] = colname - self.fetch_names.append(tnsr_op_name + ':0') + colname = _template.format(self.input_col, i) + op_name = _template.format(self.input_op_name, i) + sig_name = _template.format(self.input_sig_name, i) + tnsr_name = tfx.tensor_name(op_name) + self.input_mapping[colname] = tnsr_name + self.feed_names.append(tnsr_name) + self.sig_input_mapping[colname] = sig_name + + colname = _template.format(self.output_col, i) + op_name = _template.format(self.output_op_name, i) + sig_name = _template.format(self.output_sig_name, i) + tnsr_name = tfx.tensor_name(op_name) + self.output_mapping[tnsr_name] = colname + self.fetch_names.append(tnsr_name) + self.sig_output_mapping[sig_name] = colname else: - self.input_mapping = {self.input_col: self.input_op_name} - self.feed_names = [self.input_op_name + ':0'] - self.output_mapping = {self.output_op_name: self.output_col} - self.fetch_names = [self.output_op_name + ':0'] + self.input_mapping = {self.input_col: tfx.tensor_name(self.input_op_name)} + self.sig_input_mapping = {self.input_col: self.input_sig_name} + self.feed_names = [tfx.tensor_name(self.input_op_name)] + self.output_mapping = {tfx.tensor_name(self.output_op_name): self.output_col} + self.sig_output_mapping = {self.output_sig_name: self.output_col} + self.fetch_names = [tfx.tensor_name(self.output_op_name)] @contextmanager - def prep_tf_session(self): + def prep_tf_session(self, io_replica=1): """ Create a session to let build testing graphs Downstream classes could also choose to override this function to @@ -117,7 +137,9 @@ def prep_tf_session(self): # In each `prep_tf_session`, the implementation is expected to define ONE graph and # pass all test cases derived from it. We execute the graph and compare the result # with each test case, and return the numerical results. + self.reset_iomap(replica=io_replica) self.input_graphs = [] + self.input_graph_with_signature.clear() # Build the TensorFlow graph graph = tf.Graph() @@ -138,14 +160,16 @@ def prep_tf_session(self): for gin_info in self.input_graphs: graph_def = gin_info.gin.graph_def description = gin_info.description + input_op_name = self.input_op_name + output_op_name = self.output_op_name def gen_input_graph_test_case(): graph = tf.Graph() with tf.Session(graph=graph) as sess: namespace = 'TEST_TF_INPUT_GRAPH' tf.import_graph_def(graph_def, name=namespace) - tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, self.input_op_name), graph) - tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) + tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, input_op_name), graph) + tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, output_op_name), graph) # Run on the testing target tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: test_data}) @@ -157,7 +181,8 @@ def gen_input_graph_test_case(): err_msg = '{}: max abs diff {}'.format(description, max_diff) return TestCase(bool_result=np.allclose(ref_out, tgt_out), err_msg=err_msg) - test_case = TestFn(test_fn=gen_input_graph_test_case, description=description) + test_case = TestFn(test_fn=gen_input_graph_test_case, + description=description, metadata={}) self.test_cases.append(test_case) def register(self, gin, description): @@ -199,8 +224,6 @@ def build_from_saved_model(self): """ Build TFTransformer from saved model """ # Setup saved model export directory saved_model_dir = os.path.join(self.make_tempdir(), 'saved_model') - serving_tag = "serving_tag" - serving_sigdef_key = 'prediction_signature' builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) with self.prep_tf_session() as sess: @@ -213,29 +236,29 @@ def build_from_saved_model(self): sess.run(w.initializer) - sig_inputs = {'input_sig': tf.saved_model.utils.build_tensor_info(x)} - sig_outputs = {'output_sig': tf.saved_model.utils.build_tensor_info(z)} + sig_inputs = {self.input_sig_name: tf.saved_model.utils.build_tensor_info(x)} + sig_outputs = {self.output_sig_name: tf.saved_model.utils.build_tensor_info(z)} serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( inputs=sig_inputs, outputs=sig_outputs) builder.add_meta_graph_and_variables( - sess, [serving_tag], signature_def_map={serving_sigdef_key: serving_sigdef}) + sess, [self.serving_tag], signature_def_map={self.serving_sigdef_key: serving_sigdef}) builder.save() # Build the transformer from exported serving model # We are using signatures, thus must provide the keys - gin = TFInputGraph.fromSavedModelWithSignature(saved_model_dir, serving_tag, - serving_sigdef_key) + gin = TFInputGraph.fromSavedModelWithSignature(saved_model_dir, self.serving_tag, + self.serving_sigdef_key) self.register(gin=gin, description='saved model with signature') - imap_ref = {'input_sig': tfx.tensor_name(x)} - omap_ref = {'output_sig': tfx.tensor_name(z)} - self._add_signature_tensor_name_test_cases(gin, imap_ref, omap_ref) + _imap = {self.input_sig_name: tfx.tensor_name(x)} + _omap = {self.output_sig_name: tfx.tensor_name(z)} + self._add_signature_tensor_name_test_cases(gin, _imap, _omap) # Build the transformer from exported serving model # We are not using signatures, thus must provide tensor/operation names - gin = TFInputGraph.fromSavedModel(saved_model_dir, serving_tag, self.feed_names, + gin = TFInputGraph.fromSavedModel(saved_model_dir, self.serving_tag, self.feed_names, self.fetch_names) self.register(gin=gin, description='saved model no signature') @@ -247,7 +270,6 @@ def build_from_checkpoint(self): # Build the TensorFlow graph model_ckpt_dir = self.make_tempdir() ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - serving_sigdef_key = 'prediction_signature' with self.prep_tf_session() as sess: x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) @@ -261,8 +283,8 @@ def build_from_checkpoint(self): # Prepare the signature_def serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs={'input_sig': tf.saved_model.utils.build_tensor_info(x)}, - outputs={'output_sig': tf.saved_model.utils.build_tensor_info(z)}) + inputs={self.input_sig_name: tf.saved_model.utils.build_tensor_info(x)}, + outputs={self.output_sig_name: tf.saved_model.utils.build_tensor_info(z)}) # A rather contrived way to add signature def to a meta_graph meta_graph_def = tf.train.export_meta_graph() @@ -275,18 +297,18 @@ def build_from_checkpoint(self): # Add signature_def to the meta_graph and serialize it # This will overwrite the existing meta_graph_def file - meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) + meta_graph_def.signature_def[self.serving_sigdef_key].CopyFrom(serving_sigdef) with open(ckpt_meta_fpath, mode='wb') as fout: fout.write(meta_graph_def.SerializeToString()) # Build the transformer from exported serving model # We are using signaures, thus must provide the keys - gin = TFInputGraph.fromCheckpointWithSignature(model_ckpt_dir, serving_sigdef_key) + gin = TFInputGraph.fromCheckpointWithSignature(model_ckpt_dir, self.serving_sigdef_key) self.register(gin=gin, description='checkpoint with signature') - imap_ref = {'input_sig': tfx.tensor_name(x)} - omap_ref = {'output_sig': tfx.tensor_name(z)} - self._add_signature_tensor_name_test_cases(gin, imap_ref, omap_ref) + _imap = {self.input_sig_name: tfx.tensor_name(x)} + _omap = {self.output_sig_name: tfx.tensor_name(z)} + self._add_signature_tensor_name_test_cases(gin, _imap, _omap) # Transformer without using signature_def gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) @@ -295,22 +317,25 @@ def build_from_checkpoint(self): gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) self.register(gin=gin, description='checkpoing, graph') - def _add_signature_tensor_name_test_cases(self, gin, imap_ref, omap_ref): + def _add_signature_tensor_name_test_cases(self, gin, input_sig2tnsr, output_sig2tnsr): """ Add tests for checking signature to tensor names mapping """ imap_tgt = gin.input_tensor_name_from_signature description = 'test input signature to tensor name mapping' def check_imap(): - err_msg = '{}: {} != {}'.format(description, imap_tgt, imap_ref) - return TestCase(bool_result=imap_ref == imap_tgt, err_msg=err_msg) + err_msg = '{}: {} != {}'.format(description, imap_tgt, input_sig2tnsr) + bool_result = input_sig2tnsr == imap_tgt + return TestCase(bool_result=bool_result, err_msg=err_msg) - self.test_cases.append(TestFn(test_fn=check_imap, description=description)) + self.test_cases.append(TestFn(test_fn=check_imap, description=description, metadata={})) omap_tgt = gin.output_tensor_name_from_signature description = 'test output signature to tensor name mapping' def check_omap(): - err_msg = '{}: {} != {}'.format(description, omap_tgt, omap_ref) - return TestCase(bool_result=omap_ref == omap_tgt, err_msg=err_msg) + err_msg = '{}: {} != {}'.format(description, omap_tgt, output_sig2tnsr) + bool_result = output_sig2tnsr == omap_tgt + return TestCase(bool_result=bool_result, err_msg=err_msg) - self.test_cases.append(TestFn(test_fn=check_omap, description=description)) + self.input_graph_with_signature.add(gin) + self.test_cases.append(TestFn(test_fn=check_omap, description=description, metadata={})) diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py index 085ca75b..c0e14253 100644 --- a/python/tests/graph/test_input_graph.py +++ b/python/tests/graph/test_input_graph.py @@ -57,7 +57,7 @@ def tearDownClass(cls): clean_fn() @parameterized.expand(_ALL_TEST_CASES) - def test_tf_input_graph(self, test_fn, description): # pylint: disable=unused-argument + def test_tf_input_graph(self, test_fn, description, metadata): # pylint: disable=unused-argument """ Test build TFInputGraph from various sources """ bool_result, err_msg = test_fn() self.assertTrue(bool_result, msg=err_msg) diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py index 65c27d29..8031ec9d 100644 --- a/python/tests/transformers/tf_tensor_test.py +++ b/python/tests/transformers/tf_tensor_test.py @@ -36,20 +36,25 @@ from ..tests import SparkDLTestCase, TestSparkContext from ..graph.base_test_generators import * - class GenTFTransformerTestCases(GenTestCases): def __init__(self, vec_size=17, test_batch_size=231): super(GenTFTransformerTestCases, self).__init__(vec_size, test_batch_size) - self.input_graph_with_signature = None self.sig_input_mapping = None self.sig_output_mapping = None self.all_close_tol = 1e-8 + def build_input_graphs(self): + super(GenTFTransformerTestCases, self).build_input_graphs() + self.build_mixed_keras_graph() + self.build_multi_io() + @contextmanager - def prep_tf_session(self): + def prep_tf_session(self, io_replica=1): """ [THIS IS NOT A TEST]: encapsulate general test workflow """ + self.reset_iomap(replica=io_replica) self.input_graphs = [] + self.input_graph_with_signature.clear() # Build local features and DataFrame from it local_features = [] @@ -69,11 +74,11 @@ def prep_tf_session(self): # Get the reference data _results = [] for row in local_features: - fetches = [tfx.get_tensor(tnsr_op_name, graph) - for tnsr_op_name, _ in self.output_mapping.items()] + fetches = [tfx.get_tensor(tnsr_name, graph) + for tnsr_name, _ in self.output_mapping.items()] feed_dict = {} - for colname, tnsr_op_name in self.input_mapping.items(): - tnsr = tfx.get_tensor(tnsr_op_name, graph) + for colname, tnsr_name in self.input_mapping.items(): + tnsr = tfx.get_tensor(tnsr_name, graph) feed_dict[tnsr] = np.array(row[colname])[np.newaxis, :] curr_res = sess.run(fetches, feed_dict=feed_dict) @@ -81,17 +86,18 @@ def prep_tf_session(self): out_ref = np.hstack(_results) + # Must make sure the function does not depend on `self` + tnsr2col_mapping = self.output_mapping + all_close_tol = self.all_close_tol + # We have sessions, now create transformers out of them def check_transformer(transformer, create_dataframe_fn): df = create_dataframe_fn(local_features) analyzed_df = tfs.analyze(df) out_df = transformer.transform(analyzed_df) - out_colnames = [] - for old_colname, new_colname in self.output_mapping.items(): - out_colnames.append(new_colname) - if old_colname != new_colname: - out_df = out_df.withColumnRenamed(old_colname, new_colname) + # Collect transformed values + out_colnames = list(tnsr2col_mapping.values()) _results = [] for row in out_df.select(out_colnames).collect(): curr_res = [row[colname] for colname in out_colnames] @@ -99,19 +105,24 @@ def check_transformer(transformer, create_dataframe_fn): out_tgt = np.hstack(_results) _err_msg = 'not close => shape {} != {}, max_diff {} > {}' - bool_result = np.allclose(out_ref, out_tgt, atol=self.all_close_tol) max_diff = np.max(np.abs(out_ref - out_tgt)) err_msg = _err_msg.format(out_ref.shape, out_tgt.shape, - max_diff, self.all_close_tol) + max_diff, all_close_tol) + bool_result = np.allclose(out_ref, out_tgt, atol=all_close_tol) return TestCase(bool_result=bool_result, err_msg=err_msg) + # Apply the transform - for input_graph in self.input_graphs: + for gin_info in self.input_graphs: + input_graph = gin_info.gin transformer = TFTransformer(tfInputGraph=input_graph, inputMapping=self.input_mapping, outputMapping=self.output_mapping) + + description = '{} for TFTransformer'.format(gin_info.description) self.test_cases.append(TestFn(test_fn=lambda fn: check_transformer(transformer, fn), - description='dunno')) + description=description, + metadata=dict(need_dataframe=True))) if input_graph in self.input_graph_with_signature: _imap = input_graph.translateInputMapping(self.sig_input_mapping) @@ -120,56 +131,59 @@ def check_transformer(transformer, create_dataframe_fn): inputMapping=_imap, outputMapping=_omap) self.test_cases.append(TestFn(test_fn=lambda fn: check_transformer(transformer, fn), - description='dunno 2')) - - # def test_multi_io(self): - # """ Build TFTransformer with multiple I/O tensors """ - # self.setup_iomap(replica=3) - # with self._run_test_in_tf_session() as sess: - # xs = [] - # for tnsr_op_name in self.input_mapping.values(): - # x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=tnsr_op_name) - # xs.append(x) - - # zs = [] - # for i, tnsr_op_name in enumerate(self.output_mapping.keys()): - # z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name) - # zs.append(z) - - # gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - # self.input_graphs.append(gin) - - - # def test_mixed_keras_graph(self): - # """ Build TFTransformer from mixed keras graph """ - # with IsolatedSession(using_keras=True) as issn: - # tnsr_in = tf.placeholder( - # tf.double, shape=[None, self.vec_size], name=self.input_op_name) - # inp = tf.expand_dims(tnsr_in, axis=2) - # # Keras layers does not take tf.double - # inp = tf.cast(inp, tf.float32) - # conv = Conv1D(filters=4, kernel_size=2)(inp) - # pool = MaxPool1D(pool_size=2)(conv) - # flat = Flatten()(pool) - # dense = Dense(1)(flat) - # # We must keep the leading dimension of the output - # redsum = tf.reduce_logsumexp(dense, axis=1) - # tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) - - # # Initialize the variables - # init_op = tf.global_variables_initializer() - # issn.run(init_op) - # # We could train the model ... but skip it here - # gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) - - # self.all_close_tol = 1e-5 - # gin = TFInputGraph.fromGraphDef(gfn.graph_def, self.feed_names, self.fetch_names) - # self.input_graphs.append(gin) - - # with self._run_test_in_tf_session() as sess: - # tf.import_graph_def(gfn.graph_def, name='') - # gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - # self.input_graphs.append(gin) + description='dunno 2', + metadata=dict(need_dataframe=True))) + + def build_multi_io(self): + """ Build TFTransformer with multiple I/O tensors """ + with self.prep_tf_session(io_replica=3) as sess: + xs = [] + for tnsr_name in self.input_mapping.values(): + x = tf.placeholder(tf.float64, shape=[None, self.vec_size], + name=tfx.op_name(tnsr_name)) + xs.append(x) + + zs = [] + for i, tnsr_name in enumerate(self.output_mapping.keys()): + z = tf.reduce_mean(xs[i], axis=1, name=tfx.op_name(tnsr_name)) + zs.append(z) + + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + description = 'TFInputGraph with multiple input/output tensors' + self.register(gin=gin, description=description) + + + def build_mixed_keras_graph(self): + """ Build TFTransformer from mixed keras graph """ + with IsolatedSession(using_keras=True) as issn: + tnsr_in = tf.placeholder( + tf.double, shape=[None, self.vec_size], name=self.input_op_name) + inp = tf.expand_dims(tnsr_in, axis=2) + # Keras layers does not take tf.double + inp = tf.cast(inp, tf.float32) + conv = Conv1D(filters=4, kernel_size=2)(inp) + pool = MaxPool1D(pool_size=2)(conv) + flat = Flatten()(pool) + dense = Dense(1)(flat) + # We must keep the leading dimension of the output + redsum = tf.reduce_logsumexp(dense, axis=1) + tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) + + # Initialize the variables + init_op = tf.global_variables_initializer() + issn.run(init_op) + # We could train the model ... but skip it here + gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) + + self.all_close_tol = 1e-5 + gin = TFInputGraph.fromGraphDef(gfn.graph_def, self.feed_names, self.fetch_names) + self.input_graphs.append(gin) + + with self.prep_tf_session() as sess: + tf.import_graph_def(gfn.graph_def, name='') + gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) + description = 'TFInputGraph from Keras' + self.register(gin=gin, description=description) @@ -183,8 +197,8 @@ def _REGISTER_(obj): #======================================================================== # Register all test objects here _REGISTER_(GenTFTransformerTestCases(vec_size=23, test_batch_size=71)) -# _REGISTER_(GenTestCases(vec_size=13, test_batch_size=23)) -# _REGISTER_(GenTestCases(vec_size=5, test_batch_size=17)) +_REGISTER_(GenTFTransformerTestCases(vec_size=13, test_batch_size=23)) +_REGISTER_(GenTFTransformerTestCases(vec_size=5, test_batch_size=17)) #======================================================================== _ALL_TEST_CASES = [] @@ -203,7 +217,11 @@ def tearDownClass(cls): clean_fn() @parameterized.expand(_ALL_TEST_CASES) - def test_tf_transformers(self, test_fn, description): # pylint: disable=unused-argument + def test_tf_transformers(self, test_fn, description, metadata): # pylint: disable=unused-argument """ Test build TFInputGraph from various sources """ - bool_result, err_msg = test_fn(lambda xs: self.session.createDataFrame(xs)) + if metadata and metadata['need_dataframe']: + bool_result, err_msg = test_fn(lambda xs: self.session.createDataFrame(xs)) + else: + bool_result, err_msg = test_fn() + self.assertTrue(bool_result, msg=err_msg) From a8531ec31808d016b46014b9ab9639dd4a93d3ec Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 3 Oct 2017 16:32:32 -0700 Subject: [PATCH 72/80] buildCheckList name change and doc fixup --- python/sparkdl/param/converters.py | 2 +- python/sparkdl/param/image_params.py | 2 +- python/sparkdl/param/shared_params.py | 2 +- python/sparkdl/transformers/named_image.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/sparkdl/param/converters.py b/python/sparkdl/param/converters.py index 246d46cf..a692a013 100644 --- a/python/sparkdl/param/converters.py +++ b/python/sparkdl/param/converters.py @@ -115,7 +115,7 @@ def toTFTensorName(value): raise TypeError(err_msg.format(type(value), value, exc)) @staticmethod - def buildCheckList(supportedList): + def buildSupportedItemConverter(supportedList): """ Create a "converter" that try to check if a value is part of the supported list of values. diff --git a/python/sparkdl/param/image_params.py b/python/sparkdl/param/image_params.py index a423adae..6ca2ff6d 100644 --- a/python/sparkdl/param/image_params.py +++ b/python/sparkdl/param/image_params.py @@ -107,7 +107,7 @@ class HasOutputMode(Params): "How the output column should be formatted. 'vector' for a 1-d MLlib " + "Vector of floats. 'image' to format the output to work with the image " + "tools in this package.", - typeConverter=SparkDLTypeConverters.buildCheckList(OUTPUT_MODES)) + typeConverter=SparkDLTypeConverters.buildSupportedItemConverter(OUTPUT_MODES)) def setOutputMode(self, value): return self._set(outputMode=value) diff --git a/python/sparkdl/param/shared_params.py b/python/sparkdl/param/shared_params.py index 433c591a..432d618d 100644 --- a/python/sparkdl/param/shared_params.py +++ b/python/sparkdl/param/shared_params.py @@ -232,7 +232,7 @@ class HasTFHParams(Params): key-value object, storing parameters to be used to define the final TensorFlow graph for the Transformer. - Currently accepted values are: + Currently used values are: - `batch_size`: number of samples evaluated together in inference steps"""), typeConverter=SparkDLTypeConverters.toTFHParams) diff --git a/python/sparkdl/transformers/named_image.py b/python/sparkdl/transformers/named_image.py index 76fce766..f3139b7a 100644 --- a/python/sparkdl/transformers/named_image.py +++ b/python/sparkdl/transformers/named_image.py @@ -40,7 +40,7 @@ class DeepImagePredictor(Transformer, HasInputCol, HasOutputCol): """ modelName = Param(Params._dummy(), "modelName", "A deep learning model name", - typeConverter=SparkDLTypeConverters.buildCheckList(SUPPORTED_MODELS)) + typeConverter=SparkDLTypeConverters.buildSupportedItemConverter(SUPPORTED_MODELS)) decodePredictions = Param(Params._dummy(), "decodePredictions", "If true, output predictions in the (class, description, probability) format", typeConverter=TypeConverters.toBoolean) @@ -125,7 +125,7 @@ class DeepImageFeaturizer(Transformer, HasInputCol, HasOutputCol): """ modelName = Param(Params._dummy(), "modelName", "A deep learning model name", - typeConverter=SparkDLTypeConverters.buildCheckList(SUPPORTED_MODELS)) + typeConverter=SparkDLTypeConverters.buildSupportedItemConverter(SUPPORTED_MODELS)) @keyword_only def __init__(self, inputCol=None, outputCol=None, modelName=None): @@ -169,7 +169,7 @@ class _NamedImageTransformer(Transformer, HasInputCol, HasOutputCol): """ modelName = Param(Params._dummy(), "modelName", "A deep learning model name", - typeConverter=SparkDLTypeConverters.buildCheckList(SUPPORTED_MODELS)) + typeConverter=SparkDLTypeConverters.buildSupportedItemConverter(SUPPORTED_MODELS)) featurize = Param(Params._dummy(), "featurize", "If true, output features. If false, output predictions. Either way the output is a vector.", typeConverter=TypeConverters.toBoolean) From d7295280940a9e0783371dcc2372c448f0d4b32f Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 3 Oct 2017 18:41:51 -0700 Subject: [PATCH 73/80] PR comments --- python/sparkdl/graph/utils.py | 15 ++++++++++---- python/tests/graph/test_utils.py | 34 ++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/python/sparkdl/graph/utils.py b/python/sparkdl/graph/utils.py index 3bcbef0e..5077c9dd 100644 --- a/python/sparkdl/graph/utils.py +++ b/python/sparkdl/graph/utils.py @@ -62,6 +62,7 @@ def get_op(tfobj_or_name, graph): By default the graph we don't require this argument to be provided. """ graph = validated_graph(graph) + _assert_same_graph(tfobj_or_name, graph) if isinstance(tfobj_or_name, tf.Operation): return tfobj_or_name name = tfobj_or_name @@ -71,8 +72,7 @@ def get_op(tfobj_or_name, graph): raise TypeError('invalid op request for [type {}] {}'.format(type(name), name)) _op_name = op_name(name, graph=None) op = graph.get_operation_by_name(_op_name) - assert op is not None, \ - 'cannot locate op {} in current graph'.format(_op_name) + assert isinstance(op, tf.Operation), 'expect tf.Operation, but got {}'.format(type(op)) return op def get_tensor(tfobj_or_name, graph): @@ -85,6 +85,7 @@ def get_tensor(tfobj_or_name, graph): By default the graph we don't require this argument to be provided. """ graph = validated_graph(graph) + _assert_same_graph(tfobj_or_name, graph) if isinstance(tfobj_or_name, tf.Tensor): return tfobj_or_name name = tfobj_or_name @@ -94,8 +95,7 @@ def get_tensor(tfobj_or_name, graph): raise TypeError('invalid tensor request for {} of {}'.format(name, type(name))) _tensor_name = tensor_name(name, graph=None) tnsr = graph.get_tensor_by_name(_tensor_name) - assert tnsr is not None, \ - 'cannot locate tensor {} in current graph'.format(_tensor_name) + assert isinstance(tnsr, tf.Tensor), 'expect tf.Tensor, but got {}'.format(type(tnsr)) return tnsr def tensor_name(tfobj_or_name, graph=None): @@ -213,3 +213,10 @@ def strip_and_freeze_until(fetches, graph, sess=None, return_graph=False): return g else: return gdef_frozen + + +def _assert_same_graph(tfobj, graph): + if graph is None or not hasattr(tfobj, 'graph'): + return + assert tfobj.graph == graph, \ + 'the graph of TensorFlow element {} != graph {}'.format(tfobj, graph) diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py index b973c669..3109ce48 100644 --- a/python/tests/graph/test_utils.py +++ b/python/tests/graph/test_utils.py @@ -42,7 +42,31 @@ def _gen_invalid_tensor_or_op_input_with_wrong_types(): yield TestCase(data=wrong_val, description='wrong type {}'.format(type(wrong_val))) -def _gen_valid_tensor_op_objects(): +def _gen_invalid_tensor_or_op_with_graph_pairing(): + tnsr = tf.constant(1427.08, name='someConstOp') + other_graph = tf.Graph() + op_name = tnsr.op.name + + # Test get_tensor and get_op returns tensor or op contained in the same graph + yield TestCase(data=lambda: tfx.get_op(tnsr, other_graph), + description='test graph from getting op fron tensor') + yield TestCase(data=lambda: tfx.get_tensor(tnsr, other_graph), + description='test graph from getting tensor from tensor') + yield TestCase(data=lambda: tfx.get_op(tnsr.name, other_graph), + description='test graph from getting op fron tensor name') + yield TestCase(data=lambda: tfx.get_tensor(tnsr.name, other_graph), + description='test graph from getting tensor from tensor name') + yield TestCase(data=lambda: tfx.get_op(tnsr.op, other_graph), + description='test graph from getting op from op') + yield TestCase(data=lambda: tfx.get_tensor(tnsr.op, other_graph), + description='test graph from getting tensor from op') + yield TestCase(data=lambda: tfx.get_op(op_name, other_graph), + description='test graph from getting op from op name') + yield TestCase(data=lambda: tfx.get_tensor(op_name, other_graph), + description='test graph from getting tensor from op name') + + +def _gen_valid_tensor_op_input_combos(): op_name = 'someConstOp' tnsr_name = '{}:0'.format(op_name) tnsr = tf.constant(1427.08, name=op_name) @@ -154,8 +178,14 @@ def test_invalid_tensor_inputs_with_wrong_types(self, data, description): with self.assertRaises(TypeError, msg=description): tfx.get_tensor(data, tf.Graph()) - @parameterized.expand(_gen_valid_tensor_op_objects) + @parameterized.expand(_gen_valid_tensor_op_input_combos) def test_valid_tensor_op_object_inputs(self, data, description): """ Must get correct graph elements from valid graph elements or their names """ tfobj_or_name_a, tfobj_or_name_b = data self.assertEqual(tfobj_or_name_a, tfobj_or_name_b, msg=description) + + @parameterized.expand(_gen_invalid_tensor_or_op_with_graph_pairing) + def test_invalid_tensor_op_object_graph_pairing(self, data, description): + """ Must fail when the graph element is from a different graph than the provided """ + with self.assertRaises((KeyError, AssertionError, TypeError), msg=description): + data() From 0c2eda105a5449dba4fc89df4a6f8028273c3045 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 4 Oct 2017 12:21:16 -0700 Subject: [PATCH 74/80] PR comments --- python/sparkdl/graph/utils.py | 15 +++++++------ python/tests/graph/test_utils.py | 37 +++++++++----------------------- 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/python/sparkdl/graph/utils.py b/python/sparkdl/graph/utils.py index 5077c9dd..64e093fe 100644 --- a/python/sparkdl/graph/utils.py +++ b/python/sparkdl/graph/utils.py @@ -72,12 +72,13 @@ def get_op(tfobj_or_name, graph): raise TypeError('invalid op request for [type {}] {}'.format(type(name), name)) _op_name = op_name(name, graph=None) op = graph.get_operation_by_name(_op_name) - assert isinstance(op, tf.Operation), 'expect tf.Operation, but got {}'.format(type(op)) + err_msg = 'cannot locate op {} in the current graph, got [type {}] {}' + assert isinstance(op, tf.Operation), err_msg.format(_op_name, type(op), op) return op def get_tensor(tfobj_or_name, graph): """ - Get a tf.Tensor object + Get a :py:class:`tf.Tensor` object :param tfobj_or_name: either a :py:class:`tf.Tensor`, :py:class:`tf.Operation` or a name to either. @@ -95,7 +96,8 @@ def get_tensor(tfobj_or_name, graph): raise TypeError('invalid tensor request for {} of {}'.format(name, type(name))) _tensor_name = tensor_name(name, graph=None) tnsr = graph.get_tensor_by_name(_tensor_name) - assert isinstance(tnsr, tf.Tensor), 'expect tf.Tensor, but got {}'.format(type(tnsr)) + err_msg = 'cannot locate tensor {} in the current graph, got [type {}] {}' + assert isinstance(tnsr, tf.Tensor), err_msg.format(_tensor_name, type(tnsr), tnsr) return tnsr def tensor_name(tfobj_or_name, graph=None): @@ -216,7 +218,6 @@ def strip_and_freeze_until(fetches, graph, sess=None, return_graph=False): def _assert_same_graph(tfobj, graph): - if graph is None or not hasattr(tfobj, 'graph'): - return - assert tfobj.graph == graph, \ - 'the graph of TensorFlow element {} != graph {}'.format(tfobj, graph) + if graph is not None and hasattr(tfobj, 'graph'): + err_msg = 'the graph of TensorFlow element {} != graph {}' + assert tfobj.graph == graph, err_msg.format(tfobj, graph) diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py index 3109ce48..dab98668 100644 --- a/python/tests/graph/test_utils.py +++ b/python/tests/graph/test_utils.py @@ -47,23 +47,24 @@ def _gen_invalid_tensor_or_op_with_graph_pairing(): other_graph = tf.Graph() op_name = tnsr.op.name - # Test get_tensor and get_op returns tensor or op contained in the same graph + # Test get_tensor and get_op with non-associated tensor/op and graph inputs + _comm_suffix = ' with non-associated tensor/op and graph inputs' yield TestCase(data=lambda: tfx.get_op(tnsr, other_graph), - description='test graph from getting op fron tensor') + description='test get_op with from tensor' + _comm_suffix) yield TestCase(data=lambda: tfx.get_tensor(tnsr, other_graph), - description='test graph from getting tensor from tensor') + description='test get_tensor from tensor' + _comm_suffix) yield TestCase(data=lambda: tfx.get_op(tnsr.name, other_graph), - description='test graph from getting op fron tensor name') + description='test get_op fron tensor name' + _comm_suffix) yield TestCase(data=lambda: tfx.get_tensor(tnsr.name, other_graph), - description='test graph from getting tensor from tensor name') + description='test get_tensor from tensor name' + _comm_suffix) yield TestCase(data=lambda: tfx.get_op(tnsr.op, other_graph), - description='test graph from getting op from op') + description='test get_op from op' + _comm_suffix) yield TestCase(data=lambda: tfx.get_tensor(tnsr.op, other_graph), - description='test graph from getting tensor from op') + description='test get_tensor from op' + _comm_suffix) yield TestCase(data=lambda: tfx.get_op(op_name, other_graph), - description='test graph from getting op from op name') + description='test get_op from op name' + _comm_suffix) yield TestCase(data=lambda: tfx.get_tensor(op_name, other_graph), - description='test graph from getting tensor from op name') + description='test get_tensor from op name' + _comm_suffix) def _gen_valid_tensor_op_input_combos(): @@ -128,24 +129,6 @@ def _gen_valid_tensor_op_input_combos(): yield TestCase(data=(tnsr.op, tfx.get_op(op_name, graph)), description='test op from op name') - # Test get_tensor and get_op returns tensor or op contained in the same graph - yield TestCase(data=(graph, tfx.get_op(tnsr, graph).graph), - description='test graph from getting op fron tensor') - yield TestCase(data=(graph, tfx.get_tensor(tnsr, graph).graph), - description='test graph from getting tensor from tensor') - yield TestCase(data=(graph, tfx.get_op(tnsr_name, graph).graph), - description='test graph from getting op fron tensor name') - yield TestCase(data=(graph, tfx.get_tensor(tnsr_name, graph).graph), - description='test graph from getting tensor from tensor name') - yield TestCase(data=(graph, tfx.get_op(tnsr.op, graph).graph), - description='test graph from getting op from op') - yield TestCase(data=(graph, tfx.get_tensor(tnsr.op, graph).graph), - description='test graph from getting tensor from op') - yield TestCase(data=(graph, tfx.get_op(op_name, graph).graph), - description='test graph from getting op from op name') - yield TestCase(data=(graph, tfx.get_tensor(op_name, graph).graph), - description='test graph from getting tensor from op name') - class TFeXtensionGraphUtilsTest(PythonUnitTestCase): @parameterized.expand(_gen_tensor_op_string_input_tests) From 63967b49d9e4787d72c3aebf781e130052dfddf8 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 4 Oct 2017 17:04:17 -0700 Subject: [PATCH 75/80] PR comments --- python/tests/graph/test_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/tests/graph/test_utils.py b/python/tests/graph/test_utils.py index dab98668..4847c9b1 100644 --- a/python/tests/graph/test_utils.py +++ b/python/tests/graph/test_utils.py @@ -48,13 +48,13 @@ def _gen_invalid_tensor_or_op_with_graph_pairing(): op_name = tnsr.op.name # Test get_tensor and get_op with non-associated tensor/op and graph inputs - _comm_suffix = ' with non-associated tensor/op and graph inputs' + _comm_suffix = ' with wrong graph' yield TestCase(data=lambda: tfx.get_op(tnsr, other_graph), - description='test get_op with from tensor' + _comm_suffix) + description='test get_op from tensor' + _comm_suffix) yield TestCase(data=lambda: tfx.get_tensor(tnsr, other_graph), description='test get_tensor from tensor' + _comm_suffix) yield TestCase(data=lambda: tfx.get_op(tnsr.name, other_graph), - description='test get_op fron tensor name' + _comm_suffix) + description='test get_op from tensor name' + _comm_suffix) yield TestCase(data=lambda: tfx.get_tensor(tnsr.name, other_graph), description='test get_tensor from tensor name' + _comm_suffix) yield TestCase(data=lambda: tfx.get_op(tnsr.op, other_graph), @@ -169,6 +169,6 @@ def test_valid_tensor_op_object_inputs(self, data, description): @parameterized.expand(_gen_invalid_tensor_or_op_with_graph_pairing) def test_invalid_tensor_op_object_graph_pairing(self, data, description): - """ Must fail when the graph element is from a different graph than the provided """ + """ Must fail with non-associated tensor/op and graph inputs """ with self.assertRaises((KeyError, AssertionError, TypeError), msg=description): data() From 47d497c7e6c0aa541f61490896af9d433602fc04 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 18 Nov 2017 08:48:06 -0800 Subject: [PATCH 76/80] TFTransformer Part-4 Test Refactor (#15) * adding new tests * remove original test design * cleanup --- python/tests/transformers/tf_tensor_test.py | 227 ------------------ .../tests/transformers/tf_transformer_test.py | 145 +++++++++++ 2 files changed, 145 insertions(+), 227 deletions(-) delete mode 100644 python/tests/transformers/tf_tensor_test.py create mode 100644 python/tests/transformers/tf_transformer_test.py diff --git a/python/tests/transformers/tf_tensor_test.py b/python/tests/transformers/tf_tensor_test.py deleted file mode 100644 index 8031ec9d..00000000 --- a/python/tests/transformers/tf_tensor_test.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2017 Databricks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from __future__ import absolute_import, division, print_function - -from contextlib import contextmanager -from glob import glob -import os -import shutil -import tempfile - -from keras.layers import Conv1D, Dense, Flatten, MaxPool1D -import numpy as np -from parameterized import parameterized -import tensorflow as tf -import tensorframes as tfs - -from pyspark.sql.types import Row - -from sparkdl.graph.builder import IsolatedSession -from sparkdl.graph.input import * -import sparkdl.graph.utils as tfx -from sparkdl.transformers.tf_tensor import TFTransformer - -from ..tests import SparkDLTestCase, TestSparkContext -from ..graph.base_test_generators import * - -class GenTFTransformerTestCases(GenTestCases): - - def __init__(self, vec_size=17, test_batch_size=231): - super(GenTFTransformerTestCases, self).__init__(vec_size, test_batch_size) - self.sig_input_mapping = None - self.sig_output_mapping = None - self.all_close_tol = 1e-8 - - def build_input_graphs(self): - super(GenTFTransformerTestCases, self).build_input_graphs() - self.build_mixed_keras_graph() - self.build_multi_io() - - @contextmanager - def prep_tf_session(self, io_replica=1): - """ [THIS IS NOT A TEST]: encapsulate general test workflow """ - self.reset_iomap(replica=io_replica) - self.input_graphs = [] - self.input_graph_with_signature.clear() - - # Build local features and DataFrame from it - local_features = [] - for idx in range(self.test_batch_size): - _dict = {'idx': idx} - for colname, _ in self.input_mapping.items(): - _dict[colname] = np.random.randn(self.vec_size).tolist() - - local_features.append(Row(**_dict)) - - # Build the TensorFlow graph - graph = tf.Graph() - with tf.Session(graph=graph) as sess, graph.as_default(): - # Build test graph and transformers from here - yield sess - - # Get the reference data - _results = [] - for row in local_features: - fetches = [tfx.get_tensor(tnsr_name, graph) - for tnsr_name, _ in self.output_mapping.items()] - feed_dict = {} - for colname, tnsr_name in self.input_mapping.items(): - tnsr = tfx.get_tensor(tnsr_name, graph) - feed_dict[tnsr] = np.array(row[colname])[np.newaxis, :] - - curr_res = sess.run(fetches, feed_dict=feed_dict) - _results.append(np.ravel(curr_res)) - - out_ref = np.hstack(_results) - - # Must make sure the function does not depend on `self` - tnsr2col_mapping = self.output_mapping - all_close_tol = self.all_close_tol - - # We have sessions, now create transformers out of them - def check_transformer(transformer, create_dataframe_fn): - df = create_dataframe_fn(local_features) - analyzed_df = tfs.analyze(df) - out_df = transformer.transform(analyzed_df) - - # Collect transformed values - out_colnames = list(tnsr2col_mapping.values()) - _results = [] - for row in out_df.select(out_colnames).collect(): - curr_res = [row[colname] for colname in out_colnames] - _results.append(np.ravel(curr_res)) - out_tgt = np.hstack(_results) - - _err_msg = 'not close => shape {} != {}, max_diff {} > {}' - max_diff = np.max(np.abs(out_ref - out_tgt)) - err_msg = _err_msg.format(out_ref.shape, out_tgt.shape, - max_diff, all_close_tol) - bool_result = np.allclose(out_ref, out_tgt, atol=all_close_tol) - return TestCase(bool_result=bool_result, err_msg=err_msg) - - - # Apply the transform - for gin_info in self.input_graphs: - input_graph = gin_info.gin - transformer = TFTransformer(tfInputGraph=input_graph, - inputMapping=self.input_mapping, - outputMapping=self.output_mapping) - - description = '{} for TFTransformer'.format(gin_info.description) - self.test_cases.append(TestFn(test_fn=lambda fn: check_transformer(transformer, fn), - description=description, - metadata=dict(need_dataframe=True))) - - if input_graph in self.input_graph_with_signature: - _imap = input_graph.translateInputMapping(self.sig_input_mapping) - _omap = input_graph.translateOutputMapping(self.sig_output_mapping) - transformer = TFTransformer(tfInputGraph=input_graph, - inputMapping=_imap, - outputMapping=_omap) - self.test_cases.append(TestFn(test_fn=lambda fn: check_transformer(transformer, fn), - description='dunno 2', - metadata=dict(need_dataframe=True))) - - def build_multi_io(self): - """ Build TFTransformer with multiple I/O tensors """ - with self.prep_tf_session(io_replica=3) as sess: - xs = [] - for tnsr_name in self.input_mapping.values(): - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], - name=tfx.op_name(tnsr_name)) - xs.append(x) - - zs = [] - for i, tnsr_name in enumerate(self.output_mapping.keys()): - z = tf.reduce_mean(xs[i], axis=1, name=tfx.op_name(tnsr_name)) - zs.append(z) - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - description = 'TFInputGraph with multiple input/output tensors' - self.register(gin=gin, description=description) - - - def build_mixed_keras_graph(self): - """ Build TFTransformer from mixed keras graph """ - with IsolatedSession(using_keras=True) as issn: - tnsr_in = tf.placeholder( - tf.double, shape=[None, self.vec_size], name=self.input_op_name) - inp = tf.expand_dims(tnsr_in, axis=2) - # Keras layers does not take tf.double - inp = tf.cast(inp, tf.float32) - conv = Conv1D(filters=4, kernel_size=2)(inp) - pool = MaxPool1D(pool_size=2)(conv) - flat = Flatten()(pool) - dense = Dense(1)(flat) - # We must keep the leading dimension of the output - redsum = tf.reduce_logsumexp(dense, axis=1) - tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name) - - # Initialize the variables - init_op = tf.global_variables_initializer() - issn.run(init_op) - # We could train the model ... but skip it here - gfn = issn.asGraphFunction([tnsr_in], [tnsr_out]) - - self.all_close_tol = 1e-5 - gin = TFInputGraph.fromGraphDef(gfn.graph_def, self.feed_names, self.fetch_names) - self.input_graphs.append(gin) - - with self.prep_tf_session() as sess: - tf.import_graph_def(gfn.graph_def, name='') - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - description = 'TFInputGraph from Keras' - self.register(gin=gin, description=description) - - - -_TEST_CASES_GENERATORS = [] - - -def _REGISTER_(obj): - _TEST_CASES_GENERATORS.append(obj) - - -#======================================================================== -# Register all test objects here -_REGISTER_(GenTFTransformerTestCases(vec_size=23, test_batch_size=71)) -_REGISTER_(GenTFTransformerTestCases(vec_size=13, test_batch_size=23)) -_REGISTER_(GenTFTransformerTestCases(vec_size=5, test_batch_size=17)) -#======================================================================== - -_ALL_TEST_CASES = [] -_CLEAN_UP_TASKS = [] - -for obj in _TEST_CASES_GENERATORS: - obj.build_input_graphs() - _ALL_TEST_CASES += obj.test_cases - _CLEAN_UP_TASKS.append(obj.tear_down_env) - - -class TFTransformerTests(SparkDLTestCase): - @classmethod - def tearDownClass(cls): - for clean_fn in _CLEAN_UP_TASKS: - clean_fn() - - @parameterized.expand(_ALL_TEST_CASES) - def test_tf_transformers(self, test_fn, description, metadata): # pylint: disable=unused-argument - """ Test build TFInputGraph from various sources """ - if metadata and metadata['need_dataframe']: - bool_result, err_msg = test_fn(lambda xs: self.session.createDataFrame(xs)) - else: - bool_result, err_msg = test_fn() - - self.assertTrue(bool_result, msg=err_msg) diff --git a/python/tests/transformers/tf_transformer_test.py b/python/tests/transformers/tf_transformer_test.py new file mode 100644 index 00000000..60ced31c --- /dev/null +++ b/python/tests/transformers/tf_transformer_test.py @@ -0,0 +1,145 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import absolute_import, division, print_function + +import numpy as np +import tensorflow as tf + +from pyspark.sql.types import Row + +import tensorframes as tfs + +import sparkdl.graph.utils as tfx +from sparkdl.graph.input import TFInputGraph +from sparkdl.transformers.tf_tensor import TFTransformer + +from ..tests import SparkDLTestCase + +class TFTransformerTests(SparkDLTestCase): + def test_graph_novar(self): + transformer = _build_transformer(lambda session: + TFInputGraph.fromGraph(session.graph, session, + [_tensor_input_name], + [_tensor_output_name])) + gin = transformer.getTFInputGraph() + local_features = _build_local_features() + expected = _get_expected_result(gin, local_features) + dataset = self.session.createDataFrame(local_features) + _check_transformer_output(transformer, dataset, expected) + + +# The name of the input tensor +_tensor_input_name = "input_tensor" +# The name of the output tensor (scalar) +_tensor_output_name = "output_tensor" +# The size of the input tensor +_tensor_size = 3 +# Input mapping for the Transformer +_input_mapping = {'inputCol': tfx.tensor_name(_tensor_input_name)} +# Output mapping for the Transformer +_output_mapping = {tfx.tensor_name(_tensor_output_name): 'outputCol'} +# Numerical threshold +_all_close_tolerance = 1e-5 + + +def _build_transformer(gin_function): + """ + Makes a session and a default graph, loads the simple graph into it, and then calls + gin_function(session) to build the :py:obj:`TFInputGraph` object. + Return the :py:obj:`TFTransformer` created from it. + """ + graph = tf.Graph() + with tf.Session(graph=graph) as sess, graph.as_default(): + _build_graph(sess) + gin = gin_function(sess) + + return TFTransformer(tfInputGraph=gin, + inputMapping=_input_mapping, + outputMapping=_output_mapping) + + +def _build_graph(sess): + """ + Given a session (implicitly), adds nodes of computations + + It takes a vector input, with `_tensor_size` columns and returns an float64 scalar. + """ + x = tf.placeholder(tf.float64, shape=[None, _tensor_size], name=_tensor_input_name) + _ = tf.reduce_max(x, axis=1, name=_tensor_output_name) + +def _build_local_features(): + """ + Build numpy array (i.e. local) features. + """ + # Build local features and DataFrame from it + local_features = [] + for idx in range(100): + _dict = {'idx': idx} + for colname, _ in _input_mapping.items(): + _dict[colname] = np.random.randn(_tensor_size).tolist() + + local_features.append(Row(**_dict)) + + return local_features + +def _get_expected_result(gin, local_features): + """ + Running the graph in the :py:obj:`TFInputGraph` object and compute the expected results. + :param: gin, a :py:obj:`TFInputGraph` + :return: expected results in NumPy array + """ + graph = tf.Graph() + with tf.Session(graph=graph) as sess, graph.as_default(): + # Build test graph and transformers from here + tf.import_graph_def(gin.graph_def, name='') + + # Build the results + _results = [] + for row in local_features: + fetches = [tfx.get_tensor(tnsr_name, graph) + for tnsr_name, _ in _output_mapping.items()] + feed_dict = {} + for colname, tnsr_name in _input_mapping.items(): + tnsr = tfx.get_tensor(tnsr_name, graph) + feed_dict[tnsr] = np.array(row[colname])[np.newaxis, :] + + curr_res = sess.run(fetches, feed_dict=feed_dict) + _results.append(np.ravel(curr_res)) + + expected = np.hstack(_results) + + return expected + +def _check_transformer_output(transformer, dataset, expected): + """ + Given a transformer and a spark dataset, check if the transformer + produces the expected results. + """ + analyzed_df = tfs.analyze(dataset) + out_df = transformer.transform(analyzed_df) + + # Collect transformed values + out_colnames = list(_output_mapping.values()) + _results = [] + for row in out_df.select(out_colnames).collect(): + curr_res = [row[colname] for colname in out_colnames] + _results.append(np.ravel(curr_res)) + out_tgt = np.hstack(_results) + + _err_msg = 'not close => shape {} != {}, max_diff {} > {}' + max_diff = np.max(np.abs(expected - out_tgt)) + err_msg = _err_msg.format(expected.shape, out_tgt.shape, + max_diff, _all_close_tolerance) + assert np.allclose(expected, out_tgt, atol=_all_close_tolerance), err_msg From a39b6d3a6202547c837a55dc4f12a76e300a729a Mon Sep 17 00:00:00 2001 From: Timothy Hunter Date: Sat, 18 Nov 2017 08:48:50 -0800 Subject: [PATCH 77/80] TFTransformer Part-3 Test Refactor (#14) * profiling * tests * renamed test * removed original tests * removed the profiler utils * fixes indents * imports * added some tests * added test * fix test * one more test --- python/tests/graph/base_test_generators.py | 314 --------------------- python/tests/graph/test_import.py | 290 +++++++++++++++++++ python/tests/graph/test_input_graph.py | 63 ----- 3 files changed, 290 insertions(+), 377 deletions(-) delete mode 100644 python/tests/graph/base_test_generators.py create mode 100644 python/tests/graph/test_import.py delete mode 100644 python/tests/graph/test_input_graph.py diff --git a/python/tests/graph/base_test_generators.py b/python/tests/graph/base_test_generators.py deleted file mode 100644 index d17b7b12..00000000 --- a/python/tests/graph/base_test_generators.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2017 Databricks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from __future__ import absolute_import, division, print_function - -from collections import namedtuple -from contextlib import contextmanager -from glob import glob -import itertools -import os -import shutil -import tempfile - -import numpy as np -import tensorflow as tf - -from sparkdl.graph.input import * -import sparkdl.graph.utils as tfx - -TestCase = namedtuple('TestCase', ['bool_result', 'err_msg']) -TestFn = namedtuple('TestFn', ['test_fn', 'description']) -_GinInfo = namedtuple('_GinInfo', ['gin', 'description']) - - -class TestGenBase(object): - def __init__(self, vec_size=17, test_batch_size=231): - # Testing data spec - self.vec_size = vec_size - self.test_batch_size = test_batch_size - - # TensorFlow graph element names - self.input_op_name = 'tnsrOpIn' - self.feed_names = [] - self.output_op_name = 'tnsrOpOut' - self.fetch_names = [] - - # DataFrame column names - self.input_col = 'dfInputCol' - self.output_col = 'dfOutputCol' - - # Connecting data from Spark to TensorFlow - self.input_mapping = {} - self.output_mapping = {} - - # When testing against multiple graph inputs, - # derive new names for the DataFrame columns and TensorFlow graph elements. - self.reset_iomap(replica=1) - - # The basic stage contains the opaque :py:obj:`TFInputGraph` objects - # Any derived that override the :py:obj:`build_input_graphs` method will - # populate this field. - self.input_graphs = [] - - # Construct final test cases, which will be passed to final test cases - self.test_cases = [] - - # Build a temporary directory, which might or might not be used by the test - self._temp_dirs = [] - - def make_tempdir(self): - """ Create temp directories using this function. - At the end of this test object's life cycle, the temporary directories - will all be cleaned up. - """ - tmp_dir = tempfile.mkdtemp() - self._temp_dirs.append(tmp_dir) - return tmp_dir - - def tear_down_env(self): - for tmp_dir in self._temp_dirs: - shutil.rmtree(tmp_dir, ignore_errors=True) - - def reset_iomap(self, replica=1): - self.input_mapping = {} - self.feed_names = [] - self.output_mapping = {} - self.fetch_names = [] - - if replica > 1: - for i in range(replica): - colname = '{}_replica{:03d}'.format(self.input_col, i) - tnsr_op_name = '{}_replica{:03d}'.format(self.input_op_name, i) - self.input_mapping[colname] = tnsr_op_name - self.feed_names.append(tnsr_op_name + ':0') - - colname = '{}_replica{:03d}'.format(self.output_col, i) - tnsr_op_name = '{}_replica{:03d}'.format(self.output_op_name, i) - self.output_mapping[tnsr_op_name] = colname - self.fetch_names.append(tnsr_op_name + ':0') - else: - self.input_mapping = {self.input_col: self.input_op_name} - self.feed_names = [self.input_op_name + ':0'] - self.output_mapping = {self.output_op_name: self.output_col} - self.fetch_names = [self.output_op_name + ':0'] - - @contextmanager - def prep_tf_session(self): - """ Create a session to let build testing graphs - - Downstream classes could also choose to override this function to - build custom testing behaviors - """ - # Reset states - # In each `prep_tf_session`, the implementation is expected to define ONE graph and - # pass all test cases derived from it. We execute the graph and compare the result - # with each test case, and return the numerical results. - self.input_graphs = [] - - # Build the TensorFlow graph - graph = tf.Graph() - with tf.Session(graph=graph) as sess, graph.as_default(): - # Build test graph and transformers from here - # Notice that any TensorFlow graph could potentially be constructed in this session. - # The graph might contain variables only usable in this session. - yield sess - - # Find the input/output tensors. We expect them to use canonical names. - ref_feed = tfx.get_tensor(self.input_op_name, graph) - ref_fetch = tfx.get_tensor(self.output_op_name, graph) - - # Build test data and reference results - test_data = np.random.randn(self.test_batch_size, self.vec_size) - ref_out = sess.run(ref_fetch, feed_dict={ref_feed: test_data}) - - for gin_info in self.input_graphs: - graph_def = gin_info.gin.graph_def - description = gin_info.description - - def gen_input_graph_test_case(): - graph = tf.Graph() - with tf.Session(graph=graph) as sess: - namespace = 'TEST_TF_INPUT_GRAPH' - tf.import_graph_def(graph_def, name=namespace) - tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, self.input_op_name), graph) - tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, self.output_op_name), graph) - # Run on the testing target - tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: test_data}) - - # Uncomment to check if test cases work in parallel - # if np.random(1) < 0.3: - # raise RuntimeError('randomly killing tests') - - max_diff = np.max(np.abs(ref_out - tgt_out)) - err_msg = '{}: max abs diff {}'.format(description, max_diff) - return TestCase(bool_result=np.allclose(ref_out, tgt_out), err_msg=err_msg) - - test_case = TestFn(test_fn=gen_input_graph_test_case, description=description) - self.test_cases.append(test_case) - - def register(self, gin, description): - self.input_graphs.append(_GinInfo(gin=gin, description=description)) - - def build_input_graphs(self): - raise NotImplementedError("build your graph and test cases here") - - -class GenTestCases(TestGenBase): - """ Define various test graphs - - Please define all the graphs to be built for test cases in this class. - This is useful for other classes to inherent and change the definition of - the actual testing method. That is, by overriding :py:meth:`prep_tf_session`, - the subclass can change the evaluation behavior. - """ - - def build_input_graphs(self): - self.build_from_checkpoint() - self.build_from_graph() - self.build_from_saved_model() - - def build_from_graph(self): - """ Build TFTransformer from tf.Graph """ - with self.prep_tf_session() as sess: - # Begin building graph - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) - # End building graph - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.register(gin=gin, description='from graph with graph') - gin = TFInputGraph.fromGraphDef(sess.graph.as_graph_def(), self.feed_names, - self.fetch_names) - self.register(gin=gin, description='from graph with graph_def') - - def build_from_saved_model(self): - """ Build TFTransformer from saved model """ - # Setup saved model export directory - saved_model_dir = os.path.join(self.make_tempdir(), 'saved_model') - serving_tag = "serving_tag" - serving_sigdef_key = 'prediction_signature' - builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - - with self.prep_tf_session() as sess: - # Model definition: begin - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - w = tf.Variable( - tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - # Model definition ends - - sess.run(w.initializer) - - sig_inputs = {'input_sig': tf.saved_model.utils.build_tensor_info(x)} - sig_outputs = {'output_sig': tf.saved_model.utils.build_tensor_info(z)} - - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs=sig_inputs, outputs=sig_outputs) - - builder.add_meta_graph_and_variables( - sess, [serving_tag], signature_def_map={serving_sigdef_key: serving_sigdef}) - builder.save() - - # Build the transformer from exported serving model - # We are using signatures, thus must provide the keys - gin = TFInputGraph.fromSavedModelWithSignature(saved_model_dir, serving_tag, - serving_sigdef_key) - self.register(gin=gin, description='saved model with signature') - - imap_ref = {'input_sig': tfx.tensor_name(x)} - omap_ref = {'output_sig': tfx.tensor_name(z)} - self._add_signature_tensor_name_test_cases(gin, imap_ref, omap_ref) - - # Build the transformer from exported serving model - # We are not using signatures, thus must provide tensor/operation names - gin = TFInputGraph.fromSavedModel(saved_model_dir, serving_tag, self.feed_names, - self.fetch_names) - self.register(gin=gin, description='saved model no signature') - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.register(gin=gin, description='saved model with graph') - - def build_from_checkpoint(self): - """ Build TFTransformer from a model checkpoint """ - # Build the TensorFlow graph - model_ckpt_dir = self.make_tempdir() - ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - serving_sigdef_key = 'prediction_signature' - - with self.prep_tf_session() as sess: - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - w = tf.Variable( - tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - sess.run(w.initializer) - saver = tf.train.Saver(var_list=[w]) - _ = saver.save(sess, ckpt_path_prefix, global_step=2702) - - # Prepare the signature_def - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs={'input_sig': tf.saved_model.utils.build_tensor_info(x)}, - outputs={'output_sig': tf.saved_model.utils.build_tensor_info(z)}) - - # A rather contrived way to add signature def to a meta_graph - meta_graph_def = tf.train.export_meta_graph() - - # Find the meta_graph file (there should be only one) - _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) - assert len(_ckpt_meta_fpaths) == 1, \ - 'expected only one meta graph, but got {}'.format(','.join(_ckpt_meta_fpaths)) - ckpt_meta_fpath = _ckpt_meta_fpaths[0] - - # Add signature_def to the meta_graph and serialize it - # This will overwrite the existing meta_graph_def file - meta_graph_def.signature_def[serving_sigdef_key].CopyFrom(serving_sigdef) - with open(ckpt_meta_fpath, mode='wb') as fout: - fout.write(meta_graph_def.SerializeToString()) - - # Build the transformer from exported serving model - # We are using signaures, thus must provide the keys - gin = TFInputGraph.fromCheckpointWithSignature(model_ckpt_dir, serving_sigdef_key) - self.register(gin=gin, description='checkpoint with signature') - - imap_ref = {'input_sig': tfx.tensor_name(x)} - omap_ref = {'output_sig': tfx.tensor_name(z)} - self._add_signature_tensor_name_test_cases(gin, imap_ref, omap_ref) - - # Transformer without using signature_def - gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) - self.register(gin=gin, description='checkpoint no signature') - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.register(gin=gin, description='checkpoing, graph') - - def _add_signature_tensor_name_test_cases(self, gin, imap_ref, omap_ref): - """ Add tests for checking signature to tensor names mapping """ - imap_tgt = gin.input_tensor_name_from_signature - description = 'test input signature to tensor name mapping' - - def check_imap(): - err_msg = '{}: {} != {}'.format(description, imap_tgt, imap_ref) - return TestCase(bool_result=imap_ref == imap_tgt, err_msg=err_msg) - - self.test_cases.append(TestFn(test_fn=check_imap, description=description)) - - omap_tgt = gin.output_tensor_name_from_signature - description = 'test output signature to tensor name mapping' - - def check_omap(): - err_msg = '{}: {} != {}'.format(description, omap_tgt, omap_ref) - return TestCase(bool_result=omap_ref == omap_tgt, err_msg=err_msg) - - self.test_cases.append(TestFn(test_fn=check_omap, description=description)) diff --git a/python/tests/graph/test_import.py b/python/tests/graph/test_import.py new file mode 100644 index 00000000..a4f36244 --- /dev/null +++ b/python/tests/graph/test_import.py @@ -0,0 +1,290 @@ +# Copyright 2017 Databricks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import absolute_import, division, print_function + +import contextlib +import shutil +import numpy as np +import os +import tensorflow as tf +import tempfile +import glob + +import sparkdl.graph.utils as tfx +from sparkdl.graph.input import TFInputGraph + + +class TestGraphImport(object): + def test_graph_novar(self): + gin = _build_graph_input(lambda session: + TFInputGraph.fromGraph(session.graph, session, [_tensor_input_name], + [_tensor_output_name])) + _check_input_novar(gin) + + def test_graphdef_novar(self): + gin = _build_graph_input(lambda session: + TFInputGraph.fromGraphDef(session.graph.as_graph_def(), + [_tensor_input_name], [_tensor_output_name])) + _check_input_novar(gin) + + def test_saved_model_novar(self): + with _make_temp_directory() as tmp_dir: + saved_model_dir = os.path.join(tmp_dir, 'saved_model') + + def gin_fun(session): + _build_saved_model(session, saved_model_dir) + # Build the transformer from exported serving model + # We are using signatures, thus must provide the keys + return TFInputGraph.fromSavedModelWithSignature(saved_model_dir, _serving_tag, + _serving_sigdef_key) + + gin = _build_graph_input(gin_fun) + _check_input_novar(gin) + + def test_saved_graph_novar(self): + with _make_temp_directory() as tmp_dir: + saved_model_dir = os.path.join(tmp_dir, 'saved_model') + + def gin_fun(session): + _build_saved_model(session, saved_model_dir) + return TFInputGraph.fromGraph(session.graph, session, [_tensor_input_name], [_tensor_output_name]) + + gin = _build_graph_input(gin_fun) + _check_input_novar(gin) + + def test_checkpoint_sig_var(self): + with _make_temp_directory() as tmp_dir: + def gin_fun(session): + _build_checkpointed_model(session, tmp_dir) + return TFInputGraph.fromCheckpointWithSignature(tmp_dir, _serving_sigdef_key) + + gin = _build_graph_input_var(gin_fun) + _check_input_novar(gin) + + def test_checkpoint_nosig_var(self): + with _make_temp_directory() as tmp_dir: + def gin_fun(session): + _build_checkpointed_model(session, tmp_dir) + return TFInputGraph.fromCheckpoint(tmp_dir, + [_tensor_input_name], [_tensor_output_name]) + + gin = _build_graph_input_var(gin_fun) + _check_input_novar(gin) + + def test_checkpoint_graph_var(self): + with _make_temp_directory() as tmp_dir: + def gin_fun(session): + _build_checkpointed_model(session, tmp_dir) + return TFInputGraph.fromGraph(session.graph, session, + [_tensor_input_name], [_tensor_output_name]) + + gin = _build_graph_input_var(gin_fun) + _check_input_novar(gin) + + def test_graphdef_novar_2(self): + gin = _build_graph_input_2(lambda session: + TFInputGraph.fromGraphDef(session.graph.as_graph_def(), + [_tensor_input_name], [_tensor_output_name])) + _check_output_2(gin, np.array([1, 2, 3]), np.array([2, 2, 2]), 1) + + def test_saved_graph_novar_2(self): + with _make_temp_directory() as tmp_dir: + saved_model_dir = os.path.join(tmp_dir, 'saved_model') + + def gin_fun(session): + _build_saved_model(session, saved_model_dir) + return TFInputGraph.fromGraph(session.graph, session, [_tensor_input_name], [_tensor_output_name]) + + gin = _build_graph_input_2(gin_fun) + _check_output_2(gin, np.array([1, 2, 3]), np.array([2, 2, 2]), 1) + +_serving_tag = "serving_tag" +_serving_sigdef_key = 'prediction_signature' +# The name of the input tensor +_tensor_input_name = "input_tensor" +# For testing graphs with 2 inputs +_tensor_input_name_2 = "input_tensor_2" +# The name of the output tensor (scalar) +_tensor_output_name = "output_tensor" +# The name of the variable +_tensor_var_name = "variable" +# The size of the input tensor +_tensor_size = 3 + + +def _build_checkpointed_model(session, tmp_dir): + """ + Writes a model checkpoint in the given directory. The graph is assumed to be generated + with _build_graph_var. + """ + ckpt_path_prefix = os.path.join(tmp_dir, 'model_ckpt') + input_tensor = tfx.get_tensor(_tensor_input_name, session.graph) + output_tensor = tfx.get_tensor(_tensor_output_name, session.graph) + w = tfx.get_tensor(_tensor_var_name, session.graph) + saver = tf.train.Saver(var_list=[w]) + _ = saver.save(session, ckpt_path_prefix, global_step=2702) + sig_inputs = {'input_sig': tf.saved_model.utils.build_tensor_info(input_tensor)} + sig_outputs = {'output_sig': tf.saved_model.utils.build_tensor_info(output_tensor)} + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs=sig_inputs, outputs=sig_outputs) + + # A rather contrived way to add signature def to a meta_graph + meta_graph_def = tf.train.export_meta_graph() + + # Find the meta_graph file (there should be only one) + _ckpt_meta_fpaths = glob.glob('{}/*.meta'.format(tmp_dir)) + assert len(_ckpt_meta_fpaths) == 1, \ + 'expected only one meta graph, but got {}'.format(','.join(_ckpt_meta_fpaths)) + ckpt_meta_fpath = _ckpt_meta_fpaths[0] + + # Add signature_def to the meta_graph and serialize it + # This will overwrite the existing meta_graph_def file + meta_graph_def.signature_def[_serving_sigdef_key].CopyFrom(serving_sigdef) + with open(ckpt_meta_fpath, mode='wb') as fout: + fout.write(meta_graph_def.SerializeToString()) + + +def _build_saved_model(session, saved_model_dir): + """ + Saves a model in a file. The graph is assumed to be generated with _build_graph_novar. + """ + builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) + input_tensor = tfx.get_tensor(_tensor_input_name, session.graph) + output_tensor = tfx.get_tensor(_tensor_output_name, session.graph) + sig_inputs = {'input_sig': tf.saved_model.utils.build_tensor_info(input_tensor)} + sig_outputs = {'output_sig': tf.saved_model.utils.build_tensor_info(output_tensor)} + serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( + inputs=sig_inputs, outputs=sig_outputs) + + builder.add_meta_graph_and_variables( + session, [_serving_tag], signature_def_map={_serving_sigdef_key: serving_sigdef}) + builder.save() + + +@contextlib.contextmanager +def _make_temp_directory(): + temp_dir = tempfile.mkdtemp() + try: + yield temp_dir + finally: + shutil.rmtree(temp_dir) + + +def _build_graph_input(gin_function): + """ + Makes a session and a default graph, loads the simple graph into it, and then calls + gin_function(session) to return the graph input object + """ + graph = tf.Graph() + with tf.Session(graph=graph) as s, graph.as_default(): + _build_graph() + return gin_function(s) + + +def _build_graph_input_2(gin_function): + """ + Makes a session and a default graph, loads the simple graph into it (graph_2), and then calls + gin_function(session) to return the graph input object + """ + graph = tf.Graph() + with tf.Session(graph=graph) as s, graph.as_default(): + _build_graph_2() + return gin_function(s) + + +def _build_graph_input_var(gin_function): + """ + Makes a session and a default graph, loads the simple graph into it that contains a variable, + and then calls gin_function(session) to return the graph input object + """ + graph = tf.Graph() + with tf.Session(graph=graph) as s, graph.as_default(): + _build_graph_var(s) + return gin_function(s) + + +def _build_graph(): + """ + Given a session (implicitly), adds nodes of computations + + It takes a vector input, with vec_size columns and returns an int32 scalar. + """ + x = tf.placeholder(tf.int32, shape=[_tensor_size], name=_tensor_input_name) + _ = tf.reduce_max(x, name=_tensor_output_name) + + +def _build_graph_2(): + """ + Given a session (implicitly), adds nodes of computations with two inputs. + + It takes a vector input, with vec_size columns and returns an int32 scalar. + """ + x1 = tf.placeholder(tf.int32, shape=[_tensor_size], name=_tensor_input_name) + x2 = tf.placeholder(tf.int32, shape=[_tensor_size], name=_tensor_input_name_2) + # Make sure that the inputs are not used in a symmetric manner. + _ = tf.reduce_max(x1 - x2, name=_tensor_output_name) + + +def _build_graph_var(session): + """ + Given a session, adds nodes that include one variable. + """ + x = tf.placeholder(tf.int32, shape=[_tensor_size], name=_tensor_input_name) + w = tf.Variable(tf.ones(shape=[_tensor_size], dtype=tf.int32), name=_tensor_var_name) + _ = tf.reduce_max(x * w, name=_tensor_output_name) + session.run(w.initializer) + + +def _check_input_novar(gin): + """ + Tests that the graph from _build_graph has been serialized in the InputGraph object. + """ + _check_output(gin, np.array([1, 2, 3]), 3) + + +def _check_output(gin, tf_input, expected): + """ + Takes a TFInputGraph object (assumed to have the input and outputs of the given + names above) and compares the outcome against some expected outcome. + """ + graph = tf.Graph() + graph_def = gin.graph_def + with tf.Session(graph=graph) as sess: + tf.import_graph_def(graph_def, name="") + tgt_feed = tfx.get_tensor(_tensor_input_name, graph) + tgt_fetch = tfx.get_tensor(_tensor_output_name, graph) + # Run on the testing target + tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: tf_input}) + # Working on integers, the calculation should be exact + assert np.all(tgt_out == expected), (tgt_out, expected) + + +# TODO: we could factorize with _check_output, but this is not worth the time doing it. +def _check_output_2(gin, tf_input1, tf_input2, expected): + """ + Takes a TFInputGraph object (assumed to have the input and outputs of the given + names above) and compares the outcome against some expected outcome. + """ + graph = tf.Graph() + graph_def = gin.graph_def + with tf.Session(graph=graph) as sess: + tf.import_graph_def(graph_def, name="") + tgt_feed1 = tfx.get_tensor(_tensor_input_name, graph) + tgt_feed2 = tfx.get_tensor(_tensor_input_name_2, graph) + tgt_fetch = tfx.get_tensor(_tensor_output_name, graph) + # Run on the testing target + tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed1: tf_input1, tgt_feed2: tf_input2}) + # Working on integers, the calculation should be exact + assert np.all(tgt_out == expected), (tgt_out, expected) diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py deleted file mode 100644 index 085ca75b..00000000 --- a/python/tests/graph/test_input_graph.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2017 Databricks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from __future__ import absolute_import, division, print_function - -from collections import namedtuple -import itertools - -import numpy as np -# Use this to create parameterized test cases -from parameterized import parameterized - -from ..tests import PythonUnitTestCase -from .base_test_generators import GenTestCases - -#======================================================================== -# Don't have to modify the content below - -_TEST_CASES_GENERATORS = [] - - -def _REGISTER_(obj): - _TEST_CASES_GENERATORS.append(obj) - - -#======================================================================== -# Register all test objects here -_REGISTER_(GenTestCases(vec_size=23, test_batch_size=71)) -_REGISTER_(GenTestCases(vec_size=13, test_batch_size=23)) -_REGISTER_(GenTestCases(vec_size=5, test_batch_size=17)) -#======================================================================== - -_ALL_TEST_CASES = [] -_CLEAN_UP_TASKS = [] - -for obj in _TEST_CASES_GENERATORS: - obj.build_input_graphs() - _ALL_TEST_CASES += obj.test_cases - _CLEAN_UP_TASKS.append(obj.tear_down_env) - - -class TFInputGraphTest(PythonUnitTestCase): - @classmethod - def tearDownClass(cls): - for clean_fn in _CLEAN_UP_TASKS: - clean_fn() - - @parameterized.expand(_ALL_TEST_CASES) - def test_tf_input_graph(self, test_fn, description): # pylint: disable=unused-argument - """ Test build TFInputGraph from various sources """ - bool_result, err_msg = test_fn() - self.assertTrue(bool_result, msg=err_msg) From 07cc335752ab1440d48c88d127e89440e6dfc89b Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Sat, 18 Nov 2017 08:52:07 -0800 Subject: [PATCH 78/80] deleting original testing ideas --- python/tests/graph/base_test_generators.py | 341 --------------------- python/tests/graph/test_input_graph.py | 63 ---- 2 files changed, 404 deletions(-) delete mode 100644 python/tests/graph/base_test_generators.py delete mode 100644 python/tests/graph/test_input_graph.py diff --git a/python/tests/graph/base_test_generators.py b/python/tests/graph/base_test_generators.py deleted file mode 100644 index efd22817..00000000 --- a/python/tests/graph/base_test_generators.py +++ /dev/null @@ -1,341 +0,0 @@ -# Copyright 2017 Databricks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from __future__ import absolute_import, division, print_function - -from collections import namedtuple -from contextlib import contextmanager -from glob import glob -import itertools -import os -import shutil -import tempfile - -import numpy as np -import tensorflow as tf - -from sparkdl.graph.input import * -import sparkdl.graph.utils as tfx - -__all__ = ['TestCase', 'GenTestCases', 'TestFn'] - -TestCase = namedtuple('TestCase', ['bool_result', 'err_msg']) -TestFn = namedtuple('TestFn', ['test_fn', 'description', 'metadata']) -_GinInfo = namedtuple('_GinInfo', ['gin', 'description']) - - -class TestGenBase(object): - def __init__(self, vec_size=17, test_batch_size=231): - # Testing data spec - self.vec_size = vec_size - self.test_batch_size = test_batch_size - - # TensorFlow graph element names - self.input_op_name = 'tnsrOpIn' - self.feed_names = [] - self.output_op_name = 'tnsrOpOut' - self.fetch_names = [] - - # Serving signatures - self.serving_tag = "serving_tag" - self.serving_sigdef_key = 'prediction_signature' - self.input_sig_name = 'wellKnownInputSig' - self.output_sig_name = 'wellKnownOutputSig' - self.sig_input_mapping = {} - self.sig_output_mapping = {} - - # DataFrame column names - self.input_col = 'dfInputCol' - self.output_col = 'dfOutputCol' - - # Connecting data from Spark to TensorFlow - self.input_mapping = {} - self.output_mapping = {} - - # # When testing against multiple graph inputs, - # # derive new names for the DataFrame columns and TensorFlow graph elements. - # self.reset_iomap(replica=1) - - # The basic stage contains the opaque :py:obj:`TFInputGraph` objects - # Any derived that override the :py:obj:`build_input_graphs` method will - # populate this field. - self.input_graphs = [] - self.input_graph_with_signature = set() - - # Construct final test cases, which will be passed to final test cases - self.test_cases = [] - - # Build a temporary directory, which might or might not be used by the test - self._temp_dirs = [] - - def make_tempdir(self): - """ Create temp directories using this function. - At the end of this test object's life cycle, the temporary directories - will all be cleaned up. - """ - tmp_dir = tempfile.mkdtemp() - self._temp_dirs.append(tmp_dir) - return tmp_dir - - def tear_down_env(self): - for tmp_dir in self._temp_dirs: - shutil.rmtree(tmp_dir, ignore_errors=True) - - def reset_iomap(self, replica=1): - self.input_mapping = {} - self.sig_input_mapping = {} - self.feed_names = [] - self.output_mapping = {} - self.sig_output_mapping = {} - self.fetch_names = [] - - if replica > 1: - _template = '{}_replica{:03d}' - for i in range(replica): - colname = _template.format(self.input_col, i) - op_name = _template.format(self.input_op_name, i) - sig_name = _template.format(self.input_sig_name, i) - tnsr_name = tfx.tensor_name(op_name) - self.input_mapping[colname] = tnsr_name - self.feed_names.append(tnsr_name) - self.sig_input_mapping[colname] = sig_name - - colname = _template.format(self.output_col, i) - op_name = _template.format(self.output_op_name, i) - sig_name = _template.format(self.output_sig_name, i) - tnsr_name = tfx.tensor_name(op_name) - self.output_mapping[tnsr_name] = colname - self.fetch_names.append(tnsr_name) - self.sig_output_mapping[sig_name] = colname - else: - self.input_mapping = {self.input_col: tfx.tensor_name(self.input_op_name)} - self.sig_input_mapping = {self.input_col: self.input_sig_name} - self.feed_names = [tfx.tensor_name(self.input_op_name)] - self.output_mapping = {tfx.tensor_name(self.output_op_name): self.output_col} - self.sig_output_mapping = {self.output_sig_name: self.output_col} - self.fetch_names = [tfx.tensor_name(self.output_op_name)] - - @contextmanager - def prep_tf_session(self, io_replica=1): - """ Create a session to let build testing graphs - - Downstream classes could also choose to override this function to - build custom testing behaviors - """ - # Reset states - # In each `prep_tf_session`, the implementation is expected to define ONE graph and - # pass all test cases derived from it. We execute the graph and compare the result - # with each test case, and return the numerical results. - self.reset_iomap(replica=io_replica) - self.input_graphs = [] - self.input_graph_with_signature.clear() - - # Build the TensorFlow graph - graph = tf.Graph() - with tf.Session(graph=graph) as sess, graph.as_default(): - # Build test graph and transformers from here - # Notice that any TensorFlow graph could potentially be constructed in this session. - # The graph might contain variables only usable in this session. - yield sess - - # Find the input/output tensors. We expect them to use canonical names. - ref_feed = tfx.get_tensor(self.input_op_name, graph) - ref_fetch = tfx.get_tensor(self.output_op_name, graph) - - # Build test data and reference results - test_data = np.random.randn(self.test_batch_size, self.vec_size) - ref_out = sess.run(ref_fetch, feed_dict={ref_feed: test_data}) - - for gin_info in self.input_graphs: - graph_def = gin_info.gin.graph_def - description = gin_info.description - input_op_name = self.input_op_name - output_op_name = self.output_op_name - - def gen_input_graph_test_case(): - graph = tf.Graph() - with tf.Session(graph=graph) as sess: - namespace = 'TEST_TF_INPUT_GRAPH' - tf.import_graph_def(graph_def, name=namespace) - tgt_feed = tfx.get_tensor('{}/{}'.format(namespace, input_op_name), graph) - tgt_fetch = tfx.get_tensor('{}/{}'.format(namespace, output_op_name), graph) - # Run on the testing target - tgt_out = sess.run(tgt_fetch, feed_dict={tgt_feed: test_data}) - - # Uncomment to check if test cases work in parallel - # if np.random(1) < 0.3: - # raise RuntimeError('randomly killing tests') - - max_diff = np.max(np.abs(ref_out - tgt_out)) - err_msg = '{}: max abs diff {}'.format(description, max_diff) - return TestCase(bool_result=np.allclose(ref_out, tgt_out), err_msg=err_msg) - - test_case = TestFn(test_fn=gen_input_graph_test_case, - description=description, metadata={}) - self.test_cases.append(test_case) - - def register(self, gin, description): - self.input_graphs.append(_GinInfo(gin=gin, description=description)) - - def build_input_graphs(self): - raise NotImplementedError("build your graph and test cases here") - - -class GenTestCases(TestGenBase): - """ Define various test graphs - - Please define all the graphs to be built for test cases in this class. - This is useful for other classes to inherent and change the definition of - the actual testing method. That is, by overriding :py:meth:`prep_tf_session`, - the subclass can change the evaluation behavior. - """ - - def build_input_graphs(self): - self.build_from_checkpoint() - self.build_from_graph() - self.build_from_saved_model() - - def build_from_graph(self): - """ Build TFTransformer from tf.Graph """ - with self.prep_tf_session() as sess: - # Begin building graph - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - _ = tf.reduce_mean(x, axis=1, name=self.output_op_name) - # End building graph - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.register(gin=gin, description='from graph with graph') - gin = TFInputGraph.fromGraphDef(sess.graph.as_graph_def(), self.feed_names, - self.fetch_names) - self.register(gin=gin, description='from graph with graph_def') - - def build_from_saved_model(self): - """ Build TFTransformer from saved model """ - # Setup saved model export directory - saved_model_dir = os.path.join(self.make_tempdir(), 'saved_model') - builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - - with self.prep_tf_session() as sess: - # Model definition: begin - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - w = tf.Variable( - tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - # Model definition ends - - sess.run(w.initializer) - - sig_inputs = {self.input_sig_name: tf.saved_model.utils.build_tensor_info(x)} - sig_outputs = {self.output_sig_name: tf.saved_model.utils.build_tensor_info(z)} - - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs=sig_inputs, outputs=sig_outputs) - - builder.add_meta_graph_and_variables( - sess, [self.serving_tag], signature_def_map={self.serving_sigdef_key: serving_sigdef}) - builder.save() - - # Build the transformer from exported serving model - # We are using signatures, thus must provide the keys - gin = TFInputGraph.fromSavedModelWithSignature(saved_model_dir, self.serving_tag, - self.serving_sigdef_key) - self.register(gin=gin, description='saved model with signature') - - _imap = {self.input_sig_name: tfx.tensor_name(x)} - _omap = {self.output_sig_name: tfx.tensor_name(z)} - self._add_signature_tensor_name_test_cases(gin, _imap, _omap) - - # Build the transformer from exported serving model - # We are not using signatures, thus must provide tensor/operation names - gin = TFInputGraph.fromSavedModel(saved_model_dir, self.serving_tag, self.feed_names, - self.fetch_names) - self.register(gin=gin, description='saved model no signature') - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.register(gin=gin, description='saved model with graph') - - def build_from_checkpoint(self): - """ Build TFTransformer from a model checkpoint """ - # Build the TensorFlow graph - model_ckpt_dir = self.make_tempdir() - ckpt_path_prefix = os.path.join(model_ckpt_dir, 'model_ckpt') - - with self.prep_tf_session() as sess: - x = tf.placeholder(tf.float64, shape=[None, self.vec_size], name=self.input_op_name) - #x = tf.placeholder(tf.float64, shape=[None, vec_size], name=input_col) - w = tf.Variable( - tf.random_normal([self.vec_size], dtype=tf.float64), dtype=tf.float64, name='varW') - z = tf.reduce_mean(x * w, axis=1, name=self.output_op_name) - sess.run(w.initializer) - saver = tf.train.Saver(var_list=[w]) - _ = saver.save(sess, ckpt_path_prefix, global_step=2702) - - # Prepare the signature_def - serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( - inputs={self.input_sig_name: tf.saved_model.utils.build_tensor_info(x)}, - outputs={self.output_sig_name: tf.saved_model.utils.build_tensor_info(z)}) - - # A rather contrived way to add signature def to a meta_graph - meta_graph_def = tf.train.export_meta_graph() - - # Find the meta_graph file (there should be only one) - _ckpt_meta_fpaths = glob('{}/*.meta'.format(model_ckpt_dir)) - assert len(_ckpt_meta_fpaths) == 1, \ - 'expected only one meta graph, but got {}'.format(','.join(_ckpt_meta_fpaths)) - ckpt_meta_fpath = _ckpt_meta_fpaths[0] - - # Add signature_def to the meta_graph and serialize it - # This will overwrite the existing meta_graph_def file - meta_graph_def.signature_def[self.serving_sigdef_key].CopyFrom(serving_sigdef) - with open(ckpt_meta_fpath, mode='wb') as fout: - fout.write(meta_graph_def.SerializeToString()) - - # Build the transformer from exported serving model - # We are using signaures, thus must provide the keys - gin = TFInputGraph.fromCheckpointWithSignature(model_ckpt_dir, self.serving_sigdef_key) - self.register(gin=gin, description='checkpoint with signature') - - _imap = {self.input_sig_name: tfx.tensor_name(x)} - _omap = {self.output_sig_name: tfx.tensor_name(z)} - self._add_signature_tensor_name_test_cases(gin, _imap, _omap) - - # Transformer without using signature_def - gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names) - self.register(gin=gin, description='checkpoint no signature') - - gin = TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names) - self.register(gin=gin, description='checkpoing, graph') - - def _add_signature_tensor_name_test_cases(self, gin, input_sig2tnsr, output_sig2tnsr): - """ Add tests for checking signature to tensor names mapping """ - imap_tgt = gin.input_tensor_name_from_signature - description = 'test input signature to tensor name mapping' - - def check_imap(): - err_msg = '{}: {} != {}'.format(description, imap_tgt, input_sig2tnsr) - bool_result = input_sig2tnsr == imap_tgt - return TestCase(bool_result=bool_result, err_msg=err_msg) - - self.test_cases.append(TestFn(test_fn=check_imap, description=description, metadata={})) - - omap_tgt = gin.output_tensor_name_from_signature - description = 'test output signature to tensor name mapping' - - def check_omap(): - err_msg = '{}: {} != {}'.format(description, omap_tgt, output_sig2tnsr) - bool_result = output_sig2tnsr == omap_tgt - return TestCase(bool_result=bool_result, err_msg=err_msg) - - self.input_graph_with_signature.add(gin) - self.test_cases.append(TestFn(test_fn=check_omap, description=description, metadata={})) diff --git a/python/tests/graph/test_input_graph.py b/python/tests/graph/test_input_graph.py deleted file mode 100644 index c0e14253..00000000 --- a/python/tests/graph/test_input_graph.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2017 Databricks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from __future__ import absolute_import, division, print_function - -from collections import namedtuple -import itertools - -import numpy as np -# Use this to create parameterized test cases -from parameterized import parameterized - -from ..tests import PythonUnitTestCase -from .base_test_generators import GenTestCases - -#======================================================================== -# Don't have to modify the content below - -_TEST_CASES_GENERATORS = [] - - -def _REGISTER_(obj): - _TEST_CASES_GENERATORS.append(obj) - - -#======================================================================== -# Register all test objects here -_REGISTER_(GenTestCases(vec_size=23, test_batch_size=71)) -_REGISTER_(GenTestCases(vec_size=13, test_batch_size=23)) -_REGISTER_(GenTestCases(vec_size=5, test_batch_size=17)) -#======================================================================== - -_ALL_TEST_CASES = [] -_CLEAN_UP_TASKS = [] - -for obj in _TEST_CASES_GENERATORS: - obj.build_input_graphs() - _ALL_TEST_CASES += obj.test_cases - _CLEAN_UP_TASKS.append(obj.tear_down_env) - - -class TFInputGraphTest(PythonUnitTestCase): - @classmethod - def tearDownClass(cls): - for clean_fn in _CLEAN_UP_TASKS: - clean_fn() - - @parameterized.expand(_ALL_TEST_CASES) - def test_tf_input_graph(self, test_fn, description, metadata): # pylint: disable=unused-argument - """ Test build TFInputGraph from various sources """ - bool_result, err_msg = test_fn() - self.assertTrue(bool_result, msg=err_msg) From decdc8f539c11f368ed6ce21de80986aa01ef596 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 21 Nov 2017 17:48:31 -0800 Subject: [PATCH 79/80] PR comments --- python/sparkdl/graph/input.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/python/sparkdl/graph/input.py b/python/sparkdl/graph/input.py index 6779bbeb..04e009ec 100644 --- a/python/sparkdl/graph/input.py +++ b/python/sparkdl/graph/input.py @@ -23,7 +23,6 @@ # pylint: disable=invalid-name,wrong-spelling-in-comment,wrong-spelling-in-docstring - class TFInputGraph(object): """ An opaque object containing TensorFlow graph. @@ -39,7 +38,7 @@ class TFInputGraph(object): - :py:meth:`fromSavedModelWithSignature` - When the graph contains serving signatures in which a set of well-known names are associtated + When the graph contains serving signatures in which a set of well-known names are associated with their corresponding raw tensor names in the graph, we extract and store them here. For example, the TensorFlow saved model may contain the following structure, so that end users can retrieve the the input tensor via `well_known_input_sig` and @@ -131,9 +130,8 @@ def fromGraphDef(cls, graph_def, feed_names, fetch_names): graph = tf.Graph() with tf.Session(graph=graph) as sess: tf.import_graph_def(graph_def, name='') - gin = _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, - fetch_names=fetch_names) - return gin + return _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) @classmethod def fromCheckpoint(cls, checkpoint_dir, feed_names, fetch_names): @@ -234,12 +232,10 @@ def _from_checkpoint_impl(checkpoint_dir, signature_def_key, feed_names, fetch_n if signature_def_key is not None: sig_def = meta_graph_def.signature_def[signature_def_key] - gin = _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) + return _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) else: - gin = _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, - fetch_names=fetch_names) - return gin - + return _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key, feed_names, fetch_names): """ @@ -265,11 +261,10 @@ def _from_saved_model_impl(saved_model_dir, tag_set, signature_def_key, feed_nam if signature_def_key is not None: sig_def = tf.contrib.saved_model.get_signature_def_by_key(meta_graph_def, signature_def_key) - gin = _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) + return _build_with_sig_def(sess=sess, graph=graph, sig_def=sig_def) else: - gin = _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, - fetch_names=fetch_names) - return gin + return _build_with_feeds_fetches(sess=sess, graph=graph, feed_names=feed_names, + fetch_names=fetch_names) def _build_with_sig_def(sess, graph, sig_def): @@ -284,7 +279,6 @@ def _build_with_sig_def(sess, graph, sig_def): feed_mapping[sigdef_key] = tnsr_name feed_names.append(tnsr_name) - # TODO: IN-THIS-PR, test if these mappings are constructed correctly. fetch_mapping = {} fetch_names = [] for sigdef_key, tnsr_info in sig_def.outputs.items(): From af95b7444fd7ea6b46804a8818dc0d715468b077 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 21 Nov 2017 20:31:16 -0800 Subject: [PATCH 80/80] PR comments --- python/sparkdl/transformers/tf_tensor.py | 10 +++-- python/tests/graph/test_import.py | 40 +++++++++++++++++-- .../tests/transformers/tf_transformer_test.py | 1 + 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/python/sparkdl/transformers/tf_tensor.py b/python/sparkdl/transformers/tf_tensor.py index 1edf804d..7207f5f1 100644 --- a/python/sparkdl/transformers/tf_tensor.py +++ b/python/sparkdl/transformers/tf_tensor.py @@ -22,9 +22,8 @@ from pyspark.ml import Transformer import sparkdl.graph.utils as tfx -from sparkdl.graph.input import TFInputGraph -from sparkdl.param import (keyword_only, SparkDLTypeConverters, HasInputMapping, - HasOutputMapping, HasTFInputGraph, HasTFHParams) +from sparkdl.param import (keyword_only, HasInputMapping, HasOutputMapping, + HasTFInputGraph, HasTFHParams) __all__ = ['TFTransformer'] @@ -37,9 +36,10 @@ class TFTransformer(Transformer, HasTFInputGraph, HasTFHParams, HasInputMapping, Restrictions of the current API: We assume that - - All graphs have a "minibatch" dimension (i.e. an unknown leading + - All the inputs of the graphs have a "minibatch" dimension (i.e. an unknown leading dimension) in the tensor shapes. - Input DataFrame has an array column where all elements have the same length + - The transformer is expected to work on blocks of data at the same time. """ @keyword_only @@ -73,6 +73,8 @@ def _optimize_for_inference(self): opt_gdef = infr_opt.optimize_for_inference(gin.graph_def, input_node_names, output_node_names, + # TODO: below is the place to change for + # the `float64` data type issue. tf.float64.as_datatype_enum) return opt_gdef diff --git a/python/tests/graph/test_import.py b/python/tests/graph/test_import.py index a4f36244..36501568 100644 --- a/python/tests/graph/test_import.py +++ b/python/tests/graph/test_import.py @@ -53,6 +53,34 @@ def gin_fun(session): gin = _build_graph_input(gin_fun) _check_input_novar(gin) + def test_saved_model_iomap(self): + with _make_temp_directory() as tmp_dir: + saved_model_dir = os.path.join(tmp_dir, 'saved_model') + graph = tf.Graph() + with tf.Session(graph=graph) as sess, graph.as_default(): + _build_graph() + _build_saved_model(sess, saved_model_dir) + # Build the transformer from exported serving model + # We are using signatures, thus must provide the keys + gin = TFInputGraph.fromSavedModelWithSignature(saved_model_dir, _serving_tag, + _serving_sigdef_key) + + _input_mapping_with_sigdef = {'inputCol': _tensor_input_signature} + # Input mapping for the Transformer + _translated_input_mapping = gin.translateInputMapping(_input_mapping_with_sigdef) + _expected_input_mapping = {'inputCol': tfx.tensor_name(_tensor_input_name)} + # Output mapping for the Transformer + _output_mapping_with_sigdef = {_tensor_output_signature: 'outputCol'} + _translated_output_mapping = gin.translateOutputMapping(_output_mapping_with_sigdef) + _expected_output_mapping = {tfx.tensor_name(_tensor_output_name): 'outputCol'} + + err_msg = "signature based input mapping {} and output mapping {} " + \ + "must be translated correctly into tensor name based mappings" + assert _translated_input_mapping == _expected_input_mapping \ + and _translated_output_mapping == _expected_output_mapping, \ + err_msg.format(_translated_input_mapping, _translated_output_mapping) + + def test_saved_graph_novar(self): with _make_temp_directory() as tmp_dir: saved_model_dir = os.path.join(tmp_dir, 'saved_model') @@ -118,6 +146,10 @@ def gin_fun(session): _tensor_input_name_2 = "input_tensor_2" # The name of the output tensor (scalar) _tensor_output_name = "output_tensor" +# Input signature name +_tensor_input_signature = 'well_known_input_sig' +# Output signature name +_tensor_output_signature = 'well_known_output_sig' # The name of the variable _tensor_var_name = "variable" # The size of the input tensor @@ -135,8 +167,8 @@ def _build_checkpointed_model(session, tmp_dir): w = tfx.get_tensor(_tensor_var_name, session.graph) saver = tf.train.Saver(var_list=[w]) _ = saver.save(session, ckpt_path_prefix, global_step=2702) - sig_inputs = {'input_sig': tf.saved_model.utils.build_tensor_info(input_tensor)} - sig_outputs = {'output_sig': tf.saved_model.utils.build_tensor_info(output_tensor)} + sig_inputs = {_tensor_input_signature: tf.saved_model.utils.build_tensor_info(input_tensor)} + sig_outputs = {_tensor_output_signature: tf.saved_model.utils.build_tensor_info(output_tensor)} serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( inputs=sig_inputs, outputs=sig_outputs) @@ -163,8 +195,8 @@ def _build_saved_model(session, saved_model_dir): builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) input_tensor = tfx.get_tensor(_tensor_input_name, session.graph) output_tensor = tfx.get_tensor(_tensor_output_name, session.graph) - sig_inputs = {'input_sig': tf.saved_model.utils.build_tensor_info(input_tensor)} - sig_outputs = {'output_sig': tf.saved_model.utils.build_tensor_info(output_tensor)} + sig_inputs = {_tensor_input_signature: tf.saved_model.utils.build_tensor_info(input_tensor)} + sig_outputs = {_tensor_output_signature: tf.saved_model.utils.build_tensor_info(output_tensor)} serving_sigdef = tf.saved_model.signature_def_utils.build_signature_def( inputs=sig_inputs, outputs=sig_outputs) diff --git a/python/tests/transformers/tf_transformer_test.py b/python/tests/transformers/tf_transformer_test.py index 60ced31c..849a84d7 100644 --- a/python/tests/transformers/tf_transformer_test.py +++ b/python/tests/transformers/tf_transformer_test.py @@ -85,6 +85,7 @@ def _build_local_features(): """ # Build local features and DataFrame from it local_features = [] + np.random.seed(997) for idx in range(100): _dict = {'idx': idx} for colname, _ in _input_mapping.items():