From 50be6e6b98760baa6f0989a332d6e3cdc76b8afe Mon Sep 17 00:00:00 2001 From: dengkaipeng Date: Thu, 30 Apr 2020 07:22:47 +0000 Subject: [PATCH 1/4] refine doc for metrics --- hapi/metrics.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/hapi/metrics.py b/hapi/metrics.py index 11a174c..38b22f0 100644 --- a/hapi/metrics.py +++ b/hapi/metrics.py @@ -33,10 +33,78 @@ class Metric(object): Base class for metric, encapsulates metric logic and APIs Usage: - m = SomeMetric() - for prediction, label in ...: - m.update(prediction, label) - m.accumulate() + .. code-block:: python + + m = SomeMetric() + for prediction, label in ...: + m.update(prediction, label) + m.accumulate() + + Advanced usage for :code:`add_metric_op` + + Metric calculating con be accelerate by calucateing metric states + from model outputs and labels by Paddle OPs in :code:`add_metric_op`, + metric states will be fetch as numpy array and call :code:`update` + with states in numpy format. + + Metric calculated as follows (operations in Model and Metric are + indicated with curly brackets, while data nodes not): + + inputs & labels || + | || + {model} || + | || + outputs labels || + | || + {Metric.add_metric_op} || + | || + metric states || + | || + {fetch as numpy} || + | || + metric states(numpy) || + | || + {Metric.update} \/ + + Examples: + + For :code:`Accuracy` metric, which takes :code:`pred` and :code:`label` + as inputs, we can calculate the correct prediction matrix between + :code:`pred` and :code:`label` in :code:`add_metric_op`. + + For examples, prediction results contains 10 classes, while :code:`pred` + shape is [N, 10], :code:`label` shape is [N, 1], N is mini-batch size, + and we only need to calculate accurary of top-1 and top-5, we could + calculated the correct prediction matrix of the top-5 scores of the + prediction of each sample like follows, while the correct prediction + matrix shape is [N, 5]. + + .. code-block:: python + + def add_metric_op(pred, label): + # sort prediction and slice the top-5 scores + pred = fluid.layers.argsort(pred, descending=True)[1][:, :5] + # calculate whether the predictions are correct + correct = pred == label + return fluid.layers.cast(correct, dtype='float32') + + With the :code:`add_metric_op`, we split some calculations to OPs(which + may run on GPU devices, will be faster), and only fetch 1 tensor with + shape as [N, 5] instead of 2 tensors with shapes as [N, 10] and [N, 1]. + :code:`update` can be define as follows: + + .. code-block:: python + + def update(self, correct): + accs = [] + for i, k in enumerate(self.topk): + num_corrects = correct[:, :k].sum() + num_samples = len(correct) + accs.append(float(num_corrects) / num_samples) + self.total[i] += num_corrects + self.count[i] += num_samples + return accs + """ @abc.abstractmethod From b225b8d99b02e7f3a9de05824c6ec0f143170150 Mon Sep 17 00:00:00 2001 From: dengkaipeng Date: Thu, 30 Apr 2020 07:27:06 +0000 Subject: [PATCH 2/4] refine doc --- hapi/metrics.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hapi/metrics.py b/hapi/metrics.py index 38b22f0..1031e48 100644 --- a/hapi/metrics.py +++ b/hapi/metrics.py @@ -50,21 +50,21 @@ class Metric(object): Metric calculated as follows (operations in Model and Metric are indicated with curly brackets, while data nodes not): - inputs & labels || + inputs & labels || ------------------ | || {model} || | || outputs labels || - | || + | || tensor data {Metric.add_metric_op} || | || metric states || | || - {fetch as numpy} || + {fetch as numpy} || ------------------ | || - metric states(numpy) || + metric states(numpy) || numpy data | || - {Metric.update} \/ + {Metric.update} \/ ------------------ Examples: From f8204896b4f525c54d98af78e6182aa9e7732203 Mon Sep 17 00:00:00 2001 From: dengkaipeng Date: Thu, 30 Apr 2020 07:29:07 +0000 Subject: [PATCH 3/4] refine doc --- hapi/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hapi/metrics.py b/hapi/metrics.py index 1031e48..5f79b8f 100644 --- a/hapi/metrics.py +++ b/hapi/metrics.py @@ -56,9 +56,9 @@ class Metric(object): | || outputs labels || | || tensor data - {Metric.add_metric_op} || + {Metric.add_metric_op} || | || - metric states || + metric states(tensor) || | || {fetch as numpy} || ------------------ | || From 3b091c877f22859017fd2d7e6ab11c201bb948de Mon Sep 17 00:00:00 2001 From: dengkaipeng Date: Thu, 30 Apr 2020 07:31:59 +0000 Subject: [PATCH 4/4] refine doc --- hapi/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hapi/metrics.py b/hapi/metrics.py index 5f79b8f..056b983 100644 --- a/hapi/metrics.py +++ b/hapi/metrics.py @@ -54,7 +54,7 @@ class Metric(object): | || {model} || | || - outputs labels || + outputs & labels || | || tensor data {Metric.add_metric_op} || | ||