From a5c1038ee6ebb734f6e09d00a617d1e92e02d589 Mon Sep 17 00:00:00 2001 From: Duco Date: Fri, 19 Dec 2014 10:50:07 +0100 Subject: [PATCH 1/6] Add global loss tracker --- glove/glove.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/glove/glove.py b/glove/glove.py index d2340b4..338ce3c 100644 --- a/glove/glove.py +++ b/glove/glove.py @@ -43,6 +43,8 @@ def __init__(self, no_components=30, learning_rate=0.05, self.dictionary = None self.inverse_dictionary = None + + self.global_loss = None def fit(self, matrix, epochs=5, no_threads=2, verbose=False): """ @@ -86,6 +88,8 @@ def fit(self, matrix, epochs=5, no_threads=2, verbose=False): # Shuffle the coocurrence matrix np.random.shuffle(shuffle_indices) + + self.global_loss = np.zeros(1, dtype=np.float64) fit_vectors(self.word_vectors, self.vectors_sum_gradients, @@ -98,7 +102,10 @@ def fit(self, matrix, epochs=5, no_threads=2, verbose=False): self.learning_rate, self.max_count, self.alpha, + self.global_loss, int(no_threads)) + + print('Global loss: %d' % self.global_loss) if not np.isfinite(self.word_vectors).all(): raise Exception('Non-finite values in word vectors. ' From 6bce0da460d998295223f1299f4ef13ee19b014f Mon Sep 17 00:00:00 2001 From: Duco Date: Fri, 19 Dec 2014 10:55:33 +0100 Subject: [PATCH 2/6] Add global loss computation --- glove/glove_cython.pyx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/glove/glove_cython.pyx b/glove/glove_cython.pyx index d67f036..cdf08b9 100644 --- a/glove/glove_cython.pyx +++ b/glove/glove_cython.pyx @@ -29,6 +29,7 @@ def fit_vectors(double[:, ::1] wordvec, double initial_learning_rate, double max_count, double alpha, + double[::1] global_loss, int no_threads): """ Estimate GloVe word embeddings given the cooccurrence matrix. @@ -75,6 +76,9 @@ def fit_vectors(double[:, ::1] wordvec, # Compute loss and the example weight. entry_weight = double_min(1.0, (count / max_count)) ** alpha loss = entry_weight * (prediction - c_log(count)) + + # Compute a weighted global loss + global_loss[0] += 0.5 * entry_weight * (prediction - c_log(count)) **2 # Update step: apply gradients and reproject # onto the unit sphere. From af11285b459e2de321c2071d6f5f979a86b971d5 Mon Sep 17 00:00:00 2001 From: Duco Date: Fri, 19 Dec 2014 17:39:31 +0100 Subject: [PATCH 3/6] Verbose dependency and return fit_vectors fit_vectors returns global loss Global loss printing over epochs only enabled if verbose is True --- glove/glove.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/glove/glove.py b/glove/glove.py index 338ce3c..7d54c95 100644 --- a/glove/glove.py +++ b/glove/glove.py @@ -89,9 +89,7 @@ def fit(self, matrix, epochs=5, no_threads=2, verbose=False): # Shuffle the coocurrence matrix np.random.shuffle(shuffle_indices) - self.global_loss = np.zeros(1, dtype=np.float64) - - fit_vectors(self.word_vectors, + self.global_loss = fit_vectors(self.word_vectors, self.vectors_sum_gradients, self.word_biases, self.biases_sum_gradients, @@ -102,10 +100,10 @@ def fit(self, matrix, epochs=5, no_threads=2, verbose=False): self.learning_rate, self.max_count, self.alpha, - self.global_loss, int(no_threads)) - print('Global loss: %d' % self.global_loss) + if verbose: + print('Global loss: %d' % self.global_loss) if not np.isfinite(self.word_vectors).all(): raise Exception('Non-finite values in word vectors. ' From f6c4f9f120d37337d6e71d2d5a8444c6bd2f91c2 Mon Sep 17 00:00:00 2001 From: Duco Date: Fri, 19 Dec 2014 17:43:59 +0100 Subject: [PATCH 4/6] fit_vectors returns global_loss --- glove/glove_cython.pyx | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/glove/glove_cython.pyx b/glove/glove_cython.pyx index cdf08b9..1b8c0bf 100644 --- a/glove/glove_cython.pyx +++ b/glove/glove_cython.pyx @@ -29,7 +29,6 @@ def fit_vectors(double[:, ::1] wordvec, double initial_learning_rate, double max_count, double alpha, - double[::1] global_loss, int no_threads): """ Estimate GloVe word embeddings given the cooccurrence matrix. @@ -51,6 +50,9 @@ def fit_vectors(double[:, ::1] wordvec, # Loss and gradient variables. cdef double prediction, entry_weight, loss + + # Define global loss + cdef double global_loss # Iteration variables cdef int i, j, shuffle_index @@ -77,8 +79,8 @@ def fit_vectors(double[:, ::1] wordvec, entry_weight = double_min(1.0, (count / max_count)) ** alpha loss = entry_weight * (prediction - c_log(count)) - # Compute a weighted global loss - global_loss[0] += 0.5 * entry_weight * (prediction - c_log(count)) **2 + # Update the weighted global loss + global_loss += 0.5 * entry_weight * (prediction - c_log(count)) **2 # Update step: apply gradients and reproject # onto the unit sphere. @@ -104,7 +106,8 @@ def fit_vectors(double[:, ::1] wordvec, learning_rate = initial_learning_rate / sqrt(wordbias_sum_gradients[word_b]) wordbias[word_b] -= learning_rate * loss wordbias_sum_gradients[word_b] += loss ** 2 - + + return global_loss def transform_paragraph(double[:, ::1] wordvec, double[::1] wordbias, From 0ac14971f9d90ce9aa8a28ec4b11907bc98e2261 Mon Sep 17 00:00:00 2001 From: Duco Date: Fri, 2 Jan 2015 12:39:41 +0100 Subject: [PATCH 5/6] Indentation level in fit_vector function --- glove/glove.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/glove/glove.py b/glove/glove.py index 7d54c95..3c298bf 100644 --- a/glove/glove.py +++ b/glove/glove.py @@ -90,17 +90,17 @@ def fit(self, matrix, epochs=5, no_threads=2, verbose=False): np.random.shuffle(shuffle_indices) self.global_loss = fit_vectors(self.word_vectors, - self.vectors_sum_gradients, - self.word_biases, - self.biases_sum_gradients, - matrix.row, - matrix.col, - matrix.data, - shuffle_indices, - self.learning_rate, - self.max_count, - self.alpha, - int(no_threads)) + self.vectors_sum_gradients, + self.word_biases, + self.biases_sum_gradients, + matrix.row, + matrix.col, + matrix.data, + shuffle_indices, + self.learning_rate, + self.max_count, + self.alpha, + int(no_threads)) if verbose: print('Global loss: %d' % self.global_loss) From 654eb4f196c130385296a31bc8290a080ca1a810 Mon Sep 17 00:00:00 2001 From: Duco Date: Fri, 2 Jan 2015 16:15:41 +0100 Subject: [PATCH 6/6] Calculate weightless loss seperately --- glove/glove_cython.pyx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/glove/glove_cython.pyx b/glove/glove_cython.pyx index 1b8c0bf..a5ddc27 100644 --- a/glove/glove_cython.pyx +++ b/glove/glove_cython.pyx @@ -77,10 +77,12 @@ def fit_vectors(double[:, ::1] wordvec, # Compute loss and the example weight. entry_weight = double_min(1.0, (count / max_count)) ** alpha - loss = entry_weight * (prediction - c_log(count)) + + loss_unweighted = prediction - c_log(count) + loss = entry_weight * loss_unweighted # Update the weighted global loss - global_loss += 0.5 * entry_weight * (prediction - c_log(count)) **2 + global_loss += 0.5 * loss * loss_unweighted # Update step: apply gradients and reproject # onto the unit sphere.