Lambdamart implementation

.gitignore

@@ -109,3 +109,4 @@ copy_to_remote.sh
 copy_from_remote.sh
 
 *.sh
+/.pytest_cache/

.travis.yml

@@ -17,8 +17,14 @@ install:
   - pip install -I git+https://github.com/Syntaf/travis-sphinx.git
   - pip install coveralls nbsphinx sphinx_rtd_theme
 
+env:
+  - TESTCMD="--cov-append csrank/tests/test_choice_functions.py"
+  - TESTCMD="--cov-append csrank/tests/test_discrete_choice.py"
+  - TESTCMD="--cov-append csrank/tests/test_ranking.py"
+  - TESTCMD="--cov-append csrank/tests/test_fate.py csrank/tests/test_losses.py csrank/tests/test_metrics.py csrank/tests/test_tuning.py csrank/tests/test_util.py"
+
 script:
-  - pytest -v --cov=csrank --ignore experiments
+  - pytest -v --cov=csrank --ignore experiments $TESTCMD
   - travis-sphinx build -n --source=docs
 
 after_success:

csrank/__init__.py

@@ -1,4 +1,4 @@
-from .choicefunctions import *
+from .choicefunction import *
 from .core import *
 from .dataset_reader import *
 from .discretechoice import *

csrank/choicefunctions/choice_functions.py → csrank/choicefunction/choice_functions.py

@@ -47,7 +47,7 @@ def predict_for_scores(self, scores, **kwargs):
             result = np.array(result, dtype=int)
         return result
 
-    def _tune_threshold(self, X_val, Y_val, thin_thresholds=1):
+    def _tune_threshold(self, X_val, Y_val, thin_thresholds=1, verbose=0):
         scores = self.predict_scores(X_val)
         probabilities = np.unique(scores)[::thin_thresholds]
         threshold = 0.0
@@ -59,7 +59,8 @@ def _tune_threshold(self, X_val, Y_val, thin_thresholds=1):
                 if f1 > best:
                     threshold = p
                     best = f1
-                progress_bar(i, len(probabilities), status='Tuning threshold')
+                if verbose == 1:
+                    progress_bar(i, len(probabilities), status='Tuning threshold')
         except KeyboardInterrupt:
             self.logger.info("Keyboard interrupted")
         self.logger.info('Tuned threshold, obtained {:.2f} which achieved'

csrank/choicefunctions/cmpnet_choice.py → csrank/choicefunction/cmpnet_choice.py

@@ -4,8 +4,8 @@
 from keras.regularizers import l2
 from sklearn.model_selection import train_test_split
 
-from csrank.choicefunctions.choice_functions import ChoiceFunctions
-from csrank.choicefunctions.util import generate_complete_pairwise_dataset
+from csrank.choicefunction.choice_functions import ChoiceFunctions
+from csrank.choicefunction.util import generate_complete_pairwise_dataset
 from csrank.core.cmpnet_core import CmpNetCore
 
 
@@ -21,17 +21,19 @@ def __init__(self, n_object_features, n_hidden=2, n_units=8, loss_function='bina
             objects and the pairwise predicate is evaluated using them. The outputs of the network for each pair of
             objects :math:`U(x_1,x_2), U(x_2,x_1)` are evaluated.
             :math:`U(x_1,x_2)` is a measure of how favorable it is to choose :math:`x_1` over :math:`x_2`.
-            The utility score of object :math:`x_i` in query set :math:`Q = \{ x_1 , \ldots , x_n \}` is evaluated as:
+            The utility score of object :math:`x_i` in query set
+            :math:`Q = \\{ x_1 , \\ldots , x_n \\}` is evaluated as:
 
             .. math::
 
-                U(x_i) = \left\{ \\frac{1}{n-1} \sum_{j \in [n] \setminus \{i\}} U_1(x_i , x_j)\\right\}
+                U(x_i) = \\left\\{ \\frac{1}{n-1} \\sum_{j \\in [n]
+                \\setminus \\{i\\}} U_1(x_i , x_j)\\right\\}
 
             The choice set is defined as:
 
             .. math::
 
-                c(Q) = \{ x_i \in Q \lvert \, U(x_i) > t \}
+                c(Q) = \\{ x_i \\in Q \\lvert \\, U(x_i) > t \\}
 
             Parameters
             ----------
@@ -94,15 +96,15 @@ def fit(self, X, Y, epochs=10, callbacks=None, validation_split=0.1, tune_size=0
         """
             Fit a CmptNet model for learning a choice fucntion on the provided set of queries X and preferences Y of
             those objects. The provided queries and corresponding preferences are of a fixed size (numpy arrays). For
-            learning this network the binary cross entropy loss function for a pair of objects :math:`x_i, x_j \in Q`
+            learning this network the binary cross entropy loss function for a pair of objects :math:`x_i, x_j \\in Q`
             is defined as:
 
             .. math::
 
-                C_{ij} =  -\\tilde{P_{ij}}(0)\\cdot \log(U(x_i,x_j)) - \\tilde{P_{ij}}(1) \\cdot \log(U(x_j,x_i)) \ ,
+                C_{ij} =  -\\tilde{P_{ij}}(0)\\cdot \\log(U(x_i,x_j)) - \\tilde{P_{ij}}(1) \\cdot \\log(U(x_j,x_i)) \\ ,
 
             where :math:`\\tilde{P_{ij}}` is ground truth probability of the preference of :math:`x_i` over :math:`x_j`.
-            :math:`\\tilde{P_{ij}} = (1,0)` if :math:`x_i \succ x_j` else :math:`\\tilde{P_{ij}} = (0,1)`.
+            :math:`\\tilde{P_{ij}} = (1,0)` if :math:`x_i \\succ x_j` else :math:`\\tilde{P_{ij}} = (0,1)`.
 
             Parameters
             ----------

csrank/choicefunctions/fate_choice.py → csrank/choicefunction/fate_choice.py

@@ -27,13 +27,13 @@ def __init__(self, n_object_features, n_hidden_set_layers=2, n_hidden_set_units=
             .. math::
                 \\mu_{C(x)} = \\frac{1}{\\lvert C(x) \\lvert} \\sum_{y \\in C(x)} \\phi(y)
 
-            where :math:`\phi \colon \mathcal{X} \\to \mathcal{Z}` maps each object :math:`y` to an
-            :math:`m`-dimensional embedding space :math:`\mathcal{Z} \subseteq \mathbb{R}^m`.
+            where :math:`\\phi \\colon \\mathcal{X} \\to \\mathcal{Z}` maps each object :math:`y` to an
+            :math:`m`-dimensional embedding space :math:`\\mathcal{Z} \\subseteq \\mathbb{R}^m`.
             The choice set is defined as:
 
             .. math::
 
-                c(Q) = \{ x \in Q \lvert \, U (x, \\mu_{C(x)}) > t \}
+                c(Q) = \\{ x \\in Q \\lvert \\, U (x, \\mu_{C(x)}) > t \\}
 
 
             Parameters
@@ -153,7 +153,7 @@ def fit(self, X, Y, epochs=35, inner_epochs=1, callbacks=None, validation_split=
                 super().fit(X_train, Y_train, **kwargs)
             finally:
                 self.logger.info('Fitting utility function finished. Start tuning threshold.')
-                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds)
+                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds, verbose=verbose)
         else:
             super().fit(X, Y, **kwargs)
             self.threshold = 0.5

csrank/choicefunctions/fatelinear_choice.py → csrank/choicefunction/fatelinear_choice.py

@@ -21,14 +21,14 @@ def __init__(self, n_object_features, n_objects, n_hidden_set_units=2, loss_func
             .. math::
                 \\mu_{C(x)} = \\frac{1}{\\lvert C(x) \\lvert} \\sum_{y \\in C(x)} \\phi(y)
 
-            where :math:`\phi \colon \mathcal{X} \\to \mathcal{Z}` maps each object :math:`y` to an
-            :math:`m`-dimensional embedding space :math:`\mathcal{Z} \subseteq \mathbb{R}^m`.
+            where :math:`\\phi \\colon \\mathcal{X} \\to \\mathcal{Z}` maps each object :math:`y` to an
+            :math:`m`-dimensional embedding space :math:`\\mathcal{Z} \\subseteq \\mathbb{R}^m`.
             Training complexity is quadratic in the number of objects and prediction complexity is only linear.
             The discrete choice for the given query set :math:`Q` is defined as:
 
             .. math::
 
-                dc(Q) := \operatorname{argmax}_{x \in Q}  \;  U (x, \\mu_{C(x)})
+                dc(Q) := \\operatorname{argmax}_{x \\in Q}  \\;  U (x, \\mu_{C(x)})
 
             Parameters
             ----------
@@ -60,7 +60,7 @@ def fit(self, X, Y, epochs=10, callbacks=None, validation_split=0.1, tune_size=0
                 super().fit(X_train, Y_train, epochs, callbacks, validation_split, verbose, **kwd)
             finally:
                 self.logger.info('Fitting utility function finished. Start tuning threshold.')
-                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds)
+                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds, verbose=verbose)
         else:
             super().fit(X, Y, epochs, callbacks, validation_split, verbose,
                         **kwd)

csrank/choicefunctions/feta_choice.py → csrank/choicefunction/feta_choice.py

@@ -11,6 +11,7 @@
 
 from csrank.core.feta_network import FETANetwork
 from csrank.layers import NormalizedDense
+from csrank.numpy_util import sigmoid
 from .choice_functions import ChoiceFunctions
 
 
@@ -24,19 +25,19 @@ def __init__(self, n_objects, n_object_features, n_hidden=2, n_units=8, add_zero
         """
             Create a FETA-network architecture for learning choice functions.
             The first-evaluate-then-aggregate approach approximates the context-dependent utility function using the
-            first-order utility function :math:`U_1 \colon \mathcal{X} \\times \mathcal{X} \\rightarrow [0,1]`
-            and zeroth-order utility function  :math:`U_0 \colon \mathcal{X} \\rightarrow [0,1]`.
+            first-order utility function :math:`U_1 \\colon \\mathcal{X} \\times \\mathcal{X} \\rightarrow [0,1]`
+            and zeroth-order utility function  :math:`U_0 \\colon \\mathcal{X} \\rightarrow [0,1]`.
             The scores each object :math:`x` using a context-dependent utility function :math:`U (x, C_i)`:
 
             .. math::
-                 U(x_i, C_i) = U_0(x_i) + \\frac{1}{n-1} \sum_{x_j \in Q \\setminus \{x_i\}} U_1(x_i , x_j) \, .
+                 U(x_i, C_i) = U_0(x_i) + \\frac{1}{n-1} \\sum_{x_j \\in Q \\setminus \\{x_i\\}} U_1(x_i , x_j) \\, .
 
             Training and prediction complexity is quadratic in the number of objects.
             The choice set is defined as:
 
             .. math::
 
-                c(Q) = \{ x_i \in Q \lvert \, U (x_i, C_i) > t \}
+                c(Q) = \\{ x_i \\in Q \\lvert \\, U (x_i, C_i) > t \\}
 
             Parameters
             ----------
@@ -109,11 +110,11 @@ def construct_model(self):
         """
             Construct the :math:`1`-st order and :math:`0`-th order models, which are used to approximate the
             :math:`U_1(x, C(x))` and the :math:`U_0(x)` utilities respectively. For each pair of objects in
-            :math:`x_i, x_j \in Q` :math:`U_1(x, C(x))` we construct :class:`CmpNetCore` with weight sharing to
+            :math:`x_i, x_j \\in Q` :math:`U_1(x, C(x))` we construct :class:`CmpNetCore` with weight sharing to
             approximate a pairwise-matrix. A pairwise matrix with index (i,j) corresponds to the :math:`U_1(x_i,x_j)`
             is a measure of how favorable it is to choose :math:`x_i` over :math:`x_j`. Using this matrix we calculate
             the borda score for each object to calculate :math:`U_1(x, C(x))`. For `0`-th order model we construct
-            :math:`\lvert Q \lvert` sequential networks whose weights are shared to evaluate the :math:`U_0(x)` for
+            :math:`\\lvert Q \\lvert` sequential networks whose weights are shared to evaluate the :math:`U_0(x)` for
             each object in the query set :math:`Q`. The output mode is using sigmoid activation.
 
             Returns
@@ -176,6 +177,11 @@ def create_input_lambda(i):
         model.compile(loss=self.loss_function, optimizer=self.optimizer, metrics=self.metrics)
         return model
 
+    def _predict_scores_using_pairs(self, X, **kwd):
+        scores = super()._predict_scores_using_pairs(X=X, **kwd)
+        scores = sigmoid(scores)
+        return scores
+
     def fit(self, X, Y, epochs=10, callbacks=None, validation_split=0.1, tune_size=0.1, thin_thresholds=1, verbose=0,
             **kwd):
         """
@@ -210,7 +216,7 @@ def fit(self, X, Y, epochs=10, callbacks=None, validation_split=0.1, tune_size=0
                             validation_split, verbose, **kwd)
             finally:
                 self.logger.info('Fitting utility function finished. Start tuning threshold.')
-                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds)
+                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds, verbose=verbose)
         else:
             super().fit(X, Y, epochs, callbacks, validation_split, verbose,
                         **kwd)

csrank/choicefunctions/fetalinear_choice.py → csrank/choicefunction/fetalinear_choice.py

@@ -21,14 +21,14 @@ def __init__(self, n_object_features, n_objects, loss_function=binary_crossentro
             .. math::
                 \\mu_{C(x)} = \\frac{1}{\\lvert C(x) \\lvert} \\sum_{y \\in C(x)} \\phi(y)
 
-            where :math:`\phi \colon \mathcal{X} \\to \mathcal{Z}` maps each object :math:`y` to an
-            :math:`m`-dimensional embedding space :math:`\mathcal{Z} \subseteq \mathbb{R}^m`.
+            where :math:`\\phi \\colon \\mathcal{X} \\to \\mathcal{Z}` maps each object :math:`y` to an
+            :math:`m`-dimensional embedding space :math:`\\mathcal{Z} \\subseteq \\mathbb{R}^m`.
             Training complexity is quadratic in the number of objects and prediction complexity is only linear.
             The discrete choice for the given query set :math:`Q` is defined as:
 
             .. math::
 
-                dc(Q) := \operatorname{argmax}_{x \in Q}  \;  U (x, \\mu_{C(x)})
+                dc(Q) := \\operatorname{argmax}_{x \\in Q}  \\;  U (x, \\mu_{C(x)})
 
             Parameters
             ----------
@@ -60,7 +60,7 @@ def fit(self, X, Y, epochs=10, callbacks=None, validation_split=0.1, tune_size=0
                 super().fit(X_train, Y_train, epochs, callbacks, validation_split, verbose, **kwd)
             finally:
                 self.logger.info('Fitting utility function finished. Start tuning threshold.')
-                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds)
+                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds, verbose=verbose)
         else:
             super().fit(X, Y, epochs, callbacks, validation_split, verbose,
                         **kwd)

csrank/choicefunctions/generalized_linear_model.py → csrank/choicefunction/generalized_linear_model.py

@@ -9,7 +9,7 @@
 from sklearn.utils import check_random_state
 
 import csrank.theano_util as ttu
-from csrank.choicefunctions.util import create_weight_dictionary, BinaryCrossEntropyLikelihood
+from csrank.choicefunction.util import create_weight_dictionary, BinaryCrossEntropyLikelihood
 from csrank.discretechoice.likelihoods import fit_pymc3_model
 from csrank.learner import Learner
 from csrank.util import print_dictionary
@@ -21,7 +21,7 @@ def __init__(self, n_object_features, regularization='l2', random_state=None, **
         """
             Create an instance of the GeneralizedLinearModel model for learning the choice function. This model is
             adapted from the multinomial logit model :class:`csrank.discretechoice.multinomial_logit_model.MultinomialLogitModel`.
-            The utility score for each object in query set :math:`Q` is defined as :math:`U(x) = w \cdot x`,
+            The utility score for each object in query set :math:`Q` is defined as :math:`U(x) = w \\cdot x`,
             where :math:`w` is the weight vector. The probability of choosing an object :math:`x_i` is defined by taking
             sigmoid over the utility scores:
 
@@ -33,7 +33,7 @@ def __init__(self, n_object_features, regularization='l2', random_state=None, **
 
             .. math::
 
-                c(Q) = \{ x_i \in Q \lvert \, P(x_i \\lvert Q) > t \}
+                c(Q) = \\{ x_i \\in Q \\lvert \\, P(x_i \\lvert Q) > t \\}
 
             Parameters
             ----------
@@ -78,17 +78,17 @@ def model_configuration(self):
 
             .. math::
 
-                \\text{mu}_w \sim \\text{Normal}(\\text{mu}=0, \\text{sd}=5.0) \\\\
-                \\text{b}_w \sim \\text{HalfCauchy}(\\beta=1.0) \\\\
-                \\text{weights} \sim \\text{Laplace}(\\text{mu}=\\text{mu}_w, \\text{b}=\\text{b}_w)
+                \\text{mu}_w \\sim \\text{Normal}(\\text{mu}=0, \\text{sd}=5.0) \\\\
+                \\text{b}_w \\sim \\text{HalfCauchy}(\\beta=1.0) \\\\
+                \\text{weights} \\sim \\text{Laplace}(\\text{mu}=\\text{mu}_w, \\text{b}=\\text{b}_w)
 
             For ``l2`` regularization the priors are:
 
             .. math::
 
-                \\text{mu}_w \sim \\text{Normal}(\\text{mu}=0, \\text{sd}=5.0) \\\\
-                \\text{sd}_w \sim \\text{HalfCauchy}(\\beta=1.0) \\\\
-                \\text{weights} \sim \\text{Normal}(\\text{mu}=\\text{mu}_w, \\text{sd}=\\text{sd}_w)
+                \\text{mu}_w \\sim \\text{Normal}(\\text{mu}=0, \\text{sd}=5.0) \\\\
+                \\text{sd}_w \\sim \\text{HalfCauchy}(\\beta=1.0) \\\\
+                \\text{weights} \\sim \\text{Normal}(\\text{mu}=\\text{mu}_w, \\text{sd}=\\text{sd}_w)
         """
         if self.regularization == 'l2':
             weight = pm.Normal
@@ -103,9 +103,9 @@ def model_configuration(self):
 
     def construct_model(self, X, Y):
         """
-            Constructs the linear logit model which evaluated the utility score as :math:`U(x) = w \cdot x`, where
+            Constructs the linear logit model which evaluated the utility score as :math:`U(x) = w \\cdot x`, where
             :math:`w` is the weight vector. The probability of choosing the object :math:`x_i` from the query set
-            :math:`Q = \{x_1, \ldots ,x_n\}` is:
+            :math:`Q = \\{x_1, \\ldots ,x_n\\}` is:
 
             .. math::
 
@@ -138,15 +138,15 @@ def construct_model(self, X, Y):
         self.logger.info("Model construction completed")
 
     def fit(self, X, Y, sampler='variational', tune=500, draws=500, tune_size=0.1, thin_thresholds=1,
-            vi_params={"n": 20000, "method": "advi", "callbacks": [CheckParametersConvergence()]}, **kwargs):
+            vi_params={"n": 20000, "method": "advi", "callbacks": [CheckParametersConvergence()]}, verbose=0, **kwargs):
         """
             Fit a generalized logit model on the provided set of queries X and choices Y of those objects. The
             provided queries and corresponding preferences are of a fixed size (numpy arrays). For learning this network
-            the binary cross entropy loss function for each object :math:`x_i \in Q` is defined as:
+            the binary cross entropy loss function for each object :math:`x_i \\in Q` is defined as:
 
             .. math::
 
-                C_{i} =  -y(i)\log(P_i) - (1 - y(i))\log(1 - P_i) \enspace,
+                C_{i} =  -y(i)\\log(P_i) - (1 - y(i))\\log(1 - P_i) \\enspace,
 
             where :math:`y` is ground-truth choice vector of the objects in the given query set :math:`Q`.
             The value :math:`y(i) = 1` if object :math:`x_i` is chosen else :math:`y(i) = 0`.
@@ -176,6 +176,8 @@ def fit(self, X, Y, sampler='variational', tune=500, draws=500, tune_size=0.1, t
                 Percentage of instances to split off to tune the threshold for the choice function
             thin_thresholds: int
                 The number of instances of scores to skip while tuning the threshold
+            verbose : bool
+                Print verbose information
             **kwargs :
                 Keyword arguments for the fit function
         """
@@ -185,7 +187,7 @@ def fit(self, X, Y, sampler='variational', tune=500, draws=500, tune_size=0.1, t
                 self._fit(X_train, Y_train, sampler=sampler, vi_params=vi_params, **kwargs)
             finally:
                 self.logger.info('Fitting utility function finished. Start tuning threshold.')
-                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds)
+                self.threshold = self._tune_threshold(X_val, Y_val, thin_thresholds=thin_thresholds, verbose=verbose)
         else:
             self._fit(X, Y, sampler=sampler, sample_params={"tune": 2, "draws": 2, "chains": 4, "njobs": 8},
                       vi_params={"n": 20000, "method": "advi", "callbacks": [