metrics

Author: mgraffg

CompStats/__init__.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = '0.1.3'
+__version__ = '0.1.4'
 from CompStats.bootstrap import StatisticSamples
 from CompStats.measurements import CI, SE, difference_p_value
 from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference

CompStats/metrics.py

@@ -104,6 +104,50 @@ def inner(y, hy):
                 **kwargs)
 
 
+@perf_docs
+def top_k_accuracy_score(y_true, *y_pred, k=2,
+                         normalize=True, sample_weight=None,
+                         labels=None,
+                         num_samples: int=500,
+                         n_jobs: int=-1,
+                         use_tqdm=True,
+                         **kwargs):
+    """
+    >>> from sklearn.svm import LinearSVC
+    >>> from sklearn.ensemble import RandomForestClassifier
+    >>> from sklearn.datasets import load_iris
+    >>> from sklearn.model_selection import train_test_split
+    >>> from sklearn.base import clone
+    >>> from CompStats.metrics import top_k_accuracy_score
+    >>> X, y = load_iris(return_X_y=True)
+    >>> _ = train_test_split(X, y, test_size=0.3)
+    >>> X_train, X_val, y_train, y_val = _
+    >>> m = LinearSVC().fit(X_train, y_train)
+    >>> hy = m.decision_function(X_val)
+    >>> score = top_k_accuracy_score(y_val, hy, n_jobs=1,
+                                     labels=[0, 1, 2])
+    >>> score
+    <Perf>
+    Prediction statistics with standard error
+    forest = 0.957 (0.031)
+    alg-1 = 0.935 (0.037)
+    >>> diff = score.difference()
+    >>> diff
+    <Difference>
+    difference p-values w.r.t forest
+    alg-1 0.254  
+    """
+
+    def inner(y, hy):
+        return metrics.top_k_accuracy_score(y, hy, k=k,
+                                            normalize=normalize, sample_weight=sample_weight,
+                                            labels=labels)
+    return Perf(y_true, *y_pred, score_func=inner,
+                num_samples=num_samples, n_jobs=n_jobs,
+                use_tqdm=use_tqdm,
+                **kwargs)
+
+
 @perf_docs
 def f1_score(y_true, *y_pred, labels=None, pos_label=1,
              average='binary', sample_weight=None,

CompStats/tests/test_metrics.py

@@ -53,4 +53,18 @@ def test_balanced_accuracy_score():
     ens = RandomForestClassifier().fit(X_train, y_train)
     perf = balanced_accuracy_score(y_val, forest=ens.predict(X_val),
                                    num_samples=50)
-    assert 'forest' in perf.statistic()        
+    assert 'forest' in perf.statistic()        
+
+
+def test_top_k_accuracy_score():
+    """Test top_k_accuracy_score"""
+    from CompStats.metrics import top_k_accuracy_score
+
+    X, y = load_iris(return_X_y=True)
+    _ = train_test_split(X, y, test_size=0.3, stratify=y)
+    X_train, X_val, y_train, y_val = _
+    ens = RandomForestClassifier().fit(X_train, y_train)
+    perf = top_k_accuracy_score(y_val,
+                                forest=ens.predict_proba(X_val),
+                                num_samples=50)
+    assert 'forest' in perf.statistic()