{SLmetrics} Version 2.0-0 🚀

See NEWS.md or PR #26 for details.

.github/.gitignore

@@ -1 +1,2 @@
 *.html
+*.Rproj

.github/workflows/R-CMD-check.yaml

@@ -2,9 +2,9 @@
 # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches: [development, documentation]
+    branches: [development, documentation, weighted-metrics]
   pull_request:
-    branches: [development, documentation]
+    branches: [development, documentation, weighted-metrics]
 
 name: R-CMD-check
 

.github/workflows/test-coverage.yaml

@@ -23,21 +23,28 @@ jobs:
         with:
           use-public-rspm: true
 
+      - name: Install system dependencies (Ubuntu only)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y python3-venv
+
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
           extra-packages: any::covr, any::xml2, any::reticulate
           needs: coverage
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.x"
+          python-version: "3.10.x"
 
       - name: Setup venv with {reticulate}
         shell: Rscript {0}
         run: |
+
           path_to_python <- reticulate::virtualenv_create(
             envname = "r-reticulate",
-            python = Sys.which("python"),
+            python = Sys.which("python"), # placed on PATH by the setup-python action
             packages = c(
               "numpy",
               "scipy",

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: SLmetrics
 Title: Machine Learning Performance Evaluation on Steroids
-Version: 0.1-1
+Version: 0.2-0
 Authors@R: c(
     person(
       given   = "Serkan", 
@@ -34,5 +34,5 @@ Imports:
     Rcpp
 VignetteBuilder: knitr
 Depends: 
-    R (>= 2.10)
+    R (>= 3.5)
 URL: https://serkor1.github.io/SLmetrics/

NAMESPACE

@@ -8,6 +8,7 @@ S3method(baccuracy,factor)
 S3method(ccc,numeric)
 S3method(ckappa,cmatrix)
 S3method(ckappa,factor)
+S3method(cmatrix,factor)
 S3method(csi,cmatrix)
 S3method(csi,factor)
 S3method(dor,cmatrix)
@@ -60,7 +61,7 @@ S3method(recall,cmatrix)
 S3method(recall,factor)
 S3method(rmse,numeric)
 S3method(rmsle,numeric)
-S3method(rrmse,numeric)
+S3method(rrse,numeric)
 S3method(rsq,numeric)
 S3method(selectivity,cmatrix)
 S3method(selectivity,factor)
@@ -78,6 +79,45 @@ S3method(tpr,cmatrix)
 S3method(tpr,factor)
 S3method(tscore,cmatrix)
 S3method(tscore,factor)
+S3method(weighted.accuracy,factor)
+S3method(weighted.baccuracy,factor)
+S3method(weighted.ccc,numeric)
+S3method(weighted.ckappa,factor)
+S3method(weighted.csi,factor)
+S3method(weighted.dor,factor)
+S3method(weighted.fallout,factor)
+S3method(weighted.fbeta,factor)
+S3method(weighted.fdr,factor)
+S3method(weighted.fer,factor)
+S3method(weighted.fpr,factor)
+S3method(weighted.huberloss,numeric)
+S3method(weighted.jaccard,factor)
+S3method(weighted.mae,numeric)
+S3method(weighted.mape,numeric)
+S3method(weighted.mcc,factor)
+S3method(weighted.mpe,numeric)
+S3method(weighted.mse,numeric)
+S3method(weighted.nlr,factor)
+S3method(weighted.npv,factor)
+S3method(weighted.phi,factor)
+S3method(weighted.pinball,numeric)
+S3method(weighted.plr,factor)
+S3method(weighted.ppv,factor)
+S3method(weighted.precision,factor)
+S3method(weighted.rae,numeric)
+S3method(weighted.recall,factor)
+S3method(weighted.rmse,numeric)
+S3method(weighted.rmsle,numeric)
+S3method(weighted.rrse,numeric)
+S3method(weighted.rsq,numeric)
+S3method(weighted.selectivity,factor)
+S3method(weighted.sensitivity,factor)
+S3method(weighted.smape,numeric)
+S3method(weighted.specificity,factor)
+S3method(weighted.tnr,factor)
+S3method(weighted.tpr,factor)
+S3method(weighted.tscore,factor)
+S3method(weighted.zerooneloss,factor)
 S3method(zerooneloss,cmatrix)
 S3method(zerooneloss,factor)
 export(ROC)
@@ -113,7 +153,7 @@ export(rae)
 export(recall)
 export(rmse)
 export(rmsle)
-export(rrmse)
+export(rrse)
 export(rsq)
 export(selectivity)
 export(sensitivity)
@@ -122,6 +162,45 @@ export(specificity)
 export(tnr)
 export(tpr)
 export(tscore)
+export(weighted.accuracy)
+export(weighted.baccuracy)
+export(weighted.ccc)
+export(weighted.ckappa)
+export(weighted.csi)
+export(weighted.dor)
+export(weighted.fallout)
+export(weighted.fbeta)
+export(weighted.fdr)
+export(weighted.fer)
+export(weighted.fpr)
+export(weighted.huberloss)
+export(weighted.jaccard)
+export(weighted.mae)
+export(weighted.mape)
+export(weighted.mcc)
+export(weighted.mpe)
+export(weighted.mse)
+export(weighted.nlr)
+export(weighted.npv)
+export(weighted.phi)
+export(weighted.pinball)
+export(weighted.plr)
+export(weighted.ppv)
+export(weighted.precision)
+export(weighted.rae)
+export(weighted.recall)
+export(weighted.rmse)
+export(weighted.rmsle)
+export(weighted.rrse)
+export(weighted.rsq)
+export(weighted.selectivity)
+export(weighted.sensitivity)
+export(weighted.smape)
+export(weighted.specificity)
+export(weighted.tnr)
+export(weighted.tpr)
+export(weighted.tscore)
+export(weighted.zerooneloss)
 export(zerooneloss)
 importFrom(Rcpp,sourceCpp)
 useDynLib(SLmetrics, .registration = TRUE)

NEWS.Rmd

@@ -15,12 +15,113 @@ knitr::opts_chunk$set(
 set.seed(1903) 
 ```
 
-# Version 0.1-1
+# Version 0.2-0
 
-> Version 0.1-1 is considered pre-release of {SLmetrics}. We do not
+> Version 0.2-0 is considered pre-release of {SLmetrics}. We do not
 > expect any breaking changes, unless a major bug/issue is reported and its nature
 > forces breaking changes.
 
+## Improvements
+
+* **documentation:** The documentation has gotten some extra love, and now all functions have their formulas embedded, the details section have been freed from a general description of [factor] creation. This will make room for future expansions on the various functions where more details are required.
+
+* **weighted classification metrics:** The `cmatrix()`-function now accepts the argument `w` which is the sample weights; if passed the respective method will return the weighted metric. Below is an example using sample weights for the confusion matrix,
+
+```{r}
+# 1) define factors
+actual    <- factor(sample(letters[1:3], 100, replace = TRUE))
+predicted <- factor(sample(letters[1:3], 100, replace = TRUE))
+weights   <- runif(length(actual))
+
+# 2) without weights
+SLmetrics::cmatrix(
+    actual    = actual,
+    predicted = predicted
+)
+
+# 2) with weights
+SLmetrics::cmatrix(
+    actual    = actual,
+    predicted = predicted,
+    w         = weights
+)
+```
+
+Calculating weighted metrics manually or by using `foo.cmatrix()`-method,
+
+```{r}
+# 1) weigthed confusion matrix
+# and weighted accuray
+confusion_matrix <- SLmetrics::cmatrix(
+    actual    = actual,
+    predicted = predicted,
+    w         = weights
+)
+
+# 2) pass into accuracy
+# function
+SLmetrics::accuracy(
+    confusion_matrix
+)
+
+# 3) calculate the weighted
+# accuracy manually
+SLmetrics::weighted.accuracy(
+    actual    = actual,
+    predicted = predicted,
+    w         = weights
+)
+```
+
+Please note, however, that it is not possible to pass `cmatix()`-into `weighted.accurracy()`,
+
+* **Unit-testing:** All functions are now being tested for edge-cases in balanced and imbalanced classifcation problems, and regression problems, individually. This will enable a more robust development process and prevent avoidable bugs.
+
+```{r}
+try(
+    SLmetrics::weighted.accuracy(
+        confusion_matrix
+    )
+)
+```
+
+## Bug-fixes
+
+* **Floating precision:** Metrics would give different results based on the method used. This means that `foo.cmatrix()` and `foo.factor()` would produce different results (See Issue https://github.com/serkor1/SLmetrics/issues/16). This has been fixed by using higher precision `Rcpp::NumericMatrix` instead of `Rcpp::IntegerMatrix`.
+
+* **Miscalculation of Confusion Matrix elements:** An error in how `FN`, `TN`, `FP` and `TP` were calculated have been fixed. No issue has been raised for this bug. This was not something that was caught by the unit-tests, as the total samples were too high to spot this error. It has, however, been fixed now. This means that all metrics that uses these explicitly are now stable, and produces the desired output.
+
+* **Calculation Error in Fowlks Mallows Index:** A bug in the calculation of the `fmi()`-function has been fixed. The `fmi()`-function now correctly calculates the measure.
+
+* **Calculation Error in Pinball Deviance and Concordance Correlation Coefficient:** See issue https://github.com/serkor1/SLmetrics/issues/19. Switched to unbiased variance calculation in `ccc()`-function. The `pinball()`-function were missing a weighted quantile function. The issue is now fixed.
+
+* **Calculation Error in Balanced Accuracy:** See issue https://github.com/serkor1/SLmetrics/issues/24. The function now correctly adjusts for random chance, and the result matches that of {scikit-learn}
+
+* **Calculation Error in F-beta Score:** See issue https://github.com/serkor1/SLmetrics/issues/23. The function werent respecting `na.rm` and `micro`, this has been fixed accordingly.
+
+* **Calculation Error in Relative Absolute Error:** The function was incorrectly calculating means, instead of sums. This has been fixed.
+
+## Breaking changes
+
+* All regression metrics have had `na.rm`- and `w`-arguments removed. All  weighted regression metrics have a seperate function on the `weighted.foo()` to increase consistency across all metrics. See example below,
+
+```{r}
+# 1) define regression problem
+actual    <- rnorm(n = 1e3)
+predicted <- actual + rnorm(n = 1e3)
+w         <- runif(n = 1e3)
+
+# 2) unweighted metrics
+SLmetrics::rmse(actual, predicted)
+
+# 3) weighted metrics
+SLmetrics::weighted.rmse(actual, predicted, w = w)
+```
+
+* The `rrmse()`-function have been removed in favor of the `rrse()`-function. This function was incorrectly specified and described in the package.
+
+# Version 0.1-1
+
 ## General
 
 * **Backend changes:** All pair-wise metrics arer moved from {Rcpp} to C++, this have reduced execution time by half. All pair-wise metrics are now faster.
@@ -29,7 +130,7 @@ set.seed(1903)
 
 * **NA-controls:** All pair-wise metrics that doesn't have a `micro`-argument were handling missing values as according to C++ and {Rcpp} internals. See [Issue](https://github.com/serkor1/SLmetrics/issues/8). Thank you @EmilHvitfeldt for pointing this out. This has now been fixed so functions uses an `na.rm`-argument to explicitly control for this. See below,
 
-```{r}
+```{r, eval = FALSE}
 # 1) define factors
 actual    <- factor(c("no", "yes"))
 predicted <- factor(c(NA, "no"))
@@ -94,7 +195,6 @@ plot(
 ```
 
 
-
 # Version 0.1-0
 
 ## General