Create a set of tools

The create_toolset function takes names of predefined tools and generates a list of wrapper functions for Precision-Recall curve calculations.

create_toolset(
  tool_names = NULL,
  set_names = NULL,
  calc_auc = TRUE,
  store_res = TRUE
)

Arguments

tool_names

A character vector to specify the names of performance evaluation tools. The names for the following seven tools can be currently used.

• ROCR

• AUCCalculator

• PerfMeas

• PRROC

• precrec

• yardstick

• sklearn

The sklearn tool requires the reticulate package together with a working Python installation and numpy. When they are unavailable, the tool is created without them and returns a flat dummy curve, in the same way as AUCCalculator does without rJava.

set_names

A character vector to specify a predefined set name. Following twelve sets are currently available. The digit is the number of tools in the set, and each smaller set drops one more of the slower tools.

"def7"

ROCR, AUCCalculator, PerfMeas, PRROC, precrec, yardstick, and sklearn with calc_auc = TRUE and store_res = TRUE

"auc7"

The 7 tools of "def7" with calc_auc = TRUE and store_res = FALSE

"crv7"

The 7 tools of "def7" with calc_auc = FALSE and store_res = TRUE

"def6"

ROCR, AUCCalculator, PRROC, precrec, yardstick, and sklearn with calc_auc = TRUE and store_res = TRUE

"auc6"

The 6 tools of "def6" with calc_auc = TRUE and store_res = FALSE

"crv6"

The 6 tools of "def6" with calc_auc = FALSE and store_res = TRUE

"def5"

ROCR, PRROC, precrec, yardstick, and sklearn with calc_auc = TRUE and store_res = TRUE

"auc5"

The 5 tools of "def5" with calc_auc = TRUE and store_res = FALSE

"crv5"

The 5 tools of "def5" with calc_auc = FALSE and store_res = TRUE

"def4"

ROCR, precrec, yardstick, and sklearn with calc_auc = TRUE and store_res = TRUE

"auc4"

The 4 tools of "def4" with calc_auc = TRUE and store_res = FALSE

"crv4"

The 4 tools of "def4" with calc_auc = FALSE and store_res = TRUE

calc_auc

A Boolean value to specify whether the AUC score should be calculated.

store_res

A Boolean value to specify whether the calculated curve is retrieved and stored

Value

A list of R6 tool objects.

See also

run_benchmark and run_evalcurve require the list of the tools generated by this function ToolROCR, ToolAUCCalculator, ToolPerfMeas, ToolPRROC, Toolprecrec, Toolyardstick, and Toolsklearn as R6 tool classes.

Examples

## Create ROCR and precrec
toolset1 <- create_toolset(c("ROCR", "precrec"))
toolset1
#> $ROCR
#> 
#>     === Tool interface ===
#> 
#>     Tool name:            ROCR 
#>     Calculate AUC score:  Yes
#>     Store results:        Yes
#>     Prediction performed: No
#>     Available methods:    call(testset, calc_auc, store_res)
#>                           get_toolname()
#>                           set_toolname(toolname)
#>                           get_setname()
#>                           set_setname(setname)
#>                           get_result()
#>                           get_x()
#>                           get_y()
#>                           get_auc()
#>     Help file:            help("ToolROCR")
#> 
#> $precrec
#> 
#>     === Tool interface ===
#> 
#>     Tool name:            precrec 
#>     Calculate AUC score:  Yes
#>     Store results:        Yes
#>     Prediction performed: No
#>     Available methods:    call(testset, calc_auc, store_res)
#>                           get_toolname()
#>                           set_toolname(toolname)
#>                           get_setname()
#>                           set_setname(setname)
#>                           get_result()
#>                           get_x()
#>                           get_y()
#>                           get_auc()
#>                           set_x_bins(x_bins)
#>     Help file:            help("Toolprecrec")
#> 

## Create auc7 tools
toolset2 <- create_toolset(set_names = "auc7")
toolset2
#> $ROCR
#> 
#>     === Tool interface ===
#> 
#>     Tool name:            ROCR 
#>     Calculate AUC score:  Yes
#>     Store results:        No
#>     Prediction performed: No
#>     Available methods:    call(testset, calc_auc, store_res)
#>                           get_toolname()
#>                           set_toolname(toolname)
#>                           get_setname()
#>                           set_setname(setname)
#>                           get_result()
#>                           get_x()
#>                           get_y()
#>                           get_auc()
#>     Help file:            help("ToolROCR")
#> 
#> $AUCCalculator
#> 
#>     === Tool interface ===
#> 
#>     Tool name:            AUCCalculator 
#>     Calculate AUC score:  Yes
#>     Store results:        No
#>     Prediction performed: No
#>     Available methods:    call(testset, calc_auc, store_res)
#>                           get_toolname()
#>                           set_toolname(toolname)
#>                           get_setname()
#>                           set_setname(setname)
#>                           get_result()
#>                           get_x()
#>                           get_y()
#>                           get_auc()
#>                           set_jarpath(jarpath)
#>                           set_curvetype(curvetype)
#>                           set_auctype(auctype)
#>     Help file:            help("ToolAUCCalculator")
#> 
#> $PerfMeas
#> 
#>     === Tool interface ===
#> 
#>     Tool name:            PerfMeas 
#>     Calculate AUC score:  Yes
#>     Store results:        No
#>     Prediction performed: No
#>     Available methods:    call(testset, calc_auc, store_res)
#>                           get_toolname()
#>                           set_toolname(toolname)
#>                           get_setname()
#>                           set_setname(setname)
#>                           get_result()
#>                           get_x()
#>                           get_y()
#>                           get_auc()
#>     Help file:            help("ToolPerfMeas")
#> 
#> $PRROC
#> 
#>     === Tool interface ===
#> 
#>     Tool name:            PRROC 
#>     Calculate AUC score:  Yes
#>     Store results:        No
#>     Prediction performed: No
#>     Available methods:    call(testset, calc_auc, store_res)
#>                           get_toolname()
#>                           set_toolname(toolname)
#>                           get_setname()
#>                           set_setname(setname)
#>                           get_result()
#>                           get_x()
#>                           get_y()
#>                           get_auc()
#>                           set_curve(val)
#>                           set_minStepSize(val)
#>                           set_aucType(val)
#>     Help file:            help("ToolPRROC")
#> 
#> $precrec
#> 
#>     === Tool interface ===
#> 
#>     Tool name:            precrec 
#>     Calculate AUC score:  Yes
#>     Store results:        No
#>     Prediction performed: No
#>     Available methods:    call(testset, calc_auc, store_res)
#>                           get_toolname()
#>                           set_toolname(toolname)
#>                           get_setname()
#>                           set_setname(setname)
#>                           get_result()
#>                           get_x()
#>                           get_y()
#>                           get_auc()
#>                           set_x_bins(x_bins)
#>     Help file:            help("Toolprecrec")
#> 
#> $yardstick
#> 
#>     === Tool interface ===
#> 
#>     Tool name:            yardstick 
#>     Calculate AUC score:  Yes
#>     Store results:        No
#>     Prediction performed: No
#>     Available methods:    call(testset, calc_auc, store_res)
#>                           get_toolname()
#>                           set_toolname(toolname)
#>                           get_setname()
#>                           set_setname(setname)
#>                           get_result()
#>                           get_x()
#>                           get_y()
#>                           get_auc()
#>     Help file:            help("Toolyardstick")
#> 
#> $sklearn
#> 
#>     === Tool interface ===
#> 
#>     Tool name:            sklearn 
#>     Calculate AUC score:  Yes
#>     Store results:        No
#>     Prediction performed: No
#>     Available methods:    call(testset, calc_auc, store_res)
#>                           get_toolname()
#>                           set_toolname(toolname)
#>                           get_setname()
#>                           set_setname(setname)
#>                           get_result()
#>                           get_x()
#>                           get_y()
#>                           get_auc()
#>                           set_drop_intermediate(val)
#>                           set_aucType(val)
#>     Help file:            help("Toolsklearn")
#>