Forecast skill score measure

This function converts other error metrics such as MSE into a skill score. The reference or benchmark forecasting method is the Naive method for non-seasonal data, and the seasonal naive method for seasonal data. When used within accuracy.fbl_ts, it is important that the data contains both the training and test data, as the training data is used to compute the benchmark forecasts.

skill_score(measure)

Arguments

measure

The accuracy measure to use in computing the skill score.

Examples

skill_score(MSE)
#> function (...) 
#> {
#>     score <- measure(...)
#>     bench <- list(...)
#>     lag <- bench$.period
#>     n <- length(bench$.train)
#>     y <- bench$.train
#>     bench$.fc <- rep_len(y[c(rep(NA, max(0, lag - n)), seq_len(min(n, 
#>         lag)) + n - min(n, lag))], length(bench$.fc))
#>     bench$.resid <- bench$.actual - bench$.fc
#>     e <- y - c(rep(NA, min(lag, n)), y[seq_len(length(y) - lag)])
#>     mse <- mean(e^2, na.rm = TRUE)
#>     h <- length(bench$.actual)
#>     fullperiods <- (h - 1)/lag + 1
#>     steps <- rep(seq_len(fullperiods), rep(lag, fullperiods))[seq_len(h)]
#>     bench$.dist <- distributional::dist_normal(bench$.fc, sqrt(mse * 
#>         steps))
#>     ref_score <- do.call(measure, bench)
#>     1 - score/ref_score
#> }
#> <bytecode: 0x55fd9ed401b0>
#> <environment: 0x55fd9ec76488>

library(fable)
library(tsibble)

lung_deaths <- as_tsibble(cbind(mdeaths, fdeaths))
lung_deaths %>% 
  dplyr::filter(index < yearmonth("1979 Jan")) %>%
  model(
    ets = ETS(value ~ error("M") + trend("A") + season("A")),
    lm = TSLM(value ~ trend() + season())
  ) %>%
  forecast(h = "1 year") %>%
  accuracy(lung_deaths, measures = list(skill = skill_score(MSE)))
#> # A tibble: 4 × 4
#>   .model key     .type skill
#>   <chr>  <chr>   <chr> <dbl>
#> 1 ets    fdeaths Test  0.223
#> 2 ets    mdeaths Test  0.446
#> 3 lm     fdeaths Test  0.303
#> 4 lm     mdeaths Test  0.494