#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import numpy as np

import unittest2 as unittest

from nupic.frameworks.opf.metrics import getModule, MetricSpec, MetricMulti



class OPFMetricsTest(unittest.TestCase):

  DELTA = 0.01
  VERBOSITY = 0


  def testRMSE(self):
    rmse = getModule(MetricSpec("rmse", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
    gt = [9, 4, 5, 6]
    p = [0, 13, 8, 3]
    for i in xrange(len(gt)):
      rmse.addInstance(gt[i], p[i])
    target = 6.71

    self.assertTrue(abs(rmse.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)


  def testNRMSE(self):
    nrmse = getModule(MetricSpec("nrmse", None, None,
                                 {"verbosity" : OPFMetricsTest.VERBOSITY}))
    gt = [9, 4, 5, 6]
    p = [0, 13, 8, 3]
    for i in xrange(len(gt)):
      nrmse.addInstance(gt[i], p[i])
    target = 3.5856858280031814

    self.assertAlmostEqual(nrmse.getMetric()["value"], target)


  def testWindowedRMSE(self):
    wrmse = getModule(MetricSpec("rmse", None, None,
{"verbosity": OPFMetricsTest.VERBOSITY, "window":3}))
    gt = [9, 4, 4, 100, 44]
    p = [0, 13, 4, 6, 7]
    for gv, pv in zip(gt, p):
      wrmse.addInstance(gv, pv)
    target = 58.324

    self.assertTrue (abs(wrmse.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)


  def testAAE(self):
    aae = getModule(MetricSpec("aae", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
    gt = [9, 4, 5, 6]
    p = [0, 13, 8, 3]
    for i in xrange(len(gt)):
      aae.addInstance(gt[i], p[i])
    target = 6.0
    self.assertTrue(abs(aae.getMetric()["value"]-target) < OPFMetricsTest.DELTA)


  def testTrivialAAE(self):
    trivialaae = getModule(MetricSpec("trivial", None, None,
                {"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"aae"}))
    gt = [i/4+1 for i in range(100)]
    p = [i for i in range(100)]
    for i in xrange(len(gt)):
      trivialaae.addInstance(gt[i], p[i])
    target = .25
    self.assertTrue(abs(trivialaae.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)


  def testTrivialAccuracy(self):
    trivialaccuracy = getModule(MetricSpec("trivial", None, None,
                {"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"acc"}))
    gt = [str(i/4+1) for i in range(100)]
    p = [str(i) for i in range(100)]
    for i in xrange(len(gt)):
      trivialaccuracy.addInstance(gt[i], p[i])
    target = .75
    self.assertTrue(abs(trivialaccuracy.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)


  def testWindowedTrivialAAE (self):
    """Trivial Average Error metric test"""
    trivialAveErr = getModule(MetricSpec("trivial", None, None,
            {"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"avg_err"}))
    gt = [str(i/4+1) for i in range(100)]
    p = [str(i) for i in range(100)]
    for i in xrange(len(gt)):
      trivialAveErr.addInstance(gt[i], p[i])
    target = .25
    self.assertTrue(abs(trivialAveErr.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)


  def testWindowedTrivialAccuract(self):
    """Trivial AAE metric test"""
    trivialaae = getModule(MetricSpec("trivial", None, None,
    {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100,"errorMetric":"aae"}))
    gt = [i/4+1 for i in range(1000)]
    p = [i for i in range(1000)]
    for i in xrange(len(gt)):
      trivialaae.addInstance(gt[i], p[i])
    target = .25
    self.assertTrue(abs(trivialaae.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)


  def testWindowedTrivialAccuracy(self):
    """Trivial Accuracy metric test"""
    trivialaccuracy = getModule(MetricSpec("trivial", None, None,
 {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100,"errorMetric":"acc"}))
    gt = [str(i/4+1) for i in range(1000)]
    p = [str(i) for i in range(1000)]
    for i in xrange(len(gt)):
      trivialaccuracy.addInstance(gt[i], p[i])
    target = .75
    self.assertTrue(abs(trivialaccuracy.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)


  def testWindowedTrivialAverageError (self):
    """Trivial Average Error metric test"""
    trivialAveErr = getModule(MetricSpec("trivial", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100,"errorMetric":"avg_err"}))
    gt = [str(i/4+1) for i in range(500, 1000)]
    p = [str(i) for i in range(1000)]
    for i in xrange(len(gt)):
      trivialAveErr.addInstance(gt[i], p[i])
    target = .25
    self.assertTrue(abs(trivialAveErr.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)


  def testMultistepAAE(self):
    """Multistep AAE metric test"""
    msp = getModule(MetricSpec("multiStep", None, None,
     {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae",
           "
steps": 3}))
    
    # Make each ground truth 1 greater than the p
rediction
    gt = [i+1 for i in range(100)]
    p = [{3: {i: .7, 5: 0.3}} for i in range(100)]

    for i in xrange(len(gt)):
      msp.addInstance(gt[i], p[i])
    target = 1
    self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)


  def testMultistepAAEMultipleSteps(self):
    """Multistep AAE metric test, predicting 2 different step sizes"""
    msp = getModule(MetricSpec("multiStep", None, None,
     {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae",
           "steps": [3,6]}))
    
    # Make each 3 step prediction +1 over ground truth and each 6 step
    # prediction +0.5 over ground truth
    gt = [i for i in range(100)]
    p = [{3: {i+1: .7, 5: 0.3},
          6: {i+0.5: .7, 5: 0.3}} for i in range(100)]

    for i in xrange(len(gt)):
      msp.addInstance(gt[i], p[i])
    target = 0.75  # average of +1 error and 0.5 error
    self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)


  def testMultistepProbability(self):
    """Multistep with probabilities metric test"""
    msp = getModule(MetricSpec("multiStepProbability", None, None,
 {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae",
           "steps":3}))
    gt = [5 for i in range(1000)]
    p = [{3: {i: .3, 5: .7}} for i in range(1000)]
    for i in xrange(len(gt)):
      msp.addInstance(gt[i], p[i])
    #((999-5)(1000-5)/2-(899-5)(900-5)/2)*.3/100
    target = 283.35
    self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)


  def testMultistepProbabilityMultipleSteps(self):
    """Multistep with probabilities metric test, predicting 2 different step
    sizes"""
    msp = getModule(MetricSpec("multiStepProbability", None, None,
          {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100,
           "errorMetric":"aae", "steps": [1,3]}))
    gt = [5 for i in range(1000)]
    p = [{3: {i: .3, 5: .7},
          1: {5: 1.0}} for i in range(1000)]
    for i in xrange(len(gt)):
      msp.addInstance(gt[i], p[i])
    #(((999-5)(1000-5)/2-(899-5)(900-5)/2)*.3/100) / 2
    #  / 2 because the 1-step prediction is 100% accurate
    target = 283.35/2
    self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)


  def testMovingMeanAbsoluteError(self):
    """Moving mean Average Absolute Error metric test"""
    movingMeanAAE = getModule(MetricSpec("moving_mean", None, None,
         {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "mean_wind