# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (c) 2016-2017 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#

import collections
import logging
import pyudev
import math
import operator
import os
import platform
import re
import subprocess

from fm_api import fm_api
from fm_api import constants as fm_constants
from io_monitor import constants
from io_monitor.constants import DOMAIN
from io_monitor.utils.data_collector import DeviceDataCollector
from io_monitor.constants import _
from oslo_config import cfg

ccm_opts = [
    cfg.IntOpt('ssd_small_window_size',
               default=30,
               help=('SSD: Small moving average window size (in seconds).')),
    cfg.IntOpt('ssd_medium_window_size',
               default=60,
               help=('SSD: Medium moving average window size (in seconds).')),
    cfg.IntOpt('ssd_large_window_size',
               default=90,
               help=('SSD: Large moving average window size (in seconds).')),
    cfg.IntOpt('ssd_thresh_sustained_await',
               default=1000,
               help=('SSD: Value required in a moving average window to '
                     'trigger next state.')),
    cfg.IntOpt('ssd_thresh_max_await',
               default=5000,
               help=('SSD: Max await time. Anomalous data readings are clipped'
                     ' to this.')),
    cfg.IntOpt('hdd_small_window_size',
               default=120,
               help=('HDD: Small moving average window size (in seconds).')),
    cfg.IntOpt('hdd_medium_window_size',
               default=180,
               help=('HDD: Medium moving average window size (in seconds).')),
    cfg.IntOpt('hdd_large_window_size',
               default=240,
               help=('HDD: Large moving average window size (in seconds).')),
    cfg.IntOpt('hdd_thresh_sustained_await',
               default=1500,
               help=('HDD: Value required in a moving average window to '
                     'trigger next state.')),
    cfg.IntOpt('hdd_thresh_max_await',
               default=5000,
               help=('HDD: Max await time. Anomalous data readings are clipped'
                     ' to this.')),
    cfg.StrOpt('log_level',
               default='INFO',
               choices=('ERROR', 'WARN', 'INFO', 'DEBUG'),
               help=('Monitor debug level. Note: global level must be'
                     ' equialent or lower.')),
    cfg.FloatOpt('status_log_rate_modifier', default=0.2,
                 help=('Modify how often status messages appear in the log.'
                       '0.0 is never, 1.0 is for every iostat execution.')),
    cfg.BoolOpt('generate_fm_alarms', default=True,
                help=('Enable FM Alarm generation')),
    cfg.IntOpt('fm_alarm_debounce', default=5,
               help=('Number of consecutive same congestion states seen '
                     'before raising/clearing alarms.')),
    cfg.BoolOpt('output_write_csv', default=False,
                help=('Write monitor data to a csv for analysis')),
    cfg.StrOpt('output_csv_dir', default='/tmp',
               help=('Directory where monitor output will be located.')),
]

CONF = cfg.CONF
CONF.register_opts(ccm_opts, group="cinder_congestion")

LOG = logging.getLogger(DOMAIN)


class CinderCongestionMonitor(object):
    # Congestion States
    STATUS_NORMAL = "Normal"
    STATUS_BUILDING = "Building"
    STATUS_CONGESTED = "Limiting"

    # disk type
    CINDER_DISK_SSD = 0
    CINDER_DISK_HDD = 1

    def __init__(self):
        # Setup logging
        level_dict = {'ERROR': logging.ERROR,
                      'WARN': logging.WARN,
                      'INFO': logging.INFO,
                      'DEBUG': logging.DEBUG}

        if CONF.cinder_congestion.log_level in level_dict.keys():
            LOG.setLevel(level_dict[CONF.cinder_congestion.log_level])
        else:
            LOG.setLevel(logging.INFO)

        LOG.info("Initializing %s..." % self.__class__.__name__)

        # DRBD file
        self.drbd_file = '/etc/drbd.d/drbd-cinder.res'

        # iostat parsing regex
        self.ts_regex = re.compile(r"(\d{2}/\d{2}/\d{2,4}) "
                                   "(\d{2}:\d{2}:\d{2})")
        self.device_regex = re.compile(
            r"(\w+-?\w+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)"
            "\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+"
            "(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)")

        # window sizes
        self.s_window_sec = CONF.cinder_congestion.ssd_small_window_size
        self.m_window_sec = CONF.cinder_congestion.ssd_medium_window_size
        self.l_window_sec = CONF.cinder_congestion.ssd_large_window_size

        # state variables
        self.latest_time = None
        self.congestion_status = self.STATUS_NORMAL

        # init data collector
        self.device_dict = {}

        # devices
        self.phys_cinder_device = None
        self.base_cinder_devs = []
        self.base_cinder_tracking_devs = []
        self.non_cinder_dynamic_devs = ['drbd0', 'drbd1', 'drbd2', 'drbd3',
                                        'drbd5']
        self.non_cinder_phys_devs = []

        # set the default operational scenarios
        self.await_minimal_spike = CONF.cinder_congestion.ssd_thresh_max_await
        self.await_sustained_congestion = (
            CONF.cinder_congestion.ssd_thresh_sustained_await)

        # FM
        self.fm_api = fm_api.FaultAPIs()
        self.fm_state_count = collections.Counter()

        # CSV handle
        self.csv = None

        # status logging
        self.status_skip_count = 0

        # to compare with current g_count
        self.last_g_count = 0

        message_rate = math.ceil(60 / (CONF.wait_time+1))
        self.status_skip_total = math.ceil(
            message_rate/(message_rate *
                          CONF.cinder_congestion.status_log_rate_modifier))
        LOG.info("Display status message at %d per minute..." %
                 (message_rate *
                  CONF.cinder_congestion.status_log_rate_modifier))

        # Clear any exiting alarms
        self._clear_fm()

    def _is_number(self, s):
        try:
            float(s)
            return True
        except ValueError:
            return False

    def command(self, arguments, **kwargs):
        """ Execute e command and capture stdout, stderr & return code """
        process = subprocess.Popen(
            arguments,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            **kwargs)
        out, err = process.communicate()
        return out, err, process.returncode

    def device_path_to_device_node(self, device_path):
        try:
            output, _, _ = self.command(["udevadm", "settle", "-E",
                                        device_path])
            out, err, retcode = self. command(["readlink", "-f", device_path])
            out = out.rstrip()
        except Exception as e:
            return None

        return out

    def _get_disk_type(self, device_node):
        if device_node:
            proc_device_file = '/sys/block/' + device_node + \
                               '/queue/rotational'
            if os.path.exists(proc_device_file):
                with open(proc_device_file) as fileobject:
                    for line in fileobject:
                        return int(line.rstrip())

        # If the disk is unknown assume an SSD.
        return self.CINDER_DISK_SSD


    def _is_cinder_related_device(self,device_node):
        name = ""
        if device_node:
            proc_device_file = '/sys/block/' + device_node + \
                               '/dm/name'

            if os.path.exists(proc_device_file):
                with open(proc_device_file) as fileobject:
                    for line in fileobject:
                        name = line.rstrip()

        if constants.CINDER_DM_PREFIX in name:
            return True

        return False

    def _is_cinder_backing_device(self, device_node):
        name = ""
        if device_node:
            proc_device_file = '/sys/block/' + device_node + \
                               '/dm/name'
            if os.path.exists(proc_device_file):
                with open(proc_device_file) as fileobject:
                    for line in fileobject:
                        name = line.rstrip()

        if any(s in name for s in ['pool', 'anchor']):
            if device_node not in self.base_cinder_devs:
                self.base_cinder_devs.append(device_node)
                if any(s in name for s in ['tdata', 'tmeta']):
                    if device_node not in self.base_cinder_tracking_devs:
                        self.base_cinder_tracking_devs.append(device_node)

                LOG.info("Cinder Base Devices = %s; Tracking %s" % (
                    self.base_cinder_devs, self.base_cinder_tracking_devs))
            return True

        return False

    def _determine_cinder_devices(self):
        # Check to see if we have DRBD device we are syncing
        if os.path.exists(self.drbd_file):

            # grab the data
            with open(self.drbd_file) as fileobject:

                drbd_dev_regex = re.compile(r"device\s+/dev/(\w+);")
                drbd_disk_path_regex = re.compile(
                    r"disk\s+\"(/dev/disk/by-path/(.+))\";")
                drbd_disk_node_regex = re.compile(r"/dev/(\w+)")
                partition_regex = re.compile(r"(sd\w+)\d+")

                for line in fileobject:
                    m = drbd_dev_regex.match(line.strip())
                    if m:
                        self.base_cinder_devs.append(m.group(1))

                    m = drbd_disk_path_regex.match(line.strip())
                    if m:
                        drbd_disk = self.device_path_to_device_node(m.group(1))

                        drbd_disk_sd = drbd_disk_node_regex.match(drbd_disk)
                        if drbd_disk_sd:
                            self.base_cinder_devs.append(drbd_disk_sd.group(1))

                            d = partition_regex.match(drbd_disk_sd.group(1))
                            if d:
                                self.phys_cinder_device = d.group(1)
                                self.base_cinder_devs.append(d.group(1))

            # Which host OS?
            if platform.linux_distribution()[0] == constants.WRLINUX:
                dm_major = 252
            else:
                dm_major = 253

            # Grab the device mapper devices and pull out the base cinder
            # devices
            dmsetup_regex = re.compile(r'^([\w-]+)\s+\((\d+):(\d+)\)')

            dmsetup_command = 'dmsetup ls'
            dmsetup_process = subprocess.Popen(dmsetup_command,
                                               stdout=subprocess.PIPE,
                                               shell=True)
            dmsetup_output = dmsetup_process.stdout.read()
            lines = dmsetup_output.split('\n')
            for l in lines:
                m = dmsetup_regex.match(l.strip())
                if m:
                    if m.group(2) == str(dm_major):
                        # LOG.debug("%s %s %s" % (m.group(1),
                        #                         m.group(2),
                        #                         m.group(3)))
                        if constants.CINDER_DM_PREFIX in m.group(1):
                            if 'pool' in m.group(1) or 'anchor' in m.group(1):
                                self.base_cinder_devs.append(
                                    "dm-" + m.group(3))
                            if 'tdata' in m.group(1) or 'tmeta' in m.group(1):
                                self.base_cinder_tracking_devs.append(
                                    "dm-" + m.group(3))
                        else:
                            self.non_cinder_dynamic_devs.append(
                                "dm-" + m.group(3))

            # If the tracking devs are non existant, then we didn't find any
            # thin pool entries. Therefore we are thickly provisioned and need
            # to track the physical device
            if len(self.base_cinder_tracking_devs) == 0:
                self.base_cinder_tracking_devs.append(
                    self.phys_cinder_device)

        # Use UDEV info to grab all phyical disks
        context = pyudev.Context()
        for device in context.list_devices(subsystem='block',
                                           DEVTYPE='disk'):
            if device['MAJOR'] == '8':
                device = str(os.path.basename(device['DEVNAME']))
                if device != self.phys_cinder_device:
                    self.non_cinder_phys_devs.append(device)

    def _update_device_stats(self, ts, device, current_iops, current_await):
        if device not in self.device_dict:
             # For AIO systems nova-local will be provisioned later and
             # differently based on the instance_backing value for the compute
             # functionality. Check for cinder specific dm devices and ignore
             # all others
            if not self._is_cinder_related_device(device):
                return
            self._is_cinder_backing_device(device)
            self.device_dict.update(
                {device: DeviceDataCollector(
                    device,
                    [DeviceDataCollector.DATA_IOPS,
                     DeviceDataCollector.DATA_AWAIT],
                    self.s_window_sec,
                    self.m_window_sec,
                    self.l_window_sec)})
            self.device_dict[device].set_data_caps(
                DeviceDataCollector.DATA_AWAIT,
                self.await_minimal_spike)
            self.device_dict[device].set_congestion_thresholds(
                self.await_minimal_spike,
                self.await_sustained_congestion)

        self.device_dict[device].update_data(ts,
                                             DeviceDataCollector.DATA_IOPS,
                                             current_iops)
        self.device_dict[device].update_data(ts,
                                             DeviceDataCollector.DATA_AWAIT,
                                             current_await)
        self.device_dict[device].update_congestion_status()

    def is_system_monitorable(self):
        if not os.path.exists(self.drbd_file):
            LOG.error("%s does not exist" % self.drbd_file)
            return False

        # Discover devices on this host
        self._determine_cinder_devices()

        # Get the cinder disk type and set the monitor values accordingly
        disk_type = self._get_disk_type(self.phys_cinder_device)
        if disk_type:
            self.s_window_sec = CONF.cinder_congestion.hdd_small_window_size
            self.m_window_sec = CONF.cinder_congestion.hdd_medium_window_size
            self.l_window_sec = CONF.cinder_congestion.hdd_large_window_size
            self.await_minimal_spike = (
                CONF.cinder_congestion.hdd_thresh_max_await)
            self.await_sustained_congestion = (
                CONF.cinder_congestion.hdd_thresh_sustained_await)
        else:
            self.s_window_sec = CONF.cinder_congestion.ssd_small_window_size
            self.m_window_sec = CONF.cinder_congestion.ssd_medium_window_size
            self.l_window_sec = CONF.cinder_congestion.ssd_large_window_size
            self.await_minimal_spike = (
                CONF.cinder_congestion.ssd_thresh_max_await)
            self.await_sustained_congestion = (
                CONF.cinder_congestion.ssd_thresh_sustained_await)

        LOG.info("Physical Cinder Disk = %s - %s" %
                 (self.phys_cinder_device,
                  "HDD" if disk_type else "SSD"))
        LOG.info("Cinder Base Devices = %s; Tracking %s" % (
            self.base_cinder_devs, self.base_cinder_tracking_devs))
        LOG.info("Non-Cinder Devices = %s" % (
            self.non_cinder_dynamic_devs + self.non_cinder_phys_devs))

        return True

    def get_operational_thresholds(self):
        return (self.await_minimal_spike,
                self.await_sustained_congestion)

    def set_operational_thresholds(self,
                                   await_minimal_spike,
                                   await_sustained_congestion):
        if await_minimal_spike:
            self.await_minimal_spike = await_minimal_spike
        if await_sustained_congestion:
            self.await_sustained_congestion = await_sustained_congestion

    def _flush_stale_devices(self):
        for d in self.device_dict.keys():
            if self.device_dict[d].is_data_stale(self.latest_time):
                self.device_dict.pop(d, None)

    def _log_device_data_windows(self, device):
        LOG.debug("%-6s: %s %s" % (
            device,
            self.device_dict[device].get_element_windows_avg_string(
                DeviceDataCollector.DATA_AWAIT),
            self.device_dict[device].get_element_windows_avg_string(
                DeviceDataCollector.DATA_IOPS)))

    def _log_congestion_status(self, congestion_data):
        congestion_data.c_freq_dict.update(
            dict.fromkeys(
                set(['N', 'B', 'L']).difference(
                    congestion_data.c_freq_dict), 0))
        congestion_data.g_freq_dict.update(
            dict.fromkeys(
                set(['N', 'B', 'L']).difference(
                    congestion_data.g_freq_dict), 0))

        LOG.info("Status (%-8s): Cinder Devs IOPS [ %10.2f, %10.2f, %10.2f ] "
                 "Guests Counts %s; Guest Await[ %10.2f, %10.2f, %10.2f ]" % (
                     congestion_data.status,
                     congestion_data.c_iops_avg_list[0],
                     congestion_data.c_iops_avg_list[1],
                     congestion_data.c_iops_avg_list[2],
                     dict(congestion_data.g_freq_dict),
                     congestion_data.g_await_avg_list[0],
                     congestion_data.g_await_avg_list[1],
                     congestion_data.g_await_avg_list[2]))

    def _determine_congestion_state(self):

        # Analyze devices
        cinder_congestion_freq = collections.Counter()
        cinder_iops_avg = [0.0, 0.0, 0.0]
        guest_congestion_freq = collections.Counter()
        guest_await_avg = [0.0, 0.0, 0.0]

        for d, dc in self.device_dict.iteritems():
            if d in self.base_cinder_devs:
                if d in self.base_cinder_tracking_devs:
                    cinder_congestion_freq.update(dc.get_congestion_status())
                    cinder_iops_avg = map(operator.add,
                                          cinder_iops_avg,
                                          dc.get_element_windows_avg_list(
                                              DeviceDataCollector.DATA_IOPS))
                    # LOG.debug("C: %s " % cinder_iops_avg)
                    # self._log_device_data_windows(d)

            elif d not in (self.base_cinder_devs +
                           self.non_cinder_dynamic_devs +
                           self.non_cinder_phys_devs):
                guest_congestion_freq.update(
                    dc.get_congestion_status(debug=True))
                guest_await_avg = map(operator.add,
                                      guest_await_avg,
                                      dc.get_element_windows_avg_list(
                                          DeviceDataCollector.DATA_AWAIT))
                # LOG.debug("G: %s " % guest_await_avg)
                # self._log_device_data_windows(d)

        if list(cinder_congestion_freq.elements()):
            cinder_iops_avg[:] = [i/len(list(
                cinder_congestion_freq.elements())) for i in cinder_iops_avg]

        if list(guest_congestion_freq.elements()):
            guest_await_avg[:] = [i/len(list(
                guest_congestion_freq.elements())) for i in guest_await_avg]

        self.congestion_status = self.STATUS_NORMAL
        if DeviceDataCollector.STATUS_BUILDING in guest_congestion_freq:
            self.congestion_status = self.STATUS_BUILDING
        if DeviceDataCollector.STATUS_CONGESTED in guest_congestion_freq:
            self.congestion_status = self.STATUS_CONGESTED

        congestion_data = collections.namedtuple("congestion_data",
                                                 ["timestamp", "status",
                                                  "c_freq_dict",
                                                  "c_iops_avg_list",
                                                  "g_count",
                                                  "g_freq_dict",
                                                  "g_await_avg_list"])

        return congestion_data(self.latest_time,
                               self.congestion_status,
                               cinder_congestion_freq,
                               cinder_iops_avg,
                               sum(guest_congestion_freq.values()),
                               guest_congestion_freq,
                               guest_await_avg)

    def _clear_fm(self):
        building = fm_constants.FM_ALARM_ID_STORAGE_CINDER_IO_BUILDING
        limiting = fm_constants.FM_ALARM_ID_STORAGE_CINDER_IO_LIMITING

        entity_instance_id = "cinder_io_monitor"
        ccm_alarm_ids = [building, limiting]

        existing_alarms = []
        for alarm_id in ccm_alarm_ids:
            alarm_list = self.fm_api.get_faults_by_id(alarm_id)
            if not alarm_list:
                continue
            for alarm in alarm_list:
                existing_alarms.append(alarm)

        if len(existing_alarms) > 1:
            LOG.warn("WARNING: we have more than one existing alarm")

        for a in existing_alarms:
            self.fm_api.clear_fault(a.alarm_id, entity_instance_id)
            LOG.info(
                _("Clearing congestion alarm {} - severity: {}, "
                  "reason: {}, service_affecting: {}").format(
                      a.uuid, a.severity, a.reason_text, True))

    def _update_fm(self, debounce_count, override=None):

        building = fm_constants.FM_ALARM_ID_STORAGE_CINDER_IO_BUILDING
        limiting = fm_constants.FM_ALARM_ID_STORAGE_CINDER_IO_LIMITING

        if override:
            self.congestion_status = override

        # Update the status count
        self.fm_state_count.update(self.congestion_status[0])

        # Debounce alarms: If I have more than one congestion type then clear
        # the counts as we have crossed a threshold
        if len(self.fm_state_count) > 1:
            self.fm_state_count.clear()
            self.fm_state_count.update(self.congestion_status[0])
            return

        # Debounce alarms: Make sure we have see this alarm state for a specifc
        # number of samples
        count = self.fm_state_count.itervalues().next()
        if count < debounce_count:
            return

        # We are past the debounce state. Now take action.
        entity_instance_id = "cinder_io_monitor"
        ccm_alarm_ids = [building, limiting]

        existing_alarms = []
        for alarm_id in ccm_alarm_ids:
            alarm_list = self.fm_api.get_faults_by_id(alarm_id)
            if not alarm_list:
                continue
            for alarm in alarm_list:
                existing_alarms.append(alarm)

        if len(existing_alarms) > 1:
            LOG.warn("WARNING: we have more than one existing alarm")

        if self.congestion_status is self.STATUS_NORMAL:
            for a in existing_alarms:
                self.fm_api.clear_fault(a.alarm_id, entity_instance_id)
                LOG.info(
                    _("Clearing congestion alarm {} - severity: {}, "
                      "reason: {}, service_affecting: {}").format(
                          a.uuid, a.severity, a.reason_text, True))

        elif self.congestion_status is self.STATUS_BUILDING:
            alarm_is_raised = False
            for a in existing_alarms:
                if a.alarm_id != building:
                    self.fm_api.clear_fault(a.alarm_id, entity_instance_id)
                    LOG.info(
                        _("Clearing congestion alarm {} - severity: {}, "
                          "reason: {}, service_affecting: {}").format(
                              a.uuid, a.severity, a.reason_text, True))
                else:
                    alarm_is_raised = True

            if not alarm_is_raised:
                severity = fm_constants.FM_ALARM_SEVERITY_MAJOR
                reason_text = constants.ALARM_REASON_BUILDING

                fault = fm_api.Fault(
                    alarm_id=building,
                    alarm_type=fm_constants.FM_ALARM_TYPE_2,
                    alarm_state=fm_constants.FM_ALARM_STATE_SET,
                    entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER,
                    entity_instance_id=entity_instance_id,
                    severity=severity,
                    reason_text=reason_text,
                    probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8,
                    proposed_repair_action=constants.REPAIR_ACTION_MAJOR_ALARM,
                    service_affecting=True)
                alarm_uuid = self.fm_api.set_fault(fault)
                if alarm_uuid:
                    LOG.info(
                        _("Created congestion alarm {} - severity: {}, "
                          "reason: {}, service_affecting: {}").format(
                              alarm_uuid, severity, reason_text, True))
                else:
                    LOG.error(
                        _("Failed to create congestion alarm - severity: {},"
                          "reason: {}, service_affecting: {}").format(
                              severity, reason_text, True))

        elif self.congestion_status is self.STATUS_CONGESTED:
            alarm_is_raised = False
            for a in existing_alarms:
                if a.alarm_id != limiting:
                    self.fm_api.clear_fault(a.alarm_id, entity_instance_id)
                    LOG.info(
                        _("Clearing congestion alarm {} - severity: {}, "
                          "reason: {}, service_affecting: {}").format(
                              a.uuid, a.severity, a.reason_text, True))
                else:
                    alarm_is_raised = True

            if not alarm_is_raised:
                severity = fm_constants.FM_ALARM_SEVERITY_CRITICAL
                reason_text = constants.ALARM_REASON_CONGESTED
                repair = constants.REPAIR_ACTION_CRITICAL_ALARM
                fault = fm_api.Fault(
                    alarm_id=limiting,
                    alarm_type=fm_constants.FM_ALARM_TYPE_2,
                    alarm_state=fm_constants.FM_ALARM_STATE_SET,
                    entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER,
                    entity_instance_id=entity_instance_id,
                    severity=severity,
                    reason_text=reason_text,
                    probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8,
                    proposed_repair_action=repair,
                    service_affecting=True)
                alarm_uuid = self.fm_api.set_fault(fault)
                if alarm_uuid:
                    LOG.info(
                        _("Created congestion alarm {} - severity: {}, "
                          "reason: {}, service_affecting: {}").format(
                              alarm_uuid, severity, reason_text, True))
                else:
                    LOG.error(
                        _("Failed to congestion storage alarm - severity: {},"
                          "reason: {}, service_affecting: {}").format(
                              severity, reason_text, True))

    def _create_output(self, output_dir, congestion_data):
        if not self.csv:
            LOG.info("Creating output")
            if os.path.exists(output_dir):
                if output_dir.endswith('/'):
                    fn = output_dir + 'ccm.csv'
                else:
                    fn = output_dir + '/ccm.csv'
            else:
                fn = '/tmp/ccm.csv'
            try:
                self.csv = open(fn, 'w')
            except Exception as e:
                raise e

            self.csv.write("Timestamp, Congestion Status, "
                           "Cinder Devs Normal, "
                           "Cinder Devs Building, Cinder Devs Limiting,"
                           "Cinder IOPS Small, "
                           "Cinder IOPS Med, Cinder IOPS Large,"
                           "Guest Vols Normal, "
                           "Guest Vols Building, Guest Vols Limiting,"
                           "Guest Await Small, "
                           "Guest Await Med, Guest Await Large")
            LOG.info("Done writing")

        congestion_data.c_freq_dict.update(
            dict.fromkeys(set(['N', 'B', 'L']).difference(
                congestion_data.c_freq_dict), 0))
        congestion_data.g_freq_dict.update(
            dict.fromkeys(set(['N', 'B', 'L']).difference(
                congestion_data.g_freq_dict), 0))

        self.csv.write(
            ",".join(
                (str(congestion_data.timestamp),
                 str(congestion_data.status[0]),
                 str(congestion_data.c_freq_dict[
                     DeviceDataCollector.STATUS_NORMAL]),
                 str(congestion_data.c_freq_dict[
                     DeviceDataCollector.STATUS_BUILDING]),
                 str(congestion_data.c_freq_dict[
                     DeviceDataCollector.STATUS_CONGESTED]),
                 str(congestion_data.c_iops_avg_list[0]),
                 str(congestion_data.c_iops_avg_list[1]),
                 str(congestion_data.c_iops_avg_list[2]),
                 str(congestion_data.g_freq_dict[
                     DeviceDataCollector.STATUS_NORMAL]),
                 str(congestion_data.g_freq_dict[
                     DeviceDataCollector.STATUS_BUILDING]),
                 str(congestion_data.g_freq_dict[
                     DeviceDataCollector.STATUS_CONGESTED]),
                 str(congestion_data.g_await_avg_list[0]),
                 str(congestion_data.g_await_avg_list[1]),
                 str(congestion_data.g_await_avg_list[2]))
            ) + '\n'
        )

        # flush the python buffer
        self.csv.flush()

        # make sure the os pushes the data to disk
        os.fsync(self.csv.fileno())

    def generate_status(self):
        # Purge stale devices
        self._flush_stale_devices()

        # Get congestion state
        data = self._determine_congestion_state()
        if self.status_skip_count < self.status_skip_total:
            self.status_skip_count += 1
        else:
            self._log_congestion_status(data)
            self.status_skip_count = 0

        # Send alarm updates to FM if configured and there are guest volumes
        # present (won't be on the standby controller)
        if CONF.cinder_congestion.generate_fm_alarms:
            if data.g_count > 0:
                self._update_fm(CONF.cinder_congestion.fm_alarm_debounce)
            elif data.g_count == 0 and self.last_g_count > 0:
                self._clear_fm()

        # Save the current guest count view
        self.last_g_count = data.g_count

        # Save output
        if CONF.cinder_congestion.output_write_csv:
            self._create_output(CONF.cinder_congestion.output_csv_dir,
                                data)

    def parse_iostats(self, line):
        # LOG.debug(line)
        m = self.ts_regex.match(line)
        if m:
            self.latest_time = m.group(0)

        m = self.device_regex.match(line)
        if m:
            # LOG.debug(line)
            # LOG.debug("%s: %f %f" % (m.group(1) ,
            #                          float(m.group(4)) + float(m.group(5)),
            #                          float(m.group(10))))
            if not (self._is_number(m.group(4)) and
                    self._is_number(m.group(5)) and
                    self._is_number(m.group(10))):
                LOG.error("ValueError: invalid input: r/s = %s, w/s = %s "
                          "await = %s" % (m.group(4), m.group(5), m.group(10)))
            else:
                if not any(s in m.group(1) for s in ['loop', 'ram', 'nb',
                                                     'md', 'scd'] +
                           self.non_cinder_phys_devs):
                    self._update_device_stats(self.latest_time,
                                              m.group(1),
                                              (float(m.group(4)) +
                                               float(m.group(5))),
                                              float(m.group(10)))

# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (c) 2016-2017 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#

import collections
import logging
import pyudev
import math
import operator
import os
import platform
import re
import subprocess

from fm_api import fm_api
from fm_api import constants as fm_constants
from io_monitor import constants
from io_monitor.constants import DOMAIN
from io_monitor.utils.data_collector import DeviceDataCollector
from io_monitor.constants import _
from oslo_config import cfg

ccm_opts = [
    cfg.IntOpt('ssd_small_window_size',
               default=30,
               help=('SSD: Small moving average window size (in seconds).')),
    cfg.IntOpt('ssd_medium_window_size',
               default=60,
               help=('SSD: Medium moving average window size (in seconds).')),
    cfg.IntOpt('ssd_large_window_size',
               default=90,
               help=('SSD: Large moving average window size (in seconds).')),
    cfg.IntOpt('ssd_thresh_sustained_await',
               default=1000,
               help=('SSD: Value required in a moving average window to '
                     'trigger next state.')),
    cfg.IntOpt('ssd_thresh_max_await',
               default=5000,
               help=('SSD: Max await time. Anomalous data readings are clipped'
                     ' to this.')),
    cfg.IntOpt('hdd_small_window_size',
               default=120,
               help=('HDD: Small moving average window size (in seconds).')),
    cfg.IntOpt('hdd_medium_window_size',
               default=180,
               help=('HDD: Medium moving average window size (in seconds).')),
    cfg.IntOpt('hdd_large_window_size',
               default=240,
               help=('HDD: Large moving average window size (in seconds).')),
    cfg.IntOpt('hdd_thresh_sustained_await',
               default=1500,
               help=('HDD: Value required in a moving average window to '
                     'trigger next state.')),
    cfg.IntOpt('hdd_thresh_max_await',
               default=5000,
               help=('HDD: Max await time. Anomalous data readings are clipped'
                     ' to this.')),
    cfg.StrOpt('log_level',
               default='INFO',
               choices=('ERROR', 'WARN', 'INFO', 'DEBUG'),
               help=('Monitor debug level. Note: global level must be'
                     ' equialent or lower.')),
    cfg.FloatOpt('status_log_rate_modifier', default=0.2,
                 help=('Modify how often status messages appear in the log.'
                       '0.0 is never, 1.0 is for every iostat execution.')),
    cfg.BoolOpt('generate_fm_alarms', default=True,
                help=('Enable FM Alarm generation')),
    cfg.IntOpt('fm_alarm_debounce', default=5,
               help=('Number of consecutive same congestion states seen '
                     'before raising/clearing alarms.')),
    cfg.BoolOpt('output_write_csv', default=False,
                help=('Write monitor data to a csv for analysis')),
    cfg.StrOpt('output_csv_dir', default='/tmp',
               help=('Directory where monitor output will be located.')),
]

CONF = cfg.CONF
CONF.register_opts(ccm_opts, group="cinder_congestion")

LOG = logging.getLogger(DOMAIN)


class CinderCongestionMonitor(object):
    # Congestion States
    STATUS_NORMAL = "Normal"
    STATUS_BUILDING = "Building"
    STATUS_CONGESTED = "Limiting"

    # disk type
    CINDER_DISK_SSD = 0
    CINDER_DISK_HDD = 1

    def __init__(self):
        # Setup logging
        level_dict = {'ERROR': logging.ERROR,
                      'WARN': logging.WARN,
                      'INFO': logging.INFO,
                      'DEBUG': logging.DEBUG}

        if CONF.cinder_congestion.log_level in level_dict.keys():
            LOG.setLevel(level_dict[CONF.cinder_congestion.log_level])
        else:
            LOG.setLevel(logging.INFO)

        LOG.info("Initializing %s..." % self.__class__.__name__)

        # DRBD file
        self.drbd_file = '/etc/drbd.d/drbd-cinder.res'

        # iostat parsing regex
        self.ts_regex = re.compile(r"(\d{2}/\d{2}/\d{2,4}) "
                                   "(\d{2}:\d{2}:\d{2})")
        self.device_regex = re.compile(
            r"(\w+-?\w+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)"
            "\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+"
            "(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+(\d+.\d+)")

        # window sizes
        self.s_window_sec = CONF.cinder_congestion.ssd_small_window_size
        self.m_window_sec = CONF.cinder_congestion.ssd_medium_window_size
        self.l_window_sec = CONF.cinder_congestion.ssd_large_window_size

        # state variables
        self.latest_time = None
        self.congestion_status = self.STATUS_NORMAL

        # init data collector
        self.device_dict = {}

        # devices
        self.phys_cinder_device = None
        self.base_cinder_devs = []
        self.base_cinder_tracking_devs = []
        self.non_cinder_dynamic_devs = ['drbd0', 'drbd1', 'drbd2', 'drbd3',
                                        'drbd5']
        self.non_cinder_phys_devs = []

        # set the default operational scenarios
        self.await_minimal_spike = CONF.cinder_congestion.ssd_thresh_max_await
        self.await_sustained_congestion = (
            CONF.cinder_congestion.ssd_thresh_sustained_await)

        # FM
        self.fm_api = fm_api.FaultAPIs()
        self.fm_state_count = collections.Counter()

        # CSV handle
        self.csv = None

        # status logging
        self.status_skip_count = 0

        # to compare with current g_count
        self.last_g_count = 0

        message_rate = math.ceil(60 / (CONF.wait_time+1))
        self.status_skip_total = math.ceil(
            message_rate/(message_rate *
                          CONF.cinder_congestion.status_log_rate_modifier))
        LOG.info("Display status message at %d per minute..." %
                 (message_rate *
                  CONF.cinder_congestion.status_log_rate_modifier))

        # Clear any exiting alarms
        self._clear_fm()

    def _is_number(self, s):
        try:
            float(s)
            return True
        except ValueError:
            return False

    def command(self, arguments, **kwargs):
        """ Execute e command and capture stdout, stderr & return code """
        process = subprocess.Popen(
            arguments,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            **kwargs)
        out, err = process.communicate()
        return out, err, process.returncode

    def device_path_to_device_node(self, device_path):
        try:
            output, _, _ = self.command(["udevadm", "settle", "-E",
                                        device_path])
            out, err, retcode = self. command(["readlink", "-f", device_path])
            out = out.rstrip()
        except Exception as e:
            return None

        return out

    def _get_disk_type(self, device_node):
        if device_node:
            proc_device_file = '/sys/block/' + device_node + \
                               '/queue/rotational'
            if os.path.exists(proc_device_file):
                with open(proc_device_file) as fileobject:
                    for line in fileobject:
                        return int(line.rstrip())

        # If the disk is unknown assume an SSD.
        return self.CINDER_DISK_SSD


    def _is_cinder_related_device(self,device_node):
        name = ""
        if device_node:
            proc_device_file = '/sys/block/' + device_node + \
                               '/dm/name'

            if os.path.exists(proc_device_file):
                with open(proc_device_file) as fileobject:
                    for line in fileobject:
                        name = line.rstrip()

        if constants.CINDER_DM_PREFIX in name:
            return True

        return False

    def _is_cinder_backing_device(self, device_node):
        name = ""
        if device_node:
            proc_device_file = '/sys/block/' + device_node + \
                               '/dm/name'
            if os.path.exists(proc_device_file):
                with open(proc_device_file) as fileobject:
                    for line in fileobject:
                        name = line.rstrip()

        if any(s in name for s in ['pool', 'anchor']):
            if device_node not in self.base_cinder_devs:
                self.base_cinder_devs.append(device_node)
                if any(s in name for s in ['tdata', 'tmeta']):
                    if device_node not in self.base_cinder_tracking_devs:
                        self.base_cinder_tracking_devs.append(device_node)

                LOG.info("Cinder Base Devices = %s; Tracking %s" % (
                    self.base_cinder_devs, self.base_cinder_tracking_devs))
            return True

        return False

    def _determine_cinder_devices(self):
        # Check to see if we have DRBD device we are syncing
        if os.path.exists(self.drbd_file):

            # grab the data
            with open(self.drbd_file) as fileobject:

                drbd_dev_regex = re.compile(r"device\s+/dev/(\w+);")
                drbd_disk_path_regex = re.compile(
                    r"disk\s+\"(/dev/disk/by-path/(.+))\";")
                drbd_disk_node_regex = re.compile(r"/dev/(\w+)")
                partition_regex = re.compile(r"(sd\w+)\d+")

                for line in fileobject:
                    m = drbd_dev_regex.match(line.strip())
                    if m:
                        self.base_cinder_devs.append(m.group(1))

                    m = drbd_disk_path_regex.match(line.strip())
                    if m:
                        drbd_disk = self.device_path_to_device_node(m.group(1))

                        drbd_disk_sd = drbd_disk_node_regex.match(drbd_disk)
                        if drbd_disk_sd:
                            self.base_cinder_devs.append(drbd_disk_sd.group(1))

                            d = partition_regex.match(drbd_disk_sd.group(1))
                            if d:
                                self.phys_cinder_device = d.group(1)
                                self.base_cinder_devs.append(d.group(1))

            # Which host OS?
            if platform.linux_distribution()[0] == constants.WRLINUX:
                dm_major = 252
            else:
                dm_major = 253

            # Grab the device mapper devices and pull out the base cinder
            # devices
            dmsetup_regex = re.compile(r'^([\w-]+)\s+\((\d+):(\d+)\)')

            dmsetup_command = 'dmsetup ls'
            dmsetup_process = subprocess.Popen(dmsetup_command,
                                               stdout=subprocess.PIPE,
                                               shell=True)
            dmsetup_output = dmsetup_process.stdout.read()
            lines = dmsetup_output.split('\n')
            for l in lines:
                m = dmsetup_regex.match(l.strip())
                if m:
                    if m.group(2) == str(dm_major):
                        # LOG.debug("%s %s %s" % (m.group(1),
                        #                         m.group(2),
                        #                         m.group(3)))
                        if constants.CINDER_DM_PREFIX in m.group(1):
                            if 'pool' in m.group(1) or 'anchor' in m.group(1):
                                self.base_cinder_devs.append(
                                    "dm-" + m.group(3))
                            if 'tdata' in m.group(1) or 'tmeta' in m.group(1):
                                self.base_cinder_tracking_devs.append(
                                    "dm-" + m.group(3))
                        else:
                            self.non_cinder_dynamic_devs.append(
                                "dm-" + m.group(3))

            # If the tracking devs are non existant, then we didn't find any
            # thin pool entries. Therefore we are thickly provisioned and need
            # to track the physical device
            if len(self.base_cinder_tracking_devs) == 0:
                self.base_cinder_tracking_devs.append(
                    self.phys_cinder_device)

        # Use UDEV info to grab all phyical disks
        context = pyudev.Context()
        for device in context.list_devices(subsystem='block',
                                           DEVTYPE='disk'):
            if device['MAJOR'] == '8':
                device = str(os.path.basename(device['DEVNAME']))
                if device != self.phys_cinder_device:
                    self.non_cinder_phys_devs.append(device)

    def _update_device_stats(self, ts, device, current_iops, current_await):
        if device not in self.device_dict:
             # For AIO systems nova-local will be provisioned later and
             # differently based on the instance_backing value for the compute
             # functionality. Check for cinder specific dm devices and ignore
             # all others
            if not self._is_cinder_related_device(device):
                return
            self._is_cinder_backing_device(device)
            self.device_dict.update(
                {device: DeviceDataCollector(
                    device,
                    [DeviceDataCollector.DATA_IOPS,
                     DeviceDataCollector.DATA_AWAIT],
                    self.s_window_sec,
                    self.m_window_sec,
                    self.l_window_sec)})
            self.device_dict[device].set_data_caps(
                DeviceDataCollector.DATA_AWAIT,
                self.await_minimal_spike)
            self.device_dict[device].set_congestion_thresholds(
                self.await_minimal_spike,
                self.await_sustained_congestion)

        self.device_dict[device].update_data(ts,
                                             DeviceDataCollector.DATA_IOPS,
                                             current_iops)
        self.device_dict[device].update_data(ts,
                                             DeviceDataCollector.DATA_AWAIT,
                                             current_await)
        self.device_dict[device].update_congestion_status()

    def is_system_monitorable(self):
        if not os.path.exists(self.drbd_file):
            LOG.error("%s does not exist" % self.drbd_file)
            return False

        # Discover devices on this host
        self._determine_cinder_devices()

        # Get the cinder disk type and set the monitor values accordingly
        disk_type = self._get_disk_type(self.phys_cinder_device)
        if disk_type:
            self.s_window_sec = CONF.cinder_congestion.hdd_small_window_size
            self.m_window_sec = CONF.cinder_congestion.hdd_medium_window_size
            self.l_window_sec = CONF.cinder_congestion.hdd_large_window_size
            self.await_minimal_spike = (
                CONF.cinder_congestion.hdd_thresh_max_await)
            self.await_sustained_congestion = (
                CONF.cinder_congestion.hdd_thresh_sustained_await)
        else:
            self.s_window_sec = CONF.cinder_congestion.ssd_small_window_size
            self.m_window_sec = CONF.cinder_congestion.ssd_medium_window_size
            self.l_window_sec = CONF.cinder_congestion.ssd_large_window_size
            self.await_minimal_spike = (
                CONF.cinder_congestion.ssd_thresh_max_await)
            self.await_sustained_congestion = (
                CONF.cinder_congestion.ssd_thresh_sustained_await)

        LOG.info("Physical Cinder Disk = %s - %s" %
                 (self.phys_cinder_device,
                  "HDD" if disk_type else "SSD"))
        LOG.info("Cinder Base Devices = %s; Tracking %s" % (
            self.base_cinder_devs, self.base_cinder_tracking_devs))
        LOG.info("Non-Cinder Devices = %s" % (
            self.non_cinder_dynamic_devs + self.non_cinder_phys_devs))

        return True

    def get_operational_thresholds(self):
        return (self.await_minimal_spike,
                self.await_sustained_congestion)

    def set_operational_thresholds(self,
                                   await_minimal_spike,
                                   await_sustained_congestion):
        if await_minimal_spike:
            self.await_minimal_spike = await_minimal_spike
        if await_sustained_congestion:
            self.await_sustained_congestion = await_sustained_congestion

    def _flush_stale_devices(self):
        for d in self.device_dict.keys():
            if self.device_dict[d].is_data_stale(self.latest_time):
                self.device_dict.pop(d, None)

    def _log_device_data_windows(self, device):
        LOG.debug("%-6s: %s %s" % (
            device,
            self.device_dict[device].get_element_windows_avg_string(
                DeviceDataCollector.DATA_AWAIT),
            self.device_dict[device].get_element_windows_avg_string(
                DeviceDataCollector.DATA_IOPS)))

    def _log_congestion_status(self, congestion_data):
        congestion_data.c_freq_dict.update(
            dict.fromkeys(
                set(['N', 'B', 'L']).difference(
                    congestion_data.c_freq_dict), 0))
        congestion_data.g_freq_dict.update(
            dict.fromkeys(
                set(['N', 'B', 'L']).difference(
                    congestion_data.g_freq_dict), 0))

        LOG.info("Status (%-8s): Cinder Devs IOPS [ %10.2f, %10.2f, %10.2f ] "
                 "Guests Counts %s; Guest Await[ %10.2f, %10.2f, %10.2f ]" % (
                     congestion_data.status,
                     congestion_data.c_iops_avg_list[0],
                     congestion_data.c_iops_avg_list[1],
                     congestion_data.c_iops_avg_list[2],
                     dict(congestion_data.g_freq_dict),
                     congestion_data.g_await_avg_list[0],
                     congestion_data.g_await_avg_list[1],
                     congestion_data.g_await_avg_list[2]))

    def _determine_congestion_state(self):

        # Analyze devices
        cinder_congestion_freq = collections.Counter()
        cinder_iops_avg = [0.0, 0.0, 0.0]
        guest_congestion_freq = collections.Counter()
        guest_await_avg = [0.0, 0.0, 0.0]

        for d, dc in self.device_dict.iteritems():
            if d in self.base_cinder_devs:
                if d in self.base_cinder_tracking_devs:
                    cinder_congestion_freq.update(dc.get_congestion_status())
                    cinder_iops_avg = map(operator.add,
                                          cinder_iops_avg,
                                          dc.get_element_windows_avg_list(
                                              DeviceDataCollector.DATA_IOPS))
                    # LOG.debug("C: %s " % cinder_iops_avg)
                    # self._log_device_data_windows(d)

            elif d not in (self.base_cinder_devs +
                           self.non_cinder_dynamic_devs +
                           self.non_cinder_phys_devs):
                guest_congestion_freq.update(
                    dc.get_congestion_status(debug=True))
                guest_await_avg = map(operator.add,
                                      guest_await_avg,
                                      dc.get_element_windows_avg_list(
                                          DeviceDataCollector.DATA_AWAIT))
                # LOG.debug("G: %s " % guest_await_avg)
                # self._log_device_data_windows(d)

        if list(cinder_congestion_freq.elements()):
            cinder_iops_avg[:] = [i/len(list(
                cinder_congestion_freq.elements())) for i in cinder_iops_avg]

        if list(guest_congestion_freq.elements()):
            guest_await_avg[:] = [i/len(list(
                guest_congestion_freq.elements())) for i in guest_await_avg]

        self.congestion_status = self.STATUS_NORMAL
        if DeviceDataCollector.STATUS_BUILDING in guest_congestion_freq:
            self.congestion_status = self.STATUS_BUILDING
        if DeviceDataCollector.STATUS_CONGESTED in guest_congestion_freq:
            self.congestion_status = self.STATUS_CONGESTED

        congestion_data = collections.namedtuple("congestion_data",
                                                 ["timestamp", "status",
                                                  "c_freq_dict",
                                                  "c_iops_avg_list",
                                                  "g_count",
                                                  "g_freq_dict",
                                                  "g_await_avg_list"])

        return congestion_data(self.latest_time,
                               self.congestion_status,
                               cinder_congestion_freq,
                               cinder_iops_avg,
                               sum(guest_congestion_freq.values()),
                               guest_congestion_freq,
                               guest_await_avg)

    def _clear_fm(self):
        building = fm_constants.FM_ALARM_ID_STORAGE_CINDER_IO_BUILDING
        limiting = fm_constants.FM_ALARM_ID_STORAGE_CINDER_IO_LIMITING

        entity_instance_id = "cinder_io_monitor"
        ccm_alarm_ids = [building, limiting]

        existing_alarms = []
        for alarm_id in ccm_alarm_ids:
            alarm_list = self.fm_api.get_faults_by_id(alarm_id)
            if not alarm_list:
                continue
            for alarm in alarm_list:
                existing_alarms.append(alarm)

        if len(existing_alarms) > 1:
            LOG.warn("WARNING: we have more than one existing alarm")

        for a in existing_alarms:
            self.fm_api.clear_fault(a.alarm_id, entity_instance_id)
            LOG.info(
                _("Clearing congestion alarm {} - severity: {}, "
                  "reason: {}, service_affecting: {}").format(
                      a.uuid, a.severity, a.reason_text, True))

    def _update_fm(self, debounce_count, override=None):

        building = fm_constants.FM_ALARM_ID_STORAGE_CINDER_IO_BUILDING
        limiting = fm_constants.FM_ALARM_ID_STORAGE_CINDER_IO_LIMITING

        if override:
            self.congestion_status = override

        # Update the status count
        self.fm_state_count.update(self.congestion_status[0])

        # Debounce alarms: If I have more than one congestion type then clear
        # the counts as we have crossed a threshold
        if len(self.fm_state_count) > 1:
            self.fm_state_count.clear()
            self.fm_state_count.update(self.congestion_status[0])
            return

        # Debounce alarms: Make sure we have see this alarm state for a specifc
        # number of samples
        count = self.fm_state_count.itervalues().next()
        if count < debounce_count:
            return

        # We are past the debounce state. Now take action.
        entity_instance_id = "cinder_io_monitor"
        ccm_alarm_ids = [building, limiting]

        existing_alarms = []
        for alarm_id in ccm_alarm_ids:
            alarm_list = self.fm_api.get_faults_by_id(alarm_id)
            if not alarm_list:
                continue
            for alarm in alarm_list:
                existing_alarms.append(alarm)

        if len(existing_alarms) > 1:
            LOG.warn("WARNING: we have more than one existing alarm")

        if self.congestion_status is self.STATUS_NORMAL:
            for a in existing_alarms:
                self.fm_api.clear_fault(a.alarm_id, entity_instance_id)
                LOG.info(
                    _("Clearing congestion alarm {} - severity: {}, "
                      "reason: {}, service_affecting: {}").format(
                          a.uuid, a.severity, a.reason_text, True))

        elif self.congestion_status is self.STATUS_BUILDING:
            alarm_is_raised = False
            for a in existing_alarms:
                if a.alarm_id != building:
                    self.fm_api.clear_fault(a.alarm_id, entity_instance_id)
                    LOG.info(
                        _("Clearing congestion alarm {} - severity: {}, "
                          "reason: {}, service_affecting: {}").format(
                              a.uuid, a.severity, a.reason_text, True))
                else:
                    alarm_is_raised = True

            if not alarm_is_raised:
                severity = fm_constants.FM_ALARM_SEVERITY_MAJOR
                reason_text = constants.ALARM_REASON_BUILDING

                fault = fm_api.Fault(
                    alarm_id=building,
                    alarm_type=fm_constants.FM_ALARM_TYPE_2,
                    alarm_state=fm_constants.FM_ALARM_STATE_SET,
                    entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER,
                    entity_instance_id=entity_instance_id,
                    severity=severity,
                    reason_text=reason_text,
                    probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8,
                    proposed_repair_action=constants.REPAIR_ACTION_MAJOR_ALARM,
                    service_affecting=True)
                alarm_uuid = self.fm_api.set_fault(fault)
                if alarm_uuid:
                    LOG.info(
                        _("Created congestion alarm {} - severity: {}, "
                          "reason: {}, service_affecting: {}").format(
                              alarm_uuid, severity, reason_text, True))
                else:
                    LOG.error(
                        _("Failed to create congestion alarm - severity: {},"
                          "reason: {}, service_affecting: {}").format(
                              severity, reason_text, True))

        elif self.congestion_status is self.STATUS_CONGESTED:
            alarm_is_raised = False
            for a in existing_alarms:
                if a.alarm_id != limiting:
                    self.fm_api.clear_fault(a.alarm_id, entity_instance_id)
                    LOG.info(
                        _("Clearing congestion alarm {} - severity: {}, "
                          "reason: {}, service_affecting: {}").format(
                              a.uuid, a.severity, a.reason_text, True))
                else:
                    alarm_is_raised = True

            if not alarm_is_raised:
                severity = fm_constants.FM_ALARM_SEVERITY_CRITICAL
                reason_text = constants.ALARM_REASON_CONGESTED
                repair = constants.REPAIR_ACTION_CRITICAL_ALARM
                fault = fm_api.Fault(
                    alarm_id=limiting,
                    alarm_type=fm_constants.FM_ALARM_TYPE_2,
                    alarm_state=fm_constants.FM_ALARM_STATE_SET,
                    entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER,
                    entity_instance_id=entity_instance_id,
                    severity=severity,
                    reason_text=reason_text,
                    probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8,
                    proposed_repair_action=repair,
                    service_affecting=True)
                alarm_uuid = self.fm_api.set_fault(fault)
                if alarm_uuid:
                    LOG.info(
                        _("Created congestion alarm {} - severity: {}, "
                          "reason: {}, service_affecting: {}").format(
                              alarm_uuid, severity, reason_text, True))
                else:
                    LOG.error(
                        _("Failed to congestion storage alarm - severity: {},"
                          "reason: {}, service_affecting: {}").format(
                              severity, reason_text, True))

    def _create_output(self, output_dir, congestion_data):
        if not self.csv:
            LOG.info("Creating output")
            if os.path.exists(output_dir):
                if output_dir.endswith('/'):
                    fn = output_dir + 'ccm.csv'
                else:
                    fn = output_dir + '/ccm.csv'
            else:
                fn = '/tmp/ccm.csv'
            try:
                self.csv = open(fn, 'w')
            except Exception as e:
                raise e

            self.csv.write("Timestamp, Congestion Status, "
                           "Cinder Devs Normal, "
                           "Cinder Devs Building, Cinder Devs Limiting,"
                           "Cinder IOPS Small, "
                           "Cinder IOPS Med, Cinder IOPS Large,"
                           "Guest Vols Normal, "
                           "Guest Vols Building, Guest Vols Limiting,"
                           "Guest Await Small, "
                           "Guest Await Med, Guest Await Large")
            LOG.info("Done writing")

        congestion_data.c_freq_dict.update(
            dict.fromkeys(set(['N', 'B', 'L']).difference(
                congestion_data.c_freq_dict), 0))
        congestion_data.g_freq_dict.update(
            dict.fromkeys(set(['N', 'B', 'L']).difference(
                congestion_data.g_freq_dict), 0))

        self.csv.write(
            ",".join(
                (str(congestion_data.timestamp),
                 str(congestion_data.status[0]),
                 str(congestion_data.c_freq_dict[
                     DeviceDataCollector.STATUS_NORMAL]),
                 str(congestion_data.c_freq_dict[
                     DeviceDataCollector.STATUS_BUILDING]),
                 str(congestion_data.c_freq_dict[
                     DeviceDataCollector.STATUS_CONGESTED]),
                 str(congestion_data.c_iops_avg_list[0]),
                 str(congestion_data.c_iops_avg_list[1]),
                 str(congestion_data.c_iops_avg_list[2]),
                 str(congestion_data.g_freq_dict[
                     DeviceDataCollector.STATUS_NORMAL]),
                 str(congestion_data.g_freq_dict[
                     DeviceDataCollector.STATUS_BUILDING]),
                 str(congestion_data.g_freq_dict[
                     DeviceDataCollector.STATUS_CONGESTED]),
                 str(congestion_data.g_await_avg_list[0]),
                 str(congestion_data.g_await_avg_list[1]),
                 str(congestion_data.g_await_avg_list[2]))
            ) + '\n'
        )

        # flush the python buffer
        self.csv.flush()

        # make sure the os pushes the data to disk
        os.fsync(self.csv.fileno())

    def generate_status(self):
        # Purge stale devices
        self._flush_stale_devices()

        # Get congestion state
        data = self._determine_congestion_state()
        if self.status_skip_count < self.status_skip_total:
            self.status_skip_count += 1
        else:
            self._log_congestion_status(data)
            self.status_skip_count = 0

        # Send alarm updates to FM if configured and there are guest volumes
        # present (won't be on the standby controller)
        if CONF.cinder_congestion.generate_fm_alarms:
            if data.g_count > 0:
                self._update_fm(CONF.cinder_congestion.fm_alarm_debounce)
            elif data.g_count == 0 and self.last_g_count > 0:
                self._clear_fm()

        # Save the current guest count view
        self.last_g_count = data.g_count

        # Save output
        if CONF.cinder_congestion.output_write_csv:
            self._create_output(CONF.cinder_congestion.output_csv_dir,
                                data)

    def parse_iostats(self, line):
        # LOG.debug(line)
        m = self.ts_regex.match(line)
        if m:
            self.latest_time = m.group(0)

        m = self.device_regex.match(line)
        if m:
            # LOG.debug(line)
            # LOG.debug("%s: %f %f" % (m.group(1) ,
            #                          float(m.group(4)) + float(m.group(5)),
            #                          float(m.group(10))))
            if not (self._is_number(m.group(4)) and
                    self._is_number(m.group(5)) and
                    self._is_number(m.group(10))):
                LOG.error("ValueError: invalid input: r/s = %s, w/s = %s "
                          "await = %s" % (m.group(4), m.group(5), m.group(10)))
            else:
                if not any(s in m.group(1) for s in ['loop', 'ram', 'nb',
                                                     'md', 'scd'] +
                           self.non_cinder_phys_devs):
                    self._update_device_stats(self.latest_time,
                                              m.group(1),
                                              (float(m.group(4)) +
                                               float(m.group(5))),
                                              float(m.group(10)))

# vim: tabstop=4 shiftwidth=4 softtabstop=4 # # Copyright (c) 2016-2017 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # # Congestion States # disk type # Setup logging # DRBD file # iostat parsing regex # window sizes # state variables # init data collector # devices # set the default operational scenarios # FM # CSV handle # status logging # to compare with current g_count # Clear any exiting alarms Execute e command and capture stdout, stderr & return code # If the disk is unknown assume an SSD. # Check to see if we have DRBD device we are syncing # grab the data # Which host OS? # Grab the device mapper devices and pull out the base cinder # devices # LOG.debug("%s %s %s" % (m.group(1), #                         m.group(2), #                         m.group(3))) # If the tracking devs are non existant, then we didn't find any # thin pool entries. Therefore we are thickly provisioned and need # to track the physical device # Use UDEV info to grab all phyical disks # For AIO systems nova-local will be provisioned later and # differently based on the instance_backing value for the compute # functionality. Check for cinder specific dm devices and ignore # all others # Discover devices on this host # Get the cinder disk type and set the monitor values accordingly # Analyze devices # LOG.debug("C: %s " % cinder_iops_avg) # self._log_device_data_windows(d) # LOG.debug("G: %s " % guest_await_avg) # self._log_device_data_windows(d) # Update the status count # Debounce alarms: If I have more than one congestion type then clear # the counts as we have crossed a threshold # Debounce alarms: Make sure we have see this alarm state for a specifc # number of samples # We are past the debounce state. Now take action. # flush the python buffer # make sure the os pushes the data to disk # Purge stale devices # Get congestion state # Send alarm updates to FM if configured and there are guest volumes # present (won't be on the standby controller) # Save the current guest count view # Save output # LOG.debug(line) # LOG.debug(line) # LOG.debug("%s: %f %f" % (m.group(1) , #                          float(m.group(4)) + float(m.group(5)), #                          float(m.group(10))))