Edit File: lvechartmain.py
# coding=utf-8 # # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT from __future__ import absolute_import from __future__ import division from builtins import range from clcommon.cpapi.pluginlib import getuser from lvestats.lib.uidconverter import uid_to_username from lvestats.lib.lveinfolib import HistoryShow, FIELD_LIMIT from lvestats.lib.chart import ChartMain from lvestats.lib.commons import dateutil, sizeutil from lvestats.lib.chart.util import X_LEGEND_POINTS from lvestats.lib import lveinfolib_gov from lvestats.lib.commons import func def _4k_page_to_bytes(v): return (v or 0) * 4 * 1024 SHOW_COLUMNS = ('From', 'aCPU', 'lVMem', 'aPMem', 'lPMem', 'NprocF', 'PMemF', 'VMemF', 'aIO', 'lIO', 'lIOPS', 'IOPSf', 'IOf', 'lCPU', 'CPUf', 'aVMem', 'aEP', 'lEP', 'aNproc', 'lNproc', 'EPf', 'aIOPS', 'mIOPS',) class LveChart(ChartMain): def __init__(self, config): super(LveChart, self).__init__('lvechart', 'Creates a chart representing usage pattern for LVE/user', config) self.MINIMAL_TIME_STEP = int(config.get('aggregation_period', 60)) self.BYTES_IN_MB = 1024 * 1024 self.governor_mode = func.get_governor_mode() self.user_ignore = False def customize_parser(self, parser): is_admin = getuser() == 'root' id_user_group = parser.add_mutually_exclusive_group(required=is_admin) id_user_group.add_argument('--id', help='LVE id -- will display record only for that LVE id', dest='user_id', type=int) id_user_group.add_argument('--user', help='Use username instead of LVE id, and show only record for that user', dest='user_name', default=None) return parser def append_dbgov_data_to_dict(self, dictionary, governor_mode_is_all, num, dbcpu_sum, dbio_sum, _lve_time_index): """ :param dictionary: dict to wich we append dbgov data :param governor_mode_is_all: :param num: number of dbgov data points near this LVEStatsHistory time :param dbcpu_sum: sum value of dbgov cpu near this LVEStatsHistory time :param dbio_sum: sum value of dbgov io near this LVEStatsHistory time :param _lve_time_index: index of lve point :return: """ if num > 0: if governor_mode_is_all: # append mean value dictionary["dbcpu"].append(min(dbcpu_sum / num, dictionary['acpu'][_lve_time_index])) try: dictionary["dbio"].append( min(dbio_sum / num * self.BYTES_IN_MB, dictionary['aio'][_lve_time_index])) except (KeyError, IndexError): # if lve_version <= 4 there is no by_key['aio'] pass else: dictionary["dbcpu"].append(dbcpu_sum / num) dictionary["dbio"].append(dbio_sum / num * self.BYTES_IN_MB) # when there was no dbgov data near this LVEStatsHistory time else: dictionary["dbcpu"].append(0) dictionary["dbio"].append(0) @staticmethod def make_lvedata_with_idle(lvedata, show_columns, period_from, period_to, time_step): def get_time_from(index): try: return int(lvedata[index][show_columns.index('From')]) except IndexError: return 0 lvedata_with_idle = list() # prepare data for correct show with server idle indx_ = 0 for period_from_ in range(period_from, period_to, time_step): if lvedata and period_from_ - time_step / 2 < get_time_from(indx_) <= period_from_ + time_step / 2: lvedata_with_idle.append(lvedata[indx_]) indx_ += 1 else: line_ = list() for item in show_columns: if item == 'From': line_.append(period_from_) elif item in FIELD_LIMIT: # calculate limit for limit fields if lvedata: # use previous limit line_.append(lvedata[max(0, indx_ - 1)][show_columns.index(item)]) else: line_.append(0) else: line_.append(0) lvedata_with_idle.append(line_) return lvedata_with_idle def get_chart_data(self, engine, from_ts, to_ts, server, user_id, show_all=False): username = uid_to_username( uid=user_id, local_server_id=self.cfg.get("server_id", "localhost"), server_id=server, db_engine=engine ) self.user_ignore = func.get_governor_ignore_for_user(username) utc_from = dateutil.local_to_gm(from_ts) utc_to = dateutil.local_to_gm(to_ts) period_sec, time_step = self.get_time_step(utc_from, utc_to) lvedata_with_idle = self.load_data(engine, server, time_step, user_id, utc_from, utc_to) by_key_ = {col.lower(): [row[col_index] for row in lvedata_with_idle] for col_index, col in enumerate(SHOW_COLUMNS)} by_key = LveChart.convert_lvedata_to_dict(by_key_) lve_times = by_key['from'] del by_key['from'] show_columns = ('ts', 'cpu', 'lcpu', 'read', 'lread', 'write', 'lwrite') if self.governor_mode != "none" and not self.user_ignore: dbdata = lveinfolib_gov.HistoryShowDBGov( engine, utc_from, utc_to, uid=user_id, server_id=server, show_columns=show_columns, cfg=self.cfg, ).history_dbgov_show() data_collected = LveChart.convert_dbdata_to_dict(dbdata, show_columns) dbtimes = data_collected['ts'] del data_collected['ts'] time_step2 = time_step / 2.0 # append dbgov stats to the nearest HistoryShow points for lve_time_index, lve_time in enumerate(lve_times): dbcpu, dbio = 0, 0 tmp_dbtimes_list = dbtimes[:] # temporary list for iteration for dbtime_index, dbtime in enumerate(tmp_dbtimes_list): # check if dbgov data is near this LVEStatsHistory time if dbtime < lve_time + time_step2: dbcpu += data_collected['cpu'].pop(0) dbio += data_collected["read"].pop(0) + data_collected["write"].pop(0) dbtimes.pop(0) else: self.append_dbgov_data_to_dict( by_key, self.governor_mode == "all", dbtime_index, dbcpu, dbio, lve_time_index) break else: self.append_dbgov_data_to_dict( by_key, self.governor_mode == "all", len(tmp_dbtimes_list), dbcpu, dbio, lve_time_index) return by_key, lve_times, period_sec def load_data(self, engine, server, time_step, user_id, utc_from, utc_to): history_show = HistoryShow(dbengine=engine, period_from=utc_from, period_to=utc_to, uid=user_id, show_columns=SHOW_COLUMNS, server_id=server, time_unit=time_step) lvedata = list(history_show.proceed()) lvedata_with_idle = self.make_lvedata_with_idle(lvedata, SHOW_COLUMNS, history_show.period_from, history_show.period_to, time_step) return lvedata_with_idle def get_time_step(self, utc_from, utc_to): dt = utc_to - utc_from period_sec = dt.total_seconds() return period_sec, self.MINIMAL_TIME_STEP @staticmethod def _add_faults_graph(renderer, data_collected, times, lve_version): legend = { 'CPUf': ('CPUf', '#8DB600'), 'EPf': ('EPf', 'green',), 'VMemF': ('VMemF', '#00DDFF'), 'NprocF': ('NprocF', 'red'), 'PMemF': ('PMemF', 'blue'), 'IOf': ('IOf', '#9966CC')} if lve_version >= 8: legend.update({ 'IOPSf': ('IOPSf', '#007FFF'), }) legend_keys = tuple(key.lower() for key in list(legend.keys())) faults_present = any(tuple(any(data_collected[key]) for key in legend_keys)) message = 'No Faults' if not faults_present else None renderer.add_graph( data_collected, 'Faults', legend=legend, x_values=times, min_y=0, y_legend_converter=sizeutil.convert_powers_of_1000_for_graph, message=message) def add_graphs(self, renderer, data_collected, times, lve_version, show_all, is_user=False): title_suffix = self._get_title_suffix(data_collected) is_lcpu_limited = any(data_collected['lcpu']) if is_lcpu_limited or show_all: if is_user and self.is_normalized_user_cpu and is_lcpu_limited: self._normalize_cpu(data_collected) cpu_legend = {'aCPU': ('average', 'green'), 'lCPU': ('limit', 'red')} if show_all and not is_lcpu_limited: del cpu_legend['lCPU'] if len(data_collected['dbcpu']) > 0: cpu_legend['dbcpu'] = ('database', 'blue') renderer.add_graph(data_collected, 'CPU Usage' + title_suffix, legend=cpu_legend, x_values=times, min_y=0, unit='%', y_legend_converter=int, faults=('CPUf', 'aCPU', 'lCPU')) is_lvmem_limited = any(data_collected['lvmem']) if is_lvmem_limited or show_all: vmem_legend = { 'lVmem': ('limit', 'red', _4k_page_to_bytes), 'aVmem': ('average', 'green', _4k_page_to_bytes)} if show_all and not is_lvmem_limited: del vmem_legend['lVmem'] renderer.add_graph(data_collected, 'Virtual Memory Usage', legend=vmem_legend, min_y=0, x_values=times, y_legend_converter=sizeutil.convert_bytes_for_graph) is_lpmem_limited = any(data_collected['lpmem']) if is_lpmem_limited or show_all: pmem_legend = { 'lPMem': ('limit', 'red', _4k_page_to_bytes), 'aPMem': ('average', 'green', _4k_page_to_bytes)} if show_all and not is_lpmem_limited: del pmem_legend['lPMem'] renderer.add_graph(data_collected, 'Physical Memory Usage', legend=pmem_legend, x_values=times, min_y=0, y_legend_converter=sizeutil.convert_bytes_for_graph) is_lio_limited = any(data_collected['lio']) if is_lio_limited or show_all: io_legend = { 'lIO': ('limit', 'red'), 'aIO': ('average', 'green')} if show_all and not is_lio_limited: del io_legend['lIO'] if len(data_collected['dbio']) > 0: io_legend['dbio'] = ('database', 'blue') renderer.add_graph(data_collected, 'Input/Output Usage' + title_suffix, legend=io_legend, x_values=times, min_y=0, y_legend_converter=sizeutil.convert_bytes_for_graph, unit='/s', faults=('IOf', 'aIO', 'lIO')) if lve_version > 6: is_liops_limited = any(data_collected['liops']) if is_liops_limited or show_all: iops_legend = { 'lIOPS': ('limit', 'red'), 'aIOPS': ('average', 'green')} if show_all and not is_liops_limited: del iops_legend['lIOPS'] renderer.add_graph(data_collected, 'Io operations', legend=iops_legend, x_values=times, min_y=0, y_legend_converter=sizeutil.convert_powers_of_1000_for_graph, unit='/s', faults=('IOPSf', 'aIOPS', 'lIOPS')) is_lep_limited = any(data_collected['lep']) if is_lep_limited or show_all: ep_legend = { 'lEp': ('limit', 'red'), 'aEp': ('average', 'green')} if show_all and not is_lep_limited: del ep_legend['lEp'] renderer.add_graph(data_collected, 'Entry Processes', legend=ep_legend, x_values=times, min_y=0, y_legend_converter=sizeutil.convert_powers_of_1000_for_graph) is_lnproc_limited = any(data_collected['lnproc']) if is_lnproc_limited or show_all: nproc_legend = { 'lNproc': ('limit', 'red'), 'aNproc': ('average', 'green')} if show_all and not is_lnproc_limited: del nproc_legend['lNproc'] renderer.add_graph(data_collected, 'Processes', legend=nproc_legend, x_values=times, min_y=0, y_legend_converter=sizeutil.convert_powers_of_1000_for_graph) self._add_faults_graph(renderer, data_collected, times, lve_version) renderer.add_common_x_legend(times, X_LEGEND_POINTS) @staticmethod def _normalize_cpu(data_collected): # Magnify aCPU, dbCPU, lCPU to 100% # 1. Find max limit max_limit = max(data_collected['lcpu']) # 2. Calculate New user's data and limits a_cpu_new = [] l_cpu_new = [] db_cpu_new = [] for idx in range(0, len(data_collected['lcpu'])): # New limit l_cpu_new.append(data_collected['lcpu'][idx] * 100.0 / max_limit) # New lve cpu average a_cpu_new.append(data_collected['acpu'][idx] * 100.0 / max_limit) if len(data_collected['dbcpu']) > 0: # New database cpu average db_cpu_new.append(data_collected['dbcpu'][idx] * 100.0 / max_limit) # Store new data data_collected['lcpu'] = l_cpu_new data_collected['acpu'] = a_cpu_new if db_cpu_new: data_collected['dbcpu'] = db_cpu_new def _get_title_suffix(self, data_collected): # <lve use="on|single|off|abusers|all"/> if len(data_collected['dbcpu']) == 0 or self.governor_mode == "none" or self.user_ignore: title_suffix = "" elif self.governor_mode == "all": title_suffix = ". DB usage included" elif self.governor_mode == "abusers": title_suffix = ". DB usage included, only if restricted" elif self.governor_mode == "off": title_suffix = ". DB usage is not limited, nor accounted for" else: # single or on title_suffix = ". DB usage is not accounted as part of LVE" return title_suffix