From 4b1f5a2731179c95f258775fe3e12b973e37712a Mon Sep 17 00:00:00 2001 From: nicolargo Date: Fri, 19 Jun 2015 22:26:33 +0200 Subject: [PATCH] Allow export of Docker and sensors plugins stats to InfluxDB, StatsD... (issue #600) --- NEWS | 1 + glances/core/glances_stats.py | 11 ++++- glances/exports/glances_csv.py | 4 +- glances/exports/glances_export.py | 77 +++++++++++++++++++++++------ glances/exports/glances_influxdb.py | 1 + glances/exports/glances_statsd.py | 1 + glances/outputs/glances_bottle.py | 2 +- glances/plugins/glances_docker.py | 22 +++++++++ glances/plugins/glances_plugin.py | 4 ++ glances/plugins/glances_sensors.py | 4 ++ 10 files changed, 108 insertions(+), 19 deletions(-) diff --git a/NEWS b/NEWS index 00f176aa..bf94e756 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,7 @@ Version 2.5 Enhancements and new features: + * Allow export of Docker and sensors plugins stats to InfluxDB, StatsD... (issue #600) * Server password configuration for the browser mode (issue #500) * Display an error if export is not used in the standalone/client mode (issue #614) diff --git a/glances/core/glances_stats.py b/glances/core/glances_stats.py index 1f039eb6..bd07d290 100644 --- a/glances/core/glances_stats.py +++ b/glances/core/glances_stats.py @@ -129,14 +129,14 @@ class GlancesStats(object): # generate self._exports_list["xxx"] = ... self._exports[export_name] = export_module.Export(args=args, config=self.config) # Log plugins list - logger.debug("Available exports modules list: {0}".format(self.getAllExports())) + logger.debug("Available exports modules list: {0}".format(self.getExportList())) return True def getAllPlugins(self): """Return the plugins list.""" return [p for p in self._plugins] - def getAllExports(self): + def getExportList(self): """Return the exports modules list.""" return [p for p in self._exports] @@ -174,6 +174,13 @@ class GlancesStats(object): """Return all the stats (list).""" return [self._plugins[p].get_raw() for p in self._plugins] + def getAllExports(self): + """ + Return all the stats to be exported (list). + Default behavor is to export all the stat + """ + return [self._plugins[p].get_export() for p in self._plugins] + def getAllAsDict(self): """Return all the stats (dict).""" # Python > 2.6 diff --git a/glances/exports/glances_csv.py b/glances/exports/glances_csv.py index 54ea65c6..0d23f252 100644 --- a/glances/exports/glances_csv.py +++ b/glances/exports/glances_csv.py @@ -68,7 +68,7 @@ class Export(GlancesExport): csv_data = [] # Get the stats - all_stats = stats.getAll() + all_stats = stats.getAllExports() plugins = stats.getAllPlugins() # Loop over available plugin @@ -79,7 +79,7 @@ class Export(GlancesExport): # First line: header if self.first_line: csv_header += ('{0}_{1}_{2}'.format( - plugin, stat[stat['key']], item) for item in stat) + plugin, self.get_item_key(stat), item) for item in stat) # Others lines: stats fieldvalues = stat.values() csv_data += fieldvalues diff --git a/glances/exports/glances_export.py b/glances/exports/glances_export.py index 3bf050b3..6ec8998d 100644 --- a/glances/exports/glances_export.py +++ b/glances/exports/glances_export.py @@ -65,35 +65,84 @@ class GlancesExport(object): 'processcount', 'ip', 'system', - 'uptime'] + 'uptime', + 'sensors', + 'docker'] + + def get_item_key(self, item): + """Return the value of the item 'key'""" + try: + ret = item[item['key']] + except KeyError: + logger.error("No 'key' available in {}".format(item)) + if isinstance(ret, list): + return ret[0] + else: + return ret def update(self, stats): """Update stats to a server. The method builds two lists: names and values and calls the export method to export the stats. + + Be aware that CSV export overwrite this class and use a specific one. """ if not self.export_enable: return False - # Get the stats - all_stats = stats.getAll() + # Get all the stats & limits + all_stats = stats.getAllExports() all_limits = stats.getAllLimits() + # Get the plugins list plugins = stats.getAllPlugins() # Loop over available plugins for i, plugin in enumerate(plugins): if plugin in self.plugins_to_export(): - if isinstance(all_stats[i], list): - for item in all_stats[i]: - item.update(all_limits[i]) - export_names = list('{0}.{1}'.format(item[item['key']], key) - for key in item.keys()) - export_values = list(item.values()) - self.export(plugin, export_names, export_values) - elif isinstance(all_stats[i], dict): - export_names = list(all_stats[i].keys()) + list(all_limits[i].keys()) - export_values = list(all_stats[i].values()) + list(all_limits[i].values()) - self.export(plugin, export_names, export_values) + if isinstance(all_stats[i], dict): + all_stats[i].update(all_limits[i]) + elif isinstance(all_stats[i], list): + all_stats[i] += all_limits[i] + else: + continue + export_names, export_values = self.__build_export(all_stats[i]) + self.export(plugin, export_names, export_values) return True + + def __build_export(self, stats): + """Build the export lists""" + export_names = [] + export_values = [] + + if isinstance(stats, dict): + # Stats is a dict + # Is there a key ? + if 'key' in list(stats.keys()): + pre_key = '{}.'.format(stats[stats['key']]) + else: + pre_key = '' + # Walk through the dict + for key, value in stats.iteritems(): + if isinstance(value, list): + try: + value = value[0] + except IndexError: + value = '' + if isinstance(value, dict): + item_names, item_values = self.__build_export(value) + item_names = [pre_key + key.lower() + str(i) for i in item_names] + export_names += item_names + export_values += item_values + else: + export_names.append(pre_key + key.lower()) + export_values.append(value) + elif isinstance(stats, list): + # Stats is a list (of dict) + # Recursive loop through the list + for item in stats: + item_names, item_values = self.__build_export(item) + export_names += item_names + export_values += item_values + return export_names, export_values diff --git a/glances/exports/glances_influxdb.py b/glances/exports/glances_influxdb.py index 36a3395f..1b759303 100644 --- a/glances/exports/glances_influxdb.py +++ b/glances/exports/glances_influxdb.py @@ -129,3 +129,4 @@ class Export(GlancesExport): self.client.write_points(data) except Exception as e: logger.error("Can not export stats to InfluxDB (%s)" % e) + logger.debug("Export {} stats to InfluxDB".format(name)) diff --git a/glances/exports/glances_statsd.py b/glances/exports/glances_statsd.py index bd6eefd0..aaff939a 100644 --- a/glances/exports/glances_statsd.py +++ b/glances/exports/glances_statsd.py @@ -100,3 +100,4 @@ class Export(GlancesExport): self.client.gauge(stat_name, stat_value) except Exception as e: logger.error("Can not export stats to Statsd (%s)" % e) + logger.debug("Export {} stats to Statsd".format(name)) diff --git a/glances/outputs/glances_bottle.py b/glances/outputs/glances_bottle.py index 0374a012..d5a41c7d 100644 --- a/glances/outputs/glances_bottle.py +++ b/glances/outputs/glances_bottle.py @@ -51,7 +51,7 @@ class GlancesBottle(object): self._app.install(EnableCors()) # Password if args.password != '': - self._app.install(auth_basic(self.check_auth)); + self._app.install(auth_basic(self.check_auth)) # Define routes self._route() diff --git a/glances/plugins/glances_docker.py b/glances/plugins/glances_docker.py index 42ca7f02..ef5c5e90 100644 --- a/glances/plugins/glances_docker.py +++ b/glances/plugins/glances_docker.py @@ -60,6 +60,22 @@ class Plugin(GlancesPlugin): # Init the Docker API self.docker_client = False + def get_key(self): + """Return the key of the list.""" + return 'name' + + def get_export(self): + """Overwrite the default export method + - Only exports containers + - The key is the first container name + """ + ret = [] + try: + ret = self.stats['containers'] + except KeyError as e: + logger.debug("Docker export error {}".format(e)) + return ret + def connect(self, version=None): """Connect to the Docker server.""" # Init connection to the Docker API @@ -180,6 +196,12 @@ class Plugin(GlancesPlugin): c['memory'] = self.get_docker_memory(c['Id'], all_stats) # c['network'] = self.get_docker_network(c['Id'], all_stats) + # Export name (first name in the list, without the /) + c['name'] = c['Names'][0][1:] + + # The key is the container name and not the Id + c['key'] = self.get_key() + elif self.input_method == 'snmp': # Update stats using SNMP # Not available diff --git a/glances/plugins/glances_plugin.py b/glances/plugins/glances_plugin.py index cb3e6be0..e23595ad 100644 --- a/glances/plugins/glances_plugin.py +++ b/glances/plugins/glances_plugin.py @@ -223,6 +223,10 @@ class GlancesPlugin(object): """Return the stats object.""" return self.stats + def get_export(self): + """Return the stats object to export.""" + return self.get_raw() + def get_stats(self): """Return the stats object in JSON format.""" return json.dumps(self.stats) diff --git a/glances/plugins/glances_sensors.py b/glances/plugins/glances_sensors.py index 45464b74..e4ff1109 100644 --- a/glances/plugins/glances_sensors.py +++ b/glances/plugins/glances_sensors.py @@ -146,7 +146,11 @@ class Plugin(GlancesPlugin): - Battery capacity: 'battery' """ for i in stats: + # Set the sensors type i.update({'type': sensor_type}) + # also add the key name + i.update({'key': self.get_key()}) + return stats def update_views(self):