diff --git a/activemq_xml/check.py b/activemq_xml/check.py --- a/activemq_xml/check.py +++ b/activemq_xml/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/agent_metrics/check.py b/agent_metrics/check.py --- a/agent_metrics/check.py +++ b/agent_metrics/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/apache/check.py b/apache/check.py --- a/apache/check.py +++ b/apache/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/aspdotnet/check.py b/aspdotnet/check.py new file mode 100644 --- /dev/null +++ b/aspdotnet/check.py @@ -0,0 +1,35 @@ +# (C) Datadog, Inc. 2013-2016 +# All rights reserved +# Licensed under Simplified BSD License (see LICENSE) + +# datadog +try: + from checks.libs.win.pdhbasecheck import PDHBaseCheck +except ImportError: + def PDHBaseCheck(*args, **kwargs): + return + +EVENT_TYPE = SOURCE_TYPE_NAME = 'aspdotnet' + + +DEFAULT_COUNTERS = [ + # counterset, instance of counter, counter name, metric name + # This set is from the Microsoft recommended counters to monitor exchange: + # https://technet.microsoft.com/en-us/library/dn904093%28v=exchg.150%29.aspx?f=255&MSPPError=-2147217396 + + # ASP.Net + ["ASP.NET", None, "Application Restarts", "aspdotnet.application_restarts", "gauge"], + ["ASP.NET", None, "Worker Process Restarts", "aspdotnet.worker_process_restarts", "gauge"], + ["ASP.NET", None, "Request Wait Time", "aspdotnet.request.wait_time", "gauge"], + + # ASP.Net Applications + ["ASP.NET Applications", None, "Requests In Application Queue", "aspdotnet.applications.requests.in_queue", "gauge"], + ["ASP.NET Applications", None, "Requests Executing", "aspdotnet.applications.requests.executing", "gauge"], + ["ASP.NET Applications", None, "Requests/Sec", "aspdotnet.applications.requests.persec", "gauge"], + ["ASP.NET Applications", None, "Forms Authentication Failure", "aspdotnet.applications.forms_authentication.failure", "gauge"], + ["ASP.NET Applications", None, "Forms Authentication Successes", "aspdotnet.applications.forms_authentication.successes", "gauge"], +] + +class AspdotnetCheck(PDHBaseCheck): + def __init__(self, name, init_config, agentConfig, instances=None): + PDHBaseCheck.__init__(self, name, init_config, agentConfig, instances=instances, counter_list=DEFAULT_COUNTERS) diff --git a/btrfs/check.py b/btrfs/check.py --- a/btrfs/check.py +++ b/btrfs/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/cacti/check.py b/cacti/check.py --- a/cacti/check.py +++ b/cacti/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/cassandra_nodetool/check.py b/cassandra_nodetool/check.py --- a/cassandra_nodetool/check.py +++ b/cassandra_nodetool/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/ceph/check.py b/ceph/check.py --- a/ceph/check.py +++ b/ceph/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -24,6 +24,26 @@ class Ceph(AgentCheck): DEFAULT_CEPH_CMD = '/usr/bin/ceph' DEFAULT_CEPH_CLUSTER = 'ceph' + DEFAULT_HEALTH_CHECKS = [ + 'OSD_DOWN', + 'OSD_ORPHAN', + 'OSD_FULL', + 'OSD_NEARFULL', + 'POOL_FULL', + 'POOL_NEAR_FULL', + 'PG_AVAILABILITY', + 'PG_DEGRADED', + 'PG_DEGRADED_FULL', + 'PG_DAMAGED', + 'PG_NOT_SCRUBBED', + 'PG_NOT_DEEP_SCRUBBED', + 'CACHE_POOL_NEAR_FULL', + 'TOO_FEW_PGS', + 'TOO_MANY_PGS', + 'OBJECT_UNFOUND', + 'REQUEST_SLOW', + 'REQUEST_STUCK', + ] NAMESPACE = 'ceph' def _collect_raw(self, ceph_cmd, ceph_cluster, instance): @@ -88,21 +108,46 @@ def _extract_metrics(self, raw, tags): try: health = {'num_near_full_osds': 0, 'num_full_osds': 0} - # Health summary will be empty if no bad news - if raw['health_detail']['summary'] != []: - for osdhealth in raw['health_detail']['detail']: - osd, pct = self._osd_pct_used(osdhealth) - if osd: - local_tags = tags + ['ceph_osd:%s' % osd.replace('.','')] - - if 'near' in osdhealth: - health['num_near_full_osds'] += 1 - local_health = {'osd.pct_used': pct} - self._publish(local_health, self.gauge, ['osd.pct_used'], local_tags) - else: - health['num_full_osds'] += 1 - local_health = {'osd.pct_used': pct} - self._publish(local_health, self.gauge, ['osd.pct_used'], local_tags) + # In luminous, there are no more overall summary and detail fields, but rather + # one summary and one detail field per check type in the checks field + # For example, for near full osds, we have: + # "checks": { + # "OSD_NEARFULL": { + # "severity": "HEALTH_WARN", + # "summary": { + # "message": "1 nearfull osd(s)" + # }, + # "detail": [ + # { + # "message": "osd.0 is near full" + # } + # ] + # } + # The percentage used per osd does not appear anymore in the message, + # so we won't send the metric osd.pct_used + if 'checks' in raw['health_detail']: + checks = raw['health_detail']['checks'] + for check_name, check_detail in checks.iteritems(): + if check_name == 'OSD_NEARFULL': + health['num_near_full_osds'] = len(check_detail['detail']) + if check_name == 'OSD_FULL': + health['num_full_osds'] = len(check_detail['detail']) + else: + # Health summary will be empty if no bad news + if raw['health_detail']['summary'] != []: + for osdhealth in raw['health_detail']['detail']: + osd, pct = self._osd_pct_used(osdhealth) + if osd: + local_tags = tags + ['ceph_osd:%s' % osd.replace('.','')] + + if 'near' in osdhealth: + health['num_near_full_osds'] += 1 + local_health = {'osd.pct_used': pct} + self._publish(local_health, self.gauge, ['osd.pct_used'], local_tags) + else: + health['num_full_osds'] += 1 + local_health = {'osd.pct_used': pct} + self._publish(local_health, self.gauge, ['osd.pct_used'], local_tags) self._publish(health, self.gauge, ['num_full_osds'], tags) self._publish(health, self.gauge, ['num_near_full_osds'], tags) @@ -175,6 +220,12 @@ def _extract_metrics(self, raw, tags): except KeyError: self.log.debug('Error retrieving mon_status metrics') + try: + num_mons_active = len(raw['mon_status']['quorum']) + self.gauge(self.NAMESPACE + '.num_mons.active', num_mons_active, tags) + except KeyError: + self.log.debug('Error retrieving mon_status quorum metrics') + try: stats = raw['df_detail']['stats'] self._publish(stats, self.gauge, ['total_objects'], tags) @@ -211,9 +262,13 @@ def _osd_pct_used(self, health): else: return None, None - def _perform_service_checks(self, raw, tags): + def _perform_service_checks(self, raw, tags, health_checks): if 'status' in raw: - s_status = raw['status']['health']['overall_status'] + # In ceph luminous, the field name is now `status` + s_status = raw['status']['health'].get('status', None) + if not s_status: + s_status = raw['status']['health']['overall_status'] + if s_status.find('_OK') != -1: status = AgentCheck.OK elif s_status.find('_WARN') != -1: @@ -222,10 +277,23 @@ def _perform_service_checks(self, raw, tags): status = AgentCheck.CRITICAL self.service_check(self.NAMESPACE + '.overall_status', status) + # If we are in ceph luminous, the 'checks' fields contains details of the health checks that are not OK + # Report a service check for each of the checks listed in the yaml + if "checks" in raw['status']['health']: + for check in health_checks: + status = AgentCheck.OK + if check in raw['status']['health']['checks']: + if '_WARN' in raw['status']['health']['checks'][check]['severity']: + status = AgentCheck.WARNING + else: + status = AgentCheck.CRITICAL + self.service_check(self.NAMESPACE + '.' + check.lower(), status) + def check(self, instance): ceph_cmd = instance.get('ceph_cmd') or self.DEFAULT_CEPH_CMD ceph_cluster = instance.get('ceph_cluster') or self.DEFAULT_CEPH_CLUSTER + ceph_health_checks = instance.get('collect_service_check_for') or self.DEFAULT_HEALTH_CHECKS raw = self._collect_raw(ceph_cmd, ceph_cluster, instance) tags = self._extract_tags(raw, instance) self._extract_metrics(raw, tags) - self._perform_service_checks(raw, tags) + self._perform_service_checks(raw, tags, ceph_health_checks) diff --git a/consul/check.py b/consul/check.py --- a/consul/check.py +++ b/consul/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -119,8 +119,13 @@ def _get_cluster_leader(self, instance): def _get_agent_url(self, instance, instance_state): self.log.debug("Starting _get_agent_url") local_config = self._get_local_config(instance, instance_state) - agent_addr = local_config.get('Config', {}).get('AdvertiseAddr') - agent_port = local_config.get('Config', {}).get('Ports', {}).get('Server') + + # Member key for consul 0.7.x and up; Config key for older versions + agent_addr = local_config.get('Member', {}).get('Addr') or \ + local_config.get('Config', {}).get('AdvertiseAddr') + agent_port = local_config.get('Member', {}).get('Tags', {}).get('port') or \ + local_config.get('Config', {}).get('Ports', {}).get('Server') + agent_url = "{0}:{1}".format(agent_addr, agent_port) self.log.debug("Agent url is %s" % agent_url) return agent_url @@ -214,13 +219,13 @@ def _cull_services_list(self, services, service_whitelist, max_services=MAX_SERV if len(service_whitelist) > max_services: self.warning('More than %d services in whitelist. Service list will be truncated.' % max_services) - services = [s for s in services if s in service_whitelist][:max_services] + services = {s:services[s] for s in [s for s in services if s in service_whitelist][:max_services]} else: if len(services) <= max_services: self.log.debug('Consul service whitelist not defined. Agent will poll for all %d services found', len(services)) else: self.warning('Consul service whitelist not defined. Agent will poll for at most %d services' % max_services) - services = list(islice(services.iterkeys(), 0, max_services)) + services = {s:services[s] for s in list(islice(services.iterkeys(), 0, max_services))} return services diff --git a/couch/check.py b/couch/check.py --- a/couch/check.py +++ b/couch/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -186,7 +186,6 @@ def _build_metrics(self, data, tags, prefix = 'couchdb'): elif type(value) is dict: self._build_metrics(value, tags, "{0}.{1}".format(prefix, key)) - def _build_db_metrics(self, data, tags): for key, value in data['sizes'].items(): self.gauge("couchdb.by_db.{0}_size".format(key), value, tags) @@ -194,6 +193,20 @@ def _build_db_metrics(self, data, tags): for key in ['purge_seq', 'doc_del_count', 'doc_count']: self.gauge("couchdb.by_db.{0}".format(key), data[key], tags) + def _build_dd_metrics(self, info, tags): + data = info['view_index'] + ddtags = list(tags) + ddtags.append("design_document:{0}".format(info['name'])) + ddtags.append("language:{0}".format(data['language'])) + + for key, value in data['sizes'].items(): + self.gauge("couchdb.by_ddoc.{0}_size".format(key), value, ddtags) + + for key, value in data['updates_pending'].items(): + self.gauge("couchdb.by_ddoc.{0}_updates_pending".format(key), value, ddtags) + + self.gauge("couchdb.by_ddoc.waiting_clients", data['waiting_clients'], ddtags) + def _build_system_metrics(self, data, tags, prefix = 'couchdb.erlang'): for key, value in data.items(): if key == "message_queues": @@ -214,6 +227,47 @@ def _build_system_metrics(self, data, tags, prefix = 'couchdb.erlang'): else: self.gauge("{0}.{1}".format(prefix, key), value, tags) + def _build_active_tasks_metrics(self, data, tags, prefix = 'couchdb.active_tasks'): + counts = { + 'replication': 0, + 'database_compaction': 0, + 'indexer': 0, + 'view_compaction': 0 + } + for task in data: + counts[task['type']] += 1 + rtags = list(tags) + if task['type'] == 'replication': + for tag in ['doc_id', 'source', 'target', 'user']: + rtags.append("{0}:{1}".format(tag, task[tag])) + rtags.append("type:{0}".format('continuous' if task['continuous'] else 'one-time')) + for metric in ['doc_write_failures', 'docs_read', 'docs_written', 'missing_revisions_found', 'revisions_checked', 'changes_pending']: + if task[metric] is None: + task[metric] = 0 + self.gauge("{0}.replication.{1}".format(prefix, metric), task[metric], rtags) + elif task['type'] == 'database_compaction': + rtags.append("database:{0}".format(task['database'].split('/')[-1].split('.')[0])) + for metric in ['changes_done', 'progress', 'total_changes']: + self.gauge("{0}.db_compaction.{1}".format(prefix, metric), task[metric], rtags) + elif task['type'] == 'indexer': + rtags.append("database:{0}".format(task['database'].split('/')[-1].split('.')[0])) + rtags.append("design_document:{0}".format(task['design_document'].split('/')[-1])) + for metric in ['changes_done', 'progress', 'total_changes']: + self.gauge("{0}.indexer.{1}".format(prefix, metric), task[metric], rtags) + elif task['type'] == 'view_compaction': + rtags.append("database:{0}".format(task['database'].split('/')[-1].split('.')[0])) + rtags.append("design_document:{0}".format(task['design_document'].split('/')[-1])) + if task.get('phase', None) is not None: + rtags.append("phase:{0}".format(task['phase'])) + for metric in ['changes_done', 'progress', 'total_changes']: + if task.get(metric, None) is not None: + self.gauge("{0}.view_compaction.{1}".format(prefix, metric), task[metric], rtags) + + for metric, count in counts.items(): + if metric == "database_compaction": + metric = "db_compaction" + self.gauge("{0}.{1}.count".format(prefix, metric), count, tags) + def _get_instance_names(self, server, instance): name = instance.get('name', None) if name is None: @@ -231,6 +285,7 @@ def check(self, instance): tags = ["instance:{0}".format(name)] self._build_metrics(self._get_node_stats(server, name, instance, tags), tags) self._build_system_metrics(self._get_system_stats(server, name, instance, tags), tags) + self._build_active_tasks_metrics(self._get_active_tasks(server, name, instance, tags), tags) db_whitelist = instance.get('db_whitelist', None) db_blacklist = instance.get('db_blacklist', []) @@ -238,7 +293,10 @@ def check(self, instance): for db in self.agent_check.get(urljoin(server, "/_all_dbs"), instance, tags): if (db_whitelist is None or db in db_whitelist) and (db not in db_blacklist): tags = ["instance:{0}".format(name), "db:{0}".format(db)] - self._build_db_metrics(self.agent_check.get(urljoin(server, db), instance, tags), tags) + db_url = urljoin(server, db) + self._build_db_metrics(self.agent_check.get(db_url, instance, tags), tags) + for dd in self.agent_check.get("{0}/_all_docs?startkey=\"_design/\"&endkey=\"_design0\"".format(db_url), instance, tags)['rows']: + self._build_dd_metrics(self.agent_check.get("{0}/{1}/_info".format(db_url, dd['id']), instance, tags), tags) scanned_dbs += 1 if scanned_dbs >= max_dbs_per_check: break @@ -260,3 +318,12 @@ def _get_system_stats(self, server, name, instance, tags): # Fetch _system (Erlang) stats. return self.agent_check.get(url, instance, tags) + + def _get_active_tasks(self, server, name, instance, tags): + url = urljoin(server, "/_active_tasks") + + tasks = self.agent_check.get(url, instance, tags) + + # print tasks + + return [task for task in tasks if task['node'] == name] diff --git a/couchbase/check.py b/couchbase/check.py --- a/couchbase/check.py +++ b/couchbase/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2013-2016 +# (C) Datadog, Inc. 2013-2017 # (C) Justin Slattery 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/directory/check.py b/directory/check.py --- a/directory/check.py +++ b/directory/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2013-2016 +# (C) Datadog, Inc. 2013-2017 # (C) Brett Langdon 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/disk/check.py b/disk/check.py --- a/disk/check.py +++ b/disk/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/dns_check/check.py b/dns_check/check.py --- a/dns_check/check.py +++ b/dns_check/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/docker_daemon/check.py b/docker_daemon/check.py --- a/docker_daemon/check.py +++ b/docker_daemon/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -66,7 +66,6 @@ "docker.mem.sw_limit": (["hierarchical_memsw_limit"], lambda x: float(x) if float(x) < 2 ** 60 else None, GAUGE), "docker.mem.in_use": (["rss", "hierarchical_memory_limit"], lambda x, y: float(x)/float(y) if float(y) < 2 ** 60 else None, GAUGE), "docker.mem.sw_in_use": (["swap", "rss", "hierarchical_memsw_limit"], lambda x, y, z: float(x + y)/float(z) if float(z) < 2 ** 60 else None, GAUGE) - } }, { @@ -180,10 +179,15 @@ def __init__(self, name, init_config, agentConfig, instances=None): raise Exception("Docker check only supports one configured instance.") AgentCheck.__init__(self, name, init_config, agentConfig, instances=instances) - self.init_success = False self._service_discovery = agentConfig.get('service_discovery') and \ agentConfig.get('service_discovery_backend') == 'docker' + + global_labels_as_tags = agentConfig.get('docker_labels_as_tags') + if global_labels_as_tags: + self.collect_labels_as_tags = [label.strip() for label in global_labels_as_tags.split(',')] + else: + self.collect_labels_as_tags = DEFAULT_LABELS_AS_TAGS self.init() def init(self): @@ -216,7 +220,12 @@ def init(self): self._disable_net_metrics = False # Set tagging options - self.collect_labels_as_tags = instance.get("collect_labels_as_tags", DEFAULT_LABELS_AS_TAGS) + # The collect_labels_as_tags is legacy, only tagging docker metrics. + # It is replaced by docker_labels_as_tags in datadog.conf. + # We keep this line for backward compatibility. + if "collect_labels_as_tags" in instance: + self.collect_labels_as_tags = instance.get("collect_labels_as_tags") + self.kube_pod_tags = {} self.use_histogram = _is_affirmative(instance.get('use_histogram', False)) @@ -641,7 +650,7 @@ def _report_image_size(self, images): def _report_performance_metrics(self, containers_by_id): containers_without_proc_root = [] - for container in containers_by_id.itervalues(): + for container_id, container in containers_by_id.iteritems(): if self._is_container_excluded(container) or not self._is_container_running(container): continue @@ -654,7 +663,7 @@ def _report_performance_metrics(self, containers_by_id): continue self._report_net_metrics(container, tags) except BogusPIDException as e: - self.log.warning('Unable to report cgroup metrics: %s', e) + self.log.warning('Unable to report cgroup metrics for container %s: %s', container_id[:12], e) if containers_without_proc_root: message = "Couldn't find pid directory for containers: {0}. They'll be missing network metrics".format( @@ -1008,7 +1017,12 @@ def _parse_cgroup_file(self, stat_file): elif 'cpuacct.usage' in stat_file: return dict({'usage': str(int(fp.read())/10000000)}) elif 'memory.soft_limit_in_bytes' in stat_file: - return dict({'softlimit': int(fp.read())}) + value = int(fp.read()) + # do not report kernel max default value (uint64 * 4096) + # see https://github.com/torvalds/linux/blob/5b36577109be007a6ecf4b65b54cbc9118463c2b/mm/memcontrol.c#L2844-L2845 + # 2 ** 60 is kept for consistency of other cgroups metrics + if value < 2 ** 60: + return dict({'softlimit': value}) else: return dict(map(lambda x: x.split(' ', 1), fp.read().splitlines())) except IOError: diff --git a/dotnetclr/check.py b/dotnetclr/check.py new file mode 100644 --- /dev/null +++ b/dotnetclr/check.py @@ -0,0 +1,30 @@ +# (C) Datadog, Inc. 2013-2016 +# All rights reserved +# Licensed under Simplified BSD License (see LICENSE) + +# datadog +try: + from checks.libs.win.pdhbasecheck import PDHBaseCheck +except ImportError: + def PDHBaseCheck(*args, **kwargs): + return + +EVENT_TYPE = SOURCE_TYPE_NAME = 'dotnetclr' + +DEFAULT_COUNTERS = [ + # counterset, instance of counter, counter name, metric name + # This set is from the Microsoft recommended counters to monitor exchange: + # https://technet.microsoft.com/en-us/library/dn904093%28v=exchg.150%29.aspx?f=255&MSPPError=-2147217396 + + # .NET Exceptions counters + [".NET CLR Exceptions", None, "# of Exceps Thrown / sec", "dotnetclr.exceptions.thrown_persec", "gauge"], + + # .NET Framework counters + [".NET CLR Memory", None, "% Time in GC", "dotnetclr.memory.time_in_gc", "gauge"], + [".NET CLR Memory", None, "# Total committed Bytes", "dotnetclr.memory.heap_bytes", "gauge"], + [".NET CLR Memory", None, "# Total reserved Bytes", "dotnetclr.memory.heap_bytes", "gauge"], +] + +class DotnetclrCheck(PDHBaseCheck): + def __init__(self, name, init_config, agentConfig, instances=None): + PDHBaseCheck.__init__(self, name, init_config, agentConfig, instances=instances, counter_list=DEFAULT_COUNTERS) diff --git a/elastic/check.py b/elastic/check.py --- a/elastic/check.py +++ b/elastic/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -23,6 +23,7 @@ class NodeNotFound(Exception): ESInstanceConfig = namedtuple( 'ESInstanceConfig', [ 'pshard_stats', + 'pshard_graceful_to', 'cluster_stats', 'password', 'service_check_tags', @@ -354,6 +355,7 @@ def get_instance_config(self, instance): raise Exception("A URL must be specified in the instance") pshard_stats = _is_affirmative(instance.get('pshard_stats', False)) + pshard_graceful_to = _is_affirmative(instance.get('pshard_graceful_timeout', False)) cluster_stats = _is_affirmative(instance.get('cluster_stats', False)) if 'is_external' in instance: @@ -384,6 +386,7 @@ def get_instance_config(self, instance): config = ESInstanceConfig( pshard_stats=pshard_stats, + pshard_graceful_to=pshard_graceful_to, cluster_stats=cluster_stats, password=instance.get('password'), service_check_tags=service_check_tags, @@ -421,10 +424,17 @@ def check(self, instance): self._process_stats_data(stats_data, stats_metrics, config) # Load clusterwise data + # Note: this is a cluster-wide query, might TO. if config.pshard_stats: + send_sc = bubble_ex = not config.pshard_graceful_to pshard_stats_url = urlparse.urljoin(config.url, pshard_stats_url) - pshard_stats_data = self._get_data(pshard_stats_url, config) - self._process_pshard_stats_data(pshard_stats_data, config, pshard_stats_metrics) + try: + pshard_stats_data = self._get_data(pshard_stats_url, config, send_sc=send_sc) + self._process_pshard_stats_data(pshard_stats_data, config, pshard_stats_metrics) + except requests.ReadTimeout as e: + if bubble_ex: + raise + self.log.warning("Timed out reading pshard-stats from servers (%s) - stats will be missing", e) # Load the health data. @@ -476,8 +486,8 @@ def _define_params(self, version, cluster_stats): if version >= [0, 90, 10]: # ES versions 0.90.10 and above - health_url = "/_cluster/health?pretty=true" - pending_tasks_url = "/_cluster/pending_tasks?pretty=true" + health_url = "/_cluster/health" + pending_tasks_url = "/_cluster/pending_tasks" # For "external" clusters, we want to collect from all nodes. if cluster_stats: @@ -491,7 +501,7 @@ def _define_params(self, version, cluster_stats): additional_metrics = self.JVM_METRICS_POST_0_90_10 else: - health_url = "/_cluster/health?pretty=true" + health_url = "/_cluster/health" pending_tasks_url = None if cluster_stats: stats_url = "/_cluster/nodes/stats?all=true" diff --git a/etcd/check.py b/etcd/check.py --- a/etcd/check.py +++ b/etcd/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2015-2016 +# (C) Datadog, Inc. 2015-2017 # (C) Cory G Watson 2014-2015 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -17,6 +17,8 @@ class Etcd(AgentCheck): DEFAULT_TIMEOUT = 5 SERVICE_CHECK_NAME = 'etcd.can_connect' + HEALTH_SERVICE_CHECK_NAME = 'etcd.healthy' + HEALTH_KEY = 'health' STORE_RATES = { 'getsSuccess': 'etcd.store.gets.success', @@ -86,14 +88,25 @@ def check(self, instance): for key, param in ssl_params.items(): if param is None: del ssl_params[key] + + # Get a copy of tags for the CRIT statuses + critical_tags = list(instance_tags) + # Append the instance's URL in case there are more than one, that # way they can tell the difference! instance_tags.append("url:{0}".format(url)) timeout = float(instance.get('timeout', self.DEFAULT_TIMEOUT)) is_leader = False + # Gather self health status + sc_state = AgentCheck.UNKNOWN + health_status = self._get_health_status(url, ssl_params, timeout, critical_tags) + if health_status is not None: + sc_state = AgentCheck.OK if self._is_healthy(health_status) else AgentCheck.CRITICAL + self.service_check(self.HEALTH_SERVICE_CHECK_NAME, sc_state, tags=instance_tags) + # Gather self metrics - self_response = self._get_self_metrics(url, ssl_params, timeout) + self_response = self._get_self_metrics(url, ssl_params, timeout, critical_tags) if self_response is not None: if self_response['state'] == 'StateLeader': is_leader = True @@ -114,7 +127,7 @@ def check(self, instance): self.log.warn("Missing key {0} in stats.".format(key)) # Gather store metrics - store_response = self._get_store_metrics(url, ssl_params, timeout) + store_response = self._get_store_metrics(url, ssl_params, timeout, critical_tags) if store_response is not None: for key in self.STORE_RATES: if key in store_response: @@ -130,7 +143,7 @@ def check(self, instance): # Gather leader metrics if is_leader: - leader_response = self._get_leader_metrics(url, ssl_params, timeout) + leader_response = self._get_leader_metrics(url, ssl_params, timeout, critical_tags) if leader_response is not None and len(leader_response.get("followers", {})) > 0: # Get the followers followers = leader_response.get("followers") @@ -149,40 +162,70 @@ def check(self, instance): # Service check if self_response is not None and store_response is not None: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.OK, - tags=["url:{0}".format(url)]) + tags=instance_tags) + + def _get_health_status(self, url, ssl_params, timeout, tags): + """ + Don't send the "can connect" service check if we have troubles getting + the health status + """ + try: + r = self._perform_request(url, "/health", ssl_params, timeout) + # we don't use get() here so we can report a KeyError + return r.json()[self.HEALTH_KEY] + except Exception as e: + self.log.debug("Can't determine health status: {}".format(e)) + return None - def _get_self_metrics(self, url, ssl_params, timeout): - return self._get_json(url, "/v2/stats/self", ssl_params, timeout) + def _get_self_metrics(self, url, ssl_params, timeout, tags): + return self._get_json(url, "/v2/stats/self", ssl_params, timeout, tags) - def _get_store_metrics(self, url, ssl_params, timeout): - return self._get_json(url, "/v2/stats/store", ssl_params, timeout) + def _get_store_metrics(self, url, ssl_params, timeout, tags): + return self._get_json(url, "/v2/stats/store", ssl_params, timeout, tags) - def _get_leader_metrics(self, url, ssl_params, timeout): - return self._get_json(url, "/v2/stats/leader", ssl_params, timeout) + def _get_leader_metrics(self, url, ssl_params, timeout, tags): + return self._get_json(url, "/v2/stats/leader", ssl_params, timeout, tags) - def _get_json(self, url, path, ssl_params, timeout): + def _perform_request(self, url, path, ssl_params, timeout): + certificate = None + if 'ssl_certfile' in ssl_params and 'ssl_keyfile' in ssl_params: + certificate = (ssl_params['ssl_certfile'], ssl_params['ssl_keyfile']) + verify = ssl_params.get('ssl_ca_certs', True) if ssl_params['ssl_cert_validation'] else False + return requests.get(url + path, verify=verify, cert=certificate, timeout=timeout, headers=headers(self.agentConfig)) + + def _get_json(self, url, path, ssl_params, timeout, tags): try: - certificate = None - if 'ssl_certfile' in ssl_params and 'ssl_keyfile' in ssl_params: - certificate = (ssl_params['ssl_certfile'], ssl_params['ssl_keyfile']) - verify = ssl_params.get('ssl_ca_certs', True) if ssl_params['ssl_cert_validation'] else False - r = requests.get(url + path, verify=verify, cert=certificate, timeout=timeout, headers=headers(self.agentConfig)) + r = self._perform_request(url, path, ssl_params, timeout) except requests.exceptions.Timeout: - # If there's a timeout self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, message="Timeout when hitting %s" % url, - tags=["url:{0}".format(url)]) + tags=tags + ["url:{0}".format(url)]) raise except Exception as e: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, message="Error hitting %s. Error: %s" % (url, e.message), - tags=["url:{0}".format(url)]) + tags=tags + ["url:{0}".format(url)]) raise if r.status_code != 200: self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, message="Got %s when hitting %s" % (r.status_code, url), - tags=["url:{0}".format(url)]) + tags=tags + ["url:{0}".format(url)]) raise Exception("Http status code {0} on url {1}".format(r.status_code, url)) return r.json() + + def _is_healthy(self, status): + """ + Version of etcd prior to 3.3 return this payload when you hit /health: + {"health": "true"} + + which is wrong since the value is a `bool` on etcd. + + Version 3.3 fixed this issue in https://github.com/coreos/etcd/pull/8312 + but we need to support both. + """ + if isinstance(status, bool): + return status + + return status == "true" diff --git a/exchange_server/check.py b/exchange_server/check.py new file mode 100644 --- /dev/null +++ b/exchange_server/check.py @@ -0,0 +1,98 @@ +# (C) Datadog, Inc. 2013-2016 +# All rights reserved +# Licensed under Simplified BSD License (see LICENSE) + +# datadog +try: + from checks.libs.win.pdhbasecheck import PDHBaseCheck +except ImportError: + def PDHBaseCheck(*args, **kwargs): + return + +DEFAULT_COUNTERS = [ + # counterset, instance of counter, counter name, metric name + # This set is from the Microsoft recommended counters to monitor exchange: + # https://technet.microsoft.com/en-us/library/dn904093%28v=exchg.150%29.aspx?f=255&MSPPError=-2147217396 + # Exchange Domain Controller Connectivity Counters + ["MSExchange ADAccess Domain Controllers", None, "LDAP Read Time", "exchange.adaccess_domain_controllers.ldap_read", "gauge"], + ["MSExchange ADAccess Domain Controllers", None, "LDAP Search Time", "exchange.adaccess_domain_controllers.ldap_search", "gauge"], + ["MSExchange ADAccess Processes", None, "LDAP Read Time", "exchange.adaccess_processes.ldap_read", "gauge"], + ["MSExchange ADAccess Processes", None, "LDAP Search Time", "exchange.adaccess_processes.ldap_search", "gauge"], + # Processor and Process Counters + ["Processor", "_Total", "% Processor Time", "exchange.processor.cpu_time", "gauge"], + ["Processor", "_Total", "% User Time", "exchange.processor.cpu_user", "gauge"], + ["Processor", "_Total", "% Privileged Time", "exchange.processor.cpu_privileged", "gauge"], + ["System", None, "Processor Queue Length", "exchange.processor.queue_length", "gauge"], + # Memory Counters + ["Memory", None, "Available MBytes", "exchange.memory.available", "gauge"], + ["Memory", None, "% Committed Bytes In Use", "exchange.memory.committed", "gauge"], + # Network Counters + ["Network Interface", None, "Packets Outbound Errors", "exchange.network.outbound_errors", "gauge"], + ["TCPv6", None, "Connection Failures", "exchange.network.tcpv6.connection_failures", "gauge"], + ["TCPv4", None, "Connections Reset", "exchange.network.tcpv4.conns_reset", "count"], + ["TCPv6", None, "Connections Reset", "exchange.network.tcpv6.conns_reset", "count"], + # Netlogon Counters + ["Netlogon", None, "Semaphore Waiters", "exchange.netlogon.semaphore_waiters", "gauge"], + ["Netlogon", None, "Semaphore Holders", "exchange.netlogon.semaphore_holders", "gauge"], + ["Netlogon", None, "Semaphore Acquires", "exchange.netlogon.semaphore_acquires", "count"], + ["Netlogon", None, "Semaphore Timeouts", "exchange.netlogon.semaphore_timeouts", "count"], + ["Netlogon", None, "Average Semaphore Hold Time", "exchange.netlogon.semaphore_hold_time", "gauge"], + + # Database counters + ["MSExchange Database ==> Instances", None, "I/O Database Reads (Attached) Average Latency", "exchange.database.io_reads_avg_latency", "gauge"], + ["MSExchange Database ==> Instances", None, "I/O Database Writes (Attached) Average Latency", "exchange.database.io_writes_avg_latency", "gauge"], + ["MSExchange Database ==> Instances", None, "I/O Log Writes Average Latency", "exchange.database.io_log_writes_avg_latency", "gauge"], + ["MSExchange Database ==> Instances", None, "I/O Database Reads (Recovery) Average Latency", "exchange.database.io_db_reads_recovery_avg_latency", "gauge"], + ["MSExchange Database ==> Instances", None, "I/O Database Writes (Recovery) Average Latency", "exchange.database.io_db_writes_recovery_avg_latency", "gauge"], + ["MSExchange Database ==> Instances", None, "I/O Database Reads (Attached)/sec", "exchange.database.io_db_reads_attached_persec", "gauge"], + ["MSExchange Database ==> Instances", None, "I/O Database Writes (Attached)/sec", "exchange.database.io_db_writes_attached_persec", "gauge"], + ["MSExchange Database ==> Instances", None, "I/O Log Writes/sec", "exchange.database.io_log_writes_persec", "gauge"], + ["MSExchange Active Manager", "_total", "Database Mounted", "exchange.activemanager.database_mounted", "gauge"], + + # RPC Client Access Counters + ["MSExchange RpcClientAccess", None, "RPC Averaged Latency", "exchange.rpc.averaged_latency", "gauge"], + ["MSExchange RpcClientAccess", None, "RPC Requests", "exchange.rpc.requests", "gauge"], + ["MSExchange RpcClientAccess", None, "Active User Count", "exchange.rpc.active_user_count", "gauge"], + ["MSExchange RpcClientAccess", None, "Connection Count", "exchange.rpc.conn_count", "gauge"], + ["MSExchange RpcClientAccess", None, "RPC Operations/sec", "exchange.rpc.ops_persec", "gauge"], + ["MSExchange RpcClientAccess", None, "User Count", "exchange.rpc.user_count", "gauge"], + + # HTTP Proxy Counters + ["MSExchange HttpProxy", None, "MailboxServerLocator Average Latency", "exchange.httpproxy.server_locator_latency", "gauge"], + ["MSExchange HttpProxy", None, "Average Authentication Latency", "exchange.httpproxy.avg_auth_latency", "gauge"], + ["MSExchange HttpProxy", None, "Average ClientAccess Server Processing Latency", "exchange.httpproxy.clientaccess_processing_latency", "gauge"], + ["MSExchange HttpProxy", None, "Mailbox Server Proxy Failure Rate", "exchange.httpproxy.mailbox_proxy_failure_rate", "gauge"], + ["MSExchange HttpProxy", None, "Outstanding Proxy Requests", "exchange.httpproxy.outstanding_requests", "gauge"], + ["MSExchange HttpProxy", None, "Proxy Requests/Sec", "exchange.httpproxy.proxy_requests_persec", "gauge"], + ["MSExchange HttpProxy", None, "Requests/Sec", "exchange.httpproxy.requests_persec", "gauge"], + + # Information Store Counters + ["MSExchangeIS Store", None, "RPC Requests", "exchange.is.store.rpc_requests", "gauge"], + ["MSExchangeIS Client Type", None, "RPC Average Latency", "exchange.is.clienttype.rpc_latency", "gauge"], + ["MSExchangeIS Store", None, "RPC Average Latency", "exchange.is.store.rpc_latency", "gauge"], + ["MSExchangeIS Store", None, "RPC Operations/sec", "exchange.is.store.rpc_ops_persec", "gauge"], + ["MSExchangeIS Client Type", None, "RPC Operations/sec", "exchange.is.clienttype.rpc_ops_persec", "gauge"], + + # Client Access Server Counters + ["MSExchange ActiveSync", None, "Requests/sec", "exchange.activesync.requests_persec", "gauge"], + ["MSExchange ActiveSync", None, "Ping Commands Pending", "exchange.activesync.ping_pending", "gauge"], + ["MSExchange ActiveSync", None, "Sync Commands/sec", "exchange.activesync.sync_persec", "gauge"], + ["MSExchange OWA", None, "Current Unique Users", "exchange.owa.unique_users", "gauge"], + ["MSExchange OWA", None, "Requests/sec", "exchange.owa.requests_persec", "gauge"], + ["MSExchangeAutodiscover", None, "Requests/sec", "exchange.autodiscover.requests_persec", "gauge"], + ["MSExchangeWS", None, "Requests/sec", "exchange.ws.requests_persec", "gauge"], + ["Web Service", "_Total", "Current Connections", "exchange.ws.current_connections_total", "gauge"], + ["Web Service", None, "Current Connections", "exchange.ws.current_connections_default_website", "gauge"], + ["Web Service", "_Total", "Connection Attempts/sec", "exchange.ws.connection_attempts", "gauge"], + ["Web Service", "_Total", "Other Request Methods/sec", "exchange.ws.other_attempts", "gauge"], + + # Workload Management Counters + ["MSExchange WorkloadManagement Workloads", None, "ActiveTasks", "exchange.workload_management.active_tasks", "gauge"], + ["MSExchange WorkloadManagement Workloads", None, "CompletedTasks", "exchange.workload_management.completed_tasks", "gauge"], + ["MSExchange WorkloadManagement Workloads", None, "QueuedTasks", "exchange.workload_management.queued_tasks", "gauge"], + +] + +class ExchangeCheck(PDHBaseCheck): + def __init__(self, name, init_config, agentConfig, instances=None): + PDHBaseCheck.__init__(self, name, init_config, agentConfig, instances=instances, counter_list=DEFAULT_COUNTERS) diff --git a/fluentd/check.py b/fluentd/check.py --- a/fluentd/check.py +++ b/fluentd/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2015-2016 +# (C) Datadog, Inc. 2015-2017 # (C) Takumi Sakamoto 2014 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/gearmand/check.py b/gearmand/check.py --- a/gearmand/check.py +++ b/gearmand/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2013-2016 +# (C) Datadog, Inc. 2013-2017 # (C) Patrick Galbraith 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/gitlab/check.py b/gitlab/check.py new file mode 100644 --- /dev/null +++ b/gitlab/check.py @@ -0,0 +1,138 @@ +# stdlib +import urlparse + +# 3rd party +import requests + +from checks import CheckException +from checks.prometheus_check import PrometheusCheck +from config import _is_affirmative +from util import headers + +class GitlabCheck(PrometheusCheck): + + # Readiness signals ability to serve traffic, liveness that Gitlab is healthy overall + ALLOWED_SERVICE_CHECKS = ['readiness', 'liveness'] + EVENT_TYPE = SOURCE_TYPE_NAME = 'gitlab' + DEFAULT_CONNECT_TIMEOUT = 5 + DEFAULT_RECEIVE_TIMEOUT = 15 + + PROMETHEUS_SERVICE_CHECK_NAME = 'gitlab.prometheus_endpoint_up' + + """ + Collect Gitlab metrics from Prometheus and validates that the connectivity with Gitlab + """ + def __init__(self, name, init_config, agentConfig, instances=None): + super(GitlabCheck, self).__init__(name, init_config, agentConfig, instances) + # Mapping from Prometheus metrics names to Datadog ones + # For now it's a 1:1 mapping + # TODO: mark some metrics as rate + allowed_metrics = init_config.get('allowed_metrics') + + if not allowed_metrics: + raise CheckException("At least one metric must be whitelisted in `allowed_metrics`.") + + self.metrics_mapper = dict(zip(allowed_metrics, allowed_metrics)) + self.NAMESPACE = 'gitlab' + + def check(self, instance): + #### Metrics collection + endpoint = instance.get('prometheus_endpoint') + if endpoint is None: + raise CheckException("Unable to find prometheus_endpoint in config file.") + + # By default we send the buckets + send_buckets = _is_affirmative(instance.get('send_histograms_buckets', True)) + + try: + self.process(endpoint, send_histograms_buckets=send_buckets, instance=instance) + self.service_check(self.PROMETHEUS_SERVICE_CHECK_NAME, PrometheusCheck.OK) + except requests.exceptions.ConnectionError as e: + # Unable to connect to the metrics endpoint + self.service_check(self.PROMETHEUS_SERVICE_CHECK_NAME, PrometheusCheck.CRITICAL, + message="Unable to retrieve Prometheus metrics from endpoint %s: %s" % (endpoint, e.message)) + + #### Service check to check Gitlab's health endpoints + for check_type in self.ALLOWED_SERVICE_CHECKS: + self._check_health_endpoint(instance, check_type) + + + def _verify_ssl(self, instance): + ## Load the ssl configuration + ssl_params = { + 'ssl_cert_validation': _is_affirmative(instance.get('ssl_cert_validation', True)), + 'ssl_ca_certs': instance.get('ssl_ca_certs'), + } + + for key, param in ssl_params.items(): + if param is None: + del ssl_params[key] + + return ssl_params.get('ssl_ca_certs', True) if ssl_params['ssl_cert_validation'] else False + + def _service_check_tags(self, url): + parsed_url = urlparse.urlparse(url) + gitlab_host = parsed_url.hostname + gitlab_port = 443 if parsed_url.scheme == 'https' else (parsed_url.port or 80) + return ['gitlab_host:%s' % gitlab_host, 'gitlab_port:%s' % gitlab_port] + + # Validates an health endpoint + # + # Valid endpoints are: + # - /-/readiness + # - /-/liveness + # + # https://docs.gitlab.com/ce/user/admin_area/monitoring/health_check.html + def _check_health_endpoint(self, instance, check_type): + if check_type not in self.ALLOWED_SERVICE_CHECKS: + raise CheckException("Health endpoint %s is not a valid endpoint" % check_type) + + url = instance.get('gitlab_url') + if url is None: + # Simply ignore this service check if not configured + self.log.debug("gitlab_url not configured, service check %s skipped" % check_type) + return + + service_check_tags = self._service_check_tags(url) + verify_ssl = self._verify_ssl(instance) + + ## Timeout settings + timeouts = (int(instance.get('connect_timeout', GitlabCheck.DEFAULT_CONNECT_TIMEOUT)), + int(instance.get('receive_timeout', GitlabCheck.DEFAULT_RECEIVE_TIMEOUT))) + + ## Auth settings + auth = None + if 'gitlab_user' in instance and 'gitlab_password' in instance: + auth = (instance['gitlab_user'], instance['gitlab_password']) + + # These define which endpoint is hit and which type of check is actually performed + # TODO: parse errors and report for single sub-service failure? + service_check_name = "gitlab.%s" % check_type + check_url = "%s/-/%s" % (url, check_type) + + try: + self.log.debug('checking %s against %s' % (check_type, check_url)) + r = requests.get(check_url, auth=auth, verify=verify_ssl, timeout=timeouts, + headers=headers(self.agentConfig)) + if r.status_code != 200: + self.service_check(service_check_name, PrometheusCheck.CRITICAL, + message="Got %s when hitting %s" % (r.status_code, check_url), + tags=service_check_tags) + raise Exception("Http status code {0} on check_url {1}".format(r.status_code, check_url)) + else: + r.raise_for_status() + + except requests.exceptions.Timeout: + # If there's a timeout + self.service_check(service_check_name, PrometheusCheck.CRITICAL, + message="Timeout when hitting %s" % check_url, + tags=service_check_tags) + raise + except Exception as e: + self.service_check(service_check_name, PrometheusCheck.CRITICAL, + message="Error hitting %s. Error: %s" % (check_url, e.message), + tags=service_check_tags) + raise + else: + self.service_check(service_check_name, PrometheusCheck.OK, tags=service_check_tags) + self.log.debug("gitlab check %s succeeded" % check_type) diff --git a/gitlab_runner/check.py b/gitlab_runner/check.py new file mode 100644 --- /dev/null +++ b/gitlab_runner/check.py @@ -0,0 +1,121 @@ +# stdlib +import urlparse + +# 3rd party +import requests + +from checks import CheckException +from checks.prometheus_check import PrometheusCheck +from config import _is_affirmative +from util import headers + +class GitlabRunnerCheck(PrometheusCheck): + + EVENT_TYPE = SOURCE_TYPE_NAME = 'gitlab_runner' + MASTER_SERVICE_CHECK_NAME = 'gitlab_runner.can_connect' + PROMETHEUS_SERVICE_CHECK_NAME = 'gitlab_runner.prometheus_endpoint_up' + + DEFAULT_CONNECT_TIMEOUT = 5 + DEFAULT_RECEIVE_TIMEOUT = 15 + + """ + Collect Gitlab Runner metrics from Prometheus and validates that the connectivity with Gitlab + """ + def __init__(self, name, init_config, agentConfig, instances=None): + super(GitlabRunnerCheck, self).__init__(name, init_config, agentConfig, instances) + # Mapping from Prometheus metrics names to Datadog ones + # For now it's a 1:1 mapping + # TODO: mark some metrics as rate + allowed_metrics = init_config.get('allowed_metrics') + + if not allowed_metrics: + raise CheckException("At least one metric must be whitelisted in `allowed_metrics`.") + + self.metrics_mapper = dict(zip(allowed_metrics, allowed_metrics)) + self.NAMESPACE = 'gitlab_runner' + + def check(self, instance): + #### Metrics collection + endpoint = instance.get('prometheus_endpoint') + if endpoint is None: + raise CheckException("Unable to find prometheus_endpoint in config file.") + + # By default we send the buckets + send_buckets = _is_affirmative(instance.get('send_histograms_buckets', True)) + + try: + self.process(endpoint, send_histograms_buckets=send_buckets, instance=instance) + self.service_check(self.PROMETHEUS_SERVICE_CHECK_NAME, PrometheusCheck.OK) + except requests.exceptions.ConnectionError as e: + # Unable to connect to the metrics endpoint + self.service_check(self.PROMETHEUS_SERVICE_CHECK_NAME, PrometheusCheck.CRITICAL, + message="Unable to retrieve Prometheus metrics from endpoint %s: %s" % (endpoint, e.message)) + + + #### Service check to check whether the Runner can talk to the Gitlab master + self._check_connectivity_to_master(instance) + + # Validates that the runner can connect to Gitlab + # + # Note: Gitlab supports readiness/liveness endpoints but they require an access token + # or IP whitelisting based on the version + # https://docs.gitlab.com/ce/user/admin_area/monitoring/health_check.html + # TODO: consider using those endpoints + def _check_connectivity_to_master(self, instance): + url = instance.get('gitlab_url') + if url is None: + # Simply ignore this service check if not configured + return + + parsed_url = urlparse.urlparse(url) + gitlab_host = parsed_url.hostname + gitlab_port = 443 if parsed_url.scheme == 'https' else (parsed_url.port or 80) + service_check_tags = ['gitlab_host:%s' % gitlab_host, 'gitlab_port:%s' % gitlab_port] + + ## Load the ssl configuration + ssl_params = { + 'ssl_cert_validation': _is_affirmative(instance.get('ssl_cert_validation', True)), + 'ssl_ca_certs': instance.get('ssl_ca_certs'), + } + + for key, param in ssl_params.items(): + if param is None: + del ssl_params[key] + + verify_ssl = ssl_params.get('ssl_ca_certs', True) if ssl_params['ssl_cert_validation'] else False + + ## Timeout settings + timeouts = (int(instance.get('connect_timeout', GitlabRunnerCheck.DEFAULT_CONNECT_TIMEOUT)), + int(instance.get('receive_timeout', GitlabRunnerCheck.DEFAULT_RECEIVE_TIMEOUT))) + + ## Auth settings + auth = None + if 'gitlab_user' in instance and 'gitlab_password' in instance: + auth = (instance['gitlab_user'], instance['gitlab_password']) + + try: + self.log.debug('checking connectivity against %s' % url) + r = requests.get(url, auth=auth, verify=verify_ssl, timeout=timeouts, + headers=headers(self.agentConfig)) + if r.status_code != 200: + self.service_check(self.MASTER_SERVICE_CHECK_NAME, PrometheusCheck.CRITICAL, + message="Got %s when hitting %s" % (r.status_code, url), + tags=service_check_tags) + raise Exception("Http status code {0} on url {1}".format(r.status_code, url)) + else: + r.raise_for_status() + + except requests.exceptions.Timeout: + # If there's a timeout + self.service_check(self.MASTER_SERVICE_CHECK_NAME, PrometheusCheck.CRITICAL, + message="Timeout when hitting %s" % url, + tags=service_check_tags) + raise + except Exception as e: + self.service_check(self.MASTER_SERVICE_CHECK_NAME, PrometheusCheck.CRITICAL, + message="Error hitting %s. Error: %s" % (url, e.message), + tags=service_check_tags) + raise + else: + self.service_check(self.MASTER_SERVICE_CHECK_NAME, PrometheusCheck.OK, tags=service_check_tags) + self.log.debug("gitlab check succeeded") diff --git a/go_expvar/check.py b/go_expvar/check.py --- a/go_expvar/check.py +++ b/go_expvar/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2014-2016 +# (C) Datadog, Inc. 2014-2017 # (C) Cory Watson 2015-2016 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/gunicorn/check.py b/gunicorn/check.py --- a/gunicorn/check.py +++ b/gunicorn/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/haproxy/check.py b/haproxy/check.py --- a/haproxy/check.py +++ b/haproxy/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2012-2016 +# (C) Datadog, Inc. 2012-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -141,6 +141,7 @@ def check(self, instance): services_incl_filter = instance.get('services_include', []) services_excl_filter = instance.get('services_exclude', []) + tags_regex = instance.get('tags_regex', None) custom_tags = instance.get('tags', []) process_events = instance.get('status_check', self.init_config.get('status_check', False)) @@ -155,6 +156,7 @@ def check(self, instance): collate_status_tags_per_host=collate_status_tags_per_host, count_status_by_service=count_status_by_service, custom_tags=custom_tags, + tags_regex=tags_regex, ) def _fetch_url_data(self, url, username, password, verify): @@ -194,7 +196,7 @@ def _process_data(self, data, collect_aggregates_only, process_events, url=None, collect_status_metrics=False, collect_status_metrics_by_host=False, tag_service_check_by_host=False, services_incl_filter=None, services_excl_filter=None, collate_status_tags_per_host=False, - count_status_by_service=True, custom_tags=[]): + count_status_by_service=True, custom_tags=[], tags_regex=None): ''' Main data-processing loop. For each piece of useful data, we'll either save a metric, save an event or both. ''' @@ -228,6 +230,14 @@ def _process_data(self, data, collect_aggregates_only, process_events, url=None, data_dict, self.hosts_statuses ) + # Clone the list to avoid extending the original + # which would carry over previous iteration tags + line_tags = list(custom_tags) + + regex_tags = self._tag_from_regex(tags_regex, data_dict['pxname']) + if regex_tags: + line_tags.extend(regex_tags) + if self._should_process(data_dict, collect_aggregates_only): # update status # Send the list of data to the metric and event callbacks @@ -235,21 +245,21 @@ def _process_data(self, data, collect_aggregates_only, process_events, url=None, data_dict, url, services_incl_filter=services_incl_filter, services_excl_filter=services_excl_filter, - custom_tags=custom_tags + custom_tags=line_tags ) if process_events: self._process_event( data_dict, url, services_incl_filter=services_incl_filter, services_excl_filter=services_excl_filter, - custom_tags=custom_tags + custom_tags=line_tags ) self._process_service_check( data_dict, url, tag_by_host=tag_service_check_by_host, services_incl_filter=services_incl_filter, services_excl_filter=services_excl_filter, - custom_tags=custom_tags + custom_tags=line_tags ) if collect_status_metrics: @@ -259,14 +269,14 @@ def _process_data(self, data, collect_aggregates_only, process_events, url=None, services_excl_filter=services_excl_filter, collate_status_tags_per_host=collate_status_tags_per_host, count_status_by_service=count_status_by_service, - custom_tags=custom_tags + custom_tags=line_tags ) self._process_backend_hosts_metric( self.hosts_statuses, services_incl_filter=services_incl_filter, services_excl_filter=services_excl_filter, - custom_tags=custom_tags + custom_tags=line_tags ) return data @@ -389,6 +399,25 @@ def _tag_match_patterns(self, tag, filters): return True return False + def _tag_from_regex(self, tags_regex, service_name): + """ + Use a named regexp on the current service_name to create extra tags + Example HAProxy service name: be_edge_http_sre-prod_elk + Example named regexp: be_edge_http_(?P[a-z]+)\-(?P[a-z]+)_(?P.*) + Resulting tags: ['team:sre','env:prod','app:elk'] + """ + if not tags_regex or not service_name: + return [] + + match = re.compile(tags_regex).match(service_name) + + if not match: + return [] + + # match.groupdict() returns tags dictionary in the form of {'name': 'value'} + # convert it to Datadog tag LIST: ['name:value'] + return ["%s:%s" % (name, value) for name, value in match.groupdict().iteritems()] + @staticmethod def _normalize_status(status): """ diff --git a/hdfs/check.py b/hdfs/check.py --- a/hdfs/check.py +++ b/hdfs/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2013-2016 +# (C) Datadog, Inc. 2013-2017 # (C) Dan Crosta 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/hdfs_datanode/check.py b/hdfs_datanode/check.py --- a/hdfs_datanode/check.py +++ b/hdfs_datanode/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -62,15 +62,16 @@ def check(self, instance): jmx_address = instance.get('hdfs_datanode_jmx_uri') if jmx_address is None: raise Exception('The JMX URL must be specified in the instance configuration') + disable_ssl_validation = instance.get('disable_ssl_validation', False) # Get metrics from JMX - self._hdfs_datanode_metrics(jmx_address) + self._hdfs_datanode_metrics(jmx_address, disable_ssl_validation) - def _hdfs_datanode_metrics(self, jmx_uri): + def _hdfs_datanode_metrics(self, jmx_uri, disable_ssl_validation): ''' Get HDFS data node metrics from JMX ''' - response = self._rest_request_to_json(jmx_uri, + response = self._rest_request_to_json(jmx_uri, disable_ssl_validation, JMX_PATH, query_params={'qry':HDFS_DATANODE_BEAN_NAME}) @@ -101,7 +102,7 @@ def _set_metric(self, metric_name, metric_type, value, tags=None): else: self.log.error('Metric type "%s" unknown' % (metric_type)) - def _rest_request_to_json(self, address, object_path, query_params): + def _rest_request_to_json(self, address, disable_ssl_validation, object_path, query_params): ''' Query the given URL and return the JSON response ''' @@ -122,7 +123,7 @@ def _rest_request_to_json(self, address, object_path, query_params): self.log.debug('Attempting to connect to "%s"' % url) try: - response = requests.get(url, timeout=self.default_integration_http_timeout) + response = requests.get(url, timeout=self.default_integration_http_timeout, verify=not disable_ssl_validation) response.raise_for_status() response_json = response.json() diff --git a/hdfs_namenode/check.py b/hdfs_namenode/check.py --- a/hdfs_namenode/check.py +++ b/hdfs_namenode/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -91,15 +91,16 @@ class HDFSNameNode(AgentCheck): def check(self, instance): jmx_address = instance.get('hdfs_namenode_jmx_uri') + disable_ssl_validation = instance.get('disable_ssl_validation', False) if jmx_address is None: raise Exception('The JMX URL must be specified in the instance configuration') # Get metrics from JMX - self._hdfs_namenode_metrics(jmx_address, + self._hdfs_namenode_metrics(jmx_address, disable_ssl_validation, HDFS_NAME_SYSTEM_STATE_BEAN, HDFS_NAME_SYSTEM_STATE_METRICS) - self._hdfs_namenode_metrics(jmx_address, + self._hdfs_namenode_metrics(jmx_address, disable_ssl_validation, HDFS_NAME_SYSTEM_BEAN, HDFS_NAME_SYSTEM_METRICS) @@ -108,11 +109,11 @@ def check(self, instance): tags=['namenode_url:' + jmx_address], message='Connection to %s was successful' % jmx_address) - def _hdfs_namenode_metrics(self, jmx_uri, bean_name, metrics): + def _hdfs_namenode_metrics(self, jmx_uri, disable_ssl_validation, bean_name, metrics): ''' Get HDFS namenode metrics from JMX ''' - response = self._rest_request_to_json(jmx_uri, + response = self._rest_request_to_json(jmx_uri, disable_ssl_validation, JMX_PATH, query_params={'qry':bean_name}) @@ -147,7 +148,7 @@ def _set_metric(self, metric_name, metric_type, value, tags=None): else: self.log.error('Metric type "%s" unknown' % (metric_type)) - def _rest_request_to_json(self, address, object_path, query_params): + def _rest_request_to_json(self, address, disable_ssl_validation, object_path, query_params): ''' Query the given URL and return the JSON response ''' @@ -168,7 +169,7 @@ def _rest_request_to_json(self, address, object_path, query_params): self.log.debug('Attempting to connect to "%s"' % url) try: - response = requests.get(url, timeout=self.default_integration_http_timeout) + response = requests.get(url, timeout=self.default_integration_http_timeout, verify=not disable_ssl_validation) response.raise_for_status() response_json = response.json() diff --git a/http_check/check.py b/http_check/check.py --- a/http_check/check.py +++ b/http_check/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -275,13 +275,16 @@ def send_status_down(loginfo, message): % (str(e), length)) raise + # Store tags in a temporary list so that we don't modify the global tags data structure + tags_list = list(tags) + # Only add the URL tag if it's not already present + if not filter(re.compile('^url:').match, tags_list): + tags_list.append('url:%s' % addr) + # Only report this metric if the site is not down if response_time and not service_checks: # Stop the timer as early as possible running_time = time.time() - start - # Store tags in a temporary list so that we don't modify the global tags data structure - tags_list = list(tags) - tags_list.append('url:%s' % addr) self.gauge('network.http.response_time', running_time, tags=tags_list) # Check HTTP response status code @@ -327,6 +330,15 @@ def send_status_down(loginfo, message): else: send_status_up("%s is UP" % addr) + # Report status metrics as well + if service_checks: + can_status = 1 if service_checks[0][1] == "UP" else 0 + self.gauge('network.http.can_connect', can_status, tags=tags_list) + + # cant_connect is useful for top lists + cant_status = 0 if service_checks[0][1] == "UP" else 1 + self.gauge('network.http.cant_connect', cant_status, tags=tags_list) + if ssl_expire and parsed_uri.scheme == "https": status, days_left,msg = self.check_cert_expiration(instance, timeout, instance_ca_certs) @@ -353,7 +365,10 @@ def _create_status_event(self, sc_name, status, msg, instance): tags = instance.get('tags', []) tags_list = [] tags_list.extend(tags) - tags_list.append('url:%s' % url) + + # Only add the URL tag if it's not already present + if not filter(re.compile('^url:').match, tags_list): + tags_list.append('url:%s' % url) # Get a custom message that will be displayed in the event custom_message = instance.get('message', "") @@ -417,9 +432,12 @@ def _create_status_event(self, sc_name, status, msg, instance): def report_as_service_check(self, sc_name, status, instance, msg=None): instance_name = self.normalize(instance['name']) url = instance.get('url', None) - sc_tags = ['url:{0}'.format(url), "instance:{0}".format(instance_name)] - custom_tags = instance.get('tags', []) - tags = sc_tags + custom_tags + tags = instance.get('tags', []) + tags.append("instance:{0}".format(instance_name)) + + # Only add the URL tag if it's not already present + if not filter(re.compile('^url:').match, tags): + tags.append('url:{0}'.format(url)) if sc_name == self.SC_STATUS: # format the HTTP response body into the event @@ -446,6 +464,7 @@ def check_cert_expiration(self, instance, timeout, instance_ca_certs): o = urlparse(url) host = o.hostname + server_name = instance.get('ssl_server_name', o.hostname) port = o.port or 443 @@ -457,7 +476,7 @@ def check_cert_expiration(self, instance, timeout, instance_ca_certs): context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = True context.load_verify_locations(instance_ca_certs) - ssl_sock = context.wrap_socket(sock, server_hostname=host) + ssl_sock = context.wrap_socket(sock, server_hostname=server_name) cert = ssl_sock.getpeercert() except Exception as e: diff --git a/iis/check.py b/iis/check.py --- a/iis/check.py +++ b/iis/check.py @@ -1,187 +1,123 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) ''' Check the performance counters from IIS ''' -# 3p -import pythoncom # project from checks import AgentCheck -from checks.winwmi_check import WinWMICheck, WMIMetric -from config import _is_affirmative +try: + from checks.libs.win.pdhbasecheck import PDHBaseCheck +except ImportError: + def PDHBaseCheck(*args, **kwargs): + return + from utils.containers import hash_mutable -from utils.timeout import TimeoutException - - -class IIS(WinWMICheck): - METRICS = [ - ('ServiceUptime', 'iis.uptime', 'gauge'), - - # Network - ('TotalBytesSent','iis.net.bytes_sent', 'rate'), - ('TotalBytesReceived', 'iis.net.bytes_rcvd', 'rate'), - ('TotalBytesTransferred', 'iis.net.bytes_total', 'rate'), - ('CurrentConnections', 'iis.net.num_connections', 'gauge'), - ('TotalFilesSent', 'iis.net.files_sent', 'rate'), - ('TotalFilesReceived', 'iis.net.files_rcvd', 'rate'), - ('TotalConnectionAttemptsAllInstances', 'iis.net.connection_attempts', 'rate'), - - # HTTP Methods - ('TotalGetRequests', 'iis.httpd_request_method.get', 'rate'), - ('TotalPostRequests', 'iis.httpd_request_method.post', 'rate'), - ('TotalHeadRequests', 'iis.httpd_request_method.head', 'rate'), - ('TotalPutRequests', 'iis.httpd_request_method.put', 'rate'), - ('TotalDeleteRequests', 'iis.httpd_request_method.delete', 'rate'), - ('TotalOptionsRequests', 'iis.httpd_request_method.options', 'rate'), - ('TotalTraceRequests', 'iis.httpd_request_method.trace', 'rate'), - - # Errors - ('TotalNotFoundErrors', 'iis.errors.not_found', 'rate'), - ('TotalLockedErrors', 'iis.errors.locked', 'rate'), - - # Users - ('TotalAnonymousUsers', 'iis.users.anon', 'rate'), - ('TotalNonAnonymousUsers', 'iis.users.nonanon', 'rate'), - - # Requests - ('TotalCGIRequests', 'iis.requests.cgi', 'rate'), - ('TotalISAPIExtensionRequests', 'iis.requests.isapi', 'rate'), - ] + +DEFAULT_COUNTERS = [ + ["Web Service", None, "Service Uptime", "iis.uptime", "gauge"], + # Network + ["Web Service", None, "Bytes Sent/sec", "iis.net.bytes_sent", "gauge"], + ["Web Service", None, "Bytes Received/sec", "iis.net.bytes_rcvd", "gauge"], + ["Web Service", None, "Bytes Total/sec", "iis.net.bytes_total", "gauge"], + ["Web Service", None, "Current Connections", "iis.net.num_connections", "gauge"], + ["Web Service", None, "Files Sent/sec", "iis.net.files_sent", "gauge"], + ["Web Service", None, "Files Received/sec", "iis.net.files_rcvd" ,"gauge"], + ["Web Service", None, "Total Connection Attempts (all instances)", "iis.net.connection_attempts", "gauge"], + + # HTTP Methods + ["Web Service", None, "Get Requests/sec", "iis.httpd_request_method.get", "gauge"], + ["Web Service", None, "Post Requests/sec", "iis.httpd_request_method.post", "gauge"], + ["Web Service", None, "Head Requests/sec", "iis.httpd_request_method.head", "gauge"], + ["Web Service", None, "Put Requests/sec", "iis.httpd_request_method.put", "gauge"], + ["Web Service", None, "Delete Requests/sec", "iis.httpd_request_method.delete", "gauge"], + ["Web Service", None, "Options Requests/sec", "iis.httpd_request_method.options", "gauge"], + ["Web Service", None, "Trace Requests/sec", "iis.httpd_request_method.trace", "gauge"], + + # Errors + ["Web Service", None, "Not Found Errors/sec", "iis.errors.not_found", "gauge"], + ["Web Service", None, "Locked Errors/sec", "iis.errors.locked", "gauge"], + + # Users + ["Web Service", None, "Anonymous Users/sec", "iis.users.anon", "gauge"], + ["Web Service", None, "NonAnonymous Users/sec", "iis.users.nonanon", "gauge"], + + # Requests + ["Web Service", None, "CGI Requests/sec", "iis.requests.cgi", "gauge"], + ["Web Service", None, "ISAPI Extension Requests/sec", "iis.requests.isapi", "gauge"], +] +class IIS(PDHBaseCheck): SERVICE_CHECK = "iis.site_up" - NAMESPACE = "root\\CIMV2" - CLASS = "Win32_PerfFormattedData_W3SVC_WebService" def __init__(self, name, init_config, agentConfig, instances): - WinWMICheck.__init__(self, name, init_config, agentConfig, instances) + PDHBaseCheck.__init__(self, name, init_config, agentConfig, instances=instances, counter_list=DEFAULT_COUNTERS) def check(self, instance): - # Connect to the WMI provider - host = instance.get('host', "localhost") - provider = instance.get('provider') - user = instance.get('username', "") - password = instance.get('password', "") - instance_tags = instance.get('tags', []) - sites = instance.get('sites', ['_Total']) - is_2008 = _is_affirmative(instance.get('is_2008', False)) - - instance_hash = hash_mutable(instance) - instance_key = self._get_instance_key(host, self.NAMESPACE, self.CLASS, instance_hash) - filters = map(lambda x: {"Name": tuple(('=', x))}, sites) - - metrics_by_property, properties = self._get_wmi_properties(instance_key, self.METRICS, []) - - if is_2008: - for idx, prop in enumerate(properties): - if prop == "TotalBytesTransferred": - properties[idx] = "TotalBytesTransfered" - break - - wmi_sampler = self._get_wmi_sampler( - instance_key, - self.CLASS, properties, - filters=filters, - host=host, namespace=self.NAMESPACE, provider=provider, - username=user, password=password - ) - - # Sample, extract & submit metrics - try: - wmi_sampler.sample() - - metrics = self._extract_metrics(wmi_sampler, sites, instance_tags) - except TimeoutException: - self.log.warning( - u"[IIS] WMI query timed out." - u" class={wmi_class} - properties={wmi_properties} -" - u" filters={filters} - tags={instance_tags}".format( - wmi_class=self.CLASS, wmi_properties=properties, - filters=filters, instance_tags=instance_tags - ) - ) - except pythoncom.com_error as e: - if '0x80041017' in str(e): - self.warning( - u"You may be running IIS6/7 which reports metrics a " - u"little differently. Try enabling the is_2008 flag for this instance." - ) - raise e - else: - self._submit_events(wmi_sampler, sites) - self._submit_metrics(metrics, metrics_by_property) - - def _extract_metrics(self, wmi_sampler, sites, instance_tags): - """ - Extract and tag metrics from the WMISampler. - - Returns: List of WMIMetric - ``` - [ - WMIMetric("freemegabytes", 19742, ["name:_total"]), - WMIMetric("avgdiskbytesperwrite", 1536, ["name:c:"]), - ] - ``` - """ - metrics = [] - - for wmi_obj in wmi_sampler: - tags = list(instance_tags) if instance_tags else [] - - # Get site name - sitename = wmi_obj['Name'] - - # Skip any sites we don't specifically want. - if sitename not in sites: - continue - elif sitename != "_Total": - tags.append("site:{0}".format(self.normalize(sitename))) - - # Tag with `tag_queries` parameter - for wmi_property, wmi_value in wmi_obj.iteritems(): - # No metric extraction on 'Name' property - if wmi_property == 'name': - continue + sites = instance.get('sites') + if sites is None: + expected_sites = set() + else: + expected_sites = set(sites) + # _Total should always be in the list of expected sites; we always + # report _Total + if "_Total" not in expected_sites: + expected_sites.add("_Total") + + self.log.debug("expected sites is %s" % str(expected_sites)) + key = hash_mutable(instance) + for inst_name, dd_name, metric_func, counter in self._metrics[key]: + try: try: - metrics.append(WMIMetric(wmi_property, float(wmi_value), tags)) - except ValueError: - self.log.warning(u"When extracting metrics with WMI, found a non digit value" - " for property '{0}'.".format(wmi_property)) - continue - except TypeError: - self.log.warning(u"When extracting metrics with WMI, found a missing property" - " '{0}'".format(wmi_property)) + vals = counter.get_all_values() + except Exception as e: + self.log.error("Failed to get_all_values %s %s" % (inst_name, dd_name)) continue - return metrics - - def _submit_events(self, wmi_sampler, sites): - expected_sites = set(sites) - - for wmi_obj in wmi_sampler: - sitename = wmi_obj['Name'] - uptime = wmi_obj["ServiceUptime"] - status = AgentCheck.CRITICAL if uptime == 0 else AgentCheck.OK - self.service_check(self.SERVICE_CHECK, status, tags=['site:{0}'.format(self.normalize(sitename))]) - expected_sites.remove(sitename) + for sitename, val in vals.iteritems(): + tags = [] + if key in self._tags: + tags = self._tags[key] + + try: + if not counter.is_single_instance(): + # Skip any sites we don't specifically want. + if not sites: + tags.append("site:{0}".format(self.normalize(sitename))) + # always report total + elif sitename == "_Total": + tags.append("site:{0}".format(self.normalize(sitename))) + elif sitename not in sites: + continue + else: + tags.append("site:{0}".format(self.normalize(sitename))) + except Exception as e: + self.log.error("Caught exception %s setting tags" % str(e)) + + try: + metric_func(dd_name, val, tags) + except Exception as e: + self.log.error("metric_func: %s %s %s" % (dd_name, str(val), str(e))) + pass + + if dd_name == "iis.uptime": + uptime = int(val) + status = AgentCheck.CRITICAL if uptime == 0 else AgentCheck.OK + self.service_check(self.SERVICE_CHECK, status, tags=['site:{0}'.format(self.normalize(sitename))]) + if sitename in expected_sites: + self.log.debug("Removing %s from expected sites" % sitename) + expected_sites.remove(sitename) + else: + self.log.warning("site not in expected_sites %s" % sitename) + + except Exception as e: + # don't give up on all of the metrics because one failed + self.log.error("IIS Failed to get metric data for %s %s: %s" % (inst_name, dd_name, str(e))) + pass for site in expected_sites: self.service_check(self.SERVICE_CHECK, AgentCheck.CRITICAL, - tags=['site:{0}'.format(self.normalize(site))]) - - def _submit_metrics(self, wmi_metrics, metrics_by_property): - for m in wmi_metrics: - metric_name = m.name - # Windows 2008 sp2 reports it as TotalbytesTransfered - # instead of TotalBytesTransferred (single r) - if metric_name.lower() == "totalbytestransfered": - metric_name = "totalbytestransferred" - elif m.name not in metrics_by_property: - continue - - metric, mtype = metrics_by_property[metric_name] - submittor = getattr(self, mtype) - submittor(metric, m.value, m.tags) + tags=['site:{0}'.format(self.normalize(site))]) diff --git a/jenkins/check.py b/jenkins/check.py --- a/jenkins/check.py +++ b/jenkins/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/kafka_consumer/check.py b/kafka_consumer/check.py --- a/kafka_consumer/check.py +++ b/kafka_consumer/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -155,7 +155,17 @@ def stop(self): cli.close() def _get_instance_key(self, instance): - return self._read_config(instance, 'kafka_connect_str') + servers = self._read_config(instance, 'kafka_connect_str') + key = None + if isinstance(servers, basestring): + key = servers + elif isinstance(servers, list): + key = ",".join(servers) + else: + raise BadKafkaConsumerConfiguration('kafka_connect_str should be string or list of strings') + + return key + def _get_kafka_client(self, instance): kafka_conn_str = self._read_config(instance, 'kafka_connect_str') @@ -163,7 +173,7 @@ def _get_kafka_client(self, instance): raise BadKafkaConsumerConfiguration('Bad instance configuration') instance_key = self._get_instance_key(instance) - if kafka_conn_str not in self.kafka_clients: + if instance_key not in self.kafka_clients: cli = KafkaClient(bootstrap_servers=kafka_conn_str, client_id='dd-agent') self.kafka_clients[instance_key] = cli diff --git a/kong/check.py b/kong/check.py --- a/kong/check.py +++ b/kong/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/kubernetes/check.py b/kubernetes/check.py --- a/kubernetes/check.py +++ b/kubernetes/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/kubernetes_state/check.py b/kubernetes_state/check.py --- a/kubernetes_state/check.py +++ b/kubernetes_state/check.py @@ -3,6 +3,7 @@ # Licensed under Simplified BSD License (see LICENSE) import re +import time from collections import defaultdict from checks import CheckException @@ -10,6 +11,8 @@ METRIC_TYPES = ['counter', 'gauge'] +WHITELISTED_WAITING_REASONS = ['ErrImagePull'] +WHITELISTED_TERMINATED_REASONS = ['OOMKilled','ContainerCannotRun','Error'] class KubernetesState(PrometheusCheck): @@ -22,11 +25,11 @@ def __init__(self, name, init_config, agentConfig, instances=None): self.NAMESPACE = 'kubernetes_state' self.pod_phase_to_status = { - 'pending': self.WARNING, - 'running': self.OK, - 'succeeded': self.OK, - 'failed': self.CRITICAL, - 'unknown': self.UNKNOWN + 'Pending': self.WARNING, + 'Running': self.OK, + 'Succeeded': self.OK, + 'Failed': self.CRITICAL, + 'Unknown': self.UNKNOWN } self.condition_to_status_positive = { @@ -53,12 +56,21 @@ def __init__(self, name, init_config, agentConfig, instances=None): 'kube_deployment_status_replicas_available': 'deployment.replicas_available', 'kube_deployment_status_replicas_unavailable': 'deployment.replicas_unavailable', 'kube_deployment_status_replicas_updated': 'deployment.replicas_updated', + 'kube_hpa_spec_min_replicas': 'hpa.min_replicas', + 'kube_hpa_spec_max_replicas': 'hpa.max_replicas', + 'kube_hpa_status_desired_replicas': 'hpa.desired_replicas', + 'kube_hpa_status_current_replicas': 'hpa.current_replicas', 'kube_node_status_allocatable_cpu_cores': 'node.cpu_allocatable', 'kube_node_status_allocatable_memory_bytes': 'node.memory_allocatable', 'kube_node_status_allocatable_pods': 'node.pods_allocatable', 'kube_node_status_capacity_cpu_cores': 'node.cpu_capacity', 'kube_node_status_capacity_memory_bytes': 'node.memory_capacity', 'kube_node_status_capacity_pods': 'node.pods_capacity', + 'kube_node_status_allocatable_nvidia_gpu_cards': 'node.gpu.cards_allocatable', + 'kube_node_status_capacity_nvidia_gpu_cards': 'node.gpu.cards_capacity', + 'kube_pod_container_status_terminated': 'container.terminated', + 'kube_pod_container_status_waiting': 'container.waiting', + 'kube_persistentvolumeclaim_status_phase': 'persistentvolumeclaim.status', 'kube_pod_container_resource_limits_cpu_cores': 'container.cpu_limit', 'kube_pod_container_resource_limits_memory_bytes': 'container.memory_limit', 'kube_pod_container_resource_requests_cpu_cores': 'container.cpu_requested', @@ -66,8 +78,8 @@ def __init__(self, name, init_config, agentConfig, instances=None): 'kube_pod_container_status_ready': 'container.ready', 'kube_pod_container_status_restarts': 'container.restarts', 'kube_pod_container_status_running': 'container.running', - 'kube_pod_container_status_terminated': 'container.terminated', - 'kube_pod_container_status_waiting': 'container.waiting', + 'kube_pod_container_resource_requests_nvidia_gpu_devices': 'container.gpu.request', + 'kube_pod_container_resource_limits_nvidia_gpu_devices': 'container.gpu.limit', 'kube_pod_status_ready': 'pod.ready', 'kube_pod_status_scheduled': 'pod.scheduled', 'kube_replicaset_spec_replicas': 'replicaset.replicas_desired', @@ -81,22 +93,42 @@ def __init__(self, name, init_config, agentConfig, instances=None): 'kube_replicationcontroller_status_replicas': 'replicationcontroller.replicas', 'kube_statefulset_replicas': 'statefulset.replicas_desired', 'kube_statefulset_status_replicas': 'statefulset.replicas', + 'kube_statefulset_status_replicas_current': 'statefulset.replicas_current', + 'kube_statefulset_status_replicas_ready': 'statefulset.replicas_ready', + 'kube_statefulset_status_replicas_updated': 'statefulset.replicas_updated', } self.ignore_metrics = [ - # _info and _labels don't convey any metric + # _info, _labels and _created don't convey any metric 'kube_cronjob_info', + 'kube_cronjob_created', + 'kube_daemonset_created', + 'kube_deployment_created', + 'kube_deployment_labels', + 'kube_job_created', 'kube_job_info', + 'kube_limitrange_created', + 'kube_namespace_created', + 'kube_namespace_labels', + 'kube_node_created', 'kube_node_info', 'kube_node_labels', + 'kube_pod_created' 'kube_pod_container_info', 'kube_pod_info', + 'kube_pod_owner', + 'kube_pod_start_time', 'kube_pod_labels', + 'kube_replicaset_created', + 'kube_replicationcontroller_created', + 'kube_resourcequota_created', + 'kube_service_created', 'kube_service_info', 'kube_service_labels', + 'kube_statefulset_labels', + 'kube_statefulset_created', # _generation metrics are more metadata than metrics, no real use case for now 'kube_daemonset_metadata_generation', - 'kube_daemonset_metadata_generation', 'kube_deployment_metadata_generation', 'kube_deployment_status_observed_generation', 'kube_replicaset_metadata_generation', @@ -105,7 +137,9 @@ def __init__(self, name, init_config, agentConfig, instances=None): 'kube_replicationcontroller_status_observed_generation', 'kube_statefulset_metadata_generation', 'kube_statefulset_status_observed_generation', - # kube_node_status_phase has no use case as a service check + 'kube_hpa_metadata_generation', + # kube_node_status_phase and kube_namespace_status_phase have no use case as a service check + 'kube_namespace_status_phase', 'kube_node_status_phase', # These CronJob and Job metrics need use cases to determine how do implement 'kube_cronjob_status_active', @@ -118,7 +152,6 @@ def __init__(self, name, init_config, agentConfig, instances=None): 'kube_job_status_active', 'kube_job_status_completion_time', # We could compute the duration=completion-start as a gauge 'kube_job_status_start_time', - ] def check(self, instance): @@ -138,7 +171,6 @@ def check(self, instance): send_buckets = False else: send_buckets = True - # Job counters are monotonic: they increase at every run of the job # We want to send the delta via the `monotonic_count` method self.job_succeeded_count = defaultdict(int) @@ -176,6 +208,56 @@ def _condition_to_service_check(self, metric, sc_name, mapping, tags=None): else: self.log.debug("Unable to handle %s - unknown condition %s" % (sc_name, label.value)) + def _condition_to_tag_check(self, metric, base_sc_name, mapping, tags=None): + """ + Metrics from kube-state-metrics have changed + For example: + kube_node_status_condition{condition="Ready",node="ip-172-33-39-189.eu-west-1.compute.internal",status="true"} 1 + kube_node_status_condition{condition="OutOfDisk",node="ip-172-33-57-130.eu-west-1.compute.internal",status="false"} 1 + metric { + label { name: "condition", value: "true" + } + # other labels here + gauge { value: 1.0 } + } + + This function evaluates metrics containing conditions and sends a service check + based on a provided condition->check mapping dict + """ + if bool(metric.gauge.value) is False: + return # Ignore if gauge is not 1 and we are not processing the pod phase check + + label_value, condition_map = self._get_metric_condition_map(base_sc_name, metric.label) + service_check_name = condition_map['service_check_name'] + mapping = condition_map['mapping'] + + if base_sc_name == 'kubernetes_state.pod.phase': + message = "%s is currently reporting %s" % (self._label_to_tag('pod', metric.label), self._label_to_tag('phase', metric.label)) + else: + message = "%s is currently reporting %s" % (self._label_to_tag('node', metric.label), self._label_to_tag('condition', metric.label)) + + if condition_map['service_check_name'] is None: + self.log.debug("Unable to handle %s - unknown condition %s" % (service_check_name, label_value)) + else: + self.service_check(service_check_name, mapping[label_value], tags=tags, message=message) + self.log.debug("%s %s %s" % (service_check_name, mapping[label_value], tags)) + + def _get_metric_condition_map(self, base_sc_name, labels): + if base_sc_name == 'kubernetes_state.node': + switch = { + 'Ready': {'service_check_name': base_sc_name + '.ready', 'mapping': self.condition_to_status_positive}, + 'OutOfDisk': {'service_check_name': base_sc_name + '.out_of_disk', 'mapping': self.condition_to_status_negative}, + 'DiskPressure': {'service_check_name': base_sc_name + '.disk_pressure', 'mapping': self.condition_to_status_negative}, + 'NetworkUnavailable': {'service_check_name': base_sc_name + '.network_unavailable', 'mapping': self.condition_to_status_negative}, + 'MemoryPressure': {'service_check_name': base_sc_name + '.memory_pressure', 'mapping': self.condition_to_status_negative} + } + label_value = self._extract_label_value('status', labels) + return label_value, switch.get(self._extract_label_value('condition', labels), {'service_check_name': None, 'mapping': None}) + + elif base_sc_name == 'kubernetes_state.pod.phase': + label_value = self._extract_label_value('phase', labels) + return label_value, {'service_check_name': base_sc_name, 'mapping': self.pod_phase_to_status} + def _extract_label_value(self, name, labels): """ Search for `name` in labels name and returns @@ -213,23 +295,67 @@ def _trim_job_tag(self, name): pattern = "(-\d{4,10}$)" return re.sub(pattern, '', name) - # Labels attached: namespace, pod, phase=Pending|Running|Succeeded|Failed|Unknown - # The phase gets not passed through; rather, it becomes the service check suffix. + # Labels attached: namespace, pod + # As a message the phase=Pending|Running|Succeeded|Failed|Unknown + # From the phase the check will update its status def kube_pod_status_phase(self, message, **kwargs): """ Phase a pod is in. """ - check_basename = self.NAMESPACE + '.pod.phase.' + # Will submit a service check which status is given by its phase. + # More details about the phase in the message of the check. + check_basename = self.NAMESPACE + '.pod.phase' + for metric in message.metric: + self._condition_to_tag_check(metric, check_basename, self.pod_phase_to_status, + tags=[self._label_to_tag("pod", metric.label),self._label_to_tag("namespace", metric.label)]) + + def kube_pod_container_status_waiting_reason(self, message, **kwargs): + metric_name = self.NAMESPACE + '.container.status_report.count.waiting' for metric in message.metric: - # The gauge value is always 1, no point in fetching it. - phase = '' tags = [] + skip_metric = False for label in metric.label: - if label.name == 'phase': - phase = label.value.lower() - else: + if label.name == "reason": + if label.value in WHITELISTED_WAITING_REASONS: + tags.append(self._format_tag(label.name, label.value)) + else: + skip_metric = True + elif label.name == "container": + tags.append(self._format_tag("kube_container_name", label.value)) + elif label.name == "namespace": tags.append(self._format_tag(label.name, label.value)) - #TODO: add deployment/replicaset? - status = self.pod_phase_to_status.get(phase, self.UNKNOWN) - self.service_check(check_basename + phase, status, tags=tags) + if not skip_metric: + self.count(metric_name, metric.gauge.value, tags) + + def kube_pod_container_status_terminated_reason(self, message, **kwargs): + metric_name = self.NAMESPACE + '.container.status_report.count.terminated' + for metric in message.metric: + tags = [] + skip_metric = False + for label in metric.label: + if label.name == "reason": + if label.value in WHITELISTED_TERMINATED_REASONS: + tags.append(self._format_tag(label.name, label.value)) + else: + skip_metric = True + elif label.name == "container": + tags.append(self._format_tag("kube_container_name", label.value)) + elif label.name == "namespace": + tags.append(self._format_tag(label.name, label.value)) + if not skip_metric: + self.count(metric_name, metric.gauge.value, tags) + + def kube_cronjob_next_schedule_time(self, message, **kwargs): + """ Time until the next schedule """ + # Used as a service check so that one can be alerted if the cronjob's next schedule is in the past + check_basename = self.NAMESPACE + '.cronjob.on_schedule_check' + curr_time = int(time.time()) + for metric in message.metric: + on_schedule = int(metric.gauge.value) - curr_time + tags = [self._format_tag(label.name, label.value) for label in metric.label] + if on_schedule < 0: + message = "The service check scheduled at %s is %s seconds late" % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(int(metric.gauge.value))), on_schedule) + self.service_check(check_basename, self.CRITICAL, tags=tags, message=message) + else: + self.service_check(check_basename, self.OK, tags=tags) def kube_job_complete(self, message, **kwargs): service_check_name = self.NAMESPACE + '.job.complete' @@ -278,8 +404,15 @@ def kube_job_status_succeeded(self, message, **kwargs): tags.append(self._format_tag(label.name, label.value)) self.job_succeeded_count[frozenset(tags)] += metric.gauge.value + def kube_node_status_condition(self, message, **kwargs): + """ The ready status of a cluster node. >v1.0.0""" + base_check_name = self.NAMESPACE + '.node' + for metric in message.metric: + self._condition_to_tag_check(metric, base_check_name, self.condition_to_status_positive, + tags=[self._label_to_tag("node", metric.label)]) + def kube_node_status_ready(self, message, **kwargs): - """ The ready status of a cluster node. """ + """ The ready status of a cluster node.""" service_check_name = self.NAMESPACE + '.node.ready' for metric in message.metric: self._condition_to_service_check(metric, service_check_name, self.condition_to_status_positive, diff --git a/kyototycoon/check.py b/kyototycoon/check.py --- a/kyototycoon/check.py +++ b/kyototycoon/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # (C) Dan Crosta 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/lighttpd/check.py b/lighttpd/check.py --- a/lighttpd/check.py +++ b/lighttpd/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/linux_proc_extras/check.py b/linux_proc_extras/check.py --- a/linux_proc_extras/check.py +++ b/linux_proc_extras/check.py @@ -1,5 +1,5 @@ # (C) Cory Watson 2016 -# (C) Datadog, Inc. 2016 +# (C) Datadog, Inc. 2016-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/mapreduce/check.py b/mapreduce/check.py --- a/mapreduce/check.py +++ b/mapreduce/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/marathon/check.py b/marathon/check.py --- a/marathon/check.py +++ b/marathon/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2014-2016 +# (C) Datadog, Inc. 2014-2017 # (C) graemej 2014 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -152,10 +152,10 @@ def process_apps(self, url, timeout, auth, acs_url, ssl_verify, tags=None, group if response is not None: self.gauge('marathon.apps', len(response['apps']), tags=tags) for app in response['apps']: - tags = ['app_id:' + app['id'], 'version:' + app['version']] + tags + app_tags = ['app_id:' + app['id'], 'version:' + app['version']] + tags for attr in self.APP_METRICS: if attr in app: - self.gauge('marathon.' + attr, app[attr], tags=tags) + self.gauge('marathon.' + attr, app[attr], tags=app_tags) def process_deployments(self, url, timeout, auth, acs_url, ssl_verify, tags=None): # Number of running/pending deployments diff --git a/mcache/check.py b/mcache/check.py --- a/mcache/check.py +++ b/mcache/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/mesos/check.py b/mesos/check.py --- a/mesos/check.py +++ b/mesos/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2014-2016 +# (C) Datadog, Inc. 2014-2017 # (C) graemej 2014 # All rights reserved diff --git a/mesos_master/check.py b/mesos_master/check.py --- a/mesos_master/check.py +++ b/mesos_master/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2015-2016 +# (C) Datadog, Inc. 2015-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/mesos_slave/check.py b/mesos_slave/check.py --- a/mesos_slave/check.py +++ b/mesos_slave/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2015-2016 +# (C) Datadog, Inc. 2015-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/mongo/check.py b/mongo/check.py --- a/mongo/check.py +++ b/mongo/check.py @@ -847,7 +847,7 @@ def total_seconds(td): ) except Exception as e: - if "OperationFailure" in repr(e) and "replSetGetStatus" in str(e): + if "OperationFailure" in repr(e) and "not running with --replSet" in str(e): pass else: raise e @@ -992,8 +992,8 @@ def total_seconds(td): localdb.command("collstats", ol_collection_name)['size'] / 2.0 ** 20, 2 ) - op_asc_cursor = oplog.find().sort("$natural", pymongo.ASCENDING).limit(1) - op_dsc_cursor = oplog.find().sort("$natural", pymongo.DESCENDING).limit(1) + op_asc_cursor = oplog.find({"ts": {"$exists": 1}}).sort("$natural", pymongo.ASCENDING).limit(1) + op_dsc_cursor = oplog.find({"ts": {"$exists": 1}}).sort("$natural", pymongo.DESCENDING).limit(1) try: first_timestamp = op_asc_cursor[0]['ts'].as_datetime() diff --git a/mysql/check.py b/mysql/check.py --- a/mysql/check.py +++ b/mysql/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # (C) Datadog, Inc. Patrick Galbraith 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -528,7 +528,11 @@ def _collect_metrics(self, host, db, tags, options, queries): if _is_affirmative(options.get('replication', False)): # Get replica stats - results.update(self._get_replica_stats(db)) + is_mariadb = self._get_is_mariadb(db, host) + replication_channel = options.get('replication_channel') + if replication_channel: + self.service_check_tags.append("channel:{0}".format(replication_channel)) + results.update(self._get_replica_stats(db, is_mariadb, replication_channel)) nonblocking = _is_affirmative(options.get('replication_non_blocking_status', False)) results.update(self._get_slave_status(db, above_560, nonblocking)) metrics.update(REPLICA_VARS) @@ -543,12 +547,12 @@ def _collect_metrics(self, host, db, tags, options, queries): # MySQL 5.7.x might not have 'Slave_running'. See: https://bugs.mysql.com/bug.php?id=78544 # look at replica vars collected at the top of if-block if self._version_compatible(db, host, (5, 7, 0)): - slave_io_running = self._collect_string('Slave_IO_Running', results) - slave_sql_running = self._collect_string('Slave_SQL_Running', results) + slave_io_running = self._collect_type('Slave_IO_Running', results, dict) + slave_sql_running = self._collect_type('Slave_SQL_Running', results, dict) if slave_io_running: - slave_io_running = (slave_io_running.lower().strip() == "yes") + slave_io_running = any(v.lower().strip() == 'yes' for v in slave_io_running.itervalues()) if slave_sql_running: - slave_sql_running = (slave_sql_running.lower().strip() == "yes") + slave_sql_running = any(v.lower().strip() == 'yes' for v in slave_sql_running.itervalues()) if not (slave_io_running is None and slave_sql_running is None): if slave_io_running and slave_sql_running: @@ -672,6 +676,13 @@ def _get_version(self, db, host): self.mysql_version[hostkey] = version return version + def _get_is_mariadb(self, db, host): + with closing(db.cursor()) as cursor: + cursor.execute('SELECT VERSION() LIKE "%MariaDB%"') + result = cursor.fetchone() + + return result[0] == 1 + def _collect_all_scalars(self, key, dictionary): if key not in dictionary or dictionary[key] is None: yield None, None @@ -847,24 +858,57 @@ def _is_innodb_engine_enabled(self, db): self.warning("Possibly innodb stats unavailable - error querying engines table: %s" % str(e)) return False - def _get_replica_stats(self, db): + def _get_replica_stats(self, db, is_mariadb, replication_channel): + replica_results = {} try: with closing(db.cursor(pymysql.cursors.DictCursor)) as cursor: - replica_results = {} - cursor.execute("SHOW SLAVE STATUS;") - slave_results = cursor.fetchone() + if is_mariadb and replication_channel: + cursor.execute("SET @@default_master_connection = '{0}';".format(replication_channel)) + cursor.execute("SHOW SLAVE STATUS;") + elif replication_channel: + cursor.execute("SHOW SLAVE STATUS FOR CHANNEL '{0}';".format(replication_channel)) + else: + cursor.execute("SHOW SLAVE STATUS;") + + if replication_channel: + slave_results = cursor.fetchone() + else: + slave_results = cursor.fetchall() + if slave_results: - replica_results.update(slave_results) + if replication_channel: + replica_results.update(slave_results) + elif len(slave_results) > 0: + for slave_result in slave_results: + # MySQL <5.7 does not have Channel_Name. + # For MySQL >=5.7 'Channel_Name' is set to an empty string by default + channel = slave_result.get('Channel_Name') or 'default' + for key in slave_result: + if slave_result[key] is not None: + if key not in replica_results: + replica_results[key] = {} + replica_results[key]["channel:{0}".format(channel)] = slave_result[key] + except (pymysql.err.InternalError, pymysql.err.OperationalError) as e: + errno, msg = e.args + if errno == 1617 and msg == "There is no master connection '{0}'".format(replication_channel): + # MariaDB complains when you try to get slave status with a + # connection name on the master, without connection name it + # responds an empty string as expected. + # Mysql behaves the same with or without connection name. + pass + else: + self.warning("Privileges error getting replication status (must grant REPLICATION CLIENT): %s" % str(e)) + + try: + with closing(db.cursor(pymysql.cursors.DictCursor)) as cursor: cursor.execute("SHOW MASTER STATUS;") binlog_results = cursor.fetchone() if binlog_results: replica_results.update({'Binlog_enabled': True}) - - return replica_results - except (pymysql.err.InternalError, pymysql.err.OperationalError) as e: - self.warning("Privileges error getting replication status (must grant REPLICATION CLIENT): %s" % str(e)) - return {} + self.warning("Privileges error getting binlog information (must grant REPLICATION CLIENT): %s" % str(e)) + + return replica_results def _get_slave_status(self, db, above_560, nonblocking): """ diff --git a/nagios/check.py b/nagios/check.py --- a/nagios/check.py +++ b/nagios/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/network/check.py b/network/check.py --- a/network/check.py +++ b/network/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/nfsstat/check.py b/nfsstat/check.py --- a/nfsstat/check.py +++ b/nfsstat/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/nginx/check.py b/nginx/check.py --- a/nginx/check.py +++ b/nginx/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/ntp/check.py b/ntp/check.py --- a/ntp/check.py +++ b/ntp/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/openstack/check.py b/openstack/check.py --- a/openstack/check.py +++ b/openstack/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -85,6 +85,10 @@ ("totalRAMUsed", "total_ram_used"), ]) +DIAGNOSTICABLE_STATES = [ + 'ACTIVE' +] + class OpenStackAuthFailure(Exception): pass @@ -655,16 +659,24 @@ def get_stats_for_single_server(self, server_id, tags=None): def _is_valid_metric(label): return label in NOVA_SERVER_METRICS or any(seg in label for seg in NOVA_SERVER_INTERFACE_SEGMENTS) - url = '{0}/servers/{1}/diagnostics'.format(self.get_nova_endpoint(), server_id) + url = '{0}/servers/{1}'.format(self.get_nova_endpoint(), server_id) headers = {'X-Auth-Token': self.get_auth_token()} - server_stats = {} - + state = None try: - server_stats = self._make_request_with_auth_fallback(url, headers) - except InstancePowerOffFailure: - self.warning("Server %s is powered off and cannot be monitored" % server_id) + server_details = self._make_request_with_auth_fallback(url, headers) + state = server_details['server'].get('status') except Exception as e: - self.warning("Unknown error when monitoring %s : %s" % (server_id, e)) + self.warning("Unable to collect details for server %s : %s" % (server_id, e)) + + server_stats = {} + if state and state.upper() in DIAGNOSTICABLE_STATES: + url = '{0}/servers/{1}/diagnostics'.format(self.get_nova_endpoint(), server_id) + try: + server_stats = self._make_request_with_auth_fallback(url, headers) + except InstancePowerOffFailure: + self.warning("Server %s is powered off and cannot be monitored" % server_id) + except Exception as e: + self.warning("Unknown error when monitoring %s : %s" % (server_id, e)) if server_stats: tags = tags or [] diff --git a/pdh_check/check.py b/pdh_check/check.py --- a/pdh_check/check.py +++ b/pdh_check/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2013-2016 +# (C) Datadog, Inc. 2013-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -62,6 +62,6 @@ def check(self, instance): tags = self._tags[key] if not counter.is_single_instance(): - tag = "instance=%s" % key + tag = "instance:%s" % key tags.append(tag) metric_func(dd_name, val, tags) diff --git a/pgbouncer/check.py b/pgbouncer/check.py --- a/pgbouncer/check.py +++ b/pgbouncer/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/php_fpm/check.py b/php_fpm/check.py --- a/php_fpm/check.py +++ b/php_fpm/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -8,6 +8,9 @@ # project from checks import AgentCheck from util import headers +from config import _is_affirmative + +DEFAULT_TIMEOUT = 20 class PHPFPMCheck(AgentCheck): @@ -44,6 +47,10 @@ def check(self, instance): tags = instance.get('tags', []) http_host = instance.get('http_host') + timeout = instance.get('timeout', DEFAULT_TIMEOUT) + + disable_ssl_validation = _is_affirmative(instance.get('disable_ssl_validation', False)) + if user and password: auth = (user, password) @@ -54,25 +61,28 @@ def check(self, instance): status_exception = None if status_url is not None: try: - pool = self._process_status(status_url, auth, tags, http_host) + pool = self._process_status(status_url, auth, tags, http_host, timeout, disable_ssl_validation) except Exception as e: status_exception = e pass if ping_url is not None: - self._process_ping(ping_url, ping_reply, auth, tags, pool, http_host) + self._process_ping(ping_url, ping_reply, auth, tags, pool, http_host, timeout, disable_ssl_validation) # pylint doesn't understand that we are raising this only if it's here if status_exception is not None: raise status_exception # pylint: disable=E0702 - def _process_status(self, status_url, auth, tags, http_host): + def _process_status(self, status_url, auth, tags, http_host, timeout, disable_ssl_validation): data = {} try: # TODO: adding the 'full' parameter gets you per-process detailed # informations, which could be nice to parse and output as metrics - resp = requests.get(status_url, auth=auth, + resp = requests.get(status_url, + auth=auth, + timeout=timeout, headers=headers(self.agentConfig, http_host=http_host), + verify=not disable_ssl_validation, params={'json': True}) resp.raise_for_status() @@ -101,7 +111,7 @@ def _process_status(self, status_url, auth, tags, http_host): # return pool, to tag the service check with it if we have one return pool_name - def _process_ping(self, ping_url, ping_reply, auth, tags, pool_name, http_host): + def _process_ping(self, ping_url, ping_reply, auth, tags, pool_name, http_host, timeout, disable_ssl_validation): if ping_reply is None: ping_reply = 'pong' @@ -112,8 +122,11 @@ def _process_ping(self, ping_url, ping_reply, auth, tags, pool_name, http_host): try: # TODO: adding the 'full' parameter gets you per-process detailed # informations, which could be nice to parse and output as metrics - resp = requests.get(ping_url, auth=auth, - headers=headers(self.agentConfig, http_host=http_host)) + resp = requests.get(ping_url, + auth=auth, + timeout=timeout, + headers=headers(self.agentConfig, http_host=http_host), + verify=not disable_ssl_validation) resp.raise_for_status() if ping_reply not in resp.text: diff --git a/postfix/check.py b/postfix/check.py --- a/postfix/check.py +++ b/postfix/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2013-2016 +# (C) Datadog, Inc. 2013-2017 # (C) Josiah C Webb 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/postgres/check.py b/postgres/check.py --- a/postgres/check.py +++ b/postgres/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/powerdns_recursor/check.py b/powerdns_recursor/check.py --- a/powerdns_recursor/check.py +++ b/powerdns_recursor/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/process/check.py b/process/check.py --- a/process/check.py +++ b/process/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # (C) Luca Cipriani 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -113,6 +113,7 @@ def find_pids(self, name, search_string, exact_match, ignore_ad=True): refresh_ad_cache = self.should_refresh_ad_cache(name) matching_pids = set() + for proc in psutil.process_iter(): # Skip access denied processes if not refresh_ad_cache and proc.pid in self.ad_cache: @@ -331,6 +332,7 @@ def check(self, instance): pid = instance.get('pid') pid_file = instance.get('pid_file') collect_children = _is_affirmative(instance.get('collect_children', False)) + user = instance.get('user', False) if self._conflicting_procfs: self.warning('The `procfs_path` defined in `process.yaml` is different from the one defined in ' @@ -378,6 +380,10 @@ def check(self, instance): if collect_children: pids.update(self._get_child_processes(pids)) + + if user: + pids = self._filter_by_user(user, pids) + proc_state = self.get_process_state(name, pids) # FIXME 6.x remove the `name` tag @@ -449,3 +455,24 @@ def _process_service_check(self, name, nb_procs, bounds, tags): tags=service_check_tags, message=message_str % (status_str[status], nb_procs, name) ) + + def _filter_by_user(self, user, pids): + """ + Filter pids by it's username. + :param user: string with name of system user + :param pids: set of pids to filter + :return: set of filtered pids + """ + filtered_pids = set() + for pid in pids: + try: + proc = psutil.Process(pid) + if proc.username() == user: + self.log.debug("Collecting pid %s belonging to %s", pid, user) + filtered_pids.add(pid) + else: + self.log.debug("Discarding pid %s not belonging to %s", pid, user) + except psutil.NoSuchProcess: + pass + + return filtered_pids diff --git a/rabbitmq/check.py b/rabbitmq/check.py --- a/rabbitmq/check.py +++ b/rabbitmq/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2013-2016 +# (C) Datadog, Inc. 2013-2017 # (C) Brett Langdon 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -19,14 +19,42 @@ from config import _is_affirmative EVENT_TYPE = SOURCE_TYPE_NAME = 'rabbitmq' +EXCHANGE_TYPE = 'exchanges' QUEUE_TYPE = 'queues' NODE_TYPE = 'nodes' CONNECTION_TYPE = 'connections' +MAX_DETAILED_EXCHANGES = 50 MAX_DETAILED_QUEUES = 200 MAX_DETAILED_NODES = 100 # Post an event in the stream when the number of queues or nodes to # collect is above 90% of the limit: ALERT_THRESHOLD = 0.9 +EXCHANGE_ATTRIBUTES = [ + # Path, Name, Operation + ('message_stats/ack', 'messages.ack.count', float), + ('message_stats/ack_details/rate', 'messages.ack.rate', float), + + ('message_stats/confirm', 'messages.confirm.count', float), + ('message_stats/confirm_details/rate', 'messages.confirm.rate', float), + + ('message_stats/deliver_get', 'messages.deliver_get.count', float), + ('message_stats/deliver_get_details/rate', 'messages.deliver_get.rate', float), + + ('message_stats/publish', 'messages.publish.count', float), + ('message_stats/publish_details/rate', 'messages.publish.rate', float), + + ('message_stats/publish_in', 'messages.publish_in.count', float), + ('message_stats/publish_in_details/rate', 'messages.publish_in.rate', float), + + ('message_stats/publish_out', 'messages.publish_out.count', float), + ('message_stats/publish_out_details/rate', 'messages.publish_out.rate', float), + + ('message_stats/return_unroutable', 'messages.return_unroutable.count', float), + ('message_stats/return_unroutable_details/rate', 'messages.return_unroutable.rate', float), + + ('message_stats/redeliver', 'messages.redeliver.count', float), + ('message_stats/redeliver_details/rate', 'messages.redeliver.rate', float), +] QUEUE_ATTRIBUTES = [ # Path, Name, Operation ('active_consumers', 'active_consumers', float), @@ -62,25 +90,35 @@ NODE_ATTRIBUTES = [ ('fd_used', 'fd_used', float), + ('disk_free', 'disk_free', float), ('mem_used', 'mem_used', float), ('run_queue', 'run_queue', float), ('sockets_used', 'sockets_used', float), - ('partitions', 'partitions', len) + ('partitions', 'partitions', len), + ('running', 'running', float), + ('mem_alarm', 'mem_alarm', float), + ('disk_free_alarm', 'disk_alarm', float), ] ATTRIBUTES = { + EXCHANGE_TYPE: EXCHANGE_ATTRIBUTES, QUEUE_TYPE: QUEUE_ATTRIBUTES, NODE_TYPE: NODE_ATTRIBUTES, } TAG_PREFIX = 'rabbitmq' TAGS_MAP = { + EXCHANGE_TYPE: { + 'name': 'exchange', + 'vhost': 'vhost', + 'exchange_family': 'exchange_family', + }, QUEUE_TYPE: { 'node': 'node', - 'name': 'queue', - 'vhost': 'vhost', - 'policy': 'policy', - 'queue_family': 'queue_family', + 'name': 'queue', + 'vhost': 'vhost', + 'policy': 'policy', + 'queue_family': 'queue_family', }, NODE_TYPE: { 'name': 'node', @@ -88,6 +126,7 @@ } METRIC_SUFFIX = { + EXCHANGE_TYPE: "exchange", QUEUE_TYPE: "queue", NODE_TYPE: "node", } @@ -119,18 +158,33 @@ def _get_config(self, instance): password = instance.get('rabbitmq_pass', 'guest') custom_tags = instance.get('tags', []) parsed_url = urlparse.urlparse(base_url) + if not parsed_url.scheme or "://" not in parsed_url.geturl(): + self.log.warning('The rabbit url did not include a protocol, assuming http') + # urlparse.urljoin cannot add a protocol to the rest of the url for some reason. + # This still leaves the potential for errors, but such urls would never have been valid, either + # and it's not likely to be useful to attempt to catch all possible mistakes people could make. + # urlparse also has a known issue parsing url with no schema, but a port in the host section + # mistakingly taking the host for the schema, hence the additional validation + base_url = 'http://' + base_url + parsed_url = urlparse.urlparse(base_url) + ssl_verify = _is_affirmative(instance.get('ssl_verify', True)) if not ssl_verify and parsed_url.scheme == 'https': self.log.warning('Skipping SSL cert validation for %s based on configuration.' % (base_url)) # Limit of queues/nodes to collect metrics from max_detailed = { + EXCHANGE_TYPE: int(instance.get('max_detailed_exchanges', MAX_DETAILED_EXCHANGES)), QUEUE_TYPE: int(instance.get('max_detailed_queues', MAX_DETAILED_QUEUES)), NODE_TYPE: int(instance.get('max_detailed_nodes', MAX_DETAILED_NODES)), } # List of queues/nodes to collect metrics from specified = { + EXCHANGE_TYPE: { + 'explicit': instance.get('exchanges', []), + 'regexes': instance.get('exchanges_regexes', []), + }, QUEUE_TYPE: { 'explicit': instance.get('queues', []), 'regexes': instance.get('queues_regexes', []), @@ -166,15 +220,22 @@ def _get_vhosts(self, instance, base_url, auth=None, ssl_verify=True): def check(self, instance): base_url, max_detailed, specified, auth, ssl_verify, custom_tags = self._get_config(instance) try: - # Generate metrics from the status API. - self.get_stats(instance, base_url, QUEUE_TYPE, max_detailed[QUEUE_TYPE], specified[QUEUE_TYPE], custom_tags, - auth=auth, ssl_verify=ssl_verify) - self.get_stats(instance, base_url, NODE_TYPE, max_detailed[NODE_TYPE], specified[NODE_TYPE], custom_tags, - auth=auth, ssl_verify=ssl_verify) - vhosts = self._get_vhosts(instance, base_url, auth=auth, ssl_verify=ssl_verify) self.cached_vhosts[base_url] = vhosts - self.get_connections_stat(instance, base_url, CONNECTION_TYPE, vhosts, custom_tags, + + limit_vhosts = [] + if self._limit_vhosts(instance): + limit_vhosts = vhosts + + # Generate metrics from the status API. + self.get_stats(instance, base_url, EXCHANGE_TYPE, max_detailed[EXCHANGE_TYPE], specified[EXCHANGE_TYPE], + limit_vhosts, custom_tags, auth=auth, ssl_verify=ssl_verify) + self.get_stats(instance, base_url, QUEUE_TYPE, max_detailed[QUEUE_TYPE], specified[QUEUE_TYPE], + limit_vhosts, custom_tags, auth=auth, ssl_verify=ssl_verify) + self.get_stats(instance, base_url, NODE_TYPE, max_detailed[NODE_TYPE], specified[NODE_TYPE], + limit_vhosts, custom_tags, auth=auth, ssl_verify=ssl_verify) + + self.get_connections_stat(instance, base_url, CONNECTION_TYPE, vhosts, limit_vhosts, custom_tags, auth=auth, ssl_verify=ssl_verify) # Generate a service check from the aliveness API. In the case of an invalid response @@ -219,7 +280,10 @@ def _filter_list(self, data, explicit_filters, regex_filters, object_type, tag_f match = re.search(p, name) if match: if _is_affirmative(tag_families) and match.groups(): - data_line["queue_family"] = match.groups()[0] + if object_type == QUEUE_TYPE: + data_line["queue_family"] = match.groups()[0] + if object_type == EXCHANGE_TYPE: + data_line["exchange_family"] = match.groups()[0] matching_lines.append(data_line) match_found = True break @@ -227,8 +291,8 @@ def _filter_list(self, data, explicit_filters, regex_filters, object_type, tag_f if match_found: continue - # Absolute names work only for queues - if object_type != QUEUE_TYPE: + # Absolute names work only for queues and exchanges + if object_type != QUEUE_TYPE and object_type != EXCHANGE_TYPE: continue absolute_name = '%s/%s' % (data_line.get("vhost"), name) if absolute_name in explicit_filters: @@ -240,7 +304,10 @@ def _filter_list(self, data, explicit_filters, regex_filters, object_type, tag_f match = re.search(p, absolute_name) if match: if _is_affirmative(tag_families) and match.groups(): - data_line["queue_family"] = match.groups()[0] + if object_type == QUEUE_TYPE: + data_line["queue_family"] = match.groups()[0] + if object_type == EXCHANGE_TYPE: + data_line["exchange_family"] = match.groups()[0] matching_lines.append(data_line) match_found = True break @@ -259,23 +326,36 @@ def _get_tags(self, data, object_type, custom_tags): tags.append('%s_%s:%s' % (TAG_PREFIX, tag_list[t], tag)) return tags + custom_tags - def get_stats(self, instance, base_url, object_type, max_detailed, filters, custom_tags, auth=None, ssl_verify=True): + def get_stats(self, instance, base_url, object_type, max_detailed, filters, limit_vhosts, custom_tags, auth=None, ssl_verify=True): """ instance: the check instance base_url: the url of the rabbitmq management api (e.g. http://localhost:15672/api) - object_type: either QUEUE_TYPE or NODE_TYPE + object_type: either QUEUE_TYPE or NODE_TYPE or EXCHANGE_TYPE max_detailed: the limit of objects to collect for this type filters: explicit or regexes filters of specified queues or nodes (specified in the yaml file) """ instance_proxy = self.get_instance_proxy(instance, base_url) - data = self._get_data(urlparse.urljoin(base_url, object_type), auth=auth, - ssl_verify=ssl_verify, proxies=instance_proxy) - # Make a copy of this list as we will remove items from it at each # iteration explicit_filters = list(filters['explicit']) regex_filters = filters['regexes'] + data = [] + + # only do this if vhosts were specified, + # otherwise it'll just be making more queries for the same data + if self._limit_vhosts(instance) and object_type == QUEUE_TYPE: + for vhost in limit_vhosts: + url = '{}/{}'.format(object_type, urllib.quote_plus(vhost)) + try: + data += self._get_data(urlparse.urljoin(base_url, url), auth=auth, + ssl_verify=ssl_verify, proxies=instance_proxy) + except Exception as e: + self.log.debug("Couldn't grab queue data from vhost, {}: {}".format(vhost, e)) + else: + data = self._get_data(urlparse.urljoin(base_url, object_type), auth=auth, + ssl_verify=ssl_verify, proxies=instance_proxy) + """ data is a list of nodes or queues: data = [ {'status': 'running', 'node': 'rabbit@host', 'name': 'queue1', 'consumers': 0, 'vhost': '/', 'backing_queue_status': {'q1': 0, 'q3': 0, 'q2': 0, 'q4': 0, 'avg_ack_egress_rate': 0.0, 'ram_msg_count': 0, 'ram_ack_count': 0, 'len': 0, 'persistent_count': 0, 'target_ram_count': 'infinity', 'next_seq_id': 0, 'delta': ['delta', 'undefined', 0, 'undefined'], 'pending_acks': 0, 'avg_ack_ingress_rate': 0.0, 'avg_egress_rate': 0.0, 'avg_ingress_rate': 0.0}, 'durable': True, 'idle_since': '2013-10-03 13:38:18', 'exclusive_consumer_tag': '', 'arguments': {}, 'memory': 10956, 'policy': '', 'auto_delete': False}, @@ -299,12 +379,13 @@ def get_stats(self, instance, base_url, object_type, max_detailed, filters, cust if len(data) > max_detailed: # Display a warning in the info page self.warning( - "Too many queues to fetch. You must choose the %s you are interested in by editing the rabbitmq.yaml configuration file or get in touch with Datadog Support" % object_type) + "Too many items to fetch. You must choose the %s you are interested in by editing the rabbitmq.yaml configuration file or get in touch with Datadog Support" % object_type) for data_line in data[:max_detailed]: - # We truncate the list of nodes/queues if it's above the limit + # We truncate the list if it's above the limit self._get_metrics(data_line, object_type, custom_tags) + # get a list of the number of bindings on a given queue # /api/queues/vhost/name/bindings if object_type is QUEUE_TYPE: @@ -340,13 +421,30 @@ def _get_queue_bindings_metrics(self, base_url, custom_tags, data, instance_prox self.gauge('rabbitmq.queue.bindings.count', bindings_count, tags) - def get_connections_stat(self, instance, base_url, object_type, vhosts, custom_tags, auth=None, ssl_verify=True): + def get_connections_stat(self, instance, base_url, object_type, vhosts, limit_vhosts, custom_tags, auth=None, ssl_verify=True): """ Collect metrics on currently open connection per vhost. """ instance_proxy = self.get_instance_proxy(instance, base_url) - data = self._get_data(urlparse.urljoin(base_url, object_type), auth=auth, - ssl_verify=ssl_verify, proxies=instance_proxy) + + grab_all_data = True + + if self._limit_vhosts(instance): + grab_all_data = False + data = [] + for vhost in vhosts: + url = "vhosts/{}/{}".format(urllib.quote_plus(vhost), object_type) + try: + data += self._get_data(urlparse.urljoin(base_url, url), auth=auth, + ssl_verify=ssl_verify, proxies=instance_proxy) + except Exception as e: + # This will happen if there is no connection data to grab + self.log.debug("Couldn't grab connection data from vhost, {}: {}".format(vhost, e)) + + # sometimes it seems to need to fall back to this + if grab_all_data or not len(data): + data = self._get_data(urlparse.urljoin(base_url, object_type), auth=auth, + ssl_verify=ssl_verify, proxies=instance_proxy) stats = {vhost: 0 for vhost in vhosts} connection_states = defaultdict(int) @@ -389,6 +487,15 @@ def alert(self, base_url, max_detailed, size, object_type, custom_tags): self.event(event) + def _limit_vhosts(self, instance): + """ + Check to see if vhosts were specified in the instance + it will return a boolean, True if they were. + This allows the check to only query the wanted vhosts. + """ + vhosts = instance.get('vhosts', []) + return len(vhosts) > 0 + def _check_aliveness(self, instance, base_url, vhosts, custom_tags, auth=None, ssl_verify=True): """ Check the aliveness API against all or a subset of vhosts. The API diff --git a/redisdb/check.py b/redisdb/check.py --- a/redisdb/check.py +++ b/redisdb/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -238,24 +238,35 @@ def _check_db(self, instance, custom_tags=None): self.warning("keys in redis configuration is either not a list or empty") else: l_tags = list(tags) - for key in key_list: - key_type = conn.type(key) - key_tags = l_tags + ['key:' + key] - - if key_type == 'list': - self.gauge('redis.key.length', conn.llen(key), tags=key_tags) - elif key_type == 'set': - self.gauge('redis.key.length', conn.scard(key), tags=key_tags) - elif key_type == 'zset': - self.gauge('redis.key.length', conn.zcard(key), tags=key_tags) - elif key_type == 'hash': - self.gauge('redis.key.length', conn.hlen(key), tags=key_tags) + + for key_pattern in key_list: + if re.search(r"(? 2013 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/riakcs/check.py b/riakcs/check.py --- a/riakcs/check.py +++ b/riakcs/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # (C) Jon Glick 2014 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/snmp/check.py b/snmp/check.py --- a/snmp/check.py +++ b/snmp/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/spark/check.py b/spark/check.py --- a/spark/check.py +++ b/spark/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) ''' @@ -294,7 +294,7 @@ def _get_running_apps(self, instance, tags): return self._standalone_init(master_address, pre20) elif cluster_mode == SPARK_MESOS_MODE: - running_apps = self._mesos_init(master_address) + running_apps = self._mesos_init(instance, master_address) return self._get_spark_app_ids(running_apps) @@ -350,7 +350,7 @@ def _standalone_init(self, spark_master_address, pre_20_mode): self.log.info("Returning running apps %s" % running_apps) return running_apps - def _mesos_init(self, master_address): + def _mesos_init(self, instance, master_address): ''' Return a dictionary of {app_id: (app_name, tracking_url)} for running Spark applications. ''' @@ -367,7 +367,15 @@ def _mesos_init(self, master_address): app_name = app_json.get('name') if app_id and tracking_url and app_name: - running_apps[app_id] = (app_name, tracking_url) + spark_ports = instance.get('spark_ui_ports') + if spark_ports is None: + # No filtering by port, just return all the frameworks + running_apps[app_id] = (app_name, tracking_url) + else: + # Only return the frameworks running on the correct port + tracking_url_port = urlparse(tracking_url).port + if tracking_url_port in spark_ports: + running_apps[app_id] = (app_name, tracking_url) # Report success after gathering all metrics from ResourceManaager self.service_check(MESOS_SERVICE_CHECK, diff --git a/sqlserver/check.py b/sqlserver/check.py --- a/sqlserver/check.py +++ b/sqlserver/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) ''' @@ -121,11 +121,9 @@ def __init__(self, name, init_config, agentConfig, instances=None): self.log.error("Invalid database connector %s, defaulting to adodbapi" % self.connector) self.connector = 'adodbapi' - self.log.debug("instances: %s", str(instances)) # Pre-process the list of metrics to collect self.custom_metrics = init_config.get('custom_metrics', []) for instance in instances: - self.log.debug("initializing %s", str(instance)) try: instance_key = self._conn_key(instance, self.DEFAULT_DB_KEY) self.do_check[instance_key] = True @@ -265,8 +263,8 @@ def _make_metric_list_to_collect(self, instance, custom_metrics): elif type(m) is SqlOsWaitStat: self.log.debug("Adding SqlOsWaitStat metric %s", m.sql_name) wait_stat_metrics.append(m.sql_name) - elif type(m) is SqlOsVirtualFileStat: - self.log.debug("Adding SqlOsVirtualFileStat metric %s", m.sql_name) + elif type(m) is SqlIoVirtualFileStat: + self.log.debug("Adding SqlIoVirtualFileStat metric %s", m.sql_name) vfs_metrics.append(m.sql_name) elif type(m) is SqlOsMemoryClerksStat: self.log.debug("Adding SqlOsMemoryClerksStat metric %s", m.sql_name) @@ -275,7 +273,7 @@ def _make_metric_list_to_collect(self, instance, custom_metrics): self.instances_per_type_metrics[instance_key]["SqlSimpleMetric"] = simple_metrics self.instances_per_type_metrics[instance_key]["SqlFractionMetric"] = fraction_metrics self.instances_per_type_metrics[instance_key]["SqlOsWaitStat"] = wait_stat_metrics - self.instances_per_type_metrics[instance_key]["SqlOsVirtualFileStat"] = vfs_metrics + self.instances_per_type_metrics[instance_key]["SqlIoVirtualFileStat"] = vfs_metrics self.instances_per_type_metrics[instance_key]["SqlOsMemoryClerksStat"] = clerk_metrics def typed_metric(self, instance, cfg_inst, table, base_name, user_type, sql_type, column): @@ -305,7 +303,7 @@ def typed_metric(self, instance, cfg_inst, table, base_name, user_type, sql_type table_type_mapping = { DM_OS_WAIT_STATS_TABLE: (self.gauge, SqlOsWaitStat), DM_OS_MEMORY_CLERKS_TABLE: (self.gauge, SqlOsMemoryClerksStat), - DM_OS_VIRTUAL_FILE_STATS: (self.gauge, SqlOsVirtualFileStat) + DM_OS_VIRTUAL_FILE_STATS: (self.gauge, SqlIoVirtualFileStat) } metric_type, cls = table_type_mapping[table] @@ -470,7 +468,7 @@ def do_perf_counter_check(self, instance): simple_rows = SqlSimpleMetric.fetch_all_values(cursor, self.instances_per_type_metrics[instance_key]["SqlSimpleMetric"], self.log) fraction_results = SqlFractionMetric.fetch_all_values(cursor, self.instances_per_type_metrics[instance_key]["SqlFractionMetric"], self.log) waitstat_rows, waitstat_cols = SqlOsWaitStat.fetch_all_values(cursor, self.instances_per_type_metrics[instance_key]["SqlOsWaitStat"], self.log) - vfs_rows, vfs_cols = SqlOsVirtualFileStat.fetch_all_values(cursor, self.instances_per_type_metrics[instance_key]["SqlOsVirtualFileStat"], self.log) + vfs_rows, vfs_cols = SqlIoVirtualFileStat.fetch_all_values(cursor, self.instances_per_type_metrics[instance_key]["SqlIoVirtualFileStat"], self.log) clerk_rows, clerk_cols = SqlOsMemoryClerksStat.fetch_all_values(cursor, self.instances_per_type_metrics[instance_key]["SqlOsMemoryClerksStat"], self.log) for metric in metrics_to_collect: @@ -481,7 +479,7 @@ def do_perf_counter_check(self, instance): metric.fetch_metric(cursor, fraction_results, custom_tags) elif type(metric) is SqlOsWaitStat: metric.fetch_metric(cursor, waitstat_rows, waitstat_cols, custom_tags) - elif type(metric) is SqlOsVirtualFileStat: + elif type(metric) is SqlIoVirtualFileStat: metric.fetch_metric(cursor, vfs_rows, vfs_cols, custom_tags) elif type(metric) is SqlOsMemoryClerksStat: metric.fetch_metric(cursor, clerk_rows, clerk_cols, custom_tags) @@ -638,7 +636,7 @@ def __init__(self, connector, cfg_instance, base_name, self.connector = connector self.cfg_instance = cfg_instance self.datadog_name = cfg_instance['name'] - self.sql_name = cfg_instance['counter_name'] + self.sql_name = cfg_instance.get('counter_name', '') self.base_name = base_name self.report_function = report_function self.instance = cfg_instance.get('instance_name', '') @@ -685,7 +683,8 @@ def fetch_metric(self, cursor, rows, tags): metric_tags = metric_tags + ['%s:%s' % (self.tag_by, instance_name.strip())] self.report_function(self.datadog_name, cntr_value, tags=metric_tags) - break + if self.instance != ALL_INSTANCES: + break class SqlFractionMetric(SqlServerMetric): @@ -819,9 +818,10 @@ def fetch_metric(self, cursor, rows, columns, tags): self.log.debug("Value for %s %s is %d", self.sql_name, self.column, value) metric_tags = tags - self.report_function(self.datadog_name, value, tags=metric_tags) + metric_name = '%s.%s' % (self.datadog_name, self.column) + self.report_function(metric_name, value, tags=metric_tags) -class SqlOsVirtualFileStat(SqlServerMetric): +class SqlIoVirtualFileStat(SqlServerMetric): @classmethod def fetch_all_values(cls, cursor, counters_list, logger): @@ -836,7 +836,7 @@ def fetch_all_values(cls, cursor, counters_list, logger): def __init__(self, connector, cfg_instance, base_name, report_function, column, logger): - super(SqlOsVirtualFileStat, self).__init__(connector, cfg_instance, + super(SqlIoVirtualFileStat, self).__init__(connector, cfg_instance, base_name, report_function, column, logger) self.dbid = self.cfg_instance.get('database_id', None) diff --git a/ssh_check/check.py b/ssh_check/check.py --- a/ssh_check/check.py +++ b/ssh_check/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -56,17 +56,19 @@ def check(self, instance): tags = ["instance:{0}-{1}".format(conf.host, conf.port)] private_key = None - try: - if conf.private_key_type == 'ecdsa': - private_key = paramiko.ECDSAKey.from_private_key_file(conf.private_key_file) - else: - private_key = paramiko.RSAKey.from_private_key_file(conf.private_key_file) - except IOError: - self.warning("Unable to find private key file: {}".format(conf.private_key_file)) - except paramiko.ssh_exception.PasswordRequiredException: - self.warning("Private key file is encrypted but no password was given") - except paramiko.ssh_exception.SSHException: - self.warning("Private key file is invalid") + + if conf.private_key_file is not None: + try: + if conf.private_key_type == 'ecdsa': + private_key = paramiko.ECDSAKey.from_private_key_file(conf.private_key_file) + else: + private_key = paramiko.RSAKey.from_private_key_file(conf.private_key_file) + except IOError: + self.warning("Unable to find private key file: {}".format(conf.private_key_file)) + except paramiko.ssh_exception.PasswordRequiredException: + self.warning("Private key file is encrypted but no password was given") + except paramiko.ssh_exception.SSHException: + self.warning("Private key file is invalid") client = paramiko.SSHClient() if conf.add_missing_keys: diff --git a/statsd/check.py b/statsd/check.py --- a/statsd/check.py +++ b/statsd/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2015-2016 +# (C) Datadog, Inc. 2015-2017 # (C) Cory Watson 2015 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/supervisord/check.py b/supervisord/check.py --- a/supervisord/check.py +++ b/supervisord/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -48,14 +48,14 @@ class SupervisordCheck(AgentCheck): - def check(self, instance): server_name = instance.get('name') if not server_name or not server_name.strip(): raise Exception("Supervisor server name not specified in yaml configuration.") - server_service_check_tags = ['%s:%s' % (SERVER_TAG, server_name)] + instance_tags = instance.get('tags', []) + instance_tags.append('%s:%s' % (SERVER_TAG, server_name)) supe = self._connect(instance) count_by_status = defaultdict(int) @@ -81,8 +81,7 @@ def check(self, instance): ' has the right permissions.' % sock self.service_check(SERVER_SERVICE_CHECK, AgentCheck.CRITICAL, - tags=server_service_check_tags, - message=msg) + tags=instance_tags, message=msg) raise Exception(msg) @@ -90,17 +89,15 @@ def check(self, instance): if e.errcode == 401: # authorization error msg = 'Username or password to %s are incorrect.' % server_name else: - msg = "An error occurred while connecting to %s: "\ - "%s %s " % (server_name, e.errcode, e.errmsg) + msg = "An error occurred while connecting to %s: " \ + "%s %s " % (server_name, e.errcode, e.errmsg) self.service_check(SERVER_SERVICE_CHECK, AgentCheck.CRITICAL, - tags=server_service_check_tags, - message=msg) + tags=instance_tags, message=msg) raise Exception(msg) # If we're here, we were able to connect to the server - self.service_check(SERVER_SERVICE_CHECK, AgentCheck.OK, - tags=server_service_check_tags) + self.service_check(SERVER_SERVICE_CHECK, AgentCheck.OK, tags=instance_tags) # Filter monitored processes on configuration directives proc_regex = instance.get('proc_regex', []) @@ -129,24 +126,21 @@ def check(self, instance): # Report service checks and uptime for each process for proc in monitored_processes: proc_name = proc['name'] - tags = ['%s:%s' % (SERVER_TAG, server_name), - '%s:%s' % (PROCESS_TAG, proc_name)] + tags = instance_tags + ['%s:%s' % (PROCESS_TAG, proc_name)] # Report Service Check status = DD_STATUS[proc['statename']] msg = self._build_message(proc) count_by_status[status] += 1 - self.service_check(PROCESS_SERVICE_CHECK, - status, tags=tags, message=msg) + self.service_check(PROCESS_SERVICE_CHECK, status, tags=tags, message=msg) # Report Uptime uptime = self._extract_uptime(proc) self.gauge('supervisord.process.uptime', uptime, tags=tags) # Report counts by status - tags = ['%s:%s' % (SERVER_TAG, server_name)] for status in PROCESS_STATUS: self.gauge('supervisord.process.count', count_by_status[status], - tags=tags + ['status:%s' % PROCESS_STATUS[status]]) + tags=instance_tags + ['status:%s' % PROCESS_STATUS[status]]) @staticmethod def _connect(instance): diff --git a/system_core/check.py b/system_core/check.py --- a/system_core/check.py +++ b/system_core/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -10,15 +10,17 @@ class SystemCore(AgentCheck): - def check(self, instance): + instance_tags = instance.get('tags', []) + cpu_times = psutil.cpu_times(percpu=True) - self.gauge("system.core.count", len(cpu_times)) + self.gauge("system.core.count", len(cpu_times), tags=instance_tags) for i, cpu in enumerate(cpu_times): + tags = instance_tags + ["core:{0}".format(i)] for key, value in cpu._asdict().iteritems(): self.rate( "system.core.{0}".format(key), 100.0 * value, - tags=["core:{0}".format(i)] + tags=tags ) diff --git a/system_swap/check.py b/system_swap/check.py --- a/system_swap/check.py +++ b/system_swap/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/tcp_check/check.py b/tcp_check/check.py --- a/tcp_check/check.py +++ b/tcp_check/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/teamcity/check.py b/teamcity/check.py --- a/teamcity/check.py +++ b/teamcity/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2015-2016 +# (C) Datadog, Inc. 2015-2017 # (C) Paul Kirby 2014 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/tokumx/check.py b/tokumx/check.py --- a/tokumx/check.py +++ b/tokumx/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2014-2016 +# (C) Datadog, Inc. 2014-2017 # (C) Leif Walsh 2014 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/twemproxy/check.py b/twemproxy/check.py --- a/twemproxy/check.py +++ b/twemproxy/check.py @@ -1,3 +1,7 @@ +# (C) Datadog, Inc. 2016-2017 +# All rights reserved +# Licensed under Simplified BSD License (see LICENSE) + # stdlib import socket diff --git a/varnish/check.py b/varnish/check.py --- a/varnish/check.py +++ b/varnish/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -9,6 +9,7 @@ import re import xml.parsers.expat # python 2.4 compatible import shlex +import simplejson as json # project from checks import AgentCheck @@ -30,6 +31,11 @@ def to_check_status(cls, status): class Varnish(AgentCheck): SERVICE_CHECK_NAME = 'varnish.backend_healthy' + # Parse metrics from varnishstat. + VARNISHSTAT_FORMAT_OPTION = {"text": "-1", # version < 3.0.0 + "xml": "-x", # version >= 3.0.0 < 5.0.0 + "json": "-j", # version >=5.0.0 + } # Output of varnishstat -V : `varnishstat (varnish-4.1.1 revision 66bb824)` version_pattern = re.compile(r'(\d+\.\d+\.\d+)') @@ -97,11 +103,9 @@ def check(self, instance): raise Exception("The parameter 'metrics_filter' must be a list") # Get version and version-specific args from varnishstat -V. - version, use_xml = self._get_version_info(varnishstat_path) + version, varnishstat_format = self._get_version_info(varnishstat_path) - # Parse metrics from varnishstat. - arg = '-x' if use_xml else '-1' - cmd = varnishstat_path + [arg] + cmd = varnishstat_path + [self.VARNISHSTAT_FORMAT_OPTION[varnishstat_format]] for metric in metrics_filter: cmd.extend(["-f", metric]) @@ -113,7 +117,7 @@ def check(self, instance): output, _, _ = get_subprocess_output(cmd, self.log) - self._parse_varnishstat(output, use_xml, tags) + self._parse_varnishstat(output, varnishstat_format, tags) # Parse service checks from varnishadm. if instance.get("varnishadm", None): @@ -154,7 +158,7 @@ def _get_version_info(self, varnishstat_path): raise_on_empty_output=False) # Assumptions regarding varnish's version - use_xml = True + varnishstat_format = "json" version = LooseVersion('3.0.0') m1 = self.version_pattern.search(output, re.MULTILINE) @@ -174,13 +178,14 @@ def _get_version_info(self, varnishstat_path): # Location of varnishstat if version < LooseVersion('3.0.0'): - use_xml = False + varnishstat_format = "text" + elif version < LooseVersion('5.0.0'): # we default to json starting version 5.0.0 + varnishstat_format = "xml" - return version, use_xml - - def _parse_varnishstat(self, output, use_xml, tags=None): - """Extract stats from varnishstat -x + return version, varnishstat_format + def _parse_varnishstat(self, output, varnishstat_format, tags=None): + """ The text option (-1) is not reliable enough when counters get large. VBE.media_video_prd_services_01(10.93.67.16,,8080).happy18446744073709551615 @@ -188,35 +193,12 @@ def _parse_varnishstat(self, output, use_xml, tags=None): https://github.com/varnish/Varnish-Cache/blob/master/include/tbl/vsc_fields.h Bitmaps are not supported. - - Example XML output (with `use_xml=True`) - - - fetch_304 - 0 - a - Fetch no body (304) - - - n_sess_mem - 334 - i - N struct sess_mem - - - LCK - vcl - creat - 1 - a - Created locks - - """ + tags = tags or [] # FIXME: this check is processing an unbounded amount of data # we should explicitly list the metrics we want to get from the check - if use_xml: + if varnishstat_format == "xml": p = xml.parsers.expat.ParserCreate() p.StartElementHandler = self._start_element end_handler = lambda name: self._end_element(name, tags) @@ -224,7 +206,21 @@ def _parse_varnishstat(self, output, use_xml, tags=None): p.CharacterDataHandler = self._char_data self._reset() p.Parse(output, True) - else: + elif varnishstat_format == "json": + json_output = json.loads(output) + for name, metric in json_output.iteritems(): + if not isinstance(metric, dict): # skip 'timestamp' field + continue + + if name.startswith("MAIN."): + name = name.split('.', 1)[1] + value = metric.get("value", 0) + + if metric["flag"] in ("a", "c"): + self.rate(self.normalize(name, prefix="varnish"), long(value), tags=tags) + elif metric["flag"] in ("g", "i"): + self.gauge(self.normalize(name, prefix="varnish"), long(value), tags=tags) + elif varnishstat_format == "text": for line in output.split("\n"): self.log.debug("Parsing varnish results: %s" % line) fields = line.split() diff --git a/vsphere/check.py b/vsphere/check.py --- a/vsphere/check.py +++ b/vsphere/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/win32_event_log/check.py b/win32_event_log/check.py --- a/win32_event_log/check.py +++ b/win32_event_log/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) @@ -41,6 +41,7 @@ def __init__(self, name, init_config, agentConfig, instances=None): # Settings self._tag_event_id = init_config.get('tag_event_id', False) self._verbose = init_config.get('verbose', True) + self._default_event_priority = init_config.get('default_event_priority', 'normal') # State self.last_ts = {} @@ -52,6 +53,9 @@ def check(self, instance): password = instance.get('password', "") instance_tags = instance.get('tags', []) notify = instance.get('notify', []) + event_priority = instance.get('event_priority', self._default_event_priority) + if (event_priority.lower() != 'normal') and (event_priority.lower() != 'low'): + event_priority = 'normal' user = instance.get('user') ltypes = instance.get('type', []) @@ -138,7 +142,7 @@ def check(self, instance): hostname = None if (host == "localhost" or host == ".") else host log_ev = LogEvent( ev, self.log, hostname, instance_tags, notify, - self._tag_event_id, event_format + self._tag_event_id, event_format, event_priority ) # Since WQL only compares on the date and NOT the time, we have to @@ -162,7 +166,7 @@ def _dt_to_wmi(self, dt): class LogEvent(object): - def __init__(self, ev, log, hostname, tags, notify_list, tag_event_id, event_format): + def __init__(self, ev, log, hostname, tags, notify_list, tag_event_id, event_format, event_priority): self.event = ev self.log = log self.hostname = hostname @@ -170,6 +174,7 @@ def __init__(self, ev, log, hostname, tags, notify_list, tag_event_id, event_for self.notify_list = notify_list self.timestamp = self._wmi_to_ts(self.event['TimeGenerated']) self._format = event_format + self.event_priority = event_priority @property def _msg_title(self): @@ -237,6 +242,7 @@ def to_event_dict(self): event_dict = { 'timestamp': self.timestamp, 'event_type': EVENT_TYPE, + 'priority': self.event_priority, 'msg_title': self._msg_title, 'msg_text': self._msg_text.strip(), 'aggregation_key': self._aggregation_key, diff --git a/wmi_check/check.py b/wmi_check/check.py --- a/wmi_check/check.py +++ b/wmi_check/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2013-2016 +# (C) Datadog, Inc. 2013-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/yarn/check.py b/yarn/check.py --- a/yarn/check.py +++ b/yarn/check.py @@ -1,4 +1,4 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) diff --git a/zk/check.py b/zk/check.py --- a/zk/check.py +++ b/zk/check.py @@ -1,19 +1,8 @@ -# (C) Datadog, Inc. 2010-2016 +# (C) Datadog, Inc. 2010-2017 # All rights reserved # Licensed under Simplified BSD License (see LICENSE) -''' -As of zookeeper 3.4.0, the `mntr` admin command is provided for easy parsing of zookeeper stats. -This check first parses the `stat` admin command for a version number. -If the zookeeper version supports `mntr`, it is also parsed. - -Duplicate information is being reported by both `mntr` and `stat` to keep backwards compatibility. -Example: - `stat` reports: zookeeper.latency.avg - `mntr` reports: zookeeper.avg.latency -If available, make use of the data reported by `mntr` not `stat`. -The duplicate `stat` reports are only kept for backward compatibility. - +""" Besides the usual zookeeper state of `leader`, `follower`, `observer` and `standalone`, this check will report three other states: @@ -24,8 +13,8 @@ States can be accessed through the gauge `zookeeper.instances., through the set `zookeeper.instances`, or through the `mode:` tag. -Parses the response from zookeeper's `stat` admin command, which looks like: - +Parses the response from zookeeper's `stat` admin command, tested with Zookeeper +versions 3.0.0 to 3.4.5, which looks like: ``` Zookeeper version: 3.2.2--1, built on 03/16/2010 07:31 GMT Clients: @@ -45,10 +34,9 @@ Node count: 487 ``` -`stat` tested with Zookeeper versions 3.0.0 to 3.4.5 - -The following is an example of the `mntr` commands output: +The following is an example of the `mntr` commands output, tested with ZooKeeper +3.4.5: ``` zk_version 3.4.5-cdh4.4.0--1, built on 09/04/2013 01:46 GMT zk_avg_latency 0 @@ -67,8 +55,7 @@ zk_max_file_descriptor_count 4096 ``` -`mntr` tested with ZooKeeper 3.4.5 -''' +""" # stdlib from collections import defaultdict from distutils.version import LooseVersion # pylint: disable=E0611,E0401 @@ -163,7 +150,7 @@ def check(self, instance): 'zookeeper.ruok', status, message=message, tags=sc_tags ) - # Read metrics from the `stat` output. + # Read metrics from the `stat` output try: stat_out = self._send_command('stat', *cx_args) except ZKConnectionFailure: @@ -272,9 +259,10 @@ def _send_command(self, command, host, port, timeout): return buf def parse_stat(self, buf): - ''' `buf` is a readable file-like object - returns a tuple: (metrics, tags, mode, version) - ''' + """ + `buf` is a readable file-like object + returns a tuple: (metrics, tags, mode, version) + """ metrics = [] buf.seek(0) @@ -310,11 +298,17 @@ def parse_stat(self, buf): # Received: 101032173 _, value = buf.readline().split(':') + # Fixme: This metric name is wrong. It should be removed in a major version of the agent + # See https://github.com/DataDog/integrations-core/issues/816 metrics.append(ZKMetric('zookeeper.bytes_received', long(value.strip()))) + metrics.append(ZKMetric('zookeeper.packets.received', long(value.strip()), "rate")) # Sent: 1324 _, value = buf.readline().split(':') + # Fixme: This metric name is wrong. It should be removed in a major version of the agent + # See https://github.com/DataDog/integrations-core/issues/816 metrics.append(ZKMetric('zookeeper.bytes_sent', long(value.strip()))) + metrics.append(ZKMetric('zookeeper.packets.sent', long(value.strip()), "rate")) if has_connections_val: # Connections: 1 @@ -358,13 +352,13 @@ def parse_stat(self, buf): return metrics, tags, mode, version def parse_mntr(self, buf): - ''' + """ Parse `mntr` command's content. `buf` is a readable file-like object Returns: a tuple (metrics, mode) if mode == 'inactive', metrics will be None - ''' + """ buf.seek(0) first = buf.readline() # First is version string or error if first == 'This ZooKeeper instance is not currently serving requests':