From fb663a5ed281729bd8ab2b6b520efd07f62b2a1b Mon Sep 17 00:00:00 2001 From: Adilson Somensari Date: Thu, 1 Feb 2018 16:12:08 -0500 Subject: [PATCH 1/6] Add Support for New Relic Insights Output --- logster/outputs/builtin.py | 10 +++-- logster/outputs/insights.py | 73 +++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 logster/outputs/insights.py diff --git a/logster/outputs/builtin.py b/logster/outputs/builtin.py index f383f9e..94d78d5 100644 --- a/logster/outputs/builtin.py +++ b/logster/outputs/builtin.py @@ -1,9 +1,10 @@ -from logster.outputs.stdout import StdoutOutput -from logster.outputs.graphite import GraphiteOutput -from logster.outputs.ganglia import GangliaOutput -from logster.outputs.statsd import StatsdOutput from logster.outputs.cloudwatch import CloudwatchOutput +from logster.outputs.ganglia import GangliaOutput +from logster.outputs.graphite import GraphiteOutput +from logster.outputs.insights import InsightsOutput from logster.outputs.nsca import NSCAOutput +from logster.outputs.statsd import StatsdOutput +from logster.outputs.stdout import StdoutOutput builtin_output_classes = ( StdoutOutput, @@ -12,6 +13,7 @@ StatsdOutput, CloudwatchOutput, NSCAOutput, + InsightsOutput ) builtin_outputs = dict([(a.shortname, a) for a in builtin_output_classes]) diff --git a/logster/outputs/insights.py b/logster/outputs/insights.py new file mode 100644 index 0000000..cbc0bf6 --- /dev/null +++ b/logster/outputs/insights.py @@ -0,0 +1,73 @@ +import json +import os +from httplib import * + +from logster.logster_helper import LogsterOutput + + +class InsightsOutput(LogsterOutput): + shortname = 'insights' + + @classmethod + def add_options(cls, parser): + parser.add_option('--stdout-separator', action='store', default="_", dest="stdout_separator", + help='Separator between prefix/suffix and name for stdout. Default is \"%default\".') + parser.add_option('--event_type_name', action='store', default="Logster", dest="event_type_name", + help='Event Type as it will be used by Insights. Default is \"%default\".') + parser.add_option('--insights_api_key', action='store', + default=os.getenv('INSIGHTS_API_KEY_ID'), help='New Relic Insights API Insert key') + parser.add_option('--newrelic_account_number', action='store', + default=os.getenv('NEW_RELIC_ACCOUNT'), help='New Relic Account Number') + + def __init__(self, parser, options, logger): + super(InsightsOutput, self).__init__(parser, options, logger) + self.separator = options.stdout_separator + self.insights_url = 'insights-collector.newrelic.com' + + if not options.newrelic_account_number or not options.insights_api_key: + parser.print_help() + parser.error( + "You must supply --insights_api_key and --newrelic_account_number or Set environment variables. INSIGHTS_API_KEY_ID for --insights_api_key, NEW_RELIC_ACCOUNT for --newrelic_account_number") + + self.eventType = options.event_type_name + self.newrelic_account_number = options.newrelic_account_number + self.insights_api_key = options.insights_api_key + self.full_url = '/v1/accounts/' + self.newrelic_account_number + '/events' + + def submit(self, metrics): + + eventData = [] + + for metric in metrics: + metric.eventType = self.eventType + eventData.append(metric) + metric_name = self.get_metric_name(metric, self.separator) + """ print("%s %s %s" % (metric.timestamp, metric_name, metric.value)) """ + + try: + conn = HTTPSConnection(self.insights_url) + headers = { + "Content-Type": "application/json", + "X-Insert-Key": self.insights_api_key + } + + """ print json.dumps([ob.__dict__ for ob in eventData]) """ + eee = self.getOHIStructure(eventData) + + print json.dumps([ob.__dict__ for ob in eee.itervalues()]) + + response = conn.request("POST", self.full_url, json.dumps([ob.__dict__ for ob in eventData.itervalues()]), headers) + + except Exception as ex: + raise Exception("Can't connect ", ex) + + def getOHIStructure(self, metrics): + OHIDataEvent = {} + OHIDataEvent["name"] = "com.newrelic.ohi" + OHIDataEvent["protocol_version"] = "1" + OHIDataEvent["integration_version"] = "1.0.0" + OHIDataEvent["metrics"] = {} + OHIDataEvent["metrics"] = metrics + + return OHIDataEvent + From ab6a8f4e93b6ab42b4ba560f1ffa76812f3411bb Mon Sep 17 00:00:00 2001 From: Adilson Somensari Date: Mon, 5 Feb 2018 18:31:12 -0500 Subject: [PATCH 2/6] OHI vs Standalone Integration --- logster/outputs/insights.py | 57 +++++++++++++++++++++++-------------- setup.py | 3 +- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/logster/outputs/insights.py b/logster/outputs/insights.py index cbc0bf6..e338fe1 100644 --- a/logster/outputs/insights.py +++ b/logster/outputs/insights.py @@ -1,8 +1,10 @@ -import json -import os +from logster.logster_helper import LogsterOutput from httplib import * -from logster.logster_helper import LogsterOutput +import json +import os +import jsonpickle +import platform class InsightsOutput(LogsterOutput): @@ -18,12 +20,16 @@ def add_options(cls, parser): default=os.getenv('INSIGHTS_API_KEY_ID'), help='New Relic Insights API Insert key') parser.add_option('--newrelic_account_number', action='store', default=os.getenv('NEW_RELIC_ACCOUNT'), help='New Relic Account Number') + parser.add_option('--integration_type', action='store', + default='standalone', help='Define New Relic Integration Type (ohi or standalone)') def __init__(self, parser, options, logger): super(InsightsOutput, self).__init__(parser, options, logger) self.separator = options.stdout_separator self.insights_url = 'insights-collector.newrelic.com' + self.integration_type = options.integration_type + if not options.newrelic_account_number or not options.insights_api_key: parser.print_help() parser.error( @@ -38,30 +44,39 @@ def submit(self, metrics): eventData = [] - for metric in metrics: - metric.eventType = self.eventType - eventData.append(metric) - metric_name = self.get_metric_name(metric, self.separator) - """ print("%s %s %s" % (metric.timestamp, metric_name, metric.value)) """ + if self.integration_type == "standalone": + + for metric in metrics: + metric.eventType = self.eventType + metric.osName = os.name + metric.platformSystem = platform.system() + metric.platformRelease = platform.release() + platform.python_version() + eventData.append(metric) + + try: + conn = HTTPSConnection(self.insights_url) + headers = { + "Content-Type": "application/json", + "X-Insert-Key": self.insights_api_key + } - try: - conn = HTTPSConnection(self.insights_url) - headers = { - "Content-Type": "application/json", - "X-Insert-Key": self.insights_api_key - } + conn.request("POST", self.full_url, json.dumps([ob.__dict__ for ob in eventData.__iter__()]), headers) - """ print json.dumps([ob.__dict__ for ob in eventData]) """ - eee = self.getOHIStructure(eventData) + response = conn.getresponse() - print json.dumps([ob.__dict__ for ob in eee.itervalues()]) + print response.status, response.reason - response = conn.request("POST", self.full_url, json.dumps([ob.__dict__ for ob in eventData.itervalues()]), headers) + except Exception as ex: + raise Exception("Can't connect ", ex) + else: + for metric in metrics: + metric.event_type = self.eventType + eventData.append(metric) - except Exception as ex: - raise Exception("Can't connect ", ex) + print jsonpickle.encode(self.getohistructure(eventData)) - def getOHIStructure(self, metrics): + def getohistructure(self, metrics): OHIDataEvent = {} OHIDataEvent["name"] = "com.newrelic.ohi" OHIDataEvent["protocol_version"] = "1" diff --git a/setup.py b/setup.py index 8f93e33..58869f0 100755 --- a/setup.py +++ b/setup.py @@ -19,7 +19,8 @@ 'logster/outputs' ], install_requires = [ - 'pygtail>=0.5.1' + 'pygtail>=0.5.1', + 'jsonpickle>=0.9.5' ], zip_safe=False, scripts=[ From 5668682ca582d25cebdb62a47dda1569cea90cb8 Mon Sep 17 00:00:00 2001 From: asomensari Date: Fri, 16 Feb 2018 11:39:25 -0500 Subject: [PATCH 3/6] Update outputs.md --- docs/outputs.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/outputs.md b/docs/outputs.md index 8cc6ee9..547405c 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -26,7 +26,8 @@ you'd like to use. ## Available Third-party outputs -- +- * [New Relic Insights][insights] (`insights`): An Analytics platform that allows for querying, dashboard + and alerting on the data collected byt logster. If you have an output you'd like to be included here, please open a pull request with a link to its source/GitHub repo and a brief description of its From a61d4f67caaf76dccacbc0bdedd1c3f7e8493708 Mon Sep 17 00:00:00 2001 From: asomensari Date: Fri, 16 Feb 2018 11:49:34 -0500 Subject: [PATCH 4/6] Create insights.md --- logster/outputs/insights.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 logster/outputs/insights.md diff --git a/logster/outputs/insights.md b/logster/outputs/insights.md new file mode 100644 index 0000000..4a40469 --- /dev/null +++ b/logster/outputs/insights.md @@ -0,0 +1,9 @@ +# New Relic Insights Output + +This output class reports data to New Relic Insights Platform [https://newrelic.com/insights] + +These are the configuration options: + +- '--event_type_name': Defines the eventType name that will used to report the data (default: 'Logster') +- '--insights_api_key': New Relic Insights API Insert key. Can be configured via enviroment variable 'INSIGHTS_API_KEY_ID' +- '--newrelic_account_number': New Relic Account Number. Can be configured via environment variable 'NEW_RELIC_ACCOUNT' From 48bf05842ddc826c704b5e4e89a5b37ebd7eb9a8 Mon Sep 17 00:00:00 2001 From: asomensari Date: Fri, 16 Feb 2018 11:51:57 -0500 Subject: [PATCH 5/6] Update outputs.md --- docs/outputs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/outputs.md b/docs/outputs.md index 547405c..568c615 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -27,7 +27,7 @@ you'd like to use. ## Available Third-party outputs - * [New Relic Insights][insights] (`insights`): An Analytics platform that allows for querying, dashboard - and alerting on the data collected byt logster. + and alerting on the data collected byt logster. Setup details can be found here: [New Relic Insights Output](https://github.com/asomensari/logster/blob/master/logster/outputs/insights.md) If you have an output you'd like to be included here, please open a pull request with a link to its source/GitHub repo and a brief description of its From 4ee034c1ab86df82d353665e064f174c4d6e934e Mon Sep 17 00:00:00 2001 From: asomensari Date: Fri, 16 Feb 2018 11:55:30 -0500 Subject: [PATCH 6/6] Update outputs.md --- docs/outputs.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/outputs.md b/docs/outputs.md index 568c615..25b125f 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -29,6 +29,7 @@ you'd like to use. - * [New Relic Insights][insights] (`insights`): An Analytics platform that allows for querying, dashboard and alerting on the data collected byt logster. Setup details can be found here: [New Relic Insights Output](https://github.com/asomensari/logster/blob/master/logster/outputs/insights.md) + If you have an output you'd like to be included here, please open a pull request with a link to its source/GitHub repo and a brief description of its use.