diff --git a/logster/parsers/MetricLogster.py b/logster/parsers/MetricLogster.py index 6963d73..2904757 100644 --- a/logster/parsers/MetricLogster.py +++ b/logster/parsers/MetricLogster.py @@ -57,10 +57,13 @@ def __init__(self, option_string=None): optparser = optparse.OptionParser() optparser.add_option('--percentiles', '-p', dest='percentiles', default='90', help='Comma-separated list of integer percentiles to track: (default: "90")') + optparser.add_option('--raw', action="store_true", dest='raw', + help='Send raw metric to graphite, do not take time duration into metric calculation') opts, args = optparser.parse_args(args=options) self.percentiles = opts.percentiles.split(',') + self.raw_data = opts.raw # General regular expressions, expecting the metric name to be included in the log file. @@ -92,8 +95,10 @@ def get_state(self, duration): and return a list of metric objects.''' duration = float(duration) metrics = [] - if duration > 0: + if duration > 0 and not self.raw_data: metrics += [MetricObject(counter, self.counts[counter]/duration) for counter in self.counts] + elif self.raw_data: + metrics += [MetricObject(counter, self.counts[counter]) for counter in self.counts] for time_name in self.times: values = self.times[time_name]['values'] unit = self.times[time_name]['unit'] diff --git a/logster/parsers/OccurrenceLogster.py b/logster/parsers/OccurrenceLogster.py new file mode 100644 index 0000000..e7753ff --- /dev/null +++ b/logster/parsers/OccurrenceLogster.py @@ -0,0 +1,71 @@ +import re +import optparse + +from logster.logster_helper import MetricObject, LogsterParser, LogsterParsingException + + +class OccurrenceLogster(LogsterParser): + + def __init__(self, option_string=None): + '''Initialize any data structures or variables needed for keeping track + of the tasty bits we find in the log we are parsing.''' + self.occurrence = [] + self.pattern_list = [] + self.error_type = {} + + if option_string: + options = option_string.split(' ') + else: + options = [] + + optparser = optparse.OptionParser() + optparser.add_option('--pattern-file', '-p', dest='pattern_file', + help='Path to file with pattern list. File struct: "pattern-name";"pattern";') + optparser.add_option('--raw', action="store_true", dest='raw', + help='Send raw metric to graphite, do not take time duration into metric calculation') + + opts, args = optparser.parse_args(args=options) + + if opts.pattern_file: + for line in open(opts.pattern_file): + if len(line.strip()) > 0: + pattern_line = self.read_pattern_line(line) + self.pattern_list.append([pattern_line[0], re.compile(pattern_line[1], flags=re.IGNORECASE)]) + else: + optparser.error('File with patterns not given') + + self.raw_data = opts.raw + + def read_pattern_line(self, line): + line_split = line.split(" : ") + return [self.remove_quotation_marks(line_split[0]), self.remove_quotation_marks(line_split[1])] + + def remove_quotation_marks(self, word): + return word.replace("\"", "").strip() + + def parse_line(self, line): + '''This function should digest the contents of one line at a time, updating + object's state variables. Takes a single argument, the line to be parsed.''' + + for pattern in self.pattern_list: + try: + search = pattern[1].search(line) + if search: + self.error_type.update({pattern[0]: self.error_type.get(pattern[0], 0)+1}) + except Exception as e: + raise LogsterParsingException("Pattern search failed with %s" % e) + + def get_state(self, duration): + '''Run any necessary calculations on the data collected from the logs + and return a list of metric objects.''' + self.duration = float(duration) + + # Return a list of metrics objects + result = [] + for key, occ in self.error_type.iteritems(): + if self.raw_data: + metric = MetricObject(key, occ, "Responses") + else: + metric = MetricObject(key, (occ / self.duration), "Responses per sec") + result.append(metric) + return result diff --git a/tests/resources/patterns.txt b/tests/resources/patterns.txt new file mode 100644 index 0000000..edee37e --- /dev/null +++ b/tests/resources/patterns.txt @@ -0,0 +1,2 @@ +"errors" : "error" +"hour_pattern" : "(([0-9]{2}:){2}[0-9]{2})" diff --git a/tests/test_occureencelogster.py b/tests/test_occureencelogster.py new file mode 100644 index 0000000..d596ac8 --- /dev/null +++ b/tests/test_occureencelogster.py @@ -0,0 +1,72 @@ +from logster.parsers.OccurrenceLogster import OccurrenceLogster +import unittest, os, time + +TEST_PATTERN_FILE = os.path.join(os.path.dirname(__file__), 'resources/patterns.txt') + + +class TestOccurrenceLogster(unittest.TestCase): + + def setUp(self): + self.logster_raw = OccurrenceLogster("-p {} --raw".format(TEST_PATTERN_FILE)) + self.logster = OccurrenceLogster("-p {}".format(TEST_PATTERN_FILE)) + + def test_valid_lines_raw(self): + access_log_tmpl = "{} [%s], something goes in some way.".format(time.ctime()) + self.logster_raw.parse_line(access_log_tmpl % 'ERROR') + self.logster_raw.parse_line(access_log_tmpl % 'error') + self.logster_raw.parse_line(access_log_tmpl % 'WARN') + self.assertEqual(2, self.logster_raw.error_type['errors']) + self.assertEqual(3, self.logster_raw.error_type['hour_pattern']) + self.assertEqual(2, len(self.logster_raw.error_type)) + + def test_metrics_1sec_raw(self): + self.test_valid_lines_raw() + metrics = self.logster_raw.get_state(1) + self.assertEqual(2, len(metrics)) + + expected = {"errors": 2, + "hour_pattern": 3} + for m in metrics: + self.assertEqual(expected[m.name], m.value) + + def test_metrics_2sec_raw(self): + self.test_valid_lines_raw() + metrics = self.logster_raw.get_state(2) + self.assertEqual(2, len(metrics)) + + expected = {"errors": 2, + "hour_pattern": 3} + for m in metrics: + self.assertEqual(expected[m.name], m.value) + + def test_valid_lines(self): + access_log_tmpl = "{} [%s], something goes in some way.".format(time.ctime()) + self.logster.parse_line(access_log_tmpl % 'ERROR') + self.logster.parse_line(access_log_tmpl % 'error') + self.logster.parse_line(access_log_tmpl % 'WARN') + self.assertEqual(2, self.logster.error_type['errors']) + self.assertEqual(3, self.logster.error_type['hour_pattern']) + self.assertEqual(2, len(self.logster.error_type)) + + def test_metrics_1sec(self): + self.test_valid_lines() + metrics = self.logster.get_state(1) + self.assertEqual(2, len(metrics)) + + expected = {"errors": 2, + "hour_pattern": 3} + for m in metrics: + self.assertEqual(expected[m.name], m.value) + + def test_metrics_2sec(self): + self.test_valid_lines() + metrics = self.logster.get_state(2) + self.assertEqual(2, len(metrics)) + + expected = {"errors": 1.0, + "hour_pattern": 1.5} + for m in metrics: + self.assertEqual(expected[m.name], m.value) + +if __name__ == '__main__': + unittest.main()