From 87b176f45367cacfa9665a39ac9936995b175788 Mon Sep 17 00:00:00 2001 From: Shane Gibbons Date: Fri, 14 Jan 2011 11:33:27 -0500 Subject: [PATCH 1/4] convert log lines to UTF-8 in order to prevent encoding issues when parsing logs --- lib/oink/memory_usage_reporter.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/oink/memory_usage_reporter.rb b/lib/oink/memory_usage_reporter.rb index 7461c97..1204fee 100644 --- a/lib/oink/memory_usage_reporter.rb +++ b/lib/oink/memory_usage_reporter.rb @@ -2,6 +2,7 @@ require "oink/base" require "oink/oinked_request/oinked_memory_request" require "oink/priority_queue" +require "iconv" module Oink @@ -12,9 +13,10 @@ def print(output) output.puts "\n-- REQUESTS --\n" if @format == :verbose + ic = Iconv.new('UTF-8//IGNORE', 'UTF-8') @inputs.each do |input| input.each_line do |line| - line = line.strip + line = ic.iconv(line.strip + ' ')[0..-2] # Skip this line since we're only interested in the Hodel 3000 compliant lines next unless line =~ HODEL_LOG_FORMAT_REGEX @@ -69,4 +71,4 @@ def print(output) end -end \ No newline at end of file +end From e3a31bc74fead990401793fc32fc7077929eb807 Mon Sep 17 00:00:00 2001 From: Shane Gibbons Date: Fri, 14 Jan 2011 12:41:51 -0500 Subject: [PATCH 2/4] begin/rescue encoding errors rather than transcoding all lines by defualt - it ends up being a lot faster in the average case --- lib/oink/memory_usage_reporter.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/oink/memory_usage_reporter.rb b/lib/oink/memory_usage_reporter.rb index 1204fee..e805306 100644 --- a/lib/oink/memory_usage_reporter.rb +++ b/lib/oink/memory_usage_reporter.rb @@ -2,7 +2,6 @@ require "oink/base" require "oink/oinked_request/oinked_memory_request" require "oink/priority_queue" -require "iconv" module Oink @@ -13,13 +12,18 @@ def print(output) output.puts "\n-- REQUESTS --\n" if @format == :verbose - ic = Iconv.new('UTF-8//IGNORE', 'UTF-8') @inputs.each do |input| input.each_line do |line| - line = ic.iconv(line.strip + ' ')[0..-2] + line = line.strip - # Skip this line since we're only interested in the Hodel 3000 compliant lines - next unless line =~ HODEL_LOG_FORMAT_REGEX + # Skip this line since we're only interested in the Hodel 3000 compliant lines + # Also skip lines that have the wrong character encoding + begin + next unless line =~ HODEL_LOG_FORMAT_REGEX + rescue => e + output.puts "\nSkipping malformed line" if @format == :verbose and e =~ /invalid byte sequence/ + next + end if line =~ /rails\[(\d+)\]/ pid = $1 From 291abd32fd4c52c3c0fa906d5d8b37b91eaf62b7 Mon Sep 17 00:00:00 2001 From: Shane Gibbons Date: Fri, 14 Jan 2011 12:59:16 -0500 Subject: [PATCH 3/4] fix for malformed line skip logging, wasn't properly filtering on exception message previously --- lib/oink/memory_usage_reporter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oink/memory_usage_reporter.rb b/lib/oink/memory_usage_reporter.rb index e805306..9205d8c 100644 --- a/lib/oink/memory_usage_reporter.rb +++ b/lib/oink/memory_usage_reporter.rb @@ -21,7 +21,7 @@ def print(output) begin next unless line =~ HODEL_LOG_FORMAT_REGEX rescue => e - output.puts "\nSkipping malformed line" if @format == :verbose and e =~ /invalid byte sequence/ + output.puts "\nSkipping malformed line" if @format == :verbose and e.message =~ /invalid byte sequence/ next end From b57f5c95ac96597f6d232a63e435189a2fdb536d Mon Sep 17 00:00:00 2001 From: Shane Gibbons Date: Wed, 26 Jan 2011 13:18:16 -0500 Subject: [PATCH 4/4] make active record reporter skip log lines with bad characters --- lib/oink/active_record_instantiation_reporter.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/oink/active_record_instantiation_reporter.rb b/lib/oink/active_record_instantiation_reporter.rb index 4ab86f7..d9774c1 100644 --- a/lib/oink/active_record_instantiation_reporter.rb +++ b/lib/oink/active_record_instantiation_reporter.rb @@ -18,7 +18,12 @@ def print(output) line = line.strip # Skip this line since we're only interested in the Hodel 3000 compliant lines - next unless line =~ HODEL_LOG_FORMAT_REGEX + begin + next unless line =~ HODEL_LOG_FORMAT_REGEX + rescue => e + output.puts "\nSkipping malformed line" if @format == :verbose and e.message =~ /invalid byte sequence/ + next + end if line =~ /rails\[(\d+)\]/ pid = $1 @@ -65,4 +70,4 @@ def print(output) end -end \ No newline at end of file +end