PROGRAMMING INTERVIEWS

Link to part 1
Link to part 2
Link to part 3

Entering data in SimpleDB

Below is the script, I wrote for parsing logs and make entries in SimpleDB.

#!/usr/bin/env ruby
require 'rubygems'
require 'file/tail'
require 'right_aws'
require 'logger'
require 'geoip'
g = GeoIP.new('GeoIP.dat')

$logger = Logger.new("sdb.log")
#Connection for SimpleDB interface
@sdb = RightAws::SdbInterface.new(AWS Access Key, AWS Secret Key, {:multi_thread => true, :logger => $logger})

@alltimehash = Hash.new()   #All time hash
@h = Hash.new()                  #Hourly hash
@lasthr = 0

#Function for entering data in simpleDB
def enter_in_sdb(date, country)
  if @alltimehash[:"#{country}"]
     @alltimehash[:"#{country}"] += 1
  else
     @alltimehash[:"#{country}"] = 1
  end  

  attributes = {}
  attributes['date'] = date
  attributes['count'] = @alltimehash[:"#{country}"]
  @sdb.put_attributes('Analytics', country, attributes, true)

  hr = date.split(":")[0]  
  if hr != @lasthr
     @h.clear
     @lasthr = hr
  end

  if @h[:"#{country}"]
     @h[:"#{country}"] += 1
  else
     @h[:"#{country}"] = 1
  end

  domain_name = "Analytics" + hr.to_s
  attributes['count'] = @h[:"#{country}"]
  @sdb.put_attributes(domain_name, country, attributes, true) 
end

File.open('haproxy.log') do |log|
  log.extend(File::Tail)
  log.interval = 0
  log.backward(10)
  uuid = 1
  while true
     begin
        log.tail { |line|
           #process each line to get timestamp and IP
           country = g.country(IP)[4]                    
           enter_in_sdb(timestamp, country)
        }
     rescue Exception=>e
        puts "Met an exception"
        puts e.inspect
        next
     end
  end
end