class LogStash::Filters::Sumo

Public Instance Methods

dashvod_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 21
def dashvod_parse(uarray)
  asset_id = uarray[3].split(":")[1].split("_").first.to_i
  last_seg = uarray.last.split(".")
  video_type = last_seg.last
  chunk_type = last_seg.first.include?('chunk_ctaudio') ? :audio : (last_seg.first.include?('chunk_ctvideo') ? :video : :other)

  {path_type: "dashvod", dist_type: "vod", video_type: video_type, asset_id: asset_id, chunk_type: chunk_type}
end
filter(event) click to toggle source

def register

# File lib/logstash/filters/sumo.rb, line 286
def filter(event)

  begin
    return unless filter?(@field)

    if @field
      fieldValue = event.get(@field)

      if fieldValue
        begin
          data = parse_page(fieldValue)
          data.each { |k, v| event.set(k.to_s,v) } if data
        rescue => e
          sep = "\n"
          puts "Error : #{e} data = #{data} backtrace=#{e.backtrace[0..2].join(sep)}"
          @logger.error("SUMO> ", :pg => fieldValue, :exception => e)
        end
      end
    end


    # filter_matched should go in the last line of our successful code
    rc = filter_matched(event)

  rescue => e
    puts "SUMO error #{e} #fieldvalue #{fieldValue} "
    @logger.error("SUMO> ", :pg => fieldValue, :exception => e)
  end
  rc
end
hdsvod_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 86
def hdsvod_parse uarray
  last_seg = uarray.last.split(".")
  if uarray.count < 4
    return  {path_type: "hdshvod", dist_type: "vod",video_type: "unknown"}
  end

  asset_id = uarray[3].split(":")[1].split(".").first.to_i
  video_type = last_seg.last
  vtype = nil
  vtype = "segment" if uarray.count > 5

  {path_type: "wowza vod", dist_type: "vod", video_type: "hds", asset_id: asset_id, vtype: vtype}
end
ism_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 100
def ism_parse uarray
  page_type = uarray.pop.split(".")
  page_info = uarray.pop

  ua = page_info.split("(")
  puts "ism_parse ua = #{ua}"
  asset_id = nil
  if ua.count > 1
    asset_id = ua[1].split("_")[0].to_i
  end

  {path_type: "ism", dist_type: "vod", video_type: page_type.last, asset_id: asset_id}
end
isml_live_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 114
def isml_live_parse uarray
  # ["", "live", "727992.isml"]
  page_name = uarray.pop
  splitted_page = page_name.split(".")

  return {} if splitted_page.count<2

  page_type = splitted_page[1]
  asset_id = splitted_page[0].to_i

  {path_type: "isml", dist_type: "live", video_type: page_type, asset_id: asset_id}
end
isml_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 127
def isml_parse uarray
  # ["", "live", "727992.isml"]

  # /live/727992.isml/Manifest
  if uarray.length > 2 && uarray.last == "Manifest" && uarray[1]== "live"
    aparts = uarray[2].to_s.split(".")
    asset_id = aparts[0]

    return {path_type: "live", dist_type: "live", video_type: "ism", vtype: "manifest", asset_id: asset_id.to_s.to_i}
  end

  page_name = uarray.pop
  splitted_page = page_name.split(".")

  return {} if splitted_page.count<2

  page_type = splitted_page[1]
  asset_id = splitted_page[0].to_i

  {path_type: "isml", dist_type: "vod", video_type: page_type, asset_id: asset_id}
end
ismusp_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 54
def ismusp_parse uarray
  if uarray.last == "Manifest"
    # handle  /origin2-od.akamai.tv2.no/ismusp/isi_mp4_0/2014-04-25/G018902sumo(809735_ISMUSP).ism/Manifest
    last_seg = uarray[-2].split(".")
    asset_id = last_seg[0].split("(")[1].split("_").first.to_i
    video_type = last_seg.last

    return {path_type: "ismusp", dist_type: "vod", video_type: video_type, asset_id: asset_id,vtype: "manifest"}

  end

  last_seg = uarray.last.split(".")
  asset_id = last_seg[0].split("(")[1].split("_").first.to_i
  video_type = last_seg.last

  {path_type: "ismusp", dist_type: "vod", video_type: video_type, asset_id: asset_id}
end
ismvod_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 72
def ismvod_parse uarray
  last_seg = uarray.last.split(".")
  if uarray.count < 4
    return  {path_type: "ismvod", dist_type: "vod",video_type: "unknown"}
  end

  asset_id = uarray[3].split(":")[1].split(".").first.to_i
  video_type = last_seg.last
  vtype = nil
  vtype = "segment" if uarray.count > 5

  {path_type: "ismvod", dist_type: "vod", video_type: video_type, asset_id: asset_id, vtype: vtype}
end
out_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 149
  def out_parse uarray
    return {path_type: "unknown - to few eleements", dist_type: "unknown", uarray: uarray} if uarray.count<4

    page = uarray[3].strip
    page_parts = page.split(".")



    return {path_type: "unknown - to few page elements", dist_type: "unknown", uarray: uarray} unless page_parts && page_parts.count > 1


#    /out/u/463577.mpd                                 => path_type=“elemental live”, dist_type=“live”, video_type=“dash”, asset_id=463577, vtype="manifest"
    if page_parts[1] == "mpd"
      return {path_type: "elemental live", dist_type: "live", video_type: "dash", vtype: "manifest", asset_id: page_parts[0].to_i}
    end


#    /out/u/463577_video_1_0_3425579.mp4?m=1500616216  => path_type=“elemental live”, dist_type=“live”,video_type=“dash”, asset_id=463577 , vtype="segment"
    if page_parts[1] == "mp4"
      first_part = page_parts[0].split("_").first

      return {path_type: "elemental live", dist_type: "live", video_type: "dash", vtype: "segment", asset_id: first_part.to_i}
    end

#    /out/u/880009.m3u8                                => path_type=“elemental live”, dist_type=“live”,video_type=“hls”, asset_id=880009, vtype="playlist"
#    /out/u/880009_6.m3u8                              => path_type=“elemental live”, dist_type=“live”,video_type=“hls”, asset_id=880009, vtype="chunklist"
    if page_parts[1] == "m3u8"

      first_part = page_parts[0].split("_")

      if first_part.count == 1
        return {path_type: "elemental live", dist_type: "live", video_type: "hls", vtype: "playlist", asset_id: page_parts[0].to_i}
      end

      return {path_type: "elemental live", dist_type: "live", video_type: "hls", vtype: "chunklist", asset_id: first_part[0].to_i}
    end

#    /out/u/880009_6_3584645.ts?m=1501848254           => path_type=“elemental live”, dist_type=“live”,video_type=“hls”, asset_id=880009, vtype="segment"
    if page_parts[1] == "ts"
      first_part = page_parts[0].split("_").first


      return {path_type: "elemental live", dist_type: "live", video_type: "hls", vtype: "segment", asset_id: first_part.to_i}
    end


    return    {path_type: "unknown type - to few arguments", dist_type: "unknown"} unless uarray.count > 3


#    /out/u/880009.ism/Manifest                               => path_type=“elemental live”, dist_type=“live”,video_type=“ism”, asset_id=880009, vtype="manifest"
    if uarray[4] == "Manifest"
      return {path_type: "elemental live", dist_type: "live", video_type: "ism", vtype: "manifest", asset_id: page_parts[0].to_i}
    end

#    /out/u/880009.ism/Events(72_1501848239)/QualityLevels(2499968)/Fragments(v=77110176346666)  => path_type=“elemental live”, dist_type=“live”,video_type=“ism”, asset_id=880009, vtype="segment"
    if uarray[4].include? "Events"
      return {path_type: "elemental live", dist_type: "live", video_type: "ism", vtype: "segment", asset_id: page_parts[0].to_i}
    end

# Default empty
    {path_type: "unknown type", dist_type: "unknown"}
  end
parse_page(page) click to toggle source
# File lib/logstash/filters/sumo.rb, line 213
def parse_page page
  return {} unless page.to_s.strip.length > 0
  begin
    rc = {page: page}
    uri = URI.parse(page)

   # puts "URI = #{uri} page=#{page}"

    uarray = uri.path.split("/")
    return {} if uarray.length < 2

    cache_key = ""
    start_word = uarray[1].to_s
    if start_word.include?("origin") && start_word.include?("akamai")
      cache_key = uarray.delete_at(1)
 #     puts "CACHEKEY #{cache_key}"
    end


    page_type = uarray[-2]

    return {} unless page_type && uarray.count > 0

    root_page = uarray[1].to_s.downcase

    #puts "root_page #{root_page} page_type=#{page_type} page=#{page}"

    rc = {}

    if root_page == 'wzlive'
      rc = wzlive_parse(uarray)
    elsif root_page.include?('dashvod')
      rc = dashvod_parse(uarray)
    elsif root_page == 'vod2'
      rc = vod2_parse(uarray)
    elsif root_page.start_with? 'vod'
      rc = vod_parse(uarray)
    elsif root_page == 'ismusp'
      rc = ismusp_parse(uarray)
    elsif root_page.include?('ismvod')
      rc = ismvod_parse(uarray)
    elsif root_page.include?('hdsvod')
      rc = hdsvod_parse(uarray)
    elsif root_page == 'out'
      rc = out_parse uarray
    elsif page_type.to_s.include?(".isml") && root_page.to_s.include?("live")
      rc = isml_parse(uarray)
    elsif page_type.to_s.include?(".ism")
      rc = ism_parse(uarray)
    elsif root_page == 'live'
      rc = isml_parse uarray
    elsif root_page.include?("healthcheck")
      rc = {path_type: "healthcheck", dist_type: "healthcheck", video_type:"healthcheck"}
    end
    rc[:cache_key]= cache_key if cache_key.length > 0
  rescue => e
    rc[:exception]= e.to_json
    rc[:page] = page
    puts "SUMO parse_page #{e} rc=#{rc}"
    @logger.error("SUMO.parse_page> ", :page => page, :exception => e)
  end
  rc
end
register() click to toggle source
# File lib/logstash/filters/sumo.rb, line 279
def register

end
vod2_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 46
def vod2_parse uarray
  asset_id = uarray[3].split(":")[1].split("_").first.to_i
  last_seg = uarray.last.split(".")
  video_type = last_seg.last

  {path_type: "vod2", dist_type: "vod", video_type: video_type, asset_id: asset_id}
end
vod_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 30
def vod_parse uarray
  last_seg = uarray.last.split(".")
  if uarray.count < 4
    return  {path_type: "vod", dist_type: "vod",video_type: "unknown"}
  end

  asset_id = uarray[3].split(":")[1].split("_").first.to_i
  video_type = last_seg.last

  vtype = nil
  vtype = "segment" if uarray.count > 4

  {path_type: "vod", dist_type: "vod", video_type: video_type, asset_id: asset_id, vtype: vtype}
end
wzlive_parse(uarray) click to toggle source
# File lib/logstash/filters/sumo.rb, line 13
def wzlive_parse(uarray)
  web_stream = uarray[3].split(":")[1]
  video_type = uarray.last.split(".").last
  asset_id = uarray[4].split(".").first.split("_").first.to_i

  {path_type: "wzlive", dist_type: "live", web_stream: web_stream, video_type: video_type, asset_id: asset_id}
end