class LogStash::Inputs::Trello

A Logstash input plugin used for querying Trello at set intervals. It return events of varying types assosciated with Trello entities (ie cards, lists, etc).

Public Instance Methods

register() click to toggle source

# File lib/logstash/inputs/trello.rb, line 309
def register()
        if @fields == ["all"]
                @fields = TrelloUtils::PARAM_ALL_FIELDS
        elsif @fields == ["default"]
                @fields = TrelloUtils::PARAM_DEFAULT_FIELDS
        end

        if @filters == ["all"]
                @filters = TrelloUtils::PARAM_ALL_FILTERS
        elsif @filters == ["default"]
                @filters = TrelloUtils::PARAM_DEFAULT_FILTERS
        end

        if @output_type == ["all"]
                @output_types = [
                                        "board",
                                        "memberships",
                                        "labels",
                                        "cards",
                                        "lists",
                                        "members",
                                        "checklists",
                                        "actions"
                ]
        end
        @host = Socket.gethostname
        @client = TrelloUtils::TrelloClient.new({
                        organizations: @organizations,
                        key:           @key,
                        token:         @token,
                        board_ids:     @board_ids,
                        fields:        @fields,
                        filters:       @filters,
                        port:          @port
        })
end
run(queue) click to toggle source
# File lib/logstash/inputs/trello.rb, line 747
def run(queue)
        init_time = Time.now - @interval
        init_time = init_time.strftime('%Y-%m-%dT%H:%M:%S%z')
        query_times = {}
        board_ids.each { |board_id| query_times[board_id] = init_time}
        Stud.interval(@interval) do
                @client.board_ids.each do |board_id|
                        uri = @client.get_uri(board_id, query_times[board_id])
                        response = nil
                        begin
                                response = @client.issue_request(uri)
                        rescue RuntimeError
                                next
                        end
                        process_response(response, queue)
                        query_times[board_id] = Time.now.strftime('%Y-%m-%dT%H:%M:%S%z')
                end
        end
end

Private Instance Methods

_group(data) click to toggle source
# File lib/logstash/inputs/trello.rb, line 614
def _group(data)
        prototype = {}
        data.each do |entry|
                entry.each do |key, val|
                        prototype[key] = []
                end
        end
        data.each do |entry|
                entry.each do |key, val|
                        if val != "" and !val.nil?
                                prototype[key].push(val)
                        end
                end
        end
        return prototype
end
_nested_hash_to_matrix(data, name) click to toggle source
# File lib/logstash/inputs/trello.rb, line 656
def _nested_hash_to_matrix(data, name)
        data.each do |key, val|
                new_key = name + @sep + key.to_s
                if val.is_a?(Hash) and val != {}
                        _nested_hash_to_matrix(val, new_key)
                else
                        @output.push([new_key, val])
                end
        end
        return @output
end
clean_data(data) click to toggle source
# File lib/logstash/inputs/trello.rb, line 548
def clean_data(data)
        data = data.clone
        # remove actions data field
        if data.has_key?("data")
                data["data"].each do |key, val|
                        if not key == "old"
                                data[key] = val
                        end
                end
                data.delete("data")
        end
        if data.has_key?("board")
                data.delete("board")
        end
        return data
end
coerce_nulls(data) click to toggle source
# File lib/logstash/inputs/trello.rb, line 603
def coerce_nulls(data)
        data.each do |index, item|
                if item == ""
                        item == nil
                end
        end
        return data
end
collapse(data, source, entities) click to toggle source
# File lib/logstash/inputs/trello.rb, line 589
def collapse(data, source, entities)
        # entities might be drawn from lut.keys()
        output = { source => {} }
        data.each do |key, val|
                if entities.include?(key)
                        output[key] = val
                else
                        output[source][key] = val
                end
        end
        return output
end
conform_field_names(data, form=nil) click to toggle source
# File lib/logstash/inputs/trello.rb, line 464
def conform_field_names(data, form=nil)
        if not data.is_a?(Hash)
                return data
        end
        key_transformer = lambda do |key|
                if @@all_ids.include?(key)
                        # clobber non-id fields with id fields
                        new_key = key.gsub(/^id/, '')
                        new_key = new_key[0].downcase + new_key[1..-1]
                        if not form.nil?
                                if form == 'plural'
                                        if not /ed$/.match(new_key)
                                                new_key = new_key.pluralize
                                        end
                                elsif form == 'singular'
                                        new_key = new_key.singularize
                                end
                        end
                        return new_key
                else
                        return key
                end
        end
        return recurse(data.clone, nil, nil, key_transformer)
end
create_lut(data) click to toggle source
# File lib/logstash/inputs/trello.rb, line 381
def create_lut(data)
        # This cannot be done recursively because a recursive func will pick up
        # stubs
        lut = {}
        @@plural_entities.map { |entity| lut[entity] = {} }
        
        data.each do |key, entities|
                if lut.has_key?(key)
                        entities.each do |entity|
                                lut[key][ entity["id"] ] = entity
                        end
                end
        end
        return lut
end
exclude_fields!(event) click to toggle source
# File lib/logstash/inputs/trello.rb, line 371
def exclude_fields!(event)
        @exclude_fields.each_with_index do |field|
                f = fieldref_to_array(field)
                if recursive_has_key?(event.to_hash, f)
                        event.remove(field)
                end
        end
end
expand_entities(data, lut) click to toggle source
# File lib/logstash/inputs/trello.rb, line 491
def expand_entities(data, lut)
        func = lambda do |key, val|
                pkey = key.pluralize
                l = lut[pkey]
                if l.nil?
                        l = {}
                end

                output = val
                ds_type = get_data_structure_type(val)
                if /^array_of/.match(ds_type)
                        output = []
                        if ds_type == "array_of_hashes"
                                val.each do |item|
                                        item_type = get_data_structure_type(item)
                                        new_item = item
                                        if item_type == "hash_with_id"
                                                if l.has_key?(item["id"])
                                                        new_item = l[item["id"]]
                                                end
                                        end
                                        new_item = flatten(new_item)
                                        output.push(new_item)
                                end

                        elsif ds_type == "array_of_strings"
                                val.each do |id|
                                        if l.has_key?(id)
                                                output.push(l[id])
                                                # output.push( flatten(l[id]) )
                                        end
                                end
                        end
                        output = group(output)

                elsif ds_type == "empty_array"
                        output = nil

                elsif ds_type == "hash_with_id"
                        if l.has_key?(val["id"])
                                output = l[val["id"]]
                                # output = flatten(output)
                        end

                elsif ds_type == "String"
                        if l.has_key?(val)
                                output = l[val]
                                # output = flatten(output)
                        end
                end

                return conform_field_names(output)
        end
        return recurse(data.clone, func, func)
end
fieldref_to_array(fieldref) click to toggle source
# File lib/logstash/inputs/trello.rb, line 364
def fieldref_to_array(fieldref)
        output = fieldref.split("][")
        output.map! { |item| item.gsub(/\[|\]/, "") }
        return output
end
flatten(data) click to toggle source
# File lib/logstash/inputs/trello.rb, line 398
def flatten(data)
        func = lambda do |key, val|
                output = val
                ds_type = get_data_structure_type(val)
                if ds_type == "array_of_hashes"
                        return group(output)
                else
                        return output
                end
        end
        return recurse(data.clone, func, func)
end
get_data_structure_type(val) click to toggle source
# File lib/logstash/inputs/trello.rb, line 566
def get_data_structure_type(val)
        if val.is_a?(Array)
                if not val.empty?
                        if val[0].is_a?(Hash)
                                return "array_of_hashes"
                        elsif val[0].is_a?(String)
                                return "array_of_strings"
                        end
                else
                        return "empty_array"
                end
        elsif val.is_a?(Hash)
                if val.has_key?("id")
                        return "hash_with_id"
                else
                        return "hash_without_id"
                end
        else
                return val.class.to_s
        end
end
group(data) click to toggle source
# File lib/logstash/inputs/trello.rb, line 613
def group(data)
        def _group(data)
                prototype = {}
                data.each do |entry|
                        entry.each do |key, val|
                                prototype[key] = []
                        end
                end
                data.each do |entry|
                        entry.each do |key, val|
                                if val != "" and !val.nil?
                                        prototype[key].push(val)
                                end
                        end
                end
                return prototype
        end

        # ensure the proper data structure
        if data.is_a?(Array)
                if not data.empty?
                        if data[0].is_a?(Hash)
                                return _group(data)
                        end
                end
        else
                # return data if data structure is wrong
                return data
        end
end
matrix_to_nested_hash(data) click to toggle source
# File lib/logstash/inputs/trello.rb, line 674
def matrix_to_nested_hash(data)
        output = {}
        data.each do |keys, value|
                cursor = output
                for key in keys[0..-2]
                        if !cursor.include?(key)
                                cursor[key] = {}
                                cursor = cursor[key]
                        else
                                cursor = cursor[key]
                        end
                end
                cursor[keys[-1]] = value
        end
        return output
end
nested_hash_to_matrix(data) click to toggle source

# File lib/logstash/inputs/trello.rb, line 653
def nested_hash_to_matrix(data)
        @sep = '.'
        @output = []
        def _nested_hash_to_matrix(data, name)
                data.each do |key, val|
                        new_key = name + @sep + key.to_s
                        if val.is_a?(Hash) and val != {}
                                _nested_hash_to_matrix(val, new_key)
                        else
                                @output.push([new_key, val])
                        end
                end
                return @output
        end

        @output = _nested_hash_to_matrix(data, @sep)
        @output = @output.map { |key, val| [key.split('.')[2..-1], val] }
        return @output
end
process_response(response, queue) click to toggle source

# File lib/logstash/inputs/trello.rb, line 693
def process_response(response, queue)
        timestamp = Time.now

        response = collapse(response, "board", @@plural_entities)
        lut = create_lut(response)
        # lut = clean_lut(lut)

        @output_types.each do |out_type|
                if response.has_key?(out_type)
                        response[out_type].each do |source|
                                out_type_ = out_type.singularize
                                data = clean_data(source)
                                data = conform_field_names(data, 'plural')
                                data = expand_entities(data, lut)
                                all_ent = @@all_entities.clone
                                all_ent.delete("entities")
                                data = collapse(data, out_type_, all_ent)
                                data = flatten(data)

                                # shuffle board info into data
                                board = response["board"]
                                board = conform_field_names(board)
                                data["board"] = board

                                data = nested_hash_to_matrix(data)
                                data = coerce_nulls(data)
                                if @snake_case
                                        data = to_snake_case(data)
                                end
                                data = matrix_to_nested_hash(data)
                                event = nil
                                # set the timestamp of actions to their date field
                                _timestamp = timestamp
                                if out_type_ == "action"
                                        _timestamp = data["action"]["date"]
                                        data["action"].delete("date")
                                end
                                event = LogStash::Event.new(
                                        "host" => @host, 
                                        "type" => @type + '_' + out_type_,
                                        "@timestamp" => _timestamp,
                                        "message" => JSON.dump(source) )
                                data.each do |key, val|
                                        event[key] = val
                                end
                                exclude_fields!(event)
                                decorate(event)
                                queue << event
                        end
                end
        end
end
recurse(data, nonhash_func=nil, hash_func=nil, key_func=nil) click to toggle source
lut = lut.clone
lut.to_a.each do|ent_type, ids|
        ids.to_a.each do |item|
                temp = flatten(item.clone)
                lut[ent_type][item] = _remove_entities(temp)
        end
end
return lut

end

# File lib/logstash/inputs/trello.rb, line 442
def recurse(data, nonhash_func=nil, hash_func=nil, key_func=nil)
        hash_func = lambda { |key, val| val } if hash_func.nil?
        nonhash_func = lambda { |key, val| val } if nonhash_func.nil?
        key_func = lambda { |key| key } if key_func.nil?
        
        if not data.is_a?(Hash)
                return data # leaf (stop recursion here)
        end

        store = {}
        data.each do |key, val|
                if val.is_a?(Hash)
                        store[key_func.call(key)] = recurse(hash_func.call(key, val), 
                                                                                                nonhash_func, hash_func, key_func)
                else
                        store[key_func.call(key)] = nonhash_func.call(key, val)
                end
        end
        return store
end
recursive_has_key?(data, keys) click to toggle source

# File lib/logstash/inputs/trello.rb, line 348
def recursive_has_key?(data, keys)
        if not data.is_a?(Hash)
                return false
        end    
        if data.has_key?(keys[0])
                if keys.length > 1
                        recursive_has_key?(data[keys[0]], keys[1..-1])
                else
                        return true
                end
        else
                return false
        end
end
to_snake_case(data) click to toggle source
# File lib/logstash/inputs/trello.rb, line 645
def to_snake_case(data)
        data = data.clone
        data.each { |index, item| index.map! { |item| item.underscore } }
        return data
end