class FeedInto::Single
Constants
- SINGLE
Attributes
settings[R]
Public Class Methods
new( modules: nil, options: {} )
click to toggle source
# File lib/feed_into.rb, line 61 def initialize( modules: nil, options: {} ) mdl = modules.class.eql? String chn = options.keys.include? :channels mode = :not_found if !mdl and !chn puts 'No Channel found.' else mode = nil if chn mode = :options transfer = Marshal.load( Marshal.dump( options[:channels] ) ) options.delete( :channels ) end mdl ? mode = :folder : '' @single = Marshal.load( Marshal.dump( SINGLE ) ) if options_update( options, @single, true ) @single[:channels].concat( crl_general_channels() ) chn ? @single[:channels].concat( transfer ) : '' mdl ? @single[:channels].concat( load_modules( modules ) ) : '' @single = options_update( options, @single, false ) @settings = @single else end end end
Public Instance Methods
analyse( item: {}, trust_item: false )
click to toggle source
# File lib/feed_into.rb, line 94 def analyse( item: {}, trust_item: false ) def modul( type, channel, allow_methods, cmd, response, data, obj ) messages = [] if !channel[ type ].nil? error = nil execute = true name = nil case channel[ type ] when :self name = ( 'crl_' + channel[:name].to_s ).to_sym !allow_methods.include?( name ) ? execute = false : '' when :general name = :crl_general else name = :crl_general type = ( type.to_s + '_' + channel[ type ].to_s ).to_sym end if execute data, messages = self .method( name ) .call( type, cmd, channel, response, data, obj ) else messages = [ "Modul: #{name} not found." ] end else end return data, messages end def formats( type, cmd, channel, response, data, obj ) channel[ type ].each do | format_ | data = self .method( 'crl_general' ) .call( format_, cmd, channel, response, data, obj )[ 0 ] end return data end def set_status( messages ) s = 'Download: Status ' status = nil if messages.class.eql? Array tmp = messages .find { | a | a.start_with?( s ) } .to_s .gsub( s, '') .to_i tmp.nil? ? status = 0 : status = tmp else end return status end result = { cmd: nil, result: nil, messages: nil, time: nil, success: false, status: nil } obj = Marshal.load( Marshal.dump( @single ) ) begin start = Time.now.to_f messages = [] status = nil cmd = {} data = nil if trust_item cmd, m0 = item[:cmd], item[:messages] else cmd, m0 = cmd( item, obj ) end messages.concat( m0 ) if cmd[:valid] channel = obj[:channels].find { | c | c[:name].eql? cmd[:channel] } allow_channels = obj[:channels].map { | a | ( 'crl_' + a[:name].to_s ).to_sym } allow_methods = self.methods.select { | a | allow_channels.include?( a.to_sym ) } response, m1 = modul( :download, channel, allow_methods, cmd, response, nil, obj ) messages.concat( m1 ) data, m2 = modul( :mining, channel, allow_methods, cmd, response, nil, obj ) messages.concat( m2 ) data = formats( :pre, cmd, channel, response, data, obj ) data, m3 = modul( :transform, channel, allow_methods, cmd, response, data, obj ) messages.concat( m3 ) data = formats( :post, cmd, channel, response, data, obj ) result[:success] = true end result[:cmd] = cmd result[:result] = data result[:messages] = messages result[:time] = Time.now.to_f - start status = set_status( messages ) result[:status] = status rescue => e messages.push( "Begin/Rescue: #{e}" ) result[:cmd] = cmd result[:result] = data result[:messages] = messages result[:time] = Time.now.to_f - start status = set_status( messages ) result[:status] = status end return result end
cmd( cmd, obj )
click to toggle source
# File lib/feed_into.rb, line 224 def cmd( cmd, obj ) def validate( cmd ) check = { validation: { struct: false, url: false, channel: false, }, struct: { name: String, url: String, category: Symbol, channel: Symbol } } messages = [] check[:validation][:struct] = check[:struct] .map { | k, v | cmd[ k ].class.eql? v } .all? !check[:validation][:struct] ? messages.push( 'Structure of cmd is not valid.' ) : '' if cmd[:url] =~ URI::regexp check[:validation][:url] = true else messages.push( "'#{cmd[:url]}' is not a valid URL.") end if !cmd[:channel].eql? :not_found check[:validation][:channel] = true else messages.push( 'Channel not found' ) end cmd[:valid] = check[:validation].map { | k, v | v }.all? return cmd, messages end if [ Hash, ActiveSupport::HashWithIndifferentAccess ].include? cmd.class !cmd.key?( :name ) ? cmd[:name] = 'Unknown' : '' !cmd.key?( :category ) ? cmd[:category] = :unknown : '' cmd[:category].class.eql?( String ) ? cmd[:category] = cmd[:category].to_s.downcase.to_sym : '' else if cmd.class.eql? String cmd = { name: 'Unknown', url: cmd, category: :unknown, #channel: :not_found } else cmd = { name: 'Invalid', url: cmd.to_s, category: :invalid, channel: :not_found } end end f = obj[:channels].find do | channel | r = channel[:regexs] .map { | ps | ps.map { | p | cmd[:url].match?( p ) }.all? } .include?( true ) end !f.nil? ? cmd[:channel] = f[:name] : cmd[:channel] = :not_found valid, messages = validate( cmd ) return valid, messages end
formats( type, cmd, channel, response, data, obj )
click to toggle source
# File lib/feed_into.rb, line 126 def formats( type, cmd, channel, response, data, obj ) channel[ type ].each do | format_ | data = self .method( 'crl_general' ) .call( format_, cmd, channel, response, data, obj )[ 0 ] end return data end
modul( type, channel, allow_methods, cmd, response, data, obj )
click to toggle source
# File lib/feed_into.rb, line 95 def modul( type, channel, allow_methods, cmd, response, data, obj ) messages = [] if !channel[ type ].nil? error = nil execute = true name = nil case channel[ type ] when :self name = ( 'crl_' + channel[:name].to_s ).to_sym !allow_methods.include?( name ) ? execute = false : '' when :general name = :crl_general else name = :crl_general type = ( type.to_s + '_' + channel[ type ].to_s ).to_sym end if execute data, messages = self .method( name ) .call( type, cmd, channel, response, data, obj ) else messages = [ "Modul: #{name} not found." ] end else end return data, messages end
set_status( messages )
click to toggle source
# File lib/feed_into.rb, line 136 def set_status( messages ) s = 'Download: Status ' status = nil if messages.class.eql? Array tmp = messages .find { | a | a.start_with?( s ) } .to_s .gsub( s, '') .to_i tmp.nil? ? status = 0 : status = tmp else end return status end
validate( cmd )
click to toggle source
# File lib/feed_into.rb, line 225 def validate( cmd ) check = { validation: { struct: false, url: false, channel: false, }, struct: { name: String, url: String, category: Symbol, channel: Symbol } } messages = [] check[:validation][:struct] = check[:struct] .map { | k, v | cmd[ k ].class.eql? v } .all? !check[:validation][:struct] ? messages.push( 'Structure of cmd is not valid.' ) : '' if cmd[:url] =~ URI::regexp check[:validation][:url] = true else messages.push( "'#{cmd[:url]}' is not a valid URL.") end if !cmd[:channel].eql? :not_found check[:validation][:channel] = true else messages.push( 'Channel not found' ) end cmd[:valid] = check[:validation].map { | k, v | v }.all? return cmd, messages end
Private Instance Methods
load_modules( folder )
click to toggle source
# File lib/feed_into.rb, line 305 def load_modules( folder ) mods = [] searchs = [] Dir[ folder + '*.*' ].each do | path | require_relative path search = open( path ).read.split( "\n" ) .find { | a | a.include?( 'module' ) } .gsub('module ', '' ) searchs.push( search ) end searchs.each do | search | name = Module::const_get( search ) extend name end names = self.methods .select { | a | a.to_s.start_with?( 'crl' ) } .reject { | a | a.eql? ( ( 'crl_general' ).to_sym ) } channels = [] names.each do | n | mods.push( n.to_s.gsub( 'crl_', '' ) ) channel, messages = self .method( n.to_sym ) .call( :settings, nil, nil, nil, nil, nil ) channels.push( channel ) end puts "#{mods.length} Module#{mods.length > 2 ? 's' : ''} loaded (#{mods.join(', ')})" return channels end
options_update( options, obj, validation )
click to toggle source
# File lib/feed_into.rb, line 341 def options_update( options, obj, validation ) def str_difference( a, b ) a = a.to_s.downcase.split( '_' ).join( '' ) b = b.to_s.downcase.split( '_' ).join( '' ) longer = [ a.size, b.size ].max same = a .each_char .zip( b.each_char ) .select { | a, b | a == b } .size ( longer - same ) / a.size.to_f end allows = obj[:validation][:allows] wildcards = obj[:validation][:wildcards] messages = [] insert = Marshal.load( Marshal.dump( obj ) ) options.keys.each do | key | if allows.include?( key ) keys = key.to_s.split( '__' ).map { | a | a.to_sym } case( keys.length ) when 1 insert[ keys[ 0 ] ] = options[ key ] when 2 insert[ keys[ 0 ] ][ keys[ 1 ] ] = options[ key ] when 3 insert[ keys[ 0 ] ][ keys[ 1 ] ][ keys[ 2 ] ] = options[ key ] when 4 insert[ keys[ 0 ] ][ keys[ 1 ] ][ keys[ 2 ] ][ keys[ 3 ] ] = options[ key ] end else standard = true keys = key.to_s.split( '__' ).map { | a | a.to_sym } case keys.length when 1 inside = wildcards .map { | a | a.to_s.split( '__' ).first.to_sym } .include?( keys[ 0 ] ) if inside message = "\"#{key}\" is a potential Wildcard key but has an invalid length. Use two additional keys (plus '__') to set your option." messages.push( message ) standard = false else end when 2..3 wildcard = [ keys[ 0 ].to_s, keys[ 1 ].to_s ].join( '__' ).to_sym if wildcards.include?( wildcard ) if keys.length == 2 message = "\"#{key}\" is a Wildcard key but has an invalid length. Use an additional key (plus '__') to set your option." messages.push( message ) standard = false else !insert.keys.include?( keys[ 0 ] ) ? insert[ keys[ 0 ] ] = {} : '' !insert[ keys[ 0 ] ].keys.include?( keys[ 1 ] ) ? insert[ keys[ 0 ] ][ keys[ 1 ] ] = {} : '' insert[ keys[ 0 ] ][ keys[ 1 ] ][ keys[ 2 ] ] = options[ key ] standard = false end else end else end if standard nearest = allows .map { | word | { score: self.str_difference( key, word ), word: word } } .min_by { | item | item[:score] } if nearest.nil? message = "\"#{key}\" is not a valid key." else message = "\"#{key}\" is not a valid key, did you mean \"<--similar-->\"?" message = message.gsub( '<--similar-->', nearest[:word].to_s ) end messages.push( message ) end end end if messages.length != 0 messages.length == 1 ? puts( 'Error found:' ) : puts( 'Errors found:' ) messages.each { | m | puts( '- ' + m ) } end return validation ? messages.length == 0 : insert end
str_difference( a, b )
click to toggle source
# File lib/feed_into.rb, line 342 def str_difference( a, b ) a = a.to_s.downcase.split( '_' ).join( '' ) b = b.to_s.downcase.split( '_' ).join( '' ) longer = [ a.size, b.size ].max same = a .each_char .zip( b.each_char ) .select { | a, b | a == b } .size ( longer - same ) / a.size.to_f end