class Clockker::Watcher
Public Class Methods
new(config, logger: Logger.new)
click to toggle source
# File lib/clockker/watcher.rb, line 8 def initialize(config, logger: Logger.new) Thread.abort_on_exception = true trap "INT" do @fsevent.stop @timer.kill # the timer can just go away; it doesn't need a graceful shutdown. @safari_watcher_queue << :exit @submitter_queue << :exit # this shuts down the submitter, first letting it submit its changeset end white_black_list = WhiteBlackList.new(config) @submitter_queue = Queue.new @safari_watcher_queue = Queue.new @timer = Thread.new { create_timer(@safari_watcher_queue) } @file_watcher = Thread.new { create_file_watcher('/', white_black_list, logger) } @safari_watcher = Thread.new { create_safari_watcher(white_black_list, logger) } @submitter = Thread.new { create_submitter(config.submit_frequency, config.region, config.token, config.identifier, logger, config.development) } @file_watcher.join @safari_watcher.join @submitter.join end
Public Instance Methods
create_file_watcher(dir, white_black_list, logger)
click to toggle source
# File lib/clockker/watcher.rb, line 39 def create_file_watcher(dir, white_black_list, logger) @fsevent = FSEvent.new @fsevent.watch dir, {file_events: true} do |directories, event_meta| t = Time.now.utc count = 0 event_meta['events'].each do |event| next if white_black_list.ignore?(event['path']) count += 1 file_path = ::Pathname.new(event['path']) dirs, name = file_path.split last_dir = dirs.split[-1] title = (last_dir + name).to_s @submitter_queue << {touched_at: t, contents: file_path.to_s, meta_type: "file", metadata: {title: title, file_path: file_path.to_s}} end logger.debug "#{Time.now} checked #{event_meta['numEvents']} events and added #{count} to the submitter_queue" end logger.info "#{Time.now} now watching /" logger.info "#{Time.now} whitelist paths: #{white_black_list.whitelist}" logger.info "#{Time.now} blacklist paths: #{white_black_list.blacklist}" logger.info "#{Time.now} url whitelist: #{white_black_list.url_whitelist}" @fsevent.run logger.info "#{Time.now} fswatcher done" end
create_safari_watcher(white_black_list, logger)
click to toggle source
# File lib/clockker/watcher.rb, line 63 def create_safari_watcher(white_black_list, logger) safari_db = SQLite3::Database.new(File.expand_path("~/Library/Safari/History.db")) logger.info "#{Time.now} now watching Safari" last_time = Time.now.to_i - 978307200 # convert to macOS Core Data time loop do submit_now = false exit_now = false value = @safari_watcher_queue.pop case value when :exit submit_now = true exit_now = true when :run # we just move forward else # On the off chance the message isn't :run end safari_db.execute(" SELECT hv.id, datetime(hv.visit_time+978307200, \"unixepoch\") as visited_at, hv.visit_time, hv.title, hi.url, hv.load_successful, hv.http_non_get, hv.redirect_source, hv.redirect_destination, hv.origin, hv.generation, hv.attributes, hv.score FROM history_visits hv, history_items hi WHERE hv.history_item=hi.id AND hv.visit_time > #{last_time} ORDER BY hv.visit_time ASC ") do |row| id, visited_at, visit_time, title, url, load_successful, http_non_get, redirect_source, redirect_destination, origin, generation, attributes, score = row next if load_successful == 0 next if redirect_destination next if white_black_list.ignore_url?(url) visited_at = Time.parse(visited_at+"Z") # the time is already UTC; let's make doubly sure! logger.debug "- #{url} http_non_get: #{http_non_get} origin: #{origin} generation: #{generation} attributes: #{attributes} score: #{score}" @submitter_queue << {touched_at: visited_at, contents: url, meta_type: "url", metadata: {url: url, title: title}} last_time = visit_time end if exit_now logger.info "#{Time.now} Safari watcher preparing to exit" break end end logger.info "#{Time.now} Safari watcher done" end
create_submitter(submit_frequency, region, token, identifier, logger, dev_mode)
click to toggle source
# File lib/clockker/watcher.rb, line 110 def create_submitter(submit_frequency, region, token, identifier, logger, dev_mode) logger.info "#{Time.now} submitting to clockk.com #{"and localhost:4000 " if dev_mode}every #{submit_frequency} seconds with token #{token} from identifier #{identifier} with version #{Clockker.version}" t_start = Time.now clockk_agent_uri = URI("https://#{region}.clockk.io/api/v1/agent/artifacts") clockk_local_uri = URI("http://localhost:4000/api/v1/agent/artifacts") @changeset = [] loop do submit_now = false exit_now = false value = @submitter_queue.pop case value when :submit submit_now = true when :exit submit_now = true exit_now = true else @changeset << value end if Time.now - t_start > submit_frequency submit_now = true end if exit_now logger.info "#{Time.now} Submitter preparing to exit" end if submit_now && @changeset.length > 0 logger.info "#{Time.now} submitting #{@changeset.length} items" @changeset.each_with_index do |cs, idx| logger.info " - #{idx} - #{cs[:touched_at].strftime("%H:%M:%S")} #{cs[:contents]}" end begin dev_data = {artifacts: @changeset.map{|c| {contents: c[:contents], touched_at: c[:touched_at].strftime("%Y-%m-%d %H:%M:%S"), meta_type: c[:meta_type], metadata: c[:metadata], identifier: identifier, agent: Clockker.version}}}.to_json prod_data = dev_data # at this point the same data goes to prod as to dev header = {"Authorization" => "Bearer #{token}", "Content-Type" => "application/json"} logger.debug "PROD submission: #{prod_data.inspect}" Net::HTTP.start(clockk_agent_uri.hostname, clockk_agent_uri.port, use_ssl: clockk_agent_uri.scheme == 'https') do |http| http.post(clockk_agent_uri.path, prod_data, header) end if dev_mode begin logger.debug "DEV submission: #{dev_data.inspect}" Net::HTTP.start(clockk_local_uri.hostname, clockk_local_uri.port, use_ssl: clockk_local_uri.scheme == 'https') do |http| http.post(clockk_local_uri.path, dev_data, header) end rescue Exception => q logger.info "#{Time.now} Failed to reach localhost:4000; ignoring" end end @changeset = [] rescue Exception => e logger.info "#{Time.now} Reached exception #{e}" end t_start = Time.now end if exit_now logger.info "#{Time.now} Submitter done" break end end end
create_timer(*queues)
click to toggle source
# File lib/clockker/watcher.rb, line 32 def create_timer(*queues) loop do sleep 5 queues.each {|q| q << :run} end end