class Kennel::Syncer
Constants
- DEFAULT_BRANCH
- DELETE_ORDER
- LINE_UP
Public Class Methods
new(api, expected, project: nil)
click to toggle source
# File lib/kennel/syncer.rb, line 8 def initialize(api, expected, project: nil) @api = api @project_filter = project @expected = expected calculate_diff prevent_irreversible_partial_updates end
Public Instance Methods
confirm()
click to toggle source
# File lib/kennel/syncer.rb, line 27 def confirm return false if noop? return true if ENV["CI"] || !STDIN.tty? warn_about_deleting_resources_with_id if @project_filter Utils.ask("Execute Plan ?") end
plan()
click to toggle source
# File lib/kennel/syncer.rb, line 16 def plan Kennel.out.puts "Plan:" if noop? Kennel.out.puts Utils.color(:green, "Nothing to do") else print_plan "Create", @create, :green print_plan "Update", @update, :yellow print_plan "Delete", @delete, :red end end
update()
click to toggle source
# File lib/kennel/syncer.rb, line 34 def update each_resolved @create do |_, e| message = "#{e.class.api_resource} #{e.tracking_id}" Kennel.out.puts "Creating #{message}" reply = @api.create e.class.api_resource, e.as_json cache_metadata reply, e.class id = reply.fetch(:id) populate_id_map [], [reply] # allow resolving ids we could previously no resolve Kennel.out.puts "#{LINE_UP}Created #{message} #{e.class.url(id)}" end each_resolved @update do |id, e| message = "#{e.class.api_resource} #{e.tracking_id} #{e.class.url(id)}" Kennel.out.puts "Updating #{message}" @api.update e.class.api_resource, id, e.as_json Kennel.out.puts "#{LINE_UP}Updated #{message}" end @delete.each do |id, _, a| klass = a.fetch(:klass) message = "#{klass.api_resource} #{a.fetch(:tracking_id)} #{id}" Kennel.out.puts "Deleting #{message}" @api.delete klass.api_resource, id Kennel.out.puts "#{LINE_UP}Deleted #{message}" end end
Private Instance Methods
assert_resolved(e)
click to toggle source
raises ValidationError
when not resolved
# File lib/kennel/syncer.rb, line 114 def assert_resolved(e) resolve_linked_tracking_ids! [e], force: true end
cache_metadata(a, klass)
click to toggle source
# File lib/kennel/syncer.rb, line 174 def cache_metadata(a, klass) a[:klass] = klass a[:tracking_id] = a.fetch(:klass).parse_tracking_id(a) end
calculate_diff()
click to toggle source
# File lib/kennel/syncer.rb, line 122 def calculate_diff @update = [] @delete = [] @id_map = {} actual = Progress.progress("Downloading definitions") { download_definitions } Progress.progress "Diffing" do populate_id_map @expected, actual filter_actual_by_project! actual resolve_linked_tracking_ids! @expected # resolve dependencies to avoid diff @expected.each(&:add_tracking_id) # avoid diff with actual items = actual.map do |a| e = matching_expected(a) if e && @expected.delete(e) [e, a] else [nil, a] end end # fill details of things we need to compare details = items.map { |e, a| a if e && e.class.api_resource == "dashboard" }.compact @api.fill_details! "dashboard", details # pick out things to update or delete items.each do |e, a| id = a.fetch(:id) if e diff = e.diff(a) @update << [id, e, a, diff] if diff.any? elsif a.fetch(:tracking_id) # was previously managed @delete << [id, nil, a] end end ensure_all_ids_found @create = @expected.map { |e| [nil, e] } @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:klass).api_resource } end end
download_definitions()
click to toggle source
# File lib/kennel/syncer.rb, line 166 def download_definitions Utils.parallel(Models::Record.subclasses) do |klass| results = @api.list(klass.api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information results = results[results.keys.first] if results.is_a?(Hash) # dashboards are nested in {dashboards: []} results.each { |a| cache_metadata(a, klass) } end.flatten(1) end
each_resolved(list) { |id, e| ... }
click to toggle source
loop over items until everything is resolved or crash when we get stuck this solves cases like composite monitors depending on each other or monitor->monitor slo->slo monitor chains
# File lib/kennel/syncer.rb, line 89 def each_resolved(list) list = list.dup loop do return if list.empty? list.reject! do |id, e| if resolved?(e) yield id, e true else false end end || assert_resolved(list[0][1]) # resolve something or show a circular dependency error end end
ensure_all_ids_found()
click to toggle source
# File lib/kennel/syncer.rb, line 179 def ensure_all_ids_found @expected.each do |e| next unless id = e.id resource = e.class.api_resource raise "Unable to find existing #{resource} with id #{id}\nIf the #{resource} was deleted, remove the `id: -> { #{e.id} }` line." end end
filter_actual_by_project!(actual)
click to toggle source
# File lib/kennel/syncer.rb, line 267 def filter_actual_by_project!(actual) return unless @project_filter actual.select! do |a| tracking_id = a.fetch(:tracking_id) !tracking_id || tracking_id.start_with?("#{@project_filter}:") end end
matching_expected(a)
click to toggle source
# File lib/kennel/syncer.rb, line 187 def matching_expected(a) # index list by all the thing we look up by: tracking id and actual id @lookup_map ||= @expected.each_with_object({}) do |e, all| keys = [e.tracking_id] keys << "#{e.class.api_resource}:#{e.id}" if e.id keys.compact.each do |key| raise "Lookup #{key} is duplicated" if all[key] all[key] = e end end klass = a.fetch(:klass) @lookup_map["#{klass.api_resource}:#{a.fetch(:id)}"] || @lookup_map[a.fetch(:tracking_id)] end
noop?()
click to toggle source
# File lib/kennel/syncer.rb, line 118 def noop? @create.empty? && @update.empty? && @delete.empty? end
populate_id_map(expected, actual)
click to toggle source
# File lib/kennel/syncer.rb, line 255 def populate_id_map(expected, actual) actual.each do |a| next unless tracking_id = a.fetch(:tracking_id) @id_map[tracking_id] = a.fetch(:id) end expected.each { |e| @id_map[e.tracking_id] ||= :new } end
prevent_irreversible_partial_updates()
click to toggle source
-
do not add tracking-id when working with existing ids on a branch, so resource do not get deleted when running an update on master (for example merge->CI)
-
make sure the diff is clean, by kicking out the now noop-update
-
ideally we'd never add tracking in the first place, but when adding tracking we do not know the diff yet
# File lib/kennel/syncer.rb, line 236 def prevent_irreversible_partial_updates return unless @project_filter @update.select! do |_, e, _, diff| next true unless e.id # safe to add tracking when not having id diff.select! do |field_diff| (_, field, actual) = field_diff # TODO: refactor this so TRACKING_FIELD stays record-private next true if e.class::TRACKING_FIELD != field.to_sym # need to sym here because Hashdiff produces strings next true if e.class.parse_tracking_id(field.to_sym => actual) # already has tracking id field_diff[3] = e.remove_tracking_id # make `rake plan` output match what we are sending actual != field_diff[3] # discard diff if now nothing changes end !diff.empty? end end
print_diff(diff)
click to toggle source
# File lib/kennel/syncer.rb, line 211 def print_diff(diff) diff.each do |type, field, old, new| if type == "+" temp = Utils.pretty_inspect(new) new = Utils.pretty_inspect(old) old = temp else # ~ and - old = Utils.pretty_inspect(old) new = Utils.pretty_inspect(new) end if (old + new).size > 100 Kennel.out.puts " #{type}#{field}" Kennel.out.puts " #{old} ->" Kennel.out.puts " #{new}" else Kennel.out.puts " #{type}#{field} #{old} -> #{new}" end end end
print_plan(step, list, color)
click to toggle source
# File lib/kennel/syncer.rb, line 202 def print_plan(step, list, color) return if list.empty? list.each do |_, e, a, diff| klass = (e ? e.class : a.fetch(:klass)) Kennel.out.puts Utils.color(color, "#{step} #{klass.api_resource} #{e&.tracking_id || a.fetch(:tracking_id)}") print_diff(diff) if diff # only for update end end
resolve_linked_tracking_ids!(list, force: false)
click to toggle source
# File lib/kennel/syncer.rb, line 263 def resolve_linked_tracking_ids!(list, force: false) list.each { |e| e.resolve_linked_tracking_ids!(@id_map, force: force) } end
resolved?(e)
click to toggle source
TODO: optimize by storing an instance variable if already resolved
# File lib/kennel/syncer.rb, line 106 def resolved?(e) assert_resolved e true rescue ValidationError false end
warn_about_deleting_resources_with_id()
click to toggle source
this is brittle/hacky since it relies on knowledge from the generation + git + branch knowledge
# File lib/kennel/syncer.rb, line 64 def warn_about_deleting_resources_with_id @delete.each do |_, _, a| tracking_id = a.fetch(:tracking_id) api_resource = a.fetch(:klass).api_resource file = "generated/#{tracking_id.sub(":", "/")}.json" old = `true && git show #{DEFAULT_BRANCH}:#{file.shellescape} 2>&1` # true && to not crash on missing git next unless $?.success? old = begin JSON.parse(old) rescue StandardError false end next if !old || !old["id"] Kennel.out.puts( Utils.color(:red, "WARNING: deleting #{api_resource} #{tracking_id} will break #{DEFAULT_BRANCH} branch") ) end end