class JsonCsv
Constants
- DEFAULT_OPTS
- VERSION
- VERSION_DATE
Public Class Methods
# File lib/json-csv.rb, line 97 def convert_csv_to_json(opts) self.new(opts).convert_csv_to_json() end
# File lib/json-csv.rb, line 93 def convert_json_to_csv(opts) self.new(opts).convert_json_to_csv() end
# File lib/json-csv.rb, line 103 def initialize(opts) @opts = DEFAULT_OPTS.merge(opts) end
# File lib/json-csv.rb, line 88 def new_from_argv(argv) opts = parse_argv(argv) self.new(opts) end
# File lib/json-csv.rb, line 32 def parse_argv(argv) opts = DEFAULT_OPTS argv = OptionParser.new do |op| op.banner = <<-EOT Converts JSON to CSV, and vice versa. Usage: #{$0} [options] [--] [input-file [output-file]] EOT op.on("-i input-file", "--input input-file", "Input file (default STDIN)") do |input_file| opts[:input_file] = input_file end op.on("-o output-file", "--output output-file", "Output file (default STDOUT)") do |output_file| opts[:output_file] = output_file end op.on("-s json|csv", "--source-encoding json|csv", "Encoding of input file (default json)") do |source| opts[:source_encoding] = source end op.on("-d depth", "--depth depth", "Maximum depth of JSON-to-CSV conversion (default -1, unlimited)") do |depth| opts[:depth] = depth.to_i end op.on("-e crlf|cr|lf", "--line-ending crlf|cr|lf", "Line endings for output file (default crlf).") do |ending| opts[:line_ending] = {"crlf" => "\r\n", "cr" => "\r", "lf" => "\n"}[ending] if !opts[:line_ending] STDERR.puts "Invalid line ending '#{ending}'. Valid choices: crlf cr lf" exit 1 end end op.on_tail("--debug", "Turn debugging messages on") do opts[:debug] = true end op.on_tail("--version", "Print version info and exit") do puts "json-csv version #{VERSION} (#{VERSION_DATE})" puts "https://github.com/appcues/json-csv" exit end op.on_tail("-h", "--help", "Show this message and exit") do puts op.to_s exit end end.parse(argv) opts[:input_file] = argv.shift if argv.count > 0 opts[:output_file] = argv.shift if argv.count > 0 opts end
Public Instance Methods
# File lib/json-csv.rb, line 171 def convert_csv_to_json(opts = {}) raise NotImplementedError end
# File lib/json-csv.rb, line 124 def convert_json_to_csv(opts = {}) opts = @opts.merge(opts) ## First pass -- create CSV headers from JSON input input_fh = nil tmp_fh = nil tmp_filename = nil data_filename = nil if opts[:input_file] == "-" input_fh = STDIN data_filename = tmp_filename = "#{opts[:tmpdir]}/json-csv-#{$$}.tmp" debug(opts, "STDIN will be written to #{tmp_filename}.") tmp_fh = File.open(data_filename, "w") else input_fh = File.open(opts[:input_file], "r") data_filename = opts[:input_file] end debug(opts, "Getting headers from JSON data.") depth = opts[:depth] depth += 1 if depth > 0 # a fudge, in order to use -1 as infinity headers = get_headers_from_json(input_fh, tmp_fh, depth) input_fh.close tmp_fh.close if tmp_fh ## Second pass -- write CSV data from JSON input data_fh = File.open(data_filename, "r") output_fh = nil if opts[:output_file] == "-" output_fh = STDOUT else output_fh = File.open(opts[:output_file], "w") end debug(opts, "Writing CSV output.") output_csv(headers, data_fh, output_fh, opts[:line_ending]) data_fh.close output_fh.close debug(opts, "Removing #{tmp_filename}.") File.unlink(tmp_filename) if tmp_filename end
Performs the JSON-to-CSV or CSV-to-JSON conversion, as specified in `opts` and the options passed in during `JsonCsv.new`.
# File lib/json-csv.rb, line 110 def run(opts = {}) opts = @opts.merge(opts) enc = opts[:source_encoding] if enc == "json" convert_json_to_csv() elsif enc == "csv" convert_csv_to_json() else STDERR.puts "no such source encoding '#{enc}'" exit 1 end end
Private Instance Methods
Helper function to sort_keys
– Counts the number of dots in a string.
# File lib/json-csv.rb, line 217 def count_dots(str) str.chars.select{|c| c == "."}.count end
Helper function to output_csv
– Returns a CSV-armored version of `val`. Escapes special characters and adds double-quotes if necessary.
# File lib/json-csv.rb, line 300 def csv_armor(val) str = val.to_s.gsub('"', '""') if str.match(/[",\n]/) '"' + str + '"' else str end end
# File lib/json-csv.rb, line 178 def debug(opts, msg) STDERR.puts("#{Time.now}\t#{msg}") if opts[:debug] end
Helper function to flatten_json
– Assigns a flattened value at the current key.
# File lib/json-csv.rb, line 264 def flat_assign(dest, key, value, depth) flat_value = flatten_json(value, depth - 1) if flat_value.is_a?(Hash) flat_value.each do |k,v| dest["#{key}.#{k}"] = v end else dest["#{key}"] = flat_value end dest end
Returns a flattened representation of the given JSON-encodable data (that is: hashes, arrays, numbers, strings, and `nil`). Dot-separated string keys are used to encode nested hash and array structures.
Hashes get flattened like so:
flatten_json({a: {b: {c: 1, d: "x"}, c: nil}}) #=> {"a.b.c" => 1, "a.b.d" => "x", "a.c" => nil}
Arrays are turned into hashes like:
flatten_json([0, 1, 2, {a: "x"]) #=> {"0" => 0, "1" => 1, "2" => 2, "3.a" => "x"}
Simple data (numbers, strings, nil) passes through unchanged.
# File lib/json-csv.rb, line 240 def flatten_json(json, depth = -1) return {} if depth == 0 if json.is_a?(Hash) flat = {} json.each do |key, value| flat_assign(flat, key, value, depth) end flat elsif json.is_a?(Array) flat = {} json.each_with_index do |value, i| flat_assign(flat, i, value, depth) end flat else # number or string or nil json end end
Scans a JSON file at `input_fh` to determine the headers to use when writing CSV data. Returns a hash of `'header' => index` pairs, sorted.
# File lib/json-csv.rb, line 187 def get_headers_from_json(input_fh, tmp_fh, depth) headers = {} input_fh.each_line do |input| tmp_fh.puts(input) if tmp_fh json = JSON.parse(input) flatten_json(json, depth).each do |key, value| headers[key] = true end end sort_keys(headers) end
Reads JSON data from data_fh, and writes CSV data (with header) to output_fh.
# File lib/json-csv.rb, line 280 def output_csv(headers, data_fh, output_fh, line_ending) output_fh.write(headers.map{|h| csv_armor(h[0])}.join(",")) output_fh.write(line_ending) header_count = headers.count data_fh.each_line do |input| json = JSON.parse(input) flat = flatten_json(json) output = Array.new(header_count) flat.each do |key, value| output[headers[key]] = value if headers[key] end output_fh.write(output.map{|x| csv_armor(x)}.join(",")) output_fh.write(line_ending) end end
Helper function to get_headers_from_json
– Sorts a hash with string keys by number of dots in the string, then alphabetically. Returns a hash of `'key' => index` pairs, in order of index.
# File lib/json-csv.rb, line 203 def sort_keys(hash) sorted = {} sorted_keys = hash.keys.sort do |a, b| x = (count_dots(a) <=> count_dots(b)) x == 0 ? (a<=>b) : x end sorted_keys.each_with_index do |key, i| sorted[key] = i end sorted end