class JsonCsv

Constants

DEFAULT_OPTS
VERSION
VERSION_DATE

Public Class Methods

convert_csv_to_json(opts) click to toggle source
# File lib/json-csv.rb, line 97
def convert_csv_to_json(opts)
  self.new(opts).convert_csv_to_json()
end
convert_json_to_csv(opts) click to toggle source
# File lib/json-csv.rb, line 93
def convert_json_to_csv(opts)
  self.new(opts).convert_json_to_csv()
end
new(opts) click to toggle source
# File lib/json-csv.rb, line 103
def initialize(opts)
  @opts = DEFAULT_OPTS.merge(opts)
end
new_from_argv(argv) click to toggle source
# File lib/json-csv.rb, line 88
def new_from_argv(argv)
  opts = parse_argv(argv)
  self.new(opts)
end
parse_argv(argv) click to toggle source
# File lib/json-csv.rb, line 32
    def parse_argv(argv)
      opts = DEFAULT_OPTS

      argv = OptionParser.new do |op|
        op.banner = <<-EOT
Converts JSON to CSV, and vice versa.
Usage: #{$0} [options] [--] [input-file [output-file]]
        EOT

        op.on("-i input-file", "--input input-file", "Input file (default STDIN)") do |input_file|
          opts[:input_file] = input_file
        end

        op.on("-o output-file", "--output output-file", "Output file (default STDOUT)") do |output_file|
          opts[:output_file] = output_file
        end

        op.on("-s json|csv", "--source-encoding json|csv", "Encoding of input file (default json)") do |source|
          opts[:source_encoding] = source
        end

        op.on("-d depth", "--depth depth", "Maximum depth of JSON-to-CSV conversion (default -1, unlimited)") do |depth|
          opts[:depth] = depth.to_i
        end

        op.on("-e crlf|cr|lf", "--line-ending crlf|cr|lf", "Line endings for output file (default crlf).") do |ending|
          opts[:line_ending] = {"crlf" => "\r\n", "cr" => "\r", "lf" => "\n"}[ending]
          if !opts[:line_ending]
            STDERR.puts "Invalid line ending '#{ending}'.  Valid choices: crlf cr lf"
            exit 1
          end
        end

        op.on_tail("--debug", "Turn debugging messages on") do
          opts[:debug] = true
        end

        op.on_tail("--version", "Print version info and exit") do
          puts "json-csv version #{VERSION} (#{VERSION_DATE})"
          puts "https://github.com/appcues/json-csv"
          exit
        end

        op.on_tail("-h", "--help", "Show this message and exit") do
          puts op.to_s
          exit
        end

      end.parse(argv)

      opts[:input_file] = argv.shift if argv.count > 0
      opts[:output_file] = argv.shift if argv.count > 0

      opts
    end

Public Instance Methods

convert_csv_to_json(opts = {}) click to toggle source
# File lib/json-csv.rb, line 171
def convert_csv_to_json(opts = {})
  raise NotImplementedError
end
convert_json_to_csv(opts = {}) click to toggle source
# File lib/json-csv.rb, line 124
def convert_json_to_csv(opts = {})
  opts = @opts.merge(opts)

  ## First pass -- create CSV headers from JSON input
  input_fh = nil
  tmp_fh = nil
  tmp_filename = nil
  data_filename = nil

  if opts[:input_file] == "-"
    input_fh = STDIN
    data_filename = tmp_filename = "#{opts[:tmpdir]}/json-csv-#{$$}.tmp"
    debug(opts, "STDIN will be written to #{tmp_filename}.")
    tmp_fh = File.open(data_filename, "w")
  else
    input_fh = File.open(opts[:input_file], "r")
    data_filename = opts[:input_file]
  end

  debug(opts, "Getting headers from JSON data.")
  depth = opts[:depth]
  depth += 1 if depth > 0  # a fudge, in order to use -1 as infinity
  headers = get_headers_from_json(input_fh, tmp_fh, depth)

  input_fh.close
  tmp_fh.close if tmp_fh


  ## Second pass -- write CSV data from JSON input
  data_fh = File.open(data_filename, "r")
  output_fh = nil

  if opts[:output_file] == "-"
    output_fh = STDOUT
  else
    output_fh = File.open(opts[:output_file], "w")
  end

  debug(opts, "Writing CSV output.")
  output_csv(headers, data_fh, output_fh, opts[:line_ending])
  data_fh.close
  output_fh.close

  debug(opts, "Removing #{tmp_filename}.")
  File.unlink(tmp_filename) if tmp_filename
end
run(opts = {}) click to toggle source

Performs the JSON-to-CSV or CSV-to-JSON conversion, as specified in `opts` and the options passed in during `JsonCsv.new`.

# File lib/json-csv.rb, line 110
def run(opts = {})
  opts = @opts.merge(opts)
  enc = opts[:source_encoding]
  if enc == "json"
    convert_json_to_csv()
  elsif enc == "csv"
    convert_csv_to_json()
  else
    STDERR.puts "no such source encoding '#{enc}'"
    exit 1
  end
end

Private Instance Methods

count_dots(str) click to toggle source

Helper function to sort_keys – Counts the number of dots in a string.

# File lib/json-csv.rb, line 217
def count_dots(str)
  str.chars.select{|c| c == "."}.count
end
csv_armor(val) click to toggle source

Helper function to output_csv – Returns a CSV-armored version of `val`. Escapes special characters and adds double-quotes if necessary.

# File lib/json-csv.rb, line 300
def csv_armor(val)
  str = val.to_s.gsub('"', '""')
  if str.match(/[",\n]/)
    '"' + str + '"'
  else
    str
  end
end
debug(opts, msg) click to toggle source
# File lib/json-csv.rb, line 178
def debug(opts, msg)
  STDERR.puts("#{Time.now}\t#{msg}") if opts[:debug]
end
flat_assign(dest, key, value, depth) click to toggle source

Helper function to flatten_json – Assigns a flattened value at the current key.

# File lib/json-csv.rb, line 264
def flat_assign(dest, key, value, depth)
  flat_value = flatten_json(value, depth - 1)
  if flat_value.is_a?(Hash)
    flat_value.each do |k,v|
      dest["#{key}.#{k}"] = v
    end
  else
    dest["#{key}"] = flat_value
  end
  dest
end
flatten_json(json, depth = -1) click to toggle source

Returns a flattened representation of the given JSON-encodable data (that is: hashes, arrays, numbers, strings, and `nil`). Dot-separated string keys are used to encode nested hash and array structures.

Hashes get flattened like so:

flatten_json({a: {b: {c: 1, d: "x"}, c: nil}})
#=> {"a.b.c" => 1, "a.b.d" => "x", "a.c" => nil}

Arrays are turned into hashes like:

flatten_json([0, 1, 2, {a: "x"])
#=> {"0" => 0, "1" => 1, "2" => 2, "3.a" => "x"}

Simple data (numbers, strings, nil) passes through unchanged.

# File lib/json-csv.rb, line 240
def flatten_json(json, depth = -1)
  return {} if depth == 0

  if json.is_a?(Hash)
    flat = {}
    json.each do |key, value|
      flat_assign(flat, key, value, depth)
    end
    flat

  elsif json.is_a?(Array)
    flat = {}
    json.each_with_index do |value, i|
      flat_assign(flat, i, value, depth)
    end
    flat

  else # number or string or nil
    json
  end
end
get_headers_from_json(input_fh, tmp_fh, depth) click to toggle source

Scans a JSON file at `input_fh` to determine the headers to use when writing CSV data. Returns a hash of `'header' => index` pairs, sorted.

# File lib/json-csv.rb, line 187
def get_headers_from_json(input_fh, tmp_fh, depth)
  headers = {}
  input_fh.each_line do |input|
    tmp_fh.puts(input) if tmp_fh
    json = JSON.parse(input)
    flatten_json(json, depth).each do |key, value|
      headers[key] = true
    end
  end
  sort_keys(headers)
end
output_csv(headers, data_fh, output_fh, line_ending) click to toggle source

Reads JSON data from data_fh, and writes CSV data (with header) to output_fh.

# File lib/json-csv.rb, line 280
def output_csv(headers, data_fh, output_fh, line_ending)
  output_fh.write(headers.map{|h| csv_armor(h[0])}.join(","))
  output_fh.write(line_ending)

  header_count = headers.count
  data_fh.each_line do |input|
    json = JSON.parse(input)
    flat = flatten_json(json)
    output = Array.new(header_count)
    flat.each do |key, value|
      output[headers[key]] = value if headers[key]
    end
    output_fh.write(output.map{|x| csv_armor(x)}.join(","))
    output_fh.write(line_ending)
  end
end
sort_keys(hash) click to toggle source

Helper function to get_headers_from_json – Sorts a hash with string keys by number of dots in the string, then alphabetically. Returns a hash of `'key' => index` pairs, in order of index.

# File lib/json-csv.rb, line 203
def sort_keys(hash)
  sorted = {}
  sorted_keys = hash.keys.sort do |a, b|
    x = (count_dots(a) <=> count_dots(b))
    x == 0 ? (a<=>b) : x
  end
  sorted_keys.each_with_index do |key, i|
    sorted[key] = i
  end
  sorted
end