class Util::TextCleaner

Public Instance Methods

clean_controls(extracted_data) click to toggle source

Cleans control information from passed in file

# File lib/utilities/text_cleaner.rb, line 11
def clean_controls(extracted_data)
  controls_data = isolate_controls_data(extracted_data)
  clean_section_header = remove_section_header(controls_data)
  clean_whitespace = remove_newline_in_controls(clean_section_header)
  clean_special = remove_special(clean_whitespace)
  clean_no_space = remove_extra_space(clean_special)
  clean_pagenum = remove_pagenum(clean_no_space)
  separate_controls(clean_pagenum)
end
clean_data(data) click to toggle source

Takes in text file, cleans data and writes to new text file.

# File lib/utilities/text_cleaner.rb, line 6
def clean_data(data)
  clean_controls(data)
end
isolate_controls_data(extracted_data) click to toggle source

Removes everything before and after the controls

# File lib/utilities/text_cleaner.rb, line 22
def isolate_controls_data(extracted_data)
  extracted_data = extracted_data.gsub(/\| P a g e+/, "| P a g e\n")
  extracted_data = extracted_data.split("\n").map(&:strip).reject { |e| e.to_s.empty? }.join("\n")
  extracted_data = extracted_data.gsub('???', '')
  /^1\.1\s*[^)]*?(?=\)$)(.*\n)*?(?=\s*Appendix:)/.match(extracted_data).to_s
end
remove_extra_space(extracted_data) click to toggle source
# File lib/utilities/text_cleaner.rb, line 55
def remove_extra_space(extracted_data)
  clean_data = extracted_data.gsub(/\n\n\n/, "\n")
  clean_data = clean_data.gsub(/\t\n/, "\n")
  clean_data = clean_data.delete("\t")
  clean_data = clean_data.delete("\r")
  clean_data = clean_data.gsub(/\s\s/, ' ')
  clean_data.gsub(/(\n\n(?!^\d\.\d{1,}.*\n?.*?))/, '')
end
remove_newline_in_controls(extracted_data) click to toggle source

removes newlines between a control

# File lib/utilities/text_cleaner.rb, line 41
def remove_newline_in_controls(extracted_data)
  extracted_data.gsub(/\s\n.*?(?!d\.)/, "\n").to_s
end
remove_pagenum(extracted_data) click to toggle source

Removes all pagenumbers between the controls

# File lib/utilities/text_cleaner.rb, line 30
def remove_pagenum(extracted_data)
  clean_pagenum = extracted_data.gsub(/(\d{1,3}\|Page|\d{1,3} \| P a g e)/, '').to_s
  clean_pagenum.gsub(/(\d{1,3} \| Page)/, '').to_s
end
remove_section_header(extracted_data) click to toggle source

Removes section headers for each control

# File lib/utilities/text_cleaner.rb, line 36
def remove_section_header(extracted_data)
  extracted_data.gsub(/(?<!•)\s\n\d{1}\s.*(?:.*\n)*?(?=\d\.\d)/, "\n\n").to_s
end
remove_special(extracted_data) click to toggle source
# File lib/utilities/text_cleaner.rb, line 50
def remove_special(extracted_data)
  extracted_data = extracted_data.gsub(//, '')
  extracted_data.gsub(/•/, '')
end
separate_controls(extracted_data) click to toggle source

adds whitespace between different controls

# File lib/utilities/text_cleaner.rb, line 46
def separate_controls(extracted_data)
  extracted_data.gsub(/((?=^\s*?\d\.\d{1,}.*\n?.*?(?<=\)$)))/, "\n").to_s
end