class Util::TextCleaner
Public Instance Methods
clean_controls(extracted_data)
click to toggle source
Cleans control information from passed in file
# File lib/utilities/text_cleaner.rb, line 11 def clean_controls(extracted_data) controls_data = isolate_controls_data(extracted_data) clean_section_header = remove_section_header(controls_data) clean_whitespace = remove_newline_in_controls(clean_section_header) clean_special = remove_special(clean_whitespace) clean_no_space = remove_extra_space(clean_special) clean_pagenum = remove_pagenum(clean_no_space) separate_controls(clean_pagenum) end
clean_data(data)
click to toggle source
Takes in text file, cleans data and writes to new text file.
# File lib/utilities/text_cleaner.rb, line 6 def clean_data(data) clean_controls(data) end
isolate_controls_data(extracted_data)
click to toggle source
Removes everything before and after the controls
# File lib/utilities/text_cleaner.rb, line 22 def isolate_controls_data(extracted_data) extracted_data = extracted_data.gsub(/\| P a g e+/, "| P a g e\n") extracted_data = extracted_data.split("\n").map(&:strip).reject { |e| e.to_s.empty? }.join("\n") extracted_data = extracted_data.gsub('???', '') /^1\.1\s*[^)]*?(?=\)$)(.*\n)*?(?=\s*Appendix:)/.match(extracted_data).to_s end
remove_extra_space(extracted_data)
click to toggle source
# File lib/utilities/text_cleaner.rb, line 55 def remove_extra_space(extracted_data) clean_data = extracted_data.gsub(/\n\n\n/, "\n") clean_data = clean_data.gsub(/\t\n/, "\n") clean_data = clean_data.delete("\t") clean_data = clean_data.delete("\r") clean_data = clean_data.gsub(/\s\s/, ' ') clean_data.gsub(/(\n\n(?!^\d\.\d{1,}.*\n?.*?))/, '') end
remove_newline_in_controls(extracted_data)
click to toggle source
removes newlines between a control
# File lib/utilities/text_cleaner.rb, line 41 def remove_newline_in_controls(extracted_data) extracted_data.gsub(/\s\n.*?(?!d\.)/, "\n").to_s end
remove_pagenum(extracted_data)
click to toggle source
Removes all pagenumbers between the controls
# File lib/utilities/text_cleaner.rb, line 30 def remove_pagenum(extracted_data) clean_pagenum = extracted_data.gsub(/(\d{1,3}\|Page|\d{1,3} \| P a g e)/, '').to_s clean_pagenum.gsub(/(\d{1,3} \| Page)/, '').to_s end
remove_section_header(extracted_data)
click to toggle source
Removes section headers for each control
# File lib/utilities/text_cleaner.rb, line 36 def remove_section_header(extracted_data) extracted_data.gsub(/(?<!•)\s\n\d{1}\s.*(?:.*\n)*?(?=\d\.\d)/, "\n\n").to_s end
remove_special(extracted_data)
click to toggle source
# File lib/utilities/text_cleaner.rb, line 50 def remove_special(extracted_data) extracted_data = extracted_data.gsub(//, '') extracted_data.gsub(/•/, '') end
separate_controls(extracted_data)
click to toggle source
adds whitespace between different controls
# File lib/utilities/text_cleaner.rb, line 46 def separate_controls(extracted_data) extracted_data.gsub(/((?=^\s*?\d\.\d{1,}.*\n?.*?(?<=\)$)))/, "\n").to_s end