class Lakes::Texas
Attributes
lake_data[R]
Public Class Methods
new()
click to toggle source
# File lib/lakes/texas.rb, line 12 def initialize @lake_data = {} end
Public Instance Methods
all_details()
click to toggle source
# File lib/lakes/texas.rb, line 16 def all_details result = [] list.each do |lake_name| result << get_details(lake_name) sleep(1) end result end
get_details(lake_name)
click to toggle source
# File lib/lakes/texas.rb, line 44 def get_details(lake_name) puts "getting details for #{lake_name}" list data = lake_data[lake_name] raise 'Lake not found' if data.nil? data[:name] = lake_name parse_lake_details(data) end
list()
click to toggle source
# File lib/lakes/texas.rb, line 25 def list return @lake_data.keys unless @lake_data.empty? base_url = 'https://tpwd.texas.gov/fishboat/fish/recreational/lakes/' content = http_get("#{base_url}lakelist.phtml") html_doc = Nokogiri::HTML(content) # remove elements not needed to make parsing easier html_doc.search('div.announce, div.alert, div#bottomwrapper').each do |src| src.remove end html_doc.search('div#maincontent ul li a').each do |lake_html| lake_name = cleanup_data(lake_html.text) @lake_data[lake_name] = { details_uri: "#{base_url}#{lake_html[:href]}" } end @lake_data.keys end
Protected Instance Methods
parse_angling_opportunities(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 89 def parse_angling_opportunities(main_div, lake_data) data = main_div.xpath('//h6[contains(text(), "Angling Opportunities")]').first description = data.try(:next_element).try(:text) lake_data[:angling_opportunities_description] = description table = main_div.css('#Ratings') quality = ['Poor', 'Fair', 'Good', 'Excellent'] fish_species_elements = table.css('tr th.highlight2') lake_data[:angling_opportunities_details] = {} fish_species_elements.each do |fish_species_element| rating_index = 0 species = cleanup_data(fish_species_element.text) while fish_species_element = fish_species_element.next_element if fish_species_element.css('img').count == 0 rating_index += 1 next else lake_data[:angling_opportunities_details][species] = quality[rating_index] break end end end end
parse_aquatic_vegetation(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 167 def parse_aquatic_vegetation(main_div, lake_data) process_simple_section(main_div, lake_data, 'Aquatic Vegetation', :aquatic_vegetation, false) end
parse_current_fishing_report(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 208 def parse_current_fishing_report(main_div, lake_data) link = main_div.xpath('p/a[contains(text(), "Fishing Report")]').first fishing_report = if link.nil? nil else uri = link['href'] lake_data[:current_fishing_report_uri] = convert_relative_href(uri, lake_data[:details_uri]) content = http_get(lake_data[:current_fishing_report_uri]) report_parser = FishingReportParser.new(content) { date: report_parser.date, report: report_parser.report } end lake_data[:current_fishing_report] = fishing_report end
parse_fishing_regulations(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 114 def parse_fishing_regulations(main_div, lake_data) process_simple_section(main_div, lake_data, 'Fishing Regulations', :fishing_regulations, true) end
parse_fishing_structure(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 83 def parse_fishing_structure(main_div, lake_data) data = main_div.xpath('//h6[contains(text(), "Fishing Cover/Structure")]').first content = data.try(:next_element).try(:to_html) lake_data[:structure_and_cover_description] = content end
parse_lake_characteristics(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 122 def parse_lake_characteristics(main_div, lake_data) data = main_div.xpath("//h6[contains(text(), 'Lake Characteristics')]").first content = data.try(:next_element).try(:text).try(:strip) lake_data[:raw_lake_characteristics] = content parser = LakeCharacteristicsParser.new(content) lake_data[:lake_characteristics] = {} lake_data[:lake_characteristics][:location_desc] = parser.location_desc lake_data[:lake_characteristics][:surface_area_in_acres] = parser.surface_area_in_acres lake_data[:lake_characteristics][:max_depth_in_feet] = parser.max_depth_in_feet lake_data[:lake_characteristics][:year_impounded] = parser.year_impounded end
parse_lake_details(lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 55 def parse_lake_details(lake_data) content = http_get(lake_data[:details_uri] + '/') html_doc = Nokogiri::HTML(content) main_div = html_doc.at('div#maincontent') parse_lake_characteristics(main_div, lake_data) parse_water_conditions_and_data(main_div, lake_data) parse_reservoir_controlling_authority(main_div, lake_data) parse_aquatic_vegetation(main_div, lake_data) parse_predominant_fish_species(main_div, lake_data) parse_current_fishing_report(main_div, lake_data) parse_lake_surveys(main_div, lake_data) parse_lake_maps(main_div, lake_data) parse_fishing_regulations(main_div, lake_data) parse_angling_opportunities(main_div, lake_data) parse_fishing_structure(main_div, lake_data) parse_tips_and_tactics(main_div, lake_data) parse_lake_records(main_div, lake_data) parse_stocking_history(main_div, lake_data) lake_data end
parse_lake_maps(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 118 def parse_lake_maps(main_div, lake_data) process_simple_section(main_div, lake_data, 'Lake Maps', :lake_maps, false) end
parse_lake_records(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 227 def parse_lake_records(main_div, lake_data) link = main_div.xpath('//a[contains(text(), "Lake Records")]').first if link.nil? lake_data[:fishing_records_uri] = nil return end uri = link['href'] lake_data[:fishing_records_uri] = convert_relative_href(uri, lake_data[:details_uri]) content = http_get(lake_data[:fishing_records_uri]) parser = LakeRecordsParser.new(content) lake_data[:fishing_records] = parser.records end
parse_lake_surveys(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 178 def parse_lake_surveys(main_div, lake_data) link = main_div.xpath('p/a[contains(text(), "Latest Survey Report")]').first if link.nil? nil else uri = link['href'] lake_data[:latest_survey_report] = convert_relative_href(uri, lake_data[:details_uri]) end end
parse_predominant_fish_species(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 171 def parse_predominant_fish_species(main_div, lake_data) element = main_div.xpath('//h6[contains(text(), "Predominant Fish Species")]').first species_root = element.try(:next_element) species = species_root.nil? ? nil : species_root.css('li').map{ |e| cleanup_data(e.text) } lake_data[:predominant_fish_species] = species end
parse_stocking_history(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 188 def parse_stocking_history(main_div, lake_data) link = main_div.xpath('p/a[contains(text(), "Stocking History")]').first if link.nil? nil else uri = link['href'] lake_data[:stocking_history_uri] = convert_relative_href(uri, lake_data[:details_uri]) content = http_get(lake_data[:stocking_history_uri]) stocking_history_doc = Nokogiri::HTML(content) stocking_history_table = stocking_history_doc.at('div#maincontent table') headers = stocking_history_table.xpath('tr/th').map{ |r| r.text } rows = stocking_history_table.xpath('tr/td').map{ |r| r.text } table_data = process_data_table(headers, rows) lake_data[:stocking_history] = table_data end end
parse_tips_and_tactics(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 77 def parse_tips_and_tactics(main_div, lake_data) data = main_div.xpath('//h6[contains(text(), "Tips & Tactics")]').first content = data.try(:next_element).try(:text) lake_data[:tips_and_tactics] = content end
parse_water_conditions_and_data(main_div, lake_data)
click to toggle source
# File lib/lakes/texas.rb, line 136 def parse_water_conditions_and_data(main_div, lake_data) lake_data[:raw_water_conditions] = process_simple_section(main_div, lake_data, 'Water Conditions', :water_conditions, true) # File.write("test/data/water_conditions/#{lake_data[:name]}.txt", lake_data[:raw_water_conditions]) parser = WaterConditionsParser.new(lake_data[:raw_water_conditions]) lake_data[:water] = {} lake_data[:water][:conditions] = {} lake_data[:water][:water_data_uri] = parser.water_data_uri lake_data[:water][:conditions][:conservation_pool_elevation] = parser.conservation_pool_elevation lake_data[:water][:conditions][:fluctuation] = parser.fluctuation lake_data[:water][:conditions][:normal_clarity] = parser.normal_clarity lake_data[:water][:data] = {} return if parser.water_data_uri.nil? content = begin http_get(parser.water_data_uri) rescue Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => e puts "#{e.message} for #{lake_data[:name]}: #{parser.water_data_uri}" nil end return if content.nil? water_data_parser = WaterDataParser.new(content) lake_data[:water][:conservation_pool_elevation_in_ft_msl] = water_data_parser.conservation_pool_elevation_in_ft_msl lake_data[:water][:percentage_full] = water_data_parser.percentage_full end
process_simple_section(main_div, lake_data, section_title, data_name, html)
click to toggle source
# File lib/lakes/texas.rb, line 241 def process_simple_section(main_div, lake_data, section_title, data_name, html) data = main_div.xpath("//h6[contains(text(), \"#{section_title}\")]").first element_type_function = html ? :to_html : :text content = data.try(:next_element).try(element_type_function) lake_data[data_name] = content end