class Dependabot::Python::FileFetcher
Constants
- CHILD_REQUIREMENT_REGEX
- CONSTRAINT_REGEX
Public Class Methods
required_files_in?(filenames)
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 17 def self.required_files_in?(filenames) return true if filenames.any? { |name| name.end_with?(".txt", ".in") } # If there is a directory of requirements return true return true if filenames.include?("requirements") # If this repo is using a Pipfile return true return true if filenames.include?("Pipfile") # If this repo is using Poetry return true return true if filenames.include?("pyproject.toml") return true if filenames.include?("setup.py") filenames.include?("setup.cfg") end
required_files_message()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 34 def self.required_files_message "Repo must contain a requirements.txt, setup.py, setup.cfg, pyproject.toml, "\ "or a Pipfile." end
Private Instance Methods
cfg_files_for_setup_py(path)
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 306 def cfg_files_for_setup_py(path) cfg_path = path.gsub(/\.py$/, ".cfg") begin [ fetch_file_from_host(cfg_path, fetch_submodules: true). tap { |f| f.support_file = true } ] rescue Dependabot::DependencyFileNotFound # Ignore lack of a setup.cfg [] end end
check_required_files_present()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 82 def check_required_files_present return if requirements_txt_files.any? || setup_file || setup_cfg_file || pipfile || pyproject path = Pathname.new(File.join(directory, "requirements.txt")). cleanpath.to_path raise Dependabot::DependencyFileNotFound, path end
child_requirement_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 204 def child_requirement_files @child_requirement_files ||= begin fetched_files = req_txt_and_in_files.dup req_txt_and_in_files.flat_map do |requirement_file| child_files = fetch_child_requirement_files( file: requirement_file, previously_fetched_files: fetched_files ) fetched_files += child_files child_files end end end
child_requirement_in_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 200 def child_requirement_in_files child_requirement_files.select { |f| f.name.end_with?(".in") } end
child_requirement_txt_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 196 def child_requirement_txt_files child_requirement_files.select { |f| f.name.end_with?(".txt") } end
constraints_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 240 def constraints_files all_requirement_files = requirements_txt_files + child_requirement_txt_files constraints_paths = all_requirement_files.map do |req_file| current_dir = File.dirname(req_file.name) paths = req_file.content.scan(CONSTRAINT_REGEX).flatten paths.map do |path| path = File.join(current_dir, path) unless current_dir == "." Pathname.new(path).cleanpath.to_path end end.flatten.uniq constraints_paths.map { |path| fetch_file_from_host(path) } end
fetch_child_requirement_files(file:, previously_fetched_files:)
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 220 def fetch_child_requirement_files(file:, previously_fetched_files:) paths = file.content.scan(CHILD_REQUIREMENT_REGEX).flatten current_dir = File.dirname(file.name) paths.flat_map do |path| path = File.join(current_dir, path) unless current_dir == "." path = Pathname.new(path).cleanpath.to_path next if previously_fetched_files.map(&:name).include?(path) next if file.name == path fetched_file = fetch_file_from_host(path) grandchild_requirement_files = fetch_child_requirement_files( file: fetched_file, previously_fetched_files: previously_fetched_files + [file] ) [fetched_file, *grandchild_requirement_files] end.compact end
fetch_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 41 def fetch_files fetched_files = [] fetched_files += pipenv_files fetched_files += pyproject_files fetched_files += requirements_in_files fetched_files += requirement_files if requirements_txt_files.any? fetched_files << setup_file if setup_file fetched_files << setup_cfg_file if setup_cfg_file fetched_files += path_setup_files fetched_files << pip_conf if pip_conf fetched_files << python_version if python_version check_required_files_present uniq_files(fetched_files) end
fetch_path_setup_file(path, allow_pyproject: false)
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 278 def fetch_path_setup_file(path, allow_pyproject: false) path_setup_files = [] unless path.end_with?(".tar.gz", ".whl", ".zip") path = Pathname.new(File.join(path, "setup.py")).cleanpath.to_path end return [] if path == "setup.py" && setup_file path_setup_files << begin fetch_file_from_host( path, fetch_submodules: true ).tap { |f| f.support_file = true } rescue Dependabot::DependencyFileNotFound raise unless allow_pyproject fetch_file_from_host( path.gsub("setup.py", "pyproject.toml"), fetch_submodules: true ).tap { |f| f.support_file = true } end return path_setup_files unless path.end_with?(".py") path_setup_files + cfg_files_for_setup_py(path) end
parse_path_setup_paths(req_file)
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 350 def parse_path_setup_paths(req_file) uneditable_reqs = req_file.content. scan(/^['"]?(?:file:)?(?<path>\..*?)(?=\[|#|'|"|$)/). flatten. map(&:strip). reject { |p| p.include?("://") } editable_reqs = req_file.content. scan(/^(?:-e)\s+['"]?(?:file:)?(?<path>.*?)(?=\[|#|'|"|$)/). flatten. map(&:strip). reject { |p| p.include?("://") || p.include?("git@") } uneditable_reqs + editable_reqs end
parsed_pipfile()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 147 def parsed_pipfile raise "No Pipfile" unless pipfile @parsed_pipfile ||= TomlRB.parse(pipfile.content) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError raise Dependabot::DependencyFileNotParseable, pipfile.path end
parsed_pyproject()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 155 def parsed_pyproject raise "No pyproject.toml" unless pyproject @parsed_pyproject ||= TomlRB.parse(pyproject.content) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError raise Dependabot::DependencyFileNotParseable, pyproject.path end
path_setup_file_paths()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 332 def path_setup_file_paths requirement_txt_path_setup_file_paths + requirement_in_path_setup_file_paths + pipfile_path_setup_file_paths end
path_setup_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 257 def path_setup_files path_setup_files = [] unfetchable_files = [] path_setup_file_paths.each do |path| path_setup_files += fetch_path_setup_file(path) rescue Dependabot::DependencyFileNotFound => e unfetchable_files << e.file_path.gsub(%r{^/}, "") end poetry_path_setup_file_paths.each do |path| path_setup_files += fetch_path_setup_file(path, allow_pyproject: true) rescue Dependabot::DependencyFileNotFound => e unfetchable_files << e.file_path.gsub(%r{^/}, "") end raise Dependabot::PathDependenciesNotReachable, unfetchable_files if unfetchable_files.any? path_setup_files end
pip_conf()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 98 def pip_conf @pip_conf ||= fetch_file_if_present("pip.conf")&. tap { |f| f.support_file = true } end
pipenv_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 66 def pipenv_files [pipfile, pipfile_lock].compact end
pipfile()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 118 def pipfile @pipfile ||= fetch_file_if_present("Pipfile") end
pipfile_lock()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 122 def pipfile_lock @pipfile_lock ||= fetch_file_if_present("Pipfile.lock") end
pipfile_path_setup_file_paths()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 368 def pipfile_path_setup_file_paths return [] unless pipfile paths = [] %w(packages dev-packages).each do |dep_type| next unless parsed_pipfile[dep_type] parsed_pipfile[dep_type].each do |_, req| next unless req.is_a?(Hash) && req["path"] paths << req["path"] end end paths end
poetry_lock()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 134 def poetry_lock @poetry_lock ||= fetch_file_if_present("poetry.lock") end
poetry_path_setup_file_paths()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 385 def poetry_path_setup_file_paths return [] unless pyproject paths = [] Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |dep_type| next unless parsed_pyproject.dig("tool", "poetry", dep_type) parsed_pyproject.dig("tool", "poetry", dep_type).each do |_, req| next unless req.is_a?(Hash) && req["path"] paths << req["path"] end end paths end
pyproject()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 126 def pyproject @pyproject ||= fetch_file_if_present("pyproject.toml") end
pyproject_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 70 def pyproject_files [pyproject, pyproject_lock, poetry_lock].compact end
pyproject_lock()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 130 def pyproject_lock @pyproject_lock ||= fetch_file_if_present("pyproject.lock") end
python_version()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 103 def python_version @python_version ||= fetch_file_if_present(".python-version")&. tap { |f| f.support_file = true } return @python_version if @python_version return if [".", "/"].include?(directory) # Check the top-level for a .python-version file, too reverse_path = Pathname.new(directory[0]).relative_path_from(directory) @python_version ||= fetch_file_if_present(File.join(reverse_path, ".python-version"))&. tap { |f| f.support_file = true }&. tap { |f| f.name = ".python-version" } end
req_files_for_dir(requirements_dir)
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 183 def req_files_for_dir(requirements_dir) dir = directory.gsub(%r{(^/|/$)}, "") relative_reqs_dir = requirements_dir.path.gsub(%r{^/?#{Regexp.escape(dir)}/?}, "") repo_contents(dir: relative_reqs_dir). select { |f| f.type == "file" }. select { |f| f.name.end_with?(".txt", ".in") }. reject { |f| f.size > 200_000 }. map { |f| fetch_file_from_host("#{relative_reqs_dir}/#{f.name}") }. select { |f| requirements_file?(f) } end
req_txt_and_in_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 163 def req_txt_and_in_files return @req_txt_and_in_files if @req_txt_and_in_files @req_txt_and_in_files = [] repo_contents. select { |f| f.type == "file" }. select { |f| f.name.end_with?(".txt", ".in") }. reject { |f| f.size > 200_000 }. map { |f| fetch_file_from_host(f.name) }. select { |f| requirements_file?(f) }. each { |f| @req_txt_and_in_files << f } repo_contents. select { |f| f.type == "dir" }. each { |f| @req_txt_and_in_files += req_files_for_dir(f) } @req_txt_and_in_files end
requirement_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 74 def requirement_files [ *requirements_txt_files, *child_requirement_txt_files, *constraints_files ] end
requirement_in_path_setup_file_paths()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 344 def requirement_in_path_setup_file_paths requirements_in_files. map { |req_file| parse_path_setup_paths(req_file) }. flatten.uniq end
requirement_txt_path_setup_file_paths()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 338 def requirement_txt_path_setup_file_paths (requirements_txt_files + child_requirement_txt_files). map { |req_file| parse_path_setup_paths(req_file) }. flatten.uniq end
requirements_file?(file)
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 320 def requirements_file?(file) return false unless file.content.valid_encoding? return true if file.name.match?(/requirements/x) file.content.lines.all? do |line| next true if line.strip.empty? next true if line.strip.start_with?("#", "-r ", "-c ", "-e ", "--") line.match?(RequirementParser::VALID_REQ_TXT_REQUIREMENT) end end
requirements_in_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 142 def requirements_in_files req_txt_and_in_files.select { |f| f.name.end_with?(".in") } + child_requirement_in_files end
requirements_txt_files()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 138 def requirements_txt_files req_txt_and_in_files.select { |f| f.name.end_with?(".txt") } end
setup_cfg_file()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 94 def setup_cfg_file @setup_cfg_file ||= fetch_file_if_present("setup.cfg") end
setup_file()
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 90 def setup_file @setup_file ||= fetch_file_if_present("setup.py") end
uniq_files(fetched_files)
click to toggle source
# File lib/dependabot/python/file_fetcher.rb, line 60 def uniq_files(fetched_files) uniq_files = fetched_files.reject(&:support_file?).uniq uniq_files += fetched_files. reject { |f| uniq_files.map(&:name).include?(f.name) } end