class FPM::Package::Python

Support for python packages.

This supports input, but not output.

Example:

# Download the django python package:
pkg = FPM::Package::Python.new
pkg.input("Django")

Public Instance Methods

input(package) click to toggle source

Input a package.

The ‘package’ can be any of:

  • A name of a package on pypi (ie; easy_install some-package)

  • The path to a directory containing setup.py

  • The path to a setup.py

# File lib/fpm/package/python.rb, line 97
def input(package)
  path_to_package = download_if_necessary(package, version)

  if File.directory?(path_to_package)
    setup_py = File.join(path_to_package, "setup.py")
  else
    setup_py = path_to_package
  end

  if !File.exist?(setup_py)
    logger.error("Could not find 'setup.py'", :path => setup_py)
    raise "Unable to find python package; tried #{setup_py}"
  end

  load_package_info(setup_py)
  install_to_staging(setup_py)
end

Private Instance Methods

download_if_necessary(package, version=nil) click to toggle source

Download the given package if necessary. If version is given, that version will be downloaded, otherwise the latest is fetched.

# File lib/fpm/package/python.rb, line 117
def download_if_necessary(package, version=nil)
  # TODO(sissel): this should just be a 'download' method, the 'if_necessary'
  # part should go elsewhere.
  path = package
  # If it's a path, assume local build.
  if File.directory?(path) or (File.exist?(path) and File.basename(path) == "setup.py")
    return path
  end

  logger.info("Trying to download", :package => package)

  if version.nil?
    want_pkg = "#{package}"
  else
    want_pkg = "#{package}==#{version}"
  end

  target = build_path(package)
  FileUtils.mkdir(target) unless File.directory?(target)

  if attributes[:python_internal_pip?]
    # XXX: Should we detect if internal pip is available?
    attributes[:python_pip] = [ attributes[:python_bin], "-m", "pip"]
  end

  # attributes[:python_pip] -- expected to be a path
  if attributes[:python_pip]
    logger.debug("using pip", :pip => attributes[:python_pip])
    # TODO: Support older versions of pip

    pip = [attributes[:python_pip]] if pip.is_a?(String)
    setup_cmd = [
      *attributes[:python_pip],
      "download",
      "--no-clean",
      "--no-deps",
      "--no-binary", ":all:",
      "-d", build_path,
      "-i", attributes[:python_pypi],
    ]

    if attributes[:python_trusted_host]
      setup_cmd += [
        "--trusted-host",
        attributes[:python_trusted_host],
      ]
    end

    setup_cmd << want_pkg

    safesystem(*setup_cmd)

    # Pip removed the --build flag sometime in 2021, it seems: https://github.com/pypa/pip/issues/8333
    # A workaround for pip removing the `--build` flag. Previously, `pip download --build ...` would leave
    # behind a directory with the Python package extracted and ready to be used.
    # For example, `pip download ... Django` puts `Django-4.0.4.tar.tz` into the build_path directory.
    # If we expect `pip` to leave an unknown-named file in the `build_path` directory, let's check for
    # a single file and unpack it.
    files = ::Dir.glob(File.join(build_path, "*.{tar.gz,zip}"))
    if files.length != 1
      raise "Unexpected directory layout after `pip download ...`. This might be an fpm bug? The directory is #{build_path}"
    end

    if files[0].end_with?("tar.gz")
      safesystem("tar", "-zxf", files[0], "-C", target)
    elsif files[0].end_with?("zip")
      safesystem("unzip", files[0], "-d", target)
    else
      raise "Unexpected file format after `pip download ...`. This might be an fpm bug? The file is #{files[0]}"
    end
  else
    # no pip, use easy_install
    logger.debug("no pip, defaulting to easy_install", :easy_install => attributes[:python_easyinstall])
    safesystem(attributes[:python_easyinstall], "-i",
               attributes[:python_pypi], "--editable", "-U",
               "--build-directory", target, want_pkg)
  end

  # easy_install will put stuff in @tmpdir/packagename/, so find that:
  #  @tmpdir/somepackage/setup.py
  dirs = ::Dir.glob(File.join(target, "*"))
  if dirs.length != 1
    raise "Unexpected directory layout after easy_install. Maybe file a bug? The directory is #{build_path}"
  end
  return dirs.first
end
fix_name(name) click to toggle source

Sanitize package name. Some PyPI packages can be named ‘python-foo’, so we don’t want to end up with a package named ‘python-python-foo’. But we want packages named like ‘pythonweb’ to be suffixed ‘python-pythonweb’.

# File lib/fpm/package/python.rb, line 321
def fix_name(name)
  if name.start_with?("python")
    # If the python package is called "python-foo" strip the "python-" part while
    # prepending the package name prefix.
    return [attributes[:python_package_name_prefix], name.gsub(/^python-/, "")].join("-")
  else
    return [attributes[:python_package_name_prefix], name].join("-")
  end
end
install_to_staging(setup_py) click to toggle source

Install this package to the staging directory

# File lib/fpm/package/python.rb, line 332
def install_to_staging(setup_py)
  project_dir = File.dirname(setup_py)

  prefix = "/"
  prefix = attributes[:prefix] unless attributes[:prefix].nil?

  # Some setup.py's assume $PWD == current directory of setup.py, so let's
  # chdir first.
  ::Dir.chdir(project_dir) do
    flags = [ "--root", staging_path ]
    if !attributes[:python_install_lib].nil?
      flags += [ "--install-lib", File.join(prefix, attributes[:python_install_lib]) ]
    elsif !attributes[:prefix].nil?
      # setup.py install --prefix PREFIX still installs libs to
      # PREFIX/lib64/python2.7/site-packages/
      # but we really want something saner.
      #
      # since prefix is given, but not python_install_lib, assume PREFIX/lib
      flags += [ "--install-lib", File.join(prefix, "lib") ]
    end

    if !attributes[:python_install_data].nil?
      flags += [ "--install-data", File.join(prefix, attributes[:python_install_data]) ]
    elsif !attributes[:prefix].nil?
      # prefix given, but not python_install_data, assume PREFIX/data
      flags += [ "--install-data", File.join(prefix, "data") ]
    end

    if !attributes[:python_install_bin].nil?
      flags += [ "--install-scripts", File.join(prefix, attributes[:python_install_bin]) ]
    elsif !attributes[:prefix].nil?
      # prefix given, but not python_install_bin, assume PREFIX/bin
      flags += [ "--install-scripts", File.join(prefix, "bin") ]
    end

    if !attributes[:python_scripts_executable].nil?
      # Overwrite installed python scripts shebang binary with provided executable
      flags += [ "build_scripts", "--executable", attributes[:python_scripts_executable] ]
    end

    if !attributes[:python_setup_py_arguments].nil? and !attributes[:python_setup_py_arguments].empty?
      # Add optional setup.py arguments
      attributes[:python_setup_py_arguments].each do |a|
        flags += [ a ]
      end
    end

    safesystem(attributes[:python_bin], "setup.py", "install", *flags)
  end
end
load_package_info(setup_py) click to toggle source

Load the package information like name, version, dependencies.

# File lib/fpm/package/python.rb, line 205
def load_package_info(setup_py)
  if !attributes[:python_package_prefix].nil?
    attributes[:python_package_name_prefix] = attributes[:python_package_prefix]
  end

  begin
    json_test_code = [
      "try:",
      "  import json",
      "except ImportError:",
      "  import simplejson as json"
    ].join("\n")
    safesystem("#{attributes[:python_bin]} -c '#{json_test_code}'")
  rescue FPM::Util::ProcessFailed => e
    logger.error("Your python environment is missing json support (either json or simplejson python module). I cannot continue without this.", :python => attributes[:python_bin], :error => e)
    raise FPM::Util::ProcessFailed, "Python (#{attributes[:python_bin]}) is missing simplejson or json modules."
  end

  begin
    safesystem("#{attributes[:python_bin]} -c 'import pkg_resources'")
  rescue FPM::Util::ProcessFailed => e
    logger.error("Your python environment is missing a working setuptools module. I tried to find the 'pkg_resources' module but failed.", :python => attributes[:python_bin], :error => e)
    raise FPM::Util::ProcessFailed, "Python (#{attributes[:python_bin]}) is missing pkg_resources module."
  end

  # Add ./pyfpm/ to the python library path
  pylib = File.expand_path(File.dirname(__FILE__))

  # chdir to the directory holding setup.py because some python setup.py's assume that you are
  # in the same directory.
  setup_dir = File.dirname(setup_py)

  output = ::Dir.chdir(setup_dir) do
    tmp = build_path("metadata.json")
    setup_cmd = "env PYTHONPATH=#{pylib.shellescape}:$PYTHONPATH #{attributes[:python_bin]} " \
      "setup.py --command-packages=pyfpm get_metadata --output=#{tmp}"

    if attributes[:python_obey_requirements_txt?]
      setup_cmd += " --load-requirements-txt"
    end

    # Capture the output, which will be JSON metadata describing this python
    # package. See fpm/lib/fpm/package/pyfpm/get_metadata.py for more
    # details.
    logger.info("fetching package metadata", :setup_cmd => setup_cmd)

    success = safesystem(setup_cmd)
    #%x{#{setup_cmd}}
    if !success
      logger.error("setup.py get_metadata failed", :command => setup_cmd,
                    :exitcode => $?.exitstatus)
      raise "An unexpected error occurred while processing the setup.py file"
    end
    File.read(tmp)
  end
  logger.debug("result from `setup.py get_metadata`", :data => output)
  metadata = JSON.parse(output)
  logger.info("object output of get_metadata", :json => metadata)

  self.architecture = metadata["architecture"]
  self.description = metadata["description"]
  # Sometimes the license field is multiple lines; do best-effort and just
  # use the first line.
  if metadata["license"]
    self.license = metadata["license"].split(/[\r\n]+/).first
  end
  self.version = metadata["version"]
  self.url = metadata["url"]

  # name prefixing is optional, if enabled, a name 'foo' will become
  # 'python-foo' (depending on what the python_package_name_prefix is)
  if attributes[:python_fix_name?]
    self.name = fix_name(metadata["name"])
  else
    self.name = metadata["name"]
  end

  # convert python-Foo to python-foo if flag is set
  self.name = self.name.downcase if attributes[:python_downcase_name?]

  if !attributes[:no_auto_depends?] and attributes[:python_dependencies?]
    metadata["dependencies"].each do |dep|
      dep_re = /^([^<>!= ]+)\s*(?:([~<>!=]{1,2})\s*(.*))?$/
      match = dep_re.match(dep)
      if match.nil?
        logger.error("Unable to parse dependency", :dependency => dep)
        raise FPM::InvalidPackageConfiguration, "Invalid dependency '#{dep}'"
      end
      name, cmp, version = match.captures

      next if attributes[:python_disable_dependency].include?(name)

      # convert == to =
      if cmp == "==" or cmp == "~="
        logger.info("Converting == dependency requirement to =", :dependency => dep )
        cmp = "="
      end

      # dependency name prefixing is optional, if enabled, a name 'foo' will
      # become 'python-foo' (depending on what the python_package_name_prefix
      # is)
      name = fix_name(name) if attributes[:python_fix_dependencies?]

      # convert dependencies from python-Foo to python-foo
      name = name.downcase if attributes[:python_downcase_dependencies?]

      self.dependencies << "#{name} #{cmp} #{version}"
    end
  end # if attributes[:python_dependencies?]
end