class Pygments::Popen

Attributes

python_binary[W]

Public Instance Methods

alive?() click to toggle source

Check for a @pid variable, and then hit ‘kill -0` with the pid to check if the pid is still in the process table. If this function gives us an ENOENT or ESRCH, we can also safely return false (no process to worry about). Defensively, if EPERM is raised, in a odd/rare dying process situation (e.g., mentos is checking on the pid of a dead process and the pid has already been re-used) we’ll want to raise that as a more informative Mentos exception.

@return [Boolean] true if the child is alive.

# File lib/pygments/popen.rb, line 79
def alive?
  return true if defined?(@pid) && @pid && Process.kill(0, @pid)

  false
rescue Errno::ENOENT, Errno::ESRCH
  false
rescue Errno::EPERM
  raise MentosError, 'EPERM checking if child process is alive.'
end
css(klass = '', opts = {}) click to toggle source

@return [String] css for highlighted code

# File lib/pygments/popen.rb, line 138
def css(klass = '', opts = {})
  if klass.is_a?(Hash)
    opts = klass
    klass = ''
  end
  mentos(:css, ['html', klass], opts)
end
filters() click to toggle source

@return [Array<String>] an array of all available filters

# File lib/pygments/popen.rb, line 128
def filters
  mentos(:get_all_filters)
end
formatters() click to toggle source

Public: Get an array of available Pygments formatters

@return [Array<String>] an array of formatters

# File lib/pygments/popen.rb, line 99
def formatters
  mentos(:get_all_formatters).each_with_object({}) do |(name, desc, aliases), hash|
    # Remove the long-winded and repetitive 'Formatter' suffix
    name.sub!(/Formatter$/, '')
    hash[name] = {
      name: name,
      description: desc,
      aliases: aliases
    }
  end
end
highlight(code, opts = {}) click to toggle source

Public: Highlight code.

Takes a first-position argument of the code to be highlighted, and a second-position hash of various arguments specifying highlighting properties.

Returns the highlighted string or nil when the request to the Python process timed out.

# File lib/pygments/popen.rb, line 168
def highlight(code, opts = {})
  # If the caller didn't give us any code, we have nothing to do,
  # so return right away.
  return code if code.nil? || code.empty?

  # Callers pass along options in the hash
  opts[:options] ||= {}

  # Default to utf-8 for the output encoding, if not given.
  opts[:options][:outencoding] ||= 'utf-8'

  # Get back the string from mentos and force encoding if we can
  str = mentos(:highlight, nil, opts, code)
  str.force_encoding(opts[:options][:outencoding]) if str.respond_to?(:force_encoding)
  str
end
lexer_names_for(*args) click to toggle source

@return [[String], nil] aliases of a lexer.

# File lib/pygments/popen.rb, line 147
def lexer_names_for(*args)
  # Pop off the last arg if it's a hash, which becomes our opts
  opts = if args.last.is_a?(Hash)
           args.pop
         else
           {}
         end

  code = (args.pop if args.last.is_a?(String))

  mentos(:lexer_names_for, args, opts, code)
end
lexers!() click to toggle source

Get all available lexers from mentos itself Do not use this method directly, instead use Pygments#lexers

@return [Array<String>] an array of lexers

# File lib/pygments/popen.rb, line 115
def lexers!
  mentos(:get_all_lexers).each_with_object({}) do |lxr, hash|
    name = lxr[0]
    hash[name] = {
      name: name,
      aliases: lxr[1],
      filenames: lxr[2],
      mimetypes: lxr[3]
    }
  end
end
pygments_version() click to toggle source

Public: Returns version of underlying Pygments library

@return [Integer]

# File lib/pygments/popen.rb, line 92
def pygments_version
  mentos(:version)[0]
end
python_binary() click to toggle source
# File lib/pygments/popen.rb, line 38
def python_binary
  @python_binary ||= find_python_binary
end
start(pygments_path = File.join(__dir__, '..', '..', 'vendor', 'pygments-main')) click to toggle source

Get things started by opening a pipe to mentos (the freshmaker), a Python process that talks to the Pygments library. We’ll talk back and forth across this pipe.

# File lib/pygments/popen.rb, line 19
def start(pygments_path = File.join(__dir__, '..', '..', 'vendor', 'pygments-main'))
  @log = Logger.new(ENV.fetch('MENTOS_LOG', File::NULL))
  @log.level = Logger::INFO
  @log.datetime_format = '%Y-%m-%d %H:%M '

  ENV['PYGMENTS_PATH'] = pygments_path

  # Make sure we kill off the child when we're done
  at_exit { stop 'Exiting' }

  # A pipe to the mentos python process. #popen4 gives us
  # the pid and three IO objects to write and read.
  argv = [*python_binary, File.join(__dir__, 'mentos.py')]
  @pid, @in, @out, @err = popen4(argv)
  @in.binmode
  @out.binmode
  @log.info "Starting pid #{@pid} with python #{python_binary}."
end
stop(reason) click to toggle source

Stop the child process by issuing a kill -9.

We then call waitpid() with the pid, which waits for that particular child and reaps it.

kill() can set errno to ESRCH if, for some reason, the file is gone; regardless the final outcome of this method will be to set our @pid variable to nil.

Technically, kill() can also fail with EPERM or EINVAL (wherein the signal isn’t sent); but we have permissions, and we’re not doing anything invalid here. @param reason [String]

# File lib/pygments/popen.rb, line 57
def stop(reason)
  unless @pid.nil?
    @log.info "Killing pid: #{@pid}. Reason: #{reason}"
    begin
      Process.kill('KILL', @pid)
      Process.waitpid(@pid)
    rescue Errno::ESRCH, Errno::ECHILD => e
      @log.warn(e)
    end
  end
  @pid = nil
end
styles() click to toggle source

@return [Array<String>] an array of all available styles

# File lib/pygments/popen.rb, line 133
def styles
  mentos(:get_all_styles)
end

Private Instance Methods

find_python_binary() click to toggle source

Detect a suitable Python binary to use.

# File lib/pygments/popen.rb, line 197
def find_python_binary
  if Gem.win_platform?
    return 'python3' if which('python3')
    return 'python' if which('python')
    return %w[py -3] if which('py')
  end

  # On non-Windows platforms, we simply rely on shebang
  []
end
get_timeout(timeout) click to toggle source

@param timeout [Integer, nil] @return [Integer]

# File lib/pygments/popen.rb, line 323
def get_timeout(timeout)
  return timeout unless timeout.nil?

  Integer(ENV.fetch('MENTOS_TIMEOUT', 0))
end
handle_header_and_return(header) click to toggle source

Based on the header we receive, determine if we need to read more bytes, and read those bytes if necessary.

@param header [String, nil] @return [String, nil] either highlighted text or metadata.

# File lib/pygments/popen.rb, line 334
def handle_header_and_return(header)
  raise MentosError, 'No header received back.' if header.nil?

  @log.info "In header: #{header}"
  header = header_to_json(header)
  bytes = header[:bytes]

  # Read more bytes (the actual response body)
  res = @out.read(bytes.to_i)

  if header[:method] == 'highlight' && res.nil?
    # Make sure we have a result back; else consider this an error.
    raise MentosError, 'No highlight result back from mentos.'
  end

  res
end
header_to_json(header) click to toggle source

Convert a text header into JSON for easy access. @param header [String] @return [JSON]

# File lib/pygments/popen.rb, line 362
def header_to_json(header)
  json = JSON.parse(header, symbolize_names: true)
  raise MentosError, json[:error] unless json[:error].nil?

  json
end
mentos(method, args = [], kwargs = {}, code = nil) click to toggle source

Our ‘rpc’-ish request to mentos. Requires a method name, and then optional args, kwargs, code.

# File lib/pygments/popen.rb, line 260
def mentos(method, args = [], kwargs = {}, code = nil)
  # Open the pipe if necessary
  start unless alive?

  # Add metadata to the header and generate it.
  kwargs = kwargs.merge('bytes' => (code.nil? ? 0 : code.bytesize))
  out_header = JSON.generate(method: method, args: args, kwargs: kwargs)

  begin
    timeout = get_timeout(kwargs.delete(:timeout))
    res = with_watchdog(timeout, "Timeout on a mentos #{method} call") do
      write_header(out_header, code)

      # mentos will now return data to us. First it sends the header.
      header_len_bytes = @out.read(4)
      if header_len_bytes.nil?
        raise Errno::EPIPE, %(Failed to read response from Python process on a mentos #{method} call)
      end

      header_len = header_len_bytes.unpack1('N')
      @log.info "Size in: #{header_len}"
      header = @out.read(header_len)

      # Now handle the header, any read any more data required.
      handle_header_and_return(header)
    end

    # Finally, return what we got.
    return_result(res, method)
  rescue Errno::EPIPE => e
    begin
      error_msg = @err.read
      @log.error "Error running Python script: #{error_msg}"
      stop "Error running Python script: #{error_msg}"
      raise MentosError, %(#{e}: #{error_msg})
    rescue Errno::EPIPE
      @log.error e.to_s
      stop e.to_s
      raise e
    end
  rescue StandardError => e
    @log.error e.to_s
    stop e.to_s
    raise e
  end
end
popen4(argv) click to toggle source

@param argv [Array<String>]

# File lib/pygments/popen.rb, line 188
def popen4(argv)
  stdin, stdout, stderr, wait_thr = Open3.popen3(*argv, { close_others: true })
  while (pid = wait_thr.pid).nil? && wait_thr.alive?
    # wait_thr.pid is not immediately available on JRuby. Why???
  end
  [pid, stdin, stdout, stderr]
end
return_result(res, method) click to toggle source

@return Ruby objects for the methods that want them, text otherwise.

# File lib/pygments/popen.rb, line 353
def return_result(res, method)
  res = JSON.parse(res, symbolize_names: true) unless %i[highlight css].include?(method)
  res = res.rstrip if res.instance_of?(String)
  res
end
which(command) click to toggle source

Cross platform which command from stackoverflow.com/a/5471032/284795

# File lib/pygments/popen.rb, line 210
def which(command)
  exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
  ENV['PATH'].split(File::PATH_SEPARATOR).each do |dir|
    exts.each do |ext|
      path = File.join(dir, "#{command}#{ext}")
      return path if File.executable?(path) && !File.directory?(path)
    end
  end
  nil
end
with_watchdog(timeout, error_message) { || ... } click to toggle source

@param timeout [Integer] @param error_message [String] @yield

# File lib/pygments/popen.rb, line 224
def with_watchdog(timeout, error_message)
  state_mutex = Mutex.new
  state = :alive
  wd_cleanup = ConditionVariable.new

  watchdog = if timeout.positive?
               Thread.new do
                 state_mutex.synchronize do
                   wd_cleanup.wait(state_mutex, timeout) if state != :finished
                   if state != :finished
                     @log.error error_message
                     stop error_message
                     state = :timeout
                   end
                 end
               end
             end

  begin
    yield
  ensure
    if watchdog
      state_mutex.synchronize do
        state = :finished if state == :alive
        # wake up watchdog thread
        wd_cleanup.signal
      end
      watchdog.join
    end

    raise MentosError, error_message if state == :timeout
  end
end
write_header(header, code) click to toggle source

@param header [String] @param code [String, nil]

# File lib/pygments/popen.rb, line 309
def write_header(header, code)
  # Get the size of the header itself and write that.
  @in.write([header.bytesize].pack('N'))
  @log.info "Size out: #{header.bytesize}"

  # mentos is now waiting for the header, and, potentially, code.
  @in.write(header)
  @log.info "Out header: #{header}"
  @in.write(code) unless code.nil?
  @in.flush
end