class Liquid::C::Tokenizer

Public Class Methods

new(p1, p2) click to toggle source
static VALUE tokenizer_initialize_method(VALUE self, VALUE source, VALUE line_numbers)
{
    tokenizer_t *tokenizer;

    Check_Type(source, T_STRING);
    Tokenizer_Get_Struct(self, tokenizer);
    source = rb_str_dup_frozen(source);
    tokenizer->source = source;
    tokenizer->cursor = RSTRING_PTR(source);
    tokenizer->length = RSTRING_LEN(source);
    tokenizer->lstrip_flag = 0;
    // tokenizer->line_number keeps track of the current line number or it is 0
    // to indicate that line numbers aren't being calculated
    tokenizer->line_number = RTEST(line_numbers) ? 1 : 0;
    return Qnil;
}

Public Instance Methods

line_number() click to toggle source
static VALUE tokenizer_line_number_method(VALUE self)
{
    tokenizer_t *tokenizer;
    Tokenizer_Get_Struct(self, tokenizer);

    if (tokenizer->line_number == 0)
        return Qnil;

    return UINT2NUM(tokenizer->line_number);
}
shift() click to toggle source
static VALUE tokenizer_shift_method(VALUE self)
{
    tokenizer_t *tokenizer;
    Tokenizer_Get_Struct(self, tokenizer);

    token_t token;
    tokenizer_next(tokenizer, &token);
    if (!token.type)
        return Qnil;

    return rb_enc_str_new(token.str, token.length, utf8_encoding);
}