class Liquid::C::Tokenizer
Public Class Methods
new(p1, p2)
click to toggle source
static VALUE tokenizer_initialize_method(VALUE self, VALUE source, VALUE line_numbers) { tokenizer_t *tokenizer; Check_Type(source, T_STRING); Tokenizer_Get_Struct(self, tokenizer); source = rb_str_dup_frozen(source); tokenizer->source = source; tokenizer->cursor = RSTRING_PTR(source); tokenizer->length = RSTRING_LEN(source); tokenizer->lstrip_flag = 0; // tokenizer->line_number keeps track of the current line number or it is 0 // to indicate that line numbers aren't being calculated tokenizer->line_number = RTEST(line_numbers) ? 1 : 0; return Qnil; }
Public Instance Methods
line_number()
click to toggle source
static VALUE tokenizer_line_number_method(VALUE self) { tokenizer_t *tokenizer; Tokenizer_Get_Struct(self, tokenizer); if (tokenizer->line_number == 0) return Qnil; return UINT2NUM(tokenizer->line_number); }
shift()
click to toggle source
static VALUE tokenizer_shift_method(VALUE self) { tokenizer_t *tokenizer; Tokenizer_Get_Struct(self, tokenizer); token_t token; tokenizer_next(tokenizer, &token); if (!token.type) return Qnil; return rb_enc_str_new(token.str, token.length, utf8_encoding); }