% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/youtokentome.R
\name{bpe_encode}
\alias{bpe_encode}
\title{Tokenise text alongside a Byte Pair Encoding model}
\usage{
bpe_encode(
  model,
  x,
  type = c("subwords", "ids"),
  bos = FALSE,
  eos = FALSE,
  reverse = FALSE
)
}
\arguments{
\item{model}{an object of class \code{youtokentome} as returned by \code{\link{bpe_load_model}}}

\item{x}{a character vector of text to tokenise}

\item{type}{a character string, either 'subwords' or 'ids' to get the subwords or the corresponding ids of these subwords as defined in the vocabulary of the model. Defaults to 'subwords'.}

\item{bos}{logical if set to TRUE then token 'beginning of sentence' will be added}

\item{eos}{logical if set to TRUE then token 'end of sentence' will be added}

\item{reverse}{logical if set to TRUE the output sequence of tokens will be reversed}
}
\description{
Tokenise text alongside a Byte Pair Encoding model
}
\examples{
data(belgium_parliament, package = "tokenizers.bpe")
x <- subset(belgium_parliament, language == "french")
model <- bpe(x$text, coverage = 0.999, vocab_size = 5000, threads = 1)
model
str(model$vocabulary)

text <- c("L'appartement est grand & vraiment bien situe en plein centre",
          "Proportion de femmes dans les situations de famille monoparentale.")
bpe_encode(model, x = text, type = "subwords")
bpe_encode(model, x = text, type = "ids")

encoded <- bpe_encode(model, x = text, type = "ids")
decoded <- bpe_decode(model, encoded)
decoded

## Remove the model file (Clean up for CRAN)
file.remove(model$model_path)
}
