% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/preprocessing.R
\name{text_tokenizer}
\alias{text_tokenizer}
\title{Text tokenization utility}
\usage{
text_tokenizer(num_words = NULL,
  filters = "!\\"#$\%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n", lower = TRUE,
  split = " ", char_level = FALSE)
}
\arguments{
\item{num_words}{the maximum number of words to keep, based on word
frequency. Only the most common \code{num_words} words will be kept.}

\item{filters}{a string where each element is a character that will be
filtered from the texts. The default is all punctuation, plus tabs and line
breaks, minus the ' character.}

\item{lower}{boolean. Whether to convert the texts to lowercase.}

\item{split}{character or string to use for token splitting.}

\item{char_level}{if \code{TRUE}, every character will be treated as a token}
}
\description{
Vectorize a text corpus, by turning each text into either a sequence of
integers (each integer being the index of a token in a dictionary) or into a
vector where the coefficient for each token could be binary, based on word
count, based on tf-idf...
}
\details{
By default, all punctuation is removed, turning the texts into
space-separated sequences of words (words maybe include the ' character).
These sequences are then split into lists of tokens. They will then be
indexed or vectorized. \code{0} is a reserved index that won't be assigned to any
word.
}
\section{Attributes}{

The tokenizer object has the following attributes:
\itemize{
\item \code{word_counts} --- named list mapping words to the number of times they appeared
on during fit. Only set after \code{fit_text_tokenizer()} is called on the tokenizer.
\item \code{word_docs} --- named list mapping words to the number of documents/texts they
appeared on during fit. Only set after \code{fit_text_tokenizer()} is called on the tokenizer.
\item \code{word_index} --- named list mapping words to their rank/index (int). Only set
after \code{fit_text_tokenizer()} is called on the tokenizer.
\item \code{document_count} --- int. Number of documents (texts/sequences) the tokenizer
was trained on. Only set after \code{fit_text_tokenizer()} is called on the tokenizer.
}
}

\seealso{
Other text tokenization: \code{\link{fit_text_tokenizer}},
  \code{\link{save_text_tokenizer}},
  \code{\link{sequences_to_matrix}},
  \code{\link{texts_to_matrix}},
  \code{\link{texts_to_sequences_generator}},
  \code{\link{texts_to_sequences}}
}
