(*********************************************************************************)
(*                Higlo                                                          *)
(*                                                                               *)
(*    Copyright (C) 2014-2021 Institut National de Recherche en Informatique     *)
(*    et en Automatique. All rights reserved.                                    *)
(*                                                                               *)
(*    This program is free software; you can redistribute it and/or modify       *)
(*    it under the terms of the GNU Lesser General Public License version        *)
(*    3 as published by the Free Software Foundation.                            *)
(*                                                                               *)
(*    This program is distributed in the hope that it will be useful,            *)
(*    but WITHOUT ANY WARRANTY; without even the implied warranty of             *)
(*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *)
(*    GNU Library General Public License for more details.                       *)
(*                                                                               *)
(*    You should have received a copy of the GNU Lesser General Public           *)
(*    License along with this program; if not, write to the Free Software        *)
(*    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA                   *)
(*    02111-1307  USA                                                            *)
(*                                                                               *)
(*    Contact: Maxence.Guesdon@inria.fr                                          *)
(*                                                                               *)
(*                                                                               *)
(*********************************************************************************)

(** Syntax highligthing *)

(** Utf8 text and its length or a negative number it the length was not computed. *)
type token_text = string * int

(** Tokens read in the given code, with the corresponding text and length of
 the text (in number of codepoints). These names are inspired from
 the [highlight] tool. [Keyword] and [Symbol] are parametrized by
 an integer to be able to distinguish different families of keywords
 and symbols, as [kwa], [kwb], ..., in [highlight].
*)
type token =
| Bcomment of token_text (** block comment *)
| Constant of token_text
| Directive of token_text
| Escape of token_text (** Escape sequence like [\123] *)
| Id of token_text
| Keyword of int * token_text
| Lcomment of token_text (** one line comment *)
| Numeric of token_text
| String of token_text
| Symbol of int * token_text
| Text of token_text (** Used for everything else *)
| Title of int * token_text

(** For debug printing. *)
val string_of_token : token -> string
val string_of_token_with_length : token -> string

type error =
| Unknown_lang of string (** when the required language is not found. *)
| Lex_error of Location.t * string

exception Error of error

val string_of_error : error -> string
val pp : Format.formatter -> error -> unit

(** Lexers are based on Sedlex. A lexer returns a list of tokens,
  in the same order they appear in the read string.
  [Text] tokens are merged by the {!parse} function.
*)
type lexer = Sedlexing.lexbuf -> token list

(** [get_lexer lang] returns the lexer registered for the given language
  [lang] or raises {!Unknown_lang} if no such language was registered. *)
val get_lexer : string -> lexer

(** [registered_langs] returns the list of registered pairs (name, lexer). *)
val registered_langs : unit -> (string * lexer) list

(** If a lexer was registered for the same language, it is not
  available any more. *)
val register_lang : string -> lexer -> unit

(** [parse ?raise_exn ~lang code] gets the lexer associated to [lang]
  and uses it to build a list of tokens. Consecutive [Text]
  tokens are merged.
  If no lexer is associated to the given language, then
  the function returns [[Text code]].
  @param raise defaults to [false]. If [true], raise exceptions
  rather than returning [[Text code]].
*)
val parse : ?raise_exn:bool -> lang:string -> string -> token list

val parse_lexbuf : ?on_exn:string -> lang:string -> Sedlexing.lexbuf -> token list

