14 lines
293 B
Rust
14 lines
293 B
Rust
|
use std::{iter::Peekable, str::CharIndices};
|
||
|
|
|
||
|
|
use crate::token::Token;
|
||
|
|
|
||
|
|
pub trait Tokenizer: Send + Sync {
|
||
|
|
fn run(
|
||
|
|
&self,
|
||
|
|
c: (usize, char),
|
||
|
|
chars: &mut Peekable<CharIndices<'_>>,
|
||
|
|
source: &str,
|
||
|
|
line: usize,
|
||
|
|
) -> Option<(usize, Option<Token>)>;
|
||
|
|
}
|