use std::{iter::Peekable, str::CharIndices};

use crate::token::Token;

pub trait Tokenizer: Send + Sync {
    fn run(
        &self,
        c: (usize, char),
        chars: &mut Peekable<CharIndices<'_>>,
        source: &str,
        line: usize,
    ) -> Option<(usize, Option<Token>)>;
}