1use std::{
10 fmt::{Display, Write},
11 ops::Range,
12 str::CharIndices,
13};
14
15use base::{
16 Unsigned36Bit,
17 charset::{Script, subscript_char, superscript_char},
18 error::StringConversionFailed,
19};
20
21use super::{
22 glyph::{
23 Elevated, Glyph, GlyphShape, Unrecognised, elevate, glyph_from_name, glyph_of_char,
24 is_allowed_in_symex,
25 },
26 parser::helpers,
27 state::NumeralMode,
28};
29
30#[cfg(test)]
31mod input_file_tests;
32mod lower;
33#[cfg(test)]
34mod tests;
35
36type Span = Range<usize>;
37
38pub(crate) const DOT_CHAR: char = '·';
39pub(crate) const DOT_STR: &str = "·";
40
41#[derive(Debug, PartialEq, Eq, Clone)]
42pub(crate) struct NumericLiteral {
43 digits: String,
45
46 has_trailing_dot: bool,
57}
58
59impl Display for NumericLiteral {
60 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61 f.write_str(self.digits.as_str())?;
62 if self.has_trailing_dot {
63 f.write_char(DOT_CHAR)?;
64 }
65 Ok(())
66 }
67}
68
69impl NumericLiteral {
70 pub(crate) fn make_num(
71 &self,
72 mode: NumeralMode,
73 ) -> Result<Unsigned36Bit, StringConversionFailed> {
74 helpers::make_num(self.digits.as_str(), self.has_trailing_dot, mode)
75 }
76
77 pub(crate) fn append_digits_of_literal(&mut self, other: &NumericLiteral) {
78 assert!(!other.has_trailing_dot);
79 self.digits.push_str(&other.digits);
80 }
81
82 pub(crate) fn has_trailing_dot(&self) -> bool {
83 self.has_trailing_dot
84 }
85
86 pub(crate) fn take_digits(self) -> String {
87 self.digits
88 }
89}
90
91#[derive(Debug, PartialEq, Eq, Clone)]
94pub(crate) enum ErrorTokenKind {
95 Tab,
99 UnrecognisedGlyph(Unrecognised),
100}
101
102impl Display for ErrorTokenKind {
103 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104 match self {
105 ErrorTokenKind::Tab => {
106 const LONG_MSG: &str = concat!(
107 "Please do not use the TAB character. ",
108 "The differences between the M4 assembler's interpretation of tab and its interpreation of the space ",
109 "characer are complex, and these are not fully implemented. If you want to ",
110 "prevent two adjacent symexes being joined together, please use parentheses ",
111 "or an explicit '+' operation instead. That is, use (A)(B) or A+B instead of A<tab>B. ",
112 "If you intended to simply use TAB to produce some particular code layout, please ",
113 "use spaces instead.",
114 );
115 f.write_str(LONG_MSG)
116 }
117 ErrorTokenKind::UnrecognisedGlyph(e) => write!(f, "{e}"),
118 }
119 }
120}
121
122#[derive(Debug, PartialEq, Eq, Clone)]
124pub(crate) enum Token {
125 Error(ErrorTokenKind),
128 LeftBrace(Script),
129 RightBrace(Script),
130 Newline,
131 Tab,
132
133 LeftParen(Script),
136
137 RightParen(Script),
140
141 Hold,
149 NotHold, Arrow(Script),
151 Hand(Script),
152 Hash(Script),
153 Equals(Script),
154
155 Asterisk(Script),
163
164 Pipe(Script),
165 DoublePipe(Script),
166 ProperSuperset(Script),
167 SubsetOf(Script),
168 IdenticalTo(Script),
169 Tilde(Script),
170 LessThan(Script),
171 GreaterThan(Script),
172 Query(Script), Intersection(Script),
174 Union(Script),
175
176 Solidus(Script),
179
180 Plus(Script),
182 Minus(Script),
183 Times(Script),
184 LogicalOr(Script),
185 LogicalAnd(Script),
186 LogicalXor(Script),
187
188 Digits(Script, NumericLiteral),
190
191 BitPosition(Script, String, String),
193
194 SymexSyllable(Script, String),
204
205 Dot(Script),
217 Comma(Script),
218}
219
220impl Display for Token {
221 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
222 let mut write_elevated = |script: &Script, s: &str| -> std::fmt::Result {
223 let el = elevate(*script, s);
224 write!(f, "{el}")
225 };
226
227 match self {
228 Token::Error(msg) => write!(f, "(error: {msg})"),
229 Token::LeftBrace(script) => write_elevated(script, "{"),
230 Token::RightBrace(script) => write_elevated(script, "}"),
231 Token::Newline => f.write_char('\n'),
232 Token::Tab => f.write_char('\t'),
233 Token::LeftParen(script) => write_elevated(script, "("),
234 Token::RightParen(script) => write_elevated(script, ")"),
235 Token::Hold => f.write_char('h'),
236 Token::NotHold => f.write_char('ℏ'),
237 Token::Arrow(script) => write_elevated(script, "->"),
238 Token::Hand(script) => write_elevated(script, "☛"),
239 Token::Asterisk(script) => write_elevated(script, "*"),
240 Token::Dot(script) => write_elevated(script, DOT_STR),
241 Token::Hash(script) => write_elevated(script, "#"),
242 Token::Equals(script) => write_elevated(script, "="),
243 Token::Pipe(script) => write_elevated(script, "|"),
244 Token::DoublePipe(script) => write_elevated(script, "‖"), Token::ProperSuperset(script) => write_elevated(script, "⊃"), Token::SubsetOf(script) => write_elevated(script, "⊂"), Token::IdenticalTo(script) => write_elevated(script, "≡"),
248 Token::Tilde(script) => write_elevated(script, "~"),
249 Token::LessThan(script) => write_elevated(script, "<"),
250 Token::GreaterThan(script) => write_elevated(script, ">"),
251 Token::Query(script) => write_elevated(script, "?"),
252 Token::Intersection(script) => write_elevated(script, "∩"),
253 Token::Union(script) => write_elevated(script, "∪"),
254 Token::Solidus(script) => write_elevated(script, "/"),
255 Token::Plus(script) => write_elevated(script, "+"),
256 Token::Minus(script) => write_elevated(script, "-"),
257 Token::Times(script) => write_elevated(script, "×"),
258 Token::LogicalOr(script) => write_elevated(script, "∨"),
259 Token::LogicalAnd(script) => write_elevated(script, "∧"),
260 Token::LogicalXor(script) => write_elevated(script, "⊻"),
261 Token::Digits(script, numeric_literal) => {
262 write!(f, "{}", elevate(*script, numeric_literal.to_string()))
263 }
264 Token::BitPosition(script, quarter, bit) => {
265 let q_string = elevate(*script, quarter.clone());
266 let bit_string = elevate(*script, bit.clone());
267 let dotname = match script {
268 Script::Normal => "@dot@",
269 Script::Sub => "@sub_dot@",
270 Script::Super => "@sup_dot@",
271 };
272 write!(f, "{q_string}{dotname}{bit_string}")
273 }
274 Token::SymexSyllable(script, name) => {
275 #[allow(clippy::unnecessary_wraps)]
276 fn nochange(ch: char) -> Result<char, ()> {
277 Ok(ch)
278 }
279 fn convert_to_sup(ch: char) -> Result<char, ()> {
280 superscript_char(ch).map_err(|_| ())
281 }
282 fn convert_to_sub(ch: char) -> Result<char, ()> {
283 subscript_char(ch).map_err(|_| ())
284 }
285 type Transformer = fn(char) -> Result<char, ()>;
286 let (prefix, transform): (&'static str, Transformer) = match script {
287 Script::Super => ("super_", convert_to_sup),
288 Script::Normal => ("", nochange),
289 Script::Sub => ("sub_", convert_to_sub),
290 };
291 for ch in name.chars() {
292 match transform(ch) {
293 Ok(sup_ch) => f.write_char(sup_ch),
294 Err(()) => match ch {
295 'α' => write!(f, "@{prefix}alpha@"),
296 'β' => write!(f, "@{prefix}beta@"),
297 'γ' => write!(f, "@{prefix}gamma@"),
298 'Δ' => write!(f, "@{prefix}delta@"),
299 'ε' => write!(f, "@{prefix}eps@"),
300 'λ' => write!(f, "@{prefix}lambda@"),
301 _ => write!(f, "@{prefix}{ch}@"),
302 },
303 }?;
304 }
305 Ok(())
306 }
307 Token::Comma(script) => write_elevated(script, ","),
308 }
309 }
310}
311
312#[derive(Debug, Clone)]
314struct GlyphRecognizer<'a> {
315 it: CharIndices<'a>,
316 pos: usize,
317 glyph_start: usize,
318}
319
320impl<'a> GlyphRecognizer<'a> {
321 fn new(input: &'a str) -> GlyphRecognizer<'a> {
322 Self {
323 it: input.char_indices(),
324 pos: 0,
325 glyph_start: 0,
326 }
327 }
328
329 fn get_next_char(&mut self) -> Option<char> {
330 match self.it.next() {
331 None => None,
332 Some((i, ch)) => {
333 self.pos = i;
334 Some(ch)
335 }
336 }
337 }
338
339 fn span(&self) -> Span {
340 self.glyph_start..(self.it.offset())
341 }
342
343 fn next_named_glyph(&mut self) -> Option<Result<Elevated<&'static Glyph>, Unrecognised>> {
344 let mut name: String = String::with_capacity(10);
345 let mut got_anything = false;
346 while let Some(ch) = self.get_next_char() {
347 got_anything = true;
348 if ch == '@' {
349 break;
350 }
351 name.push(ch);
352 }
353
354 if got_anything {
358 Some(match glyph_from_name(name.as_str()) {
359 Some(g) => Ok(g),
360 None => Err(Unrecognised::UnrecognisedGlyph(name)),
361 })
362 } else {
363 None
364 }
365 }
366}
367
368impl Iterator for GlyphRecognizer<'_> {
369 type Item = Result<Elevated<&'static Glyph>, Unrecognised>;
370
371 fn next(&mut self) -> Option<Self::Item> {
372 let ch = self.get_next_char()?;
373 self.glyph_start = self.pos;
374 match ch {
375 '@' => match self.next_named_glyph() {
376 None => {
377 Some(Err(Unrecognised::InvalidChar('@')))
381 }
382 something => something,
383 },
384 ch => Some(glyph_of_char(ch)),
385 }
386 }
387}
388
389#[test]
390fn test_glyph_recognizer_next() {
391 let mut gr = GlyphRecognizer::new("W");
392 match gr.next() {
393 Some(Ok(elev)) => {
394 assert_eq!(elev.script(), Script::Normal);
395 assert_eq!(elev.get().name, "W");
396 }
397 bad => {
398 panic!("glyph should not have been recognised as {bad:?}");
399 }
400 }
401 assert_eq!(gr.next(), None);
402}
403
404#[cfg(test)]
405fn assert_glyph(
406 got: Elevated<&'static Glyph>,
407 expected_shape: GlyphShape,
408 expected_script: Script,
409) {
410 assert_eq!(got.script(), expected_script, "wrong script for {got:?}");
411 assert_eq!(got.get().shape(), expected_shape, "wrong shape for {got:?}");
412}
413
414#[test]
415fn test_glyph_scanning() {
416 let mut scanner = GlyphRecognizer::new("hs@sub_eps@@hamb@@sup_add@@nosuch@ ");
417 assert_glyph(
419 scanner.next().expect("input").expect("in character set"),
420 GlyphShape::h,
421 Script::Normal,
422 );
423 assert_eq!(scanner.next(), Some(Err(Unrecognised::InvalidChar('s'))),);
425 assert_glyph(
426 scanner.next().expect("input").expect("in character set"),
427 GlyphShape::Epsilon,
428 Script::Sub,
429 );
430 assert_glyph(
431 scanner.next().expect("input").expect("in character set"),
432 GlyphShape::IdenticalTo,
433 Script::Normal,
434 );
435 assert_glyph(
436 scanner.next().expect("input").expect("in character set"),
437 GlyphShape::Add,
438 Script::Super,
439 );
440 assert_eq!(
441 scanner.next(),
442 Some(Err(Unrecognised::UnrecognisedGlyph("nosuch".to_string())))
443 );
444 assert_glyph(
445 scanner.next().expect("input").expect("in character set"),
446 GlyphShape::Space,
447 Script::Normal,
448 );
449 assert_eq!(scanner.next(), None);
450 assert_eq!(scanner.next(), None);
452}
453
454fn tokenise_single_glyph(g: Elevated<&'static Glyph>) -> Option<Token> {
455 let script: Script = g.script();
456
457 let make_num = |ch: char| {
458 let literal = NumericLiteral {
459 digits: {
460 let mut s = String::with_capacity(12);
461 s.push(ch);
462 s
463 },
464 has_trailing_dot: false,
465 };
466 Token::Digits(script, literal)
467 };
468 let make_symex = || -> Option<Token> {
469 let name: String = g.get().get_char(Script::Normal).iter().collect();
474 match name.chars().count() {
478 0 => {
479 panic!(
480 "incoming token '{g:?}' was assigned as part of a symex syllable, but we don't have a character for it in script {script:?}"
481 );
482 }
483 1 => (),
484 n => {
485 panic!(
486 "incoming token '{g:?}' was assigned as part of a symex syllable, but the resuting initial token body unexpectedly has more than one character (specifically, {n}): {name:?}"
487 );
488 }
489 }
490 Some(Token::SymexSyllable(script, name))
491 };
492
493 #[allow(clippy::match_same_arms)] let output: Option<Token> = match g.get().shape() {
498 GlyphShape::Space | GlyphShape::Tab => None,
499 GlyphShape::Digit0 => Some(make_num('0')),
500 GlyphShape::Digit1 => Some(make_num('1')),
501 GlyphShape::Digit2 => Some(make_num('2')),
502 GlyphShape::Digit3 => Some(make_num('3')),
503 GlyphShape::Digit4 => Some(make_num('4')),
504 GlyphShape::Digit5 => Some(make_num('5')),
505 GlyphShape::Digit6 => Some(make_num('6')),
506 GlyphShape::Digit7 => Some(make_num('7')),
507 GlyphShape::Digit8 => Some(make_num('8')),
508 GlyphShape::Digit9 => Some(make_num('9')),
509 GlyphShape::Underscore
510 | GlyphShape::Circle
511 | GlyphShape::A
512 | GlyphShape::B
513 | GlyphShape::C
514 | GlyphShape::D
515 | GlyphShape::E
516 | GlyphShape::F
517 | GlyphShape::G
518 | GlyphShape::H
519 | GlyphShape::I
520 | GlyphShape::J
521 | GlyphShape::K
522 | GlyphShape::L
523 | GlyphShape::M
524 | GlyphShape::N
525 | GlyphShape::O
526 | GlyphShape::P
527 | GlyphShape::Q
528 | GlyphShape::R
529 | GlyphShape::S
530 | GlyphShape::T
531 | GlyphShape::U
532 | GlyphShape::V
533 | GlyphShape::W
534 | GlyphShape::X
535 | GlyphShape::Y
536 | GlyphShape::Z => make_symex(),
537 GlyphShape::LeftParen => Some(Token::LeftParen(script)),
538 GlyphShape::RightParen => Some(Token::RightParen(script)),
539 GlyphShape::Add => Some(Token::Plus(script)),
540 GlyphShape::Minus => Some(Token::Minus(script)),
541 GlyphShape::Comma => Some(Token::Comma(script)),
542 GlyphShape::Dot => Some(Token::Dot(script)),
543 GlyphShape::Backspace => unimplemented!("compound characters are not yet supported"),
544 GlyphShape::Hand => Some(Token::Hand(script)),
545 GlyphShape::Sigma => {
546 todo!("Sigma (which is a symex terminator) does not yet have a token")
547 }
548 GlyphShape::Pipe => Some(Token::Pipe(script)),
549 GlyphShape::DoublePipe => Some(Token::DoublePipe(script)),
550 GlyphShape::Solidus => Some(Token::Solidus(script)),
551 GlyphShape::Times => Some(Token::Times(script)),
552 GlyphShape::Hash => Some(Token::Hash(script)),
553 GlyphShape::Arrow => Some(Token::Arrow(script)),
554 GlyphShape::LessThan => Some(Token::LessThan(script)),
555 GlyphShape::GreaterThan => Some(Token::GreaterThan(script)),
556 GlyphShape::Overbar | GlyphShape::Square | GlyphShape::n => make_symex(),
557 GlyphShape::SubsetOf => Some(Token::SubsetOf(script)),
558 GlyphShape::Or => Some(Token::LogicalOr(script)),
559 GlyphShape::q
560 | GlyphShape::Gamma
561 | GlyphShape::t
562 | GlyphShape::w
563 | GlyphShape::x
564 | GlyphShape::i
565 | GlyphShape::y
566 | GlyphShape::z => make_symex(),
567 GlyphShape::Query => Some(Token::Query(script)),
568 GlyphShape::Union => Some(Token::Union(script)),
569 GlyphShape::Intersection => Some(Token::Intersection(script)),
570 GlyphShape::j | GlyphShape::k => make_symex(),
571 GlyphShape::Alpha => make_symex(),
572 GlyphShape::Delta => make_symex(),
573 GlyphShape::p => make_symex(),
574 GlyphShape::Epsilon => make_symex(),
575 GlyphShape::h => Some(match script {
576 Script::Super | Script::Sub => unimplemented!(),
579 Script::Normal => Token::Hold,
580 }),
581 GlyphShape::SupersetOf => Some(Token::ProperSuperset(script)),
583 GlyphShape::Beta => make_symex(),
584 GlyphShape::And => Some(Token::LogicalAnd(script)),
585 GlyphShape::Xor => Some(Token::LogicalXor(script)),
586 GlyphShape::Lambda => make_symex(),
587 GlyphShape::Tilde => Some(Token::Tilde(script)),
588 GlyphShape::LeftBrace => Some(Token::LeftBrace(script)),
589 GlyphShape::RightBrace => Some(Token::RightBrace(script)),
590 GlyphShape::IdenticalTo => Some(Token::IdenticalTo(script)),
591 GlyphShape::Equals => Some(Token::Equals(script)),
592 GlyphShape::Apostrophe => make_symex(),
593 GlyphShape::Asterisk => Some(Token::Asterisk(script)),
594 };
595 if let Some(t) = output.as_ref() {
596 if matches!(t, Token::SymexSyllable(_, _)) {
597 assert!(
598 is_allowed_in_symex(g.get().shape),
599 "attempted to make a symex with disallowed glyph shape {g:?}"
600 );
601 } else if matches!(t, Token::Digits(_, _) | Token::Dot(_)) {
602 assert!(
608 is_allowed_in_symex(g.get().shape),
609 "all glyphs allowed in numeric literals are also allowed in symexes, but this went wrong for {g:?}"
610 );
611 } else if g.get().shape == GlyphShape::Space {
612 } else {
614 assert!(
615 !is_allowed_in_symex(g.get().shape),
616 "glyph shape {g:?} is allowed in a symex but the scanner didn't recognise it that way"
617 );
618 }
619 }
620 output
621}
622
623#[derive(Debug, PartialEq, Eq)]
624enum TokenMergeResult {
625 Merged(Token, Span),
626 Failed {
627 current: Token,
628 current_span: Span,
629 incoming: Token,
630 incoming_span: Span,
631 },
632}
633
634fn merge_tokens(current: (Token, Span), incoming: (Token, Span)) -> TokenMergeResult {
635 let ((current, current_span), (incoming, incoming_span)) = (current, incoming);
638 if matches!(
639 (¤t, &incoming),
640 (&Token::Error(_), _) | (_, &Token::Error(_))
641 ) {
642 return TokenMergeResult::Failed {
643 current,
644 current_span,
645 incoming,
646 incoming_span,
647 };
648 }
649
650 let merged_span = current_span.start..incoming_span.end;
651 match current {
652 Token::Minus(incoming_script)
653 if incoming == Token::GreaterThan(incoming_script)
654 && incoming_script == Script::Normal =>
655 {
656 TokenMergeResult::Merged(Token::Arrow(Script::Normal), merged_span)
657 }
658 Token::SymexSyllable(existing_script, mut existing_name) => match incoming {
659 Token::Hold if existing_script == Script::Normal => {
660 if existing_name == "\u{0305}" {
662 TokenMergeResult::Merged(Token::NotHold, merged_span)
663 } else {
664 TokenMergeResult::Failed {
665 current: Token::SymexSyllable(existing_script, existing_name),
666 current_span,
667 incoming: Token::Hold,
668 incoming_span,
669 }
670 }
671 }
672 Token::SymexSyllable(incoming_script, incoming_name)
673 if existing_script == incoming_script =>
674 {
675 existing_name.push_str(&incoming_name);
676 TokenMergeResult::Merged(
677 Token::SymexSyllable(existing_script, existing_name),
678 merged_span,
679 )
680 }
681 Token::Digits(incoming_script, literal) if existing_script == incoming_script => {
682 existing_name.push_str(&literal.digits);
683 if literal.has_trailing_dot {
684 existing_name.push(DOT_CHAR);
685 }
686 TokenMergeResult::Merged(
687 Token::SymexSyllable(existing_script, existing_name),
688 merged_span,
689 )
690 }
691 other => TokenMergeResult::Failed {
692 current: Token::SymexSyllable(existing_script, existing_name),
693 current_span,
694 incoming: other,
695 incoming_span,
696 },
697 },
698 Token::Digits(existing_script, mut existing_literal) => {
699 if existing_literal.has_trailing_dot {
700 match incoming {
704 Token::Digits(
705 Script::Sub,
706 NumericLiteral {
707 digits: incoming_digit,
708 has_trailing_dot: false,
709 },
710 ) if existing_script == Script::Sub => TokenMergeResult::Merged(
711 Token::BitPosition(
712 existing_script,
713 existing_literal.digits,
714 incoming_digit,
715 ),
716 merged_span,
717 ),
718 other => TokenMergeResult::Failed {
720 current: Token::Digits(existing_script, existing_literal),
721 current_span,
722 incoming: other,
723 incoming_span,
724 },
725 }
726 } else {
727 match incoming {
728 Token::Digits(incoming_script, incoming_name)
729 if existing_script == incoming_script =>
730 {
731 existing_literal.append_digits_of_literal(&incoming_name);
732 TokenMergeResult::Merged(
733 Token::Digits(existing_script, existing_literal),
734 merged_span,
735 )
736 }
737 Token::Dot(right_script)
738 if existing_script == right_script
739 && !existing_literal.has_trailing_dot =>
740 {
741 existing_literal.has_trailing_dot = true;
742 TokenMergeResult::Merged(
743 Token::Digits(existing_script, existing_literal),
744 merged_span,
745 )
746 }
747 Token::SymexSyllable(incoming_script, sym)
748 if existing_script == incoming_script =>
749 {
750 let mut existing_name: String = existing_literal.digits;
751 existing_name.push_str(&sym);
752 TokenMergeResult::Merged(
753 Token::SymexSyllable(existing_script, existing_name),
754 merged_span,
755 )
756 }
757 other => TokenMergeResult::Failed {
758 current: Token::Digits(existing_script, existing_literal),
759 current_span,
760 incoming: other,
761 incoming_span,
762 },
763 }
764 }
765 }
766 Token::BitPosition(existing_script, existing_quarter, mut existing_bit) => match incoming {
767 Token::Digits(
768 incoming_script,
769 NumericLiteral {
770 digits: incoming_digit,
771 has_trailing_dot: false,
772 },
773 ) if existing_script == incoming_script => {
774 existing_bit.push_str(incoming_digit.as_str());
775 TokenMergeResult::Merged(
776 Token::BitPosition(existing_script, existing_quarter, existing_bit),
777 merged_span,
778 )
779 }
780 Token::Dot(incoming_script) if existing_script == incoming_script => {
781 let name = format!("{existing_quarter}\u{00B7}{existing_bit}\u{00B7}");
782 TokenMergeResult::Merged(Token::SymexSyllable(Script::Sub, name), merged_span)
783 }
784 Token::SymexSyllable(incoming_script, symbol) if existing_script == incoming_script => {
785 let name = format!("{existing_quarter}\u{00B7}{existing_bit}{symbol}");
786 TokenMergeResult::Merged(Token::SymexSyllable(Script::Sub, name), merged_span)
787 }
788 other => TokenMergeResult::Failed {
789 current: Token::BitPosition(existing_script, existing_quarter, existing_bit),
790 current_span,
791 incoming: other,
792 incoming_span,
793 },
794 },
795 existing => TokenMergeResult::Failed {
796 current: existing,
797 current_span,
798 incoming,
799 incoming_span,
800 },
801 }
802}
803
804#[derive(Debug, Clone)]
807struct GlyphTokenizer<'a> {
808 current: Option<(Token, Span)>,
809 inner: GlyphRecognizer<'a>,
810}
811
812impl<'a> GlyphTokenizer<'a> {
813 fn new(input: &'a str) -> GlyphTokenizer<'a> {
814 GlyphTokenizer {
815 current: None,
816 inner: GlyphRecognizer::new(input),
817 }
818 }
819
820 fn get_next_spanned_token(&mut self) -> Option<(Token, Span)> {
821 loop {
822 let maybe_spanned_new_token: Option<(Token, Span)> = match self.inner.next() {
823 None => None,
824 Some(Err(Unrecognised::InvalidChar('ℏ'))) => {
825 return Some((Token::NotHold, self.inner.span()));
836 }
837 Some(Err(e)) => {
838 let error_token = Token::Error(ErrorTokenKind::UnrecognisedGlyph(e));
839 Some((error_token, self.inner.span()))
840 }
841 Some(Ok(g)) => {
842 if matches!(g.get().shape(), GlyphShape::Space | GlyphShape::Tab) {
843 match self.current.take() {
844 Some(r) => {
845 return Some(r);
846 }
847 None => {
848 continue;
849 }
850 }
851 }
852 let tok_start = self.inner.span().start;
853 match tokenise_single_glyph(g) {
854 Some(token) => {
855 let span = tok_start..self.inner.span().end;
856 Some((token, span))
857 }
858 None => {
859 unimplemented!("unable) to convert glyph '{g:?}' to a token")
860 }
861 }
862 }
863 };
864 if let Some((newtoken, newtoken_span)) = maybe_spanned_new_token {
865 if let Some((current, current_span)) = self.current.take() {
866 match merge_tokens((current, current_span), (newtoken, newtoken_span)) {
867 TokenMergeResult::Merged(merged, merged_span) => {
868 self.current = Some((merged, merged_span));
869 }
870 TokenMergeResult::Failed {
871 current,
872 current_span,
873 incoming: newtoken,
874 incoming_span: newtoken_span,
875 } => {
876 let result = (current, current_span);
877 self.current = Some((newtoken, newtoken_span));
878 return Some(result);
879 }
880 }
881 } else {
882 self.current = Some((newtoken, newtoken_span));
884 }
885 } else {
886 return self.current.take();
888 }
889 }
890 }
891}
892
893#[test]
894fn test_glyph_tokenizer_simple_multi_token() {
895 let mut lex = GlyphTokenizer::new("hx");
896 assert_eq!(lex.get_next_spanned_token(), Some((Token::Hold, 0..1)));
897 assert_eq!(
898 lex.get_next_spanned_token(),
899 Some((Token::SymexSyllable(Script::Normal, "x".to_string()), 1..2))
900 );
901 assert_eq!(lex.get_next_spanned_token(), None);
902}
903
904#[test]
905fn test_glyph_tokenizer_glyph_names() {
906 let mut lex = GlyphTokenizer::new("@sup_eps@");
907 assert_eq!(
908 lex.get_next_spanned_token(),
909 Some((Token::SymexSyllable(Script::Super, "ε".to_string()), 0..9))
910 );
911 assert_eq!(lex.get_next_spanned_token(), None);
912}
913
914#[test]
915fn test_glyph_tokenizer_multi_glyph_token() {
916 let input = "@sup_eps@ᵂ";
919 let mut lex = GlyphTokenizer::new(input);
920 assert_eq!(
921 lex.get_next_spanned_token(),
922 Some((Token::SymexSyllable(Script::Super, "εW".to_string()), 0..12))
923 );
924 assert_eq!(lex.get_next_spanned_token(), None);
925}
926
927#[test]
928fn test_glyph_tokenizer_script_change_breaks_tokens() {
929 let mut lex = GlyphTokenizer::new("@sup_eps@W");
933 assert_eq!(
934 lex.get_next_spanned_token(),
935 Some((Token::SymexSyllable(Script::Super, "ε".to_string()), 0..9))
936 );
937 assert_eq!(
938 lex.get_next_spanned_token(),
939 Some((Token::SymexSyllable(Script::Normal, "W".to_string()), 9..10))
940 );
941 assert_eq!(lex.get_next_spanned_token(), None);
942}
943
944#[test]
945fn test_glyph_tokenizer_space_breaks_tokens() {
946 let mut lex = GlyphTokenizer::new("W Q");
950 assert_eq!(
951 lex.get_next_spanned_token(),
952 Some((Token::SymexSyllable(Script::Normal, "W".to_string()), 0..1))
953 );
954 assert_eq!(
955 lex.get_next_spanned_token(),
956 Some((Token::SymexSyllable(Script::Normal, "Q".to_string()), 2..3))
957 );
958 assert_eq!(lex.get_next_spanned_token(), None);
959}
960
961#[test]
962fn test_glyph_tokenizer_question_mark() {
963 let mut lex = GlyphTokenizer::new("?");
964 assert_eq!(
965 lex.get_next_spanned_token(),
966 Some((Token::Query(Script::Normal), 0..1))
967 );
968}
969
970#[derive(Debug, Clone)]
979pub(crate) struct Lexer<'a> {
980 lower: lower::LowerLexer<'a>,
981 upper: Option<GlyphTokenizer<'a>>,
982 upper_span: Option<Span>,
983}
984
985impl<'a> Lexer<'a> {
986 pub(crate) fn new(input: &'a str) -> Lexer<'a> {
987 Lexer {
988 lower: lower::LowerLexer::new(input),
989 upper: None,
990 upper_span: None,
991 }
992 }
993
994 fn adjust_span(&self, span: Range<usize>) -> Range<usize> {
995 match self.upper_span.as_ref() {
996 None => span,
997 Some(upper_span) => {
998 let offset = span.start;
999 (upper_span.start + offset)..(upper_span.end + offset)
1000 }
1001 }
1002 }
1003
1004 pub(crate) fn span(&self) -> Range<usize> {
1005 self.adjust_span(self.lower.span())
1006 }
1007
1008 pub(crate) fn spanned(&self) -> SpannedIter<'a> {
1009 let lexer: Lexer<'a> = self.clone();
1010 SpannedIter::new(lexer)
1011 }
1012
1013 fn get_next(&mut self) -> Option<Token> {
1014 use lower::Lexeme;
1015 if let Some(upper_lexer) = self.upper.as_mut()
1017 && let Some((r, span)) = upper_lexer.get_next_spanned_token()
1018 {
1019 self.upper_span = Some(span);
1020 return Some(r);
1021 }
1022
1023 self.upper = None;
1025 self.upper_span = None;
1026 match self.lower.next() {
1027 Lexeme::EndOfInput => None,
1028 Lexeme::Tok(tok) => Some(tok),
1029 Lexeme::Text(text) => {
1030 let upper = GlyphTokenizer::new(text);
1031 self.upper = Some(upper);
1032 match self
1033 .upper
1034 .as_mut()
1035 .expect("the option cannot be empty, we just filled it")
1036 .get_next_spanned_token()
1037 {
1038 Some((r, span)) => {
1039 self.upper_span = Some(span);
1040 Some(r)
1041 }
1042 None => None,
1043 }
1044 }
1045 Lexeme::Err(e) => Some(Token::Error(ErrorTokenKind::UnrecognisedGlyph(e))),
1046 }
1047 }
1048}
1049
1050impl Iterator for Lexer<'_> {
1051 type Item = Token;
1052
1053 fn next(&mut self) -> Option<Token> {
1054 self.get_next()
1055 }
1056}
1057
1058#[derive(Debug, Clone)]
1059pub(crate) struct SpannedIter<'a> {
1060 lexer: Lexer<'a>,
1061}
1062
1063impl<'a> SpannedIter<'a> {
1064 pub(crate) fn new(lexer: Lexer<'a>) -> SpannedIter<'a> {
1065 SpannedIter { lexer }
1066 }
1067}
1068
1069impl Iterator for SpannedIter<'_> {
1070 type Item = (Token, Span);
1071
1072 fn next(&mut self) -> Option<Self::Item> {
1073 let token = self.lexer.next();
1074 token.map(|t| (t, self.lexer.span()))
1075 }
1076}