Skip to main content

assembler/
lexer.rs

1//! Turn text input into a sequence of tokens.
2//!
3//! We perform two levels of lexing:
4//!
5//! - [`lower`]: handles comments, annotations, spaces and newlines.
6//!   Generates a sequence of [`lower::Lexeme`] instances.
7//! - [`Lexer`]: pulls text from the lower-level lexer and recognizes tokens.
8//!   Generates a sequence of [`Token`] instances.
9use std::{
10    fmt::{Display, Write},
11    ops::Range,
12    str::CharIndices,
13};
14
15use base::{
16    Unsigned36Bit,
17    charset::{Script, subscript_char, superscript_char},
18    error::StringConversionFailed,
19};
20
21use super::{
22    glyph::{
23        Elevated, Glyph, GlyphShape, Unrecognised, elevate, glyph_from_name, glyph_of_char,
24        is_allowed_in_symex,
25    },
26    parser::helpers,
27    state::NumeralMode,
28};
29
30#[cfg(test)]
31mod input_file_tests;
32mod lower;
33#[cfg(test)]
34mod tests;
35
36type Span = Range<usize>;
37
38pub(crate) const DOT_CHAR: char = '·';
39pub(crate) const DOT_STR: &str = "·";
40
41#[derive(Debug, PartialEq, Eq, Clone)]
42pub(crate) struct NumericLiteral {
43    /// The digits comprising the literal.
44    digits: String,
45
46    /// Indicates that the literal has a trailing dot.
47    ///
48    /// A trailing dot indicates that the literal has a non-default
49    /// base (i.e. if the default is decimal, "." signals octal and if
50    /// the default is otal, "." signals decimal).
51    ///
52    /// We don't know whether the base is decimal or octal within the
53    /// lexer, because the state information that tracks the current
54    /// default base belongs to the parser (as the parser has to
55    /// interpret the meta commands which switch the default).
56    has_trailing_dot: bool,
57}
58
59impl Display for NumericLiteral {
60    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61        f.write_str(self.digits.as_str())?;
62        if self.has_trailing_dot {
63            f.write_char(DOT_CHAR)?;
64        }
65        Ok(())
66    }
67}
68
69impl NumericLiteral {
70    pub(crate) fn make_num(
71        &self,
72        mode: NumeralMode,
73    ) -> Result<Unsigned36Bit, StringConversionFailed> {
74        helpers::make_num(self.digits.as_str(), self.has_trailing_dot, mode)
75    }
76
77    pub(crate) fn append_digits_of_literal(&mut self, other: &NumericLiteral) {
78        assert!(!other.has_trailing_dot);
79        self.digits.push_str(&other.digits);
80    }
81
82    pub(crate) fn has_trailing_dot(&self) -> bool {
83        self.has_trailing_dot
84    }
85
86    pub(crate) fn take_digits(self) -> String {
87        self.digits
88    }
89}
90
91/// Represents an item in the input we didn't recognise of which we
92/// don't support.
93#[derive(Debug, PartialEq, Eq, Clone)]
94pub(crate) enum ErrorTokenKind {
95    /// Use of a tab character causes an error diagnostic; we don't
96    /// support tabs in the same way as M4 does, since the `☛☛T` meta
97    /// command is not implemented.
98    Tab,
99    UnrecognisedGlyph(Unrecognised),
100}
101
102impl Display for ErrorTokenKind {
103    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104        match self {
105            ErrorTokenKind::Tab => {
106                const LONG_MSG: &str = concat!(
107                    "Please do not use the TAB character. ",
108                    "The differences between the M4 assembler's interpretation of tab and its interpreation of the space ",
109                    "characer are complex, and these are not fully implemented.  If you want to ",
110                    "prevent two adjacent symexes being joined together, please use parentheses ",
111                    "or an explicit '+' operation instead.  That is, use (A)(B) or A+B instead of A<tab>B. ",
112                    "If you intended to simply use TAB to produce some particular code layout, please ",
113                    "use spaces instead.",
114                );
115                f.write_str(LONG_MSG)
116            }
117            ErrorTokenKind::UnrecognisedGlyph(e) => write!(f, "{e}"),
118        }
119    }
120}
121
122/// The parser consumes these tokens.
123#[derive(Debug, PartialEq, Eq, Clone)]
124pub(crate) enum Token {
125    // In order for the parser to recover from tokenization errors, we
126    // need to be able to emit an error token.
127    Error(ErrorTokenKind),
128    LeftBrace(Script),
129    RightBrace(Script),
130    Newline,
131    Tab,
132
133    /// The parser currently only handled parenthesised expressions in
134    /// normal script.
135    LeftParen(Script),
136
137    /// The parser currently only handled parenthesised expressions in
138    /// normal script.
139    RightParen(Script),
140
141    /// Accept either 'h' or ':' signalling the hold bit (of the
142    /// instruction word) should be set.  The documentation seems to
143    /// use both, though perhaps ':' is the older usage.
144    ///
145    /// While h is indeed a letter, it is not one of the letters which
146    /// can form part of a symex.  See the TX-2 Users Handbook,
147    /// section 6-3.2, "RULES FOR SYMEX FORMATION".
148    Hold,
149    NotHold, // handled specially, there is no glyph for this.
150    Arrow(Script),
151    Hand(Script),
152    Hash(Script),
153    Equals(Script),
154
155    /// Asterisk is used quite heavily (indicating deferred addressing)
156    /// but while the TX-2 supports superscript and subscript
157    /// asterisks, they don't seem to be used.  They are not valid as
158    /// part of a symex (see User handbook, section 6-2.3) and are not
159    /// macro terminators (6-4.5).  However, they are valid as part of
160    /// a superposed character sequence making up a compound-character
161    /// macro name.
162    Asterisk(Script),
163
164    Pipe(Script),
165    DoublePipe(Script),
166    ProperSuperset(Script),
167    SubsetOf(Script),
168    IdenticalTo(Script),
169    Tilde(Script),
170    LessThan(Script),
171    GreaterThan(Script),
172    Query(Script), // question mark, i.e. "?"
173    Intersection(Script),
174    Union(Script),
175
176    /// Solidus is often called "slash" but people often confuse slash
177    /// and backslash.  So we don't call it either.
178    Solidus(Script),
179
180    // @plus@ is actually not the correct glyph name, following sub.py.
181    Plus(Script),
182    Minus(Script),
183    Times(Script),
184    LogicalOr(Script),
185    LogicalAnd(Script),
186    LogicalXor(Script),
187
188    // Any unary "-" is handled in the parser.
189    Digits(Script, NumericLiteral),
190
191    // Used as the index component of instructions like MKZ₄.₁₀
192    BitPosition(Script, String, String),
193
194    // TODO: missing from this are: overbar, square, circle.
195    /// The rules concerning which characters can be part of a symex
196    /// are given in the TX-2 Users Handbook, section 6-3.2, "RULES
197    /// FOR SYMEX FORMATION".
198    ///
199    /// We so not accept dot as part of this token becuase it behaves
200    /// differently in some circumstances (it is a macro terminator).
201    /// However it is part of a valid symex also, and so we will need
202    /// to parse it as such.
203    SymexSyllable(Script, String),
204
205    // If change the representation of the dot in the token
206    // definition, please also change DOT_CHAR.
207    //
208    // The Dot token requires care in handling.  It is valid at the
209    // end of a numeric literal (where it signals use of the alternate
210    // base).  It is also valid in a symex.  But, it is also a macro
211    // terminator.  To handle these complexities, we include the dot
212    // in numeric literals (inside which spaces are not allowed).  But
213    // we do not allow Dot inside Symex syllables - with the idea that
214    // this will help us to correctly process them when used as macro
215    // terminators.
216    Dot(Script),
217    Comma(Script),
218}
219
220impl Display for Token {
221    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
222        let mut write_elevated = |script: &Script, s: &str| -> std::fmt::Result {
223            let el = elevate(*script, s);
224            write!(f, "{el}")
225        };
226
227        match self {
228            Token::Error(msg) => write!(f, "(error: {msg})"),
229            Token::LeftBrace(script) => write_elevated(script, "{"),
230            Token::RightBrace(script) => write_elevated(script, "}"),
231            Token::Newline => f.write_char('\n'),
232            Token::Tab => f.write_char('\t'),
233            Token::LeftParen(script) => write_elevated(script, "("),
234            Token::RightParen(script) => write_elevated(script, ")"),
235            Token::Hold => f.write_char('h'),
236            Token::NotHold => f.write_char('ℏ'),
237            Token::Arrow(script) => write_elevated(script, "->"),
238            Token::Hand(script) => write_elevated(script, "☛"),
239            Token::Asterisk(script) => write_elevated(script, "*"),
240            Token::Dot(script) => write_elevated(script, DOT_STR),
241            Token::Hash(script) => write_elevated(script, "#"),
242            Token::Equals(script) => write_elevated(script, "="),
243            Token::Pipe(script) => write_elevated(script, "|"),
244            Token::DoublePipe(script) => write_elevated(script, "‖"), // U+2016
245            Token::ProperSuperset(script) => write_elevated(script, "⊃"), // U+2283
246            Token::SubsetOf(script) => write_elevated(script, "⊂"),   // U+2282
247            Token::IdenticalTo(script) => write_elevated(script, "≡"),
248            Token::Tilde(script) => write_elevated(script, "~"),
249            Token::LessThan(script) => write_elevated(script, "<"),
250            Token::GreaterThan(script) => write_elevated(script, ">"),
251            Token::Query(script) => write_elevated(script, "?"),
252            Token::Intersection(script) => write_elevated(script, "∩"),
253            Token::Union(script) => write_elevated(script, "∪"),
254            Token::Solidus(script) => write_elevated(script, "/"),
255            Token::Plus(script) => write_elevated(script, "+"),
256            Token::Minus(script) => write_elevated(script, "-"),
257            Token::Times(script) => write_elevated(script, "×"),
258            Token::LogicalOr(script) => write_elevated(script, "∨"),
259            Token::LogicalAnd(script) => write_elevated(script, "∧"),
260            Token::LogicalXor(script) => write_elevated(script, "⊻"),
261            Token::Digits(script, numeric_literal) => {
262                write!(f, "{}", elevate(*script, numeric_literal.to_string()))
263            }
264            Token::BitPosition(script, quarter, bit) => {
265                let q_string = elevate(*script, quarter.clone());
266                let bit_string = elevate(*script, bit.clone());
267                let dotname = match script {
268                    Script::Normal => "@dot@",
269                    Script::Sub => "@sub_dot@",
270                    Script::Super => "@sup_dot@",
271                };
272                write!(f, "{q_string}{dotname}{bit_string}")
273            }
274            Token::SymexSyllable(script, name) => {
275                #[allow(clippy::unnecessary_wraps)]
276                fn nochange(ch: char) -> Result<char, ()> {
277                    Ok(ch)
278                }
279                fn convert_to_sup(ch: char) -> Result<char, ()> {
280                    superscript_char(ch).map_err(|_| ())
281                }
282                fn convert_to_sub(ch: char) -> Result<char, ()> {
283                    subscript_char(ch).map_err(|_| ())
284                }
285                type Transformer = fn(char) -> Result<char, ()>;
286                let (prefix, transform): (&'static str, Transformer) = match script {
287                    Script::Super => ("super_", convert_to_sup),
288                    Script::Normal => ("", nochange),
289                    Script::Sub => ("sub_", convert_to_sub),
290                };
291                for ch in name.chars() {
292                    match transform(ch) {
293                        Ok(sup_ch) => f.write_char(sup_ch),
294                        Err(()) => match ch {
295                            'α' => write!(f, "@{prefix}alpha@"),
296                            'β' => write!(f, "@{prefix}beta@"),
297                            'γ' => write!(f, "@{prefix}gamma@"),
298                            'Δ' => write!(f, "@{prefix}delta@"),
299                            'ε' => write!(f, "@{prefix}eps@"),
300                            'λ' => write!(f, "@{prefix}lambda@"),
301                            _ => write!(f, "@{prefix}{ch}@"),
302                        },
303                    }?;
304                }
305                Ok(())
306            }
307            Token::Comma(script) => write_elevated(script, ","),
308        }
309    }
310}
311
312/// Convert a string into a sequence of [`Elevated<&Glyph>`].
313#[derive(Debug, Clone)]
314struct GlyphRecognizer<'a> {
315    it: CharIndices<'a>,
316    pos: usize,
317    glyph_start: usize,
318}
319
320impl<'a> GlyphRecognizer<'a> {
321    fn new(input: &'a str) -> GlyphRecognizer<'a> {
322        Self {
323            it: input.char_indices(),
324            pos: 0,
325            glyph_start: 0,
326        }
327    }
328
329    fn get_next_char(&mut self) -> Option<char> {
330        match self.it.next() {
331            None => None,
332            Some((i, ch)) => {
333                self.pos = i;
334                Some(ch)
335            }
336        }
337    }
338
339    fn span(&self) -> Span {
340        self.glyph_start..(self.it.offset())
341    }
342
343    fn next_named_glyph(&mut self) -> Option<Result<Elevated<&'static Glyph>, Unrecognised>> {
344        let mut name: String = String::with_capacity(10);
345        let mut got_anything = false;
346        while let Some(ch) = self.get_next_char() {
347            got_anything = true;
348            if ch == '@' {
349                break;
350            }
351            name.push(ch);
352        }
353
354        // If the input was @@, (that is, the glyph name is
355        // zero-length) name is empty but got_anything is
356        // (correctly) true.
357        if got_anything {
358            Some(match glyph_from_name(name.as_str()) {
359                Some(g) => Ok(g),
360                None => Err(Unrecognised::UnrecognisedGlyph(name)),
361            })
362        } else {
363            None
364        }
365    }
366}
367
368impl Iterator for GlyphRecognizer<'_> {
369    type Item = Result<Elevated<&'static Glyph>, Unrecognised>;
370
371    fn next(&mut self) -> Option<Self::Item> {
372        let ch = self.get_next_char()?;
373        self.glyph_start = self.pos;
374        match ch {
375            '@' => match self.next_named_glyph() {
376                None => {
377                    // There actually was input, but it was only a
378                    // single '@'.  That is not in the Lincoln
379                    // Writer character set.
380                    Some(Err(Unrecognised::InvalidChar('@')))
381                }
382                something => something,
383            },
384            ch => Some(glyph_of_char(ch)),
385        }
386    }
387}
388
389#[test]
390fn test_glyph_recognizer_next() {
391    let mut gr = GlyphRecognizer::new("W");
392    match gr.next() {
393        Some(Ok(elev)) => {
394            assert_eq!(elev.script(), Script::Normal);
395            assert_eq!(elev.get().name, "W");
396        }
397        bad => {
398            panic!("glyph should not have been recognised as {bad:?}");
399        }
400    }
401    assert_eq!(gr.next(), None);
402}
403
404#[cfg(test)]
405fn assert_glyph(
406    got: Elevated<&'static Glyph>,
407    expected_shape: GlyphShape,
408    expected_script: Script,
409) {
410    assert_eq!(got.script(), expected_script, "wrong script for {got:?}");
411    assert_eq!(got.get().shape(), expected_shape, "wrong shape for {got:?}");
412}
413
414#[test]
415fn test_glyph_scanning() {
416    let mut scanner = GlyphRecognizer::new("hs@sub_eps@@hamb@@sup_add@@nosuch@ ");
417    // h is in the Lincoln Writer character set.
418    assert_glyph(
419        scanner.next().expect("input").expect("in character set"),
420        GlyphShape::h,
421        Script::Normal,
422    );
423    // s is not in the Lincoln Writer character set.
424    assert_eq!(scanner.next(), Some(Err(Unrecognised::InvalidChar('s'))),);
425    assert_glyph(
426        scanner.next().expect("input").expect("in character set"),
427        GlyphShape::Epsilon,
428        Script::Sub,
429    );
430    assert_glyph(
431        scanner.next().expect("input").expect("in character set"),
432        GlyphShape::IdenticalTo,
433        Script::Normal,
434    );
435    assert_glyph(
436        scanner.next().expect("input").expect("in character set"),
437        GlyphShape::Add,
438        Script::Super,
439    );
440    assert_eq!(
441        scanner.next(),
442        Some(Err(Unrecognised::UnrecognisedGlyph("nosuch".to_string())))
443    );
444    assert_glyph(
445        scanner.next().expect("input").expect("in character set"),
446        GlyphShape::Space,
447        Script::Normal,
448    );
449    assert_eq!(scanner.next(), None);
450    // Verify that detection of end-of-input is sticky.
451    assert_eq!(scanner.next(), None);
452}
453
454fn tokenise_single_glyph(g: Elevated<&'static Glyph>) -> Option<Token> {
455    let script: Script = g.script();
456
457    let make_num = |ch: char| {
458        let literal = NumericLiteral {
459            digits: {
460                let mut s = String::with_capacity(12);
461                s.push(ch);
462                s
463            },
464            has_trailing_dot: false,
465        };
466        Token::Digits(script, literal)
467    };
468    let make_symex = || -> Option<Token> {
469        // The symex token always gives the characters in normal
470        // script.  The superscript/subscript information is
471        // carried in the token variant
472        // (e.g. SuperscriptSymexsyllable).
473        let name: String = g.get().get_char(Script::Normal).iter().collect();
474        // We do not use name.len() here because the number of
475        // bytes in the name is not relevant, only the number of
476        // Unicode code points.
477        match name.chars().count() {
478            0 => {
479                panic!(
480                    "incoming token '{g:?}' was assigned as part of a symex syllable, but we don't have a character for it in script {script:?}"
481                );
482            }
483            1 => (),
484            n => {
485                panic!(
486                    "incoming token '{g:?}' was assigned as part of a symex syllable, but the resuting initial token body unexpectedly has more than one character (specifically, {n}): {name:?}"
487                );
488            }
489        }
490        Some(Token::SymexSyllable(script, name))
491    };
492
493    // In the grammar described in section 6 of the Users Handbook,
494    // space and tab are not handled in quite the same way.  Space is
495    // allowed in symexes, but tab is not (tab terminates a symex).
496    #[allow(clippy::match_same_arms)] // easier to read in existing order
497    let output: Option<Token> = match g.get().shape() {
498        GlyphShape::Space | GlyphShape::Tab => None,
499        GlyphShape::Digit0 => Some(make_num('0')),
500        GlyphShape::Digit1 => Some(make_num('1')),
501        GlyphShape::Digit2 => Some(make_num('2')),
502        GlyphShape::Digit3 => Some(make_num('3')),
503        GlyphShape::Digit4 => Some(make_num('4')),
504        GlyphShape::Digit5 => Some(make_num('5')),
505        GlyphShape::Digit6 => Some(make_num('6')),
506        GlyphShape::Digit7 => Some(make_num('7')),
507        GlyphShape::Digit8 => Some(make_num('8')),
508        GlyphShape::Digit9 => Some(make_num('9')),
509        GlyphShape::Underscore
510        | GlyphShape::Circle
511        | GlyphShape::A
512        | GlyphShape::B
513        | GlyphShape::C
514        | GlyphShape::D
515        | GlyphShape::E
516        | GlyphShape::F
517        | GlyphShape::G
518        | GlyphShape::H
519        | GlyphShape::I
520        | GlyphShape::J
521        | GlyphShape::K
522        | GlyphShape::L
523        | GlyphShape::M
524        | GlyphShape::N
525        | GlyphShape::O
526        | GlyphShape::P
527        | GlyphShape::Q
528        | GlyphShape::R
529        | GlyphShape::S
530        | GlyphShape::T
531        | GlyphShape::U
532        | GlyphShape::V
533        | GlyphShape::W
534        | GlyphShape::X
535        | GlyphShape::Y
536        | GlyphShape::Z => make_symex(),
537        GlyphShape::LeftParen => Some(Token::LeftParen(script)),
538        GlyphShape::RightParen => Some(Token::RightParen(script)),
539        GlyphShape::Add => Some(Token::Plus(script)),
540        GlyphShape::Minus => Some(Token::Minus(script)),
541        GlyphShape::Comma => Some(Token::Comma(script)),
542        GlyphShape::Dot => Some(Token::Dot(script)),
543        GlyphShape::Backspace => unimplemented!("compound characters are not yet supported"),
544        GlyphShape::Hand => Some(Token::Hand(script)),
545        GlyphShape::Sigma => {
546            todo!("Sigma (which is a symex terminator) does not yet have a token")
547        }
548        GlyphShape::Pipe => Some(Token::Pipe(script)),
549        GlyphShape::DoublePipe => Some(Token::DoublePipe(script)),
550        GlyphShape::Solidus => Some(Token::Solidus(script)),
551        GlyphShape::Times => Some(Token::Times(script)),
552        GlyphShape::Hash => Some(Token::Hash(script)),
553        GlyphShape::Arrow => Some(Token::Arrow(script)),
554        GlyphShape::LessThan => Some(Token::LessThan(script)),
555        GlyphShape::GreaterThan => Some(Token::GreaterThan(script)),
556        GlyphShape::Overbar | GlyphShape::Square | GlyphShape::n => make_symex(),
557        GlyphShape::SubsetOf => Some(Token::SubsetOf(script)),
558        GlyphShape::Or => Some(Token::LogicalOr(script)),
559        GlyphShape::q
560        | GlyphShape::Gamma
561        | GlyphShape::t
562        | GlyphShape::w
563        | GlyphShape::x
564        | GlyphShape::i
565        | GlyphShape::y
566        | GlyphShape::z => make_symex(),
567        GlyphShape::Query => Some(Token::Query(script)),
568        GlyphShape::Union => Some(Token::Union(script)),
569        GlyphShape::Intersection => Some(Token::Intersection(script)),
570        GlyphShape::j | GlyphShape::k => make_symex(),
571        GlyphShape::Alpha => make_symex(),
572        GlyphShape::Delta => make_symex(),
573        GlyphShape::p => make_symex(),
574        GlyphShape::Epsilon => make_symex(),
575        GlyphShape::h => Some(match script {
576            // h cannot be part of a symex (see Users Handbook,
577            // section 6-2.3).
578            Script::Super | Script::Sub => unimplemented!(),
579            Script::Normal => Token::Hold,
580        }),
581        // Todo: Token::NotHold.
582        GlyphShape::SupersetOf => Some(Token::ProperSuperset(script)),
583        GlyphShape::Beta => make_symex(),
584        GlyphShape::And => Some(Token::LogicalAnd(script)),
585        GlyphShape::Xor => Some(Token::LogicalXor(script)),
586        GlyphShape::Lambda => make_symex(),
587        GlyphShape::Tilde => Some(Token::Tilde(script)),
588        GlyphShape::LeftBrace => Some(Token::LeftBrace(script)),
589        GlyphShape::RightBrace => Some(Token::RightBrace(script)),
590        GlyphShape::IdenticalTo => Some(Token::IdenticalTo(script)),
591        GlyphShape::Equals => Some(Token::Equals(script)),
592        GlyphShape::Apostrophe => make_symex(),
593        GlyphShape::Asterisk => Some(Token::Asterisk(script)),
594    };
595    if let Some(t) = output.as_ref() {
596        if matches!(t, Token::SymexSyllable(_, _)) {
597            assert!(
598                is_allowed_in_symex(g.get().shape),
599                "attempted to make a symex with disallowed glyph shape {g:?}"
600            );
601        } else if matches!(t, Token::Digits(_, _) | Token::Dot(_)) {
602            // Digits and dots mostly called "PERIOD" in the Users
603            // Handbook) are allowed in numeric literals but also
604            // allowed in symexes.  We scan them as numeric
605            // literals and then join them into symexes when we
606            // discover what their neighbours are.
607            assert!(
608                is_allowed_in_symex(g.get().shape),
609                "all glyphs allowed in numeric literals are also allowed in symexes, but this went wrong for {g:?}"
610            );
611        } else if g.get().shape == GlyphShape::Space {
612            // Permitted in a symex but this is implemented at the parser level.
613        } else {
614            assert!(
615                !is_allowed_in_symex(g.get().shape),
616                "glyph shape {g:?} is allowed in a symex but the scanner didn't recognise it that way"
617            );
618        }
619    }
620    output
621}
622
623#[derive(Debug, PartialEq, Eq)]
624enum TokenMergeResult {
625    Merged(Token, Span),
626    Failed {
627        current: Token,
628        current_span: Span,
629        incoming: Token,
630        incoming_span: Span,
631    },
632}
633
634fn merge_tokens(current: (Token, Span), incoming: (Token, Span)) -> TokenMergeResult {
635    // We never merge errors with non-errors, so eliminate those
636    // cases and, when both inputs are Ok variants, unwrap.
637    let ((current, current_span), (incoming, incoming_span)) = (current, incoming);
638    if matches!(
639        (&current, &incoming),
640        (&Token::Error(_), _) | (_, &Token::Error(_))
641    ) {
642        return TokenMergeResult::Failed {
643            current,
644            current_span,
645            incoming,
646            incoming_span,
647        };
648    }
649
650    let merged_span = current_span.start..incoming_span.end;
651    match current {
652        Token::Minus(incoming_script)
653            if incoming == Token::GreaterThan(incoming_script)
654                && incoming_script == Script::Normal =>
655        {
656            TokenMergeResult::Merged(Token::Arrow(Script::Normal), merged_span)
657        }
658        Token::SymexSyllable(existing_script, mut existing_name) => match incoming {
659            Token::Hold if existing_script == Script::Normal => {
660                // overbar followed by h means not-hold, and we handle this case specially.
661                if existing_name == "\u{0305}" {
662                    TokenMergeResult::Merged(Token::NotHold, merged_span)
663                } else {
664                    TokenMergeResult::Failed {
665                        current: Token::SymexSyllable(existing_script, existing_name),
666                        current_span,
667                        incoming: Token::Hold,
668                        incoming_span,
669                    }
670                }
671            }
672            Token::SymexSyllable(incoming_script, incoming_name)
673                if existing_script == incoming_script =>
674            {
675                existing_name.push_str(&incoming_name);
676                TokenMergeResult::Merged(
677                    Token::SymexSyllable(existing_script, existing_name),
678                    merged_span,
679                )
680            }
681            Token::Digits(incoming_script, literal) if existing_script == incoming_script => {
682                existing_name.push_str(&literal.digits);
683                if literal.has_trailing_dot {
684                    existing_name.push(DOT_CHAR);
685                }
686                TokenMergeResult::Merged(
687                    Token::SymexSyllable(existing_script, existing_name),
688                    merged_span,
689                )
690            }
691            other => TokenMergeResult::Failed {
692                current: Token::SymexSyllable(existing_script, existing_name),
693                current_span,
694                incoming: other,
695                incoming_span,
696            },
697        },
698        Token::Digits(existing_script, mut existing_literal) => {
699            if existing_literal.has_trailing_dot {
700                // The left-hand literal has a dot.  So the valid
701                // cases are where we're part-way through a symex, or
702                // a bit position specification.
703                match incoming {
704                    Token::Digits(
705                        Script::Sub,
706                        NumericLiteral {
707                            digits: incoming_digit,
708                            has_trailing_dot: false,
709                        },
710                    ) if existing_script == Script::Sub => TokenMergeResult::Merged(
711                        Token::BitPosition(
712                            existing_script,
713                            existing_literal.digits,
714                            incoming_digit,
715                        ),
716                        merged_span,
717                    ),
718                    // Not valid for RHS to be Dot, as we already have one.
719                    other => TokenMergeResult::Failed {
720                        current: Token::Digits(existing_script, existing_literal),
721                        current_span,
722                        incoming: other,
723                        incoming_span,
724                    },
725                }
726            } else {
727                match incoming {
728                    Token::Digits(incoming_script, incoming_name)
729                        if existing_script == incoming_script =>
730                    {
731                        existing_literal.append_digits_of_literal(&incoming_name);
732                        TokenMergeResult::Merged(
733                            Token::Digits(existing_script, existing_literal),
734                            merged_span,
735                        )
736                    }
737                    Token::Dot(right_script)
738                        if existing_script == right_script
739                            && !existing_literal.has_trailing_dot =>
740                    {
741                        existing_literal.has_trailing_dot = true;
742                        TokenMergeResult::Merged(
743                            Token::Digits(existing_script, existing_literal),
744                            merged_span,
745                        )
746                    }
747                    Token::SymexSyllable(incoming_script, sym)
748                        if existing_script == incoming_script =>
749                    {
750                        let mut existing_name: String = existing_literal.digits;
751                        existing_name.push_str(&sym);
752                        TokenMergeResult::Merged(
753                            Token::SymexSyllable(existing_script, existing_name),
754                            merged_span,
755                        )
756                    }
757                    other => TokenMergeResult::Failed {
758                        current: Token::Digits(existing_script, existing_literal),
759                        current_span,
760                        incoming: other,
761                        incoming_span,
762                    },
763                }
764            }
765        }
766        Token::BitPosition(existing_script, existing_quarter, mut existing_bit) => match incoming {
767            Token::Digits(
768                incoming_script,
769                NumericLiteral {
770                    digits: incoming_digit,
771                    has_trailing_dot: false,
772                },
773            ) if existing_script == incoming_script => {
774                existing_bit.push_str(incoming_digit.as_str());
775                TokenMergeResult::Merged(
776                    Token::BitPosition(existing_script, existing_quarter, existing_bit),
777                    merged_span,
778                )
779            }
780            Token::Dot(incoming_script) if existing_script == incoming_script => {
781                let name = format!("{existing_quarter}\u{00B7}{existing_bit}\u{00B7}");
782                TokenMergeResult::Merged(Token::SymexSyllable(Script::Sub, name), merged_span)
783            }
784            Token::SymexSyllable(incoming_script, symbol) if existing_script == incoming_script => {
785                let name = format!("{existing_quarter}\u{00B7}{existing_bit}{symbol}");
786                TokenMergeResult::Merged(Token::SymexSyllable(Script::Sub, name), merged_span)
787            }
788            other => TokenMergeResult::Failed {
789                current: Token::BitPosition(existing_script, existing_quarter, existing_bit),
790                current_span,
791                incoming: other,
792                incoming_span,
793            },
794        },
795        existing => TokenMergeResult::Failed {
796            current: existing,
797            current_span,
798            incoming,
799            incoming_span,
800        },
801    }
802}
803
804/// Tokenize a small part of the input, unifying the Unicode and
805/// `@...@` representations.
806#[derive(Debug, Clone)]
807struct GlyphTokenizer<'a> {
808    current: Option<(Token, Span)>,
809    inner: GlyphRecognizer<'a>,
810}
811
812impl<'a> GlyphTokenizer<'a> {
813    fn new(input: &'a str) -> GlyphTokenizer<'a> {
814        GlyphTokenizer {
815            current: None,
816            inner: GlyphRecognizer::new(input),
817        }
818    }
819
820    fn get_next_spanned_token(&mut self) -> Option<(Token, Span)> {
821        loop {
822            let maybe_spanned_new_token: Option<(Token, Span)> = match self.inner.next() {
823                None => None,
824                Some(Err(Unrecognised::InvalidChar('ℏ'))) => {
825                    // ℏ is Unicode code point U+210F.  There
826                    // is no glyph matching ℏ (because on the
827                    // TX-2 this was produced with an overbar
828                    // (which does not advance the carriage)
829                    // and a regular h.  We accept it as a
830                    // special case.
831                    //
832                    // Because there is no Glyph for this, we
833                    // do not accept @...@ (e.g. @hbar@) for
834                    // this.
835                    return Some((Token::NotHold, self.inner.span()));
836                }
837                Some(Err(e)) => {
838                    let error_token = Token::Error(ErrorTokenKind::UnrecognisedGlyph(e));
839                    Some((error_token, self.inner.span()))
840                }
841                Some(Ok(g)) => {
842                    if matches!(g.get().shape(), GlyphShape::Space | GlyphShape::Tab) {
843                        match self.current.take() {
844                            Some(r) => {
845                                return Some(r);
846                            }
847                            None => {
848                                continue;
849                            }
850                        }
851                    }
852                    let tok_start = self.inner.span().start;
853                    match tokenise_single_glyph(g) {
854                        Some(token) => {
855                            let span = tok_start..self.inner.span().end;
856                            Some((token, span))
857                        }
858                        None => {
859                            unimplemented!("unable) to convert glyph '{g:?}' to a token")
860                        }
861                    }
862                }
863            };
864            if let Some((newtoken, newtoken_span)) = maybe_spanned_new_token {
865                if let Some((current, current_span)) = self.current.take() {
866                    match merge_tokens((current, current_span), (newtoken, newtoken_span)) {
867                        TokenMergeResult::Merged(merged, merged_span) => {
868                            self.current = Some((merged, merged_span));
869                        }
870                        TokenMergeResult::Failed {
871                            current,
872                            current_span,
873                            incoming: newtoken,
874                            incoming_span: newtoken_span,
875                        } => {
876                            let result = (current, current_span);
877                            self.current = Some((newtoken, newtoken_span));
878                            return Some(result);
879                        }
880                    }
881                } else {
882                    // There is a new token but no existing token.
883                    self.current = Some((newtoken, newtoken_span));
884                }
885            } else {
886                // There is no new token.
887                return self.current.take();
888            }
889        }
890    }
891}
892
893#[test]
894fn test_glyph_tokenizer_simple_multi_token() {
895    let mut lex = GlyphTokenizer::new("hx");
896    assert_eq!(lex.get_next_spanned_token(), Some((Token::Hold, 0..1)));
897    assert_eq!(
898        lex.get_next_spanned_token(),
899        Some((Token::SymexSyllable(Script::Normal, "x".to_string()), 1..2))
900    );
901    assert_eq!(lex.get_next_spanned_token(), None);
902}
903
904#[test]
905fn test_glyph_tokenizer_glyph_names() {
906    let mut lex = GlyphTokenizer::new("@sup_eps@");
907    assert_eq!(
908        lex.get_next_spanned_token(),
909        Some((Token::SymexSyllable(Script::Super, "ε".to_string()), 0..9))
910    );
911    assert_eq!(lex.get_next_spanned_token(), None);
912}
913
914#[test]
915fn test_glyph_tokenizer_multi_glyph_token() {
916    // These glyphs are a single token because they are both valid
917    // in a symex and are both in superscript.
918    let input = "@sup_eps@ᵂ";
919    let mut lex = GlyphTokenizer::new(input);
920    assert_eq!(
921        lex.get_next_spanned_token(),
922        Some((Token::SymexSyllable(Script::Super, "εW".to_string()), 0..12))
923    );
924    assert_eq!(lex.get_next_spanned_token(), None);
925}
926
927#[test]
928fn test_glyph_tokenizer_script_change_breaks_tokens() {
929    // Verify that a change from superscript to normal script
930    // causes two immediately adjoining letters to be considereed
931    // separate tokens.
932    let mut lex = GlyphTokenizer::new("@sup_eps@W");
933    assert_eq!(
934        lex.get_next_spanned_token(),
935        Some((Token::SymexSyllable(Script::Super, "ε".to_string()), 0..9))
936    );
937    assert_eq!(
938        lex.get_next_spanned_token(),
939        Some((Token::SymexSyllable(Script::Normal, "W".to_string()), 9..10))
940    );
941    assert_eq!(lex.get_next_spanned_token(), None);
942}
943
944#[test]
945fn test_glyph_tokenizer_space_breaks_tokens() {
946    // Verify that a space breaks tokens.  For symexes, the parser
947    // will join the syllables together into a single name, but
948    // they are scanned as separate tokens.
949    let mut lex = GlyphTokenizer::new("W Q");
950    assert_eq!(
951        lex.get_next_spanned_token(),
952        Some((Token::SymexSyllable(Script::Normal, "W".to_string()), 0..1))
953    );
954    assert_eq!(
955        lex.get_next_spanned_token(),
956        Some((Token::SymexSyllable(Script::Normal, "Q".to_string()), 2..3))
957    );
958    assert_eq!(lex.get_next_spanned_token(), None);
959}
960
961#[test]
962fn test_glyph_tokenizer_question_mark() {
963    let mut lex = GlyphTokenizer::new("?");
964    assert_eq!(
965        lex.get_next_spanned_token(),
966        Some((Token::Query(Script::Normal), 0..1))
967    );
968}
969
970/// Tokenize the body of a source code file.
971///
972/// This is the lexer interface intended for use by the parser.
973///
974/// We use [`lower::LowerLexer`] to handle comments, line continuation
975/// and nesting of RC-blocks, and we use [`GlyphTokenizer`] to unify
976/// the several equivalent representations of some symbols (e.g. `₂`
977/// and `@sub_2@`).
978#[derive(Debug, Clone)]
979pub(crate) struct Lexer<'a> {
980    lower: lower::LowerLexer<'a>,
981    upper: Option<GlyphTokenizer<'a>>,
982    upper_span: Option<Span>,
983}
984
985impl<'a> Lexer<'a> {
986    pub(crate) fn new(input: &'a str) -> Lexer<'a> {
987        Lexer {
988            lower: lower::LowerLexer::new(input),
989            upper: None,
990            upper_span: None,
991        }
992    }
993
994    fn adjust_span(&self, span: Range<usize>) -> Range<usize> {
995        match self.upper_span.as_ref() {
996            None => span,
997            Some(upper_span) => {
998                let offset = span.start;
999                (upper_span.start + offset)..(upper_span.end + offset)
1000            }
1001        }
1002    }
1003
1004    pub(crate) fn span(&self) -> Range<usize> {
1005        self.adjust_span(self.lower.span())
1006    }
1007
1008    pub(crate) fn spanned(&self) -> SpannedIter<'a> {
1009        let lexer: Lexer<'a> = self.clone();
1010        SpannedIter::new(lexer)
1011    }
1012
1013    fn get_next(&mut self) -> Option<Token> {
1014        use lower::Lexeme;
1015        // If have more input from the upper lexer, use it.
1016        if let Some(upper_lexer) = self.upper.as_mut()
1017            && let Some((r, span)) = upper_lexer.get_next_spanned_token()
1018        {
1019            self.upper_span = Some(span);
1020            return Some(r);
1021        }
1022
1023        // Fetch more text from the lower lexer.
1024        self.upper = None;
1025        self.upper_span = None;
1026        match self.lower.next() {
1027            Lexeme::EndOfInput => None,
1028            Lexeme::Tok(tok) => Some(tok),
1029            Lexeme::Text(text) => {
1030                let upper = GlyphTokenizer::new(text);
1031                self.upper = Some(upper);
1032                match self
1033                    .upper
1034                    .as_mut()
1035                    .expect("the option cannot be empty, we just filled it")
1036                    .get_next_spanned_token()
1037                {
1038                    Some((r, span)) => {
1039                        self.upper_span = Some(span);
1040                        Some(r)
1041                    }
1042                    None => None,
1043                }
1044            }
1045            Lexeme::Err(e) => Some(Token::Error(ErrorTokenKind::UnrecognisedGlyph(e))),
1046        }
1047    }
1048}
1049
1050impl Iterator for Lexer<'_> {
1051    type Item = Token;
1052
1053    fn next(&mut self) -> Option<Token> {
1054        self.get_next()
1055    }
1056}
1057
1058#[derive(Debug, Clone)]
1059pub(crate) struct SpannedIter<'a> {
1060    lexer: Lexer<'a>,
1061}
1062
1063impl<'a> SpannedIter<'a> {
1064    pub(crate) fn new(lexer: Lexer<'a>) -> SpannedIter<'a> {
1065        SpannedIter { lexer }
1066    }
1067}
1068
1069impl Iterator for SpannedIter<'_> {
1070    type Item = (Token, Span);
1071
1072    fn next(&mut self) -> Option<Self::Item> {
1073        let token = self.lexer.next();
1074        token.map(|t| (t, self.lexer.span()))
1075    }
1076}