Skip to main content

assembler/
parser.rs

1//! Turn a sequence of input tokens into a data structure representing
2//! it.
3//!
4//! This data structure isn't a tree, because the logical
5//! structure of a TX-2 assembly-language program isn't tree-like.
6use std::{
7    fmt::Debug,
8    ops::{BitOr, Range, Shl},
9};
10
11pub(crate) mod helpers;
12mod macros;
13mod symex;
14
15#[cfg(test)]
16mod tests;
17
18use chumsky::error::Rich;
19use chumsky::extension::v1::Ext;
20use chumsky::extra::{Full, ParserExtra};
21use chumsky::input::{Emitter, MapExtra, Stream, ValueInput};
22
23use chumsky::Boxed;
24use chumsky::Parser;
25use chumsky::prelude::{Input, IterParser, Recursive, SimpleSpan, choice, just, one_of, recursive};
26use chumsky::select;
27
28use crate::collections::OneOrMore;
29
30use super::ast::{
31    ArithmeticExpression, Atom, CommaDelimitedFragment, Commas, CommasOrInstruction, ConfigValue,
32    Equality, EqualityValue, FragmentWithHold, HoldBit, InstructionFragment, LiteralValue,
33    Operator, Origin, RegisterContaining, RegistersContaining, SignedAtom, SpannedSymbolOrLiteral,
34    SymbolOrLiteral, Tag, TaggedProgramInstruction, UntaggedProgramInstruction,
35};
36use super::lexer::{self};
37use super::manuscript::{
38    MacroBody, MacroDefinition, ManuscriptLine, ManuscriptMetaCommand, SourceFile,
39    manuscript_lines_to_source_file,
40};
41use super::span::{Span, Spanned, span};
42use super::state::{MacroLookup, NumeralMode, State};
43use super::symbol::SymbolName;
44use base::charset::Script;
45use base::prelude::*;
46use helpers::Sign;
47use symex::SymexSyllableRule;
48
49pub(crate) type ExtraWithoutContext<'a> = Full<Rich<'a, lexer::Token>, State<'a>, ()>;
50
51use lexer::Token as Tok;
52
53fn maybe_sign<'a, I>(
54    script_required: Script,
55) -> impl Parser<'a, I, Option<(Sign, Span)>, ExtraWithoutContext<'a>> + Clone
56where
57    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
58{
59    choice((
60        just(Tok::Plus(script_required)).to(Sign::Plus),
61        just(Tok::Minus(script_required)).to(Sign::Minus),
62    ))
63    .map_with(|maybe_sign, extra| (maybe_sign, extra.span()))
64    .or_not()
65}
66
67#[derive(Debug, PartialEq, Eq, Clone)]
68enum BitDesignatorValidation {
69    Good(LiteralValue),
70    Suspect(u8, u8, LiteralValue),
71}
72
73fn make_bit_designator_literal(
74    script: Script,
75    quarter: u8,
76    bitnum: u8,
77    span: Span,
78) -> BitDesignatorValidation {
79    fn build(q: u64, b: u64) -> Unsigned36Bit {
80        // When used as a subscript, the quarter number goes into bits
81        // 3.6-3.5 (bits 23,22).  The bit number goes into bits
82        // 3.4-3.1 (bits 21-18).  However, subscript values are
83        // shifted left by 18 bits.  Meaning, if this is used as a
84        // normal-script value, it should not be shifted.
85        let qmod4: u64 = q % 4_u64;
86        Unsigned36Bit::ZERO.bitor(qmod4.shl(4_u32).bitor(b))
87    }
88    // Apparently-invalid bit designators should still be accepted.
89    // See for example the description of the SKM instruction (in
90    // chapter 3 of the Users Handbook which explains what the machine
91    // does with invalid bit designators.  See also the example in the
92    // table in section 6-2.4 of the Users Handbook.
93    //
94    // So we arrange to issue a warning message for this case.
95    let value = LiteralValue::from((span, script, build(quarter.into(), bitnum.into())));
96    match (quarter, bitnum) {
97        (1..=4, 1..=9) | (4, 10) => BitDesignatorValidation::Good(value),
98        _ => BitDesignatorValidation::Suspect(quarter, bitnum, value),
99    }
100}
101
102// Abbreviations for types that would otherwise be cumbersome.
103pub(crate) type MyError<'a, I> = <chumsky::extra::Full<
104    chumsky::error::Rich<'a, lexer::Token>,
105    State<'a>,
106    (),
107> as ParserExtra<'a, I>>::Error;
108type MyEmitter<'a, I> = Emitter<MyError<'a, I>>;
109type MyErrorVec<'src> = Vec<chumsky::error::Rich<'src, lexer::Token>>;
110
111fn warn_bad_bitpos<'src, I>(
112    validated: BitDesignatorValidation,
113    extra: &mut MapExtra<'src, '_, I, ExtraWithoutContext<'src>>,
114    emitter: &mut MyEmitter<'src, I>,
115) -> LiteralValue
116where
117    I: Input<'src, Token = Tok, Span = Span> + ValueInput<'src>,
118{
119    match validated {
120        // This is a warning message only, because it's
121        // allowed to specify a nonexistent bit position (see
122        // description of SKM instruction).
123        BitDesignatorValidation::Suspect(q, b, literal) => {
124            emitter.emit(Rich::custom(
125                extra.span(),
126                format!("bit position {q}\u{00B7}{b} does not exist"),
127            ));
128            literal
129        }
130        BitDesignatorValidation::Good(literal) => literal,
131    }
132}
133
134fn bit_selector<'a, I>(
135    script_required: Script,
136) -> impl Parser<'a, I, LiteralValue, ExtraWithoutContext<'a>> + Clone
137where
138    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
139{
140    select! {
141        Tok::BitPosition(script, quarter, bits) if script == script_required => (quarter, bits)
142    }
143    .try_map_with(move |(quarter, bit), extra| {
144        // Bit designators are always in decimal.  They end up in the "j bits" of the instruction word (bits 3.6 to 3.1).
145        // This is described in the Users Handbook on page 3-34 (in the description of the SKM instruction)
146        match quarter.as_str().parse::<u8>() {
147            Err(_) => Err(Rich::custom(
148                extra.span(),
149                format!("quarter {quarter} is not a valid decimal number"),
150            )),
151            Ok(q) => match bit.as_str().parse::<u8>() {
152                Ok(bit) => Ok(make_bit_designator_literal(
153                    script_required,
154                    q,
155                    bit,
156                    extra.span(),
157                )),
158                Err(_) => Err(Rich::custom(
159                    extra.span(),
160                    format!("bit position {bit} is not a valid decimal number"),
161                )),
162            },
163        }
164    })
165    .validate(warn_bad_bitpos)
166}
167
168fn literal<'a, I>(
169    script_required: Script,
170) -> impl Parser<'a, I, LiteralValue, ExtraWithoutContext<'a>> + Clone
171where
172    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
173{
174    let plain_literal = {
175        let digits = select! {
176            Tok::Digits(script, n) if script == script_required => n,
177        };
178
179        digits.try_map_with(move |digits_token_payload, extra| {
180            let state: &State = extra.state();
181            let mode: NumeralMode = state.numeral_mode;
182            match digits_token_payload.make_num(mode) {
183                Ok(value) => Ok(LiteralValue::from((extra.span(), script_required, value))),
184                Err(e) => Err(Rich::custom(extra.span(), e.to_string())),
185            }
186        })
187    };
188    choice((bit_selector(script_required), plain_literal)).labelled("numeric literal")
189}
190
191fn here<'a, I>(
192    script_required: Script,
193) -> impl Parser<'a, I, SymbolOrLiteral, ExtraWithoutContext<'a>> + Clone
194where
195    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
196{
197    select! {
198        Tok::Hash(script) if script == script_required => (),
199    }
200    .map_with(move |(), extra| SymbolOrLiteral::Here(script_required, extra.span()))
201}
202
203fn opcode_code(s: &str) -> Option<(Unsigned5Bit, Unsigned6Bit)> {
204    match s {
205        "IOS" => Some((u5!(0o00), u6!(0o04))),
206        "JMP" => Some((u5!(0o00), u6!(0o05))),
207        "BRC" => Some((u5!(0o01), u6!(0o05))),
208        "JPS" => Some((u5!(0o02), u6!(0o05))),
209        "BRS" => Some((u5!(0o03), u6!(0o05))),
210        "JPQ" => Some((u5!(0o14), u6!(0o05))),
211        "BPQ" => Some((u5!(0o15), u6!(0o05))),
212        "JES" => Some((u5!(0o16), u6!(0o05))),
213        "JPD" => Some((u5!(0o20), u6!(0o05))),
214        "BRD" => Some((u5!(0o21), u6!(0o05))),
215        "JDS" => Some((u5!(0o22), u6!(0o05))),
216        "BDS" => Some((u5!(0o23), u6!(0o05))),
217        "JPX" => Some((u5!(0o00), u6!(0o06))),
218        "JNX" => Some((u5!(0o00), u6!(0o07))),
219        "AUX" => Some((u5!(0o00), u6!(0o10))),
220        "RSX" => Some((u5!(0o00), u6!(0o11))),
221        "SKX" | "REX" | "SEX" => Some((u5!(0o00), u6!(0o12))),
222        "INX" => Some((u5!(0o02), u6!(0o12))),
223        "DEX" => Some((u5!(0o03), u6!(0o12))),
224        "SXD" => Some((u5!(0o04), u6!(0o12))),
225        "SXL" => Some((u5!(0o06), u6!(0o12))),
226        "SXG" => Some((u5!(0o07), u6!(0o12))),
227        "RXF" => Some((u5!(0o10), u6!(0o12))),
228        "RXD" => Some((u5!(0o20), u6!(0o12))),
229        "RFD" => Some((u5!(0o30), u6!(0o12))),
230        "EXX" => Some((u5!(0o00), u6!(0o14))),
231        "ADX" => Some((u5!(0o00), u6!(0o15))),
232        "DPX" => Some((u5!(0o00), u6!(0o16))),
233        "SKM" => Some((u5!(0o00), u6!(0o17))),
234        "MKC" => Some((u5!(0o01), u6!(0o17))),
235        "MKZ" => Some((u5!(0o02), u6!(0o17))),
236        "MKN" => Some((u5!(0o03), u6!(0o17))),
237        "SKU" => Some((u5!(0o10), u6!(0o17))),
238        "SUC" => Some((u5!(0o11), u6!(0o17))),
239        "SUZ" => Some((u5!(0o12), u6!(0o17))),
240        "SUN" => Some((u5!(0o13), u6!(0o17))),
241        "SKZ" => Some((u5!(0o20), u6!(0o17))),
242        "SZC" => Some((u5!(0o21), u6!(0o17))),
243        "SZZ" => Some((u5!(0o22), u6!(0o17))),
244        "SZN" => Some((u5!(0o23), u6!(0o17))),
245        "SKN" => Some((u5!(0o30), u6!(0o17))),
246        "SNC" => Some((u5!(0o31), u6!(0o17))),
247        "SNZ" => Some((u5!(0o32), u6!(0o17))),
248        "SNN" => Some((u5!(0o33), u6!(0o17))),
249        "CYR" => Some((u5!(0o04), u6!(0o17))),
250        "MCR" => Some((u5!(0o05), u6!(0o17))),
251        "MZR" => Some((u5!(0o06), u6!(0o17))),
252        "MNR" => Some((u5!(0o07), u6!(0o17))),
253        "SNR" => Some((u5!(0o34), u6!(0o17))),
254        "SZR" => Some((u5!(0o24), u6!(0o17))),
255        "SUR" => Some((u5!(0o14), u6!(0o17))),
256        "LDE" => Some((u5!(0o00), u6!(0o20))),
257        "SPF" => Some((u5!(0o00), u6!(0o21))),
258        "SPG" => Some((u5!(0o00), u6!(0o22))),
259        "LDA" => Some((u5!(0o00), u6!(0o24))),
260        "LDB" => Some((u5!(0o00), u6!(0o25))),
261        "LDC" => Some((u5!(0o00), u6!(0o26))),
262        "LDD" => Some((u5!(0o00), u6!(0o27))),
263        "STE" => Some((u5!(0o00), u6!(0o30))),
264        "FLF" => Some((u5!(0o00), u6!(0o31))),
265        "FLG" => Some((u5!(0o00), u6!(0o32))),
266        "STA" => Some((u5!(0o00), u6!(0o34))),
267        "STB" => Some((u5!(0o00), u6!(0o35))),
268        "STC" => Some((u5!(0o00), u6!(0o36))),
269        "STD" => Some((u5!(0o00), u6!(0o37))),
270        "ITE" => Some((u5!(0o00), u6!(0o40))),
271        "ITA" => Some((u5!(0o00), u6!(0o41))),
272        "UNA" => Some((u5!(0o00), u6!(0o42))),
273        "SED" => Some((u5!(0o00), u6!(0o43))),
274        "JOV" => Some((u5!(0o00), u6!(0o45))),
275        "JPA" => Some((u5!(0o00), u6!(0o46))),
276        "JNA" => Some((u5!(0o00), u6!(0o47))),
277        "EXA" => Some((u5!(0o00), u6!(0o54))),
278        "INS" => Some((u5!(0o00), u6!(0o55))),
279        "COM" => Some((u5!(0o00), u6!(0o56))),
280        "TSD" => Some((u5!(0o00), u6!(0o57))),
281        "CYA" => Some((u5!(0o00), u6!(0o60))),
282        "CYB" => Some((u5!(0o00), u6!(0o61))),
283        "CAB" => Some((u5!(0o00), u6!(0o62))),
284        "NOA" => Some((u5!(0o00), u6!(0o64))),
285        "DSA" => Some((u5!(0o00), u6!(0o65))),
286        "NAB" => Some((u5!(0o00), u6!(0o66))),
287        "ADD" => Some((u5!(0o00), u6!(0o67))),
288        "SCA" => Some((u5!(0o00), u6!(0o70))),
289        "SCB" => Some((u5!(0o00), u6!(0o71))),
290        "SAB" => Some((u5!(0o00), u6!(0o72))),
291        "TLY" => Some((u5!(0o00), u6!(0o74))),
292        "DIV" => Some((u5!(0o00), u6!(0o75))),
293        "MUL" => Some((u5!(0o00), u6!(0o76))),
294        "SUB" => Some((u5!(0o00), u6!(0o77))),
295        _ => None,
296    }
297}
298
299pub(super) fn symbol_or_literal<'a, I>(
300    script_required: Script,
301) -> impl Parser<'a, I, SymbolOrLiteral, ExtraWithoutContext<'a>> + Clone
302where
303    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
304{
305    choice((
306        literal(script_required).map(SymbolOrLiteral::Literal),
307        symex::symex_syllable(script_required).map_with(move |name, extra| {
308            SymbolOrLiteral::Symbol(script_required, SymbolName::from(name), extra.span())
309        }),
310    ))
311    .labelled(match script_required {
312        Script::Super => "superscript single-syllable symbol or literal",
313        Script::Normal => "single-syllable symbol or literal",
314        Script::Sub => "subscript single-syllable symbol or literal",
315    })
316}
317
318fn opcode_to_literal(code: Unsigned6Bit, cfgbits: Unsigned5Bit, span: Span) -> LiteralValue {
319    let bits = Unsigned36Bit::ZERO
320        .bitor(u64::from(code).shl(24))
321        .bitor(u64::from(cfgbits).shl(30))
322        .bitor(helpers::opcode_auto_hold_bit(code));
323    LiteralValue::from((span, Script::Normal, bits))
324}
325
326pub(super) fn opcode<'a, I>() -> impl Parser<'a, I, LiteralValue, ExtraWithoutContext<'a>> + Clone
327where
328    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
329{
330    symex::symex_syllable(Script::Normal)
331        .filter(|mnemonic| opcode_code(mnemonic).is_some())
332        .try_map_with(|mnemonic, extra| match opcode_code(mnemonic.as_str()) {
333            Some((cfgbits, code)) => Ok(opcode_to_literal(code, cfgbits, extra.span())),
334            None => Err(Rich::custom(
335                extra.span(),
336                format!("'{mnemonic}' is not an opcode mnemonic"),
337            )),
338        })
339        .labelled("opcode")
340}
341
342fn named_symbol<'a, I>(
343    rule: SymexSyllableRule,
344    script_required: Script,
345) -> impl Parser<'a, I, SymbolName, ExtraWithoutContext<'a>> + Clone
346where
347    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
348{
349    symex::parse_symex(rule, script_required)
350}
351
352pub(super) fn operator<'a, I>(
353    script_required: Script,
354) -> impl Parser<'a, I, Operator, ExtraWithoutContext<'a>> + Clone
355where
356    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
357{
358    select! {
359        // Solidus ("/") is used for divide.  See section 6-2.7
360        // "Word Assembly" for details.
361        Tok::Solidus(script) if script_required == script => Operator::Divide,
362        Tok::Plus(Script::Normal) => Operator::Add,
363        Tok::Times(got) if got == script_required => Operator::Multiply,
364        Tok::LogicalOr(got) if got == script_required => Operator::LogicalOr,
365        Tok::LogicalAnd(got) if got == script_required => Operator::LogicalAnd,
366        Tok::LogicalXor(got) if got == script_required => Operator::LogicalXor,
367        Tok::Minus(got) if script_required == got => Operator::Subtract,
368        Tok::Plus(got) if script_required == got => Operator::Add,
369    }
370    .labelled("arithmetic operator")
371}
372
373fn asterisk_indirection_fragment<'srcbody, I>()
374-> impl Parser<'srcbody, I, InstructionFragment, ExtraWithoutContext<'srcbody>> + Clone
375where
376    I: Input<'srcbody, Token = Tok, Span = Span> + ValueInput<'srcbody>,
377{
378    just(Tok::Asterisk(Script::Normal))
379        .map_with(|_, extra| InstructionFragment::DeferredAddressing(extra.span()))
380}
381
382/// The pipe construct is described in section 6-2.8 "SPECIAL SYMBOLS"
383/// of the Users Handbook.
384///
385/// "ADXₚ|ₜQ" should be equivalent to "ADXₚ{Qₜ}*".  So during
386/// evaluation we will need to generate an RC-word containing Qₜ.
387fn make_pipe_construct(
388    (p, (t, maybe_spanned_q)): (
389        SpannedSymbolOrLiteral,
390        (SpannedSymbolOrLiteral, Option<(InstructionFragment, Span)>),
391    ),
392) -> InstructionFragment {
393    // The variable names here are taken from the example in the
394    // documentation comment.
395    let tqspan = span(
396        t.span.start..if let Some((_, q_span)) = maybe_spanned_q {
397            q_span.end
398        } else {
399            t.span.end
400        },
401    );
402
403    let t: CommaDelimitedFragment = CommaDelimitedFragment {
404        span: t.span,
405        leading_commas: None,
406        holdbit: HoldBit::Unspecified,
407        fragment: InstructionFragment::Arithmetic(ArithmeticExpression::from(Atom::from(t.item))),
408        trailing_commas: None,
409    };
410    let qt: UntaggedProgramInstruction = if let Some((q, q_span)) = maybe_spanned_q {
411        UntaggedProgramInstruction::from(OneOrMore::with_tail(
412            CommaDelimitedFragment {
413                span: q_span,
414                holdbit: HoldBit::Unspecified,
415                leading_commas: None,
416                fragment: q,
417                trailing_commas: None,
418            },
419            vec![t],
420        ))
421    } else {
422        UntaggedProgramInstruction::from(OneOrMore::new(t))
423    };
424
425    let rc_word_value: RegisterContaining = RegisterContaining::from(TaggedProgramInstruction {
426        span: tqspan,
427        tags: Vec::new(),
428        instruction: qt,
429    });
430    InstructionFragment::PipeConstruct {
431        index: p,
432        rc_word_span: tqspan,
433        rc_word_value,
434    }
435}
436
437fn arithmetic_expression_in_any_script_allowing_spaces<'a, I>()
438-> impl Parser<'a, I, (Span, Script, ArithmeticExpression), ExtraWithoutContext<'a>>
439where
440    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
441{
442    let g = grammar();
443    choice((
444        g.normal_arithmetic_expression_allowing_spaces
445            .map_with(|expr, extra| (extra.span(), Script::Normal, expr)),
446        g.subscript_arithmetic_expression_allowing_spaces
447            .map_with(|expr, extra| (extra.span(), Script::Sub, expr)),
448        g.superscript_arithmetic_expression_allowing_spaces
449            .map_with(|expr, extra| (extra.span(), Script::Super, expr)),
450    ))
451}
452
453#[derive(Debug, PartialEq, Eq, Clone, Copy)]
454enum Metacommand {
455    Decimal,
456    Octal,
457    /// A ☛☛PUNCH meta command.
458    Punch,
459    DefineMacro,
460    EndMacroDefinition,
461}
462
463fn named_metacommand<'a, I>(
464    which: Metacommand,
465) -> impl Parser<'a, I, (), ExtraWithoutContext<'a>> + use<'a, I>
466where
467    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
468{
469    let name_match = move |actual: &str| -> bool {
470        match which {
471            Metacommand::Decimal => {
472                matches!(actual, "DECIMAL" | "DECIMA" | "DECIM" | "DECI" | "DEC")
473            }
474            Metacommand::Octal => matches!(actual, "OCTAL" | "OCTA" | "OCT" | "OC"),
475            Metacommand::Punch => matches!(actual, "PUNCH" | "PUNC" | "PUN" | "PU"),
476            Metacommand::DefineMacro => actual == "DEF",
477            Metacommand::EndMacroDefinition => matches!(actual, "EMD" | "EM"),
478        }
479    };
480
481    let matching_metacommand_name = select! {
482        Tok::SymexSyllable(Script::Normal, name) if name_match(name.as_str()) => (),
483    };
484
485    just([Tok::Hand(Script::Normal), Tok::Hand(Script::Normal)])
486        .ignored()
487        .then_ignore(matching_metacommand_name)
488}
489
490fn metacommand<'a, 'b, I>() -> impl Parser<'a, I, ManuscriptMetaCommand, ExtraWithoutContext<'a>>
491where
492    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
493{
494    fn punch<'a, I>() -> impl Parser<'a, I, ManuscriptMetaCommand, ExtraWithoutContext<'a>>
495    where
496        I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
497    {
498        // We currently have a limitation in the interpretation of
499        // "AA" in the PUNCH metacommand.  The documentation clearly
500        // states that this should be an honest tag.  We currently
501        // accept only numeric literals.
502        named_metacommand(Metacommand::Punch)
503            .ignore_then(literal(Script::Normal).or_not())
504            .try_map(|aa, span| match helpers::punch_address(aa) {
505                Ok(punch) => Ok(ManuscriptMetaCommand::Punch(punch)),
506                Err(msg) => Err(Rich::custom(span, msg)),
507            })
508            .labelled("PUNCH command")
509    }
510
511    fn base_change<'a, I>() -> impl Parser<'a, I, ManuscriptMetaCommand, ExtraWithoutContext<'a>>
512    where
513        I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
514    {
515        choice((
516            named_metacommand(Metacommand::Decimal)
517                .to(ManuscriptMetaCommand::BaseChange(NumeralMode::Decimal)),
518            named_metacommand(Metacommand::Octal)
519                .to(ManuscriptMetaCommand::BaseChange(NumeralMode::Octal)),
520        ))
521        .labelled("base-change metacommand")
522    }
523
524    choice((
525        base_change(),
526        punch(),
527        macros::macro_definition().map(ManuscriptMetaCommand::Macro),
528    ))
529    .labelled("metacommand")
530}
531
532pub(crate) fn instructions_with_comma_counts<I>(it: I) -> Vec<CommaDelimitedFragment>
533where
534    I: Iterator<Item = CommasOrInstruction>,
535{
536    /// Fold operation which estabishes an alternating pattern of
537    /// commas and instructions.
538    ///
539    /// Invariant: acc is non-empty, begins and ends with C(_)
540    /// and does not contain a consecutive pair of C(_) or a
541    /// consecutive pair of Inst(_).
542    fn fold_step(
543        mut acc: Vec<CommasOrInstruction>,
544        item: CommasOrInstruction,
545    ) -> Vec<CommasOrInstruction> {
546        fn null_instruction(span: Span) -> FragmentWithHold {
547            FragmentWithHold {
548                span,
549                holdbit: HoldBit::Unspecified,
550                fragment: InstructionFragment::Null(span),
551            }
552        }
553
554        match acc.last_mut() {
555            Some(CommasOrInstruction::C(tail_comma)) => match item {
556                CommasOrInstruction::C(maybe_commas) => {
557                    if tail_comma.is_none() {
558                        *tail_comma = maybe_commas;
559                    } else {
560                        let null_inst_span: Span = match (tail_comma, &maybe_commas) {
561                            (_, Some(ic)) => span(ic.span().start..ic.span().start),
562                            (Some(tc), _) => span(tc.span().start..tc.span().start),
563                            (None, None) => {
564                                unreachable!(
565                                    "should be no need to interpose a null instruction between two instances of zero commas"
566                                );
567                            }
568                        };
569                        acc.push(CommasOrInstruction::I(null_instruction(null_inst_span)));
570                        acc.push(CommasOrInstruction::C(maybe_commas));
571                    }
572                }
573                CommasOrInstruction::I(inst) => {
574                    acc.push(CommasOrInstruction::I(inst));
575                    acc.push(CommasOrInstruction::C(None));
576                }
577            },
578            Some(CommasOrInstruction::I(_)) => unreachable!("invariant was broken"),
579            None => unreachable!("invariant was not established"),
580        }
581        assert!(matches!(acc.first(), Some(CommasOrInstruction::C(_))));
582        assert!(matches!(acc.last(), Some(CommasOrInstruction::C(_))));
583        acc
584    }
585
586    let mut it = it.peekable();
587
588    let initial_accumulator: Vec<CommasOrInstruction> = vec![CommasOrInstruction::C({
589        match it.peek() {
590            None => {
591                return Vec::new();
592            }
593            Some(CommasOrInstruction::I(_)) => None,
594            Some(CommasOrInstruction::C(maybe_commas)) => {
595                let c = maybe_commas.clone();
596                it.next();
597                c
598            }
599        }
600    })];
601
602    let tmp = it.fold(initial_accumulator, fold_step);
603    let mut output: Vec<CommaDelimitedFragment> = Vec::with_capacity(tmp.len() / 2 + 1);
604    let mut it = tmp.into_iter().peekable();
605    loop {
606        let maybe_before_count = it.next();
607        let maybe_inst = it.next();
608        match (maybe_before_count, maybe_inst) {
609            (None, _) => {
610                break;
611            }
612            (Some(CommasOrInstruction::C(before_commas)), Some(CommasOrInstruction::I(inst))) => {
613                let after_commas: Option<Commas> = match it.peek() {
614                    Some(CommasOrInstruction::C(commas)) => commas.clone(),
615                    None => None,
616                    Some(CommasOrInstruction::I(_)) => {
617                        unreachable!("fold_step did not maintain its invariant")
618                    }
619                };
620                output.push(CommaDelimitedFragment::new(
621                    before_commas,
622                    inst,
623                    after_commas,
624                ));
625            }
626            (Some(CommasOrInstruction::C(_)), None) => {
627                // No instructions in the input.
628                break;
629            }
630            (Some(CommasOrInstruction::I(_)), _)
631            | (Some(CommasOrInstruction::C(_)), Some(CommasOrInstruction::C(_))) => {
632                unreachable!("fold_step did not maintain its invariant");
633            }
634        }
635    }
636    output
637}
638
639fn tag_definition<'a, I>() -> impl Parser<'a, I, Tag, ExtraWithoutContext<'a>> + Clone
640where
641    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
642{
643    named_symbol(SymexSyllableRule::Multiple, Script::Normal)
644        .map_with(|name, extra| Tag {
645            name,
646            span: extra.span(),
647        })
648        .then_ignore(just(Tok::Arrow(Script::Normal)))
649        .labelled("tag definition")
650}
651
652fn commas<'a, I>() -> impl Parser<'a, I, Commas, ExtraWithoutContext<'a>> + Clone
653where
654    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
655{
656    just(Tok::Comma(Script::Normal))
657        .repeated()
658        .at_least(1)
659        .at_most(3)
660        .count()
661        .map_with(|count, extra| {
662            let span = extra.span();
663            match count {
664                1 => Commas::One(span),
665                2 => Commas::Two(span),
666                3 => Commas::Three(span),
667                _ => unreachable!(),
668            }
669        })
670}
671
672fn maybe_hold<'a, I>() -> impl Parser<'a, I, Option<HoldBit>, ExtraWithoutContext<'a>> + Clone
673where
674    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
675{
676    choice((
677        one_of(Tok::Hold).to(HoldBit::Hold),
678        just(Tok::NotHold).to(HoldBit::NotHold),
679    ))
680    .or_not()
681    .labelled("instruction hold bit")
682}
683
684pub(crate) struct Grammar<'src, 'parser, I>
685where
686    I: Input<'src, Token = Tok, Span = Span> + ValueInput<'src>,
687{
688    assignment: Boxed<'src, 'parser, I, Equality, ExtraWithoutContext<'src>>,
689    tagged_program_instruction:
690        Boxed<'src, 'parser, I, TaggedProgramInstruction, ExtraWithoutContext<'src>>,
691    normal_arithmetic_expression_allowing_spaces:
692        Boxed<'src, 'parser, I, ArithmeticExpression, ExtraWithoutContext<'src>>,
693    subscript_arithmetic_expression_allowing_spaces:
694        Boxed<'src, 'parser, I, ArithmeticExpression, ExtraWithoutContext<'src>>,
695    superscript_arithmetic_expression_allowing_spaces:
696        Boxed<'src, 'parser, I, ArithmeticExpression, ExtraWithoutContext<'src>>,
697    #[cfg(test)]
698    instruction_fragment: Boxed<'src, 'parser, I, InstructionFragment, ExtraWithoutContext<'src>>,
699}
700
701fn grammar<'a: 'b, 'b, I>() -> Grammar<'a, 'b, I>
702where
703    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
704{
705    const ALLOW_SPACES: bool = true;
706
707    let mut comma_delimited_instructions = Recursive::declare();
708    let tagged_program_instruction = (tag_definition()
709        .repeated()
710        .collect()
711        .then(comma_delimited_instructions.clone()))
712    .map_with(
713        |(tags, fragments): (Vec<Tag>, OneOrMore<CommaDelimitedFragment>), extra| {
714            let span: Span = extra.span();
715            if let Some(t) = tags.first() {
716                assert_eq!(t.span.start, span.start);
717            }
718            TaggedProgramInstruction {
719                span,
720                tags,
721                instruction: UntaggedProgramInstruction::from(fragments),
722            }
723        },
724    )
725    .labelled(
726        "optional tag definition followed by a (possibly comma-delimited) program instructions",
727    );
728
729    // The contents of an RC-word can contain another RC-word.  A macro
730    // invocation inside an RC-word also needs the arithmetic-expression
731    // parser for its arguments.  Declare this parser before the expression
732    // parser and define it after both dependent parsers exist.
733    let mut register_containing = Recursive::declare();
734    let register_containing_for_expression = register_containing.clone();
735
736    let arith_expr = |allow_spaces: bool, script_required: Script| {
737        {
738            let symex_syllable_rule = if allow_spaces {
739                SymexSyllableRule::Multiple
740            } else {
741                SymexSyllableRule::OneOnly
742            };
743            // We use recursive here to prevent the parser blowing the stack
744            // when trying to parse inputs which have parentheses - that is,
745            // inputs that require recursion.
746            recursive(move |arithmetic_expr| {
747                // Parse (E) where E is some expression.
748                let parenthesised_arithmetic_expression = arithmetic_expr // this is the recursive call
749                    .clone()
750                    .delimited_by(
751                        just(Tok::LeftParen(script_required)),
752                        just(Tok::RightParen(script_required)),
753                    )
754                    .map_with(move |expr, extra| {
755                        Atom::Parens(extra.span(), script_required, Box::new(expr))
756                    })
757                    .labelled("parenthesised arithmetic expression");
758
759                // Parse a literal, symbol, #, or (recursively) an expression in parentheses.
760                let naked_atom = choice((
761                    literal(script_required).map(Atom::from),
762                    opcode().map(Atom::from),
763                    here(script_required).map(Atom::SymbolOrLiteral),
764                    named_symbol(symex_syllable_rule, script_required).map_with(
765                        move |symbol_name, extra| {
766                            Atom::SymbolOrLiteral(SymbolOrLiteral::Symbol(
767                                script_required,
768                                symbol_name,
769                                extra.span(),
770                            ))
771                        },
772                    ),
773                    register_containing_for_expression.clone(),
774                    parenthesised_arithmetic_expression,
775                ))
776                .boxed();
777
778                let signed_atom = maybe_sign(script_required).then(naked_atom).map_with(
779                    |(possible_sign, magnitude), extra| SignedAtom {
780                        span: extra.span(),
781                        negated: matches!(possible_sign, Some((Sign::Minus, _))),
782                        magnitude,
783                    },
784                );
785
786                // Parse an arithmetic operator (e.g. plus, times) followed by an atom.
787                let operator_with_signed_atom = operator(script_required).then(signed_atom.clone());
788
789                // An arithmetic expression is a signed atom followed by zero or
790                // more pairs of (arithmetic operator, signed atom).
791                signed_atom
792                    .then(operator_with_signed_atom.repeated().collect())
793                    .map(|(head, tail)| ArithmeticExpression::with_tail(head, tail))
794            })
795        }
796        .labelled("arithmetic expression")
797    };
798
799    // Parse a values (symbolic or literal) or arithmetic expression.
800    //
801    // BAT² is not an identifier but a sequence[1] whose value is
802    // computed by OR-ing the value of the symex BAT with the value of
803    // the literal "²" (which is 2<<30, or 0o20_000_000_000).  But BAT²
804    // is itself not an arithmetic_expression (because there is a script
805    // change).
806    //
807    // You could argue that (BAT²) should be parsed as an atom.  Right
808    // now that doesn't work because all the elements of an expression
809    // (i.e. everything within the parens) need to have the same script.
810    let program_instruction_fragment = recursive(|program_instruction_fragment| {
811        // Parse the pipe-construct described in the User Handbook
812        // section 2-2.8 "SPECIAL SYMBOLS" as "ₚ|ₜ" (though in reality
813        // the pipe should also be subscript and in their example they
814        // use a subscript q).
815        //
816        // The Handbook is not explicit on whether the "ₚ" or "ₜ" can
817        // contain spaces.  We will assume not, for simplicity (at
818        // least for the time being).
819        //
820        // "ADXₚ|ₜQ" should be equvialent to ADXₚ{Qₜ}*.  So we need to
821        // generate an RC-word containing Qₜ.
822        //
823        // The Sketchpad code (file_name=102726903-05-01-acc.pdf, page
824        // 16) contains a line which uses the pipe construct without a
825        // value for Q: "INX ₚ|ₜ" (p is @sub_beta@ and t is
826        // @sub_alpha@).  For this case we assume the value zero.
827        let spanned_rc_word_fragment = symbol_or_literal(Script::Sub) // this is p
828            .map_with(|p, extra| SpannedSymbolOrLiteral {
829                item: p,
830                span: extra.span(),
831            })
832            .boxed();
833
834        let q = program_instruction_fragment
835            .clone()
836            .map_with(|q, extra| (q, extra.span()));
837        let tq = symbol_or_literal(Script::Sub) // this is t
838            .map_with(|t, extra| SpannedSymbolOrLiteral {
839                item: t,
840                span: extra.span(),
841            })
842            .then(q.or_not())
843            .boxed();
844        let pipe_construct = spanned_rc_word_fragment
845            .then_ignore(just(Tok::Pipe(Script::Sub)))
846            .then(tq)
847            .map(make_pipe_construct)
848            .labelled("pipe construct");
849
850        let single_script_fragment = |script_required| {
851            arith_expr.clone()(true, script_required).map(InstructionFragment::from)
852        };
853
854        // A configuration syllable is not permitted to contain spaces
855        // (per section 6-1.2 "INSTRUCTION WORDS" of the Users
856        // Handbook).  So we need to prevent symex matching accepting
857        // spaces.
858        let config_value = choice((
859            just(Tok::DoublePipe(Script::Normal)).ignore_then(
860                arith_expr.clone()(false, Script::Normal).map(|expr| ConfigValue {
861                    already_superscript: false,
862                    expr,
863                }),
864            ),
865            arith_expr.clone()(false, Script::Super).map(|expr| ConfigValue {
866                already_superscript: true,
867                expr,
868            }),
869        ))
870        .try_map_with(|config_val, extra| {
871            let span: Span = extra.span();
872            let range: Range<usize> = span.into();
873            let slice: &str = &extra.state().body[range];
874
875            if slice.contains(' ') {
876                // Spaces in configuration values are prohibited by
877                // the rule given in the Users handbook, section 6-2.1.
878                Err(Rich::custom(
879                    span,
880                    format!("configuration value '{slice}' should not contain spaces"),
881                ))
882            } else {
883                Ok(InstructionFragment::Config(config_val))
884            }
885        })
886        .labelled("configuration value");
887
888        choice((
889            pipe_construct,
890            single_script_fragment(Script::Normal),
891            single_script_fragment(Script::Sub),
892            asterisk_indirection_fragment(),
893            config_value,
894        ))
895        .labelled("program instruction")
896    });
897
898    comma_delimited_instructions.define({
899        let untagged_program_instruction = maybe_hold()
900            .then(program_instruction_fragment.clone())
901            .map_with(
902                |(maybe_hold, fragment): (Option<HoldBit>, InstructionFragment), extra| {
903                    let span = extra.span();
904                    FragmentWithHold {
905                        span,
906                        holdbit: maybe_hold.unwrap_or(HoldBit::Unspecified),
907                        fragment,
908                    }
909                },
910            );
911
912        choice((
913            commas().map(|c| CommasOrInstruction::C(Some(c))),
914            untagged_program_instruction
915                .clone()
916                .map(CommasOrInstruction::I),
917        ))
918        .repeated()
919        .at_least(1)
920        .collect::<Vec<CommasOrInstruction>>()
921        .map(|ci_vec| instructions_with_comma_counts(ci_vec.into_iter()))
922        .map(|cdfs| match OneOrMore::try_from_iter(cdfs.into_iter()) {
923            Ok(cdfs) => cdfs,
924            Err(_) => {
925                unreachable!("instructions_with_comma_counts generated an empty output");
926            }
927        })
928    });
929
930    let nested_macro_arg = choice((
931        arith_expr.clone()(ALLOW_SPACES, Script::Normal)
932            .map_with(|expr, extra| (extra.span(), Script::Normal, expr)),
933        arith_expr.clone()(ALLOW_SPACES, Script::Sub)
934            .map_with(|expr, extra| (extra.span(), Script::Sub, expr)),
935        arith_expr.clone()(ALLOW_SPACES, Script::Super)
936            .map_with(|expr, extra| (extra.span(), Script::Super, expr)),
937    ))
938    .or_not()
939    .map_with(|got, extra| match got {
940        Some((span, script, expr)) => (span, Some((script, expr))),
941        None => (extra.span(), None),
942    })
943    .boxed();
944
945    let nested_macro_invocation = Ext(macros::MacroInvocationParser::new(
946        nested_macro_arg,
947        macros::defined_macro_name().boxed(),
948    ));
949
950    let expanded_macro = nested_macro_invocation.try_map_with(|invocation, extra| {
951        let expansion = invocation.substitute_macro_parameters(extra.state().macros());
952        if expansion
953            .local_symbols
954            .as_ref()
955            .is_some_and(|symbols| !symbols.is_empty())
956            || expansion
957                .instructions
958                .iter()
959                .any(|instruction| !instruction.tags.is_empty())
960        {
961            return Err(Rich::custom(
962                extra.span(),
963                "macro expansion inside an RC-word cannot yet contain local symbols",
964            ));
965        }
966        OneOrMore::try_from_iter(
967            expansion
968                .instructions
969                .into_iter()
970                .map(RegisterContaining::from),
971        )
972        .map_err(|_| Rich::custom(extra.span(), "macro expansion inside an RC-word is empty"))
973    });
974
975    // Parse {E} where E is either one instruction or a macro expansion.
976    register_containing.define(
977        choice((
978            expanded_macro,
979            tagged_program_instruction
980                .clone()
981                .map(|instruction| OneOrMore::new(RegisterContaining::from(instruction))),
982        ))
983        .delimited_by(
984            just(Tok::LeftBrace(Script::Normal)),
985            just(Tok::RightBrace(Script::Normal)),
986        )
987        .map_with(|words, extra| Atom::RcRef(extra.span(), RegistersContaining::from_words(words)))
988        .labelled("RC-word")
989        .boxed(),
990    );
991
992    // Assginments are called "equalities" in the TX-2 Users Handbook.
993    // See section 6-2.2, "SYMEX DEFINITON - TAGS - EQUALITIES -
994    // AUTOMATIC ASSIGNMENT".
995    let assignment = (symex::parse_symex(SymexSyllableRule::Multiple, Script::Normal)
996        .then_ignore(just(Tok::Equals(Script::Normal)))
997        .then(comma_delimited_instructions.clone().map_with(
998            |val: OneOrMore<CommaDelimitedFragment>, extra| {
999                EqualityValue::from((extra.span(), UntaggedProgramInstruction::from(val)))
1000            },
1001        )))
1002    .map_with(|(name, value), extra| Equality {
1003        span: extra.span(),
1004        name,
1005        value,
1006    })
1007    .labelled("equality (assignment)");
1008
1009    let tagged_program_instruction = tagged_program_instruction.clone();
1010
1011    Grammar {
1012        assignment: assignment.boxed(),
1013        tagged_program_instruction: tagged_program_instruction.boxed(),
1014        normal_arithmetic_expression_allowing_spaces: arith_expr.clone()(
1015            ALLOW_SPACES,
1016            Script::Normal,
1017        )
1018        .boxed(),
1019        superscript_arithmetic_expression_allowing_spaces: arith_expr.clone()(
1020            ALLOW_SPACES,
1021            Script::Super,
1022        )
1023        .boxed(),
1024        subscript_arithmetic_expression_allowing_spaces: arith_expr.clone()(
1025            ALLOW_SPACES,
1026            Script::Sub,
1027        )
1028        .boxed(),
1029        #[cfg(test)]
1030        instruction_fragment: program_instruction_fragment.boxed(),
1031    }
1032}
1033
1034#[cfg(test)]
1035fn tagged_instruction<'a, I>()
1036-> impl Parser<'a, I, TaggedProgramInstruction, ExtraWithoutContext<'a>>
1037where
1038    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
1039{
1040    grammar().tagged_program_instruction
1041}
1042
1043fn manuscript_line<'a, 'b, I>(
1044    grammar: &Grammar<'a, 'b, I>,
1045) -> impl Parser<'a, I, ManuscriptLine, ExtraWithoutContext<'a>>
1046where
1047    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
1048{
1049    fn execute_metacommand(state: &mut NumeralMode, cmd: &ManuscriptMetaCommand) {
1050        match cmd {
1051            ManuscriptMetaCommand::Punch(_) | ManuscriptMetaCommand::Macro(_) => {
1052                // Instead of executing this metacommand as we parse it,
1053                // we simply return it as part of the parser output, and
1054                // it is executed by the driver.
1055            }
1056            ManuscriptMetaCommand::BaseChange(new_base) => state.set_numeral_mode(*new_base),
1057        }
1058    }
1059
1060    fn parse_and_execute_metacommand<'a, I>()
1061    -> impl Parser<'a, I, ManuscriptLine, ExtraWithoutContext<'a>>
1062    where
1063        I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
1064    {
1065        metacommand()
1066            .map_with(|cmd, extra| {
1067                execute_metacommand(&mut extra.state().numeral_mode, &cmd);
1068                ManuscriptLine::Meta(cmd)
1069            })
1070            .labelled("metacommand")
1071    }
1072
1073    fn build_code_line(
1074        (maybe_origin, statement): (Option<Origin>, TaggedProgramInstruction),
1075    ) -> ManuscriptLine {
1076        match maybe_origin {
1077            None => ManuscriptLine::StatementOnly(statement),
1078            Some(origin) => ManuscriptLine::OriginAndStatement(origin, statement),
1079        }
1080    }
1081
1082    fn line<'a, 'b, I>(
1083        grammar: &Grammar<'a, 'b, I>,
1084    ) -> impl Parser<'a, I, ManuscriptLine, ExtraWithoutContext<'a>>
1085    where
1086        I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
1087    {
1088        fn origin<'a, I>() -> impl Parser<'a, I, Origin, ExtraWithoutContext<'a>>
1089        where
1090            I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
1091        {
1092            /// An address expression is a literal value or a symex.  That is I
1093            /// think it's not required that an arithmetic expression
1094            /// (e.g. "5+BAR") be accepted in an origin notation (such as
1095            /// "<something>|").
1096            fn literal_address_expression<'a, I>()
1097            -> impl Parser<'a, I, Origin, ExtraWithoutContext<'a>>
1098            where
1099                I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
1100            {
1101                literal(Script::Normal)
1102                    .then_ignore(just(Tok::Pipe(Script::Normal)))
1103                    .try_map(|lit, span| match Address::try_from(lit.value()) {
1104                        Ok(addr) => Ok(Origin::Literal(span, addr)),
1105                        Err(e) => Err(Rich::custom(span, format!("not a valid address: {e}"))),
1106                    })
1107                    .labelled("literal address expression")
1108            }
1109
1110            fn symbolic_address_expression<'a, I>()
1111            -> impl Parser<'a, I, Origin, ExtraWithoutContext<'a>>
1112            where
1113                I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
1114            {
1115                named_symbol(SymexSyllableRule::Multiple, Script::Normal)
1116                    .then_ignore(just(Tok::Pipe(Script::Normal)))
1117                    .map_with(|name, extra| Origin::Symbolic(extra.span(), name))
1118                    .labelled("symbolic address expression")
1119            }
1120
1121            // An origin specification is an expression followed by a
1122            // (normal-case) pipe symbol.
1123            choice((literal_address_expression(), symbolic_address_expression()))
1124                .labelled("origin specification")
1125        }
1126
1127        let optional_origin_with_statement = origin()
1128            .or_not()
1129            .then(grammar.tagged_program_instruction.clone())
1130            .map(build_code_line)
1131            .labelled("statement with origin");
1132
1133        // TODO: also allowed: "T1->T2->ORIGIN|"
1134        let origin_only = origin().map(ManuscriptLine::OriginOnly);
1135        let tags_only = tag_definition()
1136            .repeated()
1137            .at_least(1)
1138            .collect()
1139            .map(ManuscriptLine::TagsOnly);
1140        let equality = grammar.assignment.clone().map(ManuscriptLine::Eq);
1141
1142        choice((
1143            // We have to parse an assignment first here, in order to
1144            // accept "FOO=2" as an assignment rather than the instruction
1145            // fragment "FOO" followed by a syntax error.
1146            equality,
1147            macros::macro_invocation().map(ManuscriptLine::Macro),
1148            // Ignore whitespace after the metacommand but not before it.
1149            parse_and_execute_metacommand(),
1150            optional_origin_with_statement,
1151            // Because we prefer to parse a statement if one exists,
1152            // the origin_only alternative has to appear after the
1153            // alternative which parses a statement.
1154            origin_only,
1155            // Similarly for lines containing only tag efinitions.
1156            tags_only,
1157        ))
1158    }
1159
1160    line(grammar)
1161}
1162
1163fn end_of_line<'a, I>() -> impl Parser<'a, I, (), ExtraWithoutContext<'a>>
1164where
1165    I: Input<'a, Token = Tok, Span = Span>,
1166{
1167    let one_end_of_line = just(Tok::Newline).labelled("end-of-line").ignored();
1168
1169    one_end_of_line
1170        .repeated()
1171        .at_least(1)
1172        .ignored()
1173        .labelled("comment or end-of-line")
1174}
1175
1176fn terminated_manuscript_line<'a, 'b, I>(
1177    grammar: &Grammar<'a, 'b, I>,
1178) -> impl Parser<'a, I, Option<(Span, ManuscriptLine)>, ExtraWithoutContext<'a>>
1179where
1180    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
1181{
1182    // If we support INSERT, DELETE, REPLACE, we will need to
1183    // separate the processing of the metacommands and the
1184    // generation of the assembled code.
1185    manuscript_line(grammar)
1186        .or_not()
1187        .map_with(|maybe_line, extra| maybe_line.map(|line| (extra.span(), line)))
1188        .then_ignore(end_of_line())
1189}
1190
1191pub(crate) fn source_file<'a, 'b, I>(
1192    grammar: &Grammar<'a, 'b, I>,
1193) -> impl Parser<'a, I, SourceFile, ExtraWithoutContext<'a>>
1194where
1195    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
1196{
1197    fn inconsistency_error<'src>(
1198        span: Span,
1199        name: &SymbolName,
1200        what: &str,
1201    ) -> chumsky::error::Rich<'src, lexer::Token> {
1202        Rich::custom(
1203            span,
1204            format!("internal error: inconsistent parser state for macro {name}: {what}"),
1205        )
1206    }
1207
1208    fn check_consistent<'a>(
1209        macro_def_in_source_file: Option<&MacroDefinition>,
1210        macro_def_in_state: Option<&MacroDefinition>,
1211    ) -> Result<(), chumsky::error::Rich<'a, lexer::Token>> {
1212        match (macro_def_in_source_file, macro_def_in_state) {
1213            (None, None) => {
1214                panic!("all_name is incorrect");
1215            }
1216            (None, Some(state_def)) => Err(inconsistency_error(
1217                state_def.span,
1218                &state_def.name,
1219                "missing from SourceFile output",
1220            )),
1221            (Some(source_file_def), None) => Err(inconsistency_error(
1222                source_file_def.span,
1223                &source_file_def.name,
1224                "missing from State",
1225            )),
1226            (Some(source_file_def), Some(state_def)) => {
1227                if source_file_def == state_def {
1228                    Ok(())
1229                } else {
1230                    Err(inconsistency_error(
1231                        source_file_def.span,
1232                        &source_file_def.name,
1233                        "inconsistently defined",
1234                    ))
1235                }
1236            }
1237        }
1238    }
1239
1240    terminated_manuscript_line(grammar)
1241        .repeated()
1242        .collect()
1243        .try_map_with(|lines: Vec<Option<(Span, ManuscriptLine)>>, extra| {
1244            // Filter out empty lines.
1245            let lines: Vec<(Span, ManuscriptLine)> = lines.into_iter().flatten().collect();
1246            let state_macros = extra.state().macros();
1247
1248            let source_file: SourceFile = manuscript_lines_to_source_file(lines)?;
1249
1250            for name in source_file
1251                .macros
1252                .all_names()
1253                .union(&state_macros.all_names())
1254            {
1255                check_consistent(
1256                    source_file.macros.lookup_macro(name),
1257                    state_macros.lookup_macro(name),
1258                )?;
1259            }
1260            Ok(source_file)
1261        })
1262}
1263
1264type Mig<I, O> = chumsky::input::MappedInput<
1265    Tok,
1266    SimpleSpan,
1267    chumsky::input::Stream<std::vec::IntoIter<(Tok, SimpleSpan)>>,
1268    fn(I) -> O,
1269>;
1270pub(crate) type Mi = Mig<(Tok, SimpleSpan), (Tok, SimpleSpan)>;
1271
1272fn tokenize(input: &str, span_offset: usize) -> Vec<(Tok, SimpleSpan)> {
1273    let scanner = lexer::Lexer::new(input).spanned();
1274    scanner
1275        .map(|(token, tokspan): (Tok, Range<usize>)| {
1276            let adjusted_span = (tokspan.start + span_offset)..(tokspan.end + span_offset);
1277            (token, adjusted_span)
1278        })
1279        .map(|item: (Tok, Range<usize>)| -> (Tok, Span) {
1280            match item {
1281                (Tok::Tab, span) => {
1282                    // The basic problem here is that the TX-2's
1283                    // M4 assembler allows a space to occur in the
1284                    // middle of a symex.  We implement this in
1285                    // the parser by returning the individual
1286                    // parts from the lexer and having the parser
1287                    // join them together.  The lexer doesn't
1288                    // return spaces.  In order to prevent the
1289                    // parser joining together "XY\tZ" in a
1290                    // similar way we would need to return TAB as
1291                    // a lexeme.  The problem with doing that
1292                    // though is that the parser would have to
1293                    // permit the TAB token between regular
1294                    // tokens everywhere in the grammar except
1295                    // between two symex components.  That would
1296                    // make the grammar difficult to maintain (and
1297                    // difficult to specify without bugs).
1298                    (Tok::Error(lexer::ErrorTokenKind::Tab), span.into())
1299                }
1300                (tok, span) => {
1301                    // Turn the `Range<usize>` spans logos gives us into
1302                    // chumsky's `SimpleSpan` via `Into`, because it's
1303                    // easier to work with
1304                    (tok, span.into())
1305                }
1306            }
1307        })
1308        .collect()
1309}
1310
1311pub(crate) fn tokenize_and_parse_with<'a, P, T>(
1312    input: &'a str,
1313    span_offset: usize,
1314    state: &mut State<'a>,
1315    parser: P,
1316) -> (Option<T>, Vec<Rich<'a, Tok>>)
1317where
1318    P: Parser<'a, Mi, T, ExtraWithoutContext<'a>>,
1319{
1320    let tokens = tokenize(input, span_offset);
1321    let end_span: SimpleSpan = SimpleSpan::new(
1322        0,
1323        tokens.iter().map(|(_, span)| span.end).max().unwrap_or(0),
1324    );
1325    let token_stream: Mi = Stream::from_iter(tokens).map(end_span, |unchanged| unchanged);
1326    parser
1327        .parse_with_state(token_stream, state)
1328        .into_output_errors()
1329}
1330
1331fn set_initial_numeral_mode(state: &mut State) {
1332    state.numeral_mode.set_numeral_mode(NumeralMode::Octal);
1333}
1334
1335pub(crate) fn parse_source_file<'src>(
1336    source_file_body: &'src str,
1337) -> (Option<SourceFile>, Vec<Rich<'src, Tok>>) {
1338    let mut state = State::with_setup(source_file_body, set_initial_numeral_mode);
1339    set_initial_numeral_mode(&mut state);
1340
1341    let mut errors: Vec<Rich<'src, lexer::Token>> =
1342        match macros::parse_unparsed_macro_definitions_only(source_file_body, &mut state) {
1343            Ok(errs) => errs,
1344            Err(errs) => {
1345                return (None, errs);
1346            }
1347        };
1348
1349    let g = grammar();
1350    let parser = source_file(&g);
1351    let (maybe_source_file, new_errors) =
1352        tokenize_and_parse_with(source_file_body, 0, &mut state, parser);
1353    errors.extend(new_errors);
1354    (maybe_source_file, errors)
1355}
1356
1357// Local Variables:
1358// mode: rustic
1359// lsp-rust-analyzer-server-display-inlay-hints: nil
1360// End: