Skip to main content

assembler/parser/
macros.rs

1use std::collections::HashMap;
2
3use chumsky::{
4    DefaultExpected,
5    error::LabelError,
6    extension::v1::{Ext, ExtParser},
7    extra::{Full, ParserExtra},
8    input::{Cursor, InputRef, Stream, ValueInput},
9    prelude::*,
10};
11
12use base::charset::Script;
13
14use super::{
15    super::{
16        ast::ArithmeticExpression,
17        manuscript::{
18            MacroBodyLine, MacroDefinition, MacroDummyParameters, MacroInvocation, MacroParameter,
19            MacroParameterBindings, MacroParameterValue,
20        },
21        state::State,
22        symbol::SymbolName,
23    },
24    ExtraWithoutContext, Grammar, MacroBody, Metacommand, Mi, MyError, MyErrorVec, Span,
25    arithmetic_expression_in_any_script_allowing_spaces, end_of_line, grammar, lexer,
26    named_metacommand, named_symbol,
27    symex::{self, SymexSyllableRule},
28    tokenize, tokenize_and_parse_with,
29};
30
31pub(crate) struct UnparsedMacroDefinitionsParser_;
32
33use lexer::Token as Tok;
34
35fn discard_to_eol<'src, 'p, I, E>(inp: &mut InputRef<'src, 'p, I, E>)
36where
37    I: Input<'src, Token = Tok, Span = chumsky::span::SimpleSpan>,
38    E: extra::ParserExtra<'src, I>,
39{
40    while let Some(t) = inp.next_maybe().as_deref() {
41        if t == &Tok::Newline {
42            return;
43        }
44    }
45}
46
47// Read and discard tokens until we find ☛☛ at the start of a line or
48// at the start of input.  Return the position of the first ☛.
49fn advance_to_double_hand<'src, 'p, I, E>(
50    inp: &mut InputRef<'src, 'p, I, E>,
51) -> Option<Cursor<'src, 'p, I>>
52where
53    I: Input<'src, Token = Tok, Span = chumsky::span::SimpleSpan>,
54    E: extra::ParserExtra<'src, I>,
55{
56    use super::Script::Normal;
57
58    loop {
59        let before = inp.cursor();
60        match inp.next_maybe().as_deref() {
61            None => {
62                return None;
63            }
64            Some(first_tok) => {
65                match first_tok {
66                    Tok::Newline => {
67                        // Since we found a newline, we don't want to
68                        // call discard_to_eol.
69                    }
70                    Tok::Hand(Normal) => {
71                        match inp.next_maybe().as_deref() {
72                            None => {
73                                return None;
74                            }
75                            Some(second_tok) => {
76                                match second_tok {
77                                    Tok::Newline => {
78                                        // Since we found a newline, we don't
79                                        // want to call discard_to_eol.
80                                    }
81                                    Tok::Hand(Normal) => {
82                                        return Some(before);
83                                    }
84                                    _other => {
85                                        discard_to_eol(inp);
86                                    }
87                                }
88                            }
89                        }
90                    }
91                    _other => {
92                        discard_to_eol(inp);
93                    }
94                }
95            }
96        }
97    }
98}
99
100// Read and discard tokens until we find ☛☛<expected_cmd> at the start
101// of a line or at the start of input. The expected_cmd argument is
102// the literal command name we expect to find.
103fn advance_to_meta<'src, 'p, I, E>(
104    expected_cmd: &str,
105    inp: &mut InputRef<'src, 'p, I, E>,
106) -> Result<Option<Cursor<'src, 'p, I>>, E::Error>
107where
108    I: Input<'src, Token = Tok, Span = chumsky::span::SimpleSpan>,
109    E: extra::ParserExtra<'src, I>,
110{
111    use super::Script::Normal;
112    loop {
113        let Some(before) = advance_to_double_hand(inp) else {
114            // There are no more ☛☛
115            return Ok(None);
116        };
117        match inp.next_maybe().as_deref() {
118            found @ Some(Tok::SymexSyllable(Normal, actual_cmd)) => {
119                if actual_cmd == expected_cmd {
120                    return Ok(Some(before));
121                } else if actual_cmd == "EMD" {
122                    // We know we were not looking for EMD.  Hence
123                    // we have seen ☛☛EMD before ☛☛DEF.  This is
124                    // an error.
125                    return Err(LabelError::<I, _>::expected_found(
126                        [DefaultExpected::SomethingElse],
127                        found.cloned().map(Into::into),
128                        inp.span_since(&before),
129                    ));
130                }
131                // Otherwise some other metacommand
132                // (e.g. ☛☛DECIMAL).  Ignore.
133            }
134            None => {
135                return if expected_cmd == "EMD" {
136                    // EOF inside a macro body.  This is an error.
137                    Err(LabelError::<I, _>::expected_found(
138                        [DefaultExpected::SomethingElse],
139                        None,
140                        inp.span_since(&before),
141                    ))
142                } else {
143                    // EOF while looking for ☛☛DEF.
144                    Ok(None)
145                };
146            }
147            Some(_other_tok) => {
148                discard_to_eol(inp);
149            }
150        }
151    }
152}
153
154impl<'src, I, E> ExtParser<'src, I, Vec<SimpleSpan>, E> for UnparsedMacroDefinitionsParser_
155where
156    I: Input<'src, Token = Tok, Span = chumsky::span::SimpleSpan>,
157    E: extra::ParserExtra<'src, I>,
158{
159    fn parse(&self, inp: &mut InputRef<'src, '_, I, E>) -> Result<Vec<SimpleSpan>, E::Error> {
160        let mut macros = Vec::new();
161        while let Some(before) = advance_to_meta("DEF", inp)? {
162            let found_end_of_definition = advance_to_meta("EMD", inp)?.is_some();
163            macros.push(inp.span_since(&before));
164            if !found_end_of_definition {
165                break;
166            }
167        }
168        Ok(macros)
169    }
170}
171
172pub(crate) type UnparsedMacroDefinitionsParser = Ext<UnparsedMacroDefinitionsParser_>;
173
174pub(crate) fn unparsed_macro_definitions_parser() -> UnparsedMacroDefinitionsParser {
175    Ext(UnparsedMacroDefinitionsParser_)
176}
177
178fn parse_unparsed_macro_definitions<'src, 'state>(
179    source_file_body: &'src str,
180    mut macros: Vec<SimpleSpan>,
181    state: &'state mut State<'src>,
182) -> Result<MyErrorVec<'src>, MyErrorVec<'src>>
183where
184    'src: 'state,
185{
186    fn convert_errors<T>(input: HashMap<SimpleSpan, Vec<T>>) -> Vec<T> {
187        let mut tmp: Vec<(SimpleSpan, Vec<T>)> = input.into_iter().collect();
188        tmp.sort_by_key(|(span, _)| span.start);
189        tmp.into_iter().flat_map(|(_span, errs)| errs).collect()
190    }
191
192    let macro_count: usize = macros.len();
193    let full_definition_parser = macro_definition();
194    let mut per_definition_errors: HashMap<SimpleSpan, MyErrorVec<'src>> = HashMap::new();
195
196    // `macros` contains the spans of all the macro definitions in the
197    // input.  Programs are not required to define a macro before
198    // calling it, so we may not be able, yet, to parse the body of
199    // the first macro definition.  Instead, we try to parse them one
200    // by one, and when we find one that we can parse, we accept it.
201    //
202    // If we cannot parse a macro, we continue, trying to parse the
203    // remaining macro definitions.  We loop until either all
204    // definitions were parsed or we cannot parse any of the remaining
205    // macro definitions.
206    let mut iteration_number: usize = 0;
207    while !macros.is_empty() {
208        assert!(
209            iteration_number < macro_count,
210            "algorithm is not making progress; this is iteration {iteration_number} but there were only {macro_count} macros to parse"
211        );
212        iteration_number += 1;
213
214        let mut still_todo: Vec<SimpleSpan> = Vec::with_capacity(macros.len());
215        let mut any_success = false;
216        // Try to parse each of the macro definitions we have not yet
217        // been able to parse (or have not yet attempted to parse).
218        for macro_def_span in macros.drain(0..) {
219            let (maybe_parsed, errors): (Option<MacroDefinition>, MyErrorVec) =
220                parse_single_macro_definition(
221                    &full_definition_parser,
222                    source_file_body,
223                    macro_def_span,
224                    state,
225                );
226            per_definition_errors.insert(macro_def_span, errors);
227            if let Some(macro_def) = maybe_parsed {
228                // We parsed this one; record the fact that we made
229                // progress in this iteration and record the
230                // definition itself.
231                any_success = true;
232                // We record the macro definition in `state` so that
233                // later invocations of `full_definition_parser` can
234                // use the known macro definitions in its attempt to
235                // parse this macro definition.
236                state.define_macro(macro_def);
237            } else {
238                // Not successful this time; try again in the next
239                // iteration if there is one.
240                still_todo.push(macro_def_span);
241            }
242        }
243        if any_success {
244            // We made progress, so there will be another iteration.
245            macros.extend(still_todo);
246        } else {
247            // We can no longer make progress.  There are macro
248            // definitions that we cannot parse.
249            return Err(convert_errors(per_definition_errors));
250        }
251    }
252    // We have parsed all the macro definitions.  But we may still
253    // have diagnostics, so return those.
254    Ok(convert_errors(per_definition_errors))
255}
256
257/// Extract macrs defined in `source_file_body` and record them in
258/// `state`.  Return Ok if all macro definitions can be parsed, Err if
259/// at least one cannot.  In both the success and failure case, return
260/// any diagnostics the parsers produced.
261pub(crate) fn parse_unparsed_macro_definitions_only<'src, 'state>(
262    source_file_body: &'src str,
263    state: &'state mut State<'src>,
264) -> Result<MyErrorVec<'src>, MyErrorVec<'src>>
265where
266    'src: 'state,
267{
268    let span_extraction_parser = unparsed_macro_definitions_parser();
269    match tokenize_and_parse_with(source_file_body, 0, state, span_extraction_parser) {
270        (None, errors) => Err(errors),
271        (Some(macros), mut errors) => {
272            match parse_unparsed_macro_definitions(source_file_body, macros, state) {
273                Ok(mut new_errors) => {
274                    errors.extend(new_errors.drain(0..));
275                    Ok(errors)
276                }
277                Err(mut new_errors) => {
278                    errors.extend(new_errors.drain(0..));
279                    Err(errors)
280                }
281            }
282        }
283    }
284}
285
286// Exposed for testing.
287fn macro_definition_dummy_parameter<'a, I>()
288-> impl Parser<'a, I, MacroParameter, ExtraWithoutContext<'a>>
289where
290    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
291{
292    // The TX-2 Users Handbook section 6-4.4 ("DUMMY PARAMETERS")
293    // doesn't disallow spaces in macro argument names.
294    (macro_terminator().then(named_symbol(SymexSyllableRule::Multiple, Script::Normal))).map_with(
295        |(terminator, symbol), extra| MacroParameter {
296            name: symbol,
297            span: extra.span(),
298            preceding_terminator: terminator,
299        },
300    )
301}
302
303fn macro_definition_dummy_parameters<'a, I>()
304-> impl Parser<'a, I, MacroDummyParameters, ExtraWithoutContext<'a>>
305where
306    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
307{
308    choice((
309        macro_definition_dummy_parameter()
310            .repeated()
311            .at_least(1)
312            .collect::<Vec<_>>()
313            .map(MacroDummyParameters::OneOrMore),
314        macro_terminator().map(MacroDummyParameters::Zero),
315        empty().to(MacroDummyParameters::Bare),
316    ))
317}
318
319/// Macros are described in section 6-4 of the TX-2 User Handbook.
320pub(super) fn macro_definition<'src, I>()
321-> impl Parser<'src, I, MacroDefinition, ExtraWithoutContext<'src>> + use<'src, I>
322where
323    I: Input<'src, Token = Tok, Span = Span> + ValueInput<'src>,
324{
325    let g = grammar();
326    named_metacommand(Metacommand::DefineMacro)
327        .ignore_then(
328            named_symbol(SymexSyllableRule::Multiple, Script::Normal).labelled("macro name"), // the macro's name (# is not allowed)
329        )
330        .then(macro_definition_dummy_parameters())
331        .then_ignore(end_of_line())
332        .then(macro_body(&g))
333        .then_ignore(named_metacommand(Metacommand::EndMacroDefinition))
334        .try_map(|((name, params), body), span| {
335            Ok(MacroDefinition {
336                name,
337                params,
338                body,
339                span,
340            })
341        })
342    // We don't parse end-of-line here because all metacommands are supposed
343    // to be followed by end-of-line.
344}
345
346fn parse_single_macro_definition<'source, 'state, P>(
347    macro_definition_parser: &P,
348    whole_source: &'source str,
349    macro_definition_span: SimpleSpan,
350    state: &'state mut State<'source>,
351) -> (Option<MacroDefinition>, Vec<MyError<'source, Mi>>)
352where
353    P: Parser<'source, Mi, MacroDefinition, ExtraWithoutContext<'source>>,
354{
355    let macro_def_span_begin: usize = macro_definition_span.start;
356    let macro_def_span_end: usize = macro_definition_span.end;
357    let macro_definition_source_fragment = &whole_source[macro_def_span_begin..macro_def_span_end];
358
359    let spanned_tokens = tokenize(
360        macro_definition_source_fragment,
361        macro_definition_span.start,
362    );
363    let end_span: SimpleSpan = spanned_tokens
364        .iter()
365        .map(|(_, span)| span.end)
366        .max()
367        .map_or_else(
368            || SimpleSpan::new(macro_definition_span.end, macro_definition_span.end),
369            |end| SimpleSpan::new(end, end),
370        );
371    let token_stream: Mi = Stream::from_iter(spanned_tokens).map(end_span, |unchanged| unchanged);
372    macro_definition_parser
373        .parse_with_state(token_stream, state)
374        .into_output_errors()
375}
376
377/// Macro terminators are described in section 6-4.5 of the TX-2 User Handbook.
378fn macro_terminator<'a, I>() -> impl Parser<'a, I, Tok, ExtraWithoutContext<'a>>
379where
380    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
381{
382    // This list of 16 allowed terminators is exhaustive, see section
383    // 6-4.5 of the TX-2 User Handbook.
384    //
385    // ☛ , = →  | ⊃ ≡ ~ < > ∩ ∪ / × ∨ ∧
386    //
387    // The second symbol, in my scanned copy of the Users Handbook
388    // (page 6-31 of the Nov 1963 Users Handbook), looks like either a
389    // comma or a dot/full-stop/period.  Since a dot is valid in a
390    // symex name, and because the symbol seems to be taller than it
391    // is wide, I'm going to assume it is a comma.
392    //
393    // They are actually hard to distinguish in the copy of the Users
394    // Handbook I have.  But, looking at page 011 of Leonard
395    // Kleinrock's listing for his network simulator, the `HP OS`
396    // macro is definitely using as a separator a symbol that lives on
397    // the line.  If you look a little further below on the same
398    // page, the third instruction in the body of the `MV MX` macro
399    // body contains both a dot and a comma.  The dot is definitely
400    // above the line and looks rounder.  So I conclude that the
401    // separator character is a comma.
402    choice((
403        just(Tok::Hand(Script::Normal)),
404        just(Tok::Comma(Script::Normal)),
405        just(Tok::Equals(Script::Normal)),
406        just(Tok::Arrow(Script::Normal)),
407        just(Tok::Pipe(Script::Normal)),
408        just(Tok::ProperSuperset(Script::Normal)),
409        just(Tok::IdenticalTo(Script::Normal)),
410        just(Tok::Tilde(Script::Normal)),
411        just(Tok::LessThan(Script::Normal)),
412        just(Tok::GreaterThan(Script::Normal)),
413        just(Tok::Intersection(Script::Normal)),
414        just(Tok::Union(Script::Normal)),
415        just(Tok::Solidus(Script::Normal)),
416        just(Tok::Times(Script::Normal)),
417        just(Tok::LogicalOr(Script::Normal)),
418        just(Tok::LogicalAnd(Script::Normal)),
419    ))
420    .labelled("macro terminator")
421}
422
423fn macro_body<'a, 'b, I>(
424    grammar: &Grammar<'a, 'b, I>,
425) -> impl Parser<'a, I, MacroBody, ExtraWithoutContext<'a>> + use<'a, 'b, I>
426where
427    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
428{
429    let parser = (macro_body_line(grammar).then_ignore(end_of_line()))
430        .repeated()
431        .collect::<Vec<MacroBodyLine>>()
432        .map(MacroBody::from);
433    parser.labelled("macro body")
434}
435
436fn macro_body_line<'a, 'b, I>(
437    grammar: &Grammar<'a, 'b, I>,
438) -> impl Parser<'a, I, MacroBodyLine, ExtraWithoutContext<'a>> + use<'a, 'b, I>
439where
440    I: Input<'a, Token = Tok, Span = Span> + ValueInput<'a>,
441{
442    choice((
443        macro_invocation().map(MacroBodyLine::Expansion),
444        grammar.assignment.clone().map(MacroBodyLine::Equality),
445        grammar
446            .tagged_program_instruction
447            .clone()
448            .map(MacroBodyLine::Instruction),
449    ))
450}
451
452pub(super) fn macro_invocation<'src, I>()
453-> impl Parser<'src, I, MacroInvocation, ExtraWithoutContext<'src>>
454where
455    I: Input<'src, Token = Tok, Span = Span> + ValueInput<'src>,
456{
457    Ext(MacroInvocationParser::default())
458}
459
460type ParsedMacroArg = Option<(Script, ArithmeticExpression)>;
461
462#[derive(Clone)]
463pub(super) struct MacroInvocationParser<'src, I>
464where
465    I: Input<'src, Token = Tok, Span = Span> + ValueInput<'src>,
466{
467    expr_parser: Boxed<'src, 'src, I, (Span, ParsedMacroArg), ExtraWithoutContext<'src>>,
468    defined_macro_name_parser: Boxed<'src, 'src, I, MacroDefinition, ExtraWithoutContext<'src>>,
469}
470
471impl<'src, I> MacroInvocationParser<'src, I>
472where
473    I: Input<'src, Token = Tok, Span = Span> + ValueInput<'src>,
474{
475    pub(super) fn new(
476        expr_parser: Boxed<'src, 'src, I, (Span, ParsedMacroArg), ExtraWithoutContext<'src>>,
477        defined_macro_name_parser: Boxed<'src, 'src, I, MacroDefinition, ExtraWithoutContext<'src>>,
478    ) -> Self {
479        Self {
480            expr_parser,
481            defined_macro_name_parser,
482        }
483    }
484}
485
486impl<'src, I> Default for MacroInvocationParser<'src, I>
487where
488    I: Input<'src, Token = Tok, Span = Span> + ValueInput<'src>,
489{
490    fn default() -> Self {
491        Self {
492            expr_parser: arithmetic_expression_in_any_script_allowing_spaces()
493                .or_not()
494                .map_with(|got, extra| match got {
495                    Some((span, script, expr)) => (span, Some((script, expr))),
496                    None => (extra.span(), None),
497                })
498                .boxed(),
499            defined_macro_name_parser: defined_macro_name().boxed(),
500        }
501    }
502}
503
504impl<'src, I> ExtParser<'src, I, MacroInvocation, ExtraWithoutContext<'src>>
505    for MacroInvocationParser<'src, I>
506where
507    I: Input<'src, Token = Tok, Span = Span> + ValueInput<'src>,
508{
509    fn parse(
510        &self,
511        inp: &mut InputRef<'src, '_, I, ExtraWithoutContext<'src>>,
512    ) -> Result<
513        MacroInvocation,
514        <Full<Rich<'src, Tok>, State<'src>, ()> as ParserExtra<'src, I>>::Error,
515    > {
516        let before = inp.cursor();
517        let macro_def: MacroDefinition = inp.parse(&self.defined_macro_name_parser)?;
518        let param_defs: Vec<MacroParameter> = match macro_def.params {
519            MacroDummyParameters::Zero(ref expected) => {
520                if let Some(got) = inp.next_maybe().as_deref() {
521                    if got == expected {
522                        Vec::new()
523                    } else {
524                        return Err(Rich::custom(
525                            inp.span_since(&before),
526                            format!(
527                                "expected macro name {} to be followed a terminator {} but got {}",
528                                macro_def.name, expected, got
529                            ),
530                        ));
531                    }
532                } else {
533                    return Err(Rich::custom(
534                        inp.span_since(&before),
535                        format!(
536                            "expected macro name {} to be followed a terminator {}",
537                            macro_def.name, expected
538                        ),
539                    ));
540                }
541            }
542            MacroDummyParameters::Bare => Vec::new(),
543            MacroDummyParameters::OneOrMore(ref params) => params.clone(),
544        };
545        let mut param_values: MacroParameterBindings = Default::default();
546        for param_def in param_defs {
547            let before = inp.cursor();
548            if let Some(got) = inp.next_maybe().as_deref() {
549                let span = inp.span_since(&before);
550                if got == &param_def.preceding_terminator {
551                    match inp.parse(&self.expr_parser)? {
552                        (span, Some((script, expr))) => {
553                            param_values.insert(
554                                param_def.name,
555                                span,
556                                Some(MacroParameterValue::Value(script, expr)),
557                            );
558                        }
559                        (span, None) => {
560                            // Record the fact that this parameter was missing.
561                            param_values.insert(param_def.name, span, None);
562                        }
563                    }
564                } else {
565                    return Err(Rich::custom(
566                        span,
567                        format!(
568                            "in invocation of macro {}, expected macro terminator {} before parameter {} but got {}",
569                            macro_def.name, param_def.preceding_terminator, param_def.name, got
570                        ),
571                    ));
572                }
573            } else {
574                let span = inp.span_since(&before);
575                return Err(Rich::custom(
576                    span,
577                    format!(
578                        "in invocation of macro {}, expected macro terminator {} before parameter {}",
579                        macro_def.name, param_def.preceding_terminator, param_def.name
580                    ),
581                ));
582            }
583        }
584        Ok(MacroInvocation {
585            macro_def,
586            param_values,
587        })
588    }
589}
590
591pub(super) fn defined_macro_name<'src, I>()
592-> impl Parser<'src, I, MacroDefinition, ExtraWithoutContext<'src>>
593where
594    I: Input<'src, Token = Tok, Span = Span> + ValueInput<'src>,
595{
596    fn mapping<'a>(
597        name: &SymbolName,
598        state: &State,
599        span: Span,
600    ) -> Result<MacroDefinition, chumsky::error::Rich<'a, Tok>> {
601        match state.get_macro_definition(name) {
602            None => Err(Rich::custom(span, format!("unknown macro {name}"))),
603            Some(macro_def) => Ok(macro_def.clone()),
604        }
605    }
606
607    symex::parse_symex(SymexSyllableRule::OneOnly, Script::Normal).try_map_with(|name, extra| {
608        let span: Span = extra.span();
609        let state: &State = extra.state();
610        mapping(&name, state, span)
611    })
612}
613
614#[cfg(test)]
615mod tests {
616    use chumsky::prelude::SimpleSpan;
617
618    use crate::collections::OneOrMore;
619    use base::{charset::Script, u36};
620
621    use super::{
622        super::{
623            super::{
624                ast::{
625                    ArithmeticExpression, Atom, CommaDelimitedFragment, ConfigValue, EqualityValue,
626                    HoldBit, InstructionFragment, InstructionSequence, LiteralValue,
627                    SymbolOrLiteral, Tag, TaggedProgramInstruction, UntaggedProgramInstruction,
628                },
629                manuscript::{
630                    MacroBody, MacroBodyLine, MacroDefinition, MacroDummyParameters,
631                    MacroInvocation, MacroParameter, MacroParameterBindings, MacroParameterValue,
632                    ManuscriptBlock, SourceFile,
633                },
634                state::State,
635                symbol::SymbolName,
636                symtab::{ExplicitDefinition, ExplicitSymbolTable},
637            },
638            grammar,
639            lexer::Token,
640            tests::{
641                assignment_of_literal, atom_to_fragment, no_state_setup, notags,
642                parse_source_file_successfully, parse_successfully_with, parse_with, span,
643            },
644        },
645        macro_body, macro_definition, macro_definition_dummy_parameter,
646        macro_definition_dummy_parameters, macro_invocation,
647    };
648
649    #[test]
650    fn test_macro_body_accepts_two_lines() {
651        let g = grammar();
652        parse_successfully_with("SOMETHING\nWORLD\n", macro_body(&g), no_state_setup);
653    }
654
655    #[test]
656    fn test_macro_body_accepts_empty_macro_body() {
657        let input = "";
658        let g = grammar();
659        let result = parse_with(input, macro_body(&g), no_state_setup);
660        assert!(
661            result.is_ok(),
662            "expected input '{input}' to be accepted but got {result:?}"
663        );
664    }
665
666    #[cfg(test)]
667    fn check_macro_body_symex_span(input: &str, span_expected: SimpleSpan) {
668        dbg!(input, &span_expected);
669        let g = grammar();
670        let parsed_body = parse_successfully_with(input, macro_body(&g), no_state_setup);
671        let lines = parsed_body.parsed_body_lines();
672        dbg!(&lines);
673        match &lines[0] {
674            MacroBodyLine::Instruction(TaggedProgramInstruction { span: the_span, .. }) => {
675                assert_eq!(the_span, &span_expected, "incorrect span");
676            }
677            other => {
678                panic!("expected first line to be an instruction, got {other:?}");
679            }
680        }
681    }
682
683    #[test]
684    fn test_macro_body_generates_symexes_whose_span_does_not_include_newline() {
685        check_macro_body_symex_span("R\n", span(0..1));
686    }
687
688    #[test]
689    fn test_macro_body_generates_symexes_whose_span_does_not_include_comments() {
690        check_macro_body_symex_span("W ** this is a comment\n", span(0..1));
691    }
692
693    #[test]
694    fn test_macro_arg() {
695        assert_eq!(
696            parse_successfully_with(
697                "@hand@A01",
698                macro_definition_dummy_parameter(),
699                no_state_setup
700            ),
701            MacroParameter {
702                name: SymbolName::from("A01".to_string()),
703                span: span(0..9),
704                preceding_terminator: Token::Hand(Script::Normal),
705            }
706        );
707
708        assert_eq!(
709            parse_successfully_with(
710                "@comma@A02",
711                macro_definition_dummy_parameter(),
712                no_state_setup
713            ),
714            MacroParameter {
715                name: SymbolName::from("A02".to_string()),
716                span: span(0..10),
717                preceding_terminator: Token::Comma(Script::Normal),
718            }
719        );
720
721        assert_eq!(
722            parse_successfully_with("=A03", macro_definition_dummy_parameter(), no_state_setup),
723            MacroParameter {
724                name: SymbolName::from("A03".to_string()),
725                span: span(0..4),
726                preceding_terminator: Token::Equals(Script::Normal),
727            }
728        );
729
730        assert_eq!(
731            parse_successfully_with(
732                "@arr@A04",
733                macro_definition_dummy_parameter(),
734                no_state_setup
735            ),
736            MacroParameter {
737                name: SymbolName::from("A04".to_string()),
738                span: span(0..8),
739                preceding_terminator: Token::Arrow(Script::Normal),
740            }
741        );
742
743        assert_eq!(
744            // Alternative form for convenience (and consistency with
745            // what we allow for tags).
746            parse_successfully_with("->A04", macro_definition_dummy_parameter(), no_state_setup),
747            MacroParameter {
748                name: SymbolName::from("A04".to_string()),
749                span: span(0..5),
750                preceding_terminator: Token::Arrow(Script::Normal),
751            }
752        );
753
754        assert_eq!(
755            parse_successfully_with("|A05", macro_definition_dummy_parameter(), no_state_setup),
756            MacroParameter {
757                name: SymbolName::from("A05".to_string()),
758                span: span(0..4),
759                preceding_terminator: Token::Pipe(Script::Normal),
760            }
761        );
762
763        assert_eq!(
764            parse_successfully_with("⊃A06", macro_definition_dummy_parameter(), no_state_setup),
765            MacroParameter {
766                name: SymbolName::from("A06".to_string()),
767                span: span(0..6),
768                preceding_terminator: Token::ProperSuperset(Script::Normal),
769            }
770        );
771
772        assert_eq!(
773            parse_successfully_with("≡A07", macro_definition_dummy_parameter(), no_state_setup),
774            MacroParameter {
775                name: SymbolName::from("A07".to_string()),
776                span: span(0..6),
777                preceding_terminator: Token::IdenticalTo(Script::Normal),
778            }
779        );
780
781        assert_eq!(
782            parse_successfully_with(
783                "@hamb@A07",
784                macro_definition_dummy_parameter(),
785                no_state_setup
786            ),
787            MacroParameter {
788                name: SymbolName::from("A07".to_string()),
789                span: span(0..9),
790                preceding_terminator: Token::IdenticalTo(Script::Normal),
791            }
792        );
793
794        assert_eq!(
795            parse_successfully_with("~A08", macro_definition_dummy_parameter(), no_state_setup),
796            MacroParameter {
797                name: SymbolName::from("A08".to_string()),
798                span: span(0..4),
799                preceding_terminator: Token::Tilde(Script::Normal),
800            }
801        );
802
803        assert_eq!(
804            parse_successfully_with("<A09", macro_definition_dummy_parameter(), no_state_setup),
805            MacroParameter {
806                name: SymbolName::from("A09".to_string()),
807                span: span(0..4),
808                preceding_terminator: Token::LessThan(Script::Normal),
809            }
810        );
811
812        assert_eq!(
813            parse_successfully_with(">A10", macro_definition_dummy_parameter(), no_state_setup),
814            MacroParameter {
815                name: SymbolName::from("A10".to_string()),
816                span: span(0..4),
817                preceding_terminator: Token::GreaterThan(Script::Normal),
818            }
819        );
820
821        assert_eq!(
822            parse_successfully_with("∩A11", macro_definition_dummy_parameter(), no_state_setup),
823            MacroParameter {
824                name: SymbolName::from("A11".to_string()),
825                span: span(0..6),
826                preceding_terminator: Token::Intersection(Script::Normal),
827            }
828        );
829
830        assert_eq!(
831            parse_successfully_with("∪A12", macro_definition_dummy_parameter(), no_state_setup),
832            MacroParameter {
833                name: SymbolName::from("A12".to_string()),
834                span: span(0..6),
835                preceding_terminator: Token::Union(Script::Normal),
836            }
837        );
838
839        assert_eq!(
840            parse_successfully_with("/A13", macro_definition_dummy_parameter(), no_state_setup),
841            MacroParameter {
842                name: SymbolName::from("A13".to_string()),
843                span: span(0..4),
844                preceding_terminator: Token::Solidus(Script::Normal),
845            }
846        );
847
848        assert_eq!(
849            parse_successfully_with("×A14", macro_definition_dummy_parameter(), no_state_setup),
850            MacroParameter {
851                name: SymbolName::from("A14".to_string()),
852                span: span(0..5),
853                preceding_terminator: Token::Times(Script::Normal),
854            }
855        );
856
857        assert_eq!(
858            parse_successfully_with("∨A15", macro_definition_dummy_parameter(), no_state_setup),
859            MacroParameter {
860                name: SymbolName::from("A15".to_string()),
861                span: span(0..6),
862                preceding_terminator: Token::LogicalOr(Script::Normal),
863            }
864        );
865
866        assert_eq!(
867            parse_successfully_with("∧A16", macro_definition_dummy_parameter(), no_state_setup),
868            MacroParameter {
869                name: SymbolName::from("A16".to_string()),
870                span: span(0..6),
871                preceding_terminator: Token::LogicalAnd(Script::Normal),
872            }
873        );
874    }
875
876    #[test]
877    fn test_macro_args() {
878        assert_eq!(
879            parse_successfully_with(
880                "=X@hand@YZ@arr@P",
881                macro_definition_dummy_parameters(),
882                no_state_setup
883            ),
884            MacroDummyParameters::OneOrMore(vec![
885                MacroParameter {
886                    name: SymbolName::from("X".to_string()),
887                    span: span(0..2),
888                    preceding_terminator: Token::Equals(Script::Normal),
889                },
890                MacroParameter {
891                    name: SymbolName::from("YZ".to_string()),
892                    span: span(2..10),
893                    preceding_terminator: Token::Hand(Script::Normal),
894                },
895                MacroParameter {
896                    name: SymbolName::from("P".to_string()),
897                    span: span(10..16),
898                    preceding_terminator: Token::Arrow(Script::Normal),
899                },
900            ])
901        );
902    }
903
904    #[test]
905    fn test_macro_definition_with_empty_body() {
906        // TBD: where in a macro definition are comments allowed?
907        let input = concat!("☛☛DEF MYMACRO≡\n", "☛☛EMD"); // deliberately no terminating newline, see comment in macro_definition().
908        dbg!(&input[21..]);
909        let got = parse_successfully_with(input, macro_definition(), no_state_setup);
910        let expected = MacroDefinition {
911            name: SymbolName::from("MYMACRO".to_string()),
912            params: MacroDummyParameters::Zero(Token::IdenticalTo(Script::Normal)),
913            body: MacroBody::default(),
914            span: span(0..30),
915        };
916        assert_eq!(got, expected);
917    }
918
919    #[test]
920    fn test_macro_definition_with_trivial_body() {
921        let input = concat!(
922            "☛☛DEF JUST|Q\n",
923            // This macro definition has a one-line body.
924            "Q ** This is the only line in the body.\n",
925            "☛☛EMD" // deliberately no terminating newline, see comment in macro_definition().
926        );
927        dbg!(&input[0..57]);
928        dbg!(&input[0..66]);
929        let got = parse_successfully_with(input, macro_definition(), no_state_setup);
930        let expected = MacroDefinition {
931            name: SymbolName::from("JUST".to_string()),
932            params: MacroDummyParameters::OneOrMore(vec![MacroParameter {
933                name: SymbolName::from("Q".to_string()),
934                span: span(14..16),
935                preceding_terminator: Token::Pipe(Script::Normal),
936            }]),
937            body: vec![MacroBodyLine::Instruction(
938                TaggedProgramInstruction::single(
939                    notags(),
940                    HoldBit::Unspecified,
941                    span(17..18),
942                    span(17..18),
943                    InstructionFragment::Arithmetic(ArithmeticExpression::from(
944                        Atom::SymbolOrLiteral(SymbolOrLiteral::Symbol(
945                            Script::Normal,
946                            SymbolName::from("Q"),
947                            span(17..18),
948                        )),
949                    )),
950                ),
951            )]
952            .into(),
953            span: span(0..66),
954        };
955        assert_eq!(got, expected);
956    }
957
958    #[test]
959    fn test_macro_definition_as_entire_source_file() {
960        let got = parse_source_file_successfully("☛☛DEF JUST|Q\nQ\n☛☛EMD\n");
961        let expected = SourceFile {
962            blocks: Default::default(),            // empty
963            global_equalities: Default::default(), // no equalities
964            macros: [(
965                SymbolName::from("JUST".to_string()),
966                MacroDefinition {
967                    name: SymbolName {
968                        canonical: "JUST".to_string(),
969                    },
970                    params: MacroDummyParameters::OneOrMore(vec![MacroParameter {
971                        name: SymbolName {
972                            canonical: "Q".to_string(),
973                        },
974                        span: span(14..16),
975                        preceding_terminator: Token::Pipe(Script::Normal),
976                    }]),
977                    body: vec![MacroBodyLine::Instruction(
978                        TaggedProgramInstruction::single(
979                            notags(),
980                            HoldBit::Unspecified,
981                            span(17..18),
982                            span(17..18),
983                            InstructionFragment::Arithmetic(ArithmeticExpression::from(
984                                Atom::from((span(17..18), Script::Normal, SymbolName::from("Q"))),
985                            )),
986                        ),
987                    )]
988                    .into(),
989                    span: span(0..28),
990                },
991            )]
992            .into_iter()
993            .collect(),
994            punch: None,
995        };
996        assert_eq!(got, expected);
997    }
998
999    #[test]
1000    fn test_parse_macro_invocation() {
1001        let macro_definition_foo = MacroDefinition {
1002            name: SymbolName::from("FOO"),
1003            params: MacroDummyParameters::OneOrMore(vec![MacroParameter {
1004                name: SymbolName::from("A1"),
1005                span: span(10..12),
1006                preceding_terminator: Token::Tilde(Script::Normal),
1007            }]),
1008            body: vec![MacroBodyLine::Instruction(TaggedProgramInstruction {
1009                span: span(0..24),
1010                tags: notags(),
1011                instruction: UntaggedProgramInstruction::from(OneOrMore::new(
1012                    CommaDelimitedFragment {
1013                        leading_commas: None,
1014                        holdbit: HoldBit::Unspecified,
1015                        span: span(16..20),
1016                        fragment: InstructionFragment::Config(ConfigValue {
1017                            already_superscript: false,
1018                            expr: ArithmeticExpression::from(Atom::from((
1019                                span(4..5),
1020                                Script::Normal,
1021                                SymbolName::from("X"),
1022                            ))),
1023                        }),
1024                        trailing_commas: None,
1025                    },
1026                )),
1027            })]
1028            .into(),
1029            span: span(20..40),
1030        };
1031        let set_up_macro_definition = |state: &mut State| {
1032            state.define_macro(macro_definition_foo.clone());
1033        };
1034        let got = parse_successfully_with("FOO~X", macro_invocation(), set_up_macro_definition);
1035
1036        let mut bindings: MacroParameterBindings = Default::default();
1037        bindings.insert(
1038            SymbolName::from("A1"),
1039            span(4..5),
1040            Some(MacroParameterValue::Value(
1041                Script::Normal,
1042                ArithmeticExpression::from(Atom::from((
1043                    span(4..5),
1044                    Script::Normal,
1045                    SymbolName::from("X"),
1046                ))),
1047            )),
1048        );
1049
1050        assert_eq!(
1051            got,
1052            MacroInvocation {
1053                macro_def: macro_definition_foo,
1054                param_values: bindings,
1055            }
1056        );
1057    }
1058
1059    #[test]
1060    fn test_parse_macro_invocation_with_equality() {
1061        let got = parse_source_file_successfully(concat!(
1062            "☛☛DEF MYMACRO≡\n",
1063            "     X = 1\n",
1064            // Although this tag ends up in the program twice,
1065            // this is valid since each the tags in the two macro
1066            // expansions are separate.
1067            " T-> 2\n",
1068            "☛☛EMD",
1069            "\n",
1070            // We expand the macro twice.
1071            "MYMACRO≡\n",
1072            "MYMACRO≡\n",
1073        ));
1074
1075        let symtab: ExplicitSymbolTable = {
1076            let mut st = ExplicitSymbolTable::default();
1077            st.define(
1078                SymbolName::from("X"),
1079                ExplicitDefinition::Equality(EqualityValue::from((
1080                    span(30..31),
1081                    UntaggedProgramInstruction::from(OneOrMore::new(CommaDelimitedFragment {
1082                        leading_commas: None,
1083                        holdbit: HoldBit::Unspecified,
1084                        span: span(30..31),
1085                        fragment: atom_to_fragment(Atom::from(LiteralValue::from((
1086                            span(30..31),
1087                            Script::Normal,
1088                            u36!(1),
1089                        )))),
1090                        trailing_commas: None,
1091                    })),
1092                ))),
1093            )
1094            .expect("assignment should be valid");
1095            st
1096        };
1097
1098        let the_instruction = TaggedProgramInstruction::single(
1099            vec![Tag {
1100                name: SymbolName::from("T"),
1101                span: span(33..34),
1102            }],
1103            HoldBit::Unspecified,
1104            span(33..38),
1105            span(37..38),
1106            InstructionFragment::from((span(37..38), Script::Normal, u36!(2))),
1107        );
1108
1109        let expected = SourceFile {
1110            blocks: vec![ManuscriptBlock {
1111                origin: None,
1112                sequences: vec![
1113                    // The first macro expansion
1114                    InstructionSequence {
1115                        local_symbols: Some(symtab.clone()),
1116                        instructions: vec![the_instruction.clone()],
1117                    },
1118                    // The second macro expansion
1119                    InstructionSequence {
1120                        local_symbols: Some(symtab.clone()),
1121                        instructions: vec![the_instruction.clone()],
1122                    },
1123                ],
1124            }],
1125            // There are no global equalities since the only equality
1126            // is local to the macro body.
1127            global_equalities: Default::default(),
1128            macros: [(
1129                SymbolName::from("MYMACRO"),
1130                MacroDefinition {
1131                    name: SymbolName::from("MYMACRO"),
1132                    params: MacroDummyParameters::Zero(Token::IdenticalTo(Script::Normal)),
1133                    body: vec![
1134                        MacroBodyLine::Equality(assignment_of_literal(
1135                            "X",
1136                            span(26..31),
1137                            LiteralValue::from((span(30..31), Script::Normal, u36!(1))),
1138                        )),
1139                        MacroBodyLine::Instruction(the_instruction.clone()),
1140                    ]
1141                    .into(),
1142                    span: span(0..48),
1143                },
1144            )]
1145            .into_iter()
1146            .collect(),
1147            punch: None,
1148        };
1149
1150        assert_eq!(got, expected,);
1151    }
1152
1153    #[test]
1154    fn test_parse_macro_used_before_defined() {
1155        // In this test we do not actually invoke OUTER because
1156        // recursive macros are not yet supported.
1157        parse_source_file_successfully(concat!(
1158            "☛☛DEF INNER≡P\n",
1159            "     P+2\n",
1160            "☛☛EMD\n",
1161            "☛☛DEF OUTER≡P\n",
1162            "     INNER≡(P+1)\n",
1163            "☛☛EMD\n",
1164        ));
1165    }
1166
1167    #[test]
1168    fn test_parse_macro_defined_before_used() {
1169        // In this test we do not actually invoke OUTER because
1170        // recursive macros are not yet supported.
1171        parse_source_file_successfully(concat!(
1172            "☛☛DEF OUTER≡P\n",
1173            "     INNER≡(P+1)\n",
1174            "☛☛EMD\n",
1175            "☛☛DEF INNER≡P\n",
1176            "     P+2\n",
1177            "☛☛EMD\n",
1178        ));
1179    }
1180}