Improve multiline string handling

CHANGELOG.md

@@ -13,6 +13,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fixed performance issues with heavily nested conditional directives.
 - Fixed formatting of anonymous routines containing subroutines.
 
+### Changed
+
+- Unterminated string literals will never be merged with other tokens.
+- Improved support for multiline strings.
+  - A break will be inserted before each multiline string to ensure stylistic consistency.
+  - Surrounding code will be appropriately wrapped.
+  - String contents currently can _not_ be re-indented. Including the closing quotes.
+
 ## [0.5.1] - 2025-06-02
 
 ### Fixed

core/CHANGELOG.md

@@ -21,11 +21,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `break_anonymous_routine` updated to reflect the broken state.
 - When the first token of a logical line has child lines, those lines are now explored and
   formatted as usual.
+- Fixed line length calculations for multiline tokens.
 
 ### Changed
 
 - Conditional directive parsing now uses a greedy algorithm that avoids the exponential
   performance issues with the old algorithm.
+- Unterminated string literals will never mangle other tokens by always breaking after.
+- Improved support for multiline strings. They will now always break before, and
+  break surrounding contexts.
 
 ## 0.5.1 - 2025-06-02
 

core/datatests/generators/optimising_line_formatter.rs

@@ -4164,6 +4164,7 @@ mod expressions {
         mixed::generate(root_dir);
         unary::generate(root_dir);
         set::generate(root_dir);
+        strings::generate(root_dir);
     }
 
     mod boolean {
@@ -4656,6 +4657,96 @@ mod expressions {
             );
         }
     }
+
+    mod strings {
+        use super::*;
+
+        pub fn generate(root_dir: &Path) {
+            generate_test_cases!(
+                root_dir,
+                unterminated = "
+                    A :=
+                        'abc' + '
+                        ;
+                    A :=
+                        '
+                        ;
+                ",
+                multiline = "
+                    A :=
+                        '''
+                        ''';
+                    A :=
+                        '''
+                        '''
+                            + BB
+                            + CC;
+                    A :=
+                        '''
+                        '''[11111 + 11111 + 1111];
+                    A :=
+                        '''
+                        '''[
+                            11111 + 11111 + 11111
+                        ];
+                    A :=
+                        '''
+                        '''.B(1111 + 1111 + 1111);
+                    A :=
+                        '''
+                        '''.B(
+                            11111 + 11111 + 11111
+                        );
+                    A :=
+                        B(
+                            '''
+                            '''
+                        );
+                    A :=
+                        B(
+                            C,
+                            '''
+                            ''',
+                            D
+                        );
+                    A :=
+                        B(
+                            C,
+                            '''
+                            '''
+                                +
+                                '''
+                            ''',
+                            D
+                        );
+                ",
+                multiline_diff = {
+                    "
+                    A :=
+                    '''
+                         ''';
+                    A := '''
+                              '''.B(11 + 11 + 11);
+                    A :=
+                    '''
+                               '''.B(11 + 11 + 11);
+                    ",
+                    "
+                    A :=
+                        '''
+                         ''';
+                    A :=
+                        '''
+                              '''.B(11 + 11 + 11);
+                    A :=
+                        '''
+                               '''
+                            .B(11 + 11 + 11);
+                    "
+                }
+            );
+        }
+    }
 }
 
 mod attributes {

core/src/defaults/reconstructor.rs

@@ -698,8 +698,11 @@ A = 1 + {
 
 } | | | | | | | | A;
 
-A = 1     + '''
-''' + | | | | Spaces;",
+A := 1     + '''
+'''  + | | | | |Spaces;
+
+A := 1     + '''
+    '''  + | | | | | Spaces;",
                 "\
 A
     = 1
@@ -709,8 +712,19 @@ A
 }
   | | | | | | ||A;
 
-A = 1 + '''
-''' + | | | |  Spaces;",
+A :=
+    1
+        +
+        '''
+'''
+        +||| |     |Spaces;
+
+A :=
+    1
+        +
+        '''
+    '''
+        +  | | | | |Spaces;",
             );
         }
     }

core/src/rules/optimising_line_formatter/contexts.rs

@@ -445,6 +445,14 @@ impl<'a> SpecificContextStack<'a> {
 
         let curr_token_type = self.get_token_type_from_line_index(line_index);
         match (last_real_token_type, curr_token_type) {
+            (_, Some(TT::TextLiteral(TextLiteralKind::MultiLine))) => {
+                // There is necessarily a break within a multiline string
+                // literal so contexts must be updated accordingly
+                self.update_last_matching_context(node, context_matches!(_), |_, data| {
+                    data.is_broken = true;
+                });
+                self.update_operator_precedences(node, true);
+            }
             (_, Some(TT::Comment(CommentKind::InlineBlock | CommentKind::InlineLine))) => {
                 /*
                     When formatting comments, there are some considerations to be made:

core/src/rules/optimising_line_formatter/mod.rs

@@ -1055,6 +1055,19 @@ impl<'this> InternalOptimisingLineFormatter<'this, '_> {
         decision: Decision,
         token_index: Option<usize>,
     ) -> u32 {
+        if let Some((
+            TT::TextLiteral(TextLiteralKind::MultiLine) | TT::Comment(CommentKind::MultilineBlock),
+            token_content,
+        )) = token_index
+            .and_then(|index| self.formatted_tokens.get_token(index))
+            .map(|(token, _)| (token.get_token_type(), token.get_content()))
+        {
+            // Multiline tokens necessarily have a break in them, so the line
+            // length must be calculated.
+            if let Some(last_line) = token_content.lines().skip(1).last() {
+                return last_line.len() as u32;
+            }
+        }
         match (
             decision,
             prev_decision.get(),

core/src/rules/optimising_line_formatter/requirements.rs

@@ -339,18 +339,24 @@ impl InternalOptimisingLineFormatter<'_, '_> {
             }
             (
                 _,
-                Some(TT::Comment(
-                    CommentKind::IndividualLine
-                    | CommentKind::IndividualBlock
-                    | CommentKind::MultilineBlock,
-                )),
+                Some(
+                    TT::Comment(
+                        CommentKind::IndividualLine
+                        | CommentKind::IndividualBlock
+                        | CommentKind::MultilineBlock,
+                    )
+                    | TT::TextLiteral(TextLiteralKind::MultiLine),
+                ),
             ) => Some(DR::MustBreak),
             (
-                Some(TT::Comment(
-                    CommentKind::IndividualLine
-                    | CommentKind::InlineLine
-                    | CommentKind::MultilineBlock,
-                )),
+                Some(
+                    TT::Comment(
+                        CommentKind::IndividualLine
+                        | CommentKind::InlineLine
+                        | CommentKind::MultilineBlock,
+                    )
+                    | TT::TextLiteral(TextLiteralKind::Unterminated),
+                ),
                 _,
             ) => Some(DR::MustBreak),
             (Some(TT::ConditionalDirective(_)), _)