Phase 5: Editor und inkrementellen Compiler implementieren und archivieren

This commit is contained in:
2026-09-06 17:58:26 +02:00
parent df85a4b7b2
commit 9c85349a4d
36 changed files with 2993 additions and 247 deletions

View File

@@ -202,6 +202,7 @@ pub struct Proc {
pub is_static: bool,
pub body: Vec<Stmt>,
pub pos: SourcePos,
pub end_pos: SourcePos,
}
/// `OPEN … FOR modus`.

View File

@@ -0,0 +1,361 @@
//! Quelltextpflege mit dem regulären Lexer und Parser; keine zweite BASIC-Grammatik.
use crate::{
ast::{ProcKind, ProcSig, Stmt, TypeName},
lexer::{self, TokenKind},
parser,
};
pub fn normalize_line(line: &str) -> String {
let mut result = line.to_owned();
for (range, text) in normalization_edits(line).into_iter().rev() {
result.replace_range(range, &text);
}
result
}
pub fn normalization_edits(line: &str) -> Vec<(std::ops::Range<usize>, String)> {
let lexed = lexer::lex(line);
if !lexed.diagnostics.is_empty() {
return Vec::new();
}
let offsets: Vec<_> = line
.char_indices()
.map(|(i, _)| i)
.chain([line.len()])
.collect();
let mut edits = Vec::new();
for token in &lexed.tokens {
if let TokenKind::Kw(kw) = token.kind {
let start = offsets[(token.pos.column as usize - 1).min(offsets.len() - 1)];
let word = format!("{kw:?}").to_uppercase();
let end = start + word.len();
if line
.get(start..end)
.is_some_and(|s| s.eq_ignore_ascii_case(&word))
{
edits.push((start..end, word));
let gap = line[end..]
.chars()
.take_while(|c| matches!(c, ' ' | '\t'))
.count();
// DATA-Nutztext beginnt direkt hinter dem Keyword und bleibt bytegenau.
if gap > 1 && kw != lexer::Kw::Data {
edits.push((end..end + gap, " ".into()));
}
}
}
}
let mut result = line.to_owned();
edits.retain(|(r, t)| &line[r.clone()] != t);
for (range, text) in edits.iter().rev() {
result.replace_range(range.clone(), text);
}
let kinds = |s: &str| {
lexer::lex(s)
.tokens
.into_iter()
.map(|t| t.kind)
.collect::<Vec<_>>()
};
if kinds(line) == kinds(&result) {
edits
} else {
Vec::new()
}
}
pub fn type_text(ty: &TypeName) -> String {
match ty {
TypeName::Str => "STRING".into(),
TypeName::FixedStr(n) => format!("STRING * {n}"),
TypeName::Udt(name) => name.clone(),
other => format!("{other:?}").to_uppercase(),
}
}
pub fn signature(sig: &ProcSig) -> String {
let suffix = |s: Option<lexer::Suffix>| s.map(|s| s.as_char().to_string()).unwrap_or_default();
format!(
"{} {}{} ({})",
if sig.kind == ProcKind::Sub {
"SUB"
} else {
"FUNCTION"
},
sig.name,
suffix(sig.suffix),
sig.params
.iter()
.map(|p| {
format!(
"{}{}{}{}",
p.name,
suffix(p.suffix),
if p.array { "()" } else { "" },
p.as_type
.as_ref()
.map(|t| format!(" AS {}", type_text(t)))
.unwrap_or_default()
)
})
.collect::<Vec<_>>()
.join(", ")
)
}
pub const DECLARE_BEGIN: &str = "' $IDE DECLARE BEGIN";
pub const DECLARE_END: &str = "' $IDE DECLARE END";
/// Bei unsicherem Entwurf bleibt der Text vollständig unverändert; Meldung für die IDE.
pub fn maintain_declarations(source: &str) -> (String, Option<String>) {
maintain_declarations_with(
source,
|clean| {
let lexed = lexer::lex(clean);
let parsed = parser::parse("EDITOR", &lexed.tokens);
if let Some(d) = lexed.diagnostics.first().or(parsed.diagnostics.first()) {
return Err(d.to_string());
}
Ok(parsed.module)
},
|candidate| {
let analysis = crate::analyze_source("EDITOR", candidate);
analysis
.diagnostics
.first()
.map_or(Ok(()), |d| Err(d.to_string()))
},
)
}
/// Der Speicherhook kann die reguläre Projektauflösung samt Includes und Forms verwenden.
pub fn maintain_declarations_with(
source: &str,
resolve: impl Fn(&str) -> Result<crate::ast::Module, String>,
validate: impl Fn(&str) -> Result<(), String>,
) -> (String, Option<String>) {
let skip = |why: String| {
(
source.to_owned(),
Some(format!("DECLARE-Pflege übersprungen: {why}")),
)
};
let lines: Vec<_> = source.split_inclusive('\n').collect();
let begins: Vec<_> = lines
.iter()
.enumerate()
.filter(|(_, l)| l.trim_end() == DECLARE_BEGIN)
.map(|(i, _)| i)
.collect();
let ends: Vec<_> = lines
.iter()
.enumerate()
.filter(|(_, l)| l.trim_end() == DECLARE_END)
.map(|(i, _)| i)
.collect();
let old = match (begins.as_slice(), ends.as_slice()) {
([], []) => None,
([a], [b]) if a < b => Some(*a..b + 1),
_ => return skip("uneindeutiger IDE-Bereich".into()),
};
let clean: String = lines
.iter()
.enumerate()
.filter(|(i, _)| !old.as_ref().is_some_and(|r| r.contains(i)))
.map(|(_, l)| *l)
.collect();
let module = match resolve(&clean) {
Ok(module) => module,
Err(e) => return skip(e),
};
let exports = crate::sema::export_declarations(&module, &[]);
let mut generated = Vec::new();
for stmt in exports {
let Stmt::Declare { sig, .. } = stmt else {
continue;
};
let hands: Vec<_> = module
.body
.iter()
.filter_map(|s| match s {
Stmt::Declare { sig: hand, pos } if hand.name == sig.name => Some((hand, pos)),
_ => None,
})
.collect();
if hands.is_empty() {
generated.push(format!("DECLARE {}", signature(&sig)));
}
}
let mut result = clean.clone();
if !generated.is_empty() {
let nl = if source.contains("\r\n") {
"\r\n"
} else {
"\n"
};
let local = parser::parse("EDITOR", &lexer::lex(&clean).tokens);
let first_line = local.module.procs.first().map_or(1, |p| p.pos.line);
let at = if first_line <= 1 {
0
} else {
clean
.match_indices('\n')
.nth(first_line as usize - 2)
.map_or(clean.len(), |(i, _)| i + 1)
};
result.insert_str(
at,
&format!(
"{DECLARE_BEGIN}{nl}{}{nl}{DECLARE_END}{nl}",
generated.join(nl)
),
);
}
if let Err(e) = validate(&result) {
return skip(e);
}
(result, None)
}
pub fn new_procedure(
source: &str,
at: usize,
name: &str,
function: bool,
) -> Result<String, String> {
let lexed = lexer::lex(name);
if !matches!(
lexed.tokens.as_slice(),
[
lexer::Token {
kind: TokenKind::Ident { suffix: None, .. },
..
},
lexer::Token {
kind: TokenKind::Eol,
..
},
lexer::Token {
kind: TokenKind::Eof,
..
}
]
) {
return Err("Ungültiger Prozedurname".into());
}
if !matches!(&lexed.tokens[0].kind, TokenKind::Ident { name: token_name, .. } if *token_name == name.to_uppercase())
|| !source.is_char_boundary(at)
|| at > source.len()
{
return Err("Ungültiger Prozedurname oder Cursor".into());
}
let source_tokens = lexer::lex(source);
let parsed = parser::parse("EDITOR", &source_tokens.tokens);
if !source_tokens.diagnostics.is_empty() || !parsed.diagnostics.is_empty() {
return Err("Vor Prozedurerzeugung den unvollständigen Quelltext abschließen".into());
}
if !name.as_bytes().first().is_some_and(u8::is_ascii_alphabetic) {
return Err("Prozedurname muss mit ASCII-Buchstaben beginnen".into());
}
let upper = name.to_uppercase();
if parsed.module.procs.iter().any(|p| p.sig.name == upper)
|| parsed
.module
.body
.iter()
.any(|s| matches!(s, Stmt::Declare {sig, ..} if sig.name == upper))
{
return Err("Prozedurname bereits vorhanden".into());
}
let line = source[..at].bytes().filter(|b| *b == b'\n').count() as u32 + 1;
let mut defaults = std::array::from_fn::<_, 26, _>(|_| Some(TypeName::Single));
let current = parsed
.module
.procs
.iter()
.rfind(|p| p.pos.line <= line && line <= p.end_pos.line);
let module_default = |stmt: &Stmt| matches!(stmt, Stmt::DefType { .. });
if current.is_some() {
for stmt in parsed.module.body.iter().filter(|s| module_default(s)) {
if let Stmt::DefType { ty, ranges, .. } = stmt {
for (a, b) in ranges {
for c in *a..=*b {
if c.is_ascii_alphabetic() {
defaults[c.to_ascii_uppercase() as usize - 'A' as usize] =
Some(ty.clone());
}
}
}
}
}
}
let mut statements: Vec<_> = parsed
.module
.body
.iter()
.filter(|_| current.is_none())
.chain(current.into_iter().flat_map(|p| p.body.iter()))
.collect();
statements.sort_by_key(|s| crate::sema::stmt_pos(s).line);
for stmt in statements {
if let Stmt::DefType { ty, ranges, pos } = stmt {
if pos.line <= line {
for (a, b) in ranges {
for c in *a..=*b {
if c.is_ascii_alphabetic() {
defaults[c.to_ascii_uppercase() as usize - 'A' as usize] =
Some(ty.clone());
}
}
}
}
}
}
let kind = if function { "FUNCTION" } else { "SUB" };
let mut result = format!("\n{kind} {name}");
if function {
// Rückgabetyp wird am Header festgehalten.
let ty = defaults[upper.as_bytes()[0] as usize - b'A' as usize]
.as_ref()
.unwrap_or(&TypeName::Single);
result.push(match ty {
TypeName::Integer => '%',
TypeName::Long => '&',
TypeName::Double => '#',
TypeName::Currency => '@',
TypeName::Str => '$',
_ => '!',
});
}
result.push_str(" ()\n");
let mut i = 0;
while i < defaults.len() {
let Some(ty) = &defaults[i] else {
i += 1;
continue;
};
let start = i;
while i + 1 < defaults.len() && defaults[i + 1].as_ref() == Some(ty) {
i += 1;
}
let kw = match ty {
TypeName::Integer => "DEFINT",
TypeName::Long => "DEFLNG",
TypeName::Double => "DEFDBL",
TypeName::Currency => "DEFCUR",
TypeName::Str => "DEFSTR",
_ => "DEFSNG",
};
result.push_str(&format!(
"{kw} {}{}\n",
(b'A' + start as u8) as char,
if start == i {
String::new()
} else {
format!("-{}", (b'A' + i as u8) as char)
}
));
i += 1;
}
result.push_str(&format!("\nEND {kind}\n"));
Ok(result)
}

View File

@@ -673,7 +673,7 @@ pub fn event_params(event: &str) -> Option<&'static [(&'static str, EventParamTy
})
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub struct FormObject {
pub name: String,
pub class: ObjectClass,
@@ -682,7 +682,7 @@ pub struct FormObject {
pub array: bool,
}
#[derive(Debug, Clone, Default)]
#[derive(Debug, Clone, Default, PartialEq)]
pub struct FormCatalog {
pub objects: Vec<FormObject>,
}

View File

@@ -70,3 +70,5 @@ pub fn analyze_source_with_forms(
hir,
}
}
pub mod editing;

View File

@@ -9,6 +9,8 @@ use crate::lexer::{Kw, NumValue, Token, TokenKind};
use crate::{Diagnostic, SourcePos};
pub struct ParseOutput {
/// Ausschließlich fehlende Blockabschlüsse am Dateiende.
pub incomplete: bool,
pub module: Module,
pub diagnostics: Vec<Diagnostic>,
}
@@ -19,6 +21,7 @@ pub fn parse(module_name: &str, tokens: &[Token]) -> ParseOutput {
i: 0,
diags: Vec::new(),
at_line_start: true,
incomplete_errors: 0,
};
let mut body = Vec::new();
let mut procs = Vec::new();
@@ -56,6 +59,7 @@ pub fn parse(module_name: &str, tokens: &[Token]) -> ParseOutput {
}
ParseOutput {
incomplete: !p.diags.is_empty() && p.incomplete_errors == p.diags.len(),
module: Module {
name: module_name.to_string(),
body,
@@ -70,6 +74,7 @@ struct P<'a> {
i: usize,
diags: Vec<Diagnostic>,
at_line_start: bool,
incomplete_errors: usize,
}
impl<'a> P<'a> {
@@ -105,10 +110,27 @@ impl<'a> P<'a> {
}
fn err(&mut self, msg: impl Into<String>) {
let pos = self.pos();
let msg = msg.into();
if matches!(self.k(), TokenKind::Eof)
&& matches!(
msg.as_str(),
"Expected: END SUB"
| "Expected: END FUNCTION"
| "Expected: END IF"
| "Expected: END SELECT"
| "Expected: END TYPE"
| "Expected: END DEF"
| "Expected: NEXT"
| "Expected: LOOP"
| "Expected: WEND"
)
{
self.incomplete_errors += 1;
}
self.diags.push(Diagnostic {
file: None,
pos,
message: msg.into(),
message: msg,
});
}
fn expect_kw(&mut self, kw: Kw, what: &str) -> bool {
@@ -1783,6 +1805,7 @@ impl<'a> P<'a> {
ProcKind::Function => Kw::Function,
};
let body = self.parse_stmt_list(|p: &P| p.at_end_pair(end_kw));
let end_pos = self.pos();
if self.at_end_pair(end_kw) {
self.advance();
self.advance();
@@ -1797,6 +1820,7 @@ impl<'a> P<'a> {
is_static,
body,
pos,
end_pos,
})
}

View File

@@ -7,14 +7,14 @@ pub struct SourceFile {
pub path: String,
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub struct SourceSegment {
pub file: String,
pub first_line: u32,
pub text: String,
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub struct SourceUnit {
pub name: String,
pub segments: Vec<SourceSegment>,

View File

@@ -0,0 +1,95 @@
use tb_frontend::{editing::*, lexer, parser};
#[test]
fn normalization_protects_tokens_and_is_idempotent() {
for source in [
" print Wort$; \"MiX Case\" ' Kommentar bleibt",
"data MiXeD, frei Text,\"Z\": print 1",
"if x% then print x%",
"rem frei text",
"print &Hff; 2.0#",
] {
let normalized = normalize_line(source);
assert_eq!(normalize_line(&normalized), normalized);
let tokens = |s| {
lexer::lex(s)
.tokens
.into_iter()
.map(|t| t.kind)
.collect::<Vec<_>>()
};
assert_eq!(tokens(source), tokens(&normalized));
}
assert_eq!(
normalize_line("print Wert$; \"MiX\" ' Kommentar"),
"PRINT Wert$; \"MiX\" ' Kommentar"
);
assert!(normalize_line("data MiXeD, frei Text").ends_with(" MiXeD, frei Text"));
}
#[test]
fn incomplete_is_parser_context_not_a_second_grammar() {
for source in ["SUB X\n", "IF 1 THEN\n", "FUNCTION X\n"] {
assert!(
parser::parse("M", &lexer::lex(source).tokens).incomplete,
"{source}"
);
}
for source in ["PRINT )\n", "SUB X\nPRINT )\n", "PRINT 1\n"] {
assert!(
!parser::parse("M", &lexer::lex(source).tokens).incomplete,
"{source}"
);
}
}
#[test]
fn declaration_maintenance_and_procedure_generation_are_transactional() {
let source = "DEFINT A-Z\nSUB Work (x)\nx=x+1\nEND SUB\n";
let (updated, notice) = maintain_declarations(source);
assert!(notice.is_none(), "{notice:?}");
assert!(updated.contains("DECLARE SUB WORK (X%)"), "{updated}");
assert_eq!(maintain_declarations(&updated), (updated.clone(), None));
for source in [
"SUB Draft\n",
"PRINT )\n",
"DECLARE SUB Work(x$)\nSUB Work(x%)\nEND SUB\n",
] {
let (same, notice) = maintain_declarations(source);
assert_eq!(same, source);
assert!(notice.is_some(), "{source}");
}
let manual = "DECLARE SUB Work(x AS INTEGER)\nSUB Work(x%)\nEND SUB\n";
assert_eq!(maintain_declarations(manual), (manual.into(), None));
let addition = new_procedure(source, source.len(), "Fresh", true).unwrap();
assert!(addition.contains("FUNCTION Fresh%"));
assert!(
tb_frontend::analyze_source("M", &format!("{source}{addition}"))
.diagnostics
.is_empty()
);
for name in ["Work", "PRINT", "bad name", ""] {
assert!(new_procedure(source, 0, name, false).is_err());
}
}
#[test]
fn deftype_context_does_not_leak_from_another_procedure() {
let source =
"' Modulrumpf\nDEFINT A-Z\nSUB First\nDEFSTR A-Z\nx$=\"x\"\nEND SUB\nSUB Second\nEND SUB\n";
let inside = source.find("x$=").unwrap();
assert!(new_procedure(source, inside, "Fresh", true)
.unwrap()
.contains("FUNCTION Fresh$"));
let second = source.find("SUB Second").unwrap();
assert!(new_procedure(source, second, "Fresh", true)
.unwrap()
.contains("FUNCTION Fresh%"));
let before = new_procedure(source, 0, "Fresh", true).unwrap();
assert!(before.contains("FUNCTION Fresh!"));
assert!(before.contains("DEFSNG A-Z"));
for at in [inside, second, 0] {
let addition = new_procedure(source, at, "Fresh", true).unwrap();
let diagnostics =
tb_frontend::analyze_source("M", &format!("{source}{addition}")).diagnostics;
assert!(diagnostics.is_empty(), "{diagnostics:?}");
}
assert!(new_procedure(source, 0, "Foo ' comment", false).is_err());
}