817 lines
27 KiB
Rust
817 lines
27 KiB
Rust
//! Lexer: zerlegt Quelltext in Tokens.
|
||
//!
|
||
//! Besonderheiten des Dialekts:
|
||
//! - Typ-Suffixe an Bezeichnern und Literalen (`% & ! # $ @`)
|
||
//! - Zeilennummern und Labels (löst der Parser auf)
|
||
//! - Keywords case-insensitiv; nur echte Sprach-Keywords sind reserviert,
|
||
//! Bibliotheksnamen (`CLS`, `LEFT$` …) bleiben Bezeichner und werden in
|
||
//! der Semantik als Builtins aufgelöst
|
||
//! - `:` trennt Anweisungen, `'` und `REM` leiten Kommentare ein
|
||
//! - Zeilenfortsetzung: `_` als letztes Zeichen nach Leerraum
|
||
//! - Literal-Typisierung nach Sprachreferenz (docs/sprachreferenz.md §1)
|
||
|
||
use crate::{Diagnostic, SourcePos};
|
||
|
||
/// Typ-Suffix eines Bezeichners oder Literals.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||
pub enum Suffix {
|
||
Integer, // %
|
||
Long, // &
|
||
Single, // !
|
||
Double, // #
|
||
Str, // $
|
||
Currency, // @
|
||
}
|
||
|
||
impl Suffix {
|
||
pub fn from_char(c: char) -> Option<Suffix> {
|
||
match c {
|
||
'%' => Some(Suffix::Integer),
|
||
'&' => Some(Suffix::Long),
|
||
'!' => Some(Suffix::Single),
|
||
'#' => Some(Suffix::Double),
|
||
'$' => Some(Suffix::Str),
|
||
'@' => Some(Suffix::Currency),
|
||
_ => None,
|
||
}
|
||
}
|
||
pub fn as_char(self) -> char {
|
||
match self {
|
||
Suffix::Integer => '%',
|
||
Suffix::Long => '&',
|
||
Suffix::Single => '!',
|
||
Suffix::Double => '#',
|
||
Suffix::Str => '$',
|
||
Suffix::Currency => '@',
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Wert eines numerischen Literals, bereits typisiert.
|
||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||
pub enum NumValue {
|
||
Int(i16),
|
||
Long(i32),
|
||
Single(f32),
|
||
Double(f64),
|
||
/// Festkomma ×10 000
|
||
Currency(i64),
|
||
}
|
||
|
||
/// Reservierte Sprach-Keywords (bewusst schlank: Bibliotheksfunktionen
|
||
/// und -anweisungen sind KEINE Keywords, sondern Builtins der Semantik).
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||
pub enum Kw {
|
||
And,
|
||
As,
|
||
Call,
|
||
Case,
|
||
Common,
|
||
Const,
|
||
Data,
|
||
Declare,
|
||
Def,
|
||
DefCur,
|
||
DefDbl,
|
||
DefInt,
|
||
DefLng,
|
||
DefSng,
|
||
DefStr,
|
||
Dim,
|
||
Do,
|
||
Double,
|
||
Else,
|
||
ElseIf,
|
||
End,
|
||
Eqv,
|
||
Erase,
|
||
Error,
|
||
Exit,
|
||
For,
|
||
Function,
|
||
Gosub,
|
||
Goto,
|
||
If,
|
||
Imp,
|
||
Input,
|
||
Integer,
|
||
Is,
|
||
Let,
|
||
Line,
|
||
Local,
|
||
Long,
|
||
Loop,
|
||
Mod,
|
||
Next,
|
||
Not,
|
||
On,
|
||
Option,
|
||
Or,
|
||
Print,
|
||
Read,
|
||
ReDim,
|
||
Rem,
|
||
Restore,
|
||
Resume,
|
||
Return,
|
||
Select,
|
||
Shared,
|
||
Single,
|
||
Static,
|
||
Step,
|
||
Stop,
|
||
String,
|
||
Sub,
|
||
System,
|
||
Then,
|
||
To,
|
||
Type,
|
||
Until,
|
||
Using,
|
||
Wend,
|
||
While,
|
||
Xor,
|
||
Currency,
|
||
// Datei-E/A-Keywords: werden geparst, aber erst in Phase 3 implementiert
|
||
Open,
|
||
Close,
|
||
Write,
|
||
Field,
|
||
Get,
|
||
Put,
|
||
Seek,
|
||
Lset,
|
||
Rset,
|
||
}
|
||
|
||
fn keyword(upper: &str) -> Option<Kw> {
|
||
use Kw::*;
|
||
Some(match upper {
|
||
"AND" => And,
|
||
"AS" => As,
|
||
"CALL" => Call,
|
||
"CASE" => Case,
|
||
"COMMON" => Common,
|
||
"CONST" => Const,
|
||
"CURRENCY" => Currency,
|
||
"DATA" => Data,
|
||
"DECLARE" => Declare,
|
||
"DEF" => Def,
|
||
"DEFCUR" => DefCur,
|
||
"DEFDBL" => DefDbl,
|
||
"DEFINT" => DefInt,
|
||
"DEFLNG" => DefLng,
|
||
"DEFSNG" => DefSng,
|
||
"DEFSTR" => DefStr,
|
||
"DIM" => Dim,
|
||
"DO" => Do,
|
||
"DOUBLE" => Double,
|
||
"ELSE" => Else,
|
||
"ELSEIF" => ElseIf,
|
||
"END" => End,
|
||
"EQV" => Eqv,
|
||
"ERASE" => Erase,
|
||
"ERROR" => Error,
|
||
"EXIT" => Exit,
|
||
"FIELD" => Field,
|
||
"FOR" => For,
|
||
"FUNCTION" => Function,
|
||
"GET" => Get,
|
||
"GOSUB" => Gosub,
|
||
"GOTO" => Goto,
|
||
"IF" => If,
|
||
"IMP" => Imp,
|
||
"INPUT" => Input,
|
||
"INTEGER" => Integer,
|
||
"IS" => Is,
|
||
"LET" => Let,
|
||
"LINE" => Line,
|
||
"LOCAL" => Local,
|
||
"LONG" => Long,
|
||
"LOOP" => Loop,
|
||
"LSET" => Lset,
|
||
"MOD" => Mod,
|
||
"NEXT" => Next,
|
||
"NOT" => Not,
|
||
"ON" => On,
|
||
"OPEN" => Open,
|
||
"OPTION" => Option,
|
||
"OR" => Or,
|
||
"PRINT" => Print,
|
||
"PUT" => Put,
|
||
"READ" => Read,
|
||
"REDIM" => ReDim,
|
||
"REM" => Rem,
|
||
"RESTORE" => Restore,
|
||
"RESUME" => Resume,
|
||
"RETURN" => Return,
|
||
"RSET" => Rset,
|
||
"SEEK" => Seek,
|
||
"SELECT" => Select,
|
||
"SHARED" => Shared,
|
||
"SINGLE" => Single,
|
||
"STATIC" => Static,
|
||
"STEP" => Step,
|
||
"STOP" => Stop,
|
||
"STRING" => String,
|
||
"SUB" => Sub,
|
||
"SYSTEM" => System,
|
||
"THEN" => Then,
|
||
"TO" => To,
|
||
"TYPE" => Type,
|
||
"UNTIL" => Until,
|
||
"USING" => Using,
|
||
"WEND" => Wend,
|
||
"WHILE" => While,
|
||
"WRITE" => Write,
|
||
"XOR" => Xor,
|
||
"CLOSE" => Close,
|
||
_ => return None,
|
||
})
|
||
}
|
||
|
||
#[derive(Debug, Clone, PartialEq)]
|
||
pub enum TokenKind {
|
||
/// Bezeichner; `name` ist bereits in Großschreibung normalisiert.
|
||
Ident {
|
||
name: String,
|
||
suffix: Option<Suffix>,
|
||
},
|
||
Kw(Kw),
|
||
Num(NumValue),
|
||
Str(String),
|
||
Plus,
|
||
Minus,
|
||
Star,
|
||
Slash,
|
||
Backslash,
|
||
Caret,
|
||
Eq,
|
||
Ne,
|
||
Lt,
|
||
Le,
|
||
Gt,
|
||
Ge,
|
||
LParen,
|
||
RParen,
|
||
Comma,
|
||
Semicolon,
|
||
Colon,
|
||
Hash,
|
||
Dot,
|
||
/// Metabefehl `'$INCLUDE: 'datei''` (Pfad; leer = fehlerhafte Syntax).
|
||
MetaInclude(String),
|
||
/// Metabefehle `'$STATIC` / `'$DYNAMIC`.
|
||
MetaStatic,
|
||
MetaDynamic,
|
||
/// Metabefehl `'$FORM`: das Modul ist ein Formularmodul.
|
||
MetaForm,
|
||
/// Rohtext einer `DATA`-Anweisung bis zum Anweisungsende. Er wird
|
||
/// bewusst **nicht** zerlegt oder normalisiert: unquotierte Elemente
|
||
/// behalten ihre Groß- und Kleinschreibung und ihren inneren Leerraum.
|
||
DataRaw(String),
|
||
/// Ende einer logischen Zeile.
|
||
Eol,
|
||
Eof,
|
||
}
|
||
|
||
/// Prüft einen Kommentar auf Metabefehle (`$INCLUDE`, `$STATIC`, `$DYNAMIC`).
|
||
fn meta_token(rest: &[char]) -> Option<TokenKind> {
|
||
let s: String = rest.iter().collect();
|
||
let t = s.trim_start();
|
||
let up = t.to_uppercase();
|
||
if up.starts_with("$STATIC") {
|
||
return Some(TokenKind::MetaStatic);
|
||
}
|
||
if up.starts_with("$DYNAMIC") {
|
||
return Some(TokenKind::MetaDynamic);
|
||
}
|
||
if up.starts_with("$FORM") {
|
||
return Some(TokenKind::MetaForm);
|
||
}
|
||
if up.starts_with("$INCLUDE") {
|
||
if let Some(colon) = t.find(':') {
|
||
let after = t[colon + 1..].trim();
|
||
if let Some(stripped) = after.strip_prefix('\'') {
|
||
if let Some(end) = stripped.find('\'') {
|
||
return Some(TokenKind::MetaInclude(stripped[..end].to_string()));
|
||
}
|
||
}
|
||
}
|
||
return Some(TokenKind::MetaInclude(String::new()));
|
||
}
|
||
None
|
||
}
|
||
|
||
#[derive(Debug, Clone, PartialEq)]
|
||
pub struct Token {
|
||
pub kind: TokenKind,
|
||
pub pos: SourcePos,
|
||
}
|
||
|
||
pub struct LexOutput {
|
||
pub tokens: Vec<Token>,
|
||
pub diagnostics: Vec<Diagnostic>,
|
||
}
|
||
|
||
/// Signifikante Stellen einer Ziffernfolge (führende Nullen zählen nicht).
|
||
fn significant_digits(int_part: &str, frac_part: &str) -> usize {
|
||
let all: String = int_part.chars().chain(frac_part.chars()).collect();
|
||
let trimmed = all.trim_start_matches('0');
|
||
trimmed.len()
|
||
}
|
||
|
||
pub fn lex(source: &str) -> LexOutput {
|
||
let mut tokens: Vec<Token> = Vec::new();
|
||
let mut diagnostics: Vec<Diagnostic> = Vec::new();
|
||
let mut continuation = false;
|
||
|
||
for (line_idx, raw_line) in source.lines().enumerate() {
|
||
let line_no = (line_idx + 1) as u32;
|
||
let chars: Vec<char> = raw_line.chars().collect();
|
||
let mut i = 0usize;
|
||
let mut line_continued = false;
|
||
|
||
'line: while i < chars.len() {
|
||
// Leerraum überspringen
|
||
while i < chars.len() && (chars[i] == ' ' || chars[i] == '\t') {
|
||
i += 1;
|
||
}
|
||
if i >= chars.len() {
|
||
break;
|
||
}
|
||
let start = i;
|
||
let pos = SourcePos {
|
||
line: line_no,
|
||
column: (start + 1) as u32,
|
||
};
|
||
let c = chars[i];
|
||
|
||
// Zeilenfortsetzung: `_` nach Leerraum, danach nur noch Leerraum
|
||
if c == '_'
|
||
&& (start == 0 || chars[start - 1] == ' ' || chars[start - 1] == '\t')
|
||
&& chars[start + 1..].iter().all(|&ch| ch == ' ' || ch == '\t')
|
||
{
|
||
line_continued = true;
|
||
break 'line;
|
||
}
|
||
|
||
match c {
|
||
'\'' => {
|
||
// Kommentar bis Zeilenende; ggf. Metabefehl
|
||
if let Some(tok) = meta_token(&chars[i + 1..]) {
|
||
tokens.push(Token { kind: tok, pos });
|
||
}
|
||
break 'line;
|
||
}
|
||
'"' => {
|
||
i += 1;
|
||
let mut s = String::new();
|
||
let mut closed = false;
|
||
while i < chars.len() {
|
||
if chars[i] == '"' {
|
||
if i + 1 < chars.len() && chars[i + 1] == '"' {
|
||
s.push('"');
|
||
i += 2;
|
||
} else {
|
||
i += 1;
|
||
closed = true;
|
||
break;
|
||
}
|
||
} else {
|
||
s.push(chars[i]);
|
||
i += 1;
|
||
}
|
||
}
|
||
if !closed {
|
||
// Das Vorbild toleriert fehlende schließende
|
||
// Anführungszeichen am Zeilenende.
|
||
}
|
||
tokens.push(Token {
|
||
kind: TokenKind::Str(s),
|
||
pos,
|
||
});
|
||
}
|
||
'&' if i + 1 < chars.len() && matches!(chars[i + 1], 'h' | 'H' | 'o' | 'O') => {
|
||
let hex = matches!(chars[i + 1], 'h' | 'H');
|
||
i += 2;
|
||
let digit_start = i;
|
||
while i < chars.len() && chars[i].is_ascii_alphanumeric() {
|
||
i += 1;
|
||
}
|
||
let digits: String = chars[digit_start..i].iter().collect();
|
||
let long_suffix = i < chars.len() && chars[i] == '&';
|
||
if long_suffix {
|
||
i += 1;
|
||
}
|
||
let radix = if hex { 16 } else { 8 };
|
||
match u32::from_str_radix(&digits, radix) {
|
||
Ok(v) => {
|
||
let kind = if long_suffix {
|
||
TokenKind::Num(NumValue::Long(v as i32))
|
||
} else if v <= 0xFFFF {
|
||
TokenKind::Num(NumValue::Int(v as u16 as i16))
|
||
} else {
|
||
diagnostics.push(Diagnostic {
|
||
pos,
|
||
message: "Overflow".into(),
|
||
});
|
||
TokenKind::Num(NumValue::Long(v as i32))
|
||
};
|
||
tokens.push(Token { kind, pos });
|
||
}
|
||
Err(_) => diagnostics.push(Diagnostic {
|
||
pos,
|
||
message: "Syntax error".into(),
|
||
}),
|
||
}
|
||
}
|
||
'0'..='9' | '.'
|
||
if c != '.' || (i + 1 < chars.len() && chars[i + 1].is_ascii_digit()) =>
|
||
{
|
||
let int_start = i;
|
||
while i < chars.len() && chars[i].is_ascii_digit() {
|
||
i += 1;
|
||
}
|
||
let int_part: String = chars[int_start..i].iter().collect();
|
||
let mut frac_part = String::new();
|
||
let mut has_point = false;
|
||
if i < chars.len() && chars[i] == '.' {
|
||
has_point = true;
|
||
i += 1;
|
||
let fs = i;
|
||
while i < chars.len() && chars[i].is_ascii_digit() {
|
||
i += 1;
|
||
}
|
||
frac_part = chars[fs..i].iter().collect();
|
||
}
|
||
// Exponent E/D
|
||
let mut exp_kind: Option<char> = None;
|
||
let mut exp_str = String::new();
|
||
if i < chars.len() && matches!(chars[i], 'e' | 'E' | 'd' | 'D') {
|
||
let save = i;
|
||
let k = chars[i].to_ascii_uppercase();
|
||
let mut j = i + 1;
|
||
let mut e = String::new();
|
||
if j < chars.len() && (chars[j] == '+' || chars[j] == '-') {
|
||
e.push(chars[j]);
|
||
j += 1;
|
||
}
|
||
let ds = j;
|
||
while j < chars.len() && chars[j].is_ascii_digit() {
|
||
j += 1;
|
||
}
|
||
if j > ds {
|
||
for ch in &chars[ds..j] {
|
||
e.push(*ch);
|
||
}
|
||
exp_kind = Some(k);
|
||
exp_str = e;
|
||
i = j;
|
||
} else {
|
||
i = save; // kein Exponent (z. B. Variable `e`)
|
||
}
|
||
}
|
||
// Suffix
|
||
let suffix = if i < chars.len() {
|
||
Suffix::from_char(chars[i])
|
||
} else {
|
||
None
|
||
};
|
||
if suffix.is_some() {
|
||
i += 1;
|
||
}
|
||
let text = format!(
|
||
"{}{}{}{}",
|
||
int_part,
|
||
if has_point { "." } else { "" },
|
||
frac_part,
|
||
match exp_kind {
|
||
Some(_) => format!("e{exp_str}"),
|
||
None => String::new(),
|
||
}
|
||
);
|
||
let dval: f64 = text.parse().unwrap_or(0.0);
|
||
let value = match (suffix, exp_kind) {
|
||
(Some(Suffix::Integer), _) => {
|
||
if dval > i16::MAX as f64 || dval < i16::MIN as f64 {
|
||
diagnostics.push(Diagnostic {
|
||
pos,
|
||
message: "Overflow".into(),
|
||
});
|
||
}
|
||
NumValue::Int(dval as i16)
|
||
}
|
||
(Some(Suffix::Long), _) => {
|
||
if dval > i32::MAX as f64 || dval < i32::MIN as f64 {
|
||
diagnostics.push(Diagnostic {
|
||
pos,
|
||
message: "Overflow".into(),
|
||
});
|
||
}
|
||
NumValue::Long(dval as i32)
|
||
}
|
||
(Some(Suffix::Single), _) => NumValue::Single(dval as f32),
|
||
(Some(Suffix::Double), _) => NumValue::Double(dval),
|
||
(Some(Suffix::Currency), _) => {
|
||
NumValue::Currency((dval * 10_000.0).round() as i64)
|
||
}
|
||
(Some(Suffix::Str), _) => {
|
||
diagnostics.push(Diagnostic {
|
||
pos,
|
||
message: "Syntax error".into(),
|
||
});
|
||
NumValue::Double(dval)
|
||
}
|
||
(None, Some('D')) => NumValue::Double(dval),
|
||
(None, Some(_)) => NumValue::Single(dval as f32),
|
||
(None, None) => {
|
||
if !has_point {
|
||
// Ganzzahl: INTEGER → LONG → Gleitkomma
|
||
if let Ok(v) = text.parse::<i64>() {
|
||
if let Ok(v16) = i16::try_from(v) {
|
||
NumValue::Int(v16)
|
||
} else if let Ok(v32) = i32::try_from(v) {
|
||
NumValue::Long(v32)
|
||
} else {
|
||
NumValue::Double(dval)
|
||
}
|
||
} else {
|
||
NumValue::Double(dval)
|
||
}
|
||
} else {
|
||
// Entscheidung (2026-09-02, siehe
|
||
// Sprachreferenz §1): > 7 signifikante
|
||
// Stellen → DOUBLE, sonst SINGLE.
|
||
if significant_digits(&int_part, &frac_part) > 7 {
|
||
NumValue::Double(dval)
|
||
} else {
|
||
NumValue::Single(dval as f32)
|
||
}
|
||
}
|
||
}
|
||
};
|
||
tokens.push(Token {
|
||
kind: TokenKind::Num(value),
|
||
pos,
|
||
});
|
||
}
|
||
c if c.is_alphabetic() => {
|
||
i += 1;
|
||
while i < chars.len()
|
||
&& (chars[i].is_alphanumeric() || chars[i] == '.' || chars[i] == '_')
|
||
{
|
||
i += 1;
|
||
}
|
||
let mut name: String =
|
||
chars[start..i].iter().collect::<String>().to_uppercase();
|
||
let container_bang = i < chars.len()
|
||
&& chars[i] == '!'
|
||
&& i + 1 < chars.len()
|
||
&& chars[i + 1].is_alphabetic();
|
||
let suffix = if i < chars.len() && !container_bang {
|
||
Suffix::from_char(chars[i])
|
||
} else {
|
||
None
|
||
};
|
||
if suffix.is_some() {
|
||
i += 1;
|
||
}
|
||
if container_bang {
|
||
name.push('!');
|
||
i += 1;
|
||
while i < chars.len()
|
||
&& (chars[i].is_alphanumeric() || chars[i] == '.' || chars[i] == '_')
|
||
{
|
||
name.push(chars[i].to_ascii_uppercase());
|
||
i += 1;
|
||
}
|
||
}
|
||
if suffix.is_none() {
|
||
if let Some(kw) = keyword(&name) {
|
||
if kw == Kw::Rem {
|
||
// REM: Rest ist Kommentar; ggf. Metabefehl
|
||
if let Some(tok) = meta_token(&chars[i..]) {
|
||
tokens.push(Token { kind: tok, pos });
|
||
}
|
||
break 'line;
|
||
}
|
||
tokens.push(Token {
|
||
kind: TokenKind::Kw(kw),
|
||
pos,
|
||
});
|
||
if kw == Kw::Data {
|
||
// Der Rest der Anweisung ist Rohtext: bis zum
|
||
// `:` außerhalb von Anführungszeichen oder bis
|
||
// zum Zeilenende.
|
||
let start = i;
|
||
let mut in_quote = false;
|
||
while i < chars.len() {
|
||
match chars[i] {
|
||
'"' => in_quote = !in_quote,
|
||
':' if !in_quote => break,
|
||
'\'' if !in_quote => break,
|
||
_ => {}
|
||
}
|
||
i += 1;
|
||
}
|
||
let roh: String = chars[start..i].iter().collect();
|
||
tokens.push(Token {
|
||
kind: TokenKind::DataRaw(roh),
|
||
pos,
|
||
});
|
||
}
|
||
continue;
|
||
}
|
||
}
|
||
// Bezeichner dürfen nicht mit '.' enden (a.b. → a.b + .)
|
||
while name.ends_with('.') {
|
||
name.pop();
|
||
i -= 1;
|
||
}
|
||
tokens.push(Token {
|
||
kind: TokenKind::Ident { name, suffix },
|
||
pos,
|
||
});
|
||
}
|
||
_ => {
|
||
i += 1;
|
||
let kind = match c {
|
||
'+' => TokenKind::Plus,
|
||
'-' => TokenKind::Minus,
|
||
'*' => TokenKind::Star,
|
||
'/' => TokenKind::Slash,
|
||
'\\' => TokenKind::Backslash,
|
||
'^' => TokenKind::Caret,
|
||
'=' => TokenKind::Eq,
|
||
'(' => TokenKind::LParen,
|
||
')' => TokenKind::RParen,
|
||
',' => TokenKind::Comma,
|
||
';' => TokenKind::Semicolon,
|
||
':' => TokenKind::Colon,
|
||
'#' => TokenKind::Hash,
|
||
'.' => TokenKind::Dot,
|
||
'?' => TokenKind::Kw(Kw::Print), // Editor-Kurzform
|
||
'&' => TokenKind::Kw(Kw::Long), // isoliertes & (selten)
|
||
'<' => {
|
||
if i < chars.len() && chars[i] == '=' {
|
||
i += 1;
|
||
TokenKind::Le
|
||
} else if i < chars.len() && chars[i] == '>' {
|
||
i += 1;
|
||
TokenKind::Ne
|
||
} else {
|
||
TokenKind::Lt
|
||
}
|
||
}
|
||
'>' => {
|
||
if i < chars.len() && chars[i] == '=' {
|
||
i += 1;
|
||
TokenKind::Ge
|
||
} else {
|
||
TokenKind::Gt
|
||
}
|
||
}
|
||
other => {
|
||
diagnostics.push(Diagnostic {
|
||
pos,
|
||
message: format!("Syntax error ('{other}')"),
|
||
});
|
||
continue;
|
||
}
|
||
};
|
||
tokens.push(Token { kind, pos });
|
||
}
|
||
}
|
||
}
|
||
|
||
if line_continued {
|
||
continuation = true;
|
||
} else {
|
||
if !continuation || !tokens.is_empty() {
|
||
tokens.push(Token {
|
||
kind: TokenKind::Eol,
|
||
pos: SourcePos {
|
||
line: line_no,
|
||
column: (chars.len() + 1) as u32,
|
||
},
|
||
});
|
||
}
|
||
continuation = false;
|
||
}
|
||
}
|
||
|
||
tokens.push(Token {
|
||
kind: TokenKind::Eof,
|
||
pos: SourcePos {
|
||
line: (source.lines().count() + 1) as u32,
|
||
column: 1,
|
||
},
|
||
});
|
||
LexOutput {
|
||
tokens,
|
||
diagnostics,
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
fn kinds(src: &str) -> Vec<TokenKind> {
|
||
lex(src).tokens.into_iter().map(|t| t.kind).collect()
|
||
}
|
||
|
||
#[test]
|
||
fn keywords_case_insensitiv() {
|
||
let k = kinds("print If tHeN");
|
||
assert_eq!(k[0], TokenKind::Kw(Kw::Print));
|
||
assert_eq!(k[1], TokenKind::Kw(Kw::If));
|
||
assert_eq!(k[2], TokenKind::Kw(Kw::Then));
|
||
}
|
||
|
||
#[test]
|
||
fn suffix_macht_keyword_zum_bezeichner() {
|
||
// STRING$ ist die Bibliotheksfunktion, STRING das Typ-Keyword
|
||
let k = kinds("STRING$ STRING");
|
||
assert_eq!(
|
||
k[0],
|
||
TokenKind::Ident {
|
||
name: "STRING".into(),
|
||
suffix: Some(Suffix::Str)
|
||
}
|
||
);
|
||
assert_eq!(k[1], TokenKind::Kw(Kw::String));
|
||
}
|
||
|
||
#[test]
|
||
fn literal_typisierung() {
|
||
assert_eq!(kinds("42")[0], TokenKind::Num(NumValue::Int(42)));
|
||
assert_eq!(kinds("40000")[0], TokenKind::Num(NumValue::Long(40000)));
|
||
assert_eq!(kinds("1.5")[0], TokenKind::Num(NumValue::Single(1.5)));
|
||
assert_eq!(
|
||
kinds("3.14159265")[0],
|
||
TokenKind::Num(NumValue::Double(3.14159265))
|
||
);
|
||
assert_eq!(kinds("1E3")[0], TokenKind::Num(NumValue::Single(1000.0)));
|
||
assert_eq!(kinds("1D3")[0], TokenKind::Num(NumValue::Double(1000.0)));
|
||
assert_eq!(kinds("2.5@")[0], TokenKind::Num(NumValue::Currency(25000)));
|
||
assert_eq!(kinds("&HFF")[0], TokenKind::Num(NumValue::Int(255)));
|
||
assert_eq!(kinds("&HFFFF")[0], TokenKind::Num(NumValue::Int(-1)));
|
||
assert_eq!(kinds("&HFFFF&")[0], TokenKind::Num(NumValue::Long(0xFFFF)));
|
||
assert_eq!(kinds("&O777")[0], TokenKind::Num(NumValue::Int(511)));
|
||
}
|
||
|
||
#[test]
|
||
fn strings_mit_doppelten_anfuehrungszeichen() {
|
||
assert_eq!(kinds("\"a\"\"b\"")[0], TokenKind::Str("a\"b".into()));
|
||
}
|
||
|
||
#[test]
|
||
fn kommentare_und_rem() {
|
||
let k = kinds("PRINT 1 ' Kommentar\nREM ganze Zeile\nPRINT 2");
|
||
// PRINT 1 EOL EOL PRINT 2 EOL EOF
|
||
assert_eq!(k.len(), 8);
|
||
assert_eq!(k[2], TokenKind::Eol);
|
||
assert_eq!(k[3], TokenKind::Eol);
|
||
}
|
||
|
||
#[test]
|
||
fn zeilenfortsetzung() {
|
||
let k = kinds("PRINT 1, _\n 2");
|
||
// Kein Eol zwischen 1, und 2
|
||
assert!(matches!(k[3], TokenKind::Num(NumValue::Int(2))));
|
||
}
|
||
|
||
#[test]
|
||
fn bezeichner_mit_punkt_und_unterstrich() {
|
||
let k = kinds("kunde.name_2$");
|
||
assert_eq!(
|
||
k[0],
|
||
TokenKind::Ident {
|
||
name: "KUNDE.NAME_2".into(),
|
||
suffix: Some(Suffix::Str)
|
||
}
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn bang_ist_suffix_oder_containeroperator_nach_folgetoken() {
|
||
let k = kinds("Wert! = 1.5\nForm1!Text1.Text = \"a\"");
|
||
assert_eq!(
|
||
k[0],
|
||
TokenKind::Ident {
|
||
name: "WERT".into(),
|
||
suffix: Some(Suffix::Single)
|
||
}
|
||
);
|
||
assert_eq!(
|
||
k[4],
|
||
TokenKind::Ident {
|
||
name: "FORM1!TEXT1.TEXT".into(),
|
||
suffix: None
|
||
}
|
||
);
|
||
}
|
||
}
|