This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-sqlparser-rs.git
The following commit(s) were added to refs/heads/main by this push:
new ee3b6223 Tokenize empty line comments correctly (#2161)
ee3b6223 is described below
commit ee3b6223e673b9c0f737d649a7f934de0186bc64
Author: Louis Vialar <[email protected]>
AuthorDate: Tue Jan 13 11:18:32 2026 +0100
Tokenize empty line comments correctly (#2161)
---
src/tokenizer.rs | 24 +++++++++++++++++++++++-
1 file changed, 23 insertions(+), 1 deletion(-)
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index eb935a4f..a9f9fb44 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -1387,7 +1387,11 @@ impl<'a> Tokenizer<'a> {
Some('-') => {
let mut is_comment = true;
if
self.dialect.requires_single_line_comment_whitespace() {
- is_comment = Some(' ') ==
chars.peekable.clone().nth(1);
+ is_comment = chars
+ .peekable
+ .clone()
+ .nth(1)
+ .is_some_and(char::is_whitespace);
}
if is_comment {
@@ -4069,6 +4073,24 @@ mod tests {
Token::Minus,
],
);
+
+ all_dialects_where(|d|
d.requires_single_line_comment_whitespace()).tokenizes_to(
+ "--\n-- Table structure for table...\n--\n",
+ vec![
+ Token::Whitespace(Whitespace::SingleLineComment {
+ prefix: "--".to_string(),
+ comment: "\n".to_string(),
+ }),
+ Token::Whitespace(Whitespace::SingleLineComment {
+ prefix: "--".to_string(),
+ comment: " Table structure for table...\n".to_string(),
+ }),
+ Token::Whitespace(Whitespace::SingleLineComment {
+ prefix: "--".to_string(),
+ comment: "\n".to_string(),
+ }),
+ ],
+ );
}
#[test]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
