From b7f0d180ca568dc69864f4fde8db54998f6abb97 Mon Sep 17 00:00:00 2001 From: groobyming Date: Wed, 20 Mar 2024 10:36:56 +0800 Subject: [PATCH 1/2] Extended dialect triat to support numeric prefixed identifiers --- src/dialect/mod.rs | 5 +++++ src/tokenizer.rs | 48 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 2463121e7c..cd3dfc149c 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -147,6 +147,11 @@ pub trait Dialect: Debug + Any { fn supports_named_fn_args_with_eq_operator(&self) -> bool { false } + /// Returns true if the dialect supports identifiers starting with a numeric prefix + /// e.g. Table Name: 59901_user_login + fn supports_numeric_prefix(&self) -> bool { + false + } /// Returns true if the dialect has a CONVERT function which accepts a type first /// and an expression second, e.g. `CONVERT(varchar, 1)` fn convert_type_before_value(&self) -> bool { diff --git a/src/tokenizer.rs b/src/tokenizer.rs index b239d990ed..3e079d650b 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -800,7 +800,9 @@ impl<'a> Tokenizer<'a> { // mysql dialect supports identifiers that start with a numeric prefix, // as long as they aren't an exponent number. - if dialect_of!(self is MySqlDialect | HiveDialect) && exponent_part.is_empty() { + if (dialect_of!(self is MySqlDialect | HiveDialect) || self.dialect.supports_numeric_prefix()) + && exponent_part.is_empty() + { let word = peeking_take_while(chars, |ch| self.dialect.is_identifier_part(ch)); @@ -1518,6 +1520,7 @@ impl<'a: 'b, 'b> Unescape<'a, 'b> { mod tests { use super::*; use crate::dialect::{ClickHouseDialect, MsSqlDialect}; + use core::fmt::Debug; #[test] fn tokenizer_error_impl() { @@ -2386,4 +2389,47 @@ mod tests { check_unescape(r"Hello\0", None); check_unescape(r"Hello\xCADRust", None); } + + #[test] + fn tokenize_numeric_prefix() { + #[derive(Debug)] + struct NumericPrefixDialect; + + impl Dialect for NumericPrefixDialect { + fn is_identifier_start(&self, ch: char) -> bool { + ch.is_ascii_lowercase() + || ch.is_ascii_uppercase() + || ch.is_ascii_digit() + || ch == '$' + } + + fn is_identifier_part(&self, ch: char) -> bool { + ch.is_ascii_lowercase() + || ch.is_ascii_uppercase() + || ch.is_ascii_digit() + || ch == '_' + || ch == '$' + || ch == '{' + || ch == '}' + } + + fn supports_numeric_prefix(&self) -> bool { + true + } + } + + let sql = r#"SELECT * FROM 1"#; + let dialect = GenericDialect {}; + let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap(); + let expected = vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::Mul, + Token::Whitespace(Whitespace::Space), + Token::make_keyword("FROM"), + Token::Whitespace(Whitespace::Space), + Token::Number(String::from("1"), false), + ]; + compare(expected, tokens); + } } From 5d5fe5c5d5756464261cd2800c2f9025fdf2161d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 3 May 2024 14:17:25 -0400 Subject: [PATCH 2/2] Use trait based check --- src/dialect/hive.rs | 4 ++++ src/dialect/mod.rs | 4 ++-- src/dialect/mysql.rs | 4 ++++ src/tokenizer.rs | 14 ++++++-------- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/dialect/hive.rs b/src/dialect/hive.rs index 20800c1d3a..2340df611b 100644 --- a/src/dialect/hive.rs +++ b/src/dialect/hive.rs @@ -38,4 +38,8 @@ impl Dialect for HiveDialect { fn supports_filter_during_aggregation(&self) -> bool { true } + + fn supports_numeric_prefix(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index cb4dbe3f85..a043905700 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -185,8 +185,8 @@ pub trait Dialect: Debug + Any { fn supports_named_fn_args_with_eq_operator(&self) -> bool { false } - /// Returns true if the dialect supports identifiers starting with a numeric prefix - /// e.g. Table Name: 59901_user_login + /// Returns true if the dialect supports identifiers starting with a numeric + /// prefix such as tables named: `59901_user_login` fn supports_numeric_prefix(&self) -> bool { false } diff --git a/src/dialect/mysql.rs b/src/dialect/mysql.rs index f7711b2b01..32525658b9 100644 --- a/src/dialect/mysql.rs +++ b/src/dialect/mysql.rs @@ -53,6 +53,10 @@ impl Dialect for MySqlDialect { true } + fn supports_numeric_prefix(&self) -> bool { + true + } + fn parse_infix( &self, parser: &mut crate::parser::Parser, diff --git a/src/tokenizer.rs b/src/tokenizer.rs index cef10e8cd9..8e3177e9f1 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -35,11 +35,10 @@ use serde::{Deserialize, Serialize}; use sqlparser_derive::{Visit, VisitMut}; use crate::ast::DollarQuotedString; +use crate::dialect::Dialect; use crate::dialect::{ - BigQueryDialect, DuckDbDialect, GenericDialect, HiveDialect, PostgreSqlDialect, - SnowflakeDialect, + BigQueryDialect, DuckDbDialect, GenericDialect, PostgreSqlDialect, SnowflakeDialect, }; -use crate::dialect::{Dialect, MySqlDialect}; use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX}; /// SQL Token enumeration @@ -821,10 +820,7 @@ impl<'a> Tokenizer<'a> { // mysql dialect supports identifiers that start with a numeric prefix, // as long as they aren't an exponent number. - if (dialect_of!(self is MySqlDialect | HiveDialect) - || self.dialect.supports_numeric_prefix()) - && exponent_part.is_empty() - { + if self.dialect.supports_numeric_prefix() && exponent_part.is_empty() { let word = peeking_take_while(chars, |ch| self.dialect.is_identifier_part(ch)); @@ -1547,7 +1543,9 @@ impl<'a: 'b, 'b> Unescape<'a, 'b> { #[cfg(test)] mod tests { use super::*; - use crate::dialect::{BigQueryDialect, ClickHouseDialect, MsSqlDialect}; + use crate::dialect::{ + BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, + }; use core::fmt::Debug; #[test]