From c1dd27dd73a41677b7f8d1632bfd3c773b9f6854 Mon Sep 17 00:00:00 2001 From: Alex Dukhno Date: Sun, 30 Aug 2020 17:01:00 +0300 Subject: [PATCH 1/4] added parsing for PostgreSQL operations --- src/ast/mod.rs | 14 ++++- src/ast/operator.rs | 18 ++++++ src/parser.rs | 40 +++++++++++++- src/tokenizer.rs | 106 +++++++++++++++++++++++++++++++++++- tests/sqlparser_common.rs | 9 ++- tests/sqlparser_postgres.rs | 77 ++++++++++++++++++++++++++ 6 files changed, 257 insertions(+), 7 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 7b143349bb..862d77d410 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -191,7 +191,11 @@ pub enum Expr { right: Box, }, /// Unary operation e.g. `NOT foo` - UnaryOp { op: UnaryOperator, expr: Box }, + UnaryOp { + op: UnaryOperator, + expr: Box, + infix: bool, + }, /// CAST an expression to a different data type e.g. `CAST(foo AS VARCHAR(123))` Cast { expr: Box, @@ -282,7 +286,13 @@ impl fmt::Display for Expr { high ), Expr::BinaryOp { left, op, right } => write!(f, "{} {} {}", left, op, right), - Expr::UnaryOp { op, expr } => write!(f, "{} {}", op, expr), + Expr::UnaryOp { op, expr, infix } => { + if *infix { + write!(f, "{}{}", expr, op) + } else { + write!(f, "{} {}", op, expr) + } + } Expr::Cast { expr, data_type } => write!(f, "CAST({} AS {})", expr, data_type), Expr::Extract { field, expr } => write!(f, "EXTRACT({} FROM {})", field, expr), Expr::Collate { expr, collation } => write!(f, "{} COLLATE {}", expr, collation), diff --git a/src/ast/operator.rs b/src/ast/operator.rs index 63e75eead7..afcd39f0f2 100644 --- a/src/ast/operator.rs +++ b/src/ast/operator.rs @@ -21,6 +21,12 @@ pub enum UnaryOperator { Plus, Minus, Not, + PGBitwiseNot, + PGSqrt, + PGCbrt, + PGFactorial, + PGInfixFactorial, + PGAbs, } impl fmt::Display for UnaryOperator { @@ -29,6 +35,12 @@ impl fmt::Display for UnaryOperator { UnaryOperator::Plus => "+", UnaryOperator::Minus => "-", UnaryOperator::Not => "NOT", + UnaryOperator::PGBitwiseNot => "~", + UnaryOperator::PGSqrt => "|/", + UnaryOperator::PGCbrt => "||/", + UnaryOperator::PGFactorial => "!", + UnaryOperator::PGInfixFactorial => "!!", + UnaryOperator::PGAbs => "@", }) } } @@ -56,6 +68,9 @@ pub enum BinaryOperator { BitwiseOr, BitwiseAnd, BitwiseXor, + PGBitwiseXor, + PGBitwiseShiftLeft, + PGBitwiseShiftRight, } impl fmt::Display for BinaryOperator { @@ -80,6 +95,9 @@ impl fmt::Display for BinaryOperator { BinaryOperator::BitwiseOr => "|", BinaryOperator::BitwiseAnd => "&", BinaryOperator::BitwiseXor => "^", + BinaryOperator::PGBitwiseXor => "#", + BinaryOperator::PGBitwiseShiftLeft => "<<", + BinaryOperator::PGBitwiseShiftRight => ">>", }) } } diff --git a/src/parser.rs b/src/parser.rs index 431984a198..2b14c7ff4e 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -250,6 +250,7 @@ impl<'a> Parser<'a> { Keyword::NOT => Ok(Expr::UnaryOp { op: UnaryOperator::Not, expr: Box::new(self.parse_subexpr(Self::UNARY_NOT_PREC)?), + infix: false, }), // Here `w` is a word, check if it's a part of a multi-part // identifier, a function call, or a simple identifier: @@ -283,6 +284,31 @@ impl<'a> Parser<'a> { }, }, // End of Token::Word Token::Mult => Ok(Expr::Wildcard), + Token::Tilde => Ok(Expr::UnaryOp { + op: UnaryOperator::PGBitwiseNot, + expr: Box::new(self.parse_subexpr(0)?), + infix: false, + }), + Token::DoubleExclamationMark => Ok(Expr::UnaryOp { + op: UnaryOperator::PGInfixFactorial, + expr: Box::new(self.parse_subexpr(0)?), + infix: false, + }), + Token::SquareRoot => Ok(Expr::UnaryOp { + op: UnaryOperator::PGSqrt, + expr: Box::new(self.parse_subexpr(0)?), + infix: false, + }), + Token::CubeRoot => Ok(Expr::UnaryOp { + op: UnaryOperator::PGCbrt, + expr: Box::new(self.parse_subexpr(0)?), + infix: false, + }), + Token::Ampersat => Ok(Expr::UnaryOp { + op: UnaryOperator::PGAbs, + expr: Box::new(self.parse_subexpr(0)?), + infix: false, + }), tok @ Token::Minus | tok @ Token::Plus => { let op = if tok == Token::Plus { UnaryOperator::Plus @@ -292,6 +318,7 @@ impl<'a> Parser<'a> { Ok(Expr::UnaryOp { op, expr: Box::new(self.parse_subexpr(Self::PLUS_MINUS_PREC)?), + infix: false, }) } Token::Number(_) @@ -658,6 +685,9 @@ impl<'a> Parser<'a> { Token::Caret => Some(BinaryOperator::BitwiseXor), Token::Ampersand => Some(BinaryOperator::BitwiseAnd), Token::Div => Some(BinaryOperator::Divide), + Token::ShiftLeft => Some(BinaryOperator::PGBitwiseShiftLeft), + Token::ShiftRight => Some(BinaryOperator::PGBitwiseShiftRight), + Token::Sharp => Some(BinaryOperator::PGBitwiseXor), Token::Word(w) => match w.keyword { Keyword::AND => Some(BinaryOperator::And), Keyword::OR => Some(BinaryOperator::Or), @@ -707,6 +737,13 @@ impl<'a> Parser<'a> { } } else if Token::DoubleColon == tok { self.parse_pg_cast(expr) + } else if Token::ExclamationMark == tok { + // PostgreSQL factorial operation + Ok(Expr::UnaryOp { + op: UnaryOperator::PGFactorial, + expr: Box::new(expr), + infix: true, + }) } else { // Can only happen if `get_next_precedence` got out of sync with this function panic!("No infix parser for token {:?}", tok) @@ -785,11 +822,12 @@ impl<'a> Parser<'a> { Token::Word(w) if w.keyword == Keyword::LIKE => Ok(Self::BETWEEN_PREC), Token::Eq | Token::Lt | Token::LtEq | Token::Neq | Token::Gt | Token::GtEq => Ok(20), Token::Pipe => Ok(21), - Token::Caret => Ok(22), + Token::Caret | Token::Sharp | Token::ShiftRight | Token::ShiftLeft => Ok(22), Token::Ampersand => Ok(23), Token::Plus | Token::Minus => Ok(Self::PLUS_MINUS_PREC), Token::Mult | Token::Div | Token::Mod | Token::StringConcat => Ok(40), Token::DoubleColon => Ok(50), + Token::ExclamationMark => Ok(50), _ => Ok(0), } } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index f4587f3211..a2cea74c63 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -54,7 +54,7 @@ pub enum Token { Neq, /// Less Than operator `<` Lt, - /// Greater han operator `>` + /// Greater Than operator `>` Gt, /// Less Than Or Equals operator `<=` LtEq, @@ -102,6 +102,24 @@ pub enum Token { RBrace, /// Right Arrow `=>` RArrow, + /// Sharp `#` use for PostgreSQL Bitwise XOR operator + Sharp, + /// Tilde `~` use for PostgreSQL Bitwise NOT operator + Tilde, + /// Bitwise left operator `<<` use for PostgreSQL + ShiftLeft, + /// Bitwise right operator `>>` use for PostgreSQL + ShiftRight, + /// Exclamation Mark `!` use for PostgreSQL factorial operator + ExclamationMark, + /// Exclamation Mark `!!` use for PostgreSQL prefix factorial operator + DoubleExclamationMark, + /// Ampersat `@` use for PostgreSQL abs operator + Ampersat, + /// PostgreSQL square root math operator + SquareRoot, + /// PostgreSQL cube root math operator + CubeRoot, } impl fmt::Display for Token { @@ -143,6 +161,15 @@ impl fmt::Display for Token { Token::LBrace => f.write_str("{"), Token::RBrace => f.write_str("}"), Token::RArrow => f.write_str("=>"), + Token::Sharp => f.write_str("#"), + Token::ExclamationMark => f.write_str("!"), + Token::DoubleExclamationMark => f.write_str("!!"), + Token::Tilde => f.write_str("~"), + Token::Ampersat => f.write_str("@"), + Token::ShiftLeft => f.write_str("<<"), + Token::ShiftRight => f.write_str(">>"), + Token::SquareRoot => f.write_str("|/"), + Token::CubeRoot => f.write_str("||/"), } } } @@ -406,7 +433,14 @@ impl<'a> Tokenizer<'a> { '|' => { chars.next(); // consume the '|' match chars.peek() { - Some('|') => self.consume_and_return(chars, Token::StringConcat), + Some('/') => self.consume_and_return(chars, Token::SquareRoot), + Some('|') => { + chars.next(); // consume the second '|' + match chars.peek() { + Some('/') => self.consume_and_return(chars, Token::CubeRoot), + _ => Ok(Some(Token::StringConcat)), + } + } // Bitshift '|' operator _ => Ok(Some(Token::Pipe)), } @@ -423,7 +457,8 @@ impl<'a> Tokenizer<'a> { chars.next(); // consume match chars.peek() { Some('=') => self.consume_and_return(chars, Token::Neq), - _ => self.tokenizer_error("Expected to see '=' after '!' character"), + Some('!') => self.consume_and_return(chars, Token::DoubleExclamationMark), + _ => Ok(Some(Token::ExclamationMark)), } } '<' => { @@ -431,6 +466,7 @@ impl<'a> Tokenizer<'a> { match chars.peek() { Some('=') => self.consume_and_return(chars, Token::LtEq), Some('>') => self.consume_and_return(chars, Token::Neq), + Some('<') => self.consume_and_return(chars, Token::ShiftLeft), _ => Ok(Some(Token::Lt)), } } @@ -438,6 +474,7 @@ impl<'a> Tokenizer<'a> { chars.next(); // consume match chars.peek() { Some('=') => self.consume_and_return(chars, Token::GtEq), + Some('>') => self.consume_and_return(chars, Token::ShiftRight), _ => Ok(Some(Token::Gt)), } } @@ -464,6 +501,9 @@ impl<'a> Tokenizer<'a> { comment, }))) } + '~' => self.consume_and_return(chars, Token::Tilde), + '#' => self.consume_and_return(chars, Token::Sharp), + '@' => self.consume_and_return(chars, Token::Ampersat), other => self.consume_and_return(chars, Token::Char(other)), }, None => Ok(None), @@ -586,6 +626,7 @@ mod tests { use super::super::dialect::GenericDialect; use super::super::dialect::MsSqlDialect; use super::*; + use crate::dialect::PostgreSqlDialect; #[test] fn tokenize_select_1() { @@ -958,6 +999,65 @@ mod tests { compare(expected, tokens); } + #[test] + fn tokenize_postgresql_bitwise_operations() { + let sql = String::from("SELECT ~one << two # three >> four"); + let dialect = PostgreSqlDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, &sql); + let tokens = tokenizer.tokenize().unwrap(); + + let expected = vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::Tilde, + Token::make_word("one", None), + Token::Whitespace(Whitespace::Space), + Token::ShiftLeft, + Token::Whitespace(Whitespace::Space), + Token::make_word("two", None), + Token::Whitespace(Whitespace::Space), + Token::Sharp, + Token::Whitespace(Whitespace::Space), + Token::make_word("three", None), + Token::Whitespace(Whitespace::Space), + Token::ShiftRight, + Token::Whitespace(Whitespace::Space), + Token::make_word("four", None), + ]; + + compare(expected, tokens); + } + + #[test] + fn tokenize_postgresql_math_operations() { + let sql = String::from("SELECT !!5 5! @-6 |/4 ||/8"); + let dialect = PostgreSqlDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, &sql); + let tokens = tokenizer.tokenize().unwrap(); + + let expected = vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::DoubleExclamationMark, + Token::Number("5".to_string()), + Token::Whitespace(Whitespace::Space), + Token::Number("5".to_string()), + Token::ExclamationMark, + Token::Whitespace(Whitespace::Space), + Token::Ampersat, + Token::Minus, + Token::Number("6".to_string()), + Token::Whitespace(Whitespace::Space), + Token::SquareRoot, + Token::Number("4".to_string()), + Token::Whitespace(Whitespace::Space), + Token::CubeRoot, + Token::Number("8".to_string()), + ]; + + compare(expected, tokens); + } + fn compare(expected: Vec, actual: Vec) { //println!("------------------------------"); //println!("tokens = {:?}", actual); diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 0c1cea5af1..976bd651d1 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -343,7 +343,8 @@ fn parse_select_count_distinct() { name: ObjectName(vec![Ident::new("COUNT")]), args: vec![FunctionArg::Unnamed(Expr::UnaryOp { op: UnaryOperator::Plus, - expr: Box::new(Expr::Identifier(Ident::new("x"))) + expr: Box::new(Expr::Identifier(Ident::new("x"))), + infix: false, })], over: None, distinct: true, @@ -506,11 +507,13 @@ fn parse_unary_math() { left: Box::new(UnaryOp { op: UnaryOperator::Minus, expr: Box::new(Identifier(Ident::new("a"))), + infix: false, }), op: BinaryOperator::Plus, right: Box::new(UnaryOp { op: UnaryOperator::Minus, expr: Box::new(Identifier(Ident::new("b"))), + infix: false, }), }, verified_expr(sql) @@ -565,6 +568,7 @@ fn parse_not_precedence() { high: Box::new(Expr::Value(number("2"))), negated: true, }), + infix: false, }, ); @@ -579,6 +583,7 @@ fn parse_not_precedence() { op: BinaryOperator::NotLike, right: Box::new(Expr::Value(Value::SingleQuotedString("b".into()))), }), + infix: false, }, ); @@ -593,6 +598,7 @@ fn parse_not_precedence() { list: vec![Expr::Value(Value::SingleQuotedString("a".into()))], negated: true, }), + infix: false, }, ); } @@ -2604,6 +2610,7 @@ fn parse_exists_subquery() { Expr::UnaryOp { op: UnaryOperator::Not, expr: Box::new(Expr::Exists(Box::new(expected_inner))), + infix: false, }, select.selection.unwrap(), ); diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 2b950c0d39..2e60901896 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -551,6 +551,83 @@ fn parse_prepare() { ); } +#[test] +fn parse_pg_bitwise_binary_ops() { + let bitwise_ops = &[ + ("#", BinaryOperator::PGBitwiseXor), + (">>", BinaryOperator::PGBitwiseShiftRight), + ("<<", BinaryOperator::PGBitwiseShiftLeft), + ]; + + for (str_op, op) in bitwise_ops { + let select = pg().verified_only_select(&format!("SELECT a {} b", &str_op)); + assert_eq!( + SelectItem::UnnamedExpr(Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("a"))), + op: op.clone(), + right: Box::new(Expr::Identifier(Ident::new("b"))), + }), + select.projection[0] + ); + } +} + +#[test] +fn parse_pg_bitwise_unary_ops() { + let bitwise_ops = &[("~", UnaryOperator::PGBitwiseNot)]; + + for (str_op, op) in bitwise_ops { + let select = pg().verified_only_select(&format!("SELECT {} a", &str_op)); + assert_eq!( + SelectItem::UnnamedExpr(Expr::UnaryOp { + op: op.clone(), + expr: Box::new(Expr::Identifier(Ident::new("a"))), + infix: false, + }), + select.projection[0] + ); + } +} + +#[test] +fn parse_pg_infix_math_ops() { + let bitwise_ops = &[ + ("|/", UnaryOperator::PGSqrt), + ("||/", UnaryOperator::PGCbrt), + ("!!", UnaryOperator::PGInfixFactorial), + ("@", UnaryOperator::PGAbs), + ]; + + for (str_op, op) in bitwise_ops { + let select = pg().verified_only_select(&format!("SELECT {} a", &str_op)); + assert_eq!( + SelectItem::UnnamedExpr(Expr::UnaryOp { + op: op.clone(), + expr: Box::new(Expr::Identifier(Ident::new("a"))), + infix: false, + }), + select.projection[0] + ); + } +} + +#[test] +fn parse_pg_math_ops() { + let bitwise_ops = &[("!", UnaryOperator::PGFactorial)]; + + for (str_op, op) in bitwise_ops { + let select = pg().verified_only_select(&format!("SELECT a{}", &str_op)); + assert_eq!( + SelectItem::UnnamedExpr(Expr::UnaryOp { + op: op.clone(), + expr: Box::new(Expr::Identifier(Ident::new("a"))), + infix: true, + }), + select.projection[0] + ); + } +} + fn pg() -> TestedDialects { TestedDialects { dialects: vec![Box::new(PostgreSqlDialect {})], From 0df8011f92dcdf73925a2f033934bc545a6aa9ec Mon Sep 17 00:00:00 2001 From: Alex Dukhno Date: Mon, 7 Sep 2020 18:25:39 +0300 Subject: [PATCH 2/4] use dialect_of to limit tokenizer to PostgreSql dialect --- src/tokenizer.rs | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index a2cea74c63..de0a936277 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -21,6 +21,7 @@ use std::str::Chars; use super::dialect::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX}; use super::dialect::Dialect; +use super::dialect::PostgreSqlDialect; use super::dialect::SnowflakeDialect; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -433,11 +434,15 @@ impl<'a> Tokenizer<'a> { '|' => { chars.next(); // consume the '|' match chars.peek() { - Some('/') => self.consume_and_return(chars, Token::SquareRoot), + Some('/') if dialect_of!(self is PostgreSqlDialect) => { + self.consume_and_return(chars, Token::SquareRoot) + } Some('|') => { chars.next(); // consume the second '|' match chars.peek() { - Some('/') => self.consume_and_return(chars, Token::CubeRoot), + Some('/') if dialect_of!(self is PostgreSqlDialect) => { + self.consume_and_return(chars, Token::CubeRoot) + } _ => Ok(Some(Token::StringConcat)), } } @@ -457,7 +462,9 @@ impl<'a> Tokenizer<'a> { chars.next(); // consume match chars.peek() { Some('=') => self.consume_and_return(chars, Token::Neq), - Some('!') => self.consume_and_return(chars, Token::DoubleExclamationMark), + Some('!') if dialect_of!(self is PostgreSqlDialect) => { + self.consume_and_return(chars, Token::DoubleExclamationMark) + } _ => Ok(Some(Token::ExclamationMark)), } } @@ -466,7 +473,9 @@ impl<'a> Tokenizer<'a> { match chars.peek() { Some('=') => self.consume_and_return(chars, Token::LtEq), Some('>') => self.consume_and_return(chars, Token::Neq), - Some('<') => self.consume_and_return(chars, Token::ShiftLeft), + Some('<') if dialect_of!(self is PostgreSqlDialect) => { + self.consume_and_return(chars, Token::ShiftLeft) + } _ => Ok(Some(Token::Lt)), } } @@ -501,9 +510,15 @@ impl<'a> Tokenizer<'a> { comment, }))) } - '~' => self.consume_and_return(chars, Token::Tilde), - '#' => self.consume_and_return(chars, Token::Sharp), - '@' => self.consume_and_return(chars, Token::Ampersat), + '~' if dialect_of!(self is PostgreSqlDialect) => { + self.consume_and_return(chars, Token::Tilde) + } + '#' if dialect_of!(self is PostgreSqlDialect) => { + self.consume_and_return(chars, Token::Sharp) + } + '@' if dialect_of!(self is PostgreSqlDialect) => { + self.consume_and_return(chars, Token::Ampersat) + } other => self.consume_and_return(chars, Token::Char(other)), }, None => Ok(None), From 1f2b8d38d355896b7cd76cb6258c992b5cf0c7d8 Mon Sep 17 00:00:00 2001 From: Alex Dukhno Date: Sun, 27 Sep 2020 14:24:12 +0300 Subject: [PATCH 3/4] address review comments --- src/ast/mod.rs | 10 +-- src/ast/operator.rs | 22 ++++--- src/parser.rs | 71 +++++++++++---------- src/tokenizer.rs | 119 +++++++----------------------------- tests/sqlparser_common.rs | 7 --- tests/sqlparser_postgres.rs | 38 +++--------- 6 files changed, 83 insertions(+), 184 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 862d77d410..31171db7dd 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -191,11 +191,7 @@ pub enum Expr { right: Box, }, /// Unary operation e.g. `NOT foo` - UnaryOp { - op: UnaryOperator, - expr: Box, - infix: bool, - }, + UnaryOp { op: UnaryOperator, expr: Box }, /// CAST an expression to a different data type e.g. `CAST(foo AS VARCHAR(123))` Cast { expr: Box, @@ -286,8 +282,8 @@ impl fmt::Display for Expr { high ), Expr::BinaryOp { left, op, right } => write!(f, "{} {} {}", left, op, right), - Expr::UnaryOp { op, expr, infix } => { - if *infix { + Expr::UnaryOp { op, expr } => { + if op == &UnaryOperator::PGPostfixFactorial { write!(f, "{}{}", expr, op) } else { write!(f, "{} {}", op, expr) diff --git a/src/ast/operator.rs b/src/ast/operator.rs index afcd39f0f2..57e70982fa 100644 --- a/src/ast/operator.rs +++ b/src/ast/operator.rs @@ -21,11 +21,17 @@ pub enum UnaryOperator { Plus, Minus, Not, + /// Bitwise Not, e.g. `~9` (PostgreSQL-specific) PGBitwiseNot, - PGSqrt, - PGCbrt, - PGFactorial, - PGInfixFactorial, + /// Square root, e.g. `|/9` (PostgreSQL-specific) + PGSquareRoot, + /// Cube root, e.g. `||/27` (PostgreSQL-specific) + PGCubeRoot, + /// Factorial, e.g. `9!` (PostgreSQL-specific) + PGPostfixFactorial, + /// Factorial, e.g. `!!9` (PostgreSQL-specific) + PGPrefixFactorial, + /// Absolute value, e.g. `@ -9` (PostgreSQL-specific) PGAbs, } @@ -36,10 +42,10 @@ impl fmt::Display for UnaryOperator { UnaryOperator::Minus => "-", UnaryOperator::Not => "NOT", UnaryOperator::PGBitwiseNot => "~", - UnaryOperator::PGSqrt => "|/", - UnaryOperator::PGCbrt => "||/", - UnaryOperator::PGFactorial => "!", - UnaryOperator::PGInfixFactorial => "!!", + UnaryOperator::PGSquareRoot => "|/", + UnaryOperator::PGCubeRoot => "||/", + UnaryOperator::PGPostfixFactorial => "!", + UnaryOperator::PGPrefixFactorial => "!!", UnaryOperator::PGAbs => "@", }) } diff --git a/src/parser.rs b/src/parser.rs index 2b14c7ff4e..0a3e3dbae0 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -250,7 +250,6 @@ impl<'a> Parser<'a> { Keyword::NOT => Ok(Expr::UnaryOp { op: UnaryOperator::Not, expr: Box::new(self.parse_subexpr(Self::UNARY_NOT_PREC)?), - infix: false, }), // Here `w` is a word, check if it's a part of a multi-part // identifier, a function call, or a simple identifier: @@ -284,41 +283,34 @@ impl<'a> Parser<'a> { }, }, // End of Token::Word Token::Mult => Ok(Expr::Wildcard), - Token::Tilde => Ok(Expr::UnaryOp { - op: UnaryOperator::PGBitwiseNot, - expr: Box::new(self.parse_subexpr(0)?), - infix: false, - }), - Token::DoubleExclamationMark => Ok(Expr::UnaryOp { - op: UnaryOperator::PGInfixFactorial, - expr: Box::new(self.parse_subexpr(0)?), - infix: false, - }), - Token::SquareRoot => Ok(Expr::UnaryOp { - op: UnaryOperator::PGSqrt, - expr: Box::new(self.parse_subexpr(0)?), - infix: false, - }), - Token::CubeRoot => Ok(Expr::UnaryOp { - op: UnaryOperator::PGCbrt, - expr: Box::new(self.parse_subexpr(0)?), - infix: false, - }), - Token::Ampersat => Ok(Expr::UnaryOp { - op: UnaryOperator::PGAbs, - expr: Box::new(self.parse_subexpr(0)?), - infix: false, - }), - tok @ Token::Minus | tok @ Token::Plus => { - let op = if tok == Token::Plus { - UnaryOperator::Plus - } else { - UnaryOperator::Minus + tok @ Token::Plus + | tok @ Token::Minus + | tok @ Token::DoubleExclamationMark + | tok @ Token::PGSquareRoot + | tok @ Token::PGCubeRoot + | tok @ Token::AtSign + | tok @ Token::Tilde => { + let op = match tok { + Token::Plus => UnaryOperator::Plus, + Token::Minus => UnaryOperator::Minus, + Token::DoubleExclamationMark if dialect_of!(self is PostgreSqlDialect) => { + UnaryOperator::PGPrefixFactorial + } + Token::PGSquareRoot if dialect_of!(self is PostgreSqlDialect) => { + UnaryOperator::PGSquareRoot + } + Token::PGCubeRoot if dialect_of!(self is PostgreSqlDialect) => { + UnaryOperator::PGCubeRoot + } + Token::AtSign if dialect_of!(self is PostgreSqlDialect) => UnaryOperator::PGAbs, + Token::Tilde if dialect_of!(self is PostgreSqlDialect) => { + UnaryOperator::PGBitwiseNot + } + _ => unreachable!(), }; Ok(Expr::UnaryOp { op, expr: Box::new(self.parse_subexpr(Self::PLUS_MINUS_PREC)?), - infix: false, }) } Token::Number(_) @@ -685,9 +677,15 @@ impl<'a> Parser<'a> { Token::Caret => Some(BinaryOperator::BitwiseXor), Token::Ampersand => Some(BinaryOperator::BitwiseAnd), Token::Div => Some(BinaryOperator::Divide), - Token::ShiftLeft => Some(BinaryOperator::PGBitwiseShiftLeft), - Token::ShiftRight => Some(BinaryOperator::PGBitwiseShiftRight), - Token::Sharp => Some(BinaryOperator::PGBitwiseXor), + Token::ShiftLeft if dialect_of!(self is PostgreSqlDialect) => { + Some(BinaryOperator::PGBitwiseShiftLeft) + } + Token::ShiftRight if dialect_of!(self is PostgreSqlDialect) => { + Some(BinaryOperator::PGBitwiseShiftRight) + } + Token::Sharp if dialect_of!(self is PostgreSqlDialect) => { + Some(BinaryOperator::PGBitwiseXor) + } Token::Word(w) => match w.keyword { Keyword::AND => Some(BinaryOperator::And), Keyword::OR => Some(BinaryOperator::Or), @@ -740,9 +738,8 @@ impl<'a> Parser<'a> { } else if Token::ExclamationMark == tok { // PostgreSQL factorial operation Ok(Expr::UnaryOp { - op: UnaryOperator::PGFactorial, + op: UnaryOperator::PGPostfixFactorial, expr: Box::new(expr), - infix: true, }) } else { // Can only happen if `get_next_precedence` got out of sync with this function diff --git a/src/tokenizer.rs b/src/tokenizer.rs index de0a936277..2496e63a22 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -21,7 +21,6 @@ use std::str::Chars; use super::dialect::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX}; use super::dialect::Dialect; -use super::dialect::PostgreSqlDialect; use super::dialect::SnowflakeDialect; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -103,24 +102,24 @@ pub enum Token { RBrace, /// Right Arrow `=>` RArrow, - /// Sharp `#` use for PostgreSQL Bitwise XOR operator + /// Sharp `#` used for PostgreSQL Bitwise XOR operator Sharp, - /// Tilde `~` use for PostgreSQL Bitwise NOT operator + /// Tilde `~` used for PostgreSQL Bitwise NOT operator Tilde, - /// Bitwise left operator `<<` use for PostgreSQL + /// `<<`, a bitwise shift left operator in PostgreSQL ShiftLeft, - /// Bitwise right operator `>>` use for PostgreSQL + /// `>>`, a bitwise shift right operator in PostgreSQL ShiftRight, - /// Exclamation Mark `!` use for PostgreSQL factorial operator + /// Exclamation Mark `!` used for PostgreSQL factorial operator ExclamationMark, - /// Exclamation Mark `!!` use for PostgreSQL prefix factorial operator + /// Double Exclamation Mark `!!` used for PostgreSQL prefix factorial operator DoubleExclamationMark, - /// Ampersat `@` use for PostgreSQL abs operator - Ampersat, - /// PostgreSQL square root math operator - SquareRoot, - /// PostgreSQL cube root math operator - CubeRoot, + /// AtSign `@` used for PostgreSQL abs operator + AtSign, + /// `|/`, a square root math operator in PostgreSQL + PGSquareRoot, + /// `||/` , a cube root math operator in PostgreSQL + PGCubeRoot, } impl fmt::Display for Token { @@ -166,11 +165,11 @@ impl fmt::Display for Token { Token::ExclamationMark => f.write_str("!"), Token::DoubleExclamationMark => f.write_str("!!"), Token::Tilde => f.write_str("~"), - Token::Ampersat => f.write_str("@"), + Token::AtSign => f.write_str("@"), Token::ShiftLeft => f.write_str("<<"), Token::ShiftRight => f.write_str(">>"), - Token::SquareRoot => f.write_str("|/"), - Token::CubeRoot => f.write_str("||/"), + Token::PGSquareRoot => f.write_str("|/"), + Token::PGCubeRoot => f.write_str("||/"), } } } @@ -434,15 +433,11 @@ impl<'a> Tokenizer<'a> { '|' => { chars.next(); // consume the '|' match chars.peek() { - Some('/') if dialect_of!(self is PostgreSqlDialect) => { - self.consume_and_return(chars, Token::SquareRoot) - } + Some('/') => self.consume_and_return(chars, Token::PGSquareRoot), Some('|') => { chars.next(); // consume the second '|' match chars.peek() { - Some('/') if dialect_of!(self is PostgreSqlDialect) => { - self.consume_and_return(chars, Token::CubeRoot) - } + Some('/') => self.consume_and_return(chars, Token::PGCubeRoot), _ => Ok(Some(Token::StringConcat)), } } @@ -462,9 +457,7 @@ impl<'a> Tokenizer<'a> { chars.next(); // consume match chars.peek() { Some('=') => self.consume_and_return(chars, Token::Neq), - Some('!') if dialect_of!(self is PostgreSqlDialect) => { - self.consume_and_return(chars, Token::DoubleExclamationMark) - } + Some('!') => self.consume_and_return(chars, Token::DoubleExclamationMark), _ => Ok(Some(Token::ExclamationMark)), } } @@ -473,9 +466,7 @@ impl<'a> Tokenizer<'a> { match chars.peek() { Some('=') => self.consume_and_return(chars, Token::LtEq), Some('>') => self.consume_and_return(chars, Token::Neq), - Some('<') if dialect_of!(self is PostgreSqlDialect) => { - self.consume_and_return(chars, Token::ShiftLeft) - } + Some('<') => self.consume_and_return(chars, Token::ShiftLeft), _ => Ok(Some(Token::Lt)), } } @@ -510,15 +501,9 @@ impl<'a> Tokenizer<'a> { comment, }))) } - '~' if dialect_of!(self is PostgreSqlDialect) => { - self.consume_and_return(chars, Token::Tilde) - } - '#' if dialect_of!(self is PostgreSqlDialect) => { - self.consume_and_return(chars, Token::Sharp) - } - '@' if dialect_of!(self is PostgreSqlDialect) => { - self.consume_and_return(chars, Token::Ampersat) - } + '~' => self.consume_and_return(chars, Token::Tilde), + '#' => self.consume_and_return(chars, Token::Sharp), + '@' => self.consume_and_return(chars, Token::AtSign), other => self.consume_and_return(chars, Token::Char(other)), }, None => Ok(None), @@ -641,7 +626,6 @@ mod tests { use super::super::dialect::GenericDialect; use super::super::dialect::MsSqlDialect; use super::*; - use crate::dialect::PostgreSqlDialect; #[test] fn tokenize_select_1() { @@ -1014,65 +998,6 @@ mod tests { compare(expected, tokens); } - #[test] - fn tokenize_postgresql_bitwise_operations() { - let sql = String::from("SELECT ~one << two # three >> four"); - let dialect = PostgreSqlDialect {}; - let mut tokenizer = Tokenizer::new(&dialect, &sql); - let tokens = tokenizer.tokenize().unwrap(); - - let expected = vec![ - Token::make_keyword("SELECT"), - Token::Whitespace(Whitespace::Space), - Token::Tilde, - Token::make_word("one", None), - Token::Whitespace(Whitespace::Space), - Token::ShiftLeft, - Token::Whitespace(Whitespace::Space), - Token::make_word("two", None), - Token::Whitespace(Whitespace::Space), - Token::Sharp, - Token::Whitespace(Whitespace::Space), - Token::make_word("three", None), - Token::Whitespace(Whitespace::Space), - Token::ShiftRight, - Token::Whitespace(Whitespace::Space), - Token::make_word("four", None), - ]; - - compare(expected, tokens); - } - - #[test] - fn tokenize_postgresql_math_operations() { - let sql = String::from("SELECT !!5 5! @-6 |/4 ||/8"); - let dialect = PostgreSqlDialect {}; - let mut tokenizer = Tokenizer::new(&dialect, &sql); - let tokens = tokenizer.tokenize().unwrap(); - - let expected = vec![ - Token::make_keyword("SELECT"), - Token::Whitespace(Whitespace::Space), - Token::DoubleExclamationMark, - Token::Number("5".to_string()), - Token::Whitespace(Whitespace::Space), - Token::Number("5".to_string()), - Token::ExclamationMark, - Token::Whitespace(Whitespace::Space), - Token::Ampersat, - Token::Minus, - Token::Number("6".to_string()), - Token::Whitespace(Whitespace::Space), - Token::SquareRoot, - Token::Number("4".to_string()), - Token::Whitespace(Whitespace::Space), - Token::CubeRoot, - Token::Number("8".to_string()), - ]; - - compare(expected, tokens); - } - fn compare(expected: Vec, actual: Vec) { //println!("------------------------------"); //println!("tokens = {:?}", actual); diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 976bd651d1..4050cdf0b8 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -344,7 +344,6 @@ fn parse_select_count_distinct() { args: vec![FunctionArg::Unnamed(Expr::UnaryOp { op: UnaryOperator::Plus, expr: Box::new(Expr::Identifier(Ident::new("x"))), - infix: false, })], over: None, distinct: true, @@ -507,13 +506,11 @@ fn parse_unary_math() { left: Box::new(UnaryOp { op: UnaryOperator::Minus, expr: Box::new(Identifier(Ident::new("a"))), - infix: false, }), op: BinaryOperator::Plus, right: Box::new(UnaryOp { op: UnaryOperator::Minus, expr: Box::new(Identifier(Ident::new("b"))), - infix: false, }), }, verified_expr(sql) @@ -568,7 +565,6 @@ fn parse_not_precedence() { high: Box::new(Expr::Value(number("2"))), negated: true, }), - infix: false, }, ); @@ -583,7 +579,6 @@ fn parse_not_precedence() { op: BinaryOperator::NotLike, right: Box::new(Expr::Value(Value::SingleQuotedString("b".into()))), }), - infix: false, }, ); @@ -598,7 +593,6 @@ fn parse_not_precedence() { list: vec![Expr::Value(Value::SingleQuotedString("a".into()))], negated: true, }), - infix: false, }, ); } @@ -2610,7 +2604,6 @@ fn parse_exists_subquery() { Expr::UnaryOp { op: UnaryOperator::Not, expr: Box::new(Expr::Exists(Box::new(expected_inner))), - infix: false, }, select.selection.unwrap(), ); diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 2e60901896..1083e9971b 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -573,38 +573,21 @@ fn parse_pg_bitwise_binary_ops() { } #[test] -fn parse_pg_bitwise_unary_ops() { - let bitwise_ops = &[("~", UnaryOperator::PGBitwiseNot)]; - - for (str_op, op) in bitwise_ops { - let select = pg().verified_only_select(&format!("SELECT {} a", &str_op)); - assert_eq!( - SelectItem::UnnamedExpr(Expr::UnaryOp { - op: op.clone(), - expr: Box::new(Expr::Identifier(Ident::new("a"))), - infix: false, - }), - select.projection[0] - ); - } -} - -#[test] -fn parse_pg_infix_math_ops() { - let bitwise_ops = &[ - ("|/", UnaryOperator::PGSqrt), - ("||/", UnaryOperator::PGCbrt), - ("!!", UnaryOperator::PGInfixFactorial), +fn parse_pg_unary_ops() { + let pg_unary_ops = &[ + ("~", UnaryOperator::PGBitwiseNot), + ("|/", UnaryOperator::PGSquareRoot), + ("||/", UnaryOperator::PGCubeRoot), + ("!!", UnaryOperator::PGPrefixFactorial), ("@", UnaryOperator::PGAbs), ]; - for (str_op, op) in bitwise_ops { + for (str_op, op) in pg_unary_ops { let select = pg().verified_only_select(&format!("SELECT {} a", &str_op)); assert_eq!( SelectItem::UnnamedExpr(Expr::UnaryOp { op: op.clone(), expr: Box::new(Expr::Identifier(Ident::new("a"))), - infix: false, }), select.projection[0] ); @@ -612,16 +595,15 @@ fn parse_pg_infix_math_ops() { } #[test] -fn parse_pg_math_ops() { - let bitwise_ops = &[("!", UnaryOperator::PGFactorial)]; +fn parse_pg_postfix_factorial() { + let postfix_factorial = &[("!", UnaryOperator::PGPostfixFactorial)]; - for (str_op, op) in bitwise_ops { + for (str_op, op) in postfix_factorial { let select = pg().verified_only_select(&format!("SELECT a{}", &str_op)); assert_eq!( SelectItem::UnnamedExpr(Expr::UnaryOp { op: op.clone(), expr: Box::new(Expr::Identifier(Ident::new("a"))), - infix: true, }), select.projection[0] ); From 045dbf5c7188d54a69352bf313e1ec7421f619bb Mon Sep 17 00:00:00 2001 From: Alex Dukhno Date: Tue, 29 Sep 2020 10:23:03 +0300 Subject: [PATCH 4/4] address dialect_of!(self is PostgreSqlDialect) --- src/parser.rs | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 0a3e3dbae0..f7630b0372 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -283,29 +283,30 @@ impl<'a> Parser<'a> { }, }, // End of Token::Word Token::Mult => Ok(Expr::Wildcard), - tok @ Token::Plus - | tok @ Token::Minus - | tok @ Token::DoubleExclamationMark + tok @ Token::Minus | tok @ Token::Plus => { + let op = if tok == Token::Plus { + UnaryOperator::Plus + } else { + UnaryOperator::Minus + }; + Ok(Expr::UnaryOp { + op, + expr: Box::new(self.parse_subexpr(Self::PLUS_MINUS_PREC)?), + }) + } + tok @ Token::DoubleExclamationMark | tok @ Token::PGSquareRoot | tok @ Token::PGCubeRoot | tok @ Token::AtSign - | tok @ Token::Tilde => { + | tok @ Token::Tilde + if dialect_of!(self is PostgreSqlDialect) => + { let op = match tok { - Token::Plus => UnaryOperator::Plus, - Token::Minus => UnaryOperator::Minus, - Token::DoubleExclamationMark if dialect_of!(self is PostgreSqlDialect) => { - UnaryOperator::PGPrefixFactorial - } - Token::PGSquareRoot if dialect_of!(self is PostgreSqlDialect) => { - UnaryOperator::PGSquareRoot - } - Token::PGCubeRoot if dialect_of!(self is PostgreSqlDialect) => { - UnaryOperator::PGCubeRoot - } - Token::AtSign if dialect_of!(self is PostgreSqlDialect) => UnaryOperator::PGAbs, - Token::Tilde if dialect_of!(self is PostgreSqlDialect) => { - UnaryOperator::PGBitwiseNot - } + Token::DoubleExclamationMark => UnaryOperator::PGPrefixFactorial, + Token::PGSquareRoot => UnaryOperator::PGSquareRoot, + Token::PGCubeRoot => UnaryOperator::PGCubeRoot, + Token::AtSign => UnaryOperator::PGAbs, + Token::Tilde => UnaryOperator::PGBitwiseNot, _ => unreachable!(), }; Ok(Expr::UnaryOp {