From 72754b134c44946c9151efd010ab6156983e474a Mon Sep 17 00:00:00 2001 From: Denys Tsomenko Date: Fri, 6 Nov 2026 19:16:54 +0200 Subject: [PATCH] Snowflake: Parse typed PUT and GET transfer operands --- AGENTS.md | 7 + README.md | 37 +++- src/ast/helpers/stmt_data_loading.rs | 229 ++++++++++++++++++- src/ast/mod.rs | 9 +- src/ast/spans.rs | 5 +- src/dialect/mod.rs | 10 +- src/dialect/snowflake.rs | 190 +++++++++++++++- src/keywords.rs | 1 + src/tokenizer.rs | 118 +++++++++- tests/sqlparser_snowflake.rs | 314 +++++++++++++++++++++++++++ 10 files changed, 911 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cb8033c55e..1711d5a94d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,13 @@ snowflake().one_statement_parses_to( ``` ## Analyzing Parsing Issues + +For Snowflake `PUT`/`GET`, keep opaque location lexing confined to the two operands +at actual statement start. Preserve ordinary comments and identifiers/functions +named `PUT`/`GET`. Keep stage `ObjectName` quoting separate from the exact payload +prefix, avoid table-relation visitor annotations for named stages, and retain +bindings as normal expression placeholders. The parser must not resolve paths, +perform storage IO, evaluate options, or enforce an engine's transfer policy. You can try to simplify the SQL statement to identify the root cause of the parsing issue. This may involve removing certain clauses or components of the SQL statement to see if it can be parsed successfully. Additionally, you can compare the problematic SQL statement with similar statements that are parsed correctly to identify any differences that may be causing the issue. ## Pre Commit Checks diff --git a/README.md b/README.md index 775d074915..b13dff104d 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,42 @@ This crate avoids semantic analysis because it varies drastically between dialects and implementations. If you want to do semantic analysis, feel free to use this project as a base. -## Preserves Syntax Round Trip +## Snowflake transfer operands + +`SnowflakeDialect` represents `PUT` and `GET` with dedicated statement nodes and +typed local URI, stage, and binding operands. A named stage retains its real +`ObjectName` identifiers and a separate exact payload prefix. User and table +stages remain distinct AST forms. Stage operands are not annotated as table +relations for generic relation discovery. + +Only the first two operands after a statement-start transfer keyword use opaque +location tokenization. This keeps `file://`, `/*` globs, `--` filenames, `?`, and +`#` intact without changing ordinary SQL comments or identifiers/functions named +`PUT` and `GET`. Single-quoted transfer operands retain literal backslashes and +use doubled quotes for a literal quote; ordinary SQL string escape rules outside +these operands are unchanged. Whole-stage single quotes are a local grammar form, +not independent Snowflake reference evidence. Double-quoted local URIs and +`IDENTIFIER(...)` transfer operands are outside this grammar. + +Options retain their original order, duplicates, and typed `Expr` values, including +signed numeric spelling. Optional numeric spelling metadata retains forms such as +`001` even when the `bigdecimal` feature normalizes the typed number; rendering +ignores stale spelling after a visitor changes that numeric value. Location +bindings use normal placeholder expressions so +expression visitors can apply the owning engine's existing binding accounting. +`dialect::snowflake::parse_file_transfer_stage_location` converts an exact bound +stage value into a typed target without manufacturing a SQL statement. + +The parser performs no storage IO, option evaluation, range checking, async/batch +admission, or transfer execution. The execution layer must validate primitive +option shapes and supported stage kinds before planning/evaluating expressions +or submitting work. Accepting syntax does not claim a transfer transport or a +complete Snowflake workflow. In particular, path traversal/backslash admission, +`PARALLEL` range errors, duplicate/unknown options, and unsupported stage kinds +remain semantic responsibilities. These additions require owning qualification; +source tests alone do not establish runtime or reference parity. + +## Preserves Syntax Round Trip This crate allows users to recover the original SQL text (with comments removed, normalized whitespace and keyword capitalization), which is useful for tools diff --git a/src/ast/helpers/stmt_data_loading.rs b/src/ast/helpers/stmt_data_loading.rs index b37fc87d61..43679bbcd0 100644 --- a/src/ast/helpers/stmt_data_loading.rs +++ b/src/ast/helpers/stmt_data_loading.rs @@ -20,17 +20,242 @@ //! See [this page](https://docs.snowflake.com/en/sql-reference/commands-data-loading) for more details. #[cfg(not(feature = "std"))] -use alloc::string::String; +use alloc::{format, string::String, vec::Vec}; use core::fmt; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use crate::ast::helpers::key_value_options::KeyValueOptions; -use crate::ast::{Ident, ObjectName, SelectItem}; +use crate::ast::{Expr, Ident, ObjectName, SelectItem, UnaryOperator, Value}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; +/// Snowflake `PUT` upload syntax. Execution and transfer admission are separate concerns. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct Put { + /// Local source URI or a binding expression. + pub source: FileTransferLocation, + /// Stage destination or a binding expression. + pub target: FileTransferLocation, + /// Ordered options, including duplicates for owning semantic validation. + pub options: Vec, +} + +/// Snowflake `GET` download syntax. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct Get { + /// Stage source or a binding expression. + pub source: FileTransferLocation, + /// Local destination URI or a binding expression. + pub target: FileTransferLocation, + /// Ordered options, including duplicates for owning semantic validation. + pub options: Vec, +} + +/// A transfer operand, distinct from a table relation and from arbitrary SQL text. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum FileTransferLocation { + /// A local URI, preserving the `file://` prefix and path bytes. + Local { + /// Exact URI; the parser does not resolve or normalize a filesystem path. + uri: String, + /// Whether the complete operand was enclosed in single quotes. + quoted: bool, + }, + /// A typed stage reference and separate optional payload prefix. + Stage(FileTransferStageLocation), + /// A native SQL placeholder. Visitors must consume it through normal expression binding. + Binding(Expr), +} + +/// A stage reference used by transfer syntax, without table-relation visitor annotations. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct FileTransferStageLocation { + /// Named, user, or table stage syntax. Parsing does not imply execution support. + pub kind: FileTransferStage, + /// Exact bytes after the first slash outside quoted identifiers; `Some("")` retains `/`. + pub prefix: Option, + /// Whether the complete operand was enclosed in single quotes. + pub quoted: bool, +} + +/// Stage kinds must remain distinct so an execution layer can explicitly refuse unsupported ones. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum FileTransferStage { + /// A real qualified object name, retaining every identifier's quoting. + Named(ObjectName), + /// The current user's `@~` stage. + User, + /// A table's `@[namespace.]%table` stage, retaining its qualified name. + Table(ObjectName), +} + +/// One transfer option. Values use normal expression/value nodes, not enum-like placeholders. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct FileTransferOption { + /// Original option name, including its identifier quoting. + pub name: Ident, + /// Typed value traversed by normal expression binding. + pub value: Expr, + /// Original primitive numeric spelling when the numeric AST feature normalizes it. + #[cfg_attr( + feature = "serde", + serde(default, skip_serializing_if = "Option::is_none") + )] + pub number_spelling: Option, +} + +pub(crate) fn file_transfer_number_matches(value: &Expr, raw: &str) -> bool { + let (value, digits) = match value { + Expr::UnaryOp { + op: UnaryOperator::Plus, + expr, + } => { + let Some(digits) = raw.strip_prefix('+') else { + return false; + }; + (expr.as_ref(), digits) + } + Expr::UnaryOp { + op: UnaryOperator::Minus, + expr, + } => { + let Some(digits) = raw.strip_prefix('-') else { + return false; + }; + (expr.as_ref(), digits) + } + _ if raw.starts_with(['+', '-']) => return false, + _ => (value, raw), + }; + let Expr::Value(value) = value else { + return false; + }; + let Ok(number) = digits.parse() else { + return false; + }; + value.value == Value::Number(number, false) +} + +impl fmt::Display for FileTransferStage { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Named(name) => write!(f, "{name}"), + Self::User => f.write_str("~"), + Self::Table(name) => { + for (index, part) in name.0.iter().enumerate() { + if index > 0 { + f.write_str(".")?; + } + if index + 1 == name.0.len() { + f.write_str("%")?; + } + write!(f, "{part}")?; + } + Ok(()) + } + } + } +} + +impl fmt::Display for FileTransferStageLocation { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut value = format!("@{}", self.kind); + if let Some(prefix) = &self.prefix { + value.push('/'); + value.push_str(prefix); + } + display_transfer_location(f, &value, self.quoted) + } +} + +fn display_transfer_location(f: &mut fmt::Formatter, value: &str, quoted: bool) -> fmt::Result { + if quoted || file_transfer_location_needs_quotes(value) { + // Transfer paths keep backslashes literal; only the enclosing SQL quote is escaped. + write!(f, "'{}'", value.replace('\'', "''")) + } else { + f.write_str(value) + } +} + +pub(crate) fn file_transfer_location_needs_quotes(value: &str) -> bool { + let mut stage_name = value.starts_with('@'); + let mut identifier_quote = false; + let mut chars = value.chars().peekable(); + while let Some(ch) = chars.next() { + if stage_name && ch == '"' { + if identifier_quote && chars.peek() == Some(&'"') { + chars.next(); + } else { + identifier_quote = !identifier_quote; + } + } else if stage_name && ch == '/' && !identifier_quote { + stage_name = false; + } else if !identifier_quote && (ch.is_whitespace() || ch == ';') { + return true; + } + } + false +} + +impl fmt::Display for FileTransferLocation { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Local { uri, quoted } => display_transfer_location(f, uri, *quoted), + Self::Stage(stage) => write!(f, "{stage}"), + Self::Binding(expr) => write!(f, "{expr}"), + } + } +} + +impl fmt::Display for FileTransferOption { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}=", self.name)?; + if let Some(raw) = self + .number_spelling + .as_deref() + .filter(|raw| file_transfer_number_matches(&self.value, raw)) + { + f.write_str(raw) + } else { + write!(f, "{}", self.value) + } + } +} + +impl fmt::Display for Put { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "PUT {} {}", self.source, self.target)?; + for option in &self.options { + write!(f, " {option}")?; + } + Ok(()) + } +} + +impl fmt::Display for Get { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "GET {} {}", self.source, self.target)?; + for option in &self.options { + write!(f, " {option}")?; + } + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 5d6fea9bd9..07daeb2e05 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -27,7 +27,8 @@ use alloc::{ use helpers::{ attached_token::AttachedToken, stmt_data_loading::{ - AlterFileFormatOperation, AlterStageOperation, FileStagingCommand, StageLoadSelectItemKind, + AlterFileFormatOperation, AlterStageOperation, FileStagingCommand, Get, Put, + StageLoadSelectItemKind, }, }; @@ -5022,6 +5023,10 @@ pub enum Statement { /// Snowflake `REMOVE` /// See: Remove(FileStagingCommand), + /// Snowflake `PUT`: prepare an upload from a local URI to a stage. + Put(Put), + /// Snowflake `GET`: prepare a download from a stage to a local URI. + Get(Get), /// RaiseError (MSSQL) /// RAISERROR ( { msg_id | msg_str | @local_variable } /// { , severity , state } @@ -6723,6 +6728,8 @@ impl fmt::Display for Statement { Statement::Return(r) => write!(f, "{r}"), Statement::List(command) => write!(f, "LIST {command}"), Statement::Remove(command) => write!(f, "REMOVE {command}"), + Statement::Put(command) => write!(f, "{command}"), + Statement::Get(command) => write!(f, "{command}"), Statement::ExportData(e) => write!(f, "{e}"), Statement::CreateUser(s) => write!(f, "{s}"), Statement::AlterSchema(s) => write!(f, "{s}"), diff --git a/src/ast/spans.rs b/src/ast/spans.rs index c0a4499d92..b5f74df016 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -520,7 +520,10 @@ impl Spanned for Statement { Statement::Print { .. } => Span::empty(), Statement::WaitFor(_) => Span::empty(), Statement::Return { .. } => Span::empty(), - Statement::List(..) | Statement::Remove(..) => Span::empty(), + Statement::List(..) + | Statement::Remove(..) + | Statement::Put(..) + | Statement::Get(..) => Span::empty(), Statement::ExportData(ExportData { options, query, diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 85a8bac7ff..19b1ea2fb5 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -51,8 +51,8 @@ pub use self::mysql::MySqlDialect; pub use self::oracle::OracleDialect; pub use self::postgresql::PostgreSqlDialect; pub use self::redshift::RedshiftSqlDialect; -pub use self::snowflake::parse_snowflake_stage_name; pub use self::snowflake::SnowflakeDialect; +pub use self::snowflake::{parse_file_transfer_stage_location, parse_snowflake_stage_name}; pub use self::spark::SparkSqlDialect; pub use self::sqlite::SQLiteDialect; pub use self::teradata::TeradataDialect; @@ -1273,6 +1273,14 @@ pub trait Dialect: Debug + Any { false } + /// Supports `PUT`/`GET` with two opaque transfer operands at statement start. + /// + /// URI comment markers, globs, and stage prefixes are payload bytes only in those + /// operands. Normal SQL identifiers and comments retain their ordinary tokenization. + fn supports_file_transfer_commands(&self) -> bool { + false + } + /// Returns true if this dialect supports the `COMMENT` statement fn supports_comment_on(&self) -> bool { false diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index eb2e82b1e8..ddd2c2d02d 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -24,7 +24,9 @@ use crate::ast::helpers::key_value_options::{ use crate::ast::helpers::stmt_create_database::CreateDatabaseBuilder; use crate::ast::helpers::stmt_create_table::CreateTableBuilder; use crate::ast::helpers::stmt_data_loading::{ - AlterFileFormatOperation, AlterStageOperation, FileStagingCommand, StageLoadSelectItem, + file_transfer_location_needs_quotes, file_transfer_number_matches, AlterFileFormatOperation, + AlterStageOperation, FileStagingCommand, FileTransferLocation, FileTransferOption, + FileTransferStage, FileTransferStageLocation, Get, Put, StageLoadSelectItem, StageLoadSelectItemKind, StageParamsObject, }; use crate::ast::{ @@ -42,7 +44,7 @@ use crate::dialect::{Dialect, Precedence}; use crate::keywords::Keyword; use crate::parser::{IsOptional, Parser, ParserError}; use crate::tokenizer::TokenWithSpan; -use crate::tokenizer::{Span, Token}; +use crate::tokenizer::{Span, Token, Tokenizer}; #[cfg(not(feature = "std"))] use alloc::boxed::Box; #[cfg(not(feature = "std"))] @@ -272,6 +274,10 @@ impl Dialect for SnowflakeDialect { } fn parse_statement(&self, parser: &mut Parser) -> Option> { + if let Some(command) = parser.parse_one_of_keywords(&[Keyword::PUT, Keyword::GET]) { + return Some(parse_file_transfer_command(command, parser)); + } + if parser.parse_keyword(Keyword::BEGIN) { // Snowflake supports both `BEGIN TRANSACTION` and `BEGIN ... END` blocks. // If the next keyword indicates a transaction statement, let the @@ -508,6 +514,10 @@ impl Dialect for SnowflakeDialect { true } + fn supports_file_transfer_commands(&self) -> bool { + true + } + fn supports_left_associative_joins_without_parens(&self) -> bool { false } @@ -776,6 +786,182 @@ fn parse_file_staging_command(kw: Keyword, parser: &mut Parser) -> Result Result { + let source = parse_file_transfer_location(parser, command == Keyword::GET)?; + let target = parse_file_transfer_location(parser, command == Keyword::PUT)?; + let mut options = Vec::new(); + while !matches!(parser.peek_token_ref().token, Token::EOF | Token::SemiColon) { + let name = parser.parse_identifier()?; + parser.expect_token(&Token::Eq)?; + let number_spelling = match parser.peek_token().token { + Token::Number(number, false) => Some(number), + sign @ (Token::Plus | Token::Minus) => match parser.peek_nth_token(1).token { + Token::Number(number, false) => Some(format!("{sign}{number}")), + _ => None, + }, + _ => None, + }; + // Keep typed primitives, identifiers, signed numbers, and actual bindings. + // Owning execution must validate option names/value shapes without evaluating them. + let value = parser.parse_expr()?; + let number_spelling = number_spelling + .filter(|raw| raw != &value.to_string() && file_transfer_number_matches(&value, raw)); + options.push(FileTransferOption { + name, + value, + number_spelling, + }); + } + if command == Keyword::PUT { + Ok(Statement::Put(Put { + source, + target, + options, + })) + } else { + Ok(Statement::Get(Get { + source, + target, + options, + })) + } +} + +fn parse_file_transfer_location( + parser: &mut Parser, + stage: bool, +) -> Result { + let token = parser.next_token(); + let (value, quoted) = match token.token { + Token::FileTransferLocation { value, quoted } => (value, quoted), + Token::Placeholder(value) => { + return Ok(FileTransferLocation::Binding(Expr::Value( + Value::Placeholder(value).with_span(token.span), + ))); + } + _ => return parser.expected("file transfer location", token), + }; + let numeric_binding = value + .strip_prefix(':') + .is_some_and(|number| !number.is_empty() && number.chars().all(|ch| ch.is_ascii_digit())); + if !quoted && (value == "?" || numeric_binding) { + return Ok(FileTransferLocation::Binding(Expr::Value( + Value::Placeholder(value).with_span(token.span), + ))); + } + if stage && value.starts_with('@') { + let mut location = parse_file_transfer_stage_location(&value)?; + location.quoted = quoted; + return Ok(FileTransferLocation::Stage(location)); + } + if !stage && value.starts_with("file://") { + return Ok(FileTransferLocation::Local { uri: value, quoted }); + } + Err(ParserError::ParserError(if stage { + "Expected a stage reference or binding in transfer operand".into() + } else { + "Expected a file:// URI or binding in transfer operand".into() + })) +} + +/// Parse an exact bound stage operand into a typed name and separate payload prefix. +/// +/// This parses identifiers rather than manufacturing a `PUT`/`GET` SQL statement. +/// User/table forms remain distinct; parsing does not imply local execution support. +pub fn parse_file_transfer_stage_location( + value: &str, +) -> Result { + let Some(stage) = value.strip_prefix('@') else { + return Err(ParserError::ParserError( + "Expected an @stage reference".into(), + )); + }; + let mut identifier_quote = false; + let mut slash = None; + let mut chars = stage.char_indices().peekable(); + while let Some((index, ch)) = chars.next() { + if ch == '"' { + if identifier_quote && chars.peek().is_some_and(|(_, ch)| *ch == '"') { + chars.next(); + } else { + identifier_quote = !identifier_quote; + } + } else if ch == '/' && !identifier_quote { + slash = Some(index); + break; + } + } + if identifier_quote { + return Err(ParserError::ParserError( + "Unterminated stage identifier".into(), + )); + } + let (name, prefix) = match slash { + Some(index) => (&stage[..index], Some(String::from(&stage[index + 1..]))), + None => (stage, None), + }; + let kind = if name == "~" { + FileTransferStage::User + } else { + let dialect = SnowflakeDialect; + let mut tokens = Tokenizer::new(&dialect, name) + .with_file_transfer_locations(false) + .tokenize_with_location()?; + // A comment cannot silently truncate a bound identifier. Whitespace inside + // a quoted identifier is part of its Word token and is preserved normally. + if tokens + .iter() + .any(|token| matches!(token.token, Token::Whitespace(_))) + { + return Err(ParserError::ParserError( + "Unexpected whitespace or comment in stage name".into(), + )); + } + let markers = tokens + .iter() + .filter(|token| token.token == Token::Mod) + .count(); + let table = markers == 1; + if markers > 1 + || (table && (tokens.len() < 2 || tokens[tokens.len() - 2].token != Token::Mod)) + { + return Err(ParserError::ParserError( + "Expected % immediately before the final table stage identifier".into(), + )); + } + if table { + tokens.remove(tokens.len() - 2); + } + let mut parser = Parser::new(&dialect).with_tokens_with_locations(tokens); + let name = parser.parse_object_name(false)?; + if name + .0 + .iter() + .any(|part| !matches!(part, ObjectNamePart::Identifier(_))) + { + return Err(ParserError::ParserError( + "Expected identifier components in transfer stage name".into(), + )); + } + if !matches!(parser.peek_token_ref().token, Token::EOF) { + return parser.expected_ref("end of stage name", parser.peek_token_ref()); + } + if table { + FileTransferStage::Table(name) + } else { + FileTransferStage::Named(name) + } + }; + Ok(FileTransferStageLocation { + kind, + prefix, + quoted: file_transfer_location_needs_quotes(value), + }) +} + /// Parse snowflake alter dynamic table. /// fn parse_alter_dynamic_table(parser: &mut Parser) -> Result { diff --git a/src/keywords.rs b/src/keywords.rs index e51b839e31..ca6728dd82 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -826,6 +826,7 @@ define_keywords!( PUBLIC, PURCHASE, PURGE, + PUT, QUALIFY, QUARTER, QUERIES, diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 60a7e8d9a0..9fee49a743 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -219,6 +219,13 @@ pub enum Token { PGCubeRoot, /// `?` or `$` , a prepared statement arg placeholder Placeholder(String), + /// One opaque operand of a statement-start Snowflake `PUT` or `GET`. + FileTransferLocation { + /// URI or stage text with comment/glob/path bytes retained. + value: String, + /// Whether the complete operand was enclosed in single quotes. + quoted: bool, + }, /// `->`, used as a operator to extract json field in PostgreSQL Arrow, /// `->>`, used as a operator to extract json field as text in PostgreSQL @@ -383,6 +390,13 @@ impl fmt::Display for Token { Token::ShiftLeftVerticalBar => f.write_str("<<|"), Token::VerticalBarShiftRight => f.write_str("|>>"), Token::Placeholder(ref s) => write!(f, "{s}"), + Token::FileTransferLocation { value, quoted } => { + if *quoted { + write!(f, "'{value}'") + } else { + f.write_str(value) + } + } Token::Arrow => write!(f, "->"), Token::LongArrow => write!(f, "->>"), Token::HashArrow => write!(f, "#>"), @@ -872,6 +886,8 @@ pub struct Tokenizer<'a> { /// If true (the default), the tokenizer will un-escape literal /// SQL strings See [`Tokenizer::with_unescape`] for more details. unescape: bool, + /// Internal identifier-only parsing must not activate statement-start transfer syntax. + file_transfer_locations: bool, } impl<'a> Tokenizer<'a> { @@ -896,6 +912,7 @@ impl<'a> Tokenizer<'a> { dialect, query, unescape: true, + file_transfer_locations: true, } } @@ -934,6 +951,11 @@ impl<'a> Tokenizer<'a> { self } + pub(crate) fn with_file_transfer_locations(mut self, enabled: bool) -> Self { + self.file_transfer_locations = enabled; + self + } + /// Tokenize the statement and produce a vector of tokens pub fn tokenize(&mut self) -> Result, TokenizerError> { let twl = self.tokenize_with_location()?; @@ -970,9 +992,41 @@ impl<'a> Tokenizer<'a> { }; let mut location = state.location(); - while let Some(token) = self.next_token(&mut state, buf.last().map(|t| &t.token))? { + let file_transfer_locations = + self.file_transfer_locations && self.dialect.supports_file_transfer_commands(); + let mut statement_start = true; + let mut transfer_operands = 0; + loop { + let token = if transfer_operands > 0 && Self::is_transfer_operand_start(&state) { + transfer_operands -= 1; + Some(self.tokenize_file_transfer_location(&mut state)?) + } else { + self.next_token(&mut state, buf.last().map(|t| &t.token))? + }; + let Some(token) = token else { + break; + }; let span = location.span_to(state.location()); + if file_transfer_locations { + match &token { + Token::Whitespace(_) => {} + Token::SemiColon => { + statement_start = true; + transfer_operands = 0; + } + Token::Word(word) if statement_start => { + statement_start = false; + if word.quote_style.is_none() + && matches!(word.keyword, Keyword::PUT | Keyword::GET) + { + transfer_operands = 2; + } + } + _ => statement_start = false, + } + } + // Check if this is a multiline comment hint that should be expanded match &token { Token::Whitespace(Whitespace::MultiLineComment(comment)) @@ -992,6 +1046,68 @@ impl<'a> Tokenizer<'a> { Ok(()) } + /// Comments between operands retain SQL meaning. Once an operand starts, + /// its own `//`, `/*`, `--`, `?`, and `#` are opaque path bytes. + fn is_transfer_operand_start(state: &State) -> bool { + let mut chars = state.peekable.clone(); + match chars.next() { + None | Some(';' | '#' | ' ' | '\t' | '\r' | '\n') => false, + Some('-') => chars.next() != Some('-'), + Some('/') => !matches!(chars.next(), Some('/' | '*')), + Some(_) => true, + } + } + + fn tokenize_file_transfer_location(&self, state: &mut State) -> Result { + let location = state.location(); + let quoted = state.peek() == Some(&'\''); + let mut value = String::new(); + if quoted { + state.next(); + while let Some(ch) = state.next() { + if ch == '\'' { + if state.peek() == Some(&'\'') { + state.next(); + if !self.unescape { + value.push('\''); + } + } else { + return Ok(Token::FileTransferLocation { value, quoted }); + } + } + // Unlike an ordinary string literal, a transfer operand cannot + // discard a backslash before the execution layer validates paths. + value.push(ch); + } + return self.tokenizer_error(location, "Unterminated quoted file transfer location"); + } + + let mut identifier_quote = false; + let mut stage_name = state.peek() == Some(&'@'); + while let Some(&ch) = state.peek() { + if !identifier_quote && (ch.is_whitespace() || ch == ';') { + break; + } + state.next(); + value.push(ch); + if stage_name && ch == '/' && !identifier_quote { + stage_name = false; + } + if stage_name && ch == '"' { + if identifier_quote && state.peek() == Some(&'"') { + state.next(); + value.push('"'); + } else { + identifier_quote = !identifier_quote; + } + } + } + if identifier_quote { + return self.tokenizer_error(location, "Unterminated quoted file transfer identifier"); + } + Ok(Token::FileTransferLocation { value, quoted }) + } + /// Re-tokenize optimizer hints from a multiline comment and add them to the buffer. /// For example, `/*!50110 KEY_BLOCK_SIZE = 1024*/` becomes tokens for `KEY_BLOCK_SIZE = 1024` fn tokenize_comment_hints( diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 24bc67517b..5cb26a162b 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -5969,3 +5969,317 @@ fn snowflake_create_database_still_requires_statement_delimiters() { ); } } + +#[test] +fn snowflake_put_get_typed_locations_and_options() { + use sqlparser::ast::helpers::stmt_data_loading::{FileTransferLocation, FileTransferStage}; + + let sql = r#"PUT file:///tmp/a--b/*?[0-9]#part.csv @"db.with.dot"."schema/part"."stage""name"/inbox/a--b/*?x=1#part AUTO_COMPRESS=FALSE SOURCE_COMPRESSION=AUTO_DETECT OVERWRITE=TRUE PARALLEL=001"#; + let Statement::Put(command) = snowflake().one_statement_parses_to( + sql, + r#"PUT file:///tmp/a--b/*?[0-9]#part.csv @"db.with.dot"."schema/part"."stage""name"/inbox/a--b/*?x=1#part AUTO_COMPRESS=false SOURCE_COMPRESSION=AUTO_DETECT OVERWRITE=true PARALLEL=001"#, + ) else { + panic!("expected typed PUT"); + }; + assert!(matches!( + command.source, + FileTransferLocation::Local { ref uri, quoted: false } + if uri == "file:///tmp/a--b/*?[0-9]#part.csv" + )); + let FileTransferLocation::Stage(location) = command.target else { + panic!("expected typed stage target"); + }; + let FileTransferStage::Named(name) = location.kind else { + panic!("expected named stage"); + }; + assert_eq!( + name, + ObjectName::from(vec![ + Ident::with_quote('"', "db.with.dot"), + Ident::with_quote('"', "schema/part"), + Ident::with_quote('"', "stage\"name"), + ]) + ); + assert_eq!(location.prefix.as_deref(), Some("inbox/a--b/*?x=1#part")); + assert!(!location.quoted); + assert_eq!(command.options.len(), 4); + assert_eq!(command.options[0].name, Ident::new("AUTO_COMPRESS")); + assert!(matches!( + command.options[0].value, + Expr::Value(ValueWithSpan { + value: Value::Boolean(false), + .. + }) + )); + assert_eq!( + command.options[1].value, + Expr::Identifier(Ident::new("AUTO_DETECT")) + ); + assert!(matches!( + &command.options[3].value, + Expr::Value(ValueWithSpan { + value: Value::Number(_, false), + .. + }) + )); + assert_eq!(command.options[3].to_string(), "PARALLEL=001"); + let mut changed = command.options[3].clone(); + changed.value = Expr::Value(Value::Number("2".parse().unwrap(), false).with_empty_span()); + assert_eq!(changed.to_string(), "PARALLEL=2"); +} + +#[test] +fn snowflake_put_get_quoted_paths_round_trip_without_path_normalization() { + for sql in [ + r#"PUT 'file:///tmp/a b/it''s--file/*.csv?x#part' @stage/path AUTO_COMPRESS=false"#, + r#"GET '@"db.with.dot"."stage/slash"/a b/it''s--key/*?x#part' 'file:///tmp/download here' PATTERN='.*[.]csv' PARALLEL=4"#, + r#"PUT file://relative/../input.csv @stage/a/../b OVERWRITE=true"#, + r#"PUT file:///tmp/input.csv @stage/ PARALLEL=-1"#, + r#"GET @stage file:///tmp/download PARALLEL=100"#, + r#"PUT 'file:///C:\input\a.csv' @stage/a\b AUTO_COMPRESS=false"#, + r#"PUT file:///tmp/a"b.csv @stage/a"b?x#part AUTO_COMPRESS=false"#, + ] { + snowflake().verified_stmt(sql); + } +} + +#[test] +fn snowflake_put_get_preserves_option_order_duplicates_and_numeric_spelling() { + let Statement::Put(command) = snowflake().one_statement_parses_to( + "PUT file:///tmp/a.csv @stage AUTO_COMPRESS=FALSE PARALLEL=-001 AUTO_COMPRESS=TRUE UNSUPPORTED_OPTION=7", + "PUT file:///tmp/a.csv @stage AUTO_COMPRESS=false PARALLEL=-001 AUTO_COMPRESS=true UNSUPPORTED_OPTION=7", + ) else { + panic!("expected typed PUT"); + }; + assert_eq!(command.options.len(), 4); + assert_eq!(command.options[0].name, command.options[2].name); + assert_eq!(command.options[1].to_string(), "PARALLEL=-001"); + assert_eq!(command.options[3].name, Ident::new("UNSUPPORTED_OPTION")); + // Semantic option admission/range errors belong to the owning execution layer. + for value in ["0", "100", "-1", "+001", "1.5", "NULL", "'invalid'"] { + assert!(snowflake() + .parse_sql_statements(&format!("GET @stage file:///tmp/download PARALLEL={value}")) + .is_ok()); + } +} + +#[test] +fn snowflake_put_get_represents_user_and_table_stages_without_named_aliases() { + use sqlparser::ast::helpers::stmt_data_loading::{FileTransferLocation, FileTransferStage}; + for (sql, user) in [ + ("PUT file:///tmp/a.csv @~/prefix", true), + ( + r#"GET @"db"."schema".%"table/name"/prefix file:///tmp/download"#, + false, + ), + ] { + let statement = snowflake().verified_stmt(sql); + let stage = match statement { + Statement::Put(command) => command.target, + Statement::Get(command) => command.source, + _ => panic!("expected transfer statement"), + }; + let FileTransferLocation::Stage(location) = stage else { + panic!("expected stage location"); + }; + assert_eq!(location.prefix.as_deref(), Some("prefix")); + if user { + assert!(matches!(location.kind, FileTransferStage::User)); + } else { + assert!(matches!(location.kind, FileTransferStage::Table(_))); + } + } +} + +#[test] +fn snowflake_put_get_statement_boundaries_and_ordinary_sql_comments() { + use sqlparser::parser::Parser; + + let statements = Parser::parse_sql( + &SnowflakeDialect, + "SELECT 1; PUT file:///tmp/a--b/*.csv?x#part @stage/prefix; GET @stage/prefix file:///tmp/out; SELECT put, get FROM source", + ) + .unwrap(); + assert_eq!(statements.len(), 4); + assert!(matches!(statements[1], Statement::Put(_))); + assert!(matches!(statements[2], Statement::Get(_))); + assert!(matches!(statements[3], Statement::Query(_))); + snowflake().one_statement_parses_to( + "SELECT put, get FROM source // ordinary PUT file:///tmp/a.csv @stage\n", + "SELECT put, get FROM source", + ); + snowflake().one_statement_parses_to( + "SELECT put(1), get(2) /* ordinary file:///tmp/ -- text */", + "SELECT put(1), get(2)", + ); + snowflake().one_statement_parses_to( + "PUT /* before source */ file:///tmp/a.csv -- before target\n @stage/path # after target\n PARALLEL=4", + "PUT file:///tmp/a.csv @stage/path PARALLEL=4", + ); +} + +#[test] +fn snowflake_put_get_malformed_operands_and_options() { + for sql in [ + "PUT", + "PUT file:///tmp/a.csv", + "GET @stage", + "PUT file:///tmp/a.csv @", + "GET @% file:///tmp/out", + "GET @~.name file:///tmp/out", + "GET @%db.schema.table file:///tmp/out", + "GET @IDENTIFIER('stage') file:///tmp/out", + "PUT file:///tmp/a.csv @db.IDENTIFIER(:1)", + "PUT file:///tmp/a.csv @stage PARALLEL", + "GET @stage file:///tmp/out PARALLEL=", + "PUT file:///tmp/a.csv @stage AUTO_COMPRESS=TRUE, OVERWRITE=FALSE", + "PUT 'file:///tmp/unterminated @stage", + "GET @\"unterminated/file file:///tmp/out", + "PUT file:///tmp/a.csv @stage--hidden-suffix", + ] { + assert!(snowflake().parse_sql_statements(sql).is_err(), "{sql}"); + } +} + +#[test] +fn snowflake_put_get_stage_binding_parser_preserves_components_and_prefix() { + use sqlparser::ast::helpers::stmt_data_loading::FileTransferStage; + use sqlparser::dialect::parse_file_transfer_stage_location; + + let location = parse_file_transfer_stage_location( + r#"@"db.with/slash"."schema.dot"."stage""name"/prefix/../a--b/*?x#part"#, + ) + .unwrap(); + let FileTransferStage::Named(name) = location.kind else { + panic!("expected named stage"); + }; + assert_eq!(name.0.len(), 3); + assert_eq!(location.prefix.as_deref(), Some("prefix/../a--b/*?x#part")); + assert_eq!( + parse_file_transfer_stage_location("@get.schema.put") + .unwrap() + .kind, + FileTransferStage::Named(ObjectName::from(vec![ + Ident::new("get"), + Ident::new("schema"), + Ident::new("put"), + ])) + ); + assert_eq!( + parse_file_transfer_stage_location("@stage/*literal*/") + .unwrap() + .prefix + .as_deref(), + Some("*literal*/") + ); + for invalid in [ + "stage", + "@", + "@stage;SELECT 1", + "@stage--hidden", + "@IDENTIFIER('stage')", + "@db.IDENTIFIER(:1)", + ] { + assert!( + parse_file_transfer_stage_location(invalid).is_err(), + "{invalid}" + ); + } +} + +#[test] +fn snowflake_put_get_opaque_tokens_are_operand_local_and_dialect_scoped() { + let sql = r#"PUT file:///tmp/a--b/*.csv?x#part @"stage/slash"/prefix; SELECT put, get"#; + let tokens = Tokenizer::new(&SnowflakeDialect, sql) + .tokenize_with_location() + .unwrap(); + let operands = tokens + .iter() + .filter(|token| matches!(token.token, Token::FileTransferLocation { .. })) + .collect::>(); + assert_eq!(operands.len(), 2); + assert_eq!(operands[0].span.start, Location::new(1, 5)); + assert_eq!( + operands[1].span.start, + Location::new(1, u64::try_from(sql.find('@').unwrap() + 1).unwrap()) + ); + assert!(tokens.iter().any(|token| token.token == Token::SemiColon)); + let generic = Tokenizer::new(&GenericDialect, "PUT file:///tmp/a.csv @stage") + .tokenize() + .unwrap(); + assert!(!generic + .iter() + .any(|token| matches!(token, Token::FileTransferLocation { .. }))); + snowflake().one_statement_parses_to( + "// PUT file:///ignored/* @hidden\nPUT file:///tmp/a.csv @stage", + "PUT file:///tmp/a.csv @stage", + ); +} + +#[cfg(feature = "visitor")] +#[test] +fn snowflake_put_get_bindings_use_expression_visitors_not_table_relations() { + use core::ops::ControlFlow; + use sqlparser::ast::helpers::stmt_data_loading::FileTransferLocation; + use sqlparser::ast::{visit_expressions, visit_expressions_mut, visit_relations}; + + let mut statement = snowflake().verified_stmt("PUT ? :1 PARALLEL=:2"); + let mut bindings = Vec::new(); + let result = visit_expressions(&statement, |expr| { + if let Expr::Value(ValueWithSpan { + value: Value::Placeholder(value), + .. + }) = expr + { + bindings.push(value.clone()); + } + ControlFlow::<()>::Continue(()) + }); + assert_eq!(result, ControlFlow::Continue(())); + assert_eq!(bindings, vec!["?", ":1", ":2"]); + let result = visit_expressions_mut(&mut statement, |expr| { + if matches!( + expr, + Expr::Value(ValueWithSpan { + value: Value::Placeholder(_), + .. + }) + ) { + *expr = + Expr::Value(Value::SingleQuotedString("@stage/prefix".into()).with_empty_span()); + } + ControlFlow::<()>::Continue(()) + }); + assert_eq!(result, ControlFlow::Continue(())); + assert!(matches!(statement, Statement::Put(ref command) + if matches!(command.target, FileTransferLocation::Binding(Expr::Value(_))))); + + let statement = snowflake().verified_stmt("GET @db.schema.stage/prefix file:///tmp/out"); + let result = visit_relations(&statement, |_| { + ControlFlow::Break("unexpected table relation") + }); + assert_eq!(result, ControlFlow::Continue(())); +} + +#[test] +fn snowflake_bound_stage_prefix_is_data_and_display_remains_one_operand() { + use sqlparser::ast::helpers::stmt_data_loading::FileTransferLocation; + use sqlparser::dialect::parse_file_transfer_stage_location; + + let bound = + parse_file_transfer_stage_location("@stage/prefix with space;a--b/*?x#part").unwrap(); + assert!(bound.quoted); + assert_eq!( + bound.to_string(), + "'@stage/prefix with space;a--b/*?x#part'" + ); + let Statement::Get(command) = + snowflake().verified_stmt(&format!("GET {bound} file:///tmp/download")) + else { + panic!("expected typed GET"); + }; + let FileTransferLocation::Stage(reparsed) = command.source else { + panic!("expected typed stage source"); + }; + assert_eq!(reparsed, bound); +}