diff --git a/Cargo.lock b/Cargo.lock index 012573deb452d..a994ee58abfd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5737,9 +5737,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "recursive", diff --git a/Cargo.toml b/Cargo.toml index a9223c44a47a2..7194b2c63a86e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -192,7 +192,7 @@ regex = "1.12" rstest = "0.26.1" serde_json = "1" sha2 = "^0.11.0" -sqlparser = { version = "0.61.0", default-features = false, features = ["std", "visitor"] } +sqlparser = { version = "0.62.0", default-features = false, features = ["std", "visitor"] } strum = "0.28.0" strum_macros = "0.28.0" tempfile = "3" diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index 582e8e41dd0cf..8ce8a26501cf9 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -4123,8 +4123,8 @@ mod test { wildcard_with_options(wildcard_options( None, Some(ExcludeSelectItem::Multiple(vec![ - Ident::from("c1"), - Ident::from("c2") + Ident::from("c1").into(), + Ident::from("c2").into() ])), None, None, diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs index fdb4b6de7874a..e0e2330cd1ad8 100644 --- a/datafusion/expr/src/utils.rs +++ b/datafusion/expr/src/utils.rs @@ -339,7 +339,38 @@ fn get_excluded_columns( idents.push(&excepts.first_element); idents.extend(&excepts.additional_elements); } + // Declared outside the `if let` so `idents.extend(exclude_owned.iter())` + // below can borrow references that outlive the inner scope. + #[cfg(feature = "sql")] + let exclude_owned: Vec; if let Some(exclude) = opt_exclude { + #[cfg(feature = "sql")] + { + let object_name_to_ident = + |name: &sqlparser::ast::ObjectName| -> Result { + if name.0.len() != 1 { + return plan_err!( + "EXCLUDE with multi-part identifiers is not supported: {name}" + ); + } + let part = &name.0[0]; + let Some(ident) = part.as_ident() else { + return plan_err!( + "EXCLUDE with non-identifier name part is not supported: {part}" + ); + }; + Ok(ident.clone()) + }; + exclude_owned = match exclude { + ExcludeSelectItem::Single(name) => vec![object_name_to_ident(name)?], + ExcludeSelectItem::Multiple(names) => names + .iter() + .map(object_name_to_ident) + .collect::>>()?, + }; + idents.extend(exclude_owned.iter()); + } + #[cfg(not(feature = "sql"))] match exclude { ExcludeSelectItem::Single(ident) => idents.push(ident), ExcludeSelectItem::Multiple(idents_inner) => idents.extend(idents_inner), diff --git a/datafusion/sql/src/expr/function.rs b/datafusion/sql/src/expr/function.rs index 1790e66a027bb..67abb8b822063 100644 --- a/datafusion/sql/src/expr/function.rs +++ b/datafusion/sql/src/expr/function.rs @@ -927,10 +927,15 @@ impl SqlToRel<'_, S> { ); } + if lambda.params.iter().any(|p| p.data_type.is_some()) { + return not_impl_err!( + "Lambda parameters with explicit data types are not supported" + ); + } let params = lambda .params .iter() - .map(|p| crate::utils::normalize_ident(p.clone())) + .map(|p| crate::utils::normalize_ident(p.name.clone())) .collect(); let lambda_parameters = std::iter::zip(lambda_params, ¶ms) @@ -1181,19 +1186,19 @@ impl SqlToRel<'_, S> { /// After normalization with [normalize_ident], check whether all params are unique /// /// [normalize_ident]: crate::utils::normalize_ident -fn all_unique(params: &[sqlparser::ast::Ident]) -> bool { +fn all_unique(params: &[sqlparser::ast::LambdaFunctionParameter]) -> bool { match params.len() { 0 | 1 => true, 2 => { - crate::utils::normalize_ident(params[0].clone()) - != crate::utils::normalize_ident(params[1].clone()) + crate::utils::normalize_ident(params[0].name.clone()) + != crate::utils::normalize_ident(params[1].name.clone()) } _ => { let mut set = HashSet::with_capacity(params.len()); params .iter() - .map(|p| crate::utils::normalize_ident(p.clone())) + .map(|p| crate::utils::normalize_ident(p.name.clone())) .all(|p| set.insert(p)) } } diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index 5cbc1c84bdb4b..daf092ecd4cf9 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -900,7 +900,7 @@ impl SqlToRel<'_, S> { negated: bool, expr: SQLExpr, pattern: SQLExpr, - escape_char: Option, + escape_char: Option, schema: &DFSchema, planner_context: &mut PlannerContext, case_insensitive: bool, @@ -910,7 +910,7 @@ impl SqlToRel<'_, S> { return not_impl_err!("ANY in LIKE expression"); } let pattern = self.sql_expr_to_logical_expr(pattern, schema, planner_context)?; - let escape_char = match escape_char { + let escape_char = match escape_char.map(|v| v.value) { Some(Value::SingleQuotedString(char)) if char.len() == 1 => { Some(char.chars().next().unwrap()) } @@ -935,7 +935,7 @@ impl SqlToRel<'_, S> { negated: bool, expr: SQLExpr, pattern: SQLExpr, - escape_char: Option, + escape_char: Option, schema: &DFSchema, planner_context: &mut PlannerContext, ) -> Result { @@ -944,7 +944,7 @@ impl SqlToRel<'_, S> { if pattern_type != DataType::Utf8 && pattern_type != DataType::Null { return plan_err!("Invalid pattern in SIMILAR TO expression"); } - let escape_char = match escape_char { + let escape_char = match escape_char.map(|v| v.value) { Some(Value::SingleQuotedString(char)) if char.len() == 1 => { Some(char.chars().next().unwrap()) } diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index e320d2ee6e9c1..76124cbc7eb59 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -171,6 +171,7 @@ impl SqlToRel<'_, S> { // Apply to all fields columns: vec![], explicit: true, + at: None, }, ), PipeOperator::Union { diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index 09d8566c4a19e..b7f7d80e70815 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -839,6 +839,9 @@ impl SqlToRel<'_, S> { Ok(SelectExpr::Expression(expr)) } + SelectItem::ExprWithAliases { .. } => { + not_impl_err!("SELECT item with multiple aliases is not supported") + } SelectItem::Wildcard(options) => { Self::check_wildcard_options(&options)?; if empty_from { @@ -886,11 +889,14 @@ impl SqlToRel<'_, S> { opt_rename, opt_replace: _opt_replace, opt_ilike: _opt_ilike, + opt_alias, wildcard_token: _wildcard_token, } = options; if opt_rename.is_some() { not_impl_err!("wildcard * with RENAME not supported ") + } else if opt_alias.is_some() { + not_impl_err!("wildcard * with AS alias not supported") } else { Ok(()) } diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 587ed02d13188..e7088c8a3d6f1 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -344,6 +344,12 @@ impl SqlToRel<'_, S> { require_user, partition_of, for_values, + snapshot, + with_storage_lifecycle_policy, + diststyle, + distkey, + sortkey, + backup, }) => { if temporary { return not_impl_err!("Temporary tables not supported"); @@ -497,6 +503,24 @@ impl SqlToRel<'_, S> { if for_values.is_some() { return not_impl_err!("PARTITION OF .. FOR VALUES .. not supported"); } + if snapshot { + return not_impl_err!("Snapshot tables not supported"); + } + if with_storage_lifecycle_policy.is_some() { + return not_impl_err!("WITH STORAGE LIFECYCLE POLICY not supported"); + } + if diststyle.is_some() { + return not_impl_err!("DISTSTYLE not supported"); + } + if distkey.is_some() { + return not_impl_err!("DISTKEY not supported"); + } + if sortkey.is_some() { + return not_impl_err!("SORTKEY not supported"); + } + if backup.is_some() { + return not_impl_err!("BACKUP not supported"); + } // Merge inline constraints and existing constraints let mut all_constraints = constraints; let inline_constraints = calc_inline_constraints_from_columns(&columns); @@ -604,6 +628,7 @@ impl SqlToRel<'_, S> { or_alter, secure, name_before_not_exists, + copy_grants, }) => { if materialized { return not_impl_err!("Materialized views not supported")?; @@ -623,6 +648,9 @@ impl SqlToRel<'_, S> { if to.is_some() { return not_impl_err!("To not supported")?; } + if copy_grants { + return not_impl_err!("COPY GRANTS not supported")?; + } // put the statement back together temporarily to get the SQL // string representation @@ -643,6 +671,7 @@ impl SqlToRel<'_, S> { or_alter, secure, name_before_not_exists, + copy_grants, }); let sql = stmt.to_string(); let Statement::CreateView(ast::CreateView { @@ -1000,7 +1029,12 @@ impl SqlToRel<'_, S> { settings, format_clause, insert_token: _, // record the location the `INSERT` token - optimizer_hint, + optimizer_hints, + output, + multi_table_insert_type, + multi_table_into_clauses, + multi_table_when_clauses, + multi_table_else_clause, }) => { let table_name = match table { TableObject::TableName(table_name) => table_name, @@ -1009,6 +1043,11 @@ impl SqlToRel<'_, S> { "INSERT INTO Table functions not supported" ); } + TableObject::TableQuery(_) => { + return not_impl_err!( + "INSERT INTO subquery target not supported" + ); + } }; if let Some(or) = or { match or { @@ -1056,9 +1095,19 @@ impl SqlToRel<'_, S> { if format_clause.is_some() { plan_err!("Inserts with format clause not supported")?; } - if optimizer_hint.is_some() { + if !optimizer_hints.is_empty() { plan_err!("Optimizer hints not supported")?; } + if output.is_some() { + plan_err!("Insert OUTPUT clause not supported")?; + } + if multi_table_insert_type.is_some() + || !multi_table_into_clauses.is_empty() + || !multi_table_when_clauses.is_empty() + || multi_table_else_clause.is_some() + { + plan_err!("Multi-table INSERT not supported")?; + } // optional keywords don't change behavior let _ = into; let _ = has_table_keyword; @@ -1073,7 +1122,9 @@ impl SqlToRel<'_, S> { or, limit, update_token: _, - optimizer_hint, + optimizer_hints, + output, + order_by, }) => { let from_clauses = from.map(|update_table_from_kind| match update_table_from_kind { @@ -1103,9 +1154,15 @@ impl SqlToRel<'_, S> { if limit.is_some() { return not_impl_err!("Update-limit clause not supported")?; } - if optimizer_hint.is_some() { + if !optimizer_hints.is_empty() { plan_err!("Optimizer hints not supported")?; } + if output.is_some() { + plan_err!("Update OUTPUT clause not supported")?; + } + if !order_by.is_empty() { + plan_err!("Update ORDER BY not supported")?; + } self.update_to_plan(table, &assignments, update_from, selection) } @@ -1118,7 +1175,8 @@ impl SqlToRel<'_, S> { order_by, limit, delete_token: _, - optimizer_hint, + optimizer_hints, + output, }) => { if !tables.is_empty() { plan_err!("DELETE not supported")?; @@ -1136,9 +1194,12 @@ impl SqlToRel<'_, S> { plan_err!("Delete-order-by clause not yet supported")?; } - if optimizer_hint.is_some() { + if !optimizer_hints.is_empty() { plan_err!("Optimizer hints not supported")?; } + if output.is_some() { + plan_err!("Delete OUTPUT clause not supported")?; + } let table_name = self.get_delete_target(from)?; self.delete_to_plan(&table_name, selection, limit) @@ -1260,7 +1321,14 @@ impl SqlToRel<'_, S> { .. }) => { let return_type = match return_type { - Some(t) => Some(self.convert_data_type_to_field(&t)?), + Some(ast::FunctionReturnType::DataType(t)) => { + Some(self.convert_data_type_to_field(&t)?) + } + Some(ast::FunctionReturnType::SetOf(_)) => { + return not_impl_err!( + "RETURNS SETOF in CREATE FUNCTION is not supported" + ); + } None => None, }; let mut planner_context = PlannerContext::new(); @@ -1882,6 +1950,16 @@ impl SqlToRel<'_, S> { TableConstraint::FulltextOrSpatial { .. } => { _plan_err!("Indexes are not currently supported") } + TableConstraint::PrimaryKeyUsingIndex(_) => { + _plan_err!( + "PRIMARY KEY USING INDEX constraints are not currently supported" + ) + } + TableConstraint::UniqueUsingIndex(_) => { + _plan_err!( + "UNIQUE USING INDEX constraints are not currently supported" + ) + } }) .collect::>>()?; Ok(Constraints::new_unverified(constraints)) @@ -2276,7 +2354,7 @@ impl SqlToRel<'_, S> { fn insert_to_plan( &self, table_name: ObjectName, - columns: Vec, + columns: Vec, source: Box, overwrite: bool, replace_into: bool, @@ -2286,6 +2364,24 @@ impl SqlToRel<'_, S> { let table_source = self.context_provider.get_table_source(table_name.clone())?; let table_schema = DFSchema::try_from(table_source.schema())?; + let columns: Vec = columns + .into_iter() + .map(|name| { + if name.0.len() != 1 { + return not_impl_err!( + "Multi-part column names in INSERT not supported: {name}" + ); + } + let part = &name.0[0]; + let Some(ident) = part.as_ident() else { + return not_impl_err!( + "Non-identifier column name part in INSERT not supported: {part}" + ); + }; + Ok(ident.clone()) + }) + .collect::>>()?; + // Get insert fields and target table's value indices // // If value_indices[i] = Some(j), it means that the value of the i-th target table's column is @@ -2329,7 +2425,7 @@ impl SqlToRel<'_, S> { let mut prepare_param_data_types = BTreeMap::new(); if let SetExpr::Values(ast::Values { rows, .. }) = (*source.body).clone() { for row in rows.iter() { - for (idx, val) in row.iter().enumerate() { + for (idx, val) in row.content.iter().enumerate() { if let SQLExpr::Value(ValueWithSpan { value: Value::Placeholder(name), span: _, @@ -2581,7 +2677,7 @@ ON p.function_name = r.routine_name None => Ok(()), // BEGIN TRANSACTION Some(BeginTransactionKind::Transaction) => Ok(()), - Some(BeginTransactionKind::Work) => { + Some(BeginTransactionKind::Work) | Some(BeginTransactionKind::Tran) => { not_impl_err!("Transaction kind not supported: {kind:?}") } } diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rs index d8d5ec9e409fc..4b4e56c40cdc5 100644 --- a/datafusion/sql/src/unparser/ast.rs +++ b/datafusion/sql/src/unparser/ast.rs @@ -358,7 +358,7 @@ impl SelectBuilder { } pub fn build(&self) -> Result { Ok(ast::Select { - optimizer_hint: None, + optimizer_hints: vec![], distinct: self.distinct.clone(), select_modifiers: None, top_before_distinct: false, @@ -477,6 +477,9 @@ pub struct RelationBuilder { } #[derive(Clone)] +// Boxing variants would penalize the common builder path; this enum is +// constructed-then-consumed locally rather than stored at scale. +#[expect(clippy::large_enum_variant)] enum TableFactorBuilder { Table(TableRelationBuilder), Derived(DerivedRelationBuilder), @@ -794,6 +797,7 @@ impl FlattenRelationBuilder { lateral: true, name: ast::ObjectName::from(vec![ast::Ident::new("FLATTEN")]), args, + with_ordinality: false, alias: self.alias.clone(), }) } diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index e76aea5849492..5ec20a8826253 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -291,7 +291,8 @@ impl Unparser<'_> { negated: *negated, expr: Box::new(self.expr_to_sql_inner(expr)?), pattern: Box::new(self.expr_to_sql_inner(pattern)?), - escape_char: escape_char.map(|c| SingleQuotedString(c.to_string())), + escape_char: escape_char + .map(|c| SingleQuotedString(c.to_string()).into()), any: false, }), Expr::Like(Like { @@ -307,7 +308,7 @@ impl Unparser<'_> { expr: Box::new(self.expr_to_sql_inner(expr)?), pattern: Box::new(self.expr_to_sql_inner(pattern)?), escape_char: escape_char - .map(|c| SingleQuotedString(c.to_string())), + .map(|c| SingleQuotedString(c.to_string()).into()), any: false, }) } else { @@ -316,7 +317,7 @@ impl Unparser<'_> { expr: Box::new(self.expr_to_sql_inner(expr)?), pattern: Box::new(self.expr_to_sql_inner(pattern)?), escape_char: escape_char - .map(|c| SingleQuotedString(c.to_string())), + .map(|c| SingleQuotedString(c.to_string()).into()), any: false, }) } @@ -572,7 +573,10 @@ impl Unparser<'_> { params: ast::OneOrManyWithParens::Many( params .iter() - .map(|param| self.new_ident_quoted_if_needs(param.clone())) + .map(|param| ast::LambdaFunctionParameter { + name: self.new_ident_quoted_if_needs(param.clone()), + data_type: None, + }) .collect(), ), body: Box::new(self.expr_to_sql_inner(body)?), diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index 2c36fe0b2c98a..43d7ef49d9453 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -434,6 +434,7 @@ impl Unparser<'_> { name: Ident::with_quote('"', &flatten_alias_name), columns: vec![], explicit: true, + at: None, })); if !select.already_projected() { @@ -1208,6 +1209,7 @@ impl Unparser<'_> { name: Ident::with_quote('"', &alias), columns: vec![], explicit: true, + at: None, })); } relation.flatten(flatten_relation); @@ -1902,6 +1904,7 @@ impl Unparser<'_> { name: self.new_ident_quoted_if_needs(alias), columns, explicit: true, + at: None, } } diff --git a/datafusion/sql/src/values.rs b/datafusion/sql/src/values.rs index c8cdf1254f33f..a1df1fe1b18d1 100644 --- a/datafusion/sql/src/values.rs +++ b/datafusion/sql/src/values.rs @@ -43,7 +43,8 @@ impl SqlToRel<'_, S> { let values = rows .into_iter() .map(|row| { - row.into_iter() + row.content + .into_iter() .map(|v| self.sql_to_expr(v, &empty_schema, planner_context)) .collect::>>() }) diff --git a/datafusion/sqllogictest/test_files/array/array_transform.slt b/datafusion/sqllogictest/test_files/array/array_transform.slt index f726d265d9d63..f87253695d332 100644 --- a/datafusion/sqllogictest/test_files/array/array_transform.slt +++ b/datafusion/sqllogictest/test_files/array/array_transform.slt @@ -411,13 +411,13 @@ SELECT array_transform([1, 2], (e, i, j) -> i); query error DataFusion error: Error during planning: lambda parameters names must be unique, got \(v, v\) SELECT array_transform([1], (v, v) -> v*2); -query error DataFusion error: This feature is not implemented: Unsupported ast node in sqltorel: Lambda\(LambdaFunction \{ params: One\(Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,12\)\.\.Location\(1,13\)\) \}\), body: Identifier\(Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,17\)\.\.Location\(1,18\)\) \}\), syntax: Arrow \}\) +query error DataFusion error: This feature is not implemented: Unsupported ast node in sqltorel: Lambda\(LambdaFunction \{ params: One\(LambdaFunctionParameter \{ name: Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,12\)\.\.Location\(1,13\)\) \}, data_type: None \}\), body: Identifier\(Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,17\)\.\.Location\(1,18\)\) \}\), syntax: Arrow \}\) SELECT abs(v -> v); -query error DataFusion error: This feature is not implemented: Unsupported ast node in sqltorel: Lambda\(LambdaFunction \{ params: One\(Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,8\)\.\.Location\(1,9\)\) \}\), body: Identifier\(Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,13\)\.\.Location\(1,14\)\) \}\), syntax: Arrow \}\) +query error DataFusion error: This feature is not implemented: Unsupported ast node in sqltorel: Lambda\(LambdaFunction \{ params: One\(LambdaFunctionParameter \{ name: Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,8\)\.\.Location\(1,9\)\) \}, data_type: None \}\), body: Identifier\(Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,13\)\.\.Location\(1,14\)\) \}\), syntax: Arrow \}\) SELECT v -> v; -query error DataFusion error: This feature is not implemented: Unsupported ast node in sqltorel: Lambda\(LambdaFunction \{ params: One\(Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,34\)\.\.Location\(1,35\)\) \}\), body: BinaryOp \{ left: Identifier\(Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,39\)\.\.Location\(1,40\)\) \}\), op: Plus, right: Value\(ValueWithSpan \{ value: Number\("1", false\), span: Span\(Location\(1,41\)\.\.Location\(1,42\)\) \}\) \}, syntax: Arrow \}\) +query error DataFusion error: This feature is not implemented: Unsupported ast node in sqltorel: Lambda\(LambdaFunction \{ params: One\(LambdaFunctionParameter \{ name: Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,34\)\.\.Location\(1,35\)\) \}, data_type: None \}\), body: BinaryOp \{ left: Identifier\(Ident \{ value: "v", quote_style: None, span: Span\(Location\(1,39\)\.\.Location\(1,40\)\) \}\), op: Plus, right: Value\(ValueWithSpan \{ value: Number\("1", false\), span: Span\(Location\(1,41\)\.\.Location\(1,42\)\) \}\) \}, syntax: Arrow \}\) SELECT array_transform([1], v -> v -> v+1); query error DataFusion error: SQL error: ParserError\("Expected: an expression, found: \) at Line: 1, Column: 30"\)