Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
0fbf71d
Add draft for basic extension type support
tschwarzinger Feb 12, 2026
a1a3ca6
Add an example for custom extension types
tschwarzinger Feb 12, 2026
5f3781a
Further improvements of the extension type API proposal
tschwarzinger Feb 12, 2026
c0968b5
Formatting
tschwarzinger Feb 12, 2026
8b74d6b
Docs
tschwarzinger Feb 12, 2026
d2df1bb
License headers and formatting
tschwarzinger Feb 12, 2026
e7b865c
Add extension type registry implementation for mock sessions
tschwarzinger Feb 12, 2026
dcb4cab
Fix error in listing_table_factory.rs, Formatting
tschwarzinger Feb 12, 2026
3453c1e
Lints and formatting
tschwarzinger Feb 12, 2026
f07434f
first pass
paleolimbot Mar 19, 2026
c988a09
prototype cast compute
paleolimbot Mar 20, 2026
c577b00
pipe it through
paleolimbot Mar 20, 2026
e479fc1
piped with failing test
paleolimbot Mar 20, 2026
bded4fc
with passing test
paleolimbot Mar 20, 2026
2f89e97
fix test
paleolimbot Mar 20, 2026
397b2cf
fix clippy
paleolimbot Mar 20, 2026
aee9136
more clippy
paleolimbot Mar 20, 2026
567e947
more clippy
paleolimbot Mar 20, 2026
4f82129
fix more merge stuff
paleolimbot Apr 29, 2026
240adae
clean up the diff
paleolimbot Apr 29, 2026
ae03e0e
undo testing change
paleolimbot Apr 29, 2026
a1c1e02
tests passing maybe
paleolimbot Apr 29, 2026
813da1f
Merge remote-tracking branch 'upstream/main' into extension-type-regi…
paleolimbot May 7, 2026
3596273
reintroduce the cast extension
paleolimbot May 7, 2026
1a33151
reimplement the cast extension
paleolimbot May 7, 2026
8bd685f
try to align previous cast extension with the new one
paleolimbot May 7, 2026
57c4bb6
use the new trait
paleolimbot May 7, 2026
e970a2c
cast ext integration
paleolimbot May 7, 2026
b33d2fe
try to integrat into the cast expr
paleolimbot May 7, 2026
1b60f1b
remove some previous work
paleolimbot May 7, 2026
88d20f8
almost with the default extension casting
paleolimbot May 8, 2026
19bca5a
with the cast to
paleolimbot May 8, 2026
afbc183
maybe fix some tests
paleolimbot May 8, 2026
079f505
diff cleanup
paleolimbot May 8, 2026
0a0c7b8
maybe fix more errors
paleolimbot May 8, 2026
000ccf6
remove one more metadata test
paleolimbot May 8, 2026
40907ca
one more clippy thing
paleolimbot May 8, 2026
addddbb
benchark fix
paleolimbot May 8, 2026
0f186eb
more clippy
paleolimbot May 8, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 40 additions & 2 deletions datafusion/common/src/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,16 @@

use std::{collections::BTreeMap, sync::Arc};

use arrow::datatypes::{DataType, Field, FieldRef};
use arrow::{
compute::CastOptions,
datatypes::{DataType, Field, FieldRef},
};
use hashbrown::HashMap;

use crate::{DataFusionError, ScalarValue, error::_plan_err};
use crate::{
DataFusionError, ScalarValue, datatype::DataTypeExt, error::_plan_err,
nested_struct::CastExtension,
};

/// A [`ScalarValue`] with optional [`FieldMetadata`]
#[derive(Debug, Clone)]
Expand Down Expand Up @@ -62,6 +68,38 @@ impl ScalarAndMetadata {
let new_value = self.value().cast_to(target_type)?;
Ok(Self::new(new_value, self.metadata.clone()))
}

/// Try to cast this value to a ScalarValue of type `target_field` with [`CastOptions`]
pub fn cast_to_with_options(
&self,
target_field: &Field,
cast_extension: Option<&dyn CastExtension>,
cast_options: &CastOptions,
) -> Result<Self, DataFusionError> {
let mut source_field = self.value.data_type().into_nullable_field();
if let Some(metadata) = &self.metadata {
source_field = metadata.add_to_field(source_field);
}

if let Some(cast_extension) = cast_extension
&& cast_extension.can_cast_fields(&source_field, target_field)?
{
let cast_arr = cast_extension.cast_array_fields(
&self.value.to_array()?,
&source_field,
target_field,
cast_options,
)?;
let storage = ScalarValue::try_from_array(&cast_arr, 0)?;
let metadata = FieldMetadata::new_from_field(target_field);
return Ok(Self {
value: storage,
metadata: Some(metadata),
});
}

self.cast_storage_to(target_field.data_type())
}
}

/// create a new ScalarAndMetadata from a ScalarValue without
Expand Down
Loading
Loading