Skip to content

[SPARK-57716][SQL][TESTS] Add SORT / ORDER BY and window correctness tests for nanosecond-precision timestamp types #56811

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
stevomitric wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[SPARK-57716][SQL][TESTS] Add SORT / ORDER BY and window correctness tests for nanosecond-precision timestamp types #56811

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -971,3 +971,66 @@ SELECT typeof(CASE WHEN true
-- !query analysis
Project [typeof(CASE WHEN true THEN cast(cast(1969-12-31 23:59:59.1234567 as timestamp_ntz(7)) as timestamp_ltz(9)) ELSE cast(1970-01-01 00:00:00.123456789 UTC as timestamp_ltz(9)) END) AS typeof(CASE WHEN true THEN CAST(1969-12-31 23:59:59.1234567 AS TIMESTAMP_NTZ(7)) ELSE CAST(1970-01-01 00:00:00.123456789 UTC AS TIMESTAMP_LTZ(9)) END)#x]
+- OneRowRelation


-- !query
SELECT v FROM (
SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000001000' AS v
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001') ORDER BY v
-- !query analysis
Sort [v#x ASC NULLS FIRST], true
+- Project [v#x]
+- SubqueryAlias __auto_generated_subquery_name
+- Union false, false
:- Union false, false
: :- Project [2020-01-01 00:00:00.000001 AS v#x]
: : +- OneRowRelation
: +- Project [2020-01-01 00:00:00.000000999 AS TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999'#x]
: +- OneRowRelation
+- Project [2020-01-01 00:00:00.000000001 AS TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001'#x]
+- OneRowRelation


-- !query
SELECT v, row_number() OVER (ORDER BY v) AS rn FROM (
SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500') ORDER BY rn
-- !query analysis
Sort [rn#x ASC NULLS FIRST], true
+- Project [v#x, rn#x]
+- Project [v#x, rn#x, rn#x]
+- Window [row_number() windowspecdefinition(v#x ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#x], [v#x ASC NULLS FIRST]
+- Project [v#x]
+- SubqueryAlias __auto_generated_subquery_name
+- Union false, false
:- Union false, false
: :- Project [2020-01-01 00:00:00.0000009 AS v#x]
: : +- OneRowRelation
: +- Project [2020-01-01 00:00:00.0000001 AS TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100'#x]
: +- OneRowRelation
+- Project [2020-01-01 00:00:00.0000005 AS TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500'#x]
+- OneRowRelation


-- !query
SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM (
SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500') ORDER BY v
-- !query analysis
Sort [v#x ASC NULLS FIRST], true
+- Project [v#x, next_v#x]
+- Project [v#x, next_v#x, next_v#x]
+- Window [lead(v#x, 1, null) windowspecdefinition(v#x ASC NULLS FIRST, specifiedwindowframe(RowFrame, 1, 1)) AS next_v#x], [v#x ASC NULLS FIRST]
+- Project [v#x]
+- SubqueryAlias __auto_generated_subquery_name
+- Union false, false
:- Union false, false
: :- Project [2020-01-01 00:00:00.0000009 AS v#x]
: : +- OneRowRelation
: +- Project [2020-01-01 00:00:00.0000001 AS TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100'#x]
: +- OneRowRelation
+- Project [2020-01-01 00:00:00.0000005 AS TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500'#x]
+- OneRowRelation
63 changes: 63 additions & 0 deletions sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -805,3 +805,66 @@ SELECT map('min', '0001-01-01 00:00:00.000000001' :: timestamp_ntz(9),
-- !query analysis
Project [map(min, cast(0001-01-01 00:00:00.000000001 as timestamp_ntz(9)), max, cast(9999-12-31 23:59:59.999999 as timestamp_ntz(9))) AS map(min, CAST(0001-01-01 00:00:00.000000001 AS TIMESTAMP_NTZ(9)), max, TIMESTAMP_NTZ '9999-12-31 23:59:59.999999')#x]
+- OneRowRelation


-- !query
SELECT v FROM (
SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000001000' AS v
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') ORDER BY v
-- !query analysis
Sort [v#x ASC NULLS FIRST], true
+- Project [v#x]
+- SubqueryAlias __auto_generated_subquery_name
+- Union false, false
:- Union false, false
: :- Project [2020-01-01 00:00:00.000001 AS v#x]
: : +- OneRowRelation
: +- Project [2020-01-01 00:00:00.000000999 AS TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'#x]
: +- OneRowRelation
+- Project [2020-01-01 00:00:00.000000001 AS TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'#x]
+- OneRowRelation


-- !query
SELECT v, row_number() OVER (ORDER BY v) AS rn FROM (
SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500') ORDER BY rn
-- !query analysis
Sort [rn#x ASC NULLS FIRST], true
+- Project [v#x, rn#x]
+- Project [v#x, rn#x, rn#x]
+- Window [row_number() windowspecdefinition(v#x ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#x], [v#x ASC NULLS FIRST]
+- Project [v#x]
+- SubqueryAlias __auto_generated_subquery_name
+- Union false, false
:- Union false, false
: :- Project [2020-01-01 00:00:00.0000009 AS v#x]
: : +- OneRowRelation
: +- Project [2020-01-01 00:00:00.0000001 AS TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100'#x]
: +- OneRowRelation
+- Project [2020-01-01 00:00:00.0000005 AS TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500'#x]
+- OneRowRelation


-- !query
SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM (
SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500') ORDER BY v
-- !query analysis
Sort [v#x ASC NULLS FIRST], true
+- Project [v#x, next_v#x]
+- Project [v#x, next_v#x, next_v#x]
+- Window [lead(v#x, 1, null) windowspecdefinition(v#x ASC NULLS FIRST, specifiedwindowframe(RowFrame, 1, 1)) AS next_v#x], [v#x ASC NULLS FIRST]
+- Project [v#x]
+- SubqueryAlias __auto_generated_subquery_name
+- Union false, false
:- Union false, false
: :- Project [2020-01-01 00:00:00.0000009 AS v#x]
: : +- OneRowRelation
: +- Project [2020-01-01 00:00:00.0000001 AS TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100'#x]
: +- OneRowRelation
+- Project [2020-01-01 00:00:00.0000005 AS TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500'#x]
+- OneRowRelation
19 changes: 19 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,3 +309,22 @@ SELECT typeof(coalesce('0001-01-01 00:00:00.0000001' :: timestamp_ntz(7),
SELECT typeof(CASE WHEN true
THEN '1969-12-31 23:59:59.1234567' :: timestamp_ntz(7)
ELSE '1970-01-01 00:00:00.123456789 UTC' :: timestamp_ltz(9) END);

-- SORT / ORDER BY tie-breaks on the sub-microsecond remainder: 001 and 999 share a microsecond,
-- 1000 rolls into the next, so a micro-truncating sort would misorder them (full value 001<999<1000).
SELECT v FROM (
SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000001000' AS v
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001') ORDER BY v;

-- row_number() over a nanosecond ORDER BY key: the row numbers follow the sub-microsecond order.
SELECT v, row_number() OVER (ORDER BY v) AS rn FROM (
SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500') ORDER BY rn;

-- lead() over a nanosecond ORDER BY key returns the next sub-microsecond value (carrier round-trip).
SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM (
SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500') ORDER BY v;
19 changes: 19 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,3 +249,22 @@ SELECT typeof(array(TIMESTAMP_NTZ '9999-12-31 23:59:59',
'0001-01-01 00:00:00.000000001' :: timestamp_ntz(9)));
SELECT map('min', '0001-01-01 00:00:00.000000001' :: timestamp_ntz(9),
'max', TIMESTAMP_NTZ '9999-12-31 23:59:59.999999');

-- SORT / ORDER BY tie-breaks on the sub-microsecond remainder: 001 and 999 share a microsecond,
-- 1000 rolls into the next, so a micro-truncating sort would misorder them (full value 001<999<1000).
SELECT v FROM (
SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000001000' AS v
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') ORDER BY v;

-- row_number() over a nanosecond ORDER BY key: the row numbers follow the sub-microsecond order.
SELECT v, row_number() OVER (ORDER BY v) AS rn FROM (
SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500') ORDER BY rn;

-- lead() over a nanosecond ORDER BY key returns the next sub-microsecond value (carrier round-trip).
SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM (
SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500') ORDER BY v;
39 changes: 39 additions & 0 deletions sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1086,3 +1086,42 @@ SELECT typeof(CASE WHEN true
struct<typeof(CASE WHEN true THEN CAST(1969-12-31 23:59:59.1234567 AS TIMESTAMP_NTZ(7)) ELSE CAST(1970-01-01 00:00:00.123456789 UTC AS TIMESTAMP_LTZ(9)) END):string>
-- !query output
timestamp_ltz(9)


-- !query
SELECT v FROM (
SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000001000' AS v
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001') ORDER BY v
-- !query schema
struct<v:timestamp_ltz(9)>
-- !query output
2020-01-01 00:00:00.000000001
2020-01-01 00:00:00.000000999
2020-01-01 00:00:00.000001


-- !query
SELECT v, row_number() OVER (ORDER BY v) AS rn FROM (
SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500') ORDER BY rn
-- !query schema
struct<v:timestamp_ltz(9),rn:int>
-- !query output
2020-01-01 00:00:00.0000001 1
2020-01-01 00:00:00.0000005 2
2020-01-01 00:00:00.0000009 3


-- !query
SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM (
SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500') ORDER BY v
-- !query schema
struct<v:timestamp_ltz(9),next_v:timestamp_ltz(9)>
-- !query output
2020-01-01 00:00:00.0000001 2020-01-01 00:00:00.0000005
2020-01-01 00:00:00.0000005 2020-01-01 00:00:00.0000009
2020-01-01 00:00:00.0000009 NULL
39 changes: 39 additions & 0 deletions sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -883,3 +883,42 @@ SELECT map('min', '0001-01-01 00:00:00.000000001' :: timestamp_ntz(9),
struct<map(min, CAST(0001-01-01 00:00:00.000000001 AS TIMESTAMP_NTZ(9)), max, TIMESTAMP_NTZ '9999-12-31 23:59:59.999999'):map<string,timestamp_ntz(9)>>
-- !query output
{"max":9999-12-31 23:59:59.999999,"min":0001-01-01 00:00:00.000000001}


-- !query
SELECT v FROM (
SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000001000' AS v
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') ORDER BY v
-- !query schema
struct<v:timestamp_ntz(9)>
-- !query output
2020-01-01 00:00:00.000000001
2020-01-01 00:00:00.000000999
2020-01-01 00:00:00.000001


-- !query
SELECT v, row_number() OVER (ORDER BY v) AS rn FROM (
SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500') ORDER BY rn
-- !query schema
struct<v:timestamp_ntz(9),rn:int>
-- !query output
2020-01-01 00:00:00.0000001 1
2020-01-01 00:00:00.0000005 2
2020-01-01 00:00:00.0000009 3


-- !query
SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM (
SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900' AS v
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500') ORDER BY v
-- !query schema
struct<v:timestamp_ntz(9),next_v:timestamp_ntz(9)>
-- !query output
2020-01-01 00:00:00.0000001 2020-01-01 00:00:00.0000005
2020-01-01 00:00:00.0000005 2020-01-01 00:00:00.0000009
2020-01-01 00:00:00.0000009 NULL
Loading