From 47c4fe19f64765505e1daacb445fb5aa48d32e28 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Tue, 4 Jul 2023 16:40:22 +0100 Subject: [PATCH] Update scalar timestamp arithmetic tests --- datafusion/common/src/scalar.rs | 309 +------------------------------- 1 file changed, 5 insertions(+), 304 deletions(-) diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index 112cb7a3a26c..0fe5d3d16ecc 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -5756,25 +5756,11 @@ mod tests { } } - #[test] - fn timestamp_op_tests() { - // positive interval, edge cases - let test_data = get_timestamp_test_data(1); - for (lhs, rhs, expected) in test_data.into_iter() { - assert_eq!(expected, lhs.sub(rhs).unwrap()) - } - - // negative interval, edge cases - let test_data = get_timestamp_test_data(-1); - for (rhs, lhs, expected) in test_data.into_iter() { - assert_eq!(expected, lhs.sub(rhs).unwrap()); - } - } #[test] fn timestamp_op_random_tests() { // timestamp1 + (or -) interval = timestamp2 // timestamp2 - timestamp1 (or timestamp1 - timestamp2) = interval ? - let sample_size = 1000000; + let sample_size = 1000; let timestamps1 = get_random_timestamps(sample_size); let intervals = get_random_intervals(sample_size); // ts(sec) + interval(ns) = ts(sec); however, @@ -5783,18 +5769,12 @@ mod tests { for (idx, ts1) in timestamps1.iter().enumerate() { if idx % 2 == 0 { let timestamp2 = ts1.add(intervals[idx].clone()).unwrap(); - assert_eq!( - intervals[idx], - timestamp2.sub(ts1).unwrap(), - "index:{idx}, operands: {timestamp2:?} (-) {ts1:?}" - ); + let back = timestamp2.sub(intervals[idx].clone()).unwrap(); + assert_eq!(ts1, &back); } else { let timestamp2 = ts1.sub(intervals[idx].clone()).unwrap(); - assert_eq!( - intervals[idx], - ts1.sub(timestamp2.clone()).unwrap(), - "index:{idx}, operands: {ts1:?} (-) {timestamp2:?}" - ); + let back = timestamp2.add(intervals[idx].clone()).unwrap(); + assert_eq!(ts1, &back); }; } } @@ -5879,285 +5859,6 @@ mod tests { check_array(array); } - fn get_timestamp_test_data( - sign: i32, - ) -> Vec<(ScalarValue, ScalarValue, ScalarValue)> { - vec![ - ( - // 1st test case, having the same time but different with timezones - // Since they are timestamps with nanosecond precision, expected type is - // [`IntervalMonthDayNanoType`] - ScalarValue::TimestampNanosecond( - Some( - NaiveDate::from_ymd_opt(2023, 1, 1) - .unwrap() - .and_hms_nano_opt(12, 0, 0, 000_000_000) - .unwrap() - .timestamp_nanos(), - ), - Some("+12:00".into()), - ), - ScalarValue::TimestampNanosecond( - Some( - NaiveDate::from_ymd_opt(2023, 1, 1) - .unwrap() - .and_hms_nano_opt(0, 0, 0, 000_000_000) - .unwrap() - .timestamp_nanos(), - ), - Some("+00:00".into()), - ), - ScalarValue::new_interval_mdn(0, 0, 0), - ), - // 2nd test case, january with 31 days plus february with 28 days, with timezone - ( - ScalarValue::TimestampMicrosecond( - Some( - NaiveDate::from_ymd_opt(2023, 3, 1) - .unwrap() - .and_hms_micro_opt(2, 0, 0, 000_000) - .unwrap() - .timestamp_micros(), - ), - Some("+01:00".into()), - ), - ScalarValue::TimestampMicrosecond( - Some( - NaiveDate::from_ymd_opt(2023, 1, 1) - .unwrap() - .and_hms_micro_opt(0, 0, 0, 000_000) - .unwrap() - .timestamp_micros(), - ), - Some("-01:00".into()), - ), - ScalarValue::new_interval_mdn(0, sign * 59, 0), - ), - // 3rd test case, 29-days long february minus previous, year with timezone - ( - ScalarValue::TimestampMillisecond( - Some( - NaiveDate::from_ymd_opt(2024, 2, 29) - .unwrap() - .and_hms_milli_opt(10, 10, 0, 000) - .unwrap() - .timestamp_millis(), - ), - Some("+10:10".into()), - ), - ScalarValue::TimestampMillisecond( - Some( - NaiveDate::from_ymd_opt(2023, 12, 31) - .unwrap() - .and_hms_milli_opt(1, 0, 0, 000) - .unwrap() - .timestamp_millis(), - ), - Some("+01:00".into()), - ), - ScalarValue::new_interval_dt(sign * 60, 0), - ), - // 4th test case, leap years occur mostly every 4 years, but every 100 years - // we skip a leap year unless the year is divisible by 400, so 31 + 28 = 59 - ( - ScalarValue::TimestampSecond( - Some( - NaiveDate::from_ymd_opt(2100, 3, 1) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap() - .timestamp(), - ), - Some("-11:59".into()), - ), - ScalarValue::TimestampSecond( - Some( - NaiveDate::from_ymd_opt(2100, 1, 1) - .unwrap() - .and_hms_opt(23, 58, 0) - .unwrap() - .timestamp(), - ), - Some("+11:59".into()), - ), - ScalarValue::new_interval_dt(sign * 59, 0), - ), - // 5th test case, without timezone positively seemed, but with timezone, - // negative resulting interval - ( - ScalarValue::TimestampMillisecond( - Some( - NaiveDate::from_ymd_opt(2023, 1, 1) - .unwrap() - .and_hms_milli_opt(6, 00, 0, 000) - .unwrap() - .timestamp_millis(), - ), - Some("+06:00".into()), - ), - ScalarValue::TimestampMillisecond( - Some( - NaiveDate::from_ymd_opt(2023, 1, 1) - .unwrap() - .and_hms_milli_opt(0, 0, 0, 000) - .unwrap() - .timestamp_millis(), - ), - Some("-12:00".into()), - ), - ScalarValue::new_interval_dt(0, sign * -43_200_000), - ), - // 6th test case, no problem before unix epoch beginning - ( - ScalarValue::TimestampMicrosecond( - Some( - NaiveDate::from_ymd_opt(1970, 1, 1) - .unwrap() - .and_hms_micro_opt(1, 2, 3, 15) - .unwrap() - .timestamp_micros(), - ), - None, - ), - ScalarValue::TimestampMicrosecond( - Some( - NaiveDate::from_ymd_opt(1969, 1, 1) - .unwrap() - .and_hms_micro_opt(0, 0, 0, 000_000) - .unwrap() - .timestamp_micros(), - ), - None, - ), - ScalarValue::new_interval_mdn( - 0, - 365 * sign, - sign as i64 * 3_723_000_015_000, - ), - ), - // 7th test case, no problem with big intervals - ( - ScalarValue::TimestampNanosecond( - Some( - NaiveDate::from_ymd_opt(2100, 1, 1) - .unwrap() - .and_hms_nano_opt(0, 0, 0, 0) - .unwrap() - .timestamp_nanos(), - ), - None, - ), - ScalarValue::TimestampNanosecond( - Some( - NaiveDate::from_ymd_opt(2000, 1, 1) - .unwrap() - .and_hms_nano_opt(0, 0, 0, 000_000_000) - .unwrap() - .timestamp_nanos(), - ), - None, - ), - ScalarValue::new_interval_mdn(0, sign * 36525, 0), - ), - // 8th test case, no problem detecting 366-days long years - ( - ScalarValue::TimestampSecond( - Some( - NaiveDate::from_ymd_opt(2041, 1, 1) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap() - .timestamp(), - ), - None, - ), - ScalarValue::TimestampSecond( - Some( - NaiveDate::from_ymd_opt(2040, 1, 1) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap() - .timestamp(), - ), - None, - ), - ScalarValue::new_interval_dt(sign * 366, 0), - ), - // 9th test case, no problem with unrealistic timezones - ( - ScalarValue::TimestampSecond( - Some( - NaiveDate::from_ymd_opt(2023, 1, 3) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap() - .timestamp(), - ), - Some("+23:59".into()), - ), - ScalarValue::TimestampSecond( - Some( - NaiveDate::from_ymd_opt(2023, 1, 1) - .unwrap() - .and_hms_opt(0, 2, 0) - .unwrap() - .timestamp(), - ), - Some("-23:59".into()), - ), - ScalarValue::new_interval_dt(0, 0), - ), - // 10th test case, parsing different types of timezone input - ( - ScalarValue::TimestampSecond( - Some( - NaiveDate::from_ymd_opt(2023, 3, 17) - .unwrap() - .and_hms_opt(14, 10, 0) - .unwrap() - .timestamp(), - ), - Some("Europe/Istanbul".into()), - ), - ScalarValue::TimestampSecond( - Some( - NaiveDate::from_ymd_opt(2023, 3, 17) - .unwrap() - .and_hms_opt(4, 10, 0) - .unwrap() - .timestamp(), - ), - Some("America/Los_Angeles".into()), - ), - ScalarValue::new_interval_dt(0, 0), - ), - // 11th test case, negative results - ( - ScalarValue::TimestampMillisecond( - Some( - NaiveDate::from_ymd_opt(2023, 3, 17) - .unwrap() - .and_hms_milli_opt(4, 10, 0, 0) - .unwrap() - .timestamp_millis(), - ), - None, - ), - ScalarValue::TimestampMillisecond( - Some( - NaiveDate::from_ymd_opt(2023, 3, 17) - .unwrap() - .and_hms_milli_opt(4, 10, 0, 1) - .unwrap() - .timestamp_millis(), - ), - None, - ), - ScalarValue::new_interval_dt(0, -sign), - ), - ] - } - fn get_random_timestamps(sample_size: u64) -> Vec { let vector_size = sample_size; let mut timestamp = vec![];