diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index 9fbd99752ed4..fec55eff09e8 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -1910,6 +1910,14 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa assert(plan2.find(_.isInstanceOf[ProjectExecTransformer]).isDefined) } + test("cast date to timestamp with GMT session timezone") { + withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") { + runQueryAndCompare("SELECT cast(date'2023-01-02 01:01:01' as timestamp) as ts") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + test("cast timestamp to date") { val query = "select cast(ts as date) from values (timestamp'2024-01-01 00:00:00') as tab(ts)" runQueryAndCompare(query) { diff --git a/cpp/core/config/GlutenConfig.cc b/cpp/core/config/GlutenConfig.cc index 0afd458ee636..eb98f6bb9036 100644 --- a/cpp/core/config/GlutenConfig.cc +++ b/cpp/core/config/GlutenConfig.cc @@ -49,6 +49,17 @@ parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength) return sparkConfs; } +std::string normalizeSessionTimezone(std::string_view sessionTimezone) { + if (sessionTimezone == "GMT") { + return "UTC"; + } + if (sessionTimezone.size() > 3 && sessionTimezone.substr(0, 3) == "GMT" && + (sessionTimezone[3] == '+' || sessionTimezone[3] == '-')) { + return std::string("UTC").append(sessionTimezone.substr(3)); + } + return std::string(sessionTimezone); +} + std::string printConfig(const std::unordered_map& conf) { std::ostringstream oss; oss << std::endl; diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h index a082a720eadd..2b8ba54595ba 100644 --- a/cpp/core/config/GlutenConfig.h +++ b/cpp/core/config/GlutenConfig.h @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace gluten { @@ -102,5 +103,7 @@ const std::string kDebugCudfDefault = "false"; std::unordered_map parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength); +std::string normalizeSessionTimezone(std::string_view sessionTimezone); + std::string printConfig(const std::unordered_map& conf); } // namespace gluten diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 3c0505263159..1806bfdd623a 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -575,7 +575,8 @@ std::unordered_map WholeStageResultIterator::getQueryC std::to_string(veloxCfg_->get(kVeloxPreferredBatchBytes, 10L << 20)); try { configs[velox::core::QueryConfig::kSparkAnsiEnabled] = veloxCfg_->get(kAnsiEnabled, "false"); - configs[velox::core::QueryConfig::kSessionTimezone] = veloxCfg_->get(kSessionTimezone, ""); + configs[velox::core::QueryConfig::kSessionTimezone] = + normalizeSessionTimezone(veloxCfg_->get(kSessionTimezone, "")); // Adjust timestamp according to the above configured session timezone. configs[velox::core::QueryConfig::kAdjustTimestampToTimezone] = "true"; diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h index 8afc7c5bf8b2..6bfca5ec36b8 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h @@ -31,7 +31,7 @@ class SubstraitToVeloxPlanValidator { public: SubstraitToVeloxPlanValidator(memory::MemoryPool* pool) { std::unordered_map configs{ - {velox::core::QueryConfig::kSparkPartitionId, "0"}, {velox::core::QueryConfig::kSessionTimezone, "GMT"}}; + {velox::core::QueryConfig::kSparkPartitionId, "0"}, {velox::core::QueryConfig::kSessionTimezone, "UTC"}}; veloxCfg_ = std::make_shared(std::move(configs)); planConverter_ = std::make_unique( pool, veloxCfg_.get(), std::vector>{}, std::nullopt, std::nullopt, true); diff --git a/cpp/velox/utils/VeloxWriterUtils.cc b/cpp/velox/utils/VeloxWriterUtils.cc index 026418a223c4..bd3cca685411 100644 --- a/cpp/velox/utils/VeloxWriterUtils.cc +++ b/cpp/velox/utils/VeloxWriterUtils.cc @@ -89,7 +89,9 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_mapflushPolicyFactory = [maxRowGroupRows, maxRowGroupBytes]() { return std::make_unique(maxRowGroupRows, maxRowGroupBytes, [&]() { return false; }); }; - writeOption->parquetWriteTimestampTimeZone = getConfigValue(sparkConfs, kSessionTimezone, std::nullopt); + if (auto it = sparkConfs.find(kSessionTimezone); it != sparkConfs.end()) { + writeOption->parquetWriteTimestampTimeZone = normalizeSessionTimezone(it->second); + } writeOption->arrowMemoryPool = getDefaultMemoryManager()->getOrCreateArrowMemoryPool("VeloxParquetWrite.ArrowMemoryPool"); if (auto it = sparkConfs.find(kParquetDataPageSize); it != sparkConfs.end()) {