From dd61c3a2fa2a13e46e1074d8d647f19f2703da2d Mon Sep 17 00:00:00 2001 From: dirtysalt Date: Mon, 15 May 2023 22:46:41 -0700 Subject: [PATCH] [Enhancement] skip text file desc part when file is not text format (#23431) Signed-off-by: Smith Cruise --- .../connector/RemoteScanRangeLocations.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/RemoteScanRangeLocations.java b/fe/fe-core/src/main/java/com/starrocks/connector/RemoteScanRangeLocations.java index 63a75acdba78d..2b4d6805328ef 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/RemoteScanRangeLocations.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/RemoteScanRangeLocations.java @@ -16,6 +16,7 @@ import com.starrocks.sql.common.ErrorType; import com.starrocks.sql.common.StarRocksPlannerException; import com.starrocks.sql.plan.HDFSScanNodePredicates; +import com.starrocks.thrift.THdfsFileFormat; import com.starrocks.thrift.THdfsScanRange; import com.starrocks.thrift.TNetworkAddress; import com.starrocks.thrift.TScanRange; @@ -93,7 +94,9 @@ private void createScanRangeLocationsForSplit(long partitionId, RemoteFileInfo p hdfsScanRange.setFile_length(fileDesc.getLength()); hdfsScanRange.setModification_time(fileDesc.getModificationTime()); hdfsScanRange.setFile_format(partition.getFormat().toThrift()); - hdfsScanRange.setText_file_desc(fileDesc.getTextFileFormatDesc().toThrift()); + if (isTextFormat(hdfsScanRange.getFile_format())) { + hdfsScanRange.setText_file_desc(fileDesc.getTextFileFormatDesc().toThrift()); + } TScanRange scanRange = new TScanRange(); scanRange.setHdfs_scan_range(hdfsScanRange); scanRangeLocations.setScan_range(scanRange); @@ -113,6 +116,10 @@ private void createScanRangeLocationsForSplit(long partitionId, RemoteFileInfo p result.add(scanRangeLocations); } + private boolean isTextFormat(THdfsFileFormat format) { + return format == THdfsFileFormat.TEXT || format == THdfsFileFormat.LZO_TEXT; + } + private void createHudiScanRangeLocations(long partitionId, RemoteFileInfo partition, RemoteFileDesc fileDesc, @@ -126,7 +133,9 @@ private void createHudiScanRangeLocations(long partitionId, hdfsScanRange.setPartition_id(partitionId); hdfsScanRange.setFile_length(fileDesc.getLength()); hdfsScanRange.setFile_format(partition.getFormat().toThrift()); - hdfsScanRange.setText_file_desc(fileDesc.getTextFileFormatDesc().toThrift()); + if (isTextFormat(hdfsScanRange.getFile_format())) { + hdfsScanRange.setText_file_desc(fileDesc.getTextFileFormatDesc().toThrift()); + } for (String log : fileDesc.getHudiDeltaLogs()) { hdfsScanRange.addToHudi_logs(log); }