[backport] Improve sender log condition to record valuable debug info (…

…#1419)
alibaba · Mar 27, 2024 · 1c000b9 · 1c000b9
1 parent dd92bc8
commit 1c000b9
Show file tree

Hide file tree

Showing 5 changed files with 94 additions and 64 deletions.
diff --git a/core/common/LogFileCollectOffsetIndicator.cpp b/core/common/LogFileCollectOffsetIndicator.cpp
@@ -112,11 +112,11 @@ void LogFileCollectOffsetIndicator::RecordFileOffset(LoggroupTimeValue* data) {
                                                                      devInode,
                                                                      data->mLogGroupContext.mFuseMode,
                                                                      fd,
-                                                                     data->mLastUpdateTime);
+                                                                     data->mEnqueueTime);
         iter = mLogFileOffsetInfoMap.insert(std::make_pair(logFileInfo, logFileOffsetInfo)).first;
     }
     LogFileOffsetInfo* logFileOffsetInfo = iter->second;
-    logFileOffsetInfo->mLastUpdateTime = data->mLastUpdateTime;
+    logFileOffsetInfo->mLastUpdateTime = data->mEnqueueTime;
 
     LogFileOffsetInfoNode node(seqNum,
                                fileInfoPtr->offset,

diff --git a/core/common/LogstoreSenderQueue.cpp b/core/common/LogstoreSenderQueue.cpp
@@ -142,8 +142,6 @@ bool LogstoreSenderInfo::RecordSendResult(SendResult rst, LogstoreSenderStatisti
             if (++mLastNetworkErrorCount >= INT32_FLAG(max_client_send_error_count)) {
                 mLastNetworkErrorCount = INT32_FLAG(max_client_send_error_count);
                 mNetworkValidFlag = false;
-                LOG_WARNING(sLogger,
-                            ("Network fail, disable ", this->mRegion)("retry interval", mNetworkRetryInterval));
             }
             break;
         case LogstoreSenderInfo::SendResult_QuotaFail:
@@ -155,7 +153,6 @@ bool LogstoreSenderInfo::RecordSendResult(SendResult rst, LogstoreSenderStatisti
             if (++mLastQuotaExceedCount >= INT32_FLAG(max_client_quota_exceed_count)) {
                 mLastQuotaExceedCount = INT32_FLAG(max_client_quota_exceed_count);
                 mQuotaValidFlag = false;
-                LOG_WARNING(sLogger, ("QuotaF fail, disable ", this->mRegion)("retry interval", mQuotaRetryInterval));
             }
             break;
         default:

diff --git a/core/common/LogstoreSenderQueue.h b/core/common/LogstoreSenderQueue.h
@@ -47,7 +47,7 @@ struct LogstoreSenderStatistics {
 };
 
 struct LoggroupTimeValue {
-    int32_t mLastUpdateTime;
+    int32_t mEnqueueTime;
     SEND_DATA_TYPE mDataType;
     std::string mLogData;
     int32_t mRawSize;
@@ -63,6 +63,7 @@ struct LoggroupTimeValue {
 
     int32_t mSendRetryTimes;
     int32_t mLastSendTime;
+    int32_t mLastLogWarningTime;
     std::string mAliuid;
     std::string mRegion;
     std::string mShardHashKey;
@@ -99,9 +100,10 @@ struct LoggroupTimeValue {
         mDataType = dataType;
         mLogLines = lines;
         mRawSize = rawSize;
-        mLastUpdateTime = lastUpdateTime;
+        mEnqueueTime = lastUpdateTime;
         mSendRetryTimes = 0;
         mLastSendTime = 0;
+        mLastLogWarningTime = 0;
         mLogData.clear();
         mShardHashKey = shardHashKey;
         mStatus = LoggroupSendStatus_Idle;
@@ -400,11 +402,11 @@ class SingleLogstoreSenderManager : public SingleLogstoreFeedbackQueue<LoggroupT
                 continue;
             }
 
-            if (item->mLastUpdateTime < minSendTime) {
-                minSendTime = item->mLastUpdateTime;
+            if (item->mEnqueueTime < minSendTime) {
+                minSendTime = item->mEnqueueTime;
             }
-            if (item->mLastUpdateTime > maxSendTime) {
-                maxSendTime = item->mLastUpdateTime;
+            if (item->mEnqueueTime > maxSendTime) {
+                maxSendTime = item->mEnqueueTime;
             }
             ++statisticsItem.mSendQueueSize;
         }
@@ -417,15 +419,25 @@ class SingleLogstoreSenderManager : public SingleLogstoreFeedbackQueue<LoggroupT
 
     int32_t OnSendDone(LoggroupTimeValue* item, LogstoreSenderInfo::SendResult sendRst, bool& needTrigger) {
         needTrigger = mSenderInfo.RecordSendResult(sendRst, mSenderStatistics);
+        if (!mSenderInfo.mNetworkValidFlag) {
+            APSARA_LOG_WARNING(sLogger,
+                               ("Network fail, pause logstore", item->mLogstore)("project", item->mProjectName)(
+                                   "region", mSenderInfo.mRegion)("retry interval", mSenderInfo.mNetworkRetryInterval));
+        }
+        if (!mSenderInfo.mQuotaValidFlag) {
+            APSARA_LOG_WARNING(sLogger,
+                               ("Quota fail, pause logstore", item->mLogstore)("project", item->mProjectName)(
+                                   "region", mSenderInfo.mRegion)("retry interval", mSenderInfo.mQuotaRetryInterval));
+        }
         // if send error, reset status to idle, and wait to send again
         // network fail or quota fail
         if (sendRst != LogstoreSenderInfo::SendResult_OK && sendRst != LogstoreSenderInfo::SendResult_Buffered
             && sendRst != LogstoreSenderInfo::SendResult_DiscardFail) {
             item->mStatus = LoggroupSendStatus_Idle;
             return 0;
         }
-        if (mSenderStatistics.mMaxSendSuccessTime < item->mLastUpdateTime) {
-            mSenderStatistics.mMaxSendSuccessTime = item->mLastUpdateTime;
+        if (mSenderStatistics.mMaxSendSuccessTime < item->mEnqueueTime) {
+            mSenderStatistics.mMaxSendSuccessTime = item->mEnqueueTime;
         }
         // else remove item except buffered
         return RemoveItem(item, sendRst != LogstoreSenderInfo::SendResult_Buffered);

diff --git a/core/monitor/LogIntegrity.cpp b/core/monitor/LogIntegrity.cpp
@@ -272,7 +272,8 @@ void LogIntegrity::Notify(LoggroupTimeValue* data, bool flag) {
     PTScopedLock lock(mLogIntegrityMapLock);
     LogIntegrityInfo* info = NULL;
     if (FindLogIntegrityInfo(region, projectName, logstore, filename, info)) {
-        info->mLastUpdateTime = data->mLastUpdateTime;
+        info->mLastUpdateTime = data->mEnqueueTime;
+
         info->SetStatus(data->mLogGroupContext.mSeqNum,
                         data->mLogLines,
                         flag ? LogTimeInfo::LogIntegrityStatus_SendOK : LogTimeInfo::LogIntegrityStatus_SendFail);