From 35ebd1a67c080c2876930816374adca49d025ef9 Mon Sep 17 00:00:00 2001 From: Henrik Abrahamsson Date: Thu, 2 Nov 2023 16:12:27 +0100 Subject: [PATCH 1/2] Add logging of siri errors to SiriAzureETUpdater --- .../updater/azure/SiriAzureETUpdater.java | 30 ++++++++++++++++++- .../azure/SiriAzureETUpdaterParameters.java | 8 ++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdater.java b/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdater.java index ef138264999..3fba2edd93f 100644 --- a/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdater.java +++ b/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdater.java @@ -1,5 +1,7 @@ package org.opentripplanner.ext.siri.updater.azure; +import static net.logstash.logback.argument.StructuredArguments.keyValue; + import com.azure.messaging.servicebus.ServiceBusErrorContext; import com.azure.messaging.servicebus.ServiceBusReceivedMessageContext; import jakarta.xml.bind.JAXBException; @@ -14,11 +16,13 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import java.util.stream.Collectors; import javax.xml.stream.XMLStreamException; import org.apache.hc.core5.net.URIBuilder; import org.opentripplanner.ext.siri.SiriTimetableSnapshotSource; import org.opentripplanner.framework.time.DurationUtils; import org.opentripplanner.transit.service.TransitModel; +import org.opentripplanner.updater.spi.UpdateError; import org.opentripplanner.updater.spi.UpdateResult; import org.opentripplanner.updater.trip.metrics.TripUpdateMetrics; import org.rutebanken.siri20.util.SiriXml; @@ -106,13 +110,15 @@ private void processMessage(String message, String id) { } super.saveResultOnGraph.execute((graph, transitModel) -> { - snapshotSource.applyEstimatedTimetable( + var result = snapshotSource.applyEstimatedTimetable( fuzzyTripMatcher(), entityResolver(), feedId, false, updates ); + logErrors(result); + recordMetrics.accept(result); }); } catch (JAXBException | XMLStreamException e) { LOG.error(e.getLocalizedMessage(), e); @@ -138,6 +144,7 @@ private void processHistory(String message, String id) { false, updates ); + logErrors(result); recordMetrics.accept(result); setPrimed(true); @@ -174,4 +181,25 @@ private List getUpdates(String message, Str return siri.getServiceDelivery().getEstimatedTimetableDeliveries(); } + + private void logErrors(UpdateResult updateResult) { + if (updateResult.failed() == 0) { + return; + } + var errorIndex = updateResult.failures(); + errorIndex + .keySet() + .forEach(key -> { + var value = errorIndex.get(key); + var tripIds = value.stream().map(UpdateError::debugId).collect(Collectors.toSet()); + LOG.warn( + "[{} {}] {} failures of {}: {}", + keyValue("feedId", feedId), + keyValue("type", "siri-et"), + value.size(), + keyValue("errorType", key), + tripIds + ); + }); + } } diff --git a/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdaterParameters.java b/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdaterParameters.java index a1ee8c5a79e..e63615dd893 100644 --- a/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdaterParameters.java +++ b/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdaterParameters.java @@ -1,5 +1,6 @@ package org.opentripplanner.ext.siri.updater.azure; +import com.azure.core.amqp.implementation.ConnectionStringProperties; import java.time.LocalDate; import org.opentripplanner.updater.trip.UrlUpdaterParameters; @@ -23,6 +24,11 @@ public void setFromDateTime(LocalDate fromDateTime) { @Override public String url() { - return getDataInitializationUrl(); + var url = getServiceBusUrl(); + try { + return new ConnectionStringProperties(url).getEndpoint().toString(); + } catch (IllegalArgumentException e) { + return url; + } } } From a842ae88f204602133ed1fb2a7355a7ebd2afc78 Mon Sep 17 00:00:00 2001 From: Henrik Abrahamsson Date: Mon, 6 Nov 2023 10:47:13 +0100 Subject: [PATCH 2/2] Use ResultLogger in SiriAzureETUpdater --- .../updater/azure/SiriAzureETUpdater.java | 26 ++----------- .../updater/spi/ResultLogger.java | 38 +++++++++++-------- 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdater.java b/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdater.java index 3fba2edd93f..921657bb8f1 100644 --- a/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdater.java +++ b/src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdater.java @@ -22,6 +22,7 @@ import org.opentripplanner.ext.siri.SiriTimetableSnapshotSource; import org.opentripplanner.framework.time.DurationUtils; import org.opentripplanner.transit.service.TransitModel; +import org.opentripplanner.updater.spi.ResultLogger; import org.opentripplanner.updater.spi.UpdateError; import org.opentripplanner.updater.spi.UpdateResult; import org.opentripplanner.updater.trip.metrics.TripUpdateMetrics; @@ -117,7 +118,7 @@ private void processMessage(String message, String id) { false, updates ); - logErrors(result); + ResultLogger.logUpdateResultErrors(feedId, "siri-et", result); recordMetrics.accept(result); }); } catch (JAXBException | XMLStreamException e) { @@ -144,7 +145,7 @@ private void processHistory(String message, String id) { false, updates ); - logErrors(result); + ResultLogger.logUpdateResultErrors(feedId, "siri-et", result); recordMetrics.accept(result); setPrimed(true); @@ -181,25 +182,4 @@ private List getUpdates(String message, Str return siri.getServiceDelivery().getEstimatedTimetableDeliveries(); } - - private void logErrors(UpdateResult updateResult) { - if (updateResult.failed() == 0) { - return; - } - var errorIndex = updateResult.failures(); - errorIndex - .keySet() - .forEach(key -> { - var value = errorIndex.get(key); - var tripIds = value.stream().map(UpdateError::debugId).collect(Collectors.toSet()); - LOG.warn( - "[{} {}] {} failures of {}: {}", - keyValue("feedId", feedId), - keyValue("type", "siri-et"), - value.size(), - keyValue("errorType", key), - tripIds - ); - }); - } } diff --git a/src/main/java/org/opentripplanner/updater/spi/ResultLogger.java b/src/main/java/org/opentripplanner/updater/spi/ResultLogger.java index 431f630c578..97af8d650bb 100644 --- a/src/main/java/org/opentripplanner/updater/spi/ResultLogger.java +++ b/src/main/java/org/opentripplanner/updater/spi/ResultLogger.java @@ -27,24 +27,30 @@ public static void logUpdateResult(String feedId, String type, UpdateResult upda DoubleUtils.roundTo2Decimals((double) updateResult.successful() / totalUpdates * 100) ); - var errorIndex = updateResult.failures(); - - errorIndex - .keySet() - .forEach(key -> { - var value = errorIndex.get(key); - var tripIds = value.stream().map(UpdateError::debugId).collect(Collectors.toSet()); - LOG.warn( - "[{} {}] {} failures of {}: {}", - keyValue("feedId", feedId), - keyValue("type", type), - value.size(), - keyValue("errorType", key), - tripIds - ); - }); + logUpdateResultErrors(feedId, type, updateResult); } else { LOG.info("[feedId={}, type={}] Feed did not contain any updates", feedId, type); } } + + public static void logUpdateResultErrors(String feedId, String type, UpdateResult updateResult) { + if (updateResult.failed() == 0) { + return; + } + var errorIndex = updateResult.failures(); + errorIndex + .keySet() + .forEach(key -> { + var value = errorIndex.get(key); + var tripIds = value.stream().map(UpdateError::debugId).collect(Collectors.toSet()); + LOG.warn( + "[{} {}] {} failures of {}: {}", + keyValue("feedId", feedId), + keyValue("type", type), + value.size(), + keyValue("errorType", key), + tripIds + ); + }); + } }