diff --git a/doc/user/content/sql/create-source/kafka.md b/doc/user/content/sql/create-source/kafka.md index 41f6cfbd3af34..8f6c71f214e60 100644 --- a/doc/user/content/sql/create-source/kafka.md +++ b/doc/user/content/sql/create-source/kafka.md @@ -138,6 +138,13 @@ for the given Kafka message key cannot be decoded, this `error` column will cont the error message. If the most recent value for a key has been successfully decoded, this column will be `NULL`. +To use an alternative name for the error column, use `INLINE AS ..` to specify the +column name to use: + +```mzsql +ENVELOPE UPSERT (VALUE DECODING ERRORS = (INLINE AS my_error_col)) +``` + It might be convenient to implement a parsing view on top of your Kafka upsert source that excludes keys with decoding errors: diff --git a/doc/user/layouts/partials/sql-grammar/create-source-kafka.svg b/doc/user/layouts/partials/sql-grammar/create-source-kafka.svg index cb019c0eb4a8d..1c7e282b0749e 100644 --- a/doc/user/layouts/partials/sql-grammar/create-source-kafka.svg +++ b/doc/user/layouts/partials/sql-grammar/create-source-kafka.svg @@ -1,4 +1,4 @@ - + @@ -20,236 +20,236 @@ src_name - - + - ( - - - col_name - - ( + + + col_name + + - , - - , + + - ) - - ) + + - IN CLUSTER - - - cluster_name - - IN CLUSTER + + + cluster_name + + - FROM - - FROM + + - KAFKA - - KAFKA + + - CONNECTION - - - connection_name - - CONNECTION + + + connection_name + + - ( - - ( + + - TOPIC - - - topic - - TOPIC + + + topic + + - , - - - connection_option - - , + + + connection_option + + - ) - - ) + + - KEY FORMAT - - - format_spec - - KEY FORMAT + + + format_spec + + - VALUE FORMAT - - VALUE FORMAT + + - FORMAT - - - format_spec - - FORMAT + + + format_spec + + - INCLUDE - - INCLUDE + + - KEY - - KEY + + - PARTITION - - PARTITION + + - OFFSET - - OFFSET + + - TIMESTAMP - - TIMESTAMP + + - HEADERS - - HEADERS + + - AS - - - name - - AS + + + name + + - HEADER - - - key - - HEADER + + + key + + - AS - - - name - - AS + + + name + + - BYTES - - BYTES + + - , + , VALUE DECODING ERRORS = INLINE - - + + AS + + + name + + - ) - - ) + + - EXPOSE - - EXPOSE + + - PROGRESS - - PROGRESS + + - AS - - AS + + - progress_subsource_name - - progress_subsource_name + + - with_options + with_options - - + d="m17 17 h2 m0 0 h10 m140 0 h10 m20 0 h10 m0 0 h130 m-160 0 h20 m140 0 h20 m-180 0 q10 0 10 10 m160 0 q0 -10 10 -10 m-170 10 v12 m160 0 v-12 m-160 12 q0 10 10 10 m140 0 q10 0 10 -10 m-150 10 h10 m120 0 h10 m20 -32 h10 m82 0 h10 m2 0 l2 0 m2 0 l2 0 m2 0 l2 0 m-311 142 l2 0 m2 0 l2 0 m2 0 l2 0 m22 0 h10 m26 0 h10 m20 0 h10 m82 0 h10 m-122 0 l20 0 m-1 0 q-9 0 -9 -10 l0 -24 q0 -10 10 -10 m102 44 l20 0 m-20 0 q10 0 10 -10 l0 -24 q0 -10 -10 -10 m-102 0 h10 m24 0 h10 m0 0 h58 m20 44 h10 m26 0 h10 m-254 0 h20 m234 0 h20 m-274 0 q10 0 10 10 m254 0 q0 -10 10 -10 m-264 10 v14 m254 0 v-14 m-254 14 q0 10 10 10 m234 0 q10 0 10 -10 m-244 10 h10 m0 0 h224 m40 -34 h10 m0 0 h242 m-272 0 h20 m252 0 h20 m-292 0 q10 0 10 10 m272 0 q0 -10 10 -10 m-282 10 v12 m272 0 v-12 m-272 12 q0 10 10 10 m252 0 q10 0 10 -10 m-262 10 h10 m104 0 h10 m0 0 h10 m108 0 h10 m22 -32 l2 0 m2 0 l2 0 m2 0 l2 0 m-560 98 l2 0 m2 0 l2 0 m2 0 l2 0 m2 0 h10 m60 0 h10 m0 0 h10 m68 0 h10 m0 0 h10 m116 0 h10 m0 0 h10 m136 0 h10 m0 0 h10 m26 0 h10 m2 0 l2 0 m2 0 l2 0 m2 0 l2 0 m-500 66 l2 0 m2 0 l2 0 m2 0 l2 0 m2 0 h10 m64 0 h10 m0 0 h10 m52 0 h10 m20 0 h10 m0 0 h194 m-224 0 h20 m204 0 h20 m-244 0 q10 0 10 10 m224 0 q0 -10 10 -10 m-234 10 v12 m224 0 v-12 m-224 12 q0 10 10 10 m204 0 q10 0 10 -10 m-214 10 h10 m24 0 h10 m0 0 h10 m140 0 h10 m20 -32 h10 m26 0 h10 m2 0 l2 0 m2 0 l2 0 m2 0 l2 0 m-532 98 l2 0 m2 0 l2 0 m2 0 l2 0 m22 0 h10 m114 0 h10 m0 0 h10 m102 0 h10 m0 0 h10 m132 0 h10 m-428 0 h20 m408 0 h20 m-448 0 q10 0 10 10 m428 0 q0 -10 10 -10 m-438 10 v24 m428 0 v-24 m-428 24 q0 10 10 10 m408 0 q10 0 10 -10 m-418 10 h10 m80 0 h10 m0 0 h308 m20 -44 h10 m102 0 h10 m2 0 l2 0 m2 0 l2 0 m2 0 l2 0 m-632 154 l2 0 m2 0 l2 0 m2 0 l2 0 m22 0 h10 m82 0 h10 m60 0 h10 m48 0 h10 m0 0 h58 m-146 0 h20 m126 0 h20 m-166 0 q10 0 10 10 m146 0 q0 -10 10 -10 m-156 10 v24 m146 0 v-24 m-146 24 q0 10 10 10 m126 0 q10 0 10 -10 m-136 10 h10 m96 0 h10 m0 0 h10 m-136 -10 v20 m146 0 v-20 m-146 20 v24 m146 0 v-24 m-146 24 q0 10 10 10 m126 0 q10 0 10 -10 m-136 10 h10 m74 0 h10 m0 0 h32 m-136 -10 v20 m146 0 v-20 m-146 20 v24 m146 0 v-24 m-146 24 q0 10 10 10 m126 0 q10 0 10 -10 m-136 10 h10 m106 0 h10 m-136 -10 v20 m146 0 v-20 m-146 20 v24 m146 0 v-24 m-146 24 q0 10 10 10 m126 0 q10 0 10 -10 m-136 10 h10 m86 0 h10 m0 0 h20 m40 -176 h10 m0 0 h126 m-156 0 h20 m136 0 h20 m-176 0 q10 0 10 10 m156 0 q0 -10 10 -10 m-166 10 v12 m156 0 v-12 m-156 12 q0 10 10 10 m136 0 q10 0 10 -10 m-146 10 h10 m40 0 h10 m0 0 h10 m56 0 h10 m20 -32 h82 m-444 0 h20 m424 0 h20 m-464 0 q10 0 10 10 m444 0 q0 -10 10 -10 m-454 10 v200 m444 0 v-200 m-444 200 q0 10 10 10 m424 0 q10 0 10 -10 m-434 10 h10 m78 0 h10 m0 0 h10 m44 0 h10 m0 0 h10 m40 0 h10 m0 0 h10 m56 0 h10 m20 0 h10 m0 0 h76 m-106 0 h20 m86 0 h20 m-126 0 q10 0 10 10 m106 0 q0 -10 10 -10 m-116 10 v12 m106 0 v-12 m-106 12 q0 10 10 10 m86 0 q10 0 10 -10 m-96 10 h10 m66 0 h10 m-444 -252 l20 0 m-1 0 q-9 0 -9 -10 l0 -24 q0 -10 10 -10 m464 44 l20 0 m-20 0 q10 0 10 -10 l0 -24 q0 -10 -10 -10 m-464 0 h10 m24 0 h10 m0 0 h420 m-606 44 h20 m606 0 h20 m-646 0 q10 0 10 10 m626 0 q0 -10 10 -10 m-636 10 v266 m626 0 v-266 m-626 266 q0 10 10 10 m606 0 q10 0 10 -10 m-616 10 h10 m0 0 h596 m22 -286 l2 0 m2 0 l2 0 m2 0 l2 0 m-791 318 l2 0 m2 0 l2 0 m2 0 l2 0 m22 0 h10 m0 0 h838 m-868 0 h20 m848 0 h20 m-888 0 q10 0 10 10 m868 0 q0 -10 10 -10 m-878 10 v12 m868 0 v-12 m-868 12 q0 10 10 10 m848 0 q10 0 10 -10 m-858 10 h10 m94 0 h10 m20 0 h10 m60 0 h10 m0 0 h614 m-714 0 h20 m694 0 h20 m-734 0 q10 0 10 10 m714 0 q0 -10 10 -10 m-724 10 v24 m714 0 v-24 m-714 24 q0 10 10 10 m694 0 q10 0 10 -10 m-704 10 h10 m92 0 h10 m0 0 h582 m-704 -10 v20 m714 0 v-20 m-714 20 v24 m714 0 v-24 m-714 24 q0 10 10 10 m694 0 q10 0 10 -10 m-704 10 h10 m76 0 h10 m20 0 h10 m0 0 h548 m-578 0 h20 m558 0 h20 m-598 0 q10 0 10 10 m578 0 q0 -10 10 -10 m-588 10 v12 m578 0 v-12 m-578 12 q0 10 10 10 m558 0 q10 0 10 -10 m-568 10 h10 m26 0 h10 m0 0 h10 m270 0 h10 m20 0 h10 m0 0 h126 m-156 0 h20 m136 0 h20 m-176 0 q10 0 10 10 m156 0 q0 -10 10 -10 m-166 10 v12 m156 0 v-12 m-156 12 q0 10 10 10 m136 0 q10 0 10 -10 m-146 10 h10 m40 0 h10 m0 0 h10 m56 0 h10 m20 -32 h10 m26 0 h10 m62 -152 l2 0 m2 0 l2 0 m2 0 l2 0 m-732 234 l2 0 m2 0 l2 0 m2 0 l2 0 m22 0 h10 m0 0 h478 m-508 0 h20 m488 0 h20 m-528 0 q10 0 10 10 m508 0 q0 -10 10 -10 m-518 10 v12 m508 0 v-12 m-508 12 q0 10 10 10 m488 0 q10 0 10 -10 m-498 10 h10 m76 0 h10 m0 0 h10 m96 0 h10 m0 0 h10 m40 0 h10 m0 0 h10 m196 0 h10 m22 -32 l2 0 m2 0 l2 0 m2 0 l2 0 m-12 86 l2 0 m2 0 l2 0 m2 0 l2 0 m22 0 h10 m0 0 h112 m-142 0 h20 m122 0 h20 m-162 0 q10 0 10 10 m142 0 q0 -10 10 -10 m-152 10 v12 m142 0 v-12 m-142 12 q0 10 10 10 m122 0 q10 0 10 -10 m-132 10 h10 m102 0 h10 m23 -32 h-3"/> + + diff --git a/doc/user/layouts/partials/sql-grammar/sink-definition.svg b/doc/user/layouts/partials/sql-grammar/sink-definition.svg index c18c8ebdf689b..b963690cd1c0c 100644 --- a/doc/user/layouts/partials/sql-grammar/sink-definition.svg +++ b/doc/user/layouts/partials/sql-grammar/sink-definition.svg @@ -1,4 +1,4 @@ - + @@ -31,111 +31,130 @@ class="terminal" rx="10"/> INTO - - - kafka_sink_connection - - + + kafka_sink_connection + + - KEY - - KEY + + - ( - - - key_column - - ( + + + key_column + + - , - - , + + - ) - - ) + + - NOT ENFORCED - - NOT ENFORCED + + - HEADERS - - - headers_column - - HEADERS + + + headers_column + + - FORMAT - - - sink_format_spec - - FORMAT + + + KEY FORMAT + + + sink_format_spec + + + VALUE FORMAT + + + sink_format_spec + + - ENVELOPE - - ENVELOPE + + - DEBEZIUM - - DEBEZIUM + + - UPSERT - - UPSERT + + - WITH - - - with_options + WITH + + + with_options - - + d="m17 17 h2 m20 0 h10 m0 0 h242 m-272 0 h20 m252 0 h20 m-292 0 q10 0 10 10 m272 0 q0 -10 10 -10 m-282 10 v12 m272 0 v-12 m-272 12 q0 10 10 10 m252 0 q10 0 10 -10 m-262 10 h10 m104 0 h10 m0 0 h10 m108 0 h10 m20 -32 h10 m60 0 h10 m0 0 h10 m90 0 h10 m0 0 h10 m54 0 h10 m2 0 l2 0 m2 0 l2 0 m2 0 l2 0 m-343 98 l2 0 m2 0 l2 0 m2 0 l2 0 m2 0 h10 m168 0 h10 m2 0 l2 0 m2 0 l2 0 m2 0 l2 0 m-392 110 l2 0 m2 0 l2 0 m2 0 l2 0 m22 0 h10 m48 0 h10 m0 0 h10 m26 0 h10 m20 0 h10 m98 0 h10 m-138 0 l20 0 m-1 0 q-9 0 -9 -10 l0 -24 q0 -10 10 -10 m118 44 l20 0 m-20 0 q10 0 10 -10 l0 -24 q0 -10 -10 -10 m-118 0 h10 m24 0 h10 m0 0 h74 m20 44 h10 m26 0 h10 m20 0 h10 m0 0 h140 m-170 0 h20 m150 0 h20 m-190 0 q10 0 10 10 m170 0 q0 -10 10 -10 m-180 10 v12 m170 0 v-12 m-170 12 q0 10 10 10 m150 0 q10 0 10 -10 m-160 10 h10 m130 0 h10 m-508 -32 h20 m508 0 h20 m-548 0 q10 0 10 10 m528 0 q0 -10 10 -10 m-538 10 v46 m528 0 v-46 m-528 46 q0 10 10 10 m508 0 q10 0 10 -10 m-518 10 h10 m0 0 h498 m22 -66 l2 0 m2 0 l2 0 m2 0 l2 0 m-445 98 l2 0 m2 0 l2 0 m2 0 l2 0 m22 0 h10 m0 0 h244 m-274 0 h20 m254 0 h20 m-294 0 q10 0 10 10 m274 0 q0 -10 10 -10 m-284 10 v12 m274 0 v-12 m-274 12 q0 10 10 10 m254 0 q10 0 10 -10 m-264 10 h10 m86 0 h10 m0 0 h10 m128 0 h10 m22 -32 l2 0 m2 0 l2 0 m2 0 l2 0 m-508 82 l2 0 m2 0 l2 0 m2 0 l2 0 m22 0 h10 m0 0 h624 m-654 0 h20 m634 0 h20 m-674 0 q10 0 10 10 m654 0 q0 -10 10 -10 m-664 10 v12 m654 0 v-12 m-654 12 q0 10 10 10 m634 0 q10 0 10 -10 m-624 10 h10 m80 0 h10 m0 0 h340 m-460 0 h20 m440 0 h20 m-480 0 q10 0 10 10 m460 0 q0 -10 10 -10 m-470 10 v24 m460 0 v-24 m-460 24 q0 10 10 10 m440 0 q10 0 10 -10 m-450 10 h10 m114 0 h10 m0 0 h10 m134 0 h10 m0 0 h10 m132 0 h10 m20 -44 h10 m134 0 h10 m22 -32 l2 0 m2 0 l2 0 m2 0 l2 0 m-536 142 l2 0 m2 0 l2 0 m2 0 l2 0 m2 0 h10 m94 0 h10 m20 0 h10 m92 0 h10 m-132 0 h20 m112 0 h20 m-152 0 q10 0 10 10 m132 0 q0 -10 10 -10 m-142 10 v24 m132 0 v-24 m-132 24 q0 10 10 10 m112 0 q10 0 10 -10 m-122 10 h10 m76 0 h10 m0 0 h16 m40 -44 h10 m0 0 h190 m-220 0 h20 m200 0 h20 m-240 0 q10 0 10 10 m220 0 q0 -10 10 -10 m-230 10 v12 m220 0 v-12 m-220 12 q0 10 10 10 m200 0 q10 0 10 -10 m-210 10 h10 m58 0 h10 m0 0 h10 m102 0 h10 m23 -32 h-3"/> + + diff --git a/doc/user/layouts/partials/sql-grammar/sink-format-spec.svg b/doc/user/layouts/partials/sql-grammar/sink-format-spec.svg index f81649a6f7723..c870b41a493f4 100644 --- a/doc/user/layouts/partials/sql-grammar/sink-format-spec.svg +++ b/doc/user/layouts/partials/sql-grammar/sink-format-spec.svg @@ -1,4 +1,4 @@ - + @@ -20,8 +20,24 @@ class="terminal" rx="10"/> JSON + + + TEXT + + + BYTES + d="m17 17 h2 m20 0 h10 m110 0 h10 m0 0 h10 m120 0 h10 m-290 0 h20 m270 0 h20 m-310 0 q10 0 10 10 m290 0 q0 -10 10 -10 m-300 10 v24 m290 0 v-24 m-290 24 q0 10 10 10 m270 0 q10 0 10 -10 m-280 10 h10 m58 0 h10 m0 0 h192 m-280 -10 v20 m290 0 v-20 m-290 20 v24 m290 0 v-24 m-290 24 q0 10 10 10 m270 0 q10 0 10 -10 m-280 10 h10 m56 0 h10 m0 0 h194 m-280 -10 v20 m290 0 v-20 m-290 20 v24 m290 0 v-24 m-290 24 q0 10 10 10 m270 0 q10 0 10 -10 m-280 10 h10 m66 0 h10 m0 0 h184 m23 -132 h-3"/> diff --git a/doc/user/sql-grammar/sql-grammar.bnf b/doc/user/sql-grammar/sql-grammar.bnf index 0357348992471..13520f78f38b4 100644 --- a/doc/user/sql-grammar/sql-grammar.bnf +++ b/doc/user/sql-grammar/sql-grammar.bnf @@ -188,7 +188,7 @@ create_source_kafka ::= (',' ( ('KEY' | 'PARTITION' | 'OFFSET' | 'TIMESTAMP' | 'HEADERS' ) ('AS' name)? | 'HEADER' key 'AS' name ('BYTES')? ) )* )? ('ENVELOPE' ('NONE' | 'DEBEZIUM' | 'UPSERT' - ( '(' 'VALUE DECODING ERRORS = INLINE' ')' )? + ( '(' 'VALUE DECODING ERRORS = INLINE' ('AS' name)? ')' )? ))? ('EXPOSE' 'PROGRESS' 'AS' progress_subsource_name)? with_options?