From ff8d9f63e457cf3292d36c4786643df56cb14054 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Fri, 4 Oct 2024 00:58:59 +0200 Subject: [PATCH 1/2] DBZ-8275 Support for PostgreSQL 17 --- build-all-multiplatform.sh | 2 +- postgres/17-alpine/Dockerfile | 20 +++++++ postgres/17-alpine/README.md | 41 +++++++++++++ .../init-permissions.sh | 4 ++ postgres/17-alpine/postgresql.conf.sample | 16 +++++ postgres/17/Dockerfile | 59 +++++++++++++++++++ postgres/17/README.md | 42 +++++++++++++ .../init-permissions.sh | 4 ++ postgres/17/postgresql.conf.sample | 16 +++++ 9 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 postgres/17-alpine/Dockerfile create mode 100644 postgres/17-alpine/README.md create mode 100644 postgres/17-alpine/docker-entrypoint-initdb.d/init-permissions.sh create mode 100644 postgres/17-alpine/postgresql.conf.sample create mode 100644 postgres/17/Dockerfile create mode 100644 postgres/17/README.md create mode 100644 postgres/17/docker-entrypoint-initdb.d/init-permissions.sh create mode 100644 postgres/17/postgresql.conf.sample diff --git a/build-all-multiplatform.sh b/build-all-multiplatform.sh index 3ec08417..ba1b7223 100755 --- a/build-all-multiplatform.sh +++ b/build-all-multiplatform.sh @@ -33,7 +33,7 @@ MONGO_VERSIONS="3.2" MONGO_MULTIPLATFORM_VERSIONS="3.4 3.6 4.0 4.2 4.4 5.0 6.0" POSTGRES_VERSIONS="9.6 10 11 12 13 14" -POSTGRES_MULTIPLATFORM_VERSIONS="15 16 9.6-alpine 10-alpine 11-alpine 12-alpine 13-alpine 14-alpine 15-alpine 16-alpine" +POSTGRES_MULTIPLATFORM_VERSIONS="15 16 17 9.6-alpine 10-alpine 11-alpine 12-alpine 13-alpine 14-alpine 15-alpine 16-alpine 17-alpine" if shouldBuild "mongo"; then for MONGO_VERSION in $MONGO_VERSIONS; do diff --git a/postgres/17-alpine/Dockerfile b/postgres/17-alpine/Dockerfile new file mode 100644 index 00000000..34142110 --- /dev/null +++ b/postgres/17-alpine/Dockerfile @@ -0,0 +1,20 @@ +FROM postgres:17-alpine + +LABEL maintainer="Debezium Community" +ENV PLUGIN_VERSION=v3.0.0.Final + +RUN apk add --no-cache protobuf-c-dev + +# Compile the plugins from sources and install +RUN apk add --no-cache --virtual .debezium-build-deps gcc clang15 llvm15 git make musl-dev pkgconf \ + && git clone https://github.com/debezium/postgres-decoderbufs -b $PLUGIN_VERSION --single-branch \ + && (cd /postgres-decoderbufs && make && make install) \ + && rm -rf postgres-decoderbufs \ + && apk del .debezium-build-deps + +# Copy the custom configuration which will be passed down to the server (using a .sample file is the preferred way of doing it by +# the base Docker image) +COPY postgresql.conf.sample /usr/local/share/postgresql/postgresql.conf.sample + +# Copy the script which will initialize the replication permissions +COPY /docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/postgres/17-alpine/README.md b/postgres/17-alpine/README.md new file mode 100644 index 00000000..7b89a0ae --- /dev/null +++ b/postgres/17-alpine/README.md @@ -0,0 +1,41 @@ +The [Postgres](https://www.postgresql.org) relational database management system has a feature called [logical decoding](https://www.postgresql.org/docs/17/static/logicaldecoding-explanation.html) that allows clients to extract all persistent changes to a database's tables into a coherent, easy to understand format which can be interpreted without detailed knowledge of the database's internal state. An output plugin transform the data from the write-ahead log's internal representation into the format the consumer of a replication slot desires. + +This image is based upon [`postgres:17-alpine`](https://hub.docker.com/_/postgres/) and adds one logical decoding plug-in: + +* [postgres-decoderbufs](https://github.com/debezium/), based on Protocol Buffers and maintained by the Debezium community + +It is supported by the [Debezium PostgreSQL Connector](http://debezium.io/docs/connectors/postgresql/) to capture changes committed to the database and record the data change events in Kafka topics. +In addition, Debezium supports the `pgoutput` plug-in, which is available by default on Postgres 10 and later. + +This provides an example of how the Debezium output plugin can be installed and how to enable PostgreSQL's logical decoding feature. + +# What is Debezium? + +Debezium is a distributed platform that turns your existing databases into event streams, so applications can quickly react to each row-level change in the databases. Debezium is built on top of Kafka and provides Kafka Connect compatible connectors that monitor specific database management systems. Debezium records the history of data changes in Kafka logs, so your application can be stopped and restarted at any time and can easily consume all of the events it missed while it was not running, ensuring that all events are processed correctly and completely. + +Running Debezium involves Zookeeper, Kafka, and services that run Debezium's connectors. For simple evaluation and experimentation, all services can all be run on a single host machine, using the recipe outlined below. Production environments, however, require properly running and networking multiple instances of each service to provide the performance, reliability, replication, and fault tolerance. This can be done with a platform like [OpenShift](https://www.openshift.com) that manages multiple Docker containers running on multiple hosts and machines. But running Kafka in a Docker container has limitations, so for scenarios where very high throughput is required, you should run Kafka on dedicated hardware as explained in the [Kafka documentation](http://kafka.apache.org/documentation.html). + + +# How to use this image + +This image is used in the same manner as the [`postgres:17-alpine`](https://hub.docker.com/_/postgres/) image, though the `/usr/share/postgresql/postgresql.conf.sample` file configures the logical decoding feature: + +``` +# LOGGING +log_min_error_statement = fatal + +# CONNECTION +listen_addresses = '*' + +# MODULES +shared_preload_libraries = 'decoderbufs' + +# REPLICATION +wal_level = logical # minimal, archive, hot_standby, or logical (change requires restart) +max_wal_senders = 1 # max number of walsender processes (change requires restart) +#wal_keep_segments = 4 # in logfile segments, 16MB each; 0 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables +max_replication_slots = 1 # max number of replication slots (change requires restart) +``` + +This file instructs PostgreSQL to load [Debezium's logical decoding output plugin](https://github.com/debezium/postgres-decoderbufs), enable the logical decoding feature, and configure a single replication slot that will be used by the Debezium PostgreSQL Connector. diff --git a/postgres/17-alpine/docker-entrypoint-initdb.d/init-permissions.sh b/postgres/17-alpine/docker-entrypoint-initdb.d/init-permissions.sh new file mode 100644 index 00000000..5b6be68c --- /dev/null +++ b/postgres/17-alpine/docker-entrypoint-initdb.d/init-permissions.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +{ echo "host replication $POSTGRES_USER 0.0.0.0/0 trust"; } >> "$PGDATA/pg_hba.conf" \ No newline at end of file diff --git a/postgres/17-alpine/postgresql.conf.sample b/postgres/17-alpine/postgresql.conf.sample new file mode 100644 index 00000000..1c227974 --- /dev/null +++ b/postgres/17-alpine/postgresql.conf.sample @@ -0,0 +1,16 @@ +# LOGGING +# log_min_error_statement = fatal +# log_min_messages = DEBUG1 + +# CONNECTION +listen_addresses = '*' + +# MODULES +shared_preload_libraries = 'decoderbufs' + +# REPLICATION +wal_level = logical # minimal, archive, hot_standby, or logical (change requires restart) +max_wal_senders = 4 # max number of walsender processes (change requires restart) +#wal_keep_segments = 4 # in logfile segments, 16MB each; 0 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables +max_replication_slots = 4 # max number of replication slots (change requires restart) diff --git a/postgres/17/Dockerfile b/postgres/17/Dockerfile new file mode 100644 index 00000000..5317e698 --- /dev/null +++ b/postgres/17/Dockerfile @@ -0,0 +1,59 @@ +FROM postgres:17-bullseye AS build + +ARG USE_POSTGIS=true +ENV PLUGIN_VERSION=v3.0.0.Final +ENV PROTOC_VERSION=1.4 + +# Install the packages which will be required to get everything to compile +RUN apt-get update \ + && apt-get install -f -y --no-install-recommends \ + software-properties-common \ + build-essential \ + pkg-config \ + git \ + clang-11 \ + llvm-11 \ + postgresql-server-dev-$PG_MAJOR \ + && add-apt-repository "deb http://ftp.debian.org/debian testing main contrib" \ + && apt-get update && apt-get install -f -y --no-install-recommends \ + libprotobuf-c-dev=$PROTOC_VERSION.* \ + && rm -rf /var/lib/apt/lists/* + +# Compile the plugin from sources and install it +RUN git clone https://github.com/debezium/postgres-decoderbufs -b $PLUGIN_VERSION --single-branch \ + && cd /postgres-decoderbufs \ + && make && make install \ + && cd / \ + && rm -rf postgres-decoderbufs + + +FROM postgres:17-bullseye + +LABEL maintainer="Debezium Community" + +ENV POSTGIS_VERSION=3 + +RUN apt-get update \ + && apt-get install -f -y --no-install-recommends \ + software-properties-common \ + && if [ "$USE_POSTGIS" != "false" ]; then apt-get install -f -y --no-install-recommends \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_VERSION \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_VERSION-scripts \ + postgis; \ + fi \ + && if [ "$USE_PGVECTOR" != "false" ]; then apt-get install -f -y --no-install-recommends \ + postgresql-$PG_MAJOR-pgvector; \ + fi \ + && add-apt-repository "deb http://ftp.debian.org/debian testing main contrib" \ + && apt-get update && apt-get install -f -y --no-install-recommends \ + libprotobuf-c1 \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build /usr/lib/postgresql/$PG_MAJOR/lib/decoderbufs.so /usr/lib/postgresql/$PG_MAJOR/lib/ +COPY --from=build /usr/share/postgresql/$PG_MAJOR/extension/decoderbufs.control /usr/share/postgresql/$PG_MAJOR/extension/ +# Copy the custom configuration which will be passed down to the server (using a .sample file is the preferred way of doing it by +# the base Docker image) +COPY postgresql.conf.sample /usr/share/postgresql/postgresql.conf.sample + +# Copy the script which will initialize the replication permissions +COPY /docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/postgres/17/README.md b/postgres/17/README.md new file mode 100644 index 00000000..d82a0992 --- /dev/null +++ b/postgres/17/README.md @@ -0,0 +1,42 @@ +The [Postgres](https://www.postgresql.org) relational database management system has a feature called [logical decoding](https://www.postgresql.org/docs/17/static/logicaldecoding-explanation.html) that allows clients to extract all persistent changes to a database's tables into a coherent, easy to understand format which can be interpreted without detailed knowledge of the database's internal state. An output plugin transform the data from the write-ahead log's internal representation into the format the consumer of a replication slot desires. + +This image is based upon [`postgres:17`](https://hub.docker.com/_/postgres/) and adds one logical decoding plug-in: + +* [postgres-decoderbufs](https://github.com/debezium/), based on Protocol Buffers and maintained by the Debezium community + +It is supported by the [Debezium PostgreSQL Connector](http://debezium.io/docs/connectors/postgresql/) to capture changes committed to the database and record the data change events in Kafka topics. +In addition, Debezium supports the `pgoutput` plug-in, which is available by default on Postgres 10 and later. +The image also includes [PostGIS](http://www.postgis.net) spatial database extender used to provide geospatial queries, so that changes to geometric data can also be captured by Debezium. + +This provides an example of how the Debezium output plugin can be installed and how to enable PostgreSQL's logical decoding feature. + +# What is Debezium? + +Debezium is a distributed platform that turns your existing databases into event streams, so applications can quickly react to each row-level change in the databases. Debezium is built on top of Kafka and provides Kafka Connect compatible connectors that monitor specific database management systems. Debezium records the history of data changes in Kafka logs, so your application can be stopped and restarted at any time and can easily consume all of the events it missed while it was not running, ensuring that all events are processed correctly and completely. + +Running Debezium involves Zookeeper, Kafka, and services that run Debezium's connectors. For simple evaluation and experimentation, all services can all be run on a single host machine, using the recipe outlined below. Production environments, however, require properly running and networking multiple instances of each service to provide the performance, reliability, replication, and fault tolerance. This can be done with a platform like [OpenShift](https://www.openshift.com) that manages multiple Docker containers running on multiple hosts and machines. But running Kafka in a Docker container has limitations, so for scenarios where very high throughput is required, you should run Kafka on dedicated hardware as explained in the [Kafka documentation](http://kafka.apache.org/documentation.html). + + +# How to use this image + +This image is used in the same manner as the [`postgres:17`](https://hub.docker.com/_/postgres/) image, though the `/usr/share/postgresql/postgresql.conf.sample` file configures the logical decoding feature: + +``` +# LOGGING +log_min_error_statement = fatal + +# CONNECTION +listen_addresses = '*' + +# MODULES +shared_preload_libraries = 'decoderbufs' + +# REPLICATION +wal_level = logical # minimal, archive, hot_standby, or logical (change requires restart) +max_wal_senders = 1 # max number of walsender processes (change requires restart) +#wal_keep_segments = 4 # in logfile segments, 16MB each; 0 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables +max_replication_slots = 1 # max number of replication slots (change requires restart) +``` + +This file instructs PostgreSQL to load [Debezium's logical decoding output plugin](https://github.com/debezium/postgres-decoderbufs), enable the logical decoding feature, and configure a single replication slot that will be used by the Debezium PostgreSQL Connector. diff --git a/postgres/17/docker-entrypoint-initdb.d/init-permissions.sh b/postgres/17/docker-entrypoint-initdb.d/init-permissions.sh new file mode 100644 index 00000000..5b6be68c --- /dev/null +++ b/postgres/17/docker-entrypoint-initdb.d/init-permissions.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +{ echo "host replication $POSTGRES_USER 0.0.0.0/0 trust"; } >> "$PGDATA/pg_hba.conf" \ No newline at end of file diff --git a/postgres/17/postgresql.conf.sample b/postgres/17/postgresql.conf.sample new file mode 100644 index 00000000..1c227974 --- /dev/null +++ b/postgres/17/postgresql.conf.sample @@ -0,0 +1,16 @@ +# LOGGING +# log_min_error_statement = fatal +# log_min_messages = DEBUG1 + +# CONNECTION +listen_addresses = '*' + +# MODULES +shared_preload_libraries = 'decoderbufs' + +# REPLICATION +wal_level = logical # minimal, archive, hot_standby, or logical (change requires restart) +max_wal_senders = 4 # max number of walsender processes (change requires restart) +#wal_keep_segments = 4 # in logfile segments, 16MB each; 0 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables +max_replication_slots = 4 # max number of replication slots (change requires restart) From 42c74fffe781cf47cdcf7b019c893c6ea01b0fda Mon Sep 17 00:00:00 2001 From: Jiri Pechanec Date: Mon, 14 Oct 2024 12:37:27 +0200 Subject: [PATCH 2/2] DBZ-8275 Use PostgreSQL 17 compatible decoder --- postgres/16-alpine/Dockerfile | 2 +- postgres/16/Dockerfile | 2 +- postgres/17-alpine/Dockerfile | 2 +- postgres/17/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/postgres/16-alpine/Dockerfile b/postgres/16-alpine/Dockerfile index 5add7d1f..3f27cab5 100644 --- a/postgres/16-alpine/Dockerfile +++ b/postgres/16-alpine/Dockerfile @@ -1,7 +1,7 @@ FROM postgres:16-alpine LABEL maintainer="Debezium Community" -ENV PLUGIN_VERSION=v3.0.0.Final +ENV PLUGIN_VERSION=main RUN apk add --no-cache protobuf-c-dev diff --git a/postgres/16/Dockerfile b/postgres/16/Dockerfile index 64c0f1e4..2658058e 100644 --- a/postgres/16/Dockerfile +++ b/postgres/16/Dockerfile @@ -1,7 +1,7 @@ FROM postgres:16-bullseye AS build ARG USE_POSTGIS=true -ENV PLUGIN_VERSION=v3.0.0.Final +ENV PLUGIN_VERSION=main ENV PROTOC_VERSION=1.4 # Install the packages which will be required to get everything to compile diff --git a/postgres/17-alpine/Dockerfile b/postgres/17-alpine/Dockerfile index 34142110..40a60125 100644 --- a/postgres/17-alpine/Dockerfile +++ b/postgres/17-alpine/Dockerfile @@ -1,7 +1,7 @@ FROM postgres:17-alpine LABEL maintainer="Debezium Community" -ENV PLUGIN_VERSION=v3.0.0.Final +ENV PLUGIN_VERSION=main RUN apk add --no-cache protobuf-c-dev diff --git a/postgres/17/Dockerfile b/postgres/17/Dockerfile index 5317e698..1ef4b766 100644 --- a/postgres/17/Dockerfile +++ b/postgres/17/Dockerfile @@ -1,7 +1,7 @@ FROM postgres:17-bullseye AS build ARG USE_POSTGIS=true -ENV PLUGIN_VERSION=v3.0.0.Final +ENV PLUGIN_VERSION=main ENV PROTOC_VERSION=1.4 # Install the packages which will be required to get everything to compile