From f826aca881b2271eca09d541ccc51c1a8944cd78 Mon Sep 17 00:00:00 2001 From: Ahmed Hamdy Date: Sat, 2 Nov 2024 19:12:57 +0000 Subject: [PATCH] [FLINK-36651][Connectors/Elasticsearch] Fix IT test not compatible with 1.20, drop main branch tests for 1.19 and 1.18 --- .github/workflows/push_pr.yml | 9 ++---- .github/workflows/weekly.yml | 14 +++------ .../61c75739-865e-4e9d-9da6-40bcde6a923c | 8 +++++ .../archunit-violations/stored.rules | 11 ++++--- .../sink/ElasticsearchWriterITCase.java | 31 ++----------------- .../f0b21088-63c4-4e1c-bcda-d54e4a224391 | 0 .../archunit-violations/stored.rules | 11 ++++--- pom.xml | 2 +- 8 files changed, 29 insertions(+), 57 deletions(-) create mode 100644 flink-connector-elasticsearch-base/archunit-violations/61c75739-865e-4e9d-9da6-40bcde6a923c create mode 100644 flink-connector-elasticsearch6/archunit-violations/f0b21088-63c4-4e1c-bcda-d54e4a224391 diff --git a/.github/workflows/push_pr.yml b/.github/workflows/push_pr.yml index a7a38f6f..c0f25f3b 100644 --- a/.github/workflows/push_pr.yml +++ b/.github/workflows/push_pr.yml @@ -28,13 +28,8 @@ jobs: compile_and_test: strategy: matrix: - flink: [ 1.18-SNAPSHOT ] - jdk: [ '8, 11, 17' ] - include: - - flink: 1.19-SNAPSHOT - jdk: '8, 11, 17, 21' - - flink: 1.20-SNAPSHOT - jdk: '8, 11, 17, 21' + flink: [ 1.19-SNAPSHOT, 1.20-SNAPSHOT ] + jdk: [ '8, 11, 17, 21' ] uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils with: flink_version: ${{ matrix.flink }} diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 6cd802c8..3f5bebac 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -30,27 +30,21 @@ jobs: strategy: matrix: flink_branches: [{ - flink: 1.18-SNAPSHOT, - jdk: '8, 11, 17', - branch: main - }, { flink: 1.19-SNAPSHOT, - jdk: '8, 11, 17, 21', branch: main }, { flink: 1.20-SNAPSHOT, - jdk: '8, 11, 17, 21', branch: main - }, { - flink: 1.18.1, - branch: v3.0 }, { flink: 1.19.0, branch: v3.0 + }, { + flink: 1.20.0, + branch: main }] uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils with: flink_version: ${{ matrix.flink_branches.flink }} connector_branch: ${{ matrix.flink_branches.branch }} - jdk_version: ${{ matrix.flink_branches.jdk || '8, 11' }} + jdk_version: ${{ matrix.flink_branches.jdk || '8, 11, 17, 21' }} run_dependency_convergence: false diff --git a/flink-connector-elasticsearch-base/archunit-violations/61c75739-865e-4e9d-9da6-40bcde6a923c b/flink-connector-elasticsearch-base/archunit-violations/61c75739-865e-4e9d-9da6-40bcde6a923c new file mode 100644 index 00000000..6c623eab --- /dev/null +++ b/flink-connector-elasticsearch-base/archunit-violations/61c75739-865e-4e9d-9da6-40bcde6a923c @@ -0,0 +1,8 @@ +Field is annotated with in (ElasticsearchWriter.java:0) +Method is annotated with in (ElasticsearchSink.java:0) +Method is annotated with in (ElasticsearchSink.java:0) +Method is annotated with in (ElasticsearchSink.java:0) +Method calls method in (ElasticsearchSinkBuilderBase.java:370) +Method is annotated with in (ElasticsearchWriter.java:0) +Method has parameter of type <[Lorg.apache.flink.table.types.DataType;> in (IndexGeneratorFactory.java:0) +Method calls constructor ()> in (RowElasticsearchEmitter.java:74) diff --git a/flink-connector-elasticsearch-base/archunit-violations/stored.rules b/flink-connector-elasticsearch-base/archunit-violations/stored.rules index 7908b7b5..772b217a 100644 --- a/flink-connector-elasticsearch-base/archunit-violations/stored.rules +++ b/flink-connector-elasticsearch-base/archunit-violations/stored.rules @@ -1,10 +1,11 @@ # -#Thu May 12 12:03:32 CEST 2022 +#Sat Nov 02 19:05:02 GMT 2024 +Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @Public\ must\ be\ annotated\ with\ @Public.=06cb39ec-97e2-4e4e-8aa7-cd92504b759c +Connector\ production\ code\ must\ not\ depend\ on\ non-public\ API\ outside\ of\ connector\ packages=325d89d0-9c19-495f-86df-7d609b6d63dd +ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=dd583797-83e1-414c-a38d-330773978813 Production\ code\ must\ not\ call\ methods\ annotated\ with\ @VisibleForTesting=4539e81b-9c72-4fd7-b7b3-ffc2e546b968 Options\ for\ connectors\ and\ formats\ should\ reside\ in\ a\ consistent\ package\ and\ be\ public\ API.=aca99410-c331-4bc2-a45b-1363eb8dd20c Tests\ inheriting\ from\ AbstractTestBase\ should\ have\ name\ ending\ with\ ITCase=de342dd1-c974-42c9-8f64-ef182ba8c56d -Classes\ in\ API\ packages\ should\ have\ at\ least\ one\ API\ visibility\ annotation.=143fca50-a3b9-4a4b-983e-40dd2c424f62 Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @PublicEvolving\ must\ be\ annotated\ with\ @Public(Evolving).=09532d6b-7cb1-4621-90af-755558763875 -Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @Public\ must\ be\ annotated\ with\ @Public.=06cb39ec-97e2-4e4e-8aa7-cd92504b759c -ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=dd583797-83e1-414c-a38d-330773978813 -Connector\ production\ code\ must\ not\ depend\ on\ non-public\ API\ outside\ of\ connector\ packages=325d89d0-9c19-495f-86df-7d609b6d63dd +Classes\ in\ API\ packages\ should\ have\ at\ least\ one\ API\ visibility\ annotation.=143fca50-a3b9-4a4b-983e-40dd2c424f62 +Connector\ production\ code\ must\ depend\ only\ on\ public\ API\ when\ outside\ of\ connector\ packages=61c75739-865e-4e9d-9da6-40bcde6a923c diff --git a/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/connector/elasticsearch/sink/ElasticsearchWriterITCase.java b/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/connector/elasticsearch/sink/ElasticsearchWriterITCase.java index 248f6a84..1ff68a9d 100644 --- a/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/connector/elasticsearch/sink/ElasticsearchWriterITCase.java +++ b/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/connector/elasticsearch/sink/ElasticsearchWriterITCase.java @@ -17,7 +17,6 @@ package org.apache.flink.connector.elasticsearch.sink; -import org.apache.flink.api.common.operators.MailboxExecutor; import org.apache.flink.api.connector.sink2.SinkWriter; import org.apache.flink.api.connector.sink2.SinkWriter.Context; import org.apache.flink.api.java.tuple.Tuple2; @@ -31,14 +30,13 @@ import org.apache.flink.metrics.groups.SinkWriterMetricGroup; import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; import org.apache.flink.metrics.testutils.MetricListener; +import org.apache.flink.runtime.mailbox.SyncMailboxExecutor; import org.apache.flink.runtime.metrics.MetricNames; import org.apache.flink.runtime.metrics.groups.InternalSinkWriterMetricGroup; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.test.junit5.MiniClusterExtension; -import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.TestLoggerExtension; -import org.apache.flink.util.function.ThrowingRunnable; import org.apache.http.HttpHost; import org.elasticsearch.action.ActionListener; @@ -329,7 +327,7 @@ private static ElasticsearchWriter> createWriter( new DefaultBulkResponseInspector(), new NetworkClientConfig(null, null, null, null, null, null, null, null), metricGroup, - new TestMailbox()); + new SyncMailboxExecutor()); } private TestingSinkWriterMetricGroup getSinkWriterMetricGroup() { @@ -481,29 +479,4 @@ GetResponse getResponse(String index, int id) throws IOException { return client.get(new GetRequest(index, Integer.toString(id)), RequestOptions.DEFAULT); } } - - private static class TestMailbox implements MailboxExecutor { - - @Override - public void execute( - ThrowingRunnable command, - String descriptionFormat, - Object... descriptionArgs) { - try { - command.run(); - } catch (Exception e) { - throw new RuntimeException("Unexpected error", e); - } - } - - @Override - public void yield() throws InterruptedException, FlinkRuntimeException { - Thread.sleep(100); - } - - @Override - public boolean tryYield() throws FlinkRuntimeException { - return false; - } - } } diff --git a/flink-connector-elasticsearch6/archunit-violations/f0b21088-63c4-4e1c-bcda-d54e4a224391 b/flink-connector-elasticsearch6/archunit-violations/f0b21088-63c4-4e1c-bcda-d54e4a224391 new file mode 100644 index 00000000..e69de29b diff --git a/flink-connector-elasticsearch6/archunit-violations/stored.rules b/flink-connector-elasticsearch6/archunit-violations/stored.rules index 804724fb..495e05d7 100644 --- a/flink-connector-elasticsearch6/archunit-violations/stored.rules +++ b/flink-connector-elasticsearch6/archunit-violations/stored.rules @@ -1,10 +1,11 @@ # -#Thu May 12 11:59:51 CEST 2022 +#Sat Nov 02 19:05:27 GMT 2024 +Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @Public\ must\ be\ annotated\ with\ @Public.=82e2a33f-9c4b-49ed-95d5-2319a9ae5e1f +Connector\ production\ code\ must\ not\ depend\ on\ non-public\ API\ outside\ of\ connector\ packages=a2f3489b-bdd7-430b-8723-6ac3502c173e +ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=db3972e4-f3a3-45b2-9643-27cba0cef09d Production\ code\ must\ not\ call\ methods\ annotated\ with\ @VisibleForTesting=a4b5cf92-8e7d-43af-9d55-c693e352a3bc Options\ for\ connectors\ and\ formats\ should\ reside\ in\ a\ consistent\ package\ and\ be\ public\ API.=795761fe-68cf-40d9-8ed8-bc6f47cb2292 Tests\ inheriting\ from\ AbstractTestBase\ should\ have\ name\ ending\ with\ ITCase=25e52d29-fa7e-42fa-a571-b5c76235df52 -Classes\ in\ API\ packages\ should\ have\ at\ least\ one\ API\ visibility\ annotation.=07e22212-0fdc-435a-9a8b-aca438213316 Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @PublicEvolving\ must\ be\ annotated\ with\ @Public(Evolving).=3010cd85-0143-4cff-9095-1f24a1979f7f -Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @Public\ must\ be\ annotated\ with\ @Public.=82e2a33f-9c4b-49ed-95d5-2319a9ae5e1f -ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=db3972e4-f3a3-45b2-9643-27cba0cef09d -Connector\ production\ code\ must\ not\ depend\ on\ non-public\ API\ outside\ of\ connector\ packages=a2f3489b-bdd7-430b-8723-6ac3502c173e +Classes\ in\ API\ packages\ should\ have\ at\ least\ one\ API\ visibility\ annotation.=07e22212-0fdc-435a-9a8b-aca438213316 +Connector\ production\ code\ must\ depend\ only\ on\ public\ API\ when\ outside\ of\ connector\ packages=f0b21088-63c4-4e1c-bcda-d54e4a224391 diff --git a/pom.xml b/pom.xml index 3a27ceeb..a09b868c 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ under the License. - 1.18.0 + 1.20.0 2.15.3 4.13.2