[FLINK-36651][Connectors/Elasticsearch] Fix IT test not compatible wi…

…th 1.20, drop main branch tests for 1.19 and 1.18
apache · Nov 4, 2024 · f826aca · f826aca
1 parent da2ef1f
commit f826aca
Show file tree

Hide file tree

Showing 8 changed files with 29 additions and 57 deletions.
diff --git a/.github/workflows/push_pr.yml b/.github/workflows/push_pr.yml
@@ -28,13 +28,8 @@ jobs:
   compile_and_test:
     strategy:
       matrix:
-        flink: [ 1.18-SNAPSHOT ]
-        jdk: [ '8, 11, 17' ]
-        include:
-          - flink: 1.19-SNAPSHOT
-            jdk: '8, 11, 17, 21'
-          - flink: 1.20-SNAPSHOT
-            jdk: '8, 11, 17, 21'
+        flink: [ 1.19-SNAPSHOT, 1.20-SNAPSHOT ]
+        jdk: [ '8, 11, 17, 21' ]
     uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
     with:
       flink_version: ${{ matrix.flink }}

diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml
@@ -30,27 +30,21 @@ jobs:
     strategy:
       matrix:
         flink_branches: [{
-          flink: 1.18-SNAPSHOT,
-          jdk: '8, 11, 17',
-          branch: main
-        }, {
           flink: 1.19-SNAPSHOT,
-          jdk: '8, 11, 17, 21',
           branch: main
         }, {
           flink: 1.20-SNAPSHOT,
-          jdk: '8, 11, 17, 21',
           branch: main
-        }, {
-          flink: 1.18.1,
-          branch: v3.0
         }, {
           flink: 1.19.0,
           branch: v3.0
+        }, {
+          flink: 1.20.0,
+          branch: main
         }]
     uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
     with:
       flink_version: ${{ matrix.flink_branches.flink }}
       connector_branch: ${{ matrix.flink_branches.branch }}
-      jdk_version: ${{ matrix.flink_branches.jdk || '8, 11' }}
+      jdk_version: ${{ matrix.flink_branches.jdk || '8, 11, 17, 21' }}
       run_dependency_convergence: false
diff --git a/flink-connector-elasticsearch-base/archunit-violations/61c75739-865e-4e9d-9da6-40bcde6a923c b/flink-connector-elasticsearch-base/archunit-violations/61c75739-865e-4e9d-9da6-40bcde6a923c
@@ -0,0 +1,8 @@
+Field <org.apache.flink.connector.elasticsearch.sink.ElasticsearchWriter$DefaultBulkResponseInspector.failureHandler> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ElasticsearchWriter.java:0)
+Method <org.apache.flink.connector.elasticsearch.sink.ElasticsearchSink.getBulkResponseInspectorFactory()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ElasticsearchSink.java:0)
+Method <org.apache.flink.connector.elasticsearch.sink.ElasticsearchSink.getDeliveryGuarantee()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ElasticsearchSink.java:0)
+Method <org.apache.flink.connector.elasticsearch.sink.ElasticsearchSink.getNetworkClientConfig()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ElasticsearchSink.java:0)
+Method <org.apache.flink.connector.elasticsearch.sink.ElasticsearchSinkBuilderBase.build()> calls method <org.apache.flink.api.java.ClosureCleaner.clean(java.lang.Object, org.apache.flink.api.common.ExecutionConfig$ClosureCleanerLevel, boolean)> in (ElasticsearchSinkBuilderBase.java:370)
+Method <org.apache.flink.connector.elasticsearch.sink.ElasticsearchWriter.blockingFlushAllActions()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ElasticsearchWriter.java:0)
+Method <org.apache.flink.connector.elasticsearch.table.IndexGeneratorFactory.createRuntimeIndexGenerator(java.lang.String, [Ljava.lang.String;, [Lorg.apache.flink.table.types.DataType;, org.apache.flink.connector.elasticsearch.table.IndexGeneratorFactory$IndexHelper, java.time.ZoneId)> has parameter of type <[Lorg.apache.flink.table.types.DataType;> in (IndexGeneratorFactory.java:0)
+Method <org.apache.flink.connector.elasticsearch.table.RowElasticsearchEmitter$1.getMetricGroup()> calls constructor <org.apache.flink.metrics.groups.UnregisteredMetricsGroup.<init>()> in (RowElasticsearchEmitter.java:74)
diff --git a/flink-connector-elasticsearch-base/archunit-violations/stored.rules b/flink-connector-elasticsearch-base/archunit-violations/stored.rules
@@ -1,10 +1,11 @@
 #
-#Thu May 12 12:03:32 CEST 2022
+#Sat Nov 02 19:05:02 GMT 2024
+Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @Public\ must\ be\ annotated\ with\ @Public.=06cb39ec-97e2-4e4e-8aa7-cd92504b759c
+Connector\ production\ code\ must\ not\ depend\ on\ non-public\ API\ outside\ of\ connector\ packages=325d89d0-9c19-495f-86df-7d609b6d63dd
+ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=dd583797-83e1-414c-a38d-330773978813
 Production\ code\ must\ not\ call\ methods\ annotated\ with\ @VisibleForTesting=4539e81b-9c72-4fd7-b7b3-ffc2e546b968
 Options\ for\ connectors\ and\ formats\ should\ reside\ in\ a\ consistent\ package\ and\ be\ public\ API.=aca99410-c331-4bc2-a45b-1363eb8dd20c
 Tests\ inheriting\ from\ AbstractTestBase\ should\ have\ name\ ending\ with\ ITCase=de342dd1-c974-42c9-8f64-ef182ba8c56d
-Classes\ in\ API\ packages\ should\ have\ at\ least\ one\ API\ visibility\ annotation.=143fca50-a3b9-4a4b-983e-40dd2c424f62
 Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @PublicEvolving\ must\ be\ annotated\ with\ @Public(Evolving).=09532d6b-7cb1-4621-90af-755558763875
-Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @Public\ must\ be\ annotated\ with\ @Public.=06cb39ec-97e2-4e4e-8aa7-cd92504b759c
-ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=dd583797-83e1-414c-a38d-330773978813
-Connector\ production\ code\ must\ not\ depend\ on\ non-public\ API\ outside\ of\ connector\ packages=325d89d0-9c19-495f-86df-7d609b6d63dd
+Classes\ in\ API\ packages\ should\ have\ at\ least\ one\ API\ visibility\ annotation.=143fca50-a3b9-4a4b-983e-40dd2c424f62
+Connector\ production\ code\ must\ depend\ only\ on\ public\ API\ when\ outside\ of\ connector\ packages=61c75739-865e-4e9d-9da6-40bcde6a923c
diff --git a/...rc/test/java/org/apache/flink/connector/elasticsearch/sink/ElasticsearchWriterITCase.java b/...rc/test/java/org/apache/flink/connector/elasticsearch/sink/ElasticsearchWriterITCase.java
@@ -17,7 +17,6 @@
 
 package org.apache.flink.connector.elasticsearch.sink;
 
-import org.apache.flink.api.common.operators.MailboxExecutor;
 import org.apache.flink.api.connector.sink2.SinkWriter;
 import org.apache.flink.api.connector.sink2.SinkWriter.Context;
 import org.apache.flink.api.java.tuple.Tuple2;
@@ -31,14 +30,13 @@
 import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
 import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
 import org.apache.flink.metrics.testutils.MetricListener;
+import org.apache.flink.runtime.mailbox.SyncMailboxExecutor;
 import org.apache.flink.runtime.metrics.MetricNames;
 import org.apache.flink.runtime.metrics.groups.InternalSinkWriterMetricGroup;
 import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups;
 import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
 import org.apache.flink.test.junit5.MiniClusterExtension;
-import org.apache.flink.util.FlinkRuntimeException;
 import org.apache.flink.util.TestLoggerExtension;
-import org.apache.flink.util.function.ThrowingRunnable;
 
 import org.apache.http.HttpHost;
 import org.elasticsearch.action.ActionListener;
@@ -329,7 +327,7 @@ private static ElasticsearchWriter<Tuple2<Integer, String>> createWriter(
                 new DefaultBulkResponseInspector(),
                 new NetworkClientConfig(null, null, null, null, null, null, null, null),
                 metricGroup,
-                new TestMailbox());
+                new SyncMailboxExecutor());
     }
 
     private TestingSinkWriterMetricGroup getSinkWriterMetricGroup() {
@@ -481,29 +479,4 @@ GetResponse getResponse(String index, int id) throws IOException {
             return client.get(new GetRequest(index, Integer.toString(id)), RequestOptions.DEFAULT);
         }
     }
-
-    private static class TestMailbox implements MailboxExecutor {
-
-        @Override
-        public void execute(
-                ThrowingRunnable<? extends Exception> command,
-                String descriptionFormat,
-                Object... descriptionArgs) {
-            try {
-                command.run();
-            } catch (Exception e) {
-                throw new RuntimeException("Unexpected error", e);
-            }
-        }
-
-        @Override
-        public void yield() throws InterruptedException, FlinkRuntimeException {
-            Thread.sleep(100);
-        }
-
-        @Override
-        public boolean tryYield() throws FlinkRuntimeException {
-            return false;
-        }
-    }
 }
diff --git a/flink-connector-elasticsearch6/archunit-violations/f0b21088-63c4-4e1c-bcda-d54e4a224391 b/flink-connector-elasticsearch6/archunit-violations/f0b21088-63c4-4e1c-bcda-d54e4a224391
diff --git a/flink-connector-elasticsearch6/archunit-violations/stored.rules b/flink-connector-elasticsearch6/archunit-violations/stored.rules
@@ -1,10 +1,11 @@
 #
-#Thu May 12 11:59:51 CEST 2022
+#Sat Nov 02 19:05:27 GMT 2024
+Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @Public\ must\ be\ annotated\ with\ @Public.=82e2a33f-9c4b-49ed-95d5-2319a9ae5e1f
+Connector\ production\ code\ must\ not\ depend\ on\ non-public\ API\ outside\ of\ connector\ packages=a2f3489b-bdd7-430b-8723-6ac3502c173e
+ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=db3972e4-f3a3-45b2-9643-27cba0cef09d
 Production\ code\ must\ not\ call\ methods\ annotated\ with\ @VisibleForTesting=a4b5cf92-8e7d-43af-9d55-c693e352a3bc
 Options\ for\ connectors\ and\ formats\ should\ reside\ in\ a\ consistent\ package\ and\ be\ public\ API.=795761fe-68cf-40d9-8ed8-bc6f47cb2292
 Tests\ inheriting\ from\ AbstractTestBase\ should\ have\ name\ ending\ with\ ITCase=25e52d29-fa7e-42fa-a571-b5c76235df52
-Classes\ in\ API\ packages\ should\ have\ at\ least\ one\ API\ visibility\ annotation.=07e22212-0fdc-435a-9a8b-aca438213316
 Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @PublicEvolving\ must\ be\ annotated\ with\ @Public(Evolving).=3010cd85-0143-4cff-9095-1f24a1979f7f
-Return\ and\ argument\ types\ of\ methods\ annotated\ with\ @Public\ must\ be\ annotated\ with\ @Public.=82e2a33f-9c4b-49ed-95d5-2319a9ae5e1f
-ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=db3972e4-f3a3-45b2-9643-27cba0cef09d
-Connector\ production\ code\ must\ not\ depend\ on\ non-public\ API\ outside\ of\ connector\ packages=a2f3489b-bdd7-430b-8723-6ac3502c173e
+Classes\ in\ API\ packages\ should\ have\ at\ least\ one\ API\ visibility\ annotation.=07e22212-0fdc-435a-9a8b-aca438213316
+Connector\ production\ code\ must\ depend\ only\ on\ public\ API\ when\ outside\ of\ connector\ packages=f0b21088-63c4-4e1c-bcda-d54e4a224391
diff --git a/pom.xml b/pom.xml
@@ -51,7 +51,7 @@ under the License.
 	</modules>
 
 	<properties>
-		<flink.version>1.18.0</flink.version>
+		<flink.version>1.20.0</flink.version>
 
 		<jackson-bom.version>2.15.3</jackson-bom.version>
 		<junit4.version>4.13.2</junit4.version>