Merge "Hot fix for InternalIpGateway to include the case when cluster…

… is not Null" -- Branch commit log -- commit 9addb5a14f83e0f7750019e9b27131d01703548e Author: gcpdiag team <[email protected]> Date: 2024-09-16T13:52:30Z Hot fix for InternalIpGateway to include the case when cluster is not Null Change-Id: I215e8f17192f573492d20993a35b07542c6df715 GitOrigin-RevId: 7e059e3bb013223cc58979704df14042f5929a11
GoogleCloudPlatform · Sep 24, 2024 · 8a36070 · 8a36070
1 parent 7d97161
commit 8a36070
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 48 deletions.
diff --git a/gcpdiag/runbook/dataproc/cluster_creation.py b/gcpdiag/runbook/dataproc/cluster_creation.py
@@ -583,27 +583,35 @@ def execute(self):
             'The cluster and the internalIpOnly config cannot be found, skipping this step. '
             + 'Please provide internal_ip_only as input parameter ' +
             'if the cluster is deleted or keep the cluster in error state.')
+        return
       subnetwork_uri = op.get(flags.SUBNETWORK)
       if subnetwork_uri is None:
         op.add_skipped(
             cluster,
             'The cluster and the subnetworkUri config cannot be found, skipping this step. '
             + 'Please provide subnetwork_uri as input parameter ' +
             'if the cluster is deleted or keep the cluster in error state.')
-      if is_internal_ip_only is not None and subnetwork_uri is not None:
-        # Add the internal IP config of the cluster
-        if not op.get(flags.INTERNAL_IP_ONLY):
-          if cluster.is_internal_ip_only is not None:
-            op.put(flags.INTERNAL_IP_ONLY, cluster.is_internal_ip_only)
-            op.info(
-                ('Internal IP only: {}').format(cluster.is_internal_ip_only),)
-        # Add the subnetwork of the cluster
-        if not op.get(flags.SUBNETWORK):
-          op.put(flags.SUBNETWORK, subnetwork_uri)
-          op.add_ok(cluster, reason=('Subnetwork: {}').format(subnetwork_uri))
-
-        if is_internal_ip_only:
-          self.add_child(child=CheckPrivateGoogleAccess())
+        return
+    else:
+      is_internal_ip_only = cluster.is_internal_ip_only
+      subnetwork_uri = cluster.gce_subnetwork_uri
+    # Add the related configs of the cluster
+    if is_internal_ip_only is not None and subnetwork_uri is not None:
+      # Add the internal IP config of the cluster
+      if not op.get(flags.INTERNAL_IP_ONLY):
+        if cluster.is_internal_ip_only is not None:
+          op.put(flags.INTERNAL_IP_ONLY, cluster.is_internal_ip_only)
+          op.info(('Internal IP only: {}').format(cluster.is_internal_ip_only),)
+      # Add the subnetwork of the cluster
+      if not op.get(flags.SUBNETWORK):
+        op.put(flags.SUBNETWORK, subnetwork_uri)
+        op.add_ok(cluster, reason=('Subnetwork: {}').format(subnetwork_uri))
+    # If the cluster is in private subnet, check that PGA is enabled
+    # otherwise end this step
+    if is_internal_ip_only:
+      self.add_child(child=CheckPrivateGoogleAccess())
+    else:
+      op.add_ok(cluster, reason='The cluster is in a public subnet.')
 
 
 class CheckPrivateGoogleAccess(runbook.Step):

diff --git a/gcpdiag/runbook/dataproc/snapshots/cluster_creation.txt b/gcpdiag/runbook/dataproc/snapshots/cluster_creation.txt
@@ -221,6 +221,15 @@ dataproc/cluster-creation: Provides a comprehensive analysis of common issues wh
      The network communication among nodes in cluster test-deny-icmp is working.
 
 [GATEWAY]: Checking if the cluster is using internal IP only...
+[INFO]: Internal IP only: False
+
+   - gcpdiag-dataproc1-aaaa/us-central1/test-deny-icmp                    [OK]
+     [REASON]
+     Subnetwork: https://www.googleapis.com/compute/v1/projects/gcpdiag-dataproc1-aaaa/regions/us-central1/subnetworks/test-bad-subnet
+
+   - gcpdiag-dataproc1-aaaa/us-central1/test-deny-icmp                    [OK]
+     [REASON]
+     The cluster is in a public subnet.
 [GATEWAY]: Checking service account project...
 [INFO]: [email protected]
 [INFO]: VM Service Account associated with Dataproc cluster was found in the same project

diff --git a/test-data/dataproc1/json-dumps/logging-entries-1.json b/test-data/dataproc1/json-dumps/logging-entries-1.json
@@ -8,8 +8,8 @@
           "message": "Cannot start master: Timed out waiting for 2 nodes. This usually happens when VM to VM communications are blocked by firewall rules. For additional details, see https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/network#firewall_rule_requirement\nOperation timed out: Only 0 out of 2 minimum required datanodes running.\nOperation timed out: Only 0 out of 2 minimum required node managers running."
         },
         "authenticationInfo": {
-          "principalEmail": "abcdef@abcdef.com",
-          "principalSubject": "user:abcdef@abcdef.com"
+          "principalEmail": "abcde@abcde.com",
+          "principalSubject": "user:abcde@abcde.com"
         },
         "requestMetadata": {
           "requestAttributes": {},
@@ -24,9 +24,9 @@
         "type": "cloud_dataproc_cluster",
         "labels": {
           "project_id": "gcpdiag-dataproc1-aaaa",
-          "region": "us-central1",
+          "cluster_uuid": "c02d5602-14ac-4e8e-af4f-f83df01b3b16",
           "cluster_name": "test-deny-icmp",
-          "cluster_uuid": "c02d5602-14ac-4e8e-af4f-f83df01b3b16"
+          "region": "us-central1"
         }
       },
       "timestamp": "2024-06-25T13:01:48.296661Z",
@@ -44,8 +44,8 @@
         "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
         "status": {},
         "authenticationInfo": {
-          "principalEmail": "abcdef@abcdef.com",
-          "principalSubject": "user:abcdef@abcdef.com"
+          "principalEmail": "abcde@abcde.com",
+          "principalSubject": "user:abcde@abcde.com"
         },
         "requestMetadata": {
           "callerIp": "gce-internal-ip",
@@ -95,10 +95,10 @@
       "resource": {
         "type": "cloud_dataproc_cluster",
         "labels": {
-          "region": "us-central1",
-          "project_id": "gcpdiag-dataproc1-aaaa",
           "cluster_name": "test-deny-icmp",
-          "cluster_uuid": "c02d5602-14ac-4e8e-af4f-f83df01b3b16"
+          "project_id": "gcpdiag-dataproc1-aaaa",
+          "cluster_uuid": "c02d5602-14ac-4e8e-af4f-f83df01b3b16",
+          "region": "us-central1"
         }
       },
       "timestamp": "2024-06-25T12:49:49.631732Z",
@@ -116,8 +116,8 @@
         "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
         "status": {},
         "authenticationInfo": {
-          "principalEmail": "abcdef@abcdef.com",
-          "principalSubject": "user:abcdef@abcdef.com"
+          "principalEmail": "abcde@abcde.com",
+          "principalSubject": "user:abcde@abcde.com"
         },
         "requestMetadata": {
           "requestAttributes": {},
@@ -132,9 +132,9 @@
         "type": "cloud_dataproc_cluster",
         "labels": {
           "cluster_uuid": "e7e3a260-78c1-4364-8ec8-32df4ea2be57",
-          "project_id": "gcpdiag-dataproc1-aaaa",
           "region": "us-central1",
-          "cluster_name": "good"
+          "cluster_name": "good",
+          "project_id": "gcpdiag-dataproc1-aaaa"
         }
       },
       "timestamp": "2024-06-25T12:04:13.209895Z",
@@ -152,8 +152,8 @@
         "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
         "status": {},
         "authenticationInfo": {
-          "principalEmail": "abcdef@abcdef.com",
-          "principalSubject": "user:abcdef@abcdef.com"
+          "principalEmail": "abcde@abcde.com",
+          "principalSubject": "user:abcde@abcde.com"
         },
         "requestMetadata": {
           "requestAttributes": {},
@@ -167,10 +167,10 @@
       "resource": {
         "type": "cloud_dataproc_cluster",
         "labels": {
-          "cluster_name": "test-best-practices-enabled",
           "region": "us-central1",
-          "project_id": "gcpdiag-dataproc1-aaaa",
-          "cluster_uuid": "62a9bfce-295a-4637-b732-8fa8a1cc4196"
+          "cluster_uuid": "62a9bfce-295a-4637-b732-8fa8a1cc4196",
+          "cluster_name": "test-best-practices-enabled",
+          "project_id": "gcpdiag-dataproc1-aaaa"
         }
       },
       "timestamp": "2024-06-25T12:04:02.579303Z",
@@ -188,8 +188,8 @@
         "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
         "status": {},
         "authenticationInfo": {
-          "principalEmail": "abcdef@abcdef.com",
-          "principalSubject": "user:abcdef@abcdef.com"
+          "principalEmail": "abcde@abcde.com",
+          "principalSubject": "user:abcde@abcde.com"
         },
         "requestMetadata": {
           "requestAttributes": {},
@@ -203,9 +203,9 @@
       "resource": {
         "type": "cloud_dataproc_cluster",
         "labels": {
-          "region": "us-central1",
           "cluster_name": "test-best-practices-disabled",
           "project_id": "gcpdiag-dataproc1-aaaa",
+          "region": "us-central1",
           "cluster_uuid": "7426fd71-2543-44c8-aae3-c42b00069d1c"
         }
       },
@@ -224,8 +224,8 @@
         "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
         "status": {},
         "authenticationInfo": {
-          "principalEmail": "abcdef@abcdef.com",
-          "principalSubject": "user:abcdef@abcdef.com"
+          "principalEmail": "abcde@abcde.com",
+          "principalSubject": "user:abcde@abcde.com"
         },
         "requestMetadata": {
           "callerIp": "gce-internal-ip",
@@ -293,10 +293,10 @@
       "resource": {
         "type": "cloud_dataproc_cluster",
         "labels": {
+          "cluster_uuid": "62a9bfce-295a-4637-b732-8fa8a1cc4196",
           "cluster_name": "test-best-practices-enabled",
-          "region": "us-central1",
           "project_id": "gcpdiag-dataproc1-aaaa",
-          "cluster_uuid": "62a9bfce-295a-4637-b732-8fa8a1cc4196"
+          "region": "us-central1"
         }
       },
       "timestamp": "2024-06-25T12:00:10.039120Z",
@@ -314,8 +314,8 @@
         "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
         "status": {},
         "authenticationInfo": {
-          "principalEmail": "abcdef@abcdef.com",
-          "principalSubject": "user:abcdef@abcdef.com"
+          "principalEmail": "abcde@abcde.com",
+          "principalSubject": "user:abcde@abcde.com"
         },
         "requestMetadata": {
           "callerIp": "gce-internal-ip",
@@ -375,10 +375,10 @@
       "resource": {
         "type": "cloud_dataproc_cluster",
         "labels": {
-          "cluster_name": "good",
-          "region": "us-central1",
+          "project_id": "gcpdiag-dataproc1-aaaa",
           "cluster_uuid": "e7e3a260-78c1-4364-8ec8-32df4ea2be57",
-          "project_id": "gcpdiag-dataproc1-aaaa"
+          "region": "us-central1",
+          "cluster_name": "good"
         }
       },
       "timestamp": "2024-06-25T12:00:10.037846Z",
@@ -396,8 +396,8 @@
         "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
         "status": {},
         "authenticationInfo": {
-          "principalEmail": "abcdef@abcdef.com",
-          "principalSubject": "user:abcdef@abcdef.com"
+          "principalEmail": "abcde@abcde.com",
+          "principalSubject": "user:abcde@abcde.com"
         },
         "requestMetadata": {
           "callerIp": "gce-internal-ip",
@@ -463,9 +463,9 @@
         "type": "cloud_dataproc_cluster",
         "labels": {
           "region": "us-central1",
-          "cluster_name": "test-best-practices-disabled",
           "cluster_uuid": "7426fd71-2543-44c8-aae3-c42b00069d1c",
-          "project_id": "gcpdiag-dataproc1-aaaa"
+          "project_id": "gcpdiag-dataproc1-aaaa",
+          "cluster_name": "test-best-practices-disabled"
         }
       },
       "timestamp": "2024-06-25T12:00:09.939283Z",