From 8a36070edc2a17c2051f7018c03f59dd9d7f8801 Mon Sep 17 00:00:00 2001 From: gcpdiag team Date: Tue, 24 Sep 2024 15:19:09 +0000 Subject: [PATCH] Merge "Hot fix for InternalIpGateway to include the case when cluster is not Null" -- Branch commit log -- commit 9addb5a14f83e0f7750019e9b27131d01703548e Author: gcpdiag team Date: 2024-09-16T13:52:30Z Hot fix for InternalIpGateway to include the case when cluster is not Null Change-Id: I215e8f17192f573492d20993a35b07542c6df715 GitOrigin-RevId: 7e059e3bb013223cc58979704df14042f5929a11 --- gcpdiag/runbook/dataproc/cluster_creation.py | 36 ++++++---- .../dataproc/snapshots/cluster_creation.txt | 9 +++ .../json-dumps/logging-entries-1.json | 68 +++++++++---------- 3 files changed, 65 insertions(+), 48 deletions(-) diff --git a/gcpdiag/runbook/dataproc/cluster_creation.py b/gcpdiag/runbook/dataproc/cluster_creation.py index 69cc17dcc..25494dd32 100644 --- a/gcpdiag/runbook/dataproc/cluster_creation.py +++ b/gcpdiag/runbook/dataproc/cluster_creation.py @@ -583,6 +583,7 @@ def execute(self): 'The cluster and the internalIpOnly config cannot be found, skipping this step. ' + 'Please provide internal_ip_only as input parameter ' + 'if the cluster is deleted or keep the cluster in error state.') + return subnetwork_uri = op.get(flags.SUBNETWORK) if subnetwork_uri is None: op.add_skipped( @@ -590,20 +591,27 @@ def execute(self): 'The cluster and the subnetworkUri config cannot be found, skipping this step. ' + 'Please provide subnetwork_uri as input parameter ' + 'if the cluster is deleted or keep the cluster in error state.') - if is_internal_ip_only is not None and subnetwork_uri is not None: - # Add the internal IP config of the cluster - if not op.get(flags.INTERNAL_IP_ONLY): - if cluster.is_internal_ip_only is not None: - op.put(flags.INTERNAL_IP_ONLY, cluster.is_internal_ip_only) - op.info( - ('Internal IP only: {}').format(cluster.is_internal_ip_only),) - # Add the subnetwork of the cluster - if not op.get(flags.SUBNETWORK): - op.put(flags.SUBNETWORK, subnetwork_uri) - op.add_ok(cluster, reason=('Subnetwork: {}').format(subnetwork_uri)) - - if is_internal_ip_only: - self.add_child(child=CheckPrivateGoogleAccess()) + return + else: + is_internal_ip_only = cluster.is_internal_ip_only + subnetwork_uri = cluster.gce_subnetwork_uri + # Add the related configs of the cluster + if is_internal_ip_only is not None and subnetwork_uri is not None: + # Add the internal IP config of the cluster + if not op.get(flags.INTERNAL_IP_ONLY): + if cluster.is_internal_ip_only is not None: + op.put(flags.INTERNAL_IP_ONLY, cluster.is_internal_ip_only) + op.info(('Internal IP only: {}').format(cluster.is_internal_ip_only),) + # Add the subnetwork of the cluster + if not op.get(flags.SUBNETWORK): + op.put(flags.SUBNETWORK, subnetwork_uri) + op.add_ok(cluster, reason=('Subnetwork: {}').format(subnetwork_uri)) + # If the cluster is in private subnet, check that PGA is enabled + # otherwise end this step + if is_internal_ip_only: + self.add_child(child=CheckPrivateGoogleAccess()) + else: + op.add_ok(cluster, reason='The cluster is in a public subnet.') class CheckPrivateGoogleAccess(runbook.Step): diff --git a/gcpdiag/runbook/dataproc/snapshots/cluster_creation.txt b/gcpdiag/runbook/dataproc/snapshots/cluster_creation.txt index abd778343..819880a39 100644 --- a/gcpdiag/runbook/dataproc/snapshots/cluster_creation.txt +++ b/gcpdiag/runbook/dataproc/snapshots/cluster_creation.txt @@ -221,6 +221,15 @@ dataproc/cluster-creation: Provides a comprehensive analysis of common issues wh The network communication among nodes in cluster test-deny-icmp is working. [GATEWAY]: Checking if the cluster is using internal IP only... +[INFO]: Internal IP only: False + + - gcpdiag-dataproc1-aaaa/us-central1/test-deny-icmp [OK] + [REASON] + Subnetwork: https://www.googleapis.com/compute/v1/projects/gcpdiag-dataproc1-aaaa/regions/us-central1/subnetworks/test-bad-subnet + + - gcpdiag-dataproc1-aaaa/us-central1/test-deny-icmp [OK] + [REASON] + The cluster is in a public subnet. [GATEWAY]: Checking service account project... [INFO]: 12340005-compute@developer.gserviceaccount.com [INFO]: VM Service Account associated with Dataproc cluster was found in the same project diff --git a/test-data/dataproc1/json-dumps/logging-entries-1.json b/test-data/dataproc1/json-dumps/logging-entries-1.json index 45fd120ba..8497ec8f2 100644 --- a/test-data/dataproc1/json-dumps/logging-entries-1.json +++ b/test-data/dataproc1/json-dumps/logging-entries-1.json @@ -8,8 +8,8 @@ "message": "Cannot start master: Timed out waiting for 2 nodes. This usually happens when VM to VM communications are blocked by firewall rules. For additional details, see https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/network#firewall_rule_requirement\nOperation timed out: Only 0 out of 2 minimum required datanodes running.\nOperation timed out: Only 0 out of 2 minimum required node managers running." }, "authenticationInfo": { - "principalEmail": "abcdef@abcdef.com", - "principalSubject": "user:abcdef@abcdef.com" + "principalEmail": "abcde@abcde.com", + "principalSubject": "user:abcde@abcde.com" }, "requestMetadata": { "requestAttributes": {}, @@ -24,9 +24,9 @@ "type": "cloud_dataproc_cluster", "labels": { "project_id": "gcpdiag-dataproc1-aaaa", - "region": "us-central1", + "cluster_uuid": "c02d5602-14ac-4e8e-af4f-f83df01b3b16", "cluster_name": "test-deny-icmp", - "cluster_uuid": "c02d5602-14ac-4e8e-af4f-f83df01b3b16" + "region": "us-central1" } }, "timestamp": "2024-06-25T13:01:48.296661Z", @@ -44,8 +44,8 @@ "@type": "type.googleapis.com/google.cloud.audit.AuditLog", "status": {}, "authenticationInfo": { - "principalEmail": "abcdef@abcdef.com", - "principalSubject": "user:abcdef@abcdef.com" + "principalEmail": "abcde@abcde.com", + "principalSubject": "user:abcde@abcde.com" }, "requestMetadata": { "callerIp": "gce-internal-ip", @@ -95,10 +95,10 @@ "resource": { "type": "cloud_dataproc_cluster", "labels": { - "region": "us-central1", - "project_id": "gcpdiag-dataproc1-aaaa", "cluster_name": "test-deny-icmp", - "cluster_uuid": "c02d5602-14ac-4e8e-af4f-f83df01b3b16" + "project_id": "gcpdiag-dataproc1-aaaa", + "cluster_uuid": "c02d5602-14ac-4e8e-af4f-f83df01b3b16", + "region": "us-central1" } }, "timestamp": "2024-06-25T12:49:49.631732Z", @@ -116,8 +116,8 @@ "@type": "type.googleapis.com/google.cloud.audit.AuditLog", "status": {}, "authenticationInfo": { - "principalEmail": "abcdef@abcdef.com", - "principalSubject": "user:abcdef@abcdef.com" + "principalEmail": "abcde@abcde.com", + "principalSubject": "user:abcde@abcde.com" }, "requestMetadata": { "requestAttributes": {}, @@ -132,9 +132,9 @@ "type": "cloud_dataproc_cluster", "labels": { "cluster_uuid": "e7e3a260-78c1-4364-8ec8-32df4ea2be57", - "project_id": "gcpdiag-dataproc1-aaaa", "region": "us-central1", - "cluster_name": "good" + "cluster_name": "good", + "project_id": "gcpdiag-dataproc1-aaaa" } }, "timestamp": "2024-06-25T12:04:13.209895Z", @@ -152,8 +152,8 @@ "@type": "type.googleapis.com/google.cloud.audit.AuditLog", "status": {}, "authenticationInfo": { - "principalEmail": "abcdef@abcdef.com", - "principalSubject": "user:abcdef@abcdef.com" + "principalEmail": "abcde@abcde.com", + "principalSubject": "user:abcde@abcde.com" }, "requestMetadata": { "requestAttributes": {}, @@ -167,10 +167,10 @@ "resource": { "type": "cloud_dataproc_cluster", "labels": { - "cluster_name": "test-best-practices-enabled", "region": "us-central1", - "project_id": "gcpdiag-dataproc1-aaaa", - "cluster_uuid": "62a9bfce-295a-4637-b732-8fa8a1cc4196" + "cluster_uuid": "62a9bfce-295a-4637-b732-8fa8a1cc4196", + "cluster_name": "test-best-practices-enabled", + "project_id": "gcpdiag-dataproc1-aaaa" } }, "timestamp": "2024-06-25T12:04:02.579303Z", @@ -188,8 +188,8 @@ "@type": "type.googleapis.com/google.cloud.audit.AuditLog", "status": {}, "authenticationInfo": { - "principalEmail": "abcdef@abcdef.com", - "principalSubject": "user:abcdef@abcdef.com" + "principalEmail": "abcde@abcde.com", + "principalSubject": "user:abcde@abcde.com" }, "requestMetadata": { "requestAttributes": {}, @@ -203,9 +203,9 @@ "resource": { "type": "cloud_dataproc_cluster", "labels": { - "region": "us-central1", "cluster_name": "test-best-practices-disabled", "project_id": "gcpdiag-dataproc1-aaaa", + "region": "us-central1", "cluster_uuid": "7426fd71-2543-44c8-aae3-c42b00069d1c" } }, @@ -224,8 +224,8 @@ "@type": "type.googleapis.com/google.cloud.audit.AuditLog", "status": {}, "authenticationInfo": { - "principalEmail": "abcdef@abcdef.com", - "principalSubject": "user:abcdef@abcdef.com" + "principalEmail": "abcde@abcde.com", + "principalSubject": "user:abcde@abcde.com" }, "requestMetadata": { "callerIp": "gce-internal-ip", @@ -293,10 +293,10 @@ "resource": { "type": "cloud_dataproc_cluster", "labels": { + "cluster_uuid": "62a9bfce-295a-4637-b732-8fa8a1cc4196", "cluster_name": "test-best-practices-enabled", - "region": "us-central1", "project_id": "gcpdiag-dataproc1-aaaa", - "cluster_uuid": "62a9bfce-295a-4637-b732-8fa8a1cc4196" + "region": "us-central1" } }, "timestamp": "2024-06-25T12:00:10.039120Z", @@ -314,8 +314,8 @@ "@type": "type.googleapis.com/google.cloud.audit.AuditLog", "status": {}, "authenticationInfo": { - "principalEmail": "abcdef@abcdef.com", - "principalSubject": "user:abcdef@abcdef.com" + "principalEmail": "abcde@abcde.com", + "principalSubject": "user:abcde@abcde.com" }, "requestMetadata": { "callerIp": "gce-internal-ip", @@ -375,10 +375,10 @@ "resource": { "type": "cloud_dataproc_cluster", "labels": { - "cluster_name": "good", - "region": "us-central1", + "project_id": "gcpdiag-dataproc1-aaaa", "cluster_uuid": "e7e3a260-78c1-4364-8ec8-32df4ea2be57", - "project_id": "gcpdiag-dataproc1-aaaa" + "region": "us-central1", + "cluster_name": "good" } }, "timestamp": "2024-06-25T12:00:10.037846Z", @@ -396,8 +396,8 @@ "@type": "type.googleapis.com/google.cloud.audit.AuditLog", "status": {}, "authenticationInfo": { - "principalEmail": "abcdef@abcdef.com", - "principalSubject": "user:abcdef@abcdef.com" + "principalEmail": "abcde@abcde.com", + "principalSubject": "user:abcde@abcde.com" }, "requestMetadata": { "callerIp": "gce-internal-ip", @@ -463,9 +463,9 @@ "type": "cloud_dataproc_cluster", "labels": { "region": "us-central1", - "cluster_name": "test-best-practices-disabled", "cluster_uuid": "7426fd71-2543-44c8-aae3-c42b00069d1c", - "project_id": "gcpdiag-dataproc1-aaaa" + "project_id": "gcpdiag-dataproc1-aaaa", + "cluster_name": "test-best-practices-disabled" } }, "timestamp": "2024-06-25T12:00:09.939283Z",