From 9c787ca9739d90b64eb81f3ab6042e5d12065c5e Mon Sep 17 00:00:00 2001
From: Seokyun Ha <bluehornblues@gmail.com>
Date: Thu, 14 Dec 2023 18:02:37 +0900
Subject: [PATCH] feat(ingest): add ingest `--no-progress` option (#9300)

---
 docs/cli.md                                            |  1 +
 metadata-ingestion/src/datahub/cli/ingest_cli.py       | 10 ++++++++++
 .../src/datahub/ingestion/run/pipeline.py              |  6 +++++-
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/docs/cli.md b/docs/cli.md
index 8845ed5a6dac78..cb5077db429061 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -98,6 +98,7 @@ Command Options:
   --preview-workunits       The number of workunits to produce for preview
   --strict-warnings         If enabled, ingestion runs with warnings will yield a non-zero error code
   --test-source-connection  When set, ingestion will only test the source connection details from the recipe
+  --no-progress             If enabled, mute intermediate progress ingestion reports
 ```
 #### ingest --dry-run
 
diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py
index b7827ec9f050b4..569a836f3ef5c2 100644
--- a/metadata-ingestion/src/datahub/cli/ingest_cli.py
+++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py
@@ -97,6 +97,13 @@ def ingest() -> None:
 @click.option(
     "--no-spinner", type=bool, is_flag=True, default=False, help="Turn off spinner"
 )
+@click.option(
+    "--no-progress",
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="If enabled, mute intermediate progress ingestion reports",
+)
 @telemetry.with_telemetry(
     capture_kwargs=[
         "dry_run",
@@ -105,6 +112,7 @@ def ingest() -> None:
         "test_source_connection",
         "no_default_report",
         "no_spinner",
+        "no_progress",
     ]
 )
 def run(
@@ -117,6 +125,7 @@ def run(
     report_to: str,
     no_default_report: bool,
     no_spinner: bool,
+    no_progress: bool,
 ) -> None:
     """Ingest metadata into DataHub."""
 
@@ -170,6 +179,7 @@ async def run_ingestion_and_check_upgrade() -> int:
             preview_workunits,
             report_to,
             no_default_report,
+            no_progress,
             raw_pipeline_config,
         )
 
diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
index f2735c24ca19dc..25e17d692109a5 100644
--- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
+++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
@@ -173,6 +173,7 @@ def __init__(
         preview_workunits: int = 10,
         report_to: Optional[str] = None,
         no_default_report: bool = False,
+        no_progress: bool = False,
     ):
         self.config = config
         self.dry_run = dry_run
@@ -180,6 +181,7 @@ def __init__(
         self.preview_workunits = preview_workunits
         self.report_to = report_to
         self.reporters: List[PipelineRunListener] = []
+        self.no_progress = no_progress
         self.num_intermediate_workunits = 0
         self.last_time_printed = int(time.time())
         self.cli_report = CliReport()
@@ -330,6 +332,7 @@ def create(
         preview_workunits: int = 10,
         report_to: Optional[str] = "datahub",
         no_default_report: bool = False,
+        no_progress: bool = False,
         raw_config: Optional[dict] = None,
     ) -> "Pipeline":
         config = PipelineConfig.from_dict(config_dict, raw_config)
@@ -340,6 +343,7 @@ def create(
             preview_workunits=preview_workunits,
             report_to=report_to,
             no_default_report=no_default_report,
+            no_progress=no_progress,
         )
 
     def _time_to_print(self) -> bool:
@@ -379,7 +383,7 @@ def run(self) -> None:
                     self.preview_workunits if self.preview_mode else None,
                 ):
                     try:
-                        if self._time_to_print():
+                        if self._time_to_print() and not self.no_progress:
                             self.pretty_print_summary(currently_running=True)
                     except Exception as e:
                         logger.warning(f"Failed to print summary {e}")