From 159a4305ce0c68f9e7dc73576c75036e461f00ad Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Fri, 17 Jan 2025 19:55:15 +0530
Subject: [PATCH 1/2] dev(ingest): move modules from isort,flake8 to ruff

---
 .../airflow-plugin/build.gradle               |  6 +--
 .../airflow-plugin/pyproject.toml             | 53 ++++++++++++++++---
 .../airflow-plugin/setup.cfg                  | 21 --------
 .../airflow-plugin/setup.py                   |  4 +-
 .../src/datahub_airflow_plugin/_config.py     |  5 +-
 .../_datahub_ol_adapter.py                    |  3 +-
 .../src/datahub_airflow_plugin/_extractors.py | 23 ++++----
 .../client/airflow_generator.py               |  2 +-
 .../datahub_listener.py                       | 10 ++--
 .../datahub_airflow_plugin/datahub_plugin.py  | 10 ++--
 .../datahub_plugin_v22.py                     |  2 +-
 .../src/datahub_airflow_plugin/entities.py    |  1 +
 .../example_dags/generic_recipe_sample_dag.py |  1 +
 .../example_dags/graph_usage_sample_dag.py    |  2 +-
 .../example_dags/lineage_emission_dag.py      |  2 +-
 .../datahub_airflow_plugin/hooks/datahub.py   |  2 +
 .../lineage/_lineage_core.py                  |  1 -
 .../operators/datahub.py                      |  2 +-
 .../operators/datahub_assertion_operator.py   |  2 +-
 .../operators/datahub_assertion_sensor.py     |  2 +-
 .../operators/datahub_operation_operator.py   |  2 +-
 .../operators/datahub_operation_sensor.py     |  2 +-
 ...hub_emitter_operator_jinja_template_dag.py |  2 +-
 .../tests/integration/test_plugin.py          |  2 +-
 .../airflow-plugin/tests/unit/test_airflow.py |  2 +-
 .../tests/unit/test_packaging.py              |  2 +-
 .../dagster-plugin/build.gradle               |  6 +--
 .../dagster-plugin/pyproject.toml             | 52 +++++++++++++++---
 .../dagster-plugin/setup.cfg                  | 21 --------
 .../dagster-plugin/setup.py                   |  5 +-
 .../client/dagster_generator.py               |  2 +
 .../sensors/datahub_sensors.py                |  6 ++-
 .../dagster-plugin/tests/unit/test_dagster.py |  4 +-
 .../gx-plugin/pyproject.toml                  |  3 --
 .../prefect-plugin/pyproject.toml             |  3 --
 metadata-ingestion/pyproject.toml             | 12 +----
 .../api/entities/dataproduct/dataproduct.py   |  2 +-
 .../datahub/ingestion/source/abs/source.py    |  2 +-
 .../ingestion/source/dremio/dremio_api.py     |  2 +-
 .../ingestion/source/neo4j/neo4j_source.py    |  2 +-
 .../src/datahub/ingestion/source/s3/source.py |  2 +-
 .../ingestion/source/schema/json_schema.py    |  2 +-
 .../ingestion/source/sql/clickhouse.py        |  2 +-
 .../ingestion/source/tableau/tableau.py       |  2 +-
 .../src/datahub/testing/mcp_diff.py           |  2 +-
 .../src/datahub/utilities/sqllineage_patch.py |  2 +-
 .../integration/powerbi/test_m_parser.py      |  4 +-
 .../tests/integration/powerbi/test_powerbi.py |  4 +-
 .../tests/performance/data_generation.py      |  4 +-
 smoke-test/pyproject.toml                     |  2 -
 50 files changed, 168 insertions(+), 146 deletions(-)

diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle
index 68a35c0dfc417..1bcb58e6b7c54 100644
--- a/metadata-ingestion-modules/airflow-plugin/build.gradle
+++ b/metadata-ingestion-modules/airflow-plugin/build.gradle
@@ -74,16 +74,14 @@ task lint(type: Exec, dependsOn: installDev) {
     "find ${venv_name}/lib -path *airflow/_vendor/connexion/spec.py -exec sed -i.bak -e '169,169s/  # type: List\\[str\\]//g' {} \\; && " +
     "source ${venv_name}/bin/activate && set -x && " +
     "black --check --diff src/ tests/ && " +
-    "isort --check --diff src/ tests/ && " +
-    "flake8 --count --statistics src/ tests/ && " +
+    "ruff check src/ tests/ && " +
     "mypy --show-traceback --show-error-codes src/ tests/"
 }
 task lintFix(type: Exec, dependsOn: installDev) {
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "black src/ tests/ && " +
-    "isort src/ tests/ && " +
-    "flake8 src/ tests/ && " +
+    "ruff check --fix src/ tests/"
     "mypy src/ tests/ "
 }
 
diff --git a/metadata-ingestion-modules/airflow-plugin/pyproject.toml b/metadata-ingestion-modules/airflow-plugin/pyproject.toml
index 648040c1951db..7d03c2a14bf07 100644
--- a/metadata-ingestion-modules/airflow-plugin/pyproject.toml
+++ b/metadata-ingestion-modules/airflow-plugin/pyproject.toml
@@ -10,11 +10,50 @@ extend-exclude = '''
 '''
 include = '\.pyi?$'
 
-[tool.isort]
-indent = '    '
-known_future_library = ['__future__', 'datahub.utilities._markupsafe_compat', 'datahub_provider._airflow_compat']
-profile = 'black'
-sections = 'FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER'
+[tool.ruff.lint.isort]
+combine-as-imports = true
+known-first-party = ["datahub"]
+extra-standard-library = ["__future__", "datahub.utilities._markupsafe_compat", "datahub.sql_parsing._sqlglot_patch"]
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
+force-sort-within-sections = false
+force-wrap-aliases = false
+split-on-trailing-comma = false
+order-by-type = true
+relative-imports-order = "closest-to-furthest"
+force-single-line = false
+single-line-exclusions = ["typing"]
+length-sort = false
+from-first = false
+required-imports = []
+classes = ["typing"]
 
-[tool.pyright]
-extraPaths = ['tests']
\ No newline at end of file
+[tool.ruff.lint]
+select = [
+    "B",
+    "C90",
+    "E",
+    "F",
+    "I", # For isort
+    "TID",
+]
+ignore = [
+    # Ignore line length violations (handled by Black)
+    "E501",
+    # Ignore whitespace before ':' (matches Black)
+    "E203",
+    "E203",
+    # Allow usages of functools.lru_cache
+    "B019",
+    # Allow function call in argument defaults
+    "B008",
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.lint.flake8-tidy-imports]
+# Disallow all relative imports.
+ban-relative-imports = "all"
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]
\ No newline at end of file
diff --git a/metadata-ingestion-modules/airflow-plugin/setup.cfg b/metadata-ingestion-modules/airflow-plugin/setup.cfg
index c25256c5751b8..abb9040ab3535 100644
--- a/metadata-ingestion-modules/airflow-plugin/setup.cfg
+++ b/metadata-ingestion-modules/airflow-plugin/setup.cfg
@@ -1,24 +1,3 @@
-[flake8]
-max-complexity = 15
-ignore =
-    # Ignore: line length issues, since black's formatter will take care of them.
-    E501,
-    # Ignore: 1 blank line required before class docstring.
-    D203,
-    # See https://stackoverflow.com/a/57074416.
-    W503,
-    # See https://github.com/psf/black/issues/315.
-    E203
-exclude =
-    .git,
-    venv,
-    .tox,
-    __pycache__
-per-file-ignores =
-    # imported but unused
-    __init__.py: F401
-ban-relative-imports = true
-
 [mypy]
 plugins =
     sqlmypy,
diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py
index d07063dbffc5c..2fd74b37e89c0 100644
--- a/metadata-ingestion-modules/airflow-plugin/setup.py
+++ b/metadata-ingestion-modules/airflow-plugin/setup.py
@@ -73,9 +73,7 @@ def get_long_description():
     *mypy_stubs,
     "black==22.12.0",
     "coverage>=5.1",
-    "flake8>=3.8.3",
-    "flake8-tidy-imports>=4.3.0",
-    "isort>=5.7.0",
+    "ruff==0.9.1",
     "mypy==1.10.1",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
     # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py
index c4964712cf9f7..6d6ba60155678 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py
@@ -1,11 +1,12 @@
 from enum import Enum
 from typing import TYPE_CHECKING, Optional
 
-import datahub.emitter.mce_builder as builder
 from airflow.configuration import conf
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
 from pydantic.fields import Field
 
+import datahub.emitter.mce_builder as builder
+from datahub.configuration.common import AllowDenyPattern, ConfigModel
+
 if TYPE_CHECKING:
     from datahub_airflow_plugin.hooks.datahub import DatahubGenericHook
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py
index 69de61aced0a5..72cdcd8813252 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py
@@ -1,8 +1,9 @@
 import logging
 
-import datahub.emitter.mce_builder as builder
 from openlineage.client.run import Dataset as OpenLineageDataset
 
+import datahub.emitter.mce_builder as builder
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py
index 28d5775f61f54..fd01ac10f98de 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py
@@ -3,17 +3,11 @@
 import unittest.mock
 from typing import TYPE_CHECKING, Optional
 
-import datahub.emitter.mce_builder as builder
-from datahub.ingestion.source.sql.sqlalchemy_uri_mapper import (
-    get_platform_from_sqlalchemy_uri,
-)
-from datahub.sql_parsing.sqlglot_lineage import (
-    SqlParsingResult,
-    create_lineage_sql_parsed_result,
+from openlineage.airflow.extractors import (
+    BaseExtractor,
+    ExtractorManager as OLExtractorManager,
+    TaskMetadata,
 )
-from openlineage.airflow.extractors import BaseExtractor
-from openlineage.airflow.extractors import ExtractorManager as OLExtractorManager
-from openlineage.airflow.extractors import TaskMetadata
 from openlineage.airflow.extractors.snowflake_extractor import SnowflakeExtractor
 from openlineage.airflow.extractors.sql_extractor import SqlExtractor
 from openlineage.airflow.utils import get_operator_class, try_import_from_string
@@ -23,11 +17,20 @@
     SqlJobFacet,
 )
 
+import datahub.emitter.mce_builder as builder
+from datahub.ingestion.source.sql.sqlalchemy_uri_mapper import (
+    get_platform_from_sqlalchemy_uri,
+)
+from datahub.sql_parsing.sqlglot_lineage import (
+    SqlParsingResult,
+    create_lineage_sql_parsed_result,
+)
 from datahub_airflow_plugin._airflow_shims import Operator
 from datahub_airflow_plugin._datahub_ol_adapter import OL_SCHEME_TWEAKS
 
 if TYPE_CHECKING:
     from airflow.models import DagRun, TaskInstance
+
     from datahub.ingestion.graph.client import DataHubGraph
 
 logger = logging.getLogger(__name__)
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py
index e9f93c0c1eab0..c1ccdaeb0a1fb 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py
@@ -2,6 +2,7 @@
 from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union, cast
 
 from airflow.configuration import conf
+
 from datahub.api.entities.datajob import DataFlow, DataJob
 from datahub.api.entities.dataprocess.dataprocess_instance import (
     DataProcessInstance,
@@ -11,7 +12,6 @@
 from datahub.metadata.schema_classes import DataProcessTypeClass
 from datahub.utilities.urns.data_flow_urn import DataFlowUrn
 from datahub.utilities.urns.data_job_urn import DataJobUrn
-
 from datahub_airflow_plugin._airflow_compat import AIRFLOW_PATCHED
 from datahub_airflow_plugin._config import DatahubLineageConfig, DatajobUrl
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
index 640991a90a1d2..9de44811f60a4 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
@@ -8,9 +8,13 @@
 from typing import TYPE_CHECKING, Callable, Dict, List, Optional, TypeVar, cast
 
 import airflow
-import datahub.emitter.mce_builder as builder
 from airflow.models import Variable
 from airflow.models.serialized_dag import SerializedDagModel
+from openlineage.airflow.listener import TaskHolder
+from openlineage.airflow.utils import redact_with_exclusions
+from openlineage.client.serde import Serde
+
+import datahub.emitter.mce_builder as builder
 from datahub.api.entities.datajob import DataJob
 from datahub.api.entities.dataprocess.dataprocess_instance import InstanceRunResult
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -30,10 +34,6 @@
 )
 from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult
 from datahub.telemetry import telemetry
-from openlineage.airflow.listener import TaskHolder
-from openlineage.airflow.utils import redact_with_exclusions
-from openlineage.client.serde import Serde
-
 from datahub_airflow_plugin._airflow_shims import (
     HAS_AIRFLOW_DAG_LISTENER_API,
     HAS_AIRFLOW_DATASET_LISTENER_API,
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py
index 137cf97f69280..451d2dbfd7c0b 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py
@@ -15,9 +15,9 @@
 logger = logging.getLogger(__name__)
 
 
-_USE_AIRFLOW_LISTENER_INTERFACE = HAS_AIRFLOW_LISTENER_API and not os.getenv(
+_USE_AIRFLOW_LISTENER_INTERFACE = HAS_AIRFLOW_LISTENER_API and os.getenv(
     "DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN", "false"
-).lower() in ("true", "1")
+).lower() not in ("true", "1")
 
 if _USE_AIRFLOW_LISTENER_INTERFACE:
     try:
@@ -32,7 +32,7 @@
 
 
 with contextlib.suppress(Exception):
-    if not os.getenv("DATAHUB_AIRFLOW_PLUGIN_SKIP_FORK_PATCH", "false").lower() in (
+    if os.getenv("DATAHUB_AIRFLOW_PLUGIN_SKIP_FORK_PATCH", "false").lower() not in (
         "true",
         "1",
     ):
@@ -56,8 +56,8 @@ class DatahubPlugin(AirflowPlugin):
     if _USE_AIRFLOW_LISTENER_INTERFACE:
         try:
             if not NEEDS_AIRFLOW_LISTENER_MODULE:
-                from datahub_airflow_plugin.datahub_listener import (  # type: ignore[misc]
-                    get_airflow_plugin_listener,
+                from datahub_airflow_plugin.datahub_listener import (
+                    get_airflow_plugin_listener,  # type: ignore[misc]
                 )
 
                 listeners: list = list(filter(None, [get_airflow_plugin_listener()]))
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py
index db47f37bed562..4bf050d41473e 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py
@@ -7,9 +7,9 @@
 from airflow.lineage import PIPELINE_OUTLETS
 from airflow.models.baseoperator import BaseOperator
 from airflow.utils.module_loading import import_string
+
 from datahub.api.entities.dataprocess.dataprocess_instance import InstanceRunResult
 from datahub.telemetry import telemetry
-
 from datahub_airflow_plugin._airflow_shims import (
     MappedOperator,
     get_task_inlets,
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/entities.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/entities.py
index 30b35ac6d6198..f3fd17259c9f6 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/entities.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/entities.py
@@ -2,6 +2,7 @@
 from typing import List, Optional
 
 import attr
+
 import datahub.emitter.mce_builder as builder
 from datahub.utilities.urns.data_job_urn import DataJobUrn
 from datahub.utilities.urns.dataset_urn import DatasetUrn
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/generic_recipe_sample_dag.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/generic_recipe_sample_dag.py
index ff8dba457066f..ac620852c6f28 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/generic_recipe_sample_dag.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/generic_recipe_sample_dag.py
@@ -9,6 +9,7 @@
 from airflow import DAG
 from airflow.operators.python import PythonOperator
 from airflow.utils.dates import days_ago
+
 from datahub.configuration.config_loader import load_config_file
 from datahub.ingestion.run.pipeline import Pipeline
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/graph_usage_sample_dag.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/graph_usage_sample_dag.py
index d72ba67c23cd7..7951d6f7fd21e 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/graph_usage_sample_dag.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/graph_usage_sample_dag.py
@@ -4,8 +4,8 @@
 
 import pendulum
 from airflow.decorators import dag, task
-from datahub.ingestion.graph.client import DataHubGraph, RemovedStatusFilter
 
+from datahub.ingestion.graph.client import DataHubGraph, RemovedStatusFilter
 from datahub_airflow_plugin.hooks.datahub import DatahubRestHook
 
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_emission_dag.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_emission_dag.py
index 0d7cdb6b6e90a..4351f40fe7e3a 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_emission_dag.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_emission_dag.py
@@ -4,11 +4,11 @@
 """
 from datetime import timedelta
 
-import datahub.emitter.mce_builder as builder
 from airflow import DAG
 from airflow.operators.bash import BashOperator
 from airflow.utils.dates import days_ago
 
+import datahub.emitter.mce_builder as builder
 from datahub_airflow_plugin.operators.datahub import DatahubEmitterOperator
 
 default_args = {
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/hooks/datahub.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/hooks/datahub.py
index 5f4d787fb893d..26c5026c075bd 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/hooks/datahub.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/hooks/datahub.py
@@ -2,6 +2,7 @@
 
 from airflow.exceptions import AirflowException
 from airflow.hooks.base import BaseHook
+
 from datahub.emitter.generic_emitter import Emitter
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import (
@@ -11,6 +12,7 @@
 
 if TYPE_CHECKING:
     from airflow.models.connection import Connection
+
     from datahub.emitter.kafka_emitter import DatahubKafkaEmitter
     from datahub.emitter.rest_emitter import DataHubRestEmitter
     from datahub.emitter.synchronized_file_emitter import SynchronizedFileEmitter
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py
index 638458b0efd6a..db50c48dfaf08 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py
@@ -2,7 +2,6 @@
 from typing import TYPE_CHECKING, Dict, List
 
 from datahub.api.entities.dataprocess.dataprocess_instance import InstanceRunResult
-
 from datahub_airflow_plugin._config import DatahubLineageConfig
 from datahub_airflow_plugin.client.airflow_generator import AirflowGenerator
 from datahub_airflow_plugin.entities import (
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub.py
index 41d855512aa06..817db6b7480c4 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub.py
@@ -3,9 +3,9 @@
 from airflow.models import BaseOperator
 from airflow.utils.decorators import apply_defaults
 from avrogen.dict_wrapper import DictWrapper
+
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
-
 from datahub_airflow_plugin.hooks.datahub import (
     DatahubGenericHook,
     DatahubKafkaHook,
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_assertion_operator.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_assertion_operator.py
index 6f93c09a9e287..3a440b0ec14e0 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_assertion_operator.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_assertion_operator.py
@@ -2,11 +2,11 @@
 from typing import Any, List, Optional, Sequence, Union
 
 from airflow.models import BaseOperator
+
 from datahub.api.circuit_breaker import (
     AssertionCircuitBreaker,
     AssertionCircuitBreakerConfig,
 )
-
 from datahub_airflow_plugin.hooks.datahub import DatahubRestHook
 
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_assertion_sensor.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_assertion_sensor.py
index 16e5d1cbe8b1f..6a446ba1f3b55 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_assertion_sensor.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_assertion_sensor.py
@@ -2,11 +2,11 @@
 from typing import Any, List, Optional, Sequence, Union
 
 from airflow.sensors.base import BaseSensorOperator
+
 from datahub.api.circuit_breaker import (
     AssertionCircuitBreaker,
     AssertionCircuitBreakerConfig,
 )
-
 from datahub_airflow_plugin.hooks.datahub import DatahubRestHook
 
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_operation_operator.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_operation_operator.py
index 94e105309537b..eb5fe8168bccf 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_operation_operator.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_operation_operator.py
@@ -2,11 +2,11 @@
 from typing import Any, List, Optional, Sequence, Union
 
 from airflow.sensors.base import BaseSensorOperator
+
 from datahub.api.circuit_breaker import (
     OperationCircuitBreaker,
     OperationCircuitBreakerConfig,
 )
-
 from datahub_airflow_plugin.hooks.datahub import DatahubRestHook
 
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_operation_sensor.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_operation_sensor.py
index 434c60754064d..89e20e46a0074 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_operation_sensor.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub_operation_sensor.py
@@ -2,11 +2,11 @@
 from typing import Any, List, Optional, Sequence, Union
 
 from airflow.sensors.base import BaseSensorOperator
+
 from datahub.api.circuit_breaker import (
     OperationCircuitBreaker,
     OperationCircuitBreakerConfig,
 )
-
 from datahub_airflow_plugin.hooks.datahub import DatahubRestHook
 
 
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/datahub_emitter_operator_jinja_template_dag.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/datahub_emitter_operator_jinja_template_dag.py
index c1b4aa4d7b94f..04845e601d674 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/datahub_emitter_operator_jinja_template_dag.py
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/datahub_emitter_operator_jinja_template_dag.py
@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta
 
 from airflow import DAG
+
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.schema_classes import (
@@ -9,7 +10,6 @@
     DatasetPropertiesClass,
     DatasetSnapshotClass,
 )
-
 from datahub_airflow_plugin.operators.datahub import DatahubEmitterOperator
 
 default_args = {
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py
index 3b2c9140e4632..d2c9821295419 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py
@@ -17,9 +17,9 @@
 import requests
 import tenacity
 from airflow.models.connection import Connection
+
 from datahub.ingestion.sink.file import write_metadata_file
 from datahub.testing.compare_metadata_json import assert_metadata_files_equal
-
 from datahub_airflow_plugin._airflow_shims import (
     AIRFLOW_VERSION,
     HAS_AIRFLOW_DAG_LISTENER_API,
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py
index 79620f81a437b..1dc8e14a425df 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py
+++ b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py
@@ -8,12 +8,12 @@
 
 import airflow.configuration
 import airflow.version
-import datahub.emitter.mce_builder as builder
 import packaging.version
 import pytest
 from airflow.lineage import apply_lineage, prepare_lineage
 from airflow.models import DAG, Connection, DagBag, DagRun, TaskInstance
 
+import datahub.emitter.mce_builder as builder
 from datahub_airflow_plugin import get_provider_info
 from datahub_airflow_plugin._airflow_shims import (
     AIRFLOW_PATCHED,
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_packaging.py b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_packaging.py
index a9c8b7ec65fa3..a822527582c2c 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_packaging.py
+++ b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_packaging.py
@@ -1,6 +1,6 @@
 import setuptools
-from datahub.testing.check_imports import ensure_no_indirect_model_imports
 
+from datahub.testing.check_imports import ensure_no_indirect_model_imports
 from tests.utils import PytestConfig
 
 
diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle
index 0d57bb5bfdff7..503b3556a41bf 100644
--- a/metadata-ingestion-modules/dagster-plugin/build.gradle
+++ b/metadata-ingestion-modules/dagster-plugin/build.gradle
@@ -55,16 +55,14 @@ task lint(type: Exec, dependsOn: installDev) {
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "black --check --diff src/ tests/ examples/ && " +
-    "isort --check --diff src/ tests/ examples/ && " +
-    "flake8 --count --statistics src/ tests/ examples/ && " +
+    "ruff check src/ tests/ && " +
     "mypy --show-traceback --show-error-codes src/ tests/ examples/"
 }
 task lintFix(type: Exec, dependsOn: installDev) {
   commandLine 'bash', '-x', '-c',
     "source ${venv_name}/bin/activate && " +
     "black src/ tests/ examples/ && " +
-    "isort src/ tests/ examples/ && " +
-    "flake8 src/ tests/ examples/ && " +
+    "ruff check --fix src/ tests/"
     "mypy src/ tests/ examples/"
 }
 
diff --git a/metadata-ingestion-modules/dagster-plugin/pyproject.toml b/metadata-ingestion-modules/dagster-plugin/pyproject.toml
index fba81486b9f67..7d03c2a14bf07 100644
--- a/metadata-ingestion-modules/dagster-plugin/pyproject.toml
+++ b/metadata-ingestion-modules/dagster-plugin/pyproject.toml
@@ -10,10 +10,50 @@ extend-exclude = '''
 '''
 include = '\.pyi?$'
 
-[tool.isort]
-indent = '    '
-profile = 'black'
-sections = 'FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER'
+[tool.ruff.lint.isort]
+combine-as-imports = true
+known-first-party = ["datahub"]
+extra-standard-library = ["__future__", "datahub.utilities._markupsafe_compat", "datahub.sql_parsing._sqlglot_patch"]
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
+force-sort-within-sections = false
+force-wrap-aliases = false
+split-on-trailing-comma = false
+order-by-type = true
+relative-imports-order = "closest-to-furthest"
+force-single-line = false
+single-line-exclusions = ["typing"]
+length-sort = false
+from-first = false
+required-imports = []
+classes = ["typing"]
 
-[tool.pyright]
-extraPaths = ['tests']
\ No newline at end of file
+[tool.ruff.lint]
+select = [
+    "B",
+    "C90",
+    "E",
+    "F",
+    "I", # For isort
+    "TID",
+]
+ignore = [
+    # Ignore line length violations (handled by Black)
+    "E501",
+    # Ignore whitespace before ':' (matches Black)
+    "E203",
+    "E203",
+    # Allow usages of functools.lru_cache
+    "B019",
+    # Allow function call in argument defaults
+    "B008",
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.lint.flake8-tidy-imports]
+# Disallow all relative imports.
+ban-relative-imports = "all"
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]
\ No newline at end of file
diff --git a/metadata-ingestion-modules/dagster-plugin/setup.cfg b/metadata-ingestion-modules/dagster-plugin/setup.cfg
index 20a903914332a..89b28ae45f964 100644
--- a/metadata-ingestion-modules/dagster-plugin/setup.cfg
+++ b/metadata-ingestion-modules/dagster-plugin/setup.cfg
@@ -1,24 +1,3 @@
-[flake8]
-max-complexity = 15
-ignore =
-    # Ignore: line length issues, since black's formatter will take care of them.
-    E501,
-    # Ignore: 1 blank line required before class docstring.
-    D203,
-    # See https://stackoverflow.com/a/57074416.
-    W503,
-    # See https://github.com/psf/black/issues/315.
-    E203
-exclude =
-    .git,
-    venv,
-    .tox,
-    __pycache__
-per-file-ignores =
-    # imported but unused
-    __init__.py: F401
-ban-relative-imports = true
-
 [mypy]
 plugins =
     pydantic.mypy
diff --git a/metadata-ingestion-modules/dagster-plugin/setup.py b/metadata-ingestion-modules/dagster-plugin/setup.py
index 22c15497bd807..f2e90c14833f7 100644
--- a/metadata-ingestion-modules/dagster-plugin/setup.py
+++ b/metadata-ingestion-modules/dagster-plugin/setup.py
@@ -53,10 +53,7 @@ def get_long_description():
     "dagster-snowflake-pandas >= 0.11.0",
     "black==22.12.0",
     "coverage>=5.1",
-    "flake8>=6.0.0",
-    "flake8-tidy-imports>=4.3.0",
-    "flake8-bugbear==23.3.12",
-    "isort>=5.7.0",
+    "ruff==0.9.1",
     "mypy>=1.4.0",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
     # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.
diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py
index a87f490f2d947..9a0a9a1b3a75e 100644
--- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py
+++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py
@@ -13,6 +13,7 @@
     TableSchemaMetadataValue,
 )
 from dagster._core.execution.stats import RunStepKeyStatsSnapshot, StepEventStatus
+
 from datahub.sql_parsing.sqlglot_utils import get_query_fingerprint
 
 try:
@@ -23,6 +24,7 @@
 
 from dagster._core.snap.node import OpDefSnap
 from dagster._core.storage.dagster_run import DagsterRun, DagsterRunStatsSnapshot
+
 from datahub.api.entities.datajob import DataFlow, DataJob
 from datahub.api.entities.dataprocess.dataprocess_instance import (
     DataProcessInstance,
diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py
index bccdb4ac7922a..55fb6ad99ae29 100644
--- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py
+++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py
@@ -35,7 +35,9 @@
 try:
     from dagster._core.definitions.sensor_definition import SensorReturnTypesUnion
 except ImportError:
-    from dagster._core.definitions.sensor_definition import RawSensorEvaluationFunctionReturn as SensorReturnTypesUnion  # type: ignore
+    from dagster._core.definitions.sensor_definition import (
+        RawSensorEvaluationFunctionReturn as SensorReturnTypesUnion,  # type: ignore
+    )
 
 from dagster._core.definitions.target import ExecutableDefinition
 from dagster._core.definitions.unresolved_asset_job_definition import (
@@ -43,6 +45,7 @@
 )
 from dagster._core.events import DagsterEventType, HandledOutputData, LoadedInputData
 from dagster._core.execution.stats import RunStepKeyStatsSnapshot
+
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
 from datahub.metadata.schema_classes import SubTypesClass
@@ -52,7 +55,6 @@
 )
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 from datahub.utilities.urns.error import InvalidUrnError
-
 from datahub_dagster_plugin.client.dagster_generator import (
     DATAHUB_ASSET_GROUP_NAME_CACHE,
     Constant,
diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py b/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py
index c951b959f85d4..9a69822984bb8 100644
--- a/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py
+++ b/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py
@@ -22,11 +22,11 @@
     RepositoryDefinition,
 )
 from dagster._core.definitions.resource_definition import ResourceDefinition
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.graph.client import DatahubClientConfig
 from freezegun import freeze_time
 from utils.utils import PytestConfig, check_golden_file
 
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.graph.client import DatahubClientConfig
 from datahub_dagster_plugin.client.dagster_generator import DatahubDagsterSourceConfig
 from datahub_dagster_plugin.sensors.datahub_sensors import (
     DatahubSensors,
diff --git a/metadata-ingestion-modules/gx-plugin/pyproject.toml b/metadata-ingestion-modules/gx-plugin/pyproject.toml
index fba81486b9f67..bc95145217526 100644
--- a/metadata-ingestion-modules/gx-plugin/pyproject.toml
+++ b/metadata-ingestion-modules/gx-plugin/pyproject.toml
@@ -14,6 +14,3 @@ include = '\.pyi?$'
 indent = '    '
 profile = 'black'
 sections = 'FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER'
-
-[tool.pyright]
-extraPaths = ['tests']
\ No newline at end of file
diff --git a/metadata-ingestion-modules/prefect-plugin/pyproject.toml b/metadata-ingestion-modules/prefect-plugin/pyproject.toml
index fba81486b9f67..bc95145217526 100644
--- a/metadata-ingestion-modules/prefect-plugin/pyproject.toml
+++ b/metadata-ingestion-modules/prefect-plugin/pyproject.toml
@@ -14,6 +14,3 @@ include = '\.pyi?$'
 indent = '    '
 profile = 'black'
 sections = 'FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER'
-
-[tool.pyright]
-extraPaths = ['tests']
\ No newline at end of file
diff --git a/metadata-ingestion/pyproject.toml b/metadata-ingestion/pyproject.toml
index f3a51e135082e..745547f88bcb9 100644
--- a/metadata-ingestion/pyproject.toml
+++ b/metadata-ingestion/pyproject.toml
@@ -11,6 +11,7 @@ extend-exclude = '''
 include = '\.pyi?$'
 target-version = ['py38', 'py39', 'py310', 'py311']
 
+
 [tool.ruff.lint.isort]
 combine-as-imports = true
 known-first-party = ["datahub"]
@@ -28,16 +29,6 @@ from-first = false
 required-imports = []
 classes = ["typing"]
 
-[tool.pyright]
-extraPaths = ['tests']
-
-[tool.vulture]
-exclude = ["src/datahub/metadata/"]
-ignore_decorators = ["@click.*", "@validator", "@root_validator", "@pydantic.validator", "@pydantic.root_validator", "@pytest.fixture"]
-ignore_names = ["*Source", "*Sink", "*Report"]
-paths = ["src"]
-sort_by_size = true
-
 [tool.ruff]
 # Same as Black.
 line-length = 88
@@ -70,7 +61,6 @@ ignore = [
     "B008",
     # TODO: Enable these later
     "B006",  # Mutable args
-    "B007",  # Unused loop control variable
     "B017",  # Do not assert blind exception
     "B904",  # Checks for raise statements in exception handlers that lack a from clause
 ]
diff --git a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
index 2097922c15136..39de4d7f80558 100644
--- a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
+++ b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
@@ -438,7 +438,7 @@ def _patch_ownership(
             for replace_index, replace_value in patches_replace.items():
                 list_to_manipulate[replace_index] = replace_value
 
-            for drop_index, drop_value in patches_drop.items():
+            for drop_value in patches_drop.values():
                 list_to_manipulate.remove(drop_value)
 
             for add_value in patches_add:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
index ad2bc36cf558b..e4f9cd0ee7e01 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
@@ -613,7 +613,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                                 table_data.table_path
                             ].timestamp = table_data.timestamp
 
-                for guid, table_data in table_dict.items():
+                for _, table_data in table_dict.items():
                     yield from self.ingest_table(table_data, path_spec)
 
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py
index d913b7e42065d..072995c10ebce 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py
@@ -181,7 +181,7 @@ def authenticate(self, connection_args: "DremioSourceConfig") -> None:
             return
 
         # On-prem Dremio authentication (PAT or Basic Auth)
-        for retry in range(1, self._retry_count + 1):
+        for _ in range(1, self._retry_count + 1):
             try:
                 if connection_args.authentication_method == "PAT":
                     self.session.headers.update(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py b/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py
index 2c9107b967e4f..8cdd4b17733e0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py
@@ -286,7 +286,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         df = self.get_neo4j_metadata(
             "CALL apoc.meta.schema() YIELD value UNWIND keys(value) AS key RETURN key, value[key] AS value;"
         )
-        for index, row in df.iterrows():
+        for _, row in df.iterrows():
             try:
                 yield MetadataWorkUnit(
                     id=row["key"],
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
index 989d0d734352a..3173423f86a2e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
@@ -1124,7 +1124,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                                 table_data.table_path
                             ].timestamp = table_data.timestamp
 
-                for guid, table_data in table_dict.items():
+                for _, table_data in table_dict.items():
                     yield from self.ingest_table(table_data, path_spec)
 
             if not self.source_config.is_profiling_enabled():
diff --git a/metadata-ingestion/src/datahub/ingestion/source/schema/json_schema.py b/metadata-ingestion/src/datahub/ingestion/source/schema/json_schema.py
index 635e894d18c7e..a50e99393fdc2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/schema/json_schema.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/schema/json_schema.py
@@ -354,7 +354,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             browse_prefix = f"/{self.config.env.lower()}/{self.config.platform}/{self.config.platform_instance}"
 
         if os.path.isdir(self.config.path):
-            for root, dirs, files in os.walk(self.config.path, topdown=False):
+            for root, _, files in os.walk(self.config.path, topdown=False):
                 for file_name in [f for f in files if f.endswith(".json")]:
                     try:
                         yield from self._load_one_file(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py b/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py
index aeb21e88d0443..2899bcc2de37b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py
@@ -268,7 +268,7 @@ def _get_table_or_view_names(self, relkind, connection, schema=None, **kw):
     info_cache = kw.get("info_cache")
     all_relations = self._get_all_relation_info(connection, info_cache=info_cache)
     relation_names = []
-    for key, relation in all_relations.items():
+    for _, relation in all_relations.items():
         if relation.database == schema and relation.relkind == relkind:
             relation_names.append(relation.relname)
     return relation_names
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py
index ee841a2a20186..8187fff559208 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py
@@ -3605,7 +3605,7 @@ def emit_project_in_topological_order(
                 parent_container_key=parent_project_key,
             )
 
-        for id_, project in self.tableau_project_registry.items():
+        for project in self.tableau_project_registry.values():
             logger.debug(
                 f"project {project.name} and it's parent {project.parent_name} and parent id {project.parent_id}"
             )
diff --git a/metadata-ingestion/src/datahub/testing/mcp_diff.py b/metadata-ingestion/src/datahub/testing/mcp_diff.py
index 5e669a718e9ad..b58afc10148ed 100644
--- a/metadata-ingestion/src/datahub/testing/mcp_diff.py
+++ b/metadata-ingestion/src/datahub/testing/mcp_diff.py
@@ -246,7 +246,7 @@ def pretty(self, verbose: bool = False) -> str:
         for urn in self.aspect_changes.keys() - self.urns_added - self.urns_removed:
             aspect_map = self.aspect_changes[urn]
             s.append(f"Urn changed, {urn}:")
-            for aspect_name, aspect_diffs in aspect_map.items():
+            for aspect_diffs in aspect_map.values():
                 for i, ga in aspect_diffs.aspects_added.items():
                     s.append(self.report_aspect(ga, i, "added"))
                     if verbose:
diff --git a/metadata-ingestion/src/datahub/utilities/sqllineage_patch.py b/metadata-ingestion/src/datahub/utilities/sqllineage_patch.py
index afcd02478ae68..4c237d02727f7 100644
--- a/metadata-ingestion/src/datahub/utilities/sqllineage_patch.py
+++ b/metadata-ingestion/src/datahub/utilities/sqllineage_patch.py
@@ -8,7 +8,7 @@
 
 # Patch based on sqllineage v1.3.3
 def end_of_query_cleanup_patch(self, holder: SubQueryLineageHolder) -> None:  # type: ignore
-    for i, tbl in enumerate(self.tables):
+    for tbl in self.tables:
         holder.add_read(tbl)
     self.union_barriers.append((len(self.columns), len(self.tables)))
 
diff --git a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
index 832d00d9c5470..6f7a9c7833ba1 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
@@ -1070,7 +1070,7 @@ def test_unsupported_data_platform():
     )  # type :ignore
 
     is_entry_present: bool = False
-    for key, entry in info_entries.items():
+    for entry in info_entries.values():
         if entry.title == "Non-Data Platform Expression":
             is_entry_present = True
             break
@@ -1163,7 +1163,7 @@ def test_m_query_timeout(mock_get_lark_parser):
     )  # type :ignore
 
     is_entry_present: bool = False
-    for key, entry in warn_entries.items():
+    for entry in warn_entries.values():
         if entry.title == "M-Query Parsing Timeout":
             is_entry_present = True
             break
diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
index 739be7cc8408d..911d8a9f35139 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
@@ -1438,7 +1438,7 @@ def test_powerbi_cross_workspace_reference_info_message(
 
     is_entry_present: bool = False
     # Printing INFO entries
-    for key, entry in info_entries.items():
+    for entry in info_entries.values():
         if entry.title == "Missing Lineage For Tile":
             is_entry_present = True
             break
@@ -1563,7 +1563,7 @@ def test_powerbi_app_ingest_info_message(
 
     is_entry_present: bool = False
     # Printing INFO entries
-    for key, entry in info_entries.items():
+    for entry in info_entries.values():
         if entry.title == "App Ingestion Is Disabled":
             is_entry_present = True
             break
diff --git a/metadata-ingestion/tests/performance/data_generation.py b/metadata-ingestion/tests/performance/data_generation.py
index fcff13edf5936..266c0d9af0322 100644
--- a/metadata-ingestion/tests/performance/data_generation.py
+++ b/metadata-ingestion/tests/performance/data_generation.py
@@ -198,7 +198,7 @@ def generate_queries(
 
     all_tables = seed_metadata.tables + seed_metadata.views
     users = [f"user_{i}@xyz.com" for i in range(num_users)]
-    for i in range(num_selects):  # Pure SELECT statements
+    for _ in range(num_selects):  # Pure SELECT statements
         tables = _sample_list(all_tables, tables_per_select)
         all_columns = [
             FieldAccess(column, table) for table in tables for column in table.columns
@@ -213,7 +213,7 @@ def generate_queries(
             fields_accessed=_sample_list(all_columns, columns_per_select),
         )
 
-    for i in range(num_operations):
+    for _ in range(num_operations):
         modified_table = random.choice(seed_metadata.tables)
         n_col = len(modified_table.columns)
         num_columns_modified = NormalDistribution(n_col / 2, n_col / 2)
diff --git a/smoke-test/pyproject.toml b/smoke-test/pyproject.toml
index c7745d0e9a364..aeb3c03b6466d 100644
--- a/smoke-test/pyproject.toml
+++ b/smoke-test/pyproject.toml
@@ -42,5 +42,3 @@ warn_unused_configs = true
 disallow_incomplete_defs = false
 disallow_untyped_defs = false
 
-[tool.pyright]
-extraPaths = ['tests']

From 2df7673e39ec6e241ff1d010f87d82309f04167b Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Fri, 17 Jan 2025 20:26:15 +0530
Subject: [PATCH 2/2] fix lint errors

---
 .../src/datahub_airflow_plugin/datahub_plugin.py              | 4 ++--
 .../src/datahub_dagster_plugin/sensors/datahub_sensors.py     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py
index 451d2dbfd7c0b..7638720db023a 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py
@@ -56,8 +56,8 @@ class DatahubPlugin(AirflowPlugin):
     if _USE_AIRFLOW_LISTENER_INTERFACE:
         try:
             if not NEEDS_AIRFLOW_LISTENER_MODULE:
-                from datahub_airflow_plugin.datahub_listener import (
-                    get_airflow_plugin_listener,  # type: ignore[misc]
+                from datahub_airflow_plugin.datahub_listener import (  # type: ignore[misc]
+                    get_airflow_plugin_listener,
                 )
 
                 listeners: list = list(filter(None, [get_airflow_plugin_listener()]))
diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py
index 55fb6ad99ae29..b91a9cfa56d39 100644
--- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py
+++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py
@@ -35,8 +35,8 @@
 try:
     from dagster._core.definitions.sensor_definition import SensorReturnTypesUnion
 except ImportError:
-    from dagster._core.definitions.sensor_definition import (
-        RawSensorEvaluationFunctionReturn as SensorReturnTypesUnion,  # type: ignore
+    from dagster._core.definitions.sensor_definition import (  # type: ignore
+        RawSensorEvaluationFunctionReturn as SensorReturnTypesUnion,
     )
 
 from dagster._core.definitions.target import ExecutableDefinition