move from isort,flake8 to ruff

.github/scripts/pre-commit-override.yaml

@@ -5,5 +5,5 @@ repos:
         name: smoke-test cypress Lint Fix
         entry: ./gradlew :smoke-test:cypressLintFix
         language: system
-        files: ^smoke-test/tests/cypress/.*$
+        files: ^smoke-test/tests/cypress/.*\.tsx$
         pass_filenames: false

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-01-09 10:08:09 UTC
+# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-01-17 16:43:31 UTC
 # Do not edit this file directly. Run the script to regenerate.
 # Add additional hooks in .github/scripts/pre-commit-override.yaml
 repos:
@@ -442,4 +442,5 @@ repos:
         name: smoke-test cypress Lint Fix
         entry: ./gradlew :smoke-test:cypressLintFix
         language: system
-        files: ^smoke-test/tests/cypress/.*$
+        files: ^smoke-test/tests/cypress/.*\.tsx$
+        pass_filenames: false

smoke-test/build.gradle

@@ -74,16 +74,15 @@ task pythonLint(type: Exec, dependsOn: installDev) {
     commandLine 'bash', '-c',
         "source ${venv_name}/bin/activate && set -x && " +
         "black --check --diff tests/ && " +
-        "isort --check --diff tests/ && " +
-        "ruff --statistics tests/ && " +
+        "ruff check tests/ && " +
         "mypy tests/"
 }
+
 task pythonLintFix(type: Exec, dependsOn: installDev) {
     commandLine 'bash', '-c',
         "source ${venv_name}/bin/activate && set -x && " +
         "black tests/ && " +
-        "isort tests/ && " +
-        "ruff --fix tests/ && " +
+        "ruff check --fix tests/ && " +
         "mypy tests/"
 }
 

smoke-test/pyproject.toml

@@ -22,15 +22,49 @@ venv
 include = '\.pyi?$'
 target-version = ['py310']
 
-[tool.isort]
-profile = 'black'
+[tool.ruff.lint.isort]
+combine-as-imports = true
+known-first-party = ["datahub"]
+extra-standard-library = ["__future__", "datahub.utilities._markupsafe_compat", "datahub.sql_parsing._sqlglot_patch"]
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
+force-sort-within-sections = false
+force-wrap-aliases = false
+split-on-trailing-comma = false
+order-by-type = true
+relative-imports-order = "closest-to-furthest"
+force-single-line = false
+single-line-exclusions = ["typing"]
+length-sort = false
+from-first = false
+required-imports = []
+classes = ["typing"]
 
-[tool.ruff]
+[tool.ruff.lint]
+select = [
+    "B",
+    "C90",
+    "E",
+    "F",
+    "I", # For isort
+    "TID",
+]
 ignore = [
 	'E501', # Ignore line length, since black handles that.
 	'D203', # Ignore 1 blank line required before class docstring.
+	'B904', # exception with `raise ... from err` or `raise ... from None` to distinguish
+	'TID252', # Prefer absolute imports over relative imports
 ]
 
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.lint.flake8-tidy-imports]
+# Disallow all relative imports.
+ban-relative-imports = "all"
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]
+
 [tool.mypy]
 exclude = "^(venv/|build/|dist/)"
 ignore_missing_imports = true

smoke-test/requirements.txt

@@ -10,9 +10,8 @@ pytest-xdist
 networkx
 # libaries for linting below this
 black==23.7.0
-isort==5.12.0
 mypy==1.5.1
-ruff==0.0.287
+ruff==0.9.1
 # stub version are copied from metadata-ingestion/setup.py and that should be the source of truth
 types-requests>=2.28.11.6,<=2.31.0.3
 types-PyYAML

smoke-test/tests/aspect_generators/timeseries/dataset_profile_gen.py

@@ -5,7 +5,6 @@
     DatasetProfileClass,
     TimeWindowSizeClass,
 )
-
 from tests.utils import get_timestampmillis_at_start_of_day
 
 

smoke-test/tests/assertions/assertions_test.py

@@ -3,6 +3,7 @@
 
 import pytest
 import tenacity
+
 from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
@@ -22,7 +23,6 @@
     PartitionSpecClass,
     PartitionTypeClass,
 )
-
 from tests.utils import delete_urns_from_file, get_sleep_info, ingest_file_via_rest
 
 restli_default_headers = {

smoke-test/tests/assertions/custom_assertions_test.py

@@ -2,11 +2,11 @@
 from typing import Any
 
 import pytest
+
 from datahub.emitter.mce_builder import make_dataset_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import StatusClass
-
 from tests.consistency_utils import wait_for_writes_to_sync
 from tests.utils import delete_urn
 

smoke-test/tests/cli/datahub_cli.py

@@ -1,11 +1,11 @@
 import json
 
 import pytest
+
 from datahub.metadata.schema_classes import (
     BrowsePathsV2Class,
     EditableDatasetPropertiesClass,
 )
-
 from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 ingested_dataset_run_id = ""

smoke-test/tests/cli/datahub_graph_test.py

@@ -2,9 +2,9 @@
 
 import pytest
 import tenacity
+
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import KafkaSchemaClass, SchemaMetadataClass
-
 from tests.utils import delete_urns_from_file, get_sleep_info, ingest_file_via_rest
 
 sleep_sec, sleep_times = get_sleep_info()

smoke-test/tests/cli/delete_cmd/test_timeseries_delete.py

@@ -5,12 +5,12 @@
 from json import JSONDecodeError
 from typing import Any, Dict, List, Optional
 
-import datahub.emitter.mce_builder as builder
 from click.testing import CliRunner, Result
+
+import datahub.emitter.mce_builder as builder
 from datahub.emitter.serialization_helper import pre_json_transform
 from datahub.entrypoints import datahub
 from datahub.metadata.schema_classes import DatasetProfileClass
-
 from tests.aspect_generators.timeseries.dataset_profile_gen import gen_dataset_profiles
 from tests.utils import get_strftime_from_timestamp_millis, wait_for_writes_to_sync
 

smoke-test/tests/cli/ingest_cmd/test_timeseries_rollback.py

@@ -1,12 +1,12 @@
 import json
 from typing import Dict, List, Optional
 
-import datahub.emitter.mce_builder as builder
 from click.testing import CliRunner, Result
+
+import datahub.emitter.mce_builder as builder
 from datahub.emitter.serialization_helper import post_json_transform
 from datahub.entrypoints import datahub
 from datahub.metadata.schema_classes import DatasetProfileClass
-
 from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 runner = CliRunner(mix_stderr=False)

smoke-test/tests/cli/user_groups_cmd/test_group_cmd.py

@@ -5,10 +5,10 @@
 
 import yaml
 from click.testing import CliRunner, Result
+
 from datahub.api.entities.corpgroup.corpgroup import CorpGroup
 from datahub.entrypoints import datahub
 from datahub.ingestion.graph.client import DataHubGraph
-
 from tests.utils import wait_for_writes_to_sync
 
 runner = CliRunner(mix_stderr=False)

smoke-test/tests/cli/user_groups_cmd/test_user_cmd.py

@@ -5,6 +5,7 @@
 
 import yaml
 from click.testing import CliRunner, Result
+
 from datahub.api.entities.corpuser.corpuser import CorpUser
 from datahub.entrypoints import datahub
 

smoke-test/tests/cypress/integration_test.py

@@ -172,7 +172,7 @@ def ingest_cleanup_data(auth_session, graph_client):
 
 def _get_js_files(base_path: str):
     file_paths = []
-    for root, dirs, files in os.walk(base_path):
+    for root, _, files in os.walk(base_path):
         for file in files:
             if file.endswith(".js"):
                 file_paths.append(os.path.relpath(os.path.join(root, file), base_path))

smoke-test/tests/data_process_instance/test_data_process_instance.py

@@ -4,6 +4,7 @@
 from random import randint
 
 import pytest
+
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
 from datahub.ingestion.api.sink import NoopWriteCallback
@@ -23,7 +24,6 @@
     SubTypesClass,
     TimeWindowSizeClass,
 )
-
 from tests.utils import (
     delete_urns_from_file,
     ingest_file_via_rest,

smoke-test/tests/database/test_database.py

@@ -1,8 +1,8 @@
 import logging
 
 import pytest
-from datahub.emitter.mce_builder import make_dataset_urn
 
+from datahub.emitter.mce_builder import make_dataset_urn
 from tests.utilities.concurrent_openapi import run_tests
 from tests.utils import delete_urns, wait_for_writes_to_sync
 

smoke-test/tests/dataproduct/test_dataproduct.py

@@ -6,6 +6,7 @@
 
 import pytest
 import tenacity
+
 from datahub.emitter.mce_builder import datahub_guid, make_dataset_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
@@ -19,7 +20,6 @@
     DomainsClass,
 )
 from datahub.utilities.urns.urn import Urn
-
 from tests.utils import (
     delete_urns_from_file,
     get_sleep_info,

smoke-test/tests/delete/delete_test.py

@@ -2,8 +2,8 @@
 import os
 
 import pytest
-from datahub.cli.cli_utils import get_aspects_for_entity
 
+from datahub.cli.cli_utils import get_aspects_for_entity
 from tests.utils import (
     delete_urns_from_file,
     ingest_file_via_rest,

smoke-test/tests/lineage/test_lineage.py

@@ -3,9 +3,11 @@
 from enum import Enum
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
 
-import datahub.emitter.mce_builder as builder
 import networkx as nx
 import pytest
+from pydantic import BaseModel, validator
+
+import datahub.emitter.mce_builder as builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import (
@@ -18,17 +20,9 @@
     DatasetLineageTypeClass,
     DatasetPropertiesClass,
     EdgeClass,
-)
-from datahub.metadata.schema_classes import (
     FineGrainedLineageClass as FineGrainedLineage,
-)
-from datahub.metadata.schema_classes import (
     FineGrainedLineageDownstreamTypeClass as FineGrainedLineageDownstreamType,
-)
-from datahub.metadata.schema_classes import (
     FineGrainedLineageUpstreamTypeClass as FineGrainedLineageUpstreamType,
-)
-from datahub.metadata.schema_classes import (
     OtherSchemaClass,
     QueryLanguageClass,
     QueryPropertiesClass,
@@ -43,8 +37,6 @@
 )
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 from datahub.utilities.urns.urn import Urn
-from pydantic import BaseModel, validator
-
 from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 logger = logging.getLogger(__name__)

smoke-test/tests/ml_models/test_ml_models.py

@@ -4,6 +4,7 @@
 from random import randint
 
 import pytest
+
 from datahub.emitter.mce_builder import make_ml_model_group_urn, make_ml_model_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
@@ -14,7 +15,6 @@
     MLModelGroupPropertiesClass,
     MLModelPropertiesClass,
 )
-
 from tests.utils import (
     delete_urns_from_file,
     get_sleep_info,

smoke-test/tests/patch/test_datajob_patches.py

@@ -10,7 +10,6 @@
     EdgeClass,
 )
 from datahub.specific.datajob import DataJobPatchBuilder
-
 from tests.patch.common_patch_tests import (
     helper_test_custom_properties_patch,
     helper_test_dataset_tags_patch,

smoke-test/tests/patch/test_dataset_patches.py

@@ -15,7 +15,6 @@
     UpstreamLineageClass,
 )
 from datahub.specific.dataset import DatasetPatchBuilder
-
 from tests.patch.common_patch_tests import (
     helper_test_custom_properties_patch,
     helper_test_dataset_tags_patch,

smoke-test/tests/platform_resources/test_platform_resource.py

@@ -4,13 +4,13 @@
 from typing import List
 
 import pytest
+
 from datahub.api.entities.platformresource.platform_resource import (
     ElasticPlatformResourceQuery,
     PlatformResource,
     PlatformResourceKey,
     PlatformResourceSearchFields,
 )
-
 from tests.utils import wait_for_writes_to_sync
 
 logger = logging.getLogger(__name__)

smoke-test/tests/restli/restli_test.py

@@ -4,6 +4,7 @@
 from typing import List
 
 import pytest
+
 from datahub.emitter.aspect import JSON_CONTENT_TYPE
 from datahub.emitter.mce_builder import make_dashboard_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -16,7 +17,6 @@
     MetadataChangeProposalClass,
 )
 from datahub.utilities.urns.urn import guess_entity_type
-
 from tests.utils import delete_urns
 
 generated_urns: List[str] = []