diff --git a/v2.5.x/site/en/adminGuide/resource_group.md b/v2.5.x/site/en/adminGuide/resource_group.md index e2bc8ff73..c8579f99c 100644 --- a/v2.5.x/site/en/adminGuide/resource_group.md +++ b/v2.5.x/site/en/adminGuide/resource_group.md @@ -65,10 +65,10 @@ All code samples on this page are in PyMilvus 2.5.3. Upgrade your PyMilvus insta # create a resource group that exactly hold no query node. try: - utility.create_resource_group(name, config=utility.ResourceGroupConfig( + milvus_client.create_resource_group(name, config=ResourceGroupConfig( requests={"node_num": node_num}, limits={"node_num": node_num}, - ), using='default') + )) print(f"Succeeded in creating resource group {name}.") except Exception: print("Failed to create the resource group.") @@ -81,7 +81,7 @@ All code samples on this page are in PyMilvus 2.5.3. Upgrade your PyMilvus insta To view the list of resource groups in a Milvus instance, do as follows: ```python - rgs = utility.list_resource_groups(using='default') + rgs = milvus_client.list_resource_groups() print(f"Resource group list: {rgs}") # Resource group list: ['__default_resource_group', 'rg'] @@ -92,16 +92,19 @@ All code samples on this page are in PyMilvus 2.5.3. Upgrade your PyMilvus insta You can have Milvus describe a resource group in concern as follows: ```python - info = utility.describe_resource_group(name, using="default") + info = milvus_client.describe_resource_group(name) print(f"Resource group description: {info}") # Resource group description: - # , // string, rg name - # , // int, num_node which has been transfer to this rg - # , // int, available node_num, some node may shutdown - # , // map[string]int, from collection_name to loaded replica of each collecion in this rg - # , // map[string]int, from collection_name to outgoging accessed node num by replica loaded in this rg - # . // map[string]int, from collection_name to incoming accessed node num by replica loaded in other rg + # ResourceGroupInfo: + # , // resource group name + # , // resource group capacity + # , // resource group node num + # , // collection loaded replica num in resource group + # , // node num which still in use by replica in other resource group + # , // node num which is in use by replica but belong to other resource group + # , // resource group config + # // node detail info ``` 4. Transfer nodes between resource groups. @@ -117,7 +120,7 @@ All code samples on this page are in PyMilvus 2.5.3. Upgrade your PyMilvus insta expected_num_nodes_in_rg = 1 try: - utility.update_resource_groups({ + milvus_client.update_resource_groups({ source: ResourceGroupConfig( requests={"node_num": expected_num_nodes_in_default}, limits={"node_num": expected_num_nodes_in_default}, @@ -126,7 +129,7 @@ All code samples on this page are in PyMilvus 2.5.3. Upgrade your PyMilvus insta requests={"node_num": expected_num_nodes_in_rg}, limits={"node_num": expected_num_nodes_in_rg}, ) - }, using="default") + }) print(f"Succeeded in move 1 node(s) from {source} to {target}.") except Exception: print("Something went wrong while moving nodes.") @@ -141,28 +144,25 @@ All code samples on this page are in PyMilvus 2.5.3. Upgrade your PyMilvus insta ```python from pymilvus import Collection - collection = Collection('demo') + collection_name = "demo" # Milvus loads the collection to the default resource group. - collection.load(replica_number=2) + milvus_client.load_collection(collection_name, replica_number=2) # Or, you can ask Milvus load the collection to the desired resource group. # make sure that query nodes num should be greater or equal to replica_number resource_groups = ['rg'] - collection.load(replica_number=2, _resource_groups=resource_groups) + milvus_client.load_collection(replica_number=2, _resource_groups=resource_groups) ``` Also, you can just load a partition into a resource group and have its replicas distributed among several resource groups. The following assumes that a collection named `Books` already exists and it has a partition named `Novels`. ```python - collection = Collection("Books") + collection = "Books" + partition = "Novels" # Use the load method of a collection to load one of its partition - collection.load(["Novels"], replica_number=2, _resource_groups=resource_groups) - - # Or, you can use the load method of a partition directly - partition = Partition(collection, "Novels") - partition.load(replica_number=2, _resource_groups=resource_groups) + milvus_client.load_partitions(collection, [partition], replica_number=2, _resource_groups=resource_groups) ``` Note that `_resource_groups` is an optional parameter, and leaving it unspecified have Milvus load the replicas onto the query nodes in the default resource group. @@ -180,8 +180,8 @@ All code samples on this page are in PyMilvus 2.5.3. Upgrade your PyMilvus insta num_replicas = 1 try: - utility.transfer_replica(source, target, collection_name, num_replicas, using="default") - print(f"Succeeded in moving {num_node} replica(s) of {collection_name} from {source} to {target}.") + milvus_client.transfer_replica(source, target, collection_name, num_replicas) + print(f"Succeeded in moving {num_replicas} replica(s) of {collection_name} from {source} to {target}.") except Exception: print("Something went wrong while moving replicas.") @@ -193,17 +193,18 @@ All code samples on this page are in PyMilvus 2.5.3. Upgrade your PyMilvus insta You can drop a resource group that hold no query node (`limits.node_num = 0`) at any time. In this guide, resource group `rg` now has one query node. You need to change the configuration `limits.node_num` of resource group into zero first. ```python + resource_group = "rg try: - utility.update_resource_groups({ - "rg": utility.ResourceGroupConfig( + milvus_client.update_resource_groups({ + resource_group: ResourceGroupConfig( requests={"node_num": 0}, limits={"node_num": 0}, ), - }, using="default") - utility.drop_resource_group("rg", using="default") - print(f"Succeeded in dropping {source}.") + }) + milvus_client.drop_resource_group(resource_group) + print(f"Succeeded in dropping {resource_group}.") except Exception: - print(f"Something went wrong while dropping {source}.") + print(f"Something went wrong while dropping {resource_group}.") ``` For more details, please refer to the [relevant examples in pymilvus](https://github.com/milvus-io/pymilvus/blob/v2.4.3/examples/resource_group_declarative_api.py) @@ -219,7 +220,6 @@ Here is a good practice for managing QueryNodes in a cloud environment: Here is an example setup: ```python - from pymilvus import utility from pymilvus.client.types import ResourceGroupConfig _PENDING_NODES_RESOURCE_GROUP="__pending_nodes" @@ -227,26 +227,26 @@ Here is a good practice for managing QueryNodes in a cloud environment: def init_cluster(node_num: int): print(f"Init cluster with {node_num} nodes, all nodes will be put in default resource group") # create a pending resource group, which can used to hold the pending nodes that do not hold any data. - utility.create_resource_group(name=_PENDING_NODES_RESOURCE_GROUP, config=ResourceGroupConfig( + milvus_client.create_resource_group(name=_PENDING_NODES_RESOURCE_GROUP, config=ResourceGroupConfig( requests={"node_num": 0}, # this resource group can hold 0 nodes, no data will be load on it. limits={"node_num": 10000}, # this resource group can hold at most 10000 nodes )) # update default resource group, which can used to hold the nodes that all initial node in it. - utility.update_resource_groups({ + milvus_client.update_resource_groups({ "__default_resource_group": ResourceGroupConfig( requests={"node_num": node_num}, limits={"node_num": node_num}, transfer_from=[{"resource_group": _PENDING_NODES_RESOURCE_GROUP}], # recover missing node from pending resource group at high priority. transfer_to=[{"resource_group": _PENDING_NODES_RESOURCE_GROUP}], # recover redundant node to pending resource group at low priority. )}) - utility.create_resource_group(name="rg1", config=ResourceGroupConfig( + milvus_client.create_resource_group(name="rg1", config=ResourceGroupConfig( requests={"node_num": 0}, limits={"node_num": 0}, transfer_from=[{"resource_group": _PENDING_NODES_RESOURCE_GROUP}], transfer_to=[{"resource_group": _PENDING_NODES_RESOURCE_GROUP}], )) - utility.create_resource_group(name="rg2", config=ResourceGroupConfig( + milvus_client.create_resource_group(name="rg2", config=ResourceGroupConfig( requests={"node_num": 0}, limits={"node_num": 0}, transfer_from=[{"resource_group": _PENDING_NODES_RESOURCE_GROUP}], @@ -271,7 +271,7 @@ Here is a good practice for managing QueryNodes in a cloud environment: We can use the API to scale a specific resource group to a designated number of QueryNodes without affecting any other resource groups. ```python # scale rg1 into 3 nodes, rg2 into 1 nodes - utility.update_resource_groups({ + milvus_client.update_resource_groups({ "rg1": ResourceGroupConfig( requests={"node_num": 3}, limits={"node_num": 3}, @@ -295,7 +295,7 @@ Here is a good practice for managing QueryNodes in a cloud environment: ```python # scale rg1 from 3 nodes into 2 nodes - utility.update_resource_groups({ + milvus_client.update_resource_groups({ "rg1": ResourceGroupConfig( requests={"node_num": 2}, limits={"node_num": 2}, diff --git a/v2.5.x/site/en/integrations/kafka-connect-milvus.md b/v2.5.x/site/en/integrations/kafka-connect-milvus.md index b2934cbb3..73809ae6c 100644 --- a/v2.5.x/site/en/integrations/kafka-connect-milvus.md +++ b/v2.5.x/site/en/integrations/kafka-connect-milvus.md @@ -1,13 +1,24 @@ --- id: kafka-connect-milvus.md -summary: In this quick start guide we show how to setup open source kafka and Zilliz Cloud to ingest vector data. -title: Integrate Milvus with WhyHow +summary: Apache Kafka is integrated with Milvus and Zilliz Cloud to stream vector data. Learn how to use Kafka-Milvus connector to build real-time pipelines for semantic search, recommendation systems, and AI-driven analytics. +title: Connect Apache Kafka® with Milvus/Zilliz Cloud for Real-Time Vector Data Ingestion --- -# Connect Kafka with Milvus +# Connect Apache Kafka® with Milvus/Zilliz Cloud for Real-Time Vector Data Ingestion In this quick start guide we show how to setup open source kafka and Zilliz Cloud to ingest vector data. +This tutorial explains how to use Apache Kafka® to stream and ingest vector data into Milvus vector database and Zilliz Cloud (fully-managed Milvus), enabling advanced real-time applications such as semantic search, recommendation systems, and AI-powered analytics. + +Apache Kafka is a distributed event streaming platform designed for high-throughput, low-latency pipelines. It is widely used to collect, store, and process real-time data streams from sources like databases, IoT devices, mobile apps, and cloud services. Kafka’s ability to handle large volumes of data makes it an important data source of vector databases like Milvus or Zilliz Cloud. + +For example, Kafka can capture real-time data streams—such as user interactions, sensor readings, together with their embeddings from machine learning models—and publish these streams directly to Milvus or Zilliz Cloud. Once in the vector database, this data can be indexed, searched, and analyzed efficiently. + +The Kafka integration with Milvus and Zilliz Cloud provides a seamless way to build powerful pipelines for unstructured data workflows. The connector works for both open-source Kafka deployment and hosted services such as [Confluent](https://www.confluent.io/hub/zilliz/kafka-connect-milvus) and [StreamNative](https://docs.streamnative.io/hub/connector-kafka-connect-milvus-sink-v0.1). + +In this tutorial we use Zilliz Cloud as a demostration: + + ## Step 1: Download the kafka-connect-milvus plugin Complete the following steps to download the kafka-connect-milvus plugin. @@ -116,4 +127,4 @@ Ensure you have Kafka and Zilliz Cloud setup and properly configured. ### Support -If you require any assistance or have questions regarding the Kafka Connect Milvus Connector, please feel free to reach out to our support team: **Email:** [support@zilliz.com](mailto:support@zilliz.com) \ No newline at end of file +If you require any assistance or have questions regarding the Kafka Connect Milvus Connector, please feel free to reach out to the maintainer of the connector: **Email:** [support@zilliz.com](mailto:support@zilliz.com)