feat: Lazy Infra Creation with online_store Option Set to True (#144)

* test changes

* fix: allow for scylladb in feature_store.yaml

* fix: linter issue

* fix: linter issue

* fix materialization/registration logic

* fix: use kwargs in expedia provider update infra

* try to ignore keyword lint error

* fix: lint formatting

* allow for non-lazy table creation

* fix: lint formatting

* dont update for non-scylladb

* fix: lint formatting

* fix: address Bhargav's comments

* fix: address Bhargav's comments 2

* fix: update lazy_table_creation docstring

sdk/python/feast/feature_store.py

@@ -991,14 +991,15 @@ def apply(
         )
         tables_to_keep: List[FeatureView] = views_to_update + sfvs_to_update  # type: ignore
 
-        self._get_provider().update_infra(
-            project=self.project,
-            tables_to_delete=tables_to_delete,
-            tables_to_keep=tables_to_keep,
-            entities_to_delete=entities_to_delete if not partial else [],
-            entities_to_keep=entities_to_update,
-            partial=partial,
-        )
+        if not getattr(self.config.online_store, "lazy_table_creation", False):
+            self._get_provider().update_infra(
+                project=self.project,
+                tables_to_delete=tables_to_delete,
+                tables_to_keep=tables_to_keep,
+                entities_to_delete=entities_to_delete if not partial else [],
+                entities_to_keep=entities_to_update,
+                partial=partial,
+            )
 
         self._registry.commit()
 
@@ -1255,6 +1256,7 @@ def materialize_incremental(
         feature_views_to_materialize = self._get_feature_views_to_materialize(
             feature_views
         )
+
         _print_materialization_log(
             None,
             end_date,
@@ -1350,6 +1352,7 @@ def materialize(
         feature_views_to_materialize = self._get_feature_views_to_materialize(
             feature_views
         )
+
         _print_materialization_log(
             start_date,
             end_date,

sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py

@@ -124,7 +124,7 @@ class CassandraOnlineStoreConfig(FeastConfigBaseModel):
     and password being the Client ID and Client Secret of the database token.
     """
 
-    type: Literal["cassandra"] = "cassandra"
+    type: Literal["cassandra", "scylladb"] = "cassandra"
     """Online store type selector."""
 
     # settings for connection to Cassandra / Astra DB
@@ -153,6 +153,12 @@ class CassandraOnlineStoreConfig(FeastConfigBaseModel):
     request_timeout: Optional[StrictFloat] = None
     """Request timeout in seconds."""
 
+    lazy_table_creation: Optional[bool] = False
+    """
+    If True, tables will be created on during materialization, rather than registration.
+    Table deletion is not currently supported in this mode.
+    """
+
     class CassandraLoadBalancingPolicy(FeastConfigBaseModel):
         """
         Configuration block related to the Cluster's load-balancing policy.

sdk/python/feast/repo_config.py

@@ -62,6 +62,7 @@
     "postgres": "feast.infra.online_stores.contrib.postgres.PostgreSQLOnlineStore",
     "hbase": "feast.infra.online_stores.contrib.hbase_online_store.hbase.HbaseOnlineStore",
     "cassandra": "feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStore",
+    "scylladb": "feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStore",
     "mysql": "feast.infra.online_stores.contrib.mysql_online_store.mysql.MySQLOnlineStore",
     "hazelcast": "feast.infra.online_stores.contrib.hazelcast_online_store.hazelcast_online_store.HazelcastOnlineStore",
     "milvus": "feast.expediagroup.vectordb.milvus_online_store.MilvusOnlineStore",

setup.py

@@ -119,6 +119,10 @@
 
 GE_REQUIRED = ["great_expectations>=0.15.41"]
 
+SCYLLADB_REQUIRED = [
+    "scylla-driver>=3.24.0,<4",
+]
+
 AZURE_REQUIRED = [
     "azure-storage-blob>=0.37.0",
     "azure-identity>=1.6.1",
@@ -477,6 +481,7 @@ def run(self):
         "hbase": HBASE_REQUIRED,
         "docs": DOCS_REQUIRED,
         "cassandra": CASSANDRA_REQUIRED,
+        "scylladb": SCYLLADB_REQUIRED,
         "hazelcast": HAZELCAST_REQUIRED,
         "grpcio": GRPCIO_REQUIRED,
         "ibis": IBIS_REQUIRED,