Skip to content

Commit

Permalink
NPUW: Support generation & prefill configs extension for npuw::LLMCom…
Browse files Browse the repository at this point in the history
…piledModel (#28413)

Co-authored-by: Dmitry Matveev <[email protected]>
  • Loading branch information
TolyaTalamanov and dmatveev authored Jan 15, 2025
1 parent 62b720d commit 5c55539
Showing 1 changed file with 9 additions and 0 deletions.
9 changes: 9 additions & 0 deletions src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,8 @@ ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr<ov::Model>& m
// preserve them somewhere.
auto prefill_config_opt = pop_option(npuw_llm_props, std::string("NPUW_LLM_PREFILL_CONFIG"));
auto generate_config_opt = pop_option(npuw_llm_props, std::string("NPUW_LLM_GENERATE_CONFIG"));
auto prefill_config_addition = pop_option(npuw_llm_props, std::string("++NPUW_LLM_PREFILL_CONFIG"));
auto generate_config_addition = pop_option(npuw_llm_props, std::string("++NPUW_LLM_GENERATE_CONFIG"));

m_cfg.update(any_copy(npuw_llm_props));

Expand Down Expand Up @@ -501,8 +503,15 @@ ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr<ov::Model>& m
generate_config_opt.value_or(get_default_generate_config(kvcache_model, npudesc, generate_hint))
.as<ov::AnyMap>();

auto prefill_config_addition_value =
prefill_config_addition.has_value() ? prefill_config_addition.value().as<ov::AnyMap>() : ov::AnyMap{};
auto generate_config_addition_value =
generate_config_addition.has_value() ? generate_config_addition.value().as<ov::AnyMap>() : ov::AnyMap{};

merge_config_with(prefill_config, other_props);
merge_config_with(generate_config, other_props);
merge_config_with(prefill_config, prefill_config_addition_value);
merge_config_with(generate_config, generate_config_addition_value);

m_kvcache_compiled = std::dynamic_pointer_cast<ov::npuw::CompiledModel>(
ov::npuw::ICompiledModel::create(kvcache_model, plugin, generate_config));
Expand Down

0 comments on commit 5c55539

Please sign in to comment.