diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 3a37572122df..0f0f4dc7e382 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -1356,6 +1356,10 @@ void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) {
 
     GGML_ASSERT(lid <= model.hparams.n_layer());
 
+    if (cparams.embeddings_layer_inp[lid] == enable) {
+        return;
+    }
+
     cparams.embeddings_layer_inp[lid] = enable;
 
     // note: without this reserve, the draft acceptance drops to zero. not sure why - this is unexpected