use id for mmproj tensors

Author: ngxson

gguf-py/gguf/constants.py

@@ -438,8 +438,7 @@ class MODEL_TENSOR(IntEnum):
     POSNET_ATTN_V        = auto()
     POSNET_ATTN_OUT      = auto()
     # vision
-    V_MMPROJ_A           = auto()
-    V_MMPROJ_B           = auto()
+    V_MMPROJ             = auto()
     V_ENC_EMBD_CLS       = auto()
     V_ENC_EMBD_PATCH     = auto()
     V_ENC_EMBD_POS       = auto()
@@ -643,8 +642,7 @@ class MODEL_TENSOR(IntEnum):
     MODEL_TENSOR.POSNET_ATTN_V:             "posnet.{bid}.attn_v",
     MODEL_TENSOR.POSNET_ATTN_OUT:           "posnet.{bid}.attn_output",
     # vision
-    MODEL_TENSOR.V_MMPROJ_A:                "v.mmproj_a",
-    MODEL_TENSOR.V_MMPROJ_B:                "v.mmproj_b",
+    MODEL_TENSOR.V_MMPROJ:                  "v.mmproj_{bid}",
     MODEL_TENSOR.V_ENC_EMBD_CLS:            "v.enc.embd.cls",
     MODEL_TENSOR.V_ENC_EMBD_PATCH:          "v.enc.embd.patch",
     MODEL_TENSOR.V_ENC_EMBD_POS:            "v.enc.embd.pos",
@@ -1600,8 +1598,7 @@ class MODEL_TENSOR(IntEnum):
         MODEL_TENSOR.POSNET_ATTN_OUT,
     ],
     MODEL_ARCH.LLAVA_VISION: [
-        MODEL_TENSOR.V_MMPROJ_A,
-        MODEL_TENSOR.V_MMPROJ_B,
+        MODEL_TENSOR.V_MMPROJ,
         MODEL_TENSOR.V_ENC_EMBD_CLS,
         MODEL_TENSOR.V_ENC_EMBD_PATCH,
         MODEL_TENSOR.V_ENC_EMBD_POS,

gguf-py/gguf/tensor_mapping.py

@@ -726,12 +726,12 @@ class TensorNameMap:
         MODEL_TENSOR.CLS_OUT: (
             "classifier.out_proj", # roberta
 
-        MODEL_TENSOR.V_MMPROJ_A: (
-            "multi_modal_projector.linear_1",
+        MODEL_TENSOR.V_MMPROJ: (
+            "multi_modal_projector.linear_{bid}",
         ),
 
-        MODEL_TENSOR.V_MMPROJ_B: (
-            "multi_modal_projector.linear_2",
+        MODEL_TENSOR.V_MMPROJ: (
+            "multi_modal_projector.linear_{bid}",
         ),
 
         MODEL_TENSOR.V_ENC_EMBD_CLS: (

src/llama-arch.h

@@ -1420,8 +1420,7 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
 
 
 enum vision_tensor {
-    VISION_TENSOR_MMPROJ_A,
-    VISION_TENSOR_MMPROJ_B,
+    VISION_TENSOR_MMPROJ,
     VISION_TENSOR_ENC_EMBD_CLS,
     VISION_TENSOR_ENC_EMBD_PATCH,
     VISION_TENSOR_ENC_EMBD_POS,
@@ -1441,8 +1440,7 @@ static const std::map<vision_arch, std::map<vision_tensor, std::string>> VISION_
     {
         VISION_ARCH_LLAVA,
         {
-            { VISION_TENSOR_MMPROJ_A,                "v.mmproj_a"                  },
-            { VISION_TENSOR_MMPROJ_B,                "v.mmproj_b"                  },
+            { VISION_TENSOR_MMPROJ,                  "v.mmproj"                    },
             { VISION_TENSOR_ENC_EMBD_CLS,            "v.enc.embd.cls"              },
             { VISION_TENSOR_ENC_EMBD_PATCH,          "v.enc.embd.patch"            },
             { VISION_TENSOR_ENC_EMBD_POS,            "v.enc.embd.pos"              },
@@ -1472,8 +1470,7 @@ struct llm_tensor_info {
 };
 
 static const std::map<vision_tensor, llm_tensor_info> vision_tensor_info_mapping = {
-    {VISION_TENSOR_MMPROJ_A,                {LLM_TENSOR_LAYER_INPUT,     GGML_OP_GET_ROWS}},
-    {VISION_TENSOR_MMPROJ_B,                {LLM_TENSOR_LAYER_INPUT,     GGML_OP_GET_ROWS}},
+    {VISION_TENSOR_MMPROJ,                {LLM_TENSOR_LAYER_INPUT,     GGML_OP_GET_ROWS}},
     {VISION_TENSOR_ENC_EMBD_CLS,            {LLM_TENSOR_LAYER_INPUT,     GGML_OP_GET_ROWS}},
     {VISION_TENSOR_ENC_EMBD_PATCH,          {LLM_TENSOR_LAYER_INPUT,     GGML_OP_GET_ROWS}},
     {VISION_TENSOR_ENC_EMBD_POS,            {LLM_TENSOR_LAYER_INPUT,     GGML_OP_GET_ROWS}},