apply to qwen2vl

Author: rebel-thkim

src/optimum/rbln/transformers/models/decoderonly/configuration_decoderonly.py

@@ -233,7 +233,7 @@ def __init__(
         self.logits_to_keep = logits_to_keep or self._default_logits_to_keep
         if self.logits_to_keep is not None and self.logits_to_keep > 1:
             raise NotImplementedError("`logits_to_keep` > 1 is currently not supported for RBLN models.")
-    
+
         self.output_hidden_states = output_hidden_states or False
 
         self.decoder_batch_sizes = None

src/optimum/rbln/transformers/models/qwen2_5_vl/configuration_qwen2_5_vl.py

@@ -60,6 +60,7 @@ class RBLNQwen2_5_VLModelConfig(RBLNDecoderOnlyModelConfig):
     """
     Configuration class for RBLNQwen2_5_VLModel.
     """
+
     def __init__(self, visual: Optional[RBLNModelConfig] = None, **kwargs: Any):
         super().__init__(**kwargs)
         self.visual = self.initialize_submodule_config(submodule_config=visual)

src/optimum/rbln/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py

@@ -352,19 +352,22 @@ class RBLNQwen2_5_VLModel(RBLNDecoderOnlyModel):
     _rbln_submodules = [
         {"name": "visual"},
     ]
+    _config_class = Qwen2_5_VLConfig
+    _rotary_emb_class = Qwen2_5_VLRotaryEmbedding
+    _get_rope_index_func = Qwen2_5_VLModel.get_rope_index
 
     def __post_init__(self, **kwargs):
         if hasattr(self.config, "embedding_dim"):
             self.embedding_dim = self.config.embedding_dim
 
         if not isinstance(self.config.text_config, PretrainedConfig):
-            self.config = Qwen2_5_VLConfig(
+            self.config = self._config_class(
                 text_config=self.config.text_config, vision_config=self.config.vision_config
             )
 
         super().__post_init__(**kwargs)
         self.visual = self.rbln_submodules[0]
-        self.rotary_emb = Qwen2_5_VLRotaryEmbedding(self.config)
+        self.rotary_emb = self._rotary_emb_class(self.config)
         if not self.can_generate():
             self.block_tables = torch.arange(self.rbln_config.kvcache_num_blocks, dtype=torch.int16)
 
@@ -475,8 +478,7 @@ def _preprocess_prefill(
             vision_tokens = input_id[0][vision_start_indices + 1]
             image_nums = (vision_tokens == image_token_id).sum()
             video_nums = (vision_tokens == video_token_id).sum()
-            position_ids, rope_deltas = Qwen2_5_VLModel.get_rope_index(
-                self,
+            position_ids, rope_deltas = self._get_rope_index_func(
                 input_id,
                 image_grid_thw[image_idx : image_idx + image_nums] if image_grid_thw is not None else None,
                 video_grid_thw[video_idx : video_idx + video_nums] if video_grid_thw is not None else None,
@@ -723,7 +725,7 @@ def forward(
                 )
                 logits.append(output.logits)
             logits = torch.cat(logits, dim=0)
-            # Decoder
+        # Decoder
         else:
             inputs_embeds, position_embed = self._preprocess_decoder(input_ids, cache_position)
             output = self.decoder(

src/optimum/rbln/transformers/models/qwen2_5_vl/qwen2_5_vl_architecture.py

@@ -171,8 +171,8 @@ def prepare_forward_args(self, *args):
         position_embeds = args.pop(0)
         query_position = args.pop(0) if self.phase == "prefill" and self.rbln_config.logits_to_keep > 0 else None
         position_ids = None
-        lora_int_id = None
         attention_mask = args.pop(0) if self.rbln_config.use_attention_mask else None
+        lora_int_id = args.pop(0) if self.rbln_config.lora_config else None
         past_key_values = args
 
         if len(past_key_values) != 2 * self.num_hidden_layers:
@@ -202,4 +202,4 @@ def prepare_forward_args(self, *args):
             lora_int_id,
             past_key_values,
             position_embeds,
-        )
+        )