System Info
optimum habana 1.18.0.dev0
transformers 4.49.0
habana 1.17
Information
Tasks
Reproduction
my pipeline
import argparse
import json
import logging
import math
import time
from tqdm import tqdm
from pipeline import GaudiTextGenerationPipeline
from run_generation import setup_parser
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO,
)
logger = logging.getLogger(__name__)
def read_jsonl(file_path: str):
"""Read a JSONL file and return a list of dictionaries.
Args:
file_path (str): The path to the JSONL file.
Returns:
list: A list of dictionaries.
"""
with open(file_path, "r") as f:
basename = file_path.split("/")[-1]
data = f.readlines()
results = []
for line in tqdm(data, desc=basename, leave=False):
results.append(json.loads(line))
return results
PROMPT = "请根据下面输入的搜索文本，判断用户的意图是否为搜索商品，如果是的话输出1，否则输出0，请直接回答。"
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--input", "-i", type=str)
parser.add_argument("--output", "-o", type=str)
args = setup_parser(parser)
args.num_return_sequences = 1
logger.info("Initializing text-generation pipeline...")
pipe = GaudiTextGenerationPipeline(args, logger)
data_list = read_jsonl(args.input)
messages = []
y_true, y_pred = [], []
for item in data_list:
y_true.append(item["label"])
prompt = f"{PROMPT}\n{item['query']}"
messages.append(prompt)
duration = 0
for start_idx in tqdm(range(0, len(messages), args.batch_size)):
batch = messages[start_idx : start_idx + args.batch_size]
t0 = time.perf_counter()
result_batch = pipe(batch)
duration += time.perf_counter() - t0
y_pred.extend(result_batch)
throughput = args.n_iterations * args.batch_size * args.max_new_tokens / duration
print(
f"Inference Duration (for {args.n_iterations} iterations): {duration} seconds"
)
print(f"Throughput: {throughput} tokens/second")
if __name__ == "__main__":
main()my scripts:
ROOT_PATH=/mnt/shgeminicephfs/geminicephfs/searcher-others-a100/group_2/user_tongkeni/coderepo/llm-train
python run_pipeline.py \
--model_name_or_path $ROOT_PATH/outputs/ecom_intent/qwen3b/checkpoint-722 \
--input $ROOT_PATH/data/ecom-intent/v3/test.jsonl \
--output $ROOT_PATH/outputs/ecom_intent/qwen3b/checkpoint-722/eval_results.csv \
--use_kv_cache \
--max_new_tokens 1 \
--do_sample \
--sdp_on_bf16then i get:
return func(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/optimum/habana/transformers/generation/utils.py", line 1689, in generate
result = self._sample(
File "/usr/local/lib/python3.10/site-packages/optimum/habana/transformers/generation/utils.py", line 2776, in _sample
unfinished_sequences = unfinished_sequences & ~stopping_criteria(
File "/usr/local/lib/python3.10/site-packages/optimum/habana/transformers/generation/stopping_criteria.py", line 98, in gaudi_StoppingCriteriaList_call
is_done = is_done | criteria(input_ids, scores, **kwargs)
File "/usr/local/lib/python3.10/site-packages/optimum/habana/transformers/generation/stopping_criteria.py", line 73, in gaudi_EosTokenCriteria_call
is_done = torch.isin(input_ids[:, token_idx - 1], self.eos_token_id)
RuntimeError: Expected index == (this->*member).size() to be true, but got false.Expected behavior
no error.
System Info
Information
Tasks
examplesfolder (such as GLUE/SQuAD, ...)Reproduction
my pipeline
my scripts:
ROOT_PATH=/mnt/shgeminicephfs/geminicephfs/searcher-others-a100/group_2/user_tongkeni/coderepo/llm-train python run_pipeline.py \ --model_name_or_path $ROOT_PATH/outputs/ecom_intent/qwen3b/checkpoint-722 \ --input $ROOT_PATH/data/ecom-intent/v3/test.jsonl \ --output $ROOT_PATH/outputs/ecom_intent/qwen3b/checkpoint-722/eval_results.csv \ --use_kv_cache \ --max_new_tokens 1 \ --do_sample \ --sdp_on_bf16then i get:
Expected behavior
no error.