minor fixes

Author: mzegla

prepare_llm_models.sh

@@ -73,19 +73,21 @@ if [ -f "$1/$TEXT_GENERATION_MODEL/$TOKENIZER_FILE" ]; then
   echo "Models file $1/$TEXT_GENERATION_MODEL/$TOKENIZER_FILE exists. Skipping downloading models."
 else
   python3 demos/common/export_models/export_model.py text_generation --source_model "$TEXT_GENERATION_MODEL" --weight-format int8 --model_repository_path $1
-  if [ ! -f "$1/$TEXT_GENERATION_MODEL/chat_template.jinja" ]; then
+fi
+
+if [ ! -f "$1/$TEXT_GENERATION_MODEL/$TOKENIZER_FILE" ]; then
+  echo "[ERROR] Models file $1/$TEXT_GENERATION_MODEL/$TOKENIZER_FILE does not exist."
+  exit 1
+fi
+
+if [ ! -f "$1/$TEXT_GENERATION_MODEL/chat_template.jinja" ]; then
     dummy_chat_template="{% for message in messages %}\
 {% if message['role'] == 'user' %}{{ 'User: ' + message['content'] }}\
 {% elif message['role'] == 'system' %}{{ '<|system|>\n' + message['content'] + eos_token }}\
 {% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token }}\
 {% endif %}\
 {% endfor %}"
     echo "$dummy_chat_template" > "$1/$TEXT_GENERATION_MODEL/chat_template.jinja"
-  fi
-fi
-if [ ! -f "$1/$TEXT_GENERATION_MODEL/$TOKENIZER_FILE" ]; then
-  echo "[ERROR] Models file $1/$TEXT_GENERATION_MODEL/$TOKENIZER_FILE does not exist."
-  exit 1
 fi
 
 if [ -f "$1/$VLM_MODEL/$TOKENIZER_FILE" ]; then

src/llm/servable_initializer.cpp

@@ -149,6 +149,9 @@ void GenAiServableInitializer::loadPyTemplateProcessor(std::shared_ptr<GenAiServ
     properties->templateProcessor.bosToken = bosToken;
     properties->templateProcessor.eosToken = eosToken;
 
+    SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Loading Python Jinja template processor with chat template from tokenizer. Bos token: {}, Eos token: {}, chat template: \n{}",
+        bosToken, eosToken, chatTemplate);
+
     py::gil_scoped_acquire acquire;
     try {
         auto locals = py::dict("chat_template"_a = chatTemplate,
@@ -250,6 +253,13 @@ void GenAiServableInitializer::loadPyTemplateProcessor(std::shared_ptr<GenAiServ
                             if template_entry.get("name") == "tool_use":
                                 tool_chat_template = template_entry.get("template")
             
+            # Try read tool_use.jinja template file from additional_chat_templates directory if exists
+            additional_templates_dir = Path(templates_directory + "/additional_chat_templates")
+            tool_use_template_file = additional_templates_dir / "tool_use.jinja"
+            if tool_use_template_file.is_file():
+                with open(tool_use_template_file, "r", encoding="utf-8") as f:
+                    tool_chat_template = f.read()
+            
             # Load templates from strings
             template = jinja_env.from_string(chat_template)
             if tool_chat_template is not None:

windows_prepare_llm_models.bat

@@ -79,23 +79,23 @@ if exist "%~1\%TEXT_GENERATION_MODEL%\%TOKENIZER_FILE%" (
 ) else (
   echo Downloading text generation model to %~1\%TEXT_GENERATION_MODEL% directory.
   python demos\common\export_models\export_model.py text_generation --source_model "%TEXT_GENERATION_MODEL%" --weight-format int8 --model_repository_path %~1
+  if !errorlevel! neq 0 exit /b !errorlevel!
+)
+if not exist "%~1\%TEXT_GENERATION_MODEL%\%TOKENIZER_FILE%" (
+  echo Models file %~1\%TEXT_GENERATION_MODEL%\%TOKENIZER_FILE% does not exists.
+  exit /b 1
+)
 
-  if not exist "%~1\%TEXT_GENERATION_MODEL%\chat_template.jinja" (
+if not exist "%~1\%TEXT_GENERATION_MODEL%\chat_template.jinja" (
     set "dummy_chat_template={%% for message in messages %%}^
 {%% if message['role'] == 'user' %%}{{ 'User: ' + message['content'] }}^
 {%% elif message['role'] == 'system' %%}{{ '<|system|>\n' + message['content'] + eos_token }}^
 {%% elif message['role'] == 'assistant' %%}{{ message['content'] + eos_token }}^
 {%% endif %%}^
 {%% endfor %%}"
     echo !dummy_chat_template! > "%~1\%TEXT_GENERATION_MODEL%\chat_template.jinja"
-  )
-
-  if !errorlevel! neq 0 exit /b !errorlevel!
+    if !errorlevel! neq 0 exit /b !errorlevel!
 )
-if not exist "%~1\%TEXT_GENERATION_MODEL%\%TOKENIZER_FILE%" (
-  echo Models file %~1\%TEXT_GENERATION_MODEL%\%TOKENIZER_FILE% does not exists.
-  exit /b 1
-) 
 
 if exist "%~1\%EMBEDDING_MODEL%\embeddings\%LEGACY_MODEL_FILE%" (
   echo Models file %~1\%EMBEDDING_MODEL%\embeddings\%LEGACY_MODEL_FILE% exists. Skipping downloading models.