Merge branch 'mlc-ai:main' into main

Author: nihalgeorge01

.github/ISSUE_TEMPLATE/documentation.md

@@ -13,5 +13,5 @@ assignees: ''
 <!-- Please leave your general suggestion to our documentation here. -->
 
 ### Bug
-- Link to the buggy documentation/tutorial: 
+- Link to the buggy documentation/tutorial:
 - Description of the bug:

.github/ISSUE_TEMPLATE/general.md

@@ -10,4 +10,3 @@ assignees: ''
 ## ❓ General Questions
 
 <!-- Describe your questions -->
-

.github/ISSUE_TEMPLATE/speed-report.md

@@ -15,10 +15,10 @@ assignees: ''
 
 
 - The model configuration (e.g. quantization mode, running data type, etc.):
-- Device (e.g. MacBook Pro M2, PC+RTX 3080): 
+- Device (e.g. MacBook Pro M2, PC+RTX 3080):
 - OS (if applicable):
 - Encode speed (Token/s):
 - Decode speed (Token/s):
 - Memory usage (if applicable):
 
-<!-- Note that the measured speed might reflect peak performance if the prompt/chat history is short. -->
+<!-- Note that the measured speed might reflect peak performance if the prompt/chat history is short. -->

.pre-commit-config.yaml

@@ -0,0 +1,60 @@
+# To use:
+#
+#     pre-commit run -a
+#
+# Or:
+#
+#     pre-commit install  # (runs every time you commit in git)
+#
+# To update this file:
+#
+#     pre-commit autoupdate
+#
+# See https://github.com/pre-commit/pre-commit
+# Note the pre-commit hooks shoule only be used for formatting, but not for linting.
+# For linting consider using CI.
+repos:
+  # Standard hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+
+  # Changes tabs to spaces
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.5
+    hooks:
+      - id: remove-tabs
+      - id: remove-crlf
+
+  # Formatters
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.8.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v19.1.1
+    hooks:
+      - id: clang-format
+        types_or: [c++, c, cuda]
+        exclude: |
+          (?x)^(.*cubin.cpp$ | .*fmha_cubin.h | 3rdparty/.*)$
+
+  - repo: https://github.com/cheshirekow/cmake-format-precommit
+    rev: v0.6.13
+    hooks:
+      - id: cmake-format
+        additional_dependencies: [pyyaml>=5.1]

3rdparty/tvm

@@ -17,22 +17,22 @@ else()
 endif()
 
 if(NOT CMAKE_BUILD_TYPE)
-  set(
-    CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Build type" FORCE
-  )
+  set(CMAKE_BUILD_TYPE
+      RelWithDebInfo
+      CACHE STRING "Build type" FORCE)
   message(STATUS "Setting default build type to " ${CMAKE_BUILD_TYPE})
 endif(NOT CMAKE_BUILD_TYPE)
 
 option(MLC_HIDE_PRIVATE_SYMBOLS "Hide private symbols" ON)
 
-if (MLC_LLM_INSTALL_STATIC_LIB)
+if(MLC_LLM_INSTALL_STATIC_LIB)
   set(BUILD_STATIC_RUNTIME ON)
 endif()
 
 set(MLC_VISIBILITY_FLAG "")
-if (MLC_HIDE_PRIVATE_SYMBOLS)
+if(MLC_HIDE_PRIVATE_SYMBOLS)
   set(HIDE_PRIVATE_SYMBOLS ON)
-  if (NOT MSVC)
+  if(NOT MSVC)
     set(MLC_VISIBILITY_FLAG "-fvisibility=hidden")
   endif()
   message(STATUS "Hide private symbols")
@@ -54,13 +54,13 @@ set(USE_PROFILER OFF)
 set(USE_GTEST OFF)
 set(USE_LIBBACKTRACE OFF)
 set(BUILD_DUMMY_LIBTVM ON)
-if (NOT DEFINED TVM_SOURCE_DIR)
+if(NOT DEFINED TVM_SOURCE_DIR)
   if(DEFINED ENV{TVM_SOURCE_DIR})
     set(TVM_SOURCE_DIR "$ENV{TVM_SOURCE_DIR}")
   else()
     set(TVM_SOURCE_DIR 3rdparty/tvm)
   endif(DEFINED ENV{TVM_SOURCE_DIR})
-endif (NOT DEFINED TVM_SOURCE_DIR)
+endif(NOT DEFINED TVM_SOURCE_DIR)
 message(STATUS "TVM_SOURCE_DIR: ${TVM_SOURCE_DIR}")
 add_subdirectory(${TVM_SOURCE_DIR} tvm EXCLUDE_FROM_ALL)
 
@@ -75,15 +75,13 @@ list(FILTER XGRAMMAR_SRCS EXCLUDE REGEX "${XGRAMMAR_PATH}/cpp/pybind/.*\\.cc")
 list(APPEND MLC_LLM_SRCS ${XGRAMMAR_SRCS})
 add_library(mlc_llm_objs OBJECT ${MLC_LLM_SRCS})
 
-set(
-  MLC_LLM_INCLUDES
-  ${TVM_SOURCE_DIR}/include
-  ${TVM_SOURCE_DIR}/3rdparty/dlpack/include
-  ${TVM_SOURCE_DIR}/3rdparty/dmlc-core/include
-  ${TVM_SOURCE_DIR}/3rdparty/picojson
-)
+set(MLC_LLM_INCLUDES
+    ${TVM_SOURCE_DIR}/include ${TVM_SOURCE_DIR}/3rdparty/dlpack/include
+    ${TVM_SOURCE_DIR}/3rdparty/dmlc-core/include
+    ${TVM_SOURCE_DIR}/3rdparty/picojson)
 
-set(MLC_LLM_COMPILE_DEFS ${MLC_LLM_COMPILE_DEFS} DMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>)
+set(MLC_LLM_COMPILE_DEFS ${MLC_LLM_COMPILE_DEFS}
+                         DMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>)
 set(MLC_LLM_COMPILE_DEFS ${MLC_LLM_COMPILE_DEFS} __STDC_FORMAT_MACROS=1)
 set(MLC_LLM_COMPILE_DEFS ${MLC_LLM_COMPILE_DEFS} PICOJSON_USE_INT64)
 set(MLC_LLM_COMPILE_DEFS ${MLC_LLM_COMPILE_DEFS} XGRAMMAR_ENABLE_LOG_DEBUG=0)
@@ -97,41 +95,46 @@ target_include_directories(mlc_llm_objs PRIVATE ${XGRAMMAR_PATH}/include)
 
 add_library(mlc_llm SHARED $<TARGET_OBJECTS:mlc_llm_objs>)
 add_library(mlc_llm_static STATIC $<TARGET_OBJECTS:mlc_llm_objs>)
-add_dependencies(mlc_llm_static tokenizers_cpp sentencepiece-static tokenizers_c tvm_runtime)
+add_dependencies(mlc_llm_static tokenizers_cpp sentencepiece-static
+                 tokenizers_c tvm_runtime)
 set_target_properties(mlc_llm_static PROPERTIES OUTPUT_NAME mlc_llm)
 
 target_link_libraries(mlc_llm PUBLIC tvm_runtime)
 target_link_libraries(mlc_llm PRIVATE tokenizers_cpp)
 
-find_library(
-  FLASH_ATTN_LIBRARY flash_attn
-  HINTS ${TVM_SOURCE_DIR}/*/3rdparty/libflash_attn/src
-)
+find_library(FLASH_ATTN_LIBRARY flash_attn
+             HINTS ${TVM_SOURCE_DIR}/*/3rdparty/libflash_attn/src)
 
-if (FLASH_ATTN_LIBRARY STREQUAL "FLASH_ATTN_LIBRARY-NOTFOUND")
-  message(WARNING "Cannot find libflash_attn. The model must not have been built with --use-flash-attn-mqa option.")
-else ()
+if(FLASH_ATTN_LIBRARY STREQUAL "FLASH_ATTN_LIBRARY-NOTFOUND")
+  message(
+    WARNING
+      "Cannot find libflash_attn. The model must not have been built with --use-flash-attn-mqa option."
+  )
+else()
   target_link_libraries(mlc_llm PUBLIC -Wl,--no-as-needed ${FLASH_ATTN_LIBRARY})
 endif()
 
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-    target_compile_definitions(mlc_llm PRIVATE "TVM_LOG_DEBUG")
-    target_compile_definitions(mlc_llm_objs PRIVATE "TVM_LOG_DEBUG")
-    target_compile_definitions(mlc_llm_static PRIVATE "TVM_LOG_DEBUG")
+  target_compile_definitions(mlc_llm PRIVATE "TVM_LOG_DEBUG")
+  target_compile_definitions(mlc_llm_objs PRIVATE "TVM_LOG_DEBUG")
+  target_compile_definitions(mlc_llm_static PRIVATE "TVM_LOG_DEBUG")
 endif()
 
-if (BUILD_CPP_TEST)
+if(BUILD_CPP_TEST)
   message(STATUS "Building cpp unittests")
   add_subdirectory(3rdparty/googletest)
-  file(GLOB_RECURSE MLC_LLM_TEST_SRCS ${PROJECT_SOURCE_DIR}/tests/cpp/*unittest.cc)
+  file(GLOB_RECURSE MLC_LLM_TEST_SRCS
+       ${PROJECT_SOURCE_DIR}/tests/cpp/*unittest.cc)
   add_executable(mlc_llm_cpp_tests ${MLC_LLM_TEST_SRCS})
   target_include_directories(mlc_llm_cpp_tests PRIVATE ${MLC_LLM_INCLUDES})
-  target_include_directories(mlc_llm_cpp_tests PRIVATE ${PROJECT_SOURCE_DIR}/cpp)
-  target_include_directories(mlc_llm_cpp_tests PRIVATE ${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR})
+  target_include_directories(mlc_llm_cpp_tests
+                             PRIVATE ${PROJECT_SOURCE_DIR}/cpp)
+  target_include_directories(
+    mlc_llm_cpp_tests PRIVATE ${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR})
   target_link_libraries(mlc_llm_cpp_tests PUBLIC mlc_llm gtest gtest_main)
 endif(BUILD_CPP_TEST)
 
-if (CMAKE_SYSTEM_NAME STREQUAL "Android")
+if(CMAKE_SYSTEM_NAME STREQUAL "Android")
   target_link_libraries(mlc_llm PRIVATE log)
   target_link_libraries(tokenizers_cpp PRIVATE log)
 endif()
@@ -140,42 +143,43 @@ add_library(mlc_llm_module SHARED $<TARGET_OBJECTS:mlc_llm_objs>)
 target_link_libraries(mlc_llm_module PUBLIC tvm)
 target_link_libraries(mlc_llm_module PRIVATE tokenizers_cpp)
 
-set_property(TARGET mlc_llm_module APPEND PROPERTY LINK_OPTIONS "${MLC_VISIBILITY_FLAG}")
-set_property(TARGET mlc_llm APPEND PROPERTY LINK_OPTIONS "${MLC_VISIBILITY_FLAG}")
+set_property(
+  TARGET mlc_llm_module
+  APPEND
+  PROPERTY LINK_OPTIONS "${MLC_VISIBILITY_FLAG}")
+set_property(
+  TARGET mlc_llm
+  APPEND
+  PROPERTY LINK_OPTIONS "${MLC_VISIBILITY_FLAG}")
 
 find_program(CARGO_EXECUTABLE cargo)
 
 if(NOT CARGO_EXECUTABLE)
-    message(FATAL_ERROR "Cargo is not found! Please install cargo.")
+  message(FATAL_ERROR "Cargo is not found! Please install cargo.")
 endif()
 
-# when this option is on,
-# we install all static lib deps into lib
-if (MLC_LLM_INSTALL_STATIC_LIB)
-  install(TARGETS
-    mlc_llm_static
-    tokenizers_cpp
-    sentencepiece-static
-    tvm_runtime
-    LIBRARY DESTINATION lib${LIB_SUFFIX}
-    )
+# when this option is on, we install all static lib deps into lib
+if(MLC_LLM_INSTALL_STATIC_LIB)
+  install(TARGETS mlc_llm_static tokenizers_cpp sentencepiece-static tvm_runtime
+          LIBRARY DESTINATION lib${LIB_SUFFIX})
   # tokenizers need special handling as it builds from rust
   if(MSVC)
     install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/libtokenizers_c.lib
-      DESTINATION lib${LIB_SUFFIX}
-      )
+            DESTINATION lib${LIB_SUFFIX})
   else()
     install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/libtokenizers_c.a
-      DESTINATION lib${LIB_SUFFIX}
-      )
+            DESTINATION lib${LIB_SUFFIX})
   endif()
 else()
-  install(TARGETS tvm_runtime mlc_llm mlc_llm_module
-    mlc_llm_static
-    tokenizers_cpp
-    sentencepiece-static
-    RUNTIME_DEPENDENCY_SET tokenizers_c
+  install(
+    TARGETS tvm_runtime
+            mlc_llm
+            mlc_llm_module
+            mlc_llm_static
+            tokenizers_cpp
+            sentencepiece-static
+            RUNTIME_DEPENDENCY_SET
+            tokenizers_c
     RUNTIME DESTINATION bin
-    LIBRARY DESTINATION lib${LIB_SUFFIX}
-  )
+    LIBRARY DESTINATION lib${LIB_SUFFIX})
 endif()

CMakeLists.txt

@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

LICENSE

@@ -1,3 +1,3 @@
 MLC LLM
 
-Copyright (c) 2023-2024 by MLC LLM Contributors
+Copyright (c) 2023-2025 by MLC LLM Contributors

NOTICE

@@ -78,7 +78,7 @@ Please consider citing our project if you find it useful:
     author = {{MLC team}},
     title = {{MLC-LLM}},
     url = {https://github.com/mlc-ai/mlc-llm},
-    year = {2023-2024}
+    year = {2023-2025}
 }
 ```
 
@@ -131,5 +131,3 @@ The underlying techniques of MLC LLM include:
   }
   ```
 </details>
-
-

README.md

@@ -1,2 +1,2 @@
 /build
-/src/main/libs
+/src/main/libs