Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 94ed5ff

Browse files
hiento09hiento09
authored
Linux add compile python lib for convert model (#46)
Co-authored-by: Hien To <tominhhien97@gmail.com>
1 parent d2fc6b4 commit 94ed5ff

File tree

5 files changed

+63
-93
lines changed

5 files changed

+63
-93
lines changed

‎.github/runners/linux/Dockerfile.convertModel‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM nvidia/cuda:12.3.0-devel-ubuntu22.04
2+
3+
RUN apt-get update && apt-get -y install python3.10 python3-pip openmpi-bin libopenmpi-dev git git-lfs
4+
5+
RUN pip3 install tensorrt_llm==0.9.0 --extra-index-url https://pypi.nvidia.com
6+
7+
RUN pip3 install numpy==1.26.4
8+
9+
RUN git clone https://github.com/NVIDIA/TensorRT-LLM.git && cd TensorRT-LLM && git checkout v0.9.0

‎.github/workflows/build.yml‎

Lines changed: 25 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,13 @@ jobs:
4343
runs-on: "windows-tensorrt-llm-cuda-12-2"
4444
run-e2e: false
4545
s3-key-prefix: "windows-tensorrt-llm-ccache"
46-
sccache-conf-path: 'C:\sccache.conf'
46+
ccache-dir: 'C:\Users\ContainerAdministrator\AppData\Local\ccache'
4747
- os: "linux"
4848
name: "cuda-12-3"
4949
runs-on: "linux-tensorrt-llm-cuda-12-3"
5050
run-e2e: false
51-
s3-key-prefix: "linux-tensorrt-llm"
52-
sccache-conf-path: '/tmp/sccache.conf'
51+
s3-key-prefix: "linux-tensorrt-llm-0-9-0-cuda-12-3"
52+
ccache-dir: "/home/runner/.ccache"
5353
permissions:
5454
contents: write
5555
steps:
@@ -60,51 +60,34 @@ jobs:
6060
submodules: recursive
6161
lfs: true
6262

63-
- name: Install choco on Windows
63+
- name: Install tools on Windows
6464
if: runner.os == 'Windows'
6565
run: |
66-
choco install make pkgconfiglite ccache awscli -y
66+
choco install make pkgconfiglite ccache awscli 7zip -y
67+
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
68+
refreshenv
6769
68-
- name: create sccache.conf file Linux
69-
if: runner.os == 'Linux'
70-
run: |
71-
echo "[cache.s3]" > ${{ matrix.sccache-conf-path }}
72-
echo 'bucket = "${{ secrets.MINIO_BUCKET_NAME }}"' >> ${{ matrix.sccache-conf-path }}
73-
echo 'endpoint = "${{ secrets.MINIO_ENDPOINT }}"' >> ${{ matrix.sccache-conf-path }}
74-
echo 'key_prefix = "${{ matrix.s3-key-prefix }}"' >> ${{ matrix.sccache-conf-path }}
75-
echo 'use_ssl = false' >> ${{ matrix.sccache-conf-path }}
76-
echo 'server_side_encryption = false' >> ${{ matrix.sccache-conf-path }}
77-
echo 'no_credentials = false' >> ${{ matrix.sccache-conf-path }}
78-
7970
- name: Download ccache from s3
80-
continue-on-error: true
8171
if: runner.os == 'Windows'
72+
continue-on-error: true
8273
run: |
8374
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
8475
refreshenv
85-
aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} C:\Users\ContainerAdministrator\AppData\Local\ccache --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
76+
aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} ${{ matrix.ccache-dir }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
8677
env:
8778
AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
8879
AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
8980
AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
9081

91-
- name: start sccache server for linux
82+
- name: Download ccache from s3
9283
if: runner.os == 'Linux'
93-
working-directory: cpp
84+
continue-on-error: true
9485
run: |
95-
sccache --start-server
86+
aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} ${{ matrix.ccache-dir }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
9687
env:
97-
SCCACHE_BUCKET: "${{ secrets.MINIO_BUCKET_NAME }}"
98-
SCCACHE_REGION: "${{ secrets.MINIO_REGION }}"
99-
SCCACHE_ENDPOINT: "${{ secrets.MINIO_ENDPOINT }}"
100-
SCCACHE_S3_USE_SSL: "false"
101-
SCCACHE_S3_SERVER_SIDE_ENCRYPTION: "false"
102-
SCCACHE_S3_KEY_PREFIX: "${{ matrix.s3-key-prefix }}"
103-
SCCACHE_LOG: "debug"
104-
SCCACHE_CONF: '${{ matrix.sccache-conf-path }}'
10588
AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
10689
AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
107-
SCCACHE_IDLE_TIMEOUT: "0"
90+
AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
10891

10992
- name: Build Dependencies
11093
working-directory: cpp
@@ -141,25 +124,29 @@ jobs:
141124
asset_path: cpp/tensorrt_llm/cortex.tensorrt-llm/cortex.tensorrt-llm.tar.gz
142125
asset_name: cortex.tensorrt-llm-${{ needs.create-draft-release.outputs.version }}-${{ matrix.os }}-${{ matrix.name }}.tar.gz
143126
asset_content_type: application/gzip
144-
145-
- name: Clean
146-
if: always()
147-
continue-on-error: true
148-
run: |
149-
sccache --stop-server
150-
rm ${{ matrix.sccache-conf-path }}
151127

152128
- name: Upload ccache to s3
153129
continue-on-error: true
154130
if: always() && runner.os == 'Windows'
155131
run: |
156132
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
157133
refreshenv
158-
aws s3 cp C:\Users\ContainerAdministrator\AppData\Local\ccache s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
134+
aws s3 cp ${{ matrix.ccache-dir }} s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
159135
env:
160136
AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
161137
AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
162138
AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
139+
140+
- name: Upload ccache to s3
141+
continue-on-error: true
142+
if: always() && runner.os == 'Linux'
143+
run: |
144+
aws s3 cp ${{ matrix.ccache-dir }} s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
145+
env:
146+
AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
147+
AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
148+
AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
149+
163150
update_release_draft:
164151
needs: [build-and-test]
165152
permissions:
@@ -183,10 +170,3 @@ jobs:
183170
# disable-autolabeler: true
184171
env:
185172
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
186-
187-
- name: Clean
188-
if: always()
189-
continue-on-error: true
190-
run: |
191-
sccache --stop-server
192-
rm ${{ matrix.sccache-conf-path }}

‎.github/workflows/quality-gate.yml‎

Lines changed: 23 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,13 @@ jobs:
1818
runs-on: "windows-tensorrt-llm-cuda-12-2"
1919
run-e2e: false
2020
s3-key-prefix: "windows-tensorrt-llm-ccache"
21-
sccache-conf-path: 'C:\sccache.conf'
21+
ccache-dir: 'C:\Users\ContainerAdministrator\AppData\Local\ccache'
2222
- os: "linux"
2323
name: "cuda-12-3"
2424
runs-on: "linux-tensorrt-llm-cuda-12-3"
2525
run-e2e: false
26-
s3-key-prefix: "linux-tensorrt-llm"
27-
sccache-conf-path: '/tmp/sccache.conf'
26+
s3-key-prefix: "linux-tensorrt-llm-0-9-0-cuda-12-3"
27+
ccache-dir: "/home/runner/.ccache"
2828
permissions:
2929
contents: write
3030
steps:
@@ -35,51 +35,34 @@ jobs:
3535
submodules: recursive
3636
lfs: true
3737

38-
- name: Install choco on Windows
38+
- name: Install tools on Windows
3939
if: runner.os == 'Windows'
4040
run: |
41-
choco install make pkgconfiglite ccache awscli -y
41+
choco install make pkgconfiglite ccache awscli 7zip -y
42+
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
43+
refreshenv
4244
43-
- name: create sccache.conf file Linux
44-
if: runner.os == 'Linux'
45-
run: |
46-
echo "[cache.s3]" > ${{ matrix.sccache-conf-path }}
47-
echo 'bucket = "${{ secrets.MINIO_BUCKET_NAME }}"' >> ${{ matrix.sccache-conf-path }}
48-
echo 'endpoint = "${{ secrets.MINIO_ENDPOINT }}"' >> ${{ matrix.sccache-conf-path }}
49-
echo 'key_prefix = "${{ matrix.s3-key-prefix }}"' >> ${{ matrix.sccache-conf-path }}
50-
echo 'use_ssl = false' >> ${{ matrix.sccache-conf-path }}
51-
echo 'server_side_encryption = false' >> ${{ matrix.sccache-conf-path }}
52-
echo 'no_credentials = false' >> ${{ matrix.sccache-conf-path }}
53-
5445
- name: Download ccache from s3
55-
continue-on-error: true
5646
if: runner.os == 'Windows'
47+
continue-on-error: true
5748
run: |
5849
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
5950
refreshenv
60-
aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} C:\Users\ContainerAdministrator\AppData\Local\ccache --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
51+
aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} ${{ matrix.ccache-dir }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
6152
env:
6253
AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
6354
AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
6455
AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
6556

66-
- name: start sccache server for linux
57+
- name: Download ccache from s3
6758
if: runner.os == 'Linux'
68-
working-directory: cpp
59+
continue-on-error: true
6960
run: |
70-
sccache --start-server
61+
aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} ${{ matrix.ccache-dir }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
7162
env:
72-
SCCACHE_BUCKET: "${{ secrets.MINIO_BUCKET_NAME }}"
73-
SCCACHE_REGION: "${{ secrets.MINIO_REGION }}"
74-
SCCACHE_ENDPOINT: "${{ secrets.MINIO_ENDPOINT }}"
75-
SCCACHE_S3_USE_SSL: "false"
76-
SCCACHE_S3_SERVER_SIDE_ENCRYPTION: "false"
77-
SCCACHE_S3_KEY_PREFIX: "${{ matrix.s3-key-prefix }}"
78-
SCCACHE_LOG: "debug"
79-
SCCACHE_CONF: '${{ matrix.sccache-conf-path }}'
8063
AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
8164
AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
82-
SCCACHE_IDLE_TIMEOUT: "0"
65+
AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
8366

8467
- name: Build Dependencies
8568
working-directory: cpp
@@ -106,24 +89,25 @@ jobs:
10689
with:
10790
name: cortex.tensorrt-llm-${{ matrix.os }}-${{ matrix.name }}
10891
path: cpp/tensorrt_llm/cortex.tensorrt-llm/cortex.tensorrt-llm.tar.gz
109-
110-
- name: Clean
111-
if: always()
112-
continue-on-error: true
113-
run: |
114-
sccache --stop-server
115-
rm ${{ matrix.sccache-conf-path }}
11692

11793
- name: Upload ccache to s3
11894
continue-on-error: true
11995
if: always() && runner.os == 'Windows'
12096
run: |
12197
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
12298
refreshenv
123-
aws s3 cp C:\Users\ContainerAdministrator\AppData\Local\ccache s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
99+
aws s3 cp ${{ matrix.ccache-dir }} s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
124100
env:
125101
AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
126102
AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
127103
AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
128104

129-
105+
- name: Upload ccache to s3
106+
continue-on-error: true
107+
if: always() && runner.os == 'Linux'
108+
run: |
109+
aws s3 cp ${{ matrix.ccache-dir }} s3://${{ secrets.MINIO_BUCKET_NAME }}/${{ matrix.s3-key-prefix }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
110+
env:
111+
AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
112+
AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
113+
AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"

‎cpp/Makefile‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ build-deps:
1818
ifeq ($(OS),Windows_NT)
1919
@powershell -Command "cd tensorrt_llm/cortex.tensorrt-llm; cmake -S ./third-party -B ./build_deps/third-party -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Release -DCMAKE_OBJECT_PATH_MAX=500; cmake --build ./build_deps/third-party --config Release -j8;"
2020
else
21-
@cd tensorrt_llm/cortex.tensorrt-llm && cmake -S ./third-party -B ./build_deps/third-party -DCMAKE_BUILD_TYPE=Release -DCMAKE_OBJECT_PATH_MAX=500 && make -C ./build_deps/third-party -j 10 && rm -rf ./build_deps/third-party;
21+
@cd tensorrt_llm/cortex.tensorrt-llm && cmake -S ./third-party -B ./build_deps/third-party -DCMAKE_BUILD_TYPE=Release && make -C ./build_deps/third-party -j 10 && rm -rf ./build_deps/third-party;
2222
endif
2323

2424
# Build the Cortex engine
@@ -27,9 +27,8 @@ ifeq ($(OS),Windows_NT)
2727
@powershell -Command "cd ..; python .\scripts\build_wheel.py -a '80-real;86-real;89-real' --trt_root 'C:\workspace\TensorRT-9.3.0.1\' -D 'BUILD_CORTEX_TENSORRT-LLM=ON' --use_ccache"
2828
@powershell -Command "cd build; cmake .. -DCMAKE_CUDA_ARCHITECTURES='80-real;86-real;89-real' -DTRT_LIB_DIR='C:/workspace/TensorRT-9.3.0.1/lib' -DTRT_INCLUDE_DIR='C:/workspace/TensorRT-9.3.0.1/include' -DBUILD_CORTEX_TENSORRT-LLM=ON -DCMAKE_CUDA_COMPILER='C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.2/bin/nvcc.exe' -DENABLE_MULTI_DEVICE=0 -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -G Ninja; cmake --build . --parallel 2 --config Release"
2929
else
30-
@mkdir -p build && cd build; \
31-
cmake .. -GNinja -DBUILD_CORTEX_TENSORRT-LLM=ON -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache -DCMAKE_BUILD_TYPE='Release' -DBUILD_PYT='OFF' -DBUILD_PYBIND='OFF' -DNVTX_DISABLE='ON' -DCMAKE_CUDA_ARCHITECTURES='80-real;86-real;89-real' '-DENABLE_MULTI_DEVICE=0' '-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc' -DBUILD_BENCHMARKS=OFF '-DBUILD_TESTS=OFF' -DTRT_LIB_DIR=/usr/local/tensorrt/lib -DTRT_INCLUDE_DIR=/usr/local/tensorrt/include; \
32-
cmake --build . --config Release;
30+
@cd .. && python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --cuda_architectures "80-real;86-real;89-real" --extra-cmake-vars "BUILD_CORTEX_TENSORRT-LLM=ON" --use_ccache
31+
@cd build && cmake .. -DCMAKE_CUDA_ARCHITECTURES='80-real;86-real;89-real' -DTRT_LIB_DIR='/usr/local/tensorrt/lib' -DTRT_INCLUDE_DIR='/usr/local/tensorrt/include' -DBUILD_CORTEX_TENSORRT-LLM=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -G Ninja && cmake --build . --config Release;
3332
endif
3433

3534
# Prepackage the Cortex engine
@@ -41,19 +40,17 @@ ifeq ($(OS),Windows_NT)
4140
@powershell -Command "cd tensorrt_llm\cortex.tensorrt-llm\; cp -Force C:\workspace\cuDNN\bin\cudnn_ops_infer64_8.dll cortex.tensorrt-llm\;"
4241
@powershell -Command "cd tensorrt_llm\cortex.tensorrt-llm\; cp -Force ..\..\build\tensorrt_llm\plugins\nvinfer_plugin_tensorrt_llm.dll cortex.tensorrt-llm\;"
4342
@powershell -Command "cd tensorrt_llm\cortex.tensorrt-llm\; cp -Force ..\..\build\tensorrt_llm\tensorrt_llm.dll cortex.tensorrt-llm\;"
44-
@powershell -Command "cd tensorrt_llm\cortex.tensorrt-llm\; cp -Force C:\Windows\System32\msmpi.dll cortex.tensorrt-llm\;"
4543
@powershell -Command "cd tensorrt_llm\cortex.tensorrt-llm\; cp -Force .\build_deps\_install\bin\zlib.dll cortex.tensorrt-llm\;"
4644
else
4745
cd ./tensorrt_llm/cortex.tensorrt-llm && \
4846
mkdir -p cortex.tensorrt-llm && \
49-
cp ../../build/tensorrt_llm/cortex.tensorrt-llm/libengine.$(shell uname | tr '[:upper:]' '[:lower:]' | sed 's/darwin/dylib/;s/linux/so/') cortex.tensorrt-llm && \
47+
cp ../../build/tensorrt_llm/cortex.tensorrt-llm/libengine.so cortex.tensorrt-llm && \
5048
cp /usr/local/tensorrt/targets/x86_64-linux-gnu/lib/libnvinfer.so.9 cortex.tensorrt-llm && \
5149
cp /usr/lib/x86_64-linux-gnu/libcudnn.so.8 cortex.tensorrt-llm && \
5250
cp /usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8 cortex.tensorrt-llm && \
5351
cp /home/runner/actions-runner/_work/cortex.tensorrt-llm/cortex.tensorrt-llm/cpp/build/tensorrt_llm/plugins/libnvinfer_plugin_tensorrt_llm.so.9 cortex.tensorrt-llm && \
5452
cp /home/runner/actions-runner/_work/cortex.tensorrt-llm/cortex.tensorrt-llm/cpp/build/tensorrt_llm/libtensorrt_llm.so cortex.tensorrt-llm && \
55-
cp /opt/hpcx/ompi/lib/libmpi.so.40 cortex.tensorrt-llm && \
56-
cp /usr/lib/x86_64-linux-gnu/libnccl.so cortex.tensorrt-llm
53+
cp /usr/lib/x86_64-linux-gnu/libnccl.so cortex.tensorrt-llm/libnccl.so.2
5754
endif
5855

5956
codesign:

‎scripts/build_wheel.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ def main(build_type: str = "Release",
8888
if cuda_architectures is not None else "")
8989

9090
cmake_def_args = []
91-
cmake_generator = ""
91+
cmake_generator = "-GNinja"
9292

9393
hardware_arch = platform.machine()
9494

0 commit comments

Comments
 (0)