Skip to content

Commit 7be2db4

Browse files
martindevansmartindevans
authored
Merge pull request SciSharp#1165 from martindevans/update_apr_2025
Update Apr 2025
2 parents d3dd2d0 + e3f9f6c commit 7be2db4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1505
-114
lines changed

‎.github/workflows/compile.yml‎

Lines changed: 69 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,25 @@ jobs:
2828
include:
2929
- build: 'noavx'
3030
defines: '-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF'
31+
os: ubuntu-24.04
32+
arch: x64
3133
- build: 'avx2'
3234
defines: ''
35+
os: ubuntu-24.04
36+
arch: x64
3337
- build: 'avx'
3438
defines: '-DGGML_AVX2=OFF'
39+
os: ubuntu-24.04
40+
arch: x64
3541
- build: 'avx512'
3642
defines: '-DGGML_AVX512=ON'
37-
runs-on: ubuntu-24.04
43+
os: ubuntu-24.04
44+
arch: x64
45+
- build: 'aarch64'
46+
defines: '-DGGML_NATIVE=OFF -DGGML_CPU_AARCH64=ON -DGGML_CPU_ARM_ARCH=armv8-a'
47+
os: ubuntu-24.04-arm
48+
arch: arm64
49+
runs-on: ${{ matrix.os }}
3850
steps:
3951
- uses: actions/checkout@v4
4052
with:
@@ -52,28 +64,28 @@ jobs:
5264
- uses: actions/upload-artifact@v4
5365
with:
5466
path: ./build/bin/libllama.so
55-
name: llama-bin-linux-${{ matrix.build }}-x64.so
67+
name: llama-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
5668
if-no-files-found: error
5769
- uses: actions/upload-artifact@v4
5870
with:
5971
path: ./build/bin/libggml.so
60-
name: ggml-bin-linux-${{ matrix.build }}-x64.so
72+
name: ggml-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
6173
if-no-files-found: error
6274
- uses: actions/upload-artifact@v4
6375
with:
6476
path: ./build/bin/libggml-base.so
65-
name: ggml-base-bin-linux-${{ matrix.build }}-x64.so
77+
name: ggml-base-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
6678
if-no-files-found: error
6779
- uses: actions/upload-artifact@v4
6880
with:
6981
path: ./build/bin/libggml-cpu.so
70-
name: ggml-cpu-bin-linux-${{ matrix.build }}-x64.so
82+
name: ggml-cpu-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
7183
if-no-files-found: error
7284
- name: Upload Llava
7385
uses: actions/upload-artifact@v4
7486
with:
7587
path: ./build/bin/libllava_shared.so
76-
name: llava-bin-linux-${{ matrix.build }}-x64.so
88+
name: llava-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
7789
if-no-files-found: error
7890

7991
compile-musl:
@@ -527,19 +539,17 @@ jobs:
527539
if-no-files-found: error
528540

529541
compile-android:
530-
# Disable android build
531-
if: false
532-
542+
name: Compile (Android)
533543
strategy:
534544
fail-fast: true
535545
matrix:
536546
include:
537547
- build: 'x86'
538-
defines: '-DANDROID_ABI=x86'
548+
defines: '-DANDROID_ABI=x86 -DCMAKE_C_FLAGS=-march=i686 -DCMAKE_CXX_FLAGS=-march=i686'
539549
- build: 'x86_64'
540-
defines: '-DANDROID_ABI=x86_64'
550+
defines: '-DANDROID_ABI=x86_64 -DCMAKE_C_FLAGS=-march=x86-64 -DCMAKE_CXX_FLAGS=-march=x86-64'
541551
- build: 'arm64-v8a'
542-
defines: '-DANDROID_ABI=arm64-v8a'
552+
defines: '-DANDROID_ABI=arm64-v8a -DCMAKE_C_FLAGS=-march=armv8.7a -DCMAKE_C_FLAGS=-march=armv8.7a'
543553
runs-on: ubuntu-24.04
544554
steps:
545555
- uses: actions/checkout@v4
@@ -555,28 +565,39 @@ jobs:
555565
- name: Build
556566
id: cmake_build
557567
env:
558-
CMAKE_FLAGS: '-DCMAKE_TOOLCHAIN_FILE=${{ steps.setup-ndk.outputs.ndk-path }}/build/cmake/android.toolchain.cmake -DANDROID_PLATFORM=android-23'
568+
CMAKE_FLAGS: '-DCMAKE_TOOLCHAIN_FILE=${{ steps.setup-ndk.outputs.ndk-path }}/build/cmake/android.toolchain.cmake -DANDROID_PLATFORM=android-23 -DGGML_OPENMP=OFF -DGGML_LLAMAFILE=OFF'
559569
run: |
560-
mkdir build
561-
cd build
562-
cmake .. ${{ env.COMMON_DEFINE }} ${{ env.CMAKE_FLAGS }} ${{ matrix.defines }}
563-
cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS}
564-
cd ..
565-
ls -R
570+
# export-lora not supported on 32 bit machines hence breaks x86 build
571+
sed -i '/add_subdirectory(export-lora)/d' examples/CMakeLists.txt # remove export-lora from examples
572+
cmake ${{ env.COMMON_DEFINE }} ${{ env.CMAKE_FLAGS }} ${{ matrix.defines }} -B build
573+
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
566574
- name: Upload Llama
567575
uses: actions/upload-artifact@v4
568576
with:
569-
path: ./build/src/libllama.so
577+
path: ./build/bin/libllama.so
570578
name: llama-bin-android-${{ matrix.build }}.so
571-
- uses: actions/upload-artifact@v4
579+
- name: Upload GGML
580+
uses: actions/upload-artifact@v4
572581
with:
573-
path: ./build/ggml/src/libggml.so
582+
path: ./build/bin/libggml.so
574583
name: ggml-bin-android-${{ matrix.build }}.so
575584
if-no-files-found: error
585+
- name: Upload GGML Base
586+
uses: actions/upload-artifact@v4
587+
with:
588+
path: ./build/bin/libggml-base.so
589+
name: ggml-base-bin-android-${{ matrix.build }}.so
590+
if-no-files-found: error
591+
- name: Upload GGML CPU
592+
uses: actions/upload-artifact@v4
593+
with:
594+
path: ./build/bin/libggml-cpu.so
595+
name: ggml-cpu-bin-android-${{ matrix.build }}.so
596+
if-no-files-found: error
576597
- name: Upload Llava
577598
uses: actions/upload-artifact@v4
578599
with:
579-
path: ./build/examples/llava/libllava_shared.so
600+
path: ./build/bin/libllava_shared.so
580601
name: llava-bin-android-${{ matrix.build }}.so
581602

582603
build-deps:
@@ -601,7 +622,7 @@ jobs:
601622
- name: Rearrange Files
602623
run: |
603624
# Make all directories at once
604-
mkdir --parents deps/{noavx,avx,avx2,avx512,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64}
625+
mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64}
605626
606627
# Linux
607628
cp artifacts/ggml-bin-linux-noavx-x64.so/libggml.so deps/noavx/libggml.so
@@ -628,6 +649,13 @@ jobs:
628649
cp artifacts/llama-bin-linux-avx512-x64.so/libllama.so deps/avx512/libllama.so
629650
cp artifacts/llava-bin-linux-avx512-x64.so/libllava_shared.so deps/avx512/libllava_shared.so
630651
652+
# Arm64
653+
cp artifacts/ggml-bin-linux-aarch64-arm64.so/libggml.so deps/linux-arm64/libggml.so
654+
cp artifacts/ggml-base-bin-linux-aarch64-arm64.so/libggml-base.so deps/linux-arm64/libggml-base.so
655+
cp artifacts/ggml-cpu-bin-linux-aarch64-arm64.so/libggml-cpu.so deps/linux-arm64/libggml-cpu.so
656+
cp artifacts/llama-bin-linux-aarch64-arm64.so/libllama.so deps/linux-arm64/libllama.so
657+
cp artifacts/llava-bin-linux-aarch64-arm64.so/libllava_shared.so deps/linux-arm64/libllava_shared.so
658+
631659
# Musl
632660
cp artifacts/ggml-bin-musl-noavx-x64.so/libggml.so deps/musl-noavx/libggml.so
633661
cp artifacts/ggml-base-bin-musl-noavx-x64.so/libggml-base.so deps/musl-noavx/libggml-base.so
@@ -703,17 +731,23 @@ jobs:
703731
cp artifacts/llava-bin-osx-x64-rosetta2.dylib/libllava_shared.dylib deps/osx-x64-rosetta2/libllava_shared.dylib
704732
705733
# Android
706-
#cp artifacts/ggml-bin-android-arm64-v8a.so/libggml.so deps/android-arm64-v8a/libggml.so
707-
#cp artifacts/llama-bin-android-arm64-v8a.so/libllama.so deps/android-arm64-v8a/libllama.so
708-
#cp artifacts/llava-bin-android-arm64-v8a.so/libllava_shared.so deps/android-arm64-v8a/libllava_shared.so
709-
710-
#cp artifacts/ggml-bin-android-x86.so/libggml.so deps/android-x86/libggml.so
711-
#cp artifacts/llama-bin-android-x86.so/libllama.so deps/android-x86/libllama.so
712-
#cp artifacts/llava-bin-android-x86.so/libllava_shared.so deps/android-x86/libllava_shared.so
713-
714-
#cp artifacts/ggml-bin-android-x86_64.so/libggml.so deps/android-x86_64/libggml.so
715-
#cp artifacts/llama-bin-android-x86_64.so/libllama.so deps/android-x86_64/libllama.so
716-
#cp artifacts/llava-bin-android-x86_64.so/libllava_shared.so deps/android-x86_64/libllava_shared.so
734+
cp artifacts/ggml-bin-android-arm64-v8a.so/libggml.so deps/android-arm64-v8a/libggml.so
735+
cp artifacts/ggml-base-bin-android-arm64-v8a.so/libggml-base.so deps/android-arm64-v8a/libggml-base.so
736+
cp artifacts/ggml-cpu-bin-android-arm64-v8a.so/libggml-cpu.so deps/android-arm64-v8a/libggml-cpu.so
737+
cp artifacts/llama-bin-android-arm64-v8a.so/libllama.so deps/android-arm64-v8a/libllama.so
738+
cp artifacts/llava-bin-android-arm64-v8a.so/libllava_shared.so deps/android-arm64-v8a/libllava_shared.so
739+
740+
cp artifacts/ggml-bin-android-x86.so/libggml.so deps/android-x86/libggml.so
741+
cp artifacts/ggml-base-bin-android-x86.so/libggml-base.so deps/android-x86/libggml-base.so
742+
cp artifacts/ggml-cpu-bin-android-x86.so/libggml-cpu.so deps/android-x86/libggml-cpu.so
743+
cp artifacts/llama-bin-android-x86.so/libllama.so deps/android-x86/libllama.so
744+
cp artifacts/llava-bin-android-x86.so/libllava_shared.so deps/android-x86/libllava_shared.so
745+
746+
cp artifacts/ggml-bin-android-x86_64.so/libggml.so deps/android-x86_64/libggml.so
747+
cp artifacts/ggml-base-bin-android-x86_64.so/libggml-base.so deps/android-x86_64/libggml-base.so
748+
cp artifacts/ggml-cpu-bin-android-x86_64.so/libggml-cpu.so deps/android-x86_64/libggml-cpu.so
749+
cp artifacts/llama-bin-android-x86_64.so/libllama.so deps/android-x86_64/libllama.so
750+
cp artifacts/llava-bin-android-x86_64.so/libllava_shared.so deps/android-x86_64/libllava_shared.so
717751
718752
# Windows CUDA
719753
cp artifacts/ggml-bin-win-cublas-cu11.7.1-x64.dll/ggml.dll deps/cu11.7.1/ggml.dll

‎.github/workflows/main.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ jobs:
3838
with:
3939
dotnet-version: |
4040
8.0.x
41+
- name: Install Mobile Workloads
42+
run: |
43+
dotnet workload install android --ignore-failed-sources
44+
dotnet workload install maui --ignore-failed-sources
4145
- name: Cache Packages
4246
uses: actions/cache@v4
4347
with:

‎LLama/LLamaSharp.Runtime.targets‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,28 @@
202202
</None>
203203

204204

205+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libllama.so">
206+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
207+
<Link>runtimes/linux-arm64/native/libllama.so</Link>
208+
</None>
209+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libggml.so">
210+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
211+
<Link>runtimes/linux-arm64/native/libggml.so</Link>
212+
</None>
213+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libggml-base.so">
214+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
215+
<Link>runtimes/linux-arm64/native/libggml-base.so</Link>
216+
</None>
217+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libggml-cpu.so">
218+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
219+
<Link>runtimes/linux-arm64/native/libggml-cpu.so</Link>
220+
</None>
221+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libllava_shared.so">
222+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
223+
<Link>runtimes/linux-arm64/native/libllava_shared.so</Link>
224+
</None>
225+
226+
205227
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libllama.so">
206228
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
207229
<Link>runtimes/linux-x64/native/cuda11/libllama.so</Link>
@@ -466,4 +488,94 @@
466488
<Link>runtimes/linux-x64/native/vulkan/libllava_shared.so</Link>
467489
</None>
468490
</ItemGroup>
491+
492+
<!-- Android Native Libs (Start) -->
493+
<ItemGroup
494+
Condition="$(AndroidSupportedAbis.Contains('x86')) or $(RuntimeIdentifiers.Contains('android-x86'))">
495+
<AndroidNativeLibrary Visible="false"
496+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libllama.so">
497+
<Link>runtimes/android-x86/native/libllama.so</Link>
498+
<Abi>x86</Abi>
499+
</AndroidNativeLibrary>
500+
<AndroidNativeLibrary Visible="false"
501+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libggml.so">
502+
<Link>runtimes/android-x86/native/libggml.so</Link>
503+
<Abi>x86</Abi>
504+
</AndroidNativeLibrary>
505+
<AndroidNativeLibrary Visible="false"
506+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libggml-base.so">
507+
<Link>runtimes/android-x86/native/libggml-base.so</Link>
508+
<Abi>x86</Abi>
509+
</AndroidNativeLibrary>
510+
<AndroidNativeLibrary Visible="false"
511+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libggml-cpu.so">
512+
<Link>runtimes/android-x86/native/libggml-cpu.so</Link>
513+
<Abi>x86</Abi>
514+
</AndroidNativeLibrary>
515+
<AndroidNativeLibrary Visible="false"
516+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libllava_shared.so">
517+
<Link>runtimes/android-x86/native/libllava_shared.so</Link>
518+
<Abi>x86</Abi>
519+
</AndroidNativeLibrary>
520+
</ItemGroup>
521+
522+
<ItemGroup
523+
Condition="$(AndroidSupportedAbis.Contains('x86_64')) or $(RuntimeIdentifiers.Contains('android-x64'))">
524+
<AndroidNativeLibrary Visible="false"
525+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libllama.so">
526+
<Link>lib/x86_64/libllama.so</Link>
527+
<Abi>x86_64</Abi>
528+
</AndroidNativeLibrary>
529+
<AndroidNativeLibrary Visible="false"
530+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libggml.so">
531+
<Link>lib/x86_64/libggml.so</Link>
532+
<Abi>x86_64</Abi>
533+
</AndroidNativeLibrary>
534+
<AndroidNativeLibrary Visible="false"
535+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libggml-base.so">
536+
<Link>lib/x86_64/libggml-base.so</Link>
537+
<Abi>x86_64</Abi>
538+
</AndroidNativeLibrary>
539+
<AndroidNativeLibrary Visible="false"
540+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libggml-cpu.so">
541+
<Link>lib/x86_64/libggml-cpu.so</Link>
542+
<Abi>x86_64</Abi>
543+
</AndroidNativeLibrary>
544+
<AndroidNativeLibrary Visible="false"
545+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libllava_shared.so">
546+
<Link>lib/x86_64/libllava_shared.so</Link>
547+
<Abi>x86_64</Abi>
548+
</AndroidNativeLibrary>
549+
</ItemGroup>
550+
551+
<ItemGroup
552+
Condition="$(AndroidSupportedAbis.Contains('arm64-v8a')) or $(RuntimeIdentifiers.Contains('android-arm64'))">
553+
<AndroidNativeLibrary Visible="false"
554+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libllama.so">
555+
<Link>lib/arm64-v8a/libllama.so</Link>
556+
<Abi>arm64-v8a</Abi>
557+
</AndroidNativeLibrary>
558+
<AndroidNativeLibrary Visible="false"
559+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libggml.so">
560+
<Link>lib/arm64-v8a/libggml.so</Link>
561+
<Abi>arm64-v8a</Abi>
562+
</AndroidNativeLibrary>
563+
<AndroidNativeLibrary Visible="false"
564+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libggml-base.so">
565+
<Link>lib/arm64-v8a/libggml-base.so</Link>
566+
<Abi>arm64-v8a</Abi>
567+
</AndroidNativeLibrary>
568+
<AndroidNativeLibrary Visible="false"
569+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libggml-cpu.so">
570+
<Link>lib/arm64-v8a/libggml-cpu.so</Link>
571+
<Abi>arm64-v8a</Abi>
572+
</AndroidNativeLibrary>
573+
<AndroidNativeLibrary Visible="false"
574+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libllava_shared.so">
575+
<Link>lib/arm64-v8a/libllava_shared.so</Link>
576+
<Abi>arm64-v8a</Abi>
577+
</AndroidNativeLibrary>
578+
</ItemGroup>
579+
<!-- Android Native Libs (End) -->
580+
469581
</Project>

‎LLama/Native/LLamaModelParams.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ public unsafe struct LLamaModelParams
1313
/// todo: add support for llama_model_params.devices
1414
/// </summary>
1515
private IntPtr devices;
16+
17+
// NULL-terminated list of buffer types to use for tensors that match a pattern
18+
// actual type: llama_model_tensor_buft_override*
19+
// todo: add support for tensor_buft_overrides
20+
private IntPtr tensor_buft_overrides;
1621

1722
/// <summary>
1823
/// // number of layers to store in VRAM

‎LLama/Native/LLamaModelQuantizeParams.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,11 @@ public bool keep_split
8989
/// </summary>
9090
public IntPtr kv_overrides;
9191

92+
/// <summary>
93+
/// pointer to vector containing tensor types
94+
/// </summary>
95+
public IntPtr tensor_types;
96+
9297
/// <summary>
9398
/// Create a LLamaModelQuantizeParams with default values
9499
/// </summary>

‎LLama/Native/LLamaVocabPreType.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,9 @@ internal enum LLamaVocabPreType
3838
MINERVA = 27,
3939
DEEPSEEK3_LLM = 28,
4040
GPT4O = 29,
41+
SUPERBPE = 30,
42+
TRILLION = 31,
43+
BAILINGMOE = 32,
44+
LLAMA4 = 33,
4145
}
4246
// ReSharper restore InconsistentNaming

0 commit comments

Comments
 (0)