Skip to content
This repository was archived by the owner on Aug 29, 2024. It is now read-only.

test tokenizer #3

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

test tokenizer #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Torchsharp-llama/ITokenizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,6 @@ public int[] Encode(string input, bool bos, bool eos)
tokens = tokens.Concat(new int[] { this.EosId }).ToArray();
}

Console.WriteLine($"tokens: {string.Join(",", tokens)}");

return tokens;
}
}
27 changes: 8 additions & 19 deletions Torchsharp-llama/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,30 +6,19 @@
var vocabPath = @"vocab.json";
var mergesPath = @"merges.txt";
var tokenizer = new BPETokenizer(vocabPath, mergesPath);
var checkpointDirectory = "/home/xiaoyuz/llama/llama-2-7b";
var device = "cuda";

if (device == "cuda")
{
torch.InitializeDeviceType(DeviceType.CUDA);
torch.cuda.is_available().Should().BeTrue();
}

torch.manual_seed(100);
var model = LLaMA.Build(
checkPointsDirectory: checkpointDirectory,
tokenizer: tokenizer,
maxSeqLen: 128,
maxBatchSize: 1,
device: device);

var prompts = new[]
{
"I believe the meaning of life is",
"Replace this text in the input field to see how llama tokenization works."
};
var result = model.TextCompletion(prompts, temperature: 0, echo: true, device: device);

foreach (var item in result)
foreach (var prompt in prompts)
{
Console.WriteLine($"generation: {item.generation}");
var tokens = tokenizer.Encode(prompt, bos: true, eos: false);
var decoded = tokenizer.Decode(tokens);
Console.WriteLine($"Prompt: {prompt}");
Console.WriteLine($"Tokens: {string.Join(", ", tokens)}");
Console.WriteLine($"Decoded: {decoded}");
Console.WriteLine();
}
2 changes: 1 addition & 1 deletion Torchsharp-llama/Torchsharp-llama.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="7.0.0-alpha.3" />
<PackageReference Include="Microsoft.ML.Tokenizers" Version="0.21.1" />
<PackageReference Include="TorchSharp-cuda-linux" Version="0.100.7" />
<PackageReference Include="TorchSharp-cuda-windows" Version="0.100.7" />
<!-- <PackageReference Include="TorchSharp-cpu" Version="0.100.7" /> -->
<!-- <PackageReference Include="libtorch-cuda-12.1-linux-x64" Version="2.1.0.1" /> -->
</ItemGroup>
Expand Down