Skip to content

Commit 0447db7

Browse files
authored
Flush (#4)
Adds new flush options, updates .NET version, remove old versions
1 parent 0e36897 commit 0447db7

File tree

11 files changed

+161
-139
lines changed

11 files changed

+161
-139
lines changed

README.md

Lines changed: 42 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ Below is a sample configuration for the file provider. The values shown are the
3737
"MaxFileSizeInMB": 50, // this can be decimal
3838
"FormatterName": "simple", // simple or json
3939
"Append": true,
40+
"FlushToDisk" : false, // true to indicate that flush operations should flush all the way to disk (ie fsync)
41+
"MaxFlushInterval" : null, // max ms for which to allow before a flush operation on log writes
4042
"formatterOptions" : {
4143
// see formatter options below
4244
}
@@ -140,72 +142,71 @@ Example configuration
140142
Log files can have a max file size at which time a new file will be create with a incremented id appended. You may also specify a maximum number of files to retain. Once the maximum number of files has been reached, the oldest will be overwritten.
141143
Using RollInterval setting, you can also specify that a date will be appended to the file name and the files will roll according to the date in 'yyyyMMddHH' format.
142144

143-
## Custom Formatters
145+
## Custom Formatters
144146

145147
To implment a custom formatter, create a class that derives from ```FileFormatter<TOptions>```. Then register the formatter using a name and specify that name on the Logger options
146148

147-
## Benchmarks
149+
## Flush behavior
150+
151+
For performance reasons, this library uses a message queue and a dedicated thread for writing log messages to the file. By default, a flush() is called when the write thread empties the queue of log messages. In situations where there is a high throughput of log messages, file buffers may not flush for quite some time as the queue is not cleared. You can optionally specify a maximum amount of time in (MaxFlushInterval) ms to go without a file flush operation. A value of 0 would force a flush on every messsage write. Note that even a flush() operation may not actually ensure that the data is written to the physical disk. By default, the flush ensures that the data is written to the drive cache. If you wish to ensure that a flush will ensure the data is written to disk set "FlushToDisk" to true. This could have some minor performance implications.
148152

149-
The library has be benchmarked against a couple of other popular file loggers for .NET core. Specically NReco and Karambolo loggers. See the results below.
153+
## Benchmarks
150154

151-
#### Simple Text Message
152-
```
155+
The library has been benchmarked against a couple of other popular file loggers for .NET core. Specically NReco and Karambolo loggers. See the results below.
153156

154-
BenchmarkDotNet v0.13.12, Windows 11 (10.0.22631.3007/23H2/2023Update/SunValley3)
157+
### Simple Text Message
158+
```
159+
BenchmarkDotNet v0.14.0, Windows 11 (10.0.26100.2605)
155160
AMD Ryzen 5 5600H with Radeon Graphics, 1 CPU, 12 logical and 6 physical cores
156-
.NET SDK 8.0.100
157-
[Host] : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2
158-
Job-HSNTJM : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2
159-
160-
IterationCount=10 LaunchCount=2 WarmupCount=10
161+
.NET SDK 9.0.101
162+
[Host] : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2
163+
Job-BNUXKU : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2
161164

165+
IterationCount=10 LaunchCount=2 WarmupCount=10
162166
```
163-
| Method | Mean | Error | StdDev | Gen0 | Gen1 | Allocated |
164-
|----------------------- |----------------:|--------------:|--------------:|----------:|---------:|-----------:|
165-
| Bleess_single_write | 659.6 ns | 20.31 ns | 22.57 ns | 0.1068 | 0.0038 | 904 B |
166-
| Karambolo_single_write | 679.7 ns | 82.32 ns | 94.80 ns | 0.0839 | - | 706 B |
167-
| NReco_single_write | 1,547.5 ns | 13.10 ns | 14.56 ns | 0.1640 | 0.0057 | 1373 B |
168-
| Bleess_10000_writes | 6,469,397.9 ns | 163,216.34 ns | 181,414.53 ns | 1078.1250 | - | 9040006 B |
169-
| Karambolo_10000_write | 6,522,278.8 ns | 318,328.54 ns | 366,587.62 ns | 843.7500 | - | 7083785 B |
170-
| NReco_10000_write | 15,962,119.6 ns | 380,106.19 ns | 406,709.37 ns | 1625.0000 | 125.0000 | 13713687 B |
167+
| Method | Mean | Error | StdDev | Median | Gen0 | Gen1 | Allocated |
168+
|----------------------- |----------------:|----------------:|----------------:|----------------:|---------:|-------:|----------:|
169+
| Bleess_single_write | 594.1 ns | 19.90 ns | 22.11 ns | 588.8 ns | 0.0877 | 0.0191 | 744 B |
170+
| Karambolo_single_write | 541.3 ns | 14.60 ns | 16.23 ns | 539.3 ns | 0.0839 | - | 707 B |
171+
| NReco_single_write | 2,199.3 ns | 59.97 ns | 66.66 ns | 2,171.6 ns | 0.0839 | 0.0019 | 718 B |
172+
| Bleess_10000_writes | 7,285,301.5 ns | 1,745,837.96 ns | 1,868,027.06 ns | 5,723,939.1 ns | 875.0000 | - | 7440023 B |
173+
| Karambolo_10000_write | 8,466,456.8 ns | 144,135.60 ns | 165,986.77 ns | 8,468,877.3 ns | 843.7500 | - | 7067165 B |
174+
| NReco_10000_write | 20,273,803.3 ns | 2,614,162.62 ns | 3,010,473.56 ns | 21,856,078.1 ns | 843.7500 | - | 7165933 B |
171175

172176

173-
#### Json file
177+
### Json file
174178
```
175-
176-
BenchmarkDotNet v0.13.12, Windows 11 (10.0.22631.3007/23H2/2023Update/SunValley3)
179+
BenchmarkDotNet v0.14.0, Windows 11 (10.0.26100.2605)
177180
AMD Ryzen 5 5600H with Radeon Graphics, 1 CPU, 12 logical and 6 physical cores
178-
.NET SDK 8.0.100
179-
[Host] : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2
180-
Job-HSNTJM : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2
181-
182-
IterationCount=10 LaunchCount=2 WarmupCount=10
181+
.NET SDK 9.0.101
182+
[Host] : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2
183+
Job-BNUXKU : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2
183184

185+
IterationCount=10 LaunchCount=2 WarmupCount=10
184186
```
187+
188+
185189
| Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated |
186190
|---------------------------- |---------:|----------:|----------:|-------:|-------:|-------:|----------:|
187-
| Bleess_single_write_json | 1.990 μs | 0.0469 μs | 0.0521 μs | 0.9766 | - | - | 7.99 KB |
188-
| Karambolo_single_write_json | 2.118 μs | 0.1781 μs | 0.2051 μs | 0.3204 | 0.0458 | 0.0153 | 2.65 KB |
189-
191+
| Bleess_single_write_json | 1.950 us | 0.0633 us | 0.0704 us | 0.9766 | 0.0305 | - | 7.99 KB |
192+
| Karambolo_single_write_json | 1.717 us | 0.0329 us | 0.0352 us | 0.3204 | 0.0381 | 0.0229 | 2.64 KB |
190193

191194

192-
#### Multi-File
195+
### Multi-File
193196
```
194-
195-
BenchmarkDotNet v0.13.12, Windows 11 (10.0.22631.3007/23H2/2023Update/SunValley3)
197+
BenchmarkDotNet v0.14.0, Windows 11 (10.0.26100.2605)
196198
AMD Ryzen 5 5600H with Radeon Graphics, 1 CPU, 12 logical and 6 physical cores
197-
.NET SDK 8.0.100
198-
[Host] : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2
199-
Job-HSNTJM : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2
200-
201-
IterationCount=10 LaunchCount=2 WarmupCount=10
199+
.NET SDK 9.0.101
200+
[Host] : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2
201+
Job-BNUXKU : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2
202202

203+
IterationCount=10 LaunchCount=2 WarmupCount=10
203204
```
205+
204206
| Method | Mean | Error | StdDev | Gen0 | Gen1 | Allocated |
205207
|--------------------------------- |---------:|----------:|----------:|-------:|-------:|----------:|
206-
| Bleess_multifile_single_write | 1.374 μs | 0.0396 μs | 0.0441 μs | 0.1984 | - | 1.67 KB |
207-
| Karambolo_multifile_single_write | 1.554 μs | 0.1430 μs | 0.1647 μs | 0.1602 | 0.0229 | 1.33 KB |
208-
208+
| Bleess_multifile_single_write | 1.249 us | 0.0267 us | 0.0286 us | 0.1602 | - | 1.36 KB |
209+
| Karambolo_multifile_single_write | 1.281 us | 0.1082 us | 0.1246 us | 0.1602 | 0.0153 | 1.31 KB |
209210

210211
## Credits
211212
- Some of the code was a adapted from dotnet source code (specifically Microsoft.Extensions.Logging.Console) https://github.com/dotnet/runtime/tree/master/src/libraries/Microsoft.Extensions.Logging.Console

benchmark/Benchmarks/Benchmarks.csproj

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@
88
</PropertyGroup>
99

1010
<ItemGroup>
11-
<PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
12-
<PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.5.0" Aliases="Karambolo" />
13-
<PackageReference Include="Karambolo.Extensions.Logging.File.Json" Version="3.5.0" Aliases="Karambolo" />
14-
<PackageReference Include="NReco.Logging.File" Version="1.2.0" Aliases="NReco" />
11+
<PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
12+
<PackageReference Include="Karambolo.Extensions.Logging.File" Version="3.6.0" Aliases="Karambolo" />
13+
<PackageReference Include="Karambolo.Extensions.Logging.File.Json" Version="3.6.0" Aliases="Karambolo" />
14+
<PackageReference Include="NReco.Logging.File" Version="1.2.2" Aliases="NReco" />
1515
</ItemGroup>
1616

1717
<ItemGroup>

benchmark/Benchmarks/MultiFileBenchmarks.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ namespace Benchmarks
2020
[SimpleJob(2, 10, 10)]
2121
public partial class MultiFileBenchmarks
2222
{
23-
24-
2523
public MultiFileBenchmarks()
2624
{
2725
this.SetupBleessLogging();

samples/Sample/appsettings.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525

2626
"File": {
2727
"Path": "logs/sample.txt",
28-
28+
"MaxFlushInterval": 2000, // flush every write (0)
29+
"FlushToDisk" : true,
2930
"formatterOptions": {
3031
"IncludeScopes": false,
3132
"SingleLine": false,

src/Bleess.Extensions.Logging.File/Bleess.Extensions.Logging.File.csproj

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFrameworks>net8.0;net7.0;net6.0;net5.0</TargetFrameworks>
4+
<TargetFrameworks>net9.0;net8.0;net6.0</TargetFrameworks>
55
<LangVersion>latest</LangVersion>
66

77
<RepoRoot>$([System.IO.Path]::GetDirectoryName($([MSBuild]::GetPathOfFileAbove('.gitignore', '$(MSBuildThisFileDirectory)'))))</RepoRoot>
@@ -10,7 +10,7 @@
1010

1111
<BuildNum>1</BuildNum>
1212

13-
<VersionPrefix>2.0.1</VersionPrefix>
13+
<VersionPrefix>3.0.0</VersionPrefix>
1414
<VersionSuffix></VersionSuffix>
1515

1616
<FileVersion>$(VersionPrefix).$(BuildNum)</FileVersion>
@@ -32,7 +32,7 @@
3232
How to use: https://github.com/pableess/Bleess.Extensions.Logging.File
3333

3434
- Basic file logging features: rolling files by date, maximum file length, max number of files, etc
35-
- Simple text and json formatters with options for single line, multi-line, and separated log statements
35+
- Simple text and json formatters with options for sing7le line, multi-line, and separated log statements
3636
- Support for logging scopes
3737
- Support for idiomatic configuration via IConfiguration sources or in code
3838
- Ability to change settings without restarting application
@@ -78,33 +78,21 @@
7878
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All" />
7979
</ItemGroup>
8080

81-
<ItemGroup Condition="$(TargetFramework) == 'netstandard2.0'">
82-
<PackageReference Include="Microsoft.Extensions.Logging" Version="3.0.0" />
83-
<PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="3.0.0" />
84-
<PackageReference Include="System.Text.Json" Version="4.7.2" />
81+
<ItemGroup Condition="$(TargetFramework) == 'net9.0'">
82+
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />
83+
<PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="9.0.0" />
84+
<PackageReference Include="System.Text.Json" Version="9.0.0" />
8585
</ItemGroup>
8686

8787
<ItemGroup Condition="$(TargetFramework) == 'net8.0'">
8888
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
8989
<PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="8.0.0" />
90-
<PackageReference Include="System.Text.Json" Version="8.0.1" />
91-
</ItemGroup>
92-
93-
<ItemGroup Condition="$(TargetFramework) == 'net7.0'">
94-
<PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
95-
<PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="7.0.0" />
96-
<PackageReference Include="System.Text.Json" Version="7.0.4" />
90+
<PackageReference Include="System.Text.Json" Version="8.0.5" />
9791
</ItemGroup>
9892

9993
<ItemGroup Condition="$(TargetFramework) == 'net6.0'">
10094
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
10195
<PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="6.0.0" />
102-
<PackageReference Include="System.Text.Json" Version="6.0.9" />
103-
</ItemGroup>
104-
105-
<ItemGroup Condition="$(TargetFramework) == 'net5.0'">
106-
<PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />
107-
<PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="5.0.0" />
108-
<PackageReference Include="System.Text.Json" Version="5.0.2" />
96+
<PackageReference Include="System.Text.Json" Version="6.0.11" />
10997
</ItemGroup>
11098
</Project>

src/Bleess.Extensions.Logging.File/FileLogger.cs

Lines changed: 45 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -9,67 +9,65 @@ namespace Bleess.Extensions.Logging.File
99
{
1010
// adapted from ConsoleLogger from https://github.com/dotnet/runtime
1111
class FileLogger : ILogger
12-
{
13-
private readonly string _name;
14-
private readonly FileLoggerProcessor _queueProcessor;
12+
{
13+
private readonly string _name;
14+
private readonly FileLoggerProcessor _queueProcessor;
1515

16-
internal FileLogger(string name, FileLoggerProcessor loggerProcessor)
16+
internal FileLogger(string name, FileLoggerProcessor loggerProcessor)
17+
{
18+
if (name == null)
1719
{
18-
if (name == null)
19-
{
20-
throw new ArgumentNullException(nameof(name));
21-
}
22-
23-
_name = name;
24-
_queueProcessor = loggerProcessor;
20+
throw new ArgumentNullException(nameof(name));
2521
}
2622

27-
internal FileFormatter Formatter { get; set; }
23+
_name = name;
24+
_queueProcessor = loggerProcessor;
25+
}
2826

29-
/// fallback setting to include scopes, deprecated API for setting this on the log provider
30-
internal bool? FallbackIncludeScope { get; set; }
31-
32-
internal string SubProviderName { get; set; }
27+
internal FileFormatter Formatter { get; set; }
3328

34-
internal IExternalScopeProvider ScopeProvider { get; set; }
29+
/// fallback setting to include scopes, deprecated API for setting this on the log provider
30+
internal bool? FallbackIncludeScope { get; set; }
3531

36-
[ThreadStatic]
37-
private static StringWriter t_stringWriter;
32+
internal string SubProviderName { get; set; }
3833

39-
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
40-
{
41-
if (!IsEnabled(logLevel))
42-
{
43-
return;
44-
}
45-
if (formatter == null)
46-
{
47-
throw new ArgumentNullException(nameof(formatter));
48-
}
49-
t_stringWriter ??= new StringWriter();
50-
LogEntry<TState> logEntry = new LogEntry<TState>(logLevel, _name, eventId, state, exception, formatter);
51-
Formatter.Write(in logEntry, ScopeProvider, t_stringWriter, SubProviderName, FallbackIncludeScope);
34+
internal IExternalScopeProvider ScopeProvider { get; set; }
35+
36+
[ThreadStatic]
37+
private static StringWriter t_stringWriter;
5238

53-
var sb = t_stringWriter.GetStringBuilder();
54-
if (sb.Length == 0)
55-
{
56-
return;
57-
}
58-
string computedAnsiString = sb.ToString();
59-
sb.Clear();
60-
if (sb.Capacity > 1024)
61-
{
62-
sb.Capacity = 1024;
63-
}
64-
_queueProcessor.EnqueueMessage(new LogMessageEntry(computedAnsiString));
39+
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
40+
{
41+
if (!IsEnabled(logLevel))
42+
{
43+
return;
44+
}
45+
if (formatter == null)
46+
{
47+
throw new ArgumentNullException(nameof(formatter));
6548
}
49+
t_stringWriter ??= new StringWriter();
50+
LogEntry<TState> logEntry = new LogEntry<TState>(logLevel, _name, eventId, state, exception, formatter);
51+
Formatter.Write(in logEntry, ScopeProvider, t_stringWriter, SubProviderName, FallbackIncludeScope);
6652

67-
public bool IsEnabled(LogLevel logLevel)
53+
var sb = t_stringWriter.GetStringBuilder();
54+
if (sb.Length == 0)
6855
{
69-
return logLevel != LogLevel.None;
56+
return;
7057
}
58+
string computedAnsiString = sb.ToString();
59+
sb.Clear();
60+
if (sb.Capacity > 1024)
61+
{
62+
sb.Capacity = 1024;
63+
}
64+
var msg = new LogMessageEntry(computedAnsiString);
65+
_queueProcessor.EnqueueMessage(ref msg);
66+
}
67+
68+
public bool IsEnabled(LogLevel logLevel) => logLevel != LogLevel.None;
7169

72-
public IDisposable BeginScope<TState>(TState state) => ScopeProvider?.Push(state) ?? NullScope.Instance;
70+
public IDisposable BeginScope<TState>(TState state) => ScopeProvider?.Push(state) ?? NullScope.Instance;
7371
}
7472

7573
internal class NullScope : IDisposable

src/Bleess.Extensions.Logging.File/FileLoggerOptions.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,18 @@ public FileLoggerOptions()
6262
/// </summary>
6363
public bool Append { get; set; } = true;
6464

65+
/// <summary>
66+
/// If true, forces the flush operations to ensure that all data is written to the underlying disk (ie fsync on linux)
67+
/// </summary>
68+
public bool FlushToDisk { get; set; } = false;
69+
70+
/// <summary>
71+
/// In high throughput scenarios, log messages are queued up and written from a background thread.
72+
/// By default, flushing of the log file to disk only happens when the queue is emptied. This setting, ensures that
73+
/// a flush to disk will happen if a previous flush has not occured within this interval. Setting is in MS
74+
/// </summary>
75+
public int? MaxFlushInterval { get; set; }
76+
6577
internal long MaxFileSizeInBytes => (long)(MaxFileSizeInMB * 1024 * 1024);
6678
}
6779
}

0 commit comments

Comments
 (0)