Merge pull request #957 from ZachBray/feature/dtos

[C#,C++,Java] Generate DTOs for non-perf-sensitive usecases.

Author: ZachBray

.github/workflows/codeql.yml

@@ -88,7 +88,7 @@ jobs:
       fail-fast: false
       matrix:
         language: [ 'csharp' ]
-        dotnet: [ '3.1.x' ]
+        dotnet: [ '8.0.x' ]
     env:
       DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
       DOTNET_CLI_TELEMETRY_OPTOUT: 1

.github/workflows/slow.yml

@@ -0,0 +1,62 @@
+name: Slow checks
+
+on:
+  workflow_dispatch:
+    branches:
+      - '**'
+  schedule:
+    - cron: '0 12 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  GRADLE_OPTS: '-Dorg.gradle.daemon=false -Dorg.gradle.java.installations.auto-detect=false -Dorg.gradle.warning.mode=fail'
+
+permissions:
+  contents: read
+
+jobs:
+  property-tests:
+    name: Property tests
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        java: [ '21' ]
+        dotnet: [ '8.0.x' ]
+    env:
+      DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
+      DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Setup java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ matrix.java }}
+      - name: Setup BUILD_JAVA_HOME & BUILD_JAVA_VERSION
+        run: |
+          java -Xinternalversion
+          echo "BUILD_JAVA_HOME=${JAVA_HOME}" >> $GITHUB_ENV
+          echo "BUILD_JAVA_VERSION=${{ matrix.java }}" >> $GITHUB_ENV
+      - name: Setup java 8 to run the Gradle script
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: 8
+      - name: Setup dotnet
+        uses: actions/setup-dotnet@v2
+        with:
+          dotnet-version: ${{ matrix.dotnet }}
+      - name: Build .NET library
+        run: ./csharp/build.sh
+      - name: Run property tests
+        run: ./gradlew propertyTest
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          name: property-tests
+          path: sbe-tool/build/reports/tests/propertyTest

.gitignore

@@ -119,4 +119,7 @@ rust/Cargo.lock
 .DS_Store
 /sbe-tool/src/main/golang/uk_co_real_logic_sbe_ir_generated/
 
+# JQwik
+*.jqwik-database
+
 /generated/

build.gradle

@@ -31,6 +31,7 @@ buildscript {
 
 plugins {
     id 'java-library'
+    id 'jvm-test-suite'
     id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
     id 'com.github.ben-manes.versions' version '0.51.0'
 }
@@ -57,6 +58,8 @@ def checkstyleVersion = '9.3'
 def hamcrestVersion = '2.2'
 def mockitoVersion = '4.11.0'
 def junitVersion = '5.10.2'
+def jqwikVersion = '1.8.1'
+def jsonVersion = '20230618'
 def jmhVersion = '1.37'
 def agronaVersion = '1.21.2'
 def agronaVersionRange = '[1.21.2,2.0[' // allow any release >= 1.21.2 and < 2.0.0
@@ -164,6 +167,7 @@ jar.enabled = false
 
 subprojects {
     apply plugin: 'java-library'
+    apply plugin: 'jvm-test-suite'
     apply plugin: 'checkstyle'
 
     group = sbeGroup
@@ -216,22 +220,34 @@ subprojects {
         }
     }
 
-    test {
-        useJUnitPlatform()
+    testing {
+        suites {
+            test {
+                useJUnitJupiter junitVersion
 
-        testLogging {
-            for (def level : LogLevel.values())
-            {
-                def testLogging = get(level)
-                testLogging.exceptionFormat = 'full'
-                testLogging.events = ["FAILED", "STANDARD_OUT", "STANDARD_ERROR"]
-            }
-        }
+                targets {
+                    all {
+                        testTask.configure {
+                            useJUnitPlatform()
 
-        javaLauncher.set(toolchainLauncher)
+                            testLogging {
+                                for (def level : LogLevel.values())
+                                {
+                                    def testLogging = get(level)
+                                    testLogging.exceptionFormat = 'full'
+                                    testLogging.events = ["FAILED", "STANDARD_OUT", "STANDARD_ERROR"]
+                                }
+                            }
+
+                            javaLauncher.set(toolchainLauncher)
 
-        systemProperty 'sbe.enable.ir.precedence.checks', 'true'
-        systemProperty 'sbe.enable.test.precedence.checks', 'true'
+                            systemProperty 'sbe.enable.ir.precedence.checks', 'true'
+                            systemProperty 'sbe.enable.test.precedence.checks', 'true'
+                        }
+                    }
+                }
+            }
+        }
     }
 }
 
@@ -248,11 +264,6 @@ project(':sbe-tool') {
                 prefer(agronaVersion)
             }
         }
-        testImplementation files('build/classes/java/generated')
-        testImplementation "org.hamcrest:hamcrest:${hamcrestVersion}"
-        testImplementation "org.mockito:mockito-core:${mockitoVersion}"
-        testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}"
-        testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}"
     }
 
     def generatedDir = 'build/generated-src'
@@ -269,11 +280,55 @@ project(':sbe-tool') {
 
     compileGeneratedJava {
         dependsOn 'generateTestCodecs'
+        dependsOn 'generateTestDtos'
         classpath += sourceSets.main.runtimeClasspath
     }
 
     compileTestJava.dependsOn compileGeneratedJava
 
+    testing {
+        suites {
+            test {
+                dependencies {
+                    implementation files('build/classes/java/generated')
+                    implementation "org.hamcrest:hamcrest:${hamcrestVersion}"
+                    implementation "org.mockito:mockito-core:${mockitoVersion}"
+                    implementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}"
+                }
+            }
+
+            propertyTest(JvmTestSuite) {
+                // We should be able to use _only_ the JQwik engine, but this issue is outstanding:
+                // https://github.com/gradle/gradle/issues/21299
+                useJUnitJupiter junitVersion
+
+                dependencies {
+                    implementation project()
+                    implementation("net.jqwik:jqwik:${jqwikVersion}") {
+                        // Exclude JUnit 5 dependencies that are already provided due to useJUnitJupiter
+                        exclude group: 'org.junit.platform', module: 'junit-platform-commons'
+                        exclude group: 'org.junit.platform', module: 'junit-platform-engine'
+                    }
+                    implementation "org.json:json:${jsonVersion}"
+                }
+
+
+                targets {
+                    all {
+                        testTask.configure {
+                            minHeapSize = '2g'
+                            maxHeapSize = '2g'
+
+                            javaLauncher.set(toolchainLauncher)
+
+                            systemProperty 'sbe.dll', "${rootProject.projectDir}/csharp/sbe-dll/bin/Release/netstandard2.0/SBE.dll"
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     tasks.register('generateTestCodecs', JavaExec) {
         dependsOn 'compileJava'
         mainClass.set('uk.co.real_logic.sbe.SbeTool')
@@ -290,6 +345,21 @@ project(':sbe-tool') {
                 'src/test/resources/field-order-check-schema.xml']
     }
 
+    tasks.register('generateTestDtos', JavaExec) {
+        dependsOn 'compileJava'
+        mainClass.set('uk.co.real_logic.sbe.SbeTool')
+        classpath = sourceSets.main.runtimeClasspath
+        systemProperties(
+                'sbe.output.dir': generatedDir,
+                'sbe.target.language': 'java',
+                'sbe.validation.stop.on.error': 'true',
+                'sbe.validation.xsd': validationXsdPath,
+                'sbe.generate.precedence.checks': 'true',
+                'sbe.java.precedence.checks.property.name': 'sbe.enable.test.precedence.checks',
+                'sbe.java.generate.dtos': 'true')
+        args = ['src/test/resources/example-extension-schema.xml']
+    }
+
     jar {
         manifest.attributes(
             'Specification-Title': 'Simple Binary Encoding',
@@ -735,7 +805,7 @@ tasks.register('generateCSharpCodecsWithXIncludes', JavaExec) {
             'sbe-samples/src/main/resources/example-extension-schema.xml']
 }
 
-tasks.register('generateCSharpCodecsTests', JavaExec) {
+tasks.register('generateCSharpTestCodecs', JavaExec) {
     mainClass.set('uk.co.real_logic.sbe.SbeTool')
     classpath = project(':sbe-tool').sourceSets.main.runtimeClasspath
     systemProperties(
@@ -754,9 +824,21 @@ tasks.register('generateCSharpCodecsTests', JavaExec) {
             'sbe-benchmarks/src/main/resources/fix-message-samples.xml']
 }
 
+tasks.register('generateCSharpTestDtos', JavaExec) {
+    mainClass.set('uk.co.real_logic.sbe.SbeTool')
+    classpath = project(':sbe-tool').sourceSets.main.runtimeClasspath
+    systemProperties(
+            'sbe.output.dir': 'csharp/sbe-generated',
+            'sbe.target.language': 'uk.co.real_logic.sbe.generation.csharp.CSharpDtos',
+            'sbe.xinclude.aware': 'true',
+            'sbe.validation.xsd': validationXsdPath)
+    args = ['sbe-samples/src/main/resources/example-extension-schema.xml']
+}
+
 tasks.register('generateCSharpCodecs') {
     description = 'Generate csharp codecs'
-    dependsOn 'generateCSharpCodecsTests',
+    dependsOn 'generateCSharpTestCodecs',
+            'generateCSharpTestDtos',
             'generateCSharpCodecsWithXIncludes'
 }
 

config/checkstyle/suppressions.xml

@@ -6,5 +6,6 @@
     <suppress files=".*generated-src.*" checks="."/>
     <suppress files=".*generated.*" checks="."/>
     <suppress files="[\\/]test[\\/]" checks="MissingJavadoc.*"/>
+    <suppress files="[\\/]propertyTest[\\/]" checks="MissingJavadoc.*"/>
     <suppress files="[\\/]sbe-benchmarks[\\/]" checks="MissingJavadoc.*"/>
 </suppressions>

csharp/sbe-dll/DirectBuffer.cs

@@ -694,21 +694,16 @@ public int SetBytes(int index, ReadOnlySpan<byte> src)
 
         /// <summary>
         /// Writes a string into the underlying buffer, encoding using the provided <see cref="System.Text.Encoding"/>.
-        /// If there is not enough room in the buffer for the bytes it will throw IndexOutOfRangeException.
         /// </summary>
         /// <param name="encoding">encoding to use to write the bytes from the string</param>
         /// <param name="src">source string</param>
         /// <param name="index">index in the underlying buffer to start writing bytes</param>
         /// <returns>count of bytes written</returns>
         public unsafe int SetBytesFromString(Encoding encoding, string src, int index)
         {
-           int available = _capacity - index;
            int byteCount = encoding.GetByteCount(src);
 
-           if (byteCount > available)
-           {
-               ThrowHelper.ThrowIndexOutOfRangeException(_capacity);
-           }
+           CheckLimit(index + byteCount);
 
            fixed (char* ptr = src)
            {