From e6a0e6bb6d7c844d8a473bbe73896af9e9b6957f Mon Sep 17 00:00:00 2001 From: "Alan D. Cabrera" Date: Tue, 23 Apr 2019 09:22:43 -0700 Subject: [PATCH] Add support for multiline strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - backticks, `, and triple quotes, “””, now allow multiline strings - split strings into STRING_NAME and STRING_VALUE --- .../protostuff/compiler/parser/ProtoLexer.g4 | 21 +++- .../protostuff/compiler/parser/ProtoParser.g4 | 10 +- .../compiler/model/DynamicMessage.java | 2 +- .../compiler/parser/MessageParseListener.java | 2 +- .../compiler/parser/OptionParseListener.java | 12 ++- .../compiler/parser/ProtoParseListener.java | 4 +- .../io/protostuff/compiler/parser/Util.java | 30 ++++++ .../protostuff/compiler/parser/UtilTest.java | 14 ++- .../protobuf_unittest/unittest.proto | 5 + .../unittest_custom_options.proto | 99 +++++++++++++------ 10 files changed, 153 insertions(+), 46 deletions(-) diff --git a/protostuff-parser/src/main/antlr4/io/protostuff/compiler/parser/ProtoLexer.g4 b/protostuff-parser/src/main/antlr4/io/protostuff/compiler/parser/ProtoLexer.g4 index 65de41fb..9fe9d1e6 100644 --- a/protostuff-parser/src/main/antlr4/io/protostuff/compiler/parser/ProtoLexer.g4 +++ b/protostuff-parser/src/main/antlr4/io/protostuff/compiler/parser/ProtoLexer.g4 @@ -182,6 +182,10 @@ IDENT : (ALPHA | UNDERSCORE) (ALPHA | DIGIT | UNDERSCORE)* ; STRING_VALUE + : BACKTICK_STRING + | TRIPLE_DOUBLE_QUOTE_STRING + ; +STRING_NAME : DOUBLE_QUOTE_STRING | SINGLE_QUOTE_STRING ; @@ -202,6 +206,17 @@ fragment DOUBLE_QUOTE_STRING fragment SINGLE_QUOTE_STRING : '\'' ( ESC_SEQ | ~('\\' | '\'' | '\r' | '\n') )* '\'' ; +fragment BACKTICK_STRING + : '`' ( ESC_SEQ | ~('\\' | '`') )* '`' + ; +fragment TRIPLE_DOUBLE_QUOTE_STRING + : '"""' TDQ_STRING_CHAR*? '"""' + ; +fragment TDQ_STRING_CHAR + : ~["\\] + | '"' { !(_input.LA(1) == '"' && _input.LA(2) == '"') }? + | ESC_SEQ + ; fragment EXPONENT : (FLOAT_LIT|DEC_VALUE) EXP DEC_VALUE ; @@ -212,7 +227,7 @@ fragment FLOAT_LIT fragment INF : 'inf' ; -fragment NAN +fragment NAN : 'nan' ; fragment EXP @@ -247,7 +262,7 @@ fragment UNDERSCORE : '_' ; fragment ESC_SEQ - : '\\' ('a'|'v'|'b'|'t'|'n'|'f'|'r'|'?'|'"'|'\''|'\\') + : '\\' ('a'|'v'|'b'|'t'|'n'|'f'|'r'|'?'|'"'|'\''|'\\'|'`') | '\\' ('x'|'X') HEX_DIGIT HEX_DIGIT | UNICODE_ESC | OCTAL_ESC @@ -269,4 +284,4 @@ fragment UNICODE_ESC */ ERRCHAR : . -> channel(HIDDEN) - ; \ No newline at end of file + ; diff --git a/protostuff-parser/src/main/antlr4/io/protostuff/compiler/parser/ProtoParser.g4 b/protostuff-parser/src/main/antlr4/io/protostuff/compiler/parser/ProtoParser.g4 index 68795fa9..ee7a2d42 100644 --- a/protostuff-parser/src/main/antlr4/io/protostuff/compiler/parser/ProtoParser.g4 +++ b/protostuff-parser/src/main/antlr4/io/protostuff/compiler/parser/ProtoParser.g4 @@ -20,7 +20,7 @@ syntaxStatement : SYNTAX ASSIGN syntaxName SEMICOLON ; syntaxName - : STRING_VALUE + : STRING_NAME ; packageStatement : PACKAGE packageName SEMICOLON @@ -32,7 +32,7 @@ importStatement : IMPORT PUBLIC? fileReference SEMICOLON ; fileReference - : STRING_VALUE + : STRING_NAME ; optionEntry : OPTION option SEMICOLON @@ -160,7 +160,7 @@ reservedFieldNames : RESERVED reservedFieldName (COMMA reservedFieldName)* SEMICOLON ; reservedFieldName - : STRING_VALUE + : STRING_NAME ; field : fieldModifier? typeReference fieldName ASSIGN tag fieldOptions? SEMICOLON @@ -211,6 +211,7 @@ optionValue : INTEGER_VALUE | FLOAT_VALUE | BOOLEAN_VALUE + | STRING_NAME | STRING_VALUE | IDENT | textFormat @@ -230,6 +231,7 @@ textFormatOptionValue : INTEGER_VALUE | FLOAT_VALUE | BOOLEAN_VALUE + | STRING_NAME | STRING_VALUE | IDENT ; @@ -276,4 +278,4 @@ ident | BOOL | STRING | BYTES - ; \ No newline at end of file + ; diff --git a/protostuff-parser/src/main/java/io/protostuff/compiler/model/DynamicMessage.java b/protostuff-parser/src/main/java/io/protostuff/compiler/model/DynamicMessage.java index 977f2a41..cdeafc6e 100644 --- a/protostuff-parser/src/main/java/io/protostuff/compiler/model/DynamicMessage.java +++ b/protostuff-parser/src/main/java/io/protostuff/compiler/model/DynamicMessage.java @@ -148,7 +148,7 @@ private void merge(DynamicMessage message) { private Key createKey(String fieldName) { Key key; if (fieldName.startsWith("(")) { - String name = Util.removeFirstAndLastChar(fieldName); + String name = Util.trimStringName(fieldName); if (name.startsWith(".")) { name = name.substring(1); } diff --git a/protostuff-parser/src/main/java/io/protostuff/compiler/parser/MessageParseListener.java b/protostuff-parser/src/main/java/io/protostuff/compiler/parser/MessageParseListener.java index 9d034f44..ef8cbeab 100644 --- a/protostuff-parser/src/main/java/io/protostuff/compiler/parser/MessageParseListener.java +++ b/protostuff-parser/src/main/java/io/protostuff/compiler/parser/MessageParseListener.java @@ -72,7 +72,7 @@ public void exitReservedFieldNames(ProtoParser.ReservedFieldNamesContext ctx) { UserType userType = context.peek(UserType.class); for (ProtoParser.ReservedFieldNameContext fieldNameContext : ctx.reservedFieldName()) { String fieldName = fieldNameContext.getText(); - fieldName = Util.removeFirstAndLastChar(fieldName); + fieldName = Util.trimStringName(fieldName); userType.addReservedFieldName(fieldName); } } diff --git a/protostuff-parser/src/main/java/io/protostuff/compiler/parser/OptionParseListener.java b/protostuff-parser/src/main/java/io/protostuff/compiler/parser/OptionParseListener.java index 3c7ab1ce..bb3dc3c6 100644 --- a/protostuff-parser/src/main/java/io/protostuff/compiler/parser/OptionParseListener.java +++ b/protostuff-parser/src/main/java/io/protostuff/compiler/parser/OptionParseListener.java @@ -52,10 +52,14 @@ private DynamicMessage.Value getOptionValue(ProtoParser.OptionValueContext optio } else if (optionValueContext.INTEGER_VALUE() != null) { String text = optionValueContext.INTEGER_VALUE().getText(); optionValue = parseInteger(sourceCodeLocation, text); + } else if (optionValueContext.STRING_NAME() != null) { + String text = optionValueContext.STRING_NAME().getText(); + // TODO: unescape + optionValue = DynamicMessage.Value.createString(sourceCodeLocation, Util.trimStringName(text)); } else if (optionValueContext.STRING_VALUE() != null) { String text = optionValueContext.STRING_VALUE().getText(); // TODO: unescape - optionValue = DynamicMessage.Value.createString(sourceCodeLocation, Util.removeFirstAndLastChar(text)); + optionValue = DynamicMessage.Value.createString(sourceCodeLocation, Util.trimStringValue(text)); } else if (optionValueContext.IDENT() != null) { String text = optionValueContext.IDENT().getText(); optionValue = DynamicMessage.Value.createEnum(sourceCodeLocation, text); @@ -142,10 +146,14 @@ private DynamicMessage.Value getTextFormatOptionValue(ProtoParser.TextFormatEntr } else if (ctx.textFormatOptionValue().INTEGER_VALUE() != null) { String text = ctx.textFormatOptionValue().INTEGER_VALUE().getText(); optionValue = parseInteger(sourceCodeLocation, text); + } else if (ctx.textFormatOptionValue().STRING_NAME() != null) { + String text = ctx.textFormatOptionValue().STRING_NAME().getText(); + // TODO: unescape + optionValue = DynamicMessage.Value.createString(sourceCodeLocation, Util.trimStringName(text)); } else if (ctx.textFormatOptionValue().STRING_VALUE() != null) { String text = ctx.textFormatOptionValue().STRING_VALUE().getText(); // TODO: unescape - optionValue = DynamicMessage.Value.createString(sourceCodeLocation, Util.removeFirstAndLastChar(text)); + optionValue = DynamicMessage.Value.createString(sourceCodeLocation, Util.trimStringValue(text)); } else if (ctx.textFormatOptionValue().IDENT() != null) { String text = ctx.textFormatOptionValue().IDENT().getText(); optionValue = DynamicMessage.Value.createEnum(sourceCodeLocation, text); diff --git a/protostuff-parser/src/main/java/io/protostuff/compiler/parser/ProtoParseListener.java b/protostuff-parser/src/main/java/io/protostuff/compiler/parser/ProtoParseListener.java index 8ee7d45a..bed5a07a 100644 --- a/protostuff-parser/src/main/java/io/protostuff/compiler/parser/ProtoParseListener.java +++ b/protostuff-parser/src/main/java/io/protostuff/compiler/parser/ProtoParseListener.java @@ -74,7 +74,7 @@ private boolean isCommentBlockOwner(Token token) { public void exitSyntaxStatement(ProtoParser.SyntaxStatementContext ctx) { Proto proto = context.peek(Proto.class); String text = ctx.syntaxName().getText(); - String value = Util.removeFirstAndLastChar(text); + String value = Util.trimStringName(text); Syntax syntax = new Syntax(proto, value); syntax.setSourceCodeLocation(getSourceCodeLocation(ctx)); proto.setSyntax(syntax); @@ -93,7 +93,7 @@ public void exitPackageStatement(ProtoParser.PackageStatementContext ctx) { public void exitImportStatement(ProtoParser.ImportStatementContext ctx) { Proto proto = context.peek(Proto.class); String text = ctx.fileReference().getText(); - String fileName = Util.removeFirstAndLastChar(text); + String fileName = Util.trimStringName(text); Import anImport = new Import(proto, fileName, ctx.PUBLIC() != null); anImport.setSourceCodeLocation(getSourceCodeLocation(ctx)); proto.addImport(anImport); diff --git a/protostuff-parser/src/main/java/io/protostuff/compiler/parser/Util.java b/protostuff-parser/src/main/java/io/protostuff/compiler/parser/Util.java index fb7dd9d1..2b927e14 100644 --- a/protostuff-parser/src/main/java/io/protostuff/compiler/parser/Util.java +++ b/protostuff-parser/src/main/java/io/protostuff/compiler/parser/Util.java @@ -1,6 +1,7 @@ package io.protostuff.compiler.parser; import com.google.common.base.Preconditions; + import javax.annotation.ParametersAreNonnullByDefault; /** @@ -26,13 +27,42 @@ private Util() { * @param text given string * * @return substring of given string - without first and last characters + * @deprecated replaced by {@link #trimStringName(String)} */ public static String removeFirstAndLastChar(String text) { + return trimStringName(text); + } + + /** + * Remove first and last character from given string name and return result. + * + * @param text given string + * + * @return substring of given string - without first and last characters + */ + public static String trimStringName(String text) { Preconditions.checkNotNull(text, "text can not be null"); int n = text.length(); return text.substring(1, n - 1); } + /** + * Remove first and last character from given string value and return result. + * + * A string value can be wrapped by either a backtick, `, or triple double + * quote, """. + * + * @param text given string value + * + * @return substring of given string - without first and last characters + */ + public static String trimStringValue(String text) { + Preconditions.checkNotNull(text, "text can not be null"); + int size = text.startsWith("`") ? 1 : 3; + int n = text.length(); + return text.substring(size, n - size); + } + /** * Returns file name by given absolute or relative file location. * TODO: remove unused? diff --git a/protostuff-parser/src/test/java/io/protostuff/compiler/parser/UtilTest.java b/protostuff-parser/src/test/java/io/protostuff/compiler/parser/UtilTest.java index 861b76ee..aaee8395 100644 --- a/protostuff-parser/src/test/java/io/protostuff/compiler/parser/UtilTest.java +++ b/protostuff-parser/src/test/java/io/protostuff/compiler/parser/UtilTest.java @@ -2,7 +2,8 @@ import org.junit.jupiter.api.Test; -import static io.protostuff.compiler.parser.Util.removeFirstAndLastChar; +import static io.protostuff.compiler.parser.Util.trimStringName; +import static io.protostuff.compiler.parser.Util.trimStringValue; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -11,8 +12,13 @@ public class UtilTest { @Test - public void testRemoveQuotes_expected() throws Exception { - assertEquals("abc", removeFirstAndLastChar("\"abc\"")); + public void testTrimStringName_expected() throws Exception { + assertEquals("abc", trimStringName("\"abc\"")); } -} \ No newline at end of file + @Test + public void testTrimStringValue_expected() throws Exception { + assertEquals("abc", trimStringValue("`abc`")); + assertEquals("abc", trimStringValue("\"\"\"abc\"\"\"")); + } +} diff --git a/protostuff-parser/src/test/resources/protobuf_unittest/unittest.proto b/protostuff-parser/src/test/resources/protobuf_unittest/unittest.proto index c5aafcb3..6f93feaf 100644 --- a/protostuff-parser/src/test/resources/protobuf_unittest/unittest.proto +++ b/protostuff-parser/src/test/resources/protobuf_unittest/unittest.proto @@ -161,6 +161,8 @@ message TestAllTypes { optional bool default_bool = 73 [default = true]; optional string default_string = 74 [default = "hello"]; optional bytes default_bytes = 75 [default = "world"]; + optional string default_m_string = 76 [default = `hello +world`]; optional NestedEnum default_nested_enum = 81 [default = BAR]; optional ForeignEnum default_foreign_enum = 82 [default = FOREIGN_BAR]; @@ -560,6 +562,9 @@ message TestExtremeDefaultValues { optional string string_piece_with_zero = 25 [ctype = STRING_PIECE, default = "ab\000c"]; optional string cord_with_zero = 26 [ctype = CORD, default = "12\0003"]; optional string replacement_string = 27 [default = "${unknown}"]; + optional string tick_multiline_string = 28 [default=`backtick +multiline +default`]; } message SparseEnumMessage { diff --git a/protostuff-parser/src/test/resources/protobuf_unittest/unittest_custom_options.proto b/protostuff-parser/src/test/resources/protobuf_unittest/unittest_custom_options.proto index 34a68dda..607329db 100644 --- a/protostuff-parser/src/test/resources/protobuf_unittest/unittest_custom_options.proto +++ b/protostuff-parser/src/test/resources/protobuf_unittest/unittest_custom_options.proto @@ -150,21 +150,22 @@ message DummyMessageInvalidAsOptionType { } extend google.protobuf.MessageOptions { - optional bool bool_opt = 7706090; - optional int32 int32_opt = 7705709; - optional int64 int64_opt = 7705542; - optional uint32 uint32_opt = 7704880; - optional uint64 uint64_opt = 7702367; - optional sint32 sint32_opt = 7701568; - optional sint64 sint64_opt = 7700863; - optional fixed32 fixed32_opt = 7700307; - optional fixed64 fixed64_opt = 7700194; - optional sfixed32 sfixed32_opt = 7698645; - optional sfixed64 sfixed64_opt = 7685475; - optional float float_opt = 7675390; - optional double double_opt = 7673293; - optional string string_opt = 7673285; - optional bytes bytes_opt = 7673238; + optional bool bool_opt = 7706090; + optional int32 int32_opt = 7705709; + optional int64 int64_opt = 7705542; + optional uint32 uint32_opt = 7704880; + optional uint64 uint64_opt = 7702367; + optional sint32 sint32_opt = 7701568; + optional sint64 sint64_opt = 7700863; + optional fixed32 fixed32_opt = 7700307; + optional fixed64 fixed64_opt = 7700194; + optional sfixed32 sfixed32_opt = 7698645; + optional sfixed64 sfixed64_opt = 7685475; + optional float float_opt = 7675390; + optional double double_opt = 7673293; + optional string string_opt = 7673285; + optional string mstring_opt = 7673286; + optional bytes bytes_opt = 7673238; optional DummyMessageContainingEnum.TestEnumType enum_opt = 7673233; optional DummyMessageInvalidAsOptionType message_type_opt = 7665967; } @@ -198,12 +199,14 @@ message CustomOptionMaxIntegerValues { } message CustomOptionOtherValues { - option (int32_opt) = -100; // To test sign-extension. - option (float_opt) = 12.3456789; - option (double_opt) = 1.234567890123456789; - option (string_opt) = "Hello, \"World\""; - option (bytes_opt) = "Hello\0World"; - option (enum_opt) = TEST_OPTION_ENUM_TYPE2; + option (int32_opt) = -100; // To test sign-extension. + option (float_opt) = 12.3456789; + option (double_opt) = 1.234567890123456789; + option (string_opt) = "Hello, \"World\""; + option (mstring_opt) = `Hello +World`; + option (bytes_opt) = "Hello\0World"; + option (enum_opt) = TEST_OPTION_ENUM_TYPE2; } message SettingRealsFromPositiveInts { @@ -311,12 +314,14 @@ message AggregateMessageSetElement { optional AggregateMessageSetElement message_set_extension = 15447542; } optional string s = 1; + optional string ms = 2; } // A helper type used to test aggregate option parsing message Aggregate { optional int32 i = 1; optional string s = 2; + optional string ms = 6; // A nested object optional Aggregate sub = 3; @@ -358,16 +363,25 @@ extend google.protobuf.MethodOptions { // Try using AggregateOption at different points in the proto grammar option (fileopt) = { s: 'FileAnnotation' + ms: `multiline +FileAnnotation` + // Also test the handling of comments /* of both types */ i: 100 - sub { s: 'NestedFileAnnotation' } + sub { + s: 'NestedFileAnnotation' + ms: `multiline +NestedFileAnnotation` + } // Include a google.protobuf.FileOptions and recursively extend it with // another fileopt. file { [protobuf_unittest.fileopt] { s:'FileExtensionAnnotation' + ms: `multiline +FileExtensionAnnotation` } } @@ -375,25 +389,52 @@ option (fileopt) = { mset { [protobuf_unittest.AggregateMessageSetElement.message_set_extension] { s: 'EmbeddedMessageSetElement' + ms: `multiline +EmbeddedMessageSetElement` } } }; message AggregateMessage { - option (msgopt) = { i:101 s:'MessageAnnotation' }; - optional int32 fieldname = 1 [(fieldopt) = { s:'FieldAnnotation' }]; + option (msgopt) = { + i: 101 + s: 'MessageAnnotation' + ms: `multiline +MessageAnnotation` + }; + optional int32 fieldname = 1 [(fieldopt) = { + s: 'FieldAnnotation' + ms: `multiline +FieldAnnotation` + }]; } service AggregateService { - option (serviceopt) = { s:'ServiceAnnotation' }; + option (serviceopt) = { + s: 'ServiceAnnotation' + ms: `multiline +ServiceAnnotation` + }; rpc Method (AggregateMessage) returns (AggregateMessage) { - option (methodopt) = { s:'MethodAnnotation' }; + option (methodopt) = { + s: 'MethodAnnotation' + ms: `multiline +MethodAnnotation` + }; } } enum AggregateEnum { - option (enumopt) = { s:'EnumAnnotation' }; - VALUE = 1 [(enumvalopt) = { s:'EnumValueAnnotation' }]; + option (enumopt) = { + s: 'EnumAnnotation' + ms: `multiline +EnumAnnotation` + }; + VALUE = 1 [(enumvalopt) = { + s: 'EnumValueAnnotation' + ms: `multiline +EnumValueAnnotation` + }]; } // Test custom options for nested type. @@ -436,4 +477,4 @@ extend google.protobuf.MessageOptions { // Test message using the "required_enum_opt" option defined above. message TestMessageWithRequiredEnumOption { option (required_enum_opt) = { value: OLD_VALUE }; -} \ No newline at end of file +}