diff --git a/cmake/examples.cmake b/cmake/examples.cmake index 0a651051..e5cad63f 100644 --- a/cmake/examples.cmake +++ b/cmake/examples.cmake @@ -1,57 +1,57 @@ -if(protobuf_VERBOSE) - message(STATUS "Protocol Buffers Examples Configuring...") -endif() - -get_filename_component(examples_dir "../examples" ABSOLUTE) - -if(protobuf_VERBOSE) - message(STATUS "Protocol Buffers Examples Configuring done") -endif() -include(ExternalProject) - -# Internal utility function: Create a custom target representing a build of examples with custom options. -function(add_examples_build NAME) - - ExternalProject_Add(${NAME} - PREFIX ${NAME} - SOURCE_DIR "${examples_dir}" - BINARY_DIR ${NAME} - STAMP_DIR ${NAME}/logs - INSTALL_COMMAND "" #Skip - LOG_CONFIGURE 1 - CMAKE_CACHE_ARGS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" - "-Dprotobuf_VERBOSE:BOOL=${protobuf_VERBOSE}" - ${ARGN} - ) - set_property(TARGET ${NAME} PROPERTY FOLDER "Examples") - set_property(TARGET ${NAME} PROPERTY EXCLUDE_FROM_ALL TRUE) -endfunction() - -# Add examples as an external project. -# sub_directory cannot be used because the find_package(protobuf) call would cause failures with redefined targets. -add_examples_build(examples "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}") -add_dependencies(examples libprotobuf protoc) - -option(protobuf_BUILD_EXAMPLES_MULTITEST "Build Examples in multiple configurations. Useful for testing." OFF) -mark_as_advanced(protobuf_BUILD_EXAMPLES_MULTITEST) -if(protobuf_BUILD_EXAMPLES_MULTITEST) - set_property(GLOBAL PROPERTY USE_FOLDERS ON) - - #Build using the legacy compatibility module. - add_examples_build(examples-legacy - "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}" - "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE" - ) - add_dependencies(examples-legacy libprotobuf protoc) - - #Build using the installed library. - add_examples_build(examples-installed - "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}" - ) - - #Build using the installed library in legacy compatibility mode. - add_examples_build(examples-installed-legacy - "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}" - "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE" - ) -endif() +if(protobuf_VERBOSE) + message(STATUS "Protocol Buffers Examples Configuring...") +endif() + +get_filename_component(examples_dir "../examples" ABSOLUTE) + +if(protobuf_VERBOSE) + message(STATUS "Protocol Buffers Examples Configuring done") +endif() +include(ExternalProject) + +# Internal utility function: Create a custom target representing a build of examples with custom options. +function(add_examples_build NAME) + + ExternalProject_Add(${NAME} + PREFIX ${NAME} + SOURCE_DIR "${examples_dir}" + BINARY_DIR ${NAME} + STAMP_DIR ${NAME}/logs + INSTALL_COMMAND "" #Skip + LOG_CONFIGURE 1 + CMAKE_CACHE_ARGS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" + "-Dprotobuf_VERBOSE:BOOL=${protobuf_VERBOSE}" + ${ARGN} + ) + set_property(TARGET ${NAME} PROPERTY FOLDER "Examples") + set_property(TARGET ${NAME} PROPERTY EXCLUDE_FROM_ALL TRUE) +endfunction() + +# Add examples as an external project. +# sub_directory cannot be used because the find_package(protobuf) call would cause failures with redefined targets. +add_examples_build(examples "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}") +add_dependencies(examples libprotobuf protoc) + +option(protobuf_BUILD_EXAMPLES_MULTITEST "Build Examples in multiple configurations. Useful for testing." OFF) +mark_as_advanced(protobuf_BUILD_EXAMPLES_MULTITEST) +if(protobuf_BUILD_EXAMPLES_MULTITEST) + set_property(GLOBAL PROPERTY USE_FOLDERS ON) + + #Build using the legacy compatibility module. + add_examples_build(examples-legacy + "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}" + "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE" + ) + add_dependencies(examples-legacy libprotobuf protoc) + + #Build using the installed library. + add_examples_build(examples-installed + "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}" + ) + + #Build using the installed library in legacy compatibility mode. + add_examples_build(examples-installed-legacy + "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}" + "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE" + ) +endif() diff --git a/cmake/protobuf-options.cmake b/cmake/protobuf-options.cmake index 99c85ebe..47fb1582 100644 --- a/cmake/protobuf-options.cmake +++ b/cmake/protobuf-options.cmake @@ -1,7 +1,7 @@ -# Verbose output -option(protobuf_VERBOSE "Enable for verbose output" OFF) -mark_as_advanced(protobuf_VERBOSE) - -# FindProtobuf module compatibel -option(protobuf_MODULE_COMPATIBLE "CMake build-in FindProtobuf.cmake module compatible" OFF) -mark_as_advanced(protobuf_MODULE_COMPATIBLE) +# Verbose output +option(protobuf_VERBOSE "Enable for verbose output" OFF) +mark_as_advanced(protobuf_VERBOSE) + +# FindProtobuf module compatibel +option(protobuf_MODULE_COMPATIBLE "CMake build-in FindProtobuf.cmake module compatible" OFF) +mark_as_advanced(protobuf_MODULE_COMPATIBLE) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index c9d46885..2cd2acc0 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,63 +1,63 @@ -# Minimum CMake required -cmake_minimum_required(VERSION 2.8.12) - -# Project -project(protobuf-examples) - -# Find required protobuf package -find_package(protobuf CONFIG REQUIRED) - -if(protobuf_VERBOSE) - message(STATUS "Using Protocol Buffers ${Protobuf_VERSION}") -endif() - -set(CMAKE_INCLUDE_CURRENT_DIR TRUE) - -# http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F -if(MSVC AND protobuf_MSVC_STATIC_RUNTIME) - foreach(flag_var - CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE - CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) - if(${flag_var} MATCHES "/MD") - string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") - endif(${flag_var} MATCHES "/MD") - endforeach() -endif() - -foreach(example add_person list_people) - set(${example}_SRCS ${example}.cc) - set(${example}_PROTOS addressbook.proto) - - #Code Generation - if(protobuf_MODULE_COMPATIBLE) #Legacy Support - protobuf_generate_cpp(${example}_PROTO_SRCS ${example}_PROTO_HDRS ${${example}_PROTOS}) - list(APPEND ${example}_SRCS ${${example}_PROTO_SRCS} ${${example}_PROTO_HDRS}) - else() - - foreach(proto_file ${${example}_PROTOS}) - get_filename_component(proto_file_abs ${proto_file} ABSOLUTE) - get_filename_component(basename ${proto_file} NAME_WE) - set(generated_files ${basename}.pb.cc ${basename}.pb.h) - list(APPEND ${example}_SRCS ${generated_files}) - - add_custom_command( - OUTPUT ${generated_files} - COMMAND protobuf::protoc - ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${CMAKE_CURRENT_SOURCE_DIR} ${proto_file_abs} - COMMENT "Generating ${generated_files} from ${proto_file}" - VERBATIM - ) - endforeach() - endif() - - #Executable setup - set(executable_name ${example}_cpp) - add_executable(${executable_name} ${${example}_SRCS} ${${example}_PROTOS}) - if(protobuf_MODULE_COMPATIBLE) #Legacy mode - target_include_directories(${executable_name} PUBLIC ${PROTOBUF_INCLUDE_DIRS}) - target_link_libraries(${executable_name} ${PROTOBUF_LIBRARIES}) - else() - target_link_libraries(${executable_name} protobuf::libprotobuf) - endif() - -endforeach() +# Minimum CMake required +cmake_minimum_required(VERSION 2.8.12) + +# Project +project(protobuf-examples) + +# Find required protobuf package +find_package(protobuf CONFIG REQUIRED) + +if(protobuf_VERBOSE) + message(STATUS "Using Protocol Buffers ${Protobuf_VERSION}") +endif() + +set(CMAKE_INCLUDE_CURRENT_DIR TRUE) + +# http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F +if(MSVC AND protobuf_MSVC_STATIC_RUNTIME) + foreach(flag_var + CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE + CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) + if(${flag_var} MATCHES "/MD") + string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") + endif(${flag_var} MATCHES "/MD") + endforeach() +endif() + +foreach(example add_person list_people) + set(${example}_SRCS ${example}.cc) + set(${example}_PROTOS addressbook.proto) + + #Code Generation + if(protobuf_MODULE_COMPATIBLE) #Legacy Support + protobuf_generate_cpp(${example}_PROTO_SRCS ${example}_PROTO_HDRS ${${example}_PROTOS}) + list(APPEND ${example}_SRCS ${${example}_PROTO_SRCS} ${${example}_PROTO_HDRS}) + else() + + foreach(proto_file ${${example}_PROTOS}) + get_filename_component(proto_file_abs ${proto_file} ABSOLUTE) + get_filename_component(basename ${proto_file} NAME_WE) + set(generated_files ${basename}.pb.cc ${basename}.pb.h) + list(APPEND ${example}_SRCS ${generated_files}) + + add_custom_command( + OUTPUT ${generated_files} + COMMAND protobuf::protoc + ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${CMAKE_CURRENT_SOURCE_DIR} ${proto_file_abs} + COMMENT "Generating ${generated_files} from ${proto_file}" + VERBATIM + ) + endforeach() + endif() + + #Executable setup + set(executable_name ${example}_cpp) + add_executable(${executable_name} ${${example}_SRCS} ${${example}_PROTOS}) + if(protobuf_MODULE_COMPATIBLE) #Legacy mode + target_include_directories(${executable_name} PUBLIC ${PROTOBUF_INCLUDE_DIRS}) + target_link_libraries(${executable_name} ${PROTOBUF_LIBRARIES}) + else() + target_link_libraries(${executable_name} protobuf::libprotobuf) + endif() + +endforeach() diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java index 576a350f..4f5a9b77 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -144,6 +144,44 @@ public abstract class CodedOutputStream extends ByteOutput { return new NioEncoder(byteBuffer); } + /** + * Configures serialization to be deterministic. + * + *

The deterministic serialization guarantees that for a given binary, equal (defined by the + * {@code equals()} methods in protos) messages will always be serialized to the same bytes. This + * implies: + * + *

+ * + *

Note the deterministic serialization is NOT canonical across languages; it is also unstable + * across different builds with schema changes due to unknown fields. Users who need canonical + * serialization, e.g. persistent storage in a canonical form, fingerprinting, etc, should define + * their own canonicalization specification and implement the serializer using reflection APIs + * rather than relying on this API. + * + *

Once set, the serializer will: (Note this is an implementation detail and may subject to + * change in the future) + * + *

+ */ + public final void useDeterministicSerialization() { + serializationDeterministic = true; + } + + boolean isSerializationDeterministic() { + return serializationDeterministic; + } + private boolean serializationDeterministic; + /** * Create a new {@code CodedOutputStream} that writes to the given {@link ByteBuffer}. * diff --git a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java index 859a9e8f..c54da67f 100644 --- a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java +++ b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java @@ -526,6 +526,14 @@ public final class DynamicMessage extends AbstractMessage { fields.clearField(oldField); } oneofCases[index] = field; + } else if (field.getFile().getSyntax() == Descriptors.FileDescriptor.Syntax.PROTO3) { + if (!field.isRepeated() + && field.getJavaType() != FieldDescriptor.JavaType.MESSAGE + && value.equals(field.getDefaultValue())) { + // In proto3, setting a field to its default value is equivalent to clearing the field. + fields.clearField(field); + return this; + } } fields.setField(field, value); return this; diff --git a/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java b/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java index 82f4216b..4a42c897 100644 --- a/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java +++ b/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java @@ -31,6 +31,8 @@ package com.google.protobuf; import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.EnumDescriptor; +import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.FieldPresenceTestProto.TestAllTypes; import com.google.protobuf.FieldPresenceTestProto.TestOptionalFieldsOnly; @@ -253,6 +255,54 @@ public class FieldPresenceTest extends TestCase { assertEquals(4, message.getAllFields().size()); } + public void testFieldPresenceDynamicMessage() { + Descriptor descriptor = TestAllTypes.getDescriptor(); + FieldDescriptor optionalInt32Field = descriptor.findFieldByName("optional_int32"); + FieldDescriptor optionalStringField = descriptor.findFieldByName("optional_string"); + FieldDescriptor optionalBytesField = descriptor.findFieldByName("optional_bytes"); + FieldDescriptor optionalNestedEnumField = descriptor.findFieldByName("optional_nested_enum"); + EnumDescriptor enumDescriptor = optionalNestedEnumField.getEnumType(); + EnumValueDescriptor defaultEnumValueDescriptor = enumDescriptor.getValues().get(0); + EnumValueDescriptor nonDefaultEnumValueDescriptor = enumDescriptor.getValues().get(1); + + DynamicMessage defaultInstance = DynamicMessage.getDefaultInstance(descriptor); + // Field not present. + DynamicMessage message = defaultInstance.newBuilderForType().build(); + assertFalse(message.hasField(optionalInt32Field)); + assertFalse(message.hasField(optionalStringField)); + assertFalse(message.hasField(optionalBytesField)); + assertFalse(message.hasField(optionalNestedEnumField)); + assertEquals(0, message.getAllFields().size()); + + // Field set to non-default value is seen as present. + message = + defaultInstance + .newBuilderForType() + .setField(optionalInt32Field, 1) + .setField(optionalStringField, "x") + .setField(optionalBytesField, ByteString.copyFromUtf8("y")) + .setField(optionalNestedEnumField, nonDefaultEnumValueDescriptor) + .build(); + assertTrue(message.hasField(optionalInt32Field)); + assertTrue(message.hasField(optionalStringField)); + assertTrue(message.hasField(optionalBytesField)); + assertTrue(message.hasField(optionalNestedEnumField)); + assertEquals(4, message.getAllFields().size()); + + // Field set to default value is seen as not present. + message = message.toBuilder() + .setField(optionalInt32Field, 0) + .setField(optionalStringField, "") + .setField(optionalBytesField, ByteString.EMPTY) + .setField(optionalNestedEnumField, defaultEnumValueDescriptor) + .build(); + assertFalse(message.hasField(optionalInt32Field)); + assertFalse(message.hasField(optionalStringField)); + assertFalse(message.hasField(optionalBytesField)); + assertFalse(message.hasField(optionalNestedEnumField)); + assertEquals(0, message.getAllFields().size()); + } + public void testMessageField() { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); assertFalse(builder.hasOptionalNestedMessage()); diff --git a/java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java b/java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java index 0f42ac50..497c4df2 100644 --- a/java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java +++ b/java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java @@ -35,6 +35,7 @@ import static java.util.Arrays.asList; import junit.framework.TestCase; import java.util.ArrayList; +import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.List; @@ -233,7 +234,7 @@ public class LazyStringArrayListTest extends TestCase { } try { - list.addAllByteArray(asList(BYTE_STRING_A.toByteArray())); + list.addAllByteArray(Collections.singletonList(BYTE_STRING_A.toByteArray())); fail(); } catch (UnsupportedOperationException e) { // expected diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index 297545e5..ad50cc0e 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -116,7 +116,8 @@ public class JsonFormat { private Printer( TypeRegistry registry, boolean includingDefaultValueFields, - boolean preservingProtoFieldNames, boolean omittingInsignificantWhitespace) { + boolean preservingProtoFieldNames, + boolean omittingInsignificantWhitespace) { this.registry = registry; this.includingDefaultValueFields = includingDefaultValueFields; this.preservingProtoFieldNames = preservingProtoFieldNames; @@ -133,7 +134,11 @@ public class JsonFormat { if (this.registry != TypeRegistry.getEmptyTypeRegistry()) { throw new IllegalArgumentException("Only one registry is allowed."); } - return new Printer(registry, includingDefaultValueFields, preservingProtoFieldNames, omittingInsignificantWhitespace); + return new Printer( + registry, + includingDefaultValueFields, + preservingProtoFieldNames, + omittingInsignificantWhitespace); } /** @@ -143,7 +148,8 @@ public class JsonFormat { * {@link Printer}. */ public Printer includingDefaultValueFields() { - return new Printer(registry, true, preservingProtoFieldNames, omittingInsignificantWhitespace); + return new Printer( + registry, true, preservingProtoFieldNames, omittingInsignificantWhitespace); } /** @@ -153,7 +159,8 @@ public class JsonFormat { * current {@link Printer}. */ public Printer preservingProtoFieldNames() { - return new Printer(registry, includingDefaultValueFields, true, omittingInsignificantWhitespace); + return new Printer( + registry, includingDefaultValueFields, true, omittingInsignificantWhitespace); } @@ -172,7 +179,7 @@ public class JsonFormat { * See https://tools.ietf.org/html/rfc7159 * current {@link Printer}. */ - public Printer omittingInsignificantWhitespace(){ + public Printer omittingInsignificantWhitespace() { return new Printer(registry, includingDefaultValueFields, preservingProtoFieldNames, true); } @@ -186,7 +193,12 @@ public class JsonFormat { public void appendTo(MessageOrBuilder message, Appendable output) throws IOException { // TODO(xiaofeng): Investigate the allocation overhead and optimize for // mobile. - new PrinterImpl(registry, includingDefaultValueFields, preservingProtoFieldNames, output, omittingInsignificantWhitespace) + new PrinterImpl( + registry, + includingDefaultValueFields, + preservingProtoFieldNames, + output, + omittingInsignificantWhitespace) .print(message); } @@ -379,18 +391,18 @@ public class JsonFormat { */ interface TextGenerator { void indent(); + void outdent(); + void print(final CharSequence text) throws IOException; } - /** * Format the json without indentation */ - private static final class CompactTextGenerator implements TextGenerator{ + private static final class CompactTextGenerator implements TextGenerator { private final Appendable output; - private CompactTextGenerator(final Appendable output) { this.output = output; } @@ -411,12 +423,11 @@ public class JsonFormat { public void print(final CharSequence text) throws IOException { output.append(text); } - } /** * A TextGenerator adds indentation when writing formatted text. */ - private static final class PrettyTextGenerator implements TextGenerator{ + private static final class PrettyTextGenerator implements TextGenerator { private final Appendable output; private final StringBuilder indent = new StringBuilder(); private boolean atStartOfLine = true; @@ -496,7 +507,8 @@ public class JsonFormat { TypeRegistry registry, boolean includingDefaultValueFields, boolean preservingProtoFieldNames, - Appendable jsonOutput, boolean omittingInsignificantWhitespace) { + Appendable jsonOutput, + boolean omittingInsignificantWhitespace) { this.registry = registry; this.includingDefaultValueFields = includingDefaultValueFields; this.preservingProtoFieldNames = preservingProtoFieldNames; @@ -734,9 +746,7 @@ public class JsonFormat { } /** Prints a regular message with an optional type URL. */ - - private void print(MessageOrBuilder message, String typeUrl) - throws IOException { + private void print(MessageOrBuilder message, String typeUrl) throws IOException { generator.print("{" + blankOrNewLine); generator.indent(); diff --git a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java index e68c7be1..6fc784ef 100644 --- a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java @@ -140,7 +140,7 @@ public class JsonFormatTest extends TestCase { private String toJsonString(Message message) throws IOException { return JsonFormat.printer().print(message); } - private String toCompactJsonString(Message message) throws IOException{ + private String toCompactJsonString(Message message) throws IOException { return JsonFormat.printer().omittingInsignificantWhitespace().print(message); } @@ -1172,7 +1172,9 @@ public class JsonFormatTest extends TestCase { public void testOmittingInsignificantWhiteSpace() throws Exception { TestAllTypes message = TestAllTypes.newBuilder().setOptionalInt32(12345).build(); - assertEquals("{" + "\"optionalInt32\":12345" + "}", JsonFormat.printer().omittingInsignificantWhitespace().print(message)); + assertEquals( + "{" + "\"optionalInt32\":12345" + "}", + JsonFormat.printer().omittingInsignificantWhitespace().print(message)); TestAllTypes message1 = TestAllTypes.getDefaultInstance(); assertEquals("{}", JsonFormat.printer().omittingInsignificantWhitespace().print(message1)); TestAllTypes.Builder builder = TestAllTypes.newBuilder(); @@ -1224,4 +1226,20 @@ public class JsonFormatTest extends TestCase { toCompactJsonString(message2)); } + // Regression test for b/29892357 + public void testEmptyWrapperTypesInAny() throws Exception { + JsonFormat.TypeRegistry registry = + JsonFormat.TypeRegistry.newBuilder().add(TestAllTypes.getDescriptor()).build(); + JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(registry); + + Any.Builder builder = Any.newBuilder(); + parser.merge( + "{\n" + + " \"@type\": \"type.googleapis.com/google.protobuf.BoolValue\",\n" + + " \"value\": false\n" + + "}\n", + builder); + Any any = builder.build(); + assertEquals(0, any.getValue().size()); + } } diff --git a/js/message_test.js b/js/message_test.js index 0b0c0172..11792423 100644 --- a/js/message_test.js +++ b/js/message_test.js @@ -215,6 +215,10 @@ describe('Message test suite', function() { assertEquals(true, response.getBoolField()); assertEquals(11, response.getIntField()); assertEquals(13, response.getEnumField()); + assertFalse(response.hasStringField()); + assertFalse(response.hasBoolField()); + assertFalse(response.hasIntField()); + assertFalse(response.hasEnumField()); // Test with null values, as would be returned by a JSON serializer. response = makeDefault([null, null, null, null]); @@ -222,6 +226,10 @@ describe('Message test suite', function() { assertEquals(true, response.getBoolField()); assertEquals(11, response.getIntField()); assertEquals(13, response.getEnumField()); + assertFalse(response.hasStringField()); + assertFalse(response.hasBoolField()); + assertFalse(response.hasIntField()); + assertFalse(response.hasEnumField()); // Test with false-like values. response = makeDefault(['', false, 0, 0]); @@ -229,6 +237,10 @@ describe('Message test suite', function() { assertEquals(false, response.getBoolField()); assertEquals(true, response.getIntField() == 0); assertEquals(true, response.getEnumField() == 0); + assertTrue(response.hasStringField()); + assertTrue(response.hasBoolField()); + assertTrue(response.hasIntField()); + assertTrue(response.hasEnumField()); // Test that clearing the values reverts them to the default state. response = makeDefault(['blah', false, 111, 77]); @@ -238,6 +250,10 @@ describe('Message test suite', function() { assertEquals(true, response.getBoolField()); assertEquals(11, response.getIntField()); assertEquals(13, response.getEnumField()); + assertFalse(response.hasStringField()); + assertFalse(response.hasBoolField()); + assertFalse(response.hasIntField()); + assertFalse(response.hasEnumField()); // Test that setFoo(null) clears the values. response = makeDefault(['blah', false, 111, 77]); @@ -247,6 +263,10 @@ describe('Message test suite', function() { assertEquals(true, response.getBoolField()); assertEquals(11, response.getIntField()); assertEquals(13, response.getEnumField()); + assertFalse(response.hasStringField()); + assertFalse(response.hasBoolField()); + assertFalse(response.hasIntField()); + assertFalse(response.hasEnumField()); }); it('testMessageRegistration', function() { @@ -269,6 +289,8 @@ describe('Message test suite', function() { assertUndefined(foo.getAString()); assertUndefined(foo.getABool()); assertUndefined(foo.getANestedMessage()); + assertFalse(foo.hasAString()); + assertFalse(foo.hasABool()); assertObjectEquals([], foo.getARepeatedMessageList()); assertObjectEquals([], foo.getARepeatedStringList()); // NOTE: We want the missing fields in 'expected' to be undefined, @@ -291,6 +313,8 @@ describe('Message test suite', function() { assertNull(foo.getAString()); assertNull(foo.getABool()); assertNull(foo.getANestedMessage()); + assertFalse(foo.hasAString()); + assertFalse(foo.hasABool()); assertObjectEquals([], foo.getARepeatedMessageList()); assertObjectEquals([], foo.getARepeatedStringList()); assertObjectEquals([null, null, null, [], []], foo.toArray()); @@ -307,6 +331,8 @@ describe('Message test suite', function() { assertUndefined(foo.getAString()); assertUndefined(foo.getABool()); assertUndefined(foo.getANestedMessage()); + assertFalse(foo.hasAString()); + assertFalse(foo.hasABool()); assertObjectEquals([], foo.getARepeatedMessageList()); assertObjectEquals([], foo.getARepeatedStringList()); expected = [,,, [], []]; @@ -800,14 +826,20 @@ describe('Message test suite', function() { var message = new proto.jspb.test.TestMessageWithOneof; assertUndefined(message.getPone()); assertUndefined(message.getPthree()); + assertFalse(message.hasPone()); + assertFalse(message.hasPthree()); message.setPone('hi'); assertEquals('hi', message.getPone()); assertUndefined(message.getPthree()); + assertTrue(message.hasPone()); + assertFalse(message.hasPthree()); message.setPthree('bye'); assertUndefined(message.getPone()); assertEquals('bye', message.getPthree()); + assertFalse(message.hasPone()); + assertTrue(message.hasPthree()); }); it('testSettingOneofFieldDoesNotClearFieldsFromOtherUnions', function() { @@ -816,17 +848,23 @@ describe('Message test suite', function() { assertUndefined(message.getPone()); assertUndefined(message.getPthree()); assertUndefined(message.getRone()); + assertFalse(message.hasPone()); + assertFalse(message.hasPthree()); message.setPone('hi'); message.setRone(other); assertEquals('hi', message.getPone()); assertUndefined(message.getPthree()); assertEquals(other, message.getRone()); + assertTrue(message.hasPone()); + assertFalse(message.hasPthree()); message.setPthree('bye'); assertUndefined(message.getPone()); assertEquals('bye', message.getPthree()); assertEquals(other, message.getRone()); + assertFalse(message.hasPone()); + assertTrue(message.hasPthree()); }); it('testUnsetsOneofCaseWhenFieldIsCleared', function() { @@ -884,6 +922,8 @@ describe('Message test suite', function() { var message = new proto.jspb.test.TestMessageWithOneof; assertUndefined(message.getBone()); assertEquals(1234, message.getBtwo()); + assertFalse(message.hasBone()); + assertFalse(message.hasBtwo()); assertEquals( proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase .DEFAULT_ONEOF_B_NOT_SET, @@ -892,12 +932,16 @@ describe('Message test suite', function() { message.setBone(2); assertEquals(2, message.getBone()); assertEquals(1234, message.getBtwo()); + assertTrue(message.hasBone()); + assertFalse(message.hasBtwo()); assertEquals( proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BONE, message.getDefaultOneofBCase()); message.setBtwo(3); assertUndefined(message.getBone()); + assertFalse(message.hasBone()); + assertTrue(message.hasBtwo()); assertEquals(3, message.getBtwo()); assertEquals( proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BTWO, @@ -905,6 +949,8 @@ describe('Message test suite', function() { message.clearBtwo(); assertUndefined(message.getBone()); + assertFalse(message.hasBone()); + assertFalse(message.hasBtwo()); assertEquals(1234, message.getBtwo()); assertEquals( proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase diff --git a/js/proto3_test.js b/js/proto3_test.js index 4dd7790f..7f76006a 100644 --- a/js/proto3_test.js +++ b/js/proto3_test.js @@ -225,12 +225,18 @@ describe('proto3Test', function() { assertEquals(msg.getOneofForeignMessage(), undefined); assertEquals(msg.getOneofString(), undefined); assertEquals(msg.getOneofBytes(), undefined); + assertFalse(msg.hasOneofUint32()); + assertFalse(msg.hasOneofString()); + assertFalse(msg.hasOneofBytes()); msg.setOneofUint32(42); assertEquals(msg.getOneofUint32(), 42); assertEquals(msg.getOneofForeignMessage(), undefined); assertEquals(msg.getOneofString(), undefined); assertEquals(msg.getOneofBytes(), undefined); + assertTrue(msg.hasOneofUint32()); + assertFalse(msg.hasOneofString()); + assertFalse(msg.hasOneofBytes()); var submsg = new proto.jspb.test.ForeignMessage(); @@ -239,12 +245,18 @@ describe('proto3Test', function() { assertEquals(msg.getOneofForeignMessage(), submsg); assertEquals(msg.getOneofString(), undefined); assertEquals(msg.getOneofBytes(), undefined); + assertFalse(msg.hasOneofUint32()); + assertFalse(msg.hasOneofString()); + assertFalse(msg.hasOneofBytes()); msg.setOneofString('hello'); assertEquals(msg.getOneofUint32(), undefined); assertEquals(msg.getOneofForeignMessage(), undefined); assertEquals(msg.getOneofString(), 'hello'); assertEquals(msg.getOneofBytes(), undefined); + assertFalse(msg.hasOneofUint32()); + assertTrue(msg.hasOneofString()); + assertFalse(msg.hasOneofBytes()); msg.setOneofBytes(goog.crypt.base64.encodeString('\u00FF\u00FF')); assertEquals(msg.getOneofUint32(), undefined); @@ -252,6 +264,9 @@ describe('proto3Test', function() { assertEquals(msg.getOneofString(), undefined); assertEquals(msg.getOneofBytes_asB64(), goog.crypt.base64.encodeString('\u00FF\u00FF')); + assertFalse(msg.hasOneofUint32()); + assertFalse(msg.hasOneofString()); + assertTrue(msg.hasOneofBytes()); }); diff --git a/js/test.proto b/js/test.proto index cf2eafef..937ffb89 100644 --- a/js/test.proto +++ b/js/test.proto @@ -233,3 +233,4 @@ message TestEndsWithBytes { optional int32 value = 1; optional bytes data = 2; } + diff --git a/python/google/protobuf/internal/json_format_test.py b/python/google/protobuf/internal/json_format_test.py index 6df12bea..a5ee8ace 100644 --- a/python/google/protobuf/internal/json_format_test.py +++ b/python/google/protobuf/internal/json_format_test.py @@ -252,10 +252,7 @@ class JsonFormatTest(JsonFormatBase): message = json_format_proto3_pb2.TestMessage() json_format.Parse('{"stringValue": "\\uD83D\\uDE01"}', message) self.assertEqual(message.string_value, - b'\xF0\x9F\x98\x81'.decode("utf-8", "strict")) - - # TODO: add test that UTF-8 encoded surrogate code points are rejected. - # UTF-8 does not allow them. + b'\xF0\x9F\x98\x81'.decode('utf-8', 'strict')) # Error case: unpaired high surrogate. self.CheckError( @@ -267,7 +264,6 @@ class JsonFormatTest(JsonFormatBase): '{"stringValue": "\\uDE01"}', r'Invalid \\uXXXX escape|Unpaired.*surrogate') - def testTimestampMessage(self): message = json_format_proto3_pb2.TestTimestamp() message.value.seconds = 0 diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index f8f73dd2..c0d0ad45 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -76,7 +76,6 @@ from google.protobuf.internal import well_known_types from google.protobuf.internal import wire_format from google.protobuf import descriptor as descriptor_mod from google.protobuf import message as message_mod -from google.protobuf import symbol_database from google.protobuf import text_format _FieldDescriptor = descriptor_mod.FieldDescriptor @@ -98,16 +97,12 @@ class GeneratedProtocolMessageType(type): classes at runtime, as in this example: mydescriptor = Descriptor(.....) - class MyProtoClass(Message): - __metaclass__ = GeneratedProtocolMessageType - DESCRIPTOR = mydescriptor + factory = symbol_database.Default() + factory.pool.AddDescriptor(mydescriptor) + MyProtoClass = factory.GetPrototype(mydescriptor) myproto_instance = MyProtoClass() myproto.foo_field = 23 ... - - The above example will not work for nested types. If you wish to include them, - use reflection.MakeClass() instead of manually instantiating the class in - order to create the appropriate class structure. """ # Must be consistent with the protocol-compiler code in @@ -926,26 +921,33 @@ def _InternalUnpackAny(msg): Returns: The unpacked message. """ + # TODO(amauryfa): Don't use the factory of generated messages. + # To make Any work with custom factories, use the message factory of the + # parent message. + # pylint: disable=g-import-not-at-top + from google.protobuf import symbol_database + factory = symbol_database.Default() + type_url = msg.type_url - db = symbol_database.Default() if not type_url: return None # TODO(haberman): For now we just strip the hostname. Better logic will be # required. - type_name = type_url.split("/")[-1] - descriptor = db.pool.FindMessageTypeByName(type_name) + type_name = type_url.split('/')[-1] + descriptor = factory.pool.FindMessageTypeByName(type_name) if descriptor is None: return None - message_class = db.GetPrototype(descriptor) + message_class = factory.GetPrototype(descriptor) message = message_class() message.ParseFromString(msg.value) return message + def _AddEqualsMethod(message_descriptor, cls): """Helper for _AddMessageMethods().""" def __eq__(self, other): diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index 6dc2fffe..20e5d245 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -972,6 +972,7 @@ class ReflectionTest(unittest.TestCase): proto.repeated_nested_message.add(bb=23) self.assertEqual(1, len(proto.repeated_nested_message)) self.assertEqual(23, proto.repeated_nested_message[0].bb) + self.assertRaises(TypeError, proto.repeated_nested_message.add, 23) def testRepeatedCompositeRemove(self): proto = unittest_pb2.TestAllTypes() diff --git a/python/google/protobuf/internal/symbol_database_test.py b/python/google/protobuf/internal/symbol_database_test.py index c99b426d..4f5173b2 100644 --- a/python/google/protobuf/internal/symbol_database_test.py +++ b/python/google/protobuf/internal/symbol_database_test.py @@ -39,26 +39,28 @@ except ImportError: from google.protobuf import unittest_pb2 from google.protobuf import descriptor +from google.protobuf import descriptor_pool from google.protobuf import symbol_database + class SymbolDatabaseTest(unittest.TestCase): def _Database(self): - # TODO(b/17734095): Remove this difference when the C++ implementation - # supports multiple databases. if descriptor._USE_C_DESCRIPTORS: - return symbol_database.Default() + # The C++ implementation does not allow mixing descriptors from + # different pools. + db = symbol_database.SymbolDatabase(pool=descriptor_pool.Default()) else: db = symbol_database.SymbolDatabase() - # Register representative types from unittest_pb2. - db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR) - db.RegisterMessage(unittest_pb2.TestAllTypes) - db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage) - db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup) - db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup) - db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) - db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) - return db + # Register representative types from unittest_pb2. + db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR) + db.RegisterMessage(unittest_pb2.TestAllTypes) + db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage) + db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup) + db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup) + db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) + db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) + return db def testGetPrototype(self): instance = self._Database().GetPrototype( diff --git a/python/google/protobuf/pyext/cpp_message.py b/python/google/protobuf/pyext/cpp_message.py index b215211e..fc8eb32d 100644 --- a/python/google/protobuf/pyext/cpp_message.py +++ b/python/google/protobuf/pyext/cpp_message.py @@ -48,9 +48,9 @@ class GeneratedProtocolMessageType(_message.MessageMeta): classes at runtime, as in this example: mydescriptor = Descriptor(.....) - class MyProtoClass(Message): - __metaclass__ = GeneratedProtocolMessageType - DESCRIPTOR = mydescriptor + factory = symbol_database.Default() + factory.pool.AddDescriptor(mydescriptor) + MyProtoClass = factory.GetPrototype(mydescriptor) myproto_instance = MyProtoClass() myproto.foo_field = 23 ... diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc index 90438df1..0987b898 100644 --- a/python/google/protobuf/pyext/map_container.cc +++ b/python/google/protobuf/pyext/map_container.cc @@ -348,9 +348,10 @@ PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) { } // Initializes the underlying Message object of "to" so it becomes a new parent -// repeated scalar, and copies all the values from "from" to it. A child scalar +// map container, and copies all the values from "from" to it. A child map // container can be released by passing it as both from and to (e.g. making it // the recipient of the new parent message and copying the values from itself). +// In fact, this is the only supported use at the moment. static int InitializeAndCopyToParentContainer(MapContainer* from, MapContainer* to) { // For now we require from == to, re-evaluate if we want to support deep copy diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index a9261f20..5535338d 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -1041,7 +1041,12 @@ int InternalDeleteRepeatedField( } // Initializes fields of a message. Used in constructors. -int InitAttributes(CMessage* self, PyObject* kwargs) { +int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) { + if (args != NULL && PyTuple_Size(args) != 0) { + PyErr_SetString(PyExc_TypeError, "No positional arguments allowed"); + return -1; + } + if (kwargs == NULL) { return 0; } @@ -1167,7 +1172,7 @@ int InitAttributes(CMessage* self, PyObject* kwargs) { } CMessage* cmessage = reinterpret_cast(message.get()); if (PyDict_Check(value)) { - if (InitAttributes(cmessage, value) < 0) { + if (InitAttributes(cmessage, NULL, value) < 0) { return -1; } } else { @@ -1245,12 +1250,7 @@ static PyObject* New(PyTypeObject* cls, // The __init__ method of Message classes. // It initializes fields from keywords passed to the constructor. static int Init(CMessage* self, PyObject* args, PyObject* kwargs) { - if (PyTuple_Size(args) != 0) { - PyErr_SetString(PyExc_TypeError, "No positional arguments allowed"); - return -1; - } - - return InitAttributes(self, kwargs); + return InitAttributes(self, args, kwargs); } // --------------------------------------------------------------------- diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index 8b399e05..c44a2ae2 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -237,7 +237,9 @@ PyObject* HasFieldByDescriptor( PyObject* HasField(CMessage* self, PyObject* arg); // Initializes values of fields on a newly constructed message. -int InitAttributes(CMessage* self, PyObject* kwargs); +// Note that positional arguments are disallowed: 'args' must be NULL or the +// empty tuple. +int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs); PyObject* MergeFrom(CMessage* self, PyObject* arg); diff --git a/python/google/protobuf/pyext/repeated_composite_container.cc b/python/google/protobuf/pyext/repeated_composite_container.cc index 4f339e77..bb2f6db2 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.cc +++ b/python/google/protobuf/pyext/repeated_composite_container.cc @@ -146,7 +146,7 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self, cmsg->owner = self->owner; cmsg->message = sub_message; cmsg->parent = self->parent; - if (cmessage::InitAttributes(cmsg, kwargs) < 0) { + if (cmessage::InitAttributes(cmsg, args, kwargs) < 0) { Py_DECREF(cmsg); return NULL; } @@ -166,7 +166,7 @@ static PyObject* AddToReleased(RepeatedCompositeContainer* self, // Create a new Message detached from the rest. PyObject* py_cmsg = PyEval_CallObjectWithKeywords( - self->child_message_class->AsPyObject(), NULL, kwargs); + self->child_message_class->AsPyObject(), args, kwargs); if (py_cmsg == NULL) return NULL; diff --git a/python/google/protobuf/reflection.py b/python/google/protobuf/reflection.py index 0c757264..51c83321 100755 --- a/python/google/protobuf/reflection.py +++ b/python/google/protobuf/reflection.py @@ -58,13 +58,7 @@ else: from google.protobuf.internal import python_message as message_impl # The type of all Message classes. -# Part of the public interface. -# -# Used by generated files, but clients can also use it at runtime: -# mydescriptor = pool.FindDescriptor(.....) -# class MyProtoClass(Message): -# __metaclass__ = GeneratedProtocolMessageType -# DESCRIPTOR = mydescriptor +# Part of the public interface, but normally only used by message factories. GeneratedProtocolMessageType = message_impl.GeneratedProtocolMessageType diff --git a/python/google/protobuf/symbol_database.py b/python/google/protobuf/symbol_database.py index 87760f26..aa466abd 100644 --- a/python/google/protobuf/symbol_database.py +++ b/python/google/protobuf/symbol_database.py @@ -30,11 +30,9 @@ """A database of Python protocol buffer generated symbols. -SymbolDatabase makes it easy to create new instances of a registered type, given -only the type's protocol buffer symbol name. Once all symbols are registered, -they can be accessed using either the MessageFactory interface which -SymbolDatabase exposes, or the DescriptorPool interface of the underlying -pool. +SymbolDatabase is the MessageFactory for messages generated at compile time, +and makes it easy to create new instances of a registered type, given only the +type's protocol buffer symbol name. Example usage: @@ -61,27 +59,17 @@ Example usage: from google.protobuf import descriptor_pool +from google.protobuf import message_factory -class SymbolDatabase(object): - """A database of Python generated symbols. - - SymbolDatabase also models message_factory.MessageFactory. - - The symbol database can be used to keep a global registry of all protocol - buffer types used within a program. - """ - - def __init__(self, pool=None): - """Constructor.""" - - self._symbols = {} - self._symbols_by_file = {} - self.pool = pool or descriptor_pool.Default() +class SymbolDatabase(message_factory.MessageFactory): + """A database of Python generated symbols.""" def RegisterMessage(self, message): """Registers the given message type in the local database. + Calls to GetSymbol() and GetMessages() will return messages registered here. + Args: message: a message.Message, to be registered. @@ -90,10 +78,7 @@ class SymbolDatabase(object): """ desc = message.DESCRIPTOR - self._symbols[desc.full_name] = message - if desc.file.name not in self._symbols_by_file: - self._symbols_by_file[desc.file.name] = {} - self._symbols_by_file[desc.file.name][desc.full_name] = message + self._classes[desc.full_name] = message self.pool.AddDescriptor(desc) return message @@ -136,47 +121,46 @@ class SymbolDatabase(object): KeyError: if the symbol could not be found. """ - return self._symbols[symbol] - - def GetPrototype(self, descriptor): - """Builds a proto2 message class based on the passed in descriptor. - - Passing a descriptor with a fully qualified name matching a previous - invocation will cause the same class to be returned. - - Args: - descriptor: The descriptor to build from. - - Returns: - A class describing the passed in descriptor. - """ - - return self.GetSymbol(descriptor.full_name) + return self._classes[symbol] def GetMessages(self, files): - """Gets all the messages from a specified file. - - This will find and resolve dependencies, failing if they are not registered - in the symbol database. + # TODO(amauryfa): Fix the differences with MessageFactory. + """Gets all registered messages from a specified file. + Only messages already created and registered will be returned; (this is the + case for imported _pb2 modules) + But unlike MessageFactory, this version also returns nested messages. Args: files: The file names to extract messages from. Returns: - A dictionary mapping proto names to the message classes. This will include - any dependent messages as well as any messages defined in the same file as - a specified message. + A dictionary mapping proto names to the message classes. Raises: KeyError: if a file could not be found. """ + def _GetAllMessageNames(desc): + """Walk a message Descriptor and recursively yields all message names.""" + yield desc.full_name + for msg_desc in desc.nested_types: + for full_name in _GetAllMessageNames(msg_desc): + yield full_name + result = {} - for f in files: - result.update(self._symbols_by_file[f]) + for file_name in files: + file_desc = self.pool.FindFileByName(file_name) + for msg_desc in file_desc.message_types_by_name.values(): + for full_name in _GetAllMessageNames(msg_desc): + try: + result[full_name] = self._classes[full_name] + except KeyError: + # This descriptor has no registered class, skip it. + pass return result + _DEFAULT = SymbolDatabase(pool=descriptor_pool.Default()) diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc index 385b973e..b3eca660 100644 --- a/src/google/protobuf/compiler/cpp/cpp_file.cc +++ b/src/google/protobuf/compiler/cpp/cpp_file.cc @@ -336,19 +336,6 @@ void FileGenerator::GenerateSource(io::Printer* printer) { // Generate classes. for (int i = 0; i < file_->message_type_count(); i++) { - if (i == 0 && HasGeneratedMethods(file_, options_)) { - printer->Print( - "\n" - "namespace {\n" - "\n" - "static void MergeFromFail(int line) GOOGLE_ATTRIBUTE_COLD;\n" - "static void MergeFromFail(int line) {\n" - " GOOGLE_CHECK(false) << __FILE__ << \":\" << line;\n" - "}\n" - "\n" - "} // namespace\n" - "\n"); - } printer->Print("\n"); printer->Print(kThickSeparator); printer->Print("\n"); @@ -464,9 +451,10 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) { // and we only use AddDescriptors() to allocate default instances. if (HasDescriptorMethods(file_, options_)) { printer->Print( - "\n" - "void $assigndescriptorsname$() {\n", - "assigndescriptorsname", GlobalAssignDescriptorsName(file_->name())); + "\n" + "void $assigndescriptorsname$() GOOGLE_ATTRIBUTE_COLD;\n" + "void $assigndescriptorsname$() {\n", + "assigndescriptorsname", GlobalAssignDescriptorsName(file_->name())); printer->Indent(); // Make sure the file has found its way into the pool. If a descriptor @@ -525,8 +513,9 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) { // protobuf_RegisterTypes(): Calls // MessageFactory::InternalRegisterGeneratedType() for each message type. printer->Print( - "void protobuf_RegisterTypes(const ::std::string&) {\n" - " protobuf_AssignDescriptorsOnce();\n"); + "void protobuf_RegisterTypes(const ::std::string&) GOOGLE_ATTRIBUTE_COLD;\n" + "void protobuf_RegisterTypes(const ::std::string&) {\n" + " protobuf_AssignDescriptorsOnce();\n"); printer->Indent(); for (int i = 0; i < file_->message_type_count(); i++) { @@ -566,6 +555,7 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) { // Note that we don't need any special synchronization in the following // code // because it is called at static init time before any threads exist. + "void $adddescriptorsname$() GOOGLE_ATTRIBUTE_COLD;\n" "void $adddescriptorsname$() {\n" " static bool already_here = false;\n" " if (already_here) return;\n" diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.cc b/src/google/protobuf/compiler/cpp/cpp_map_field.cc index dd9f1887..0588e34e 100644 --- a/src/google/protobuf/compiler/cpp/cpp_map_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_map_field.cc @@ -251,117 +251,148 @@ GenerateMergeFromCodedStream(io::Printer* printer) const { } } +static void GenerateSerializationLoop(io::Printer* printer, + const map& variables, + bool supports_arenas, + const string& utf8_check, + const string& loop_header, + const string& ptr, + bool loop_via_iterators) { + printer->Print(variables, + StrCat("::google::protobuf::scoped_ptr<$map_classname$> entry;\n", + loop_header, " {\n").c_str()); + printer->Indent(); + + printer->Print(variables, StrCat( + "entry.reset($name$_.New$wrapper$(\n" + " ", ptr, "->first, ", ptr, "->second));\n" + "$write_entry$;\n").c_str()); + + // If entry is allocated by arena, its desctructor should be avoided. + if (supports_arenas) { + printer->Print( + "if (entry->GetArena() != NULL) {\n" + " entry.release();\n" + "}\n"); + } + + if (!utf8_check.empty()) { + // If loop_via_iterators is true then ptr is actually an iterator, and we + // create a pointer by prefixing it with "&*". + printer->Print( + StrCat(utf8_check, "(", (loop_via_iterators ? "&*" : ""), ptr, ");\n") + .c_str()); + } + + printer->Outdent(); + printer->Print( + "}\n"); +} + void MapFieldGenerator:: GenerateSerializeWithCachedSizes(io::Printer* printer) const { - printer->Print(variables_, - "{\n" - " ::google::protobuf::scoped_ptr<$map_classname$> entry;\n" - " for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n" - " it = this->$name$().begin();\n" - " it != this->$name$().end(); ++it) {\n"); - - // If entry is allocated by arena, its desctructor should be avoided. - if (SupportsArenas(descriptor_)) { - printer->Print(variables_, - " if (entry.get() != NULL && entry->GetArena() != NULL) {\n" - " entry.release();\n" - " }\n"); - } - - printer->Print(variables_, - " entry.reset($name$_.New$wrapper$(it->first, it->second));\n" - " ::google::protobuf::internal::WireFormatLite::Write$stream_writer$(\n" - " $number$, *entry, output);\n"); - - printer->Indent(); - printer->Indent(); - - const FieldDescriptor* key_field = - descriptor_->message_type()->FindFieldByName("key"); - const FieldDescriptor* value_field = - descriptor_->message_type()->FindFieldByName("value"); - if (key_field->type() == FieldDescriptor::TYPE_STRING) { - GenerateUtf8CheckCodeForString(key_field, options_, false, variables_, - "it->first.data(), it->first.length(),\n", - printer); - } - if (value_field->type() == FieldDescriptor::TYPE_STRING) { - GenerateUtf8CheckCodeForString(value_field, options_, false, variables_, - "it->second.data(), it->second.length(),\n", - printer); - } - - printer->Outdent(); - printer->Outdent(); - - printer->Print( - " }\n"); - - // If entry is allocated by arena, its desctructor should be avoided. - if (SupportsArenas(descriptor_)) { - printer->Print(variables_, - " if (entry.get() != NULL && entry->GetArena() != NULL) {\n" - " entry.release();\n" - " }\n"); - } - - printer->Print("}\n"); + map variables(variables_); + variables["write_entry"] = "::google::protobuf::internal::WireFormatLite::Write" + + variables["stream_writer"] + "(\n " + + variables["number"] + ", *entry, output)"; + variables["deterministic"] = "output->IsSerializationDeterminstic()"; + GenerateSerializeWithCachedSizes(printer, variables); } void MapFieldGenerator:: GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const { - printer->Print(variables_, - "{\n" - " ::google::protobuf::scoped_ptr<$map_classname$> entry;\n" - " for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n" - " it = this->$name$().begin();\n" - " it != this->$name$().end(); ++it) {\n"); - - // If entry is allocated by arena, its desctructor should be avoided. - if (SupportsArenas(descriptor_)) { - printer->Print(variables_, - " if (entry.get() != NULL && entry->GetArena() != NULL) {\n" - " entry.release();\n" - " }\n"); - } - - printer->Print(variables_, - " entry.reset($name$_.New$wrapper$(it->first, it->second));\n" - " target = ::google::protobuf::internal::WireFormatLite::\n" - " InternalWrite$declared_type$NoVirtualToArray(\n" - " $number$, *entry, false, target);\n"); + map variables(variables_); + variables["write_entry"] = + "target = ::google::protobuf::internal::WireFormatLite::\n" + " InternalWrite" + variables["declared_type"] + + "NoVirtualToArray(\n " + variables["number"] + + ", *entry, deterministic, target);\n"; + variables["deterministic"] = "deterministic"; + GenerateSerializeWithCachedSizes(printer, variables); +} +void MapFieldGenerator::GenerateSerializeWithCachedSizes( + io::Printer* printer, const map& variables) const { + printer->Print(variables, + "if (!this->$name$().empty()) {\n"); printer->Indent(); - printer->Indent(); - const FieldDescriptor* key_field = descriptor_->message_type()->FindFieldByName("key"); const FieldDescriptor* value_field = descriptor_->message_type()->FindFieldByName("value"); - if (key_field->type() == FieldDescriptor::TYPE_STRING) { - GenerateUtf8CheckCodeForString(key_field, options_, false, variables_, - "it->first.data(), it->first.length(),\n", - printer); + const bool string_key = key_field->type() == FieldDescriptor::TYPE_STRING; + const bool string_value = value_field->type() == FieldDescriptor::TYPE_STRING; + + printer->Print(variables, + "typedef ::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_pointer\n" + " ConstPtr;\n"); + if (string_key) { + printer->Print(variables, + "typedef ConstPtr SortItem;\n" + "typedef ::google::protobuf::internal::" + "CompareByDerefFirst Less;\n"); + } else { + printer->Print(variables, + "typedef ::google::protobuf::internal::SortItem< $key_cpp$, ConstPtr > " + "SortItem;\n" + "typedef ::google::protobuf::internal::CompareByFirstField Less;\n"); } - if (value_field->type() == FieldDescriptor::TYPE_STRING) { - GenerateUtf8CheckCodeForString(value_field, options_, false, variables_, - "it->second.data(), it->second.length(),\n", - printer); + string utf8_check; + if (string_key || string_value) { + printer->Print( + "struct Utf8Check {\n" + " static void Check(ConstPtr p) {\n"); + printer->Indent(); + printer->Indent(); + if (string_key) { + GenerateUtf8CheckCodeForString(key_field, options_, false, variables, + "p->first.data(), p->first.length(),\n", + printer); + } + if (string_value) { + GenerateUtf8CheckCodeForString(value_field, options_, false, variables, + "p->second.data(), p->second.length(),\n", + printer); + } + printer->Outdent(); + printer->Outdent(); + printer->Print( + " }\n" + "};\n"); + utf8_check = "Utf8Check::Check"; } - printer->Outdent(); + printer->Print(variables, + "\n" + "if ($deterministic$ &&\n" + " this->$name$().size() > 1) {\n" + " ::google::protobuf::scoped_array items(\n" + " new SortItem[this->$name$().size()]);\n" + " typedef ::google::protobuf::Map< $key_cpp$, $val_cpp$ >::size_type size_type;\n" + " size_type n = 0;\n" + " for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n" + " it = this->$name$().begin();\n" + " it != this->$name$().end(); ++it, ++n) {\n" + " items[n] = SortItem(&*it);\n" + " }\n" + " ::std::sort(&items[0], &items[n], Less());\n"); + printer->Indent(); + GenerateSerializationLoop(printer, variables, SupportsArenas(descriptor_), + utf8_check, "for (size_type i = 0; i < n; i++)", + string_key ? "items[i]" : "items[i].second", false); printer->Outdent(); printer->Print( - " }\n"); - - // If entry is allocated by arena, its desctructor should be avoided. - if (SupportsArenas(descriptor_)) { - printer->Print(variables_, - " if (entry.get() != NULL && entry->GetArena() != NULL) {\n" - " entry.release();\n" - " }\n"); - } - + "} else {\n"); + printer->Indent(); + GenerateSerializationLoop( + printer, variables, SupportsArenas(descriptor_), utf8_check, + "for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n" + " it = this->$name$().begin();\n" + " it != this->$name$().end(); ++it)", + "it", true); + printer->Outdent(); + printer->Print("}\n"); + printer->Outdent(); printer->Print("}\n"); } diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.h b/src/google/protobuf/compiler/cpp/cpp_map_field.h index 087dcde0..2930fe59 100644 --- a/src/google/protobuf/compiler/cpp/cpp_map_field.h +++ b/src/google/protobuf/compiler/cpp/cpp_map_field.h @@ -61,6 +61,10 @@ class MapFieldGenerator : public FieldGenerator { void GenerateByteSize(io::Printer* printer) const; private: + // A helper for GenerateSerializeWithCachedSizes{,ToArray}. + void GenerateSerializeWithCachedSizes( + io::Printer* printer, const map& variables) const; + const FieldDescriptor* descriptor_; const bool dependent_field_; map variables_; diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc index 32f63152..405a5fed 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message.cc @@ -2715,7 +2715,9 @@ GenerateMergeFrom(io::Printer* printer) { "void $classname$::MergeFrom(const ::google::protobuf::Message& from) {\n" "// @@protoc_insertion_point(generalized_merge_from_start:" "$full_name$)\n" - " if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);\n", + " if (GOOGLE_PREDICT_FALSE(&from == this)) {\n" + " ::google::protobuf::internal::MergeFromFail(__FILE__, __LINE__);\n" + " }\n", "classname", classname_, "full_name", descriptor_->full_name()); printer->Indent(); @@ -2756,7 +2758,9 @@ GenerateMergeFrom(io::Printer* printer) { "void $classname$::MergeFrom(const $classname$& from) {\n" "// @@protoc_insertion_point(class_specific_merge_from_start:" "$full_name$)\n" - " if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);\n", + " if (GOOGLE_PREDICT_FALSE(&from == this)) {\n" + " ::google::protobuf::internal::MergeFromFail(__FILE__, __LINE__);\n" + " }\n", "classname", classname_, "full_name", descriptor_->full_name()); printer->Indent(); diff --git a/src/google/protobuf/compiler/java/java_file.cc b/src/google/protobuf/compiler/java/java_file.cc index a06c5f6d..5e387285 100644 --- a/src/google/protobuf/compiler/java/java_file.cc +++ b/src/google/protobuf/compiler/java/java_file.cc @@ -266,9 +266,7 @@ void FileGenerator::Generate(io::Printer* printer) { printer->Print( "public static void registerAllExtensions(\n" - " com.google.protobuf.ExtensionRegistry$lite$ registry) {\n", - "lite", - HasDescriptorMethods(file_, context_->EnforceLite()) ? "" : "Lite"); + " com.google.protobuf.ExtensionRegistryLite registry) {\n"); printer->Indent(); @@ -283,6 +281,20 @@ void FileGenerator::Generate(io::Printer* printer) { printer->Outdent(); printer->Print( "}\n"); + if (HasDescriptorMethods(file_, context_->EnforceLite())) { + // Overload registerAllExtensions for the non-lite usage to + // redundantly maintain the original signature (this is + // redundant because ExtensionRegistryLite now invokes + // ExtensionRegistry in the non-lite usage). Intent is + // to remove this in the future. + printer->Print( + "\n" + "public static void registerAllExtensions(\n" + " com.google.protobuf.ExtensionRegistry registry) {\n" + " registerAllExtensions(\n" + " (com.google.protobuf.ExtensionRegistryLite) registry);\n" + "}\n"); + } // ----------------------------------------------------------------- diff --git a/src/google/protobuf/compiler/java/java_generator.cc b/src/google/protobuf/compiler/java/java_generator.cc index 3c545e15..b1ab4043 100644 --- a/src/google/protobuf/compiler/java/java_generator.cc +++ b/src/google/protobuf/compiler/java/java_generator.cc @@ -79,8 +79,6 @@ bool JavaGenerator::Generate(const FileDescriptor* file, file_options.generate_mutable_code = true; } else if (options[i].first == "shared") { file_options.generate_shared_code = true; - } else if (options[i].first == "lite") { - file_options.enforce_lite = true; } else if (options[i].first == "annotate_code") { file_options.annotate_code = true; } else if (options[i].first == "annotation_list_file") { diff --git a/src/google/protobuf/compiler/js/js_generator.cc b/src/google/protobuf/compiler/js/js_generator.cc index 8a2633d7..ad8cb166 100755 --- a/src/google/protobuf/compiler/js/js_generator.cc +++ b/src/google/protobuf/compiler/js/js_generator.cc @@ -2031,9 +2031,8 @@ void Generator::GenerateClassFieldToObject(const GeneratorOptions& options, "getter", JSGetterName(options, field, BYTES_B64)); } else { if (field->has_default_value()) { - printer->Print("jspb.Message.getField(msg, $index$) == null ? " - "$defaultValue$ : ", - "index", JSFieldIndex(field), + printer->Print("!msg.has$name$() ? $defaultValue$ : ", + "name", JSGetterName(options, field), "defaultValue", JSFieldDefault(field)); } if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT || @@ -2408,9 +2407,8 @@ void Generator::GenerateClassField(const GeneratorOptions& options, "default", Proto3PrimitiveFieldDefault(field)); } else { if (field->has_default_value()) { - printer->Print("jspb.Message.getField(this, $index$) == null ? " - "$defaultValue$ : ", - "index", JSFieldIndex(field), + printer->Print("!this.has$name$() ? $defaultValue$ : ", + "name", JSGetterName(options, field), "defaultValue", JSFieldDefault(field)); } if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT || @@ -2515,6 +2513,20 @@ void Generator::GenerateClassField(const GeneratorOptions& options, "\n", "clearedvalue", (field->is_repeated() ? "[]" : "undefined"), "returnvalue", JSReturnClause(field)); + + printer->Print( + "/**\n" + " * Returns whether this field is set.\n" + " * @return{!boolean}\n" + " */\n" + "$class$.prototype.has$name$ = function() {\n" + " return jspb.Message.getField(this, $index$) != null;\n" + "};\n" + "\n" + "\n", + "class", GetPath(options, field->containing_type()), + "name", JSGetterName(options, field), + "index", JSFieldIndex(field)); } } } diff --git a/src/google/protobuf/compiler/python/python_generator.cc b/src/google/protobuf/compiler/python/python_generator.cc index cc54a8ab..d5468a0c 100644 --- a/src/google/protobuf/compiler/python/python_generator.cc +++ b/src/google/protobuf/compiler/python/python_generator.cc @@ -44,6 +44,7 @@ // performance-minded Python code leverage the fast C++ implementation // directly. +#include #include #include #include @@ -107,20 +108,25 @@ string ModuleAlias(const string& filename) { return module_name; } +// Keywords reserved by the Python language. +const char* const kKeywords[] = { + "False", "None", "True", "and", "as", "assert", "break", + "class", "continue", "def", "del", "elif", "else", "except", + "finally", "for", "from", "global", "if", "import", "in", + "is", "lambda", "nonlocal", "not", "or", "pass", "raise", + "return", "try", "while", "with", "yield", +}; +const char* const* kKeywordsEnd = + kKeywords + (sizeof(kKeywords) / sizeof(kKeywords[0])); -// Returns an import statement of form "from X.Y.Z import T" for the given -// .proto filename. -string ModuleImportStatement(const string& filename) { - string module_name = ModuleName(filename); - int last_dot_pos = module_name.rfind('.'); - if (last_dot_pos == string::npos) { - // NOTE(petya): this is not tested as it would require a protocol buffer - // outside of any package, and I don't think that is easily achievable. - return "import " + module_name; - } else { - return "from " + module_name.substr(0, last_dot_pos) + " import " + - module_name.substr(last_dot_pos + 1); +bool ContainsPythonKeyword(const string& module_name) { + vector tokens = Split(module_name, "."); + for (int i = 0; i < tokens.size(); ++i) { + if (std::find(kKeywords, kKeywordsEnd, tokens[i]) != kKeywordsEnd) { + return true; + } } + return false; } @@ -359,10 +365,32 @@ bool Generator::Generate(const FileDescriptor* file, void Generator::PrintImports() const { for (int i = 0; i < file_->dependency_count(); ++i) { const string& filename = file_->dependency(i)->name(); - string import_statement = ModuleImportStatement(filename); + + string module_name = ModuleName(filename); string module_alias = ModuleAlias(filename); - printer_->Print("$statement$ as $alias$\n", "statement", - import_statement, "alias", module_alias); + if (ContainsPythonKeyword(module_name)) { + // If the module path contains a Python keyword, we have to quote the + // module name and import it using importlib. Otherwise the usual kind of + // import statement would result in a syntax error from the presence of + // the keyword. + printer_->Print("import importlib\n"); + printer_->Print("$alias$ = importlib.import_module('$name$')\n", "alias", + module_alias, "name", module_name); + } else { + int last_dot_pos = module_name.rfind('.'); + string import_statement; + if (last_dot_pos == string::npos) { + // NOTE(petya): this is not tested as it would require a protocol buffer + // outside of any package, and I don't think that is easily achievable. + import_statement = "import " + module_name; + } else { + import_statement = "from " + module_name.substr(0, last_dot_pos) + + " import " + module_name.substr(last_dot_pos + 1); + } + printer_->Print("$statement$ as $alias$\n", "statement", import_statement, + "alias", module_alias); + } + CopyPublicDependenciesAliases(module_alias, file_->dependency(i)); } printer_->Print("\n"); diff --git a/src/google/protobuf/compiler/python/python_plugin_unittest.cc b/src/google/protobuf/compiler/python/python_plugin_unittest.cc index 23f2449c..34f857fd 100644 --- a/src/google/protobuf/compiler/python/python_plugin_unittest.cc +++ b/src/google/protobuf/compiler/python/python_plugin_unittest.cc @@ -46,6 +46,7 @@ #include #include +#include #include #include @@ -115,6 +116,53 @@ TEST(PythonPluginTest, PluginTest) { EXPECT_EQ(0, cli.Run(5, argv)); } +// This test verifies that the generated Python output uses regular imports (as +// opposed to importlib) in the usual case where the .proto file paths do not +// not contain any Python keywords. +TEST(PythonPluginTest, ImportTest) { + // Create files test1.proto and test2.proto with the former importing the + // latter. + GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/test1.proto", + "syntax = \"proto3\";\n" + "package foo;\n" + "import \"test2.proto\";" + "message Message1 {\n" + " Message2 message_2 = 1;\n" + "}\n", + true)); + GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/test2.proto", + "syntax = \"proto3\";\n" + "package foo;\n" + "message Message2 {}\n", + true)); + + google::protobuf::compiler::CommandLineInterface cli; + cli.SetInputsAreProtoPathRelative(true); + python::Generator python_generator; + cli.RegisterGenerator("--python_out", &python_generator, ""); + string proto_path = "-I" + TestTempDir(); + string python_out = "--python_out=" + TestTempDir(); + const char* argv[] = {"protoc", proto_path.c_str(), "-I.", python_out.c_str(), + "test1.proto"}; + ASSERT_EQ(0, cli.Run(5, argv)); + + // Loop over the lines of the generated code and verify that we find an + // ordinary Python import but do not find the string "importlib". + string output; + GOOGLE_CHECK_OK(File::GetContents(TestTempDir() + "/test1_pb2.py", &output, + true)); + std::vector lines = Split(output, "\n"); + string expected_import = "import test2_pb2"; + bool found_expected_import = false; + for (int i = 0; i < lines.size(); ++i) { + if (lines[i].find(expected_import) != string::npos) { + found_expected_import = true; + } + EXPECT_EQ(string::npos, lines[i].find("importlib")); + } + EXPECT_TRUE(found_expected_import); +} + } // namespace } // namespace python } // namespace compiler diff --git a/src/google/protobuf/descriptor.proto b/src/google/protobuf/descriptor.proto index da853dbc..28410d4a 100644 --- a/src/google/protobuf/descriptor.proto +++ b/src/google/protobuf/descriptor.proto @@ -45,6 +45,7 @@ option java_package = "com.google.protobuf"; option java_outer_classname = "DescriptorProtos"; option csharp_namespace = "Google.Protobuf.Reflection"; option objc_class_prefix = "GPB"; +option java_generate_equals_and_hash = true; // descriptor.proto must be optimized for speed because reflection-based // algorithms don't work during bootstrapping. diff --git a/src/google/protobuf/extension_set_heavy.cc b/src/google/protobuf/extension_set_heavy.cc index 5dd171ed..b26a246c 100644 --- a/src/google/protobuf/extension_set_heavy.cc +++ b/src/google/protobuf/extension_set_heavy.cc @@ -341,7 +341,7 @@ bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input, int ExtensionSet::SpaceUsedExcludingSelf() const { int total_size = - extensions_.size() * sizeof(map::value_type); + extensions_.size() * sizeof(ExtensionMap::value_type); for (ExtensionMap::const_iterator iter = extensions_.begin(), end = extensions_.end(); iter != end; diff --git a/src/google/protobuf/generated_message_util.cc b/src/google/protobuf/generated_message_util.cc index 7b813f8a..7ad6d61c 100644 --- a/src/google/protobuf/generated_message_util.cc +++ b/src/google/protobuf/generated_message_util.cc @@ -73,6 +73,12 @@ int StringSpaceUsedExcludingSelf(const string& str) { +void MergeFromFail(const char* file, int line) { + GOOGLE_CHECK(false) << file << ":" << line; + // Open-source GOOGLE_CHECK(false) is not NORETURN. + exit(1); +} + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/generated_message_util.h b/src/google/protobuf/generated_message_util.h index 56f5afd2..dc815189 100644 --- a/src/google/protobuf/generated_message_util.h +++ b/src/google/protobuf/generated_message_util.h @@ -41,8 +41,8 @@ #include #include -#include #include +#include namespace google { @@ -115,6 +115,10 @@ class ArenaString; ArenaString* ReadArenaString(::google::protobuf::io::CodedInputStream* input, ::google::protobuf::Arena* arena); +// Helper function to crash on merge failure. +// Moved out of generated code to reduce binary size. +void MergeFromFail(const char* file, int line) GOOGLE_ATTRIBUTE_NORETURN; + } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/io/coded_stream.cc b/src/google/protobuf/io/coded_stream.cc index 148eee0e..a5675e79 100644 --- a/src/google/protobuf/io/coded_stream.cc +++ b/src/google/protobuf/io/coded_stream.cc @@ -649,13 +649,16 @@ bool CodedInputStream::Refresh() { // CodedOutputStream ================================================= +bool CodedOutputStream::default_serialization_deterministic_ = false; + CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output) : output_(output), buffer_(NULL), buffer_size_(0), total_bytes_(0), had_error_(false), - aliasing_enabled_(false) { + aliasing_enabled_(false), + serialization_deterministic_is_overridden_(false) { // Eagerly Refresh() so buffer space is immediately available. Refresh(); // The Refresh() may have failed. If the client doesn't write any data, @@ -671,7 +674,8 @@ CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output, buffer_size_(0), total_bytes_(0), had_error_(false), - aliasing_enabled_(false) { + aliasing_enabled_(false), + serialization_deterministic_is_overridden_(false) { if (do_eager_refresh) { // Eagerly Refresh() so buffer space is immediately available. Refresh(); diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index eb320745..316da765 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -813,6 +813,44 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { // created. bool HadError() const { return had_error_; } + // Deterministic serialization, if requested, guarantees that for a given + // binary, equal messages will always be serialized to the same bytes. This + // implies: + // . repeated serialization of a message will return the same bytes + // . different processes of the same binary (which may be executing on + // different machines) will serialize equal messages to the same bytes. + // + // Note the deterministic serialization is NOT canonical across languages; it + // is also unstable across different builds with schema changes due to unknown + // fields. Users who need canonical serialization, e.g., persistent storage in + // a canonical form, fingerprinting, etc., should define their own + // canonicalization specification and implement the serializer using + // reflection APIs rather than relying on this API. + // + // If determinisitc serialization is requested, the serializer will + // sort map entries by keys in lexicographical order or numerical order. + // (This is an implementation detail and may subject to change.) + // + // There are two ways to determine whether serialization should be + // deterministic for this CodedOutputStream. If SetSerializationDeterministic + // has not yet been called, then the default comes from the global default, + // which is false, until SetDefaultSerializationDeterministic has been called. + // Otherwise, SetSerializationDeterministic has been called, and the last + // value passed to it is all that matters. + void SetSerializationDeterministic(bool value) { + serialization_deterministic_is_overridden_ = true; + serialization_deterministic_override_ = value; + } + // See above. Also, note that users of this CodedOutputStream may need to + // call IsSerializationDeterminstic() to serialize in the intended way. This + // CodedOutputStream cannot enforce a desire for deterministic serialization + // by itself. + bool IsSerializationDeterminstic() const { + return serialization_deterministic_is_overridden_ ? + serialization_deterministic_override_ : + default_serialization_deterministic_; + } + private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedOutputStream); @@ -822,6 +860,10 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { int total_bytes_; // Sum of sizes of all buffers seen so far. bool had_error_; // Whether an error occurred during output. bool aliasing_enabled_; // See EnableAliasing(). + // See SetSerializationDeterministic() regarding these three fields. + bool serialization_deterministic_is_overridden_; + bool serialization_deterministic_override_; + static bool default_serialization_deterministic_; // Advance the buffer by a given number of bytes. void Advance(int amount); @@ -849,6 +891,11 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { uint64 value, uint8* target); static int VarintSize32Fallback(uint32 value); + + // See above. Other projects may use "friend" to allow them to call this. + static void SetDefaultSerializationDeterministic() { + default_serialization_deterministic_ = true; + } }; // inline methods ==================================================== diff --git a/src/google/protobuf/map_entry_lite.h b/src/google/protobuf/map_entry_lite.h index 23ac7b8a..4dedfd57 100644 --- a/src/google/protobuf/map_entry_lite.h +++ b/src/google/protobuf/map_entry_lite.h @@ -535,6 +535,32 @@ class MapEntryLite : public MessageLite { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MapEntryLite); }; +// Helpers for deterministic serialization ============================= + +// This struct can be used with any generic sorting algorithm. If the Key +// type is relatively small and easy to copy then copying Keys into an +// array of SortItems can be beneficial. Then all the data the sorting +// algorithm needs to touch is in that one array. +template struct SortItem { + SortItem() {} + explicit SortItem(PtrToKeyValuePair p) : first(p->first), second(p) {} + + Key first; + PtrToKeyValuePair second; +}; + +template struct CompareByFirstField { + bool operator()(const T& a, const T& b) const { + return a.first < b.first; + } +}; + +template struct CompareByDerefFirst { + bool operator()(const T& a, const T& b) const { + return a->first < b->first; + } +}; + } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/map_proto2_unittest.proto b/src/google/protobuf/map_proto2_unittest.proto index 916cc546..ddc2a582 100644 --- a/src/google/protobuf/map_proto2_unittest.proto +++ b/src/google/protobuf/map_proto2_unittest.proto @@ -64,3 +64,23 @@ message TestEnumMapPlusExtra { message TestImportEnumMap { map import_enum_amp = 1; } + +message TestIntIntMap { + map m = 1; +} + +// Test all key types: string, plus the non-floating-point scalars. +message TestMaps { + map m_int32 = 1; + map m_int64 = 2; + map m_uint32 = 3; + map m_uint64 = 4; + map m_sint32 = 5; + map m_sint64 = 6; + map m_fixed32 = 7; + map m_fixed64 = 8; + map m_sfixed32 = 9; + map m_sfixed64 = 10; + map m_bool = 11; + map m_string = 12; +} diff --git a/src/google/protobuf/map_test.cc b/src/google/protobuf/map_test.cc index cdd1ccd5..03954e75 100644 --- a/src/google/protobuf/map_test.cc +++ b/src/google/protobuf/map_test.cc @@ -74,6 +74,7 @@ #include #include #include +#include #include #include #include @@ -2869,6 +2870,82 @@ TEST(WireFormatForMapFieldTest, MapParseHelpers) { } } +// Deterministic Serialization Test ========================================== + +template +static string DeterministicSerialization(const T& t) { + const int size = t.ByteSize(); + string result(size, '\0'); + io::ArrayOutputStream array_stream(string_as_array(&result), size); + io::CodedOutputStream output_stream(&array_stream); + output_stream.SetSerializationDeterministic(true); + t.SerializeWithCachedSizes(&output_stream); + EXPECT_FALSE(output_stream.HadError()); + EXPECT_EQ(size, output_stream.ByteCount()); + return result; +} + +// Helper to test the serialization of the first arg against a golden file. +static void TestDeterministicSerialization(const protobuf_unittest::TestMaps& t, + const string& filename) { + string expected; + GOOGLE_CHECK_OK(File::GetContents( + TestSourceDir() + "/google/protobuf/testdata/" + filename, + &expected, true)); + const string actual = DeterministicSerialization(t); + EXPECT_EQ(expected, actual); + protobuf_unittest::TestMaps u; + EXPECT_TRUE(u.ParseFromString(actual)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(u, t)); +} + +// Helper for MapSerializationTest. Return a 7-bit ASCII string. +static string ConstructKey(uint64 n) { + string s(n % static_cast(9), '\0'); + if (s.empty()) { + return StrCat(n); + } else { + while (n != 0) { + s[n % s.size()] = (n >> 10) & 0x7f; + n /= 888; + } + return s; + } +} + +TEST(MapSerializationTest, Deterministic) { + const int kIters = 25; + protobuf_unittest::TestMaps t; + protobuf_unittest::TestIntIntMap inner; + (*inner.mutable_m())[0] = (*inner.mutable_m())[10] = + (*inner.mutable_m())[-200] = 0; + uint64 frog = 9; + const uint64 multiplier = 0xa29cd16f; + for (int i = 0; i < kIters; i++) { + const int32 i32 = static_cast(frog & 0xffffffff); + const uint32 u32 = static_cast(i32) * 91919; + const int64 i64 = static_cast(frog); + const uint64 u64 = frog * static_cast(187321); + const bool b = i32 > 0; + const string s = ConstructKey(frog); + (*inner.mutable_m())[i] = i32; + (*t.mutable_m_int32())[i32] = (*t.mutable_m_sint32())[i32] = + (*t.mutable_m_sfixed32())[i32] = inner; + (*t.mutable_m_uint32())[u32] = (*t.mutable_m_fixed32())[u32] = inner; + (*t.mutable_m_int64())[i64] = (*t.mutable_m_sint64())[i64] = + (*t.mutable_m_sfixed64())[i64] = inner; + (*t.mutable_m_uint64())[u64] = (*t.mutable_m_fixed64())[u64] = inner; + (*t.mutable_m_bool())[b] = inner; + (*t.mutable_m_string())[s] = inner; + (*t.mutable_m_string())[s + string(1 << (u32 % static_cast(9)), + b)] = inner; + inner.mutable_m()->erase(i); + frog = frog * multiplier + i; + frog ^= (frog >> 41); + } + TestDeterministicSerialization(t, "golden_message_maps"); +} + // Text Format Test ================================================= TEST(TextFormatMapTest, SerializeAndParse) { diff --git a/src/google/protobuf/map_type_handler.h b/src/google/protobuf/map_type_handler.h index 74e8bb50..685a770f 100644 --- a/src/google/protobuf/map_type_handler.h +++ b/src/google/protobuf/map_type_handler.h @@ -166,10 +166,10 @@ class MapTypeHandler { io::CodedOutputStream* output); static inline uint8* InternalWriteToArray(int field, const MapEntryAccessorType& value, - bool deterministic, uint8* output); + bool deterministic, uint8* target); static inline uint8* WriteToArray(int field, const MapEntryAccessorType& value, - uint8* output); + uint8* target); // Functions to manipulate data on memory. ======================== static inline const Type& GetExternalReference(const Type* value); @@ -227,11 +227,11 @@ class MapTypeHandler { int field, \ const MapEntryAccessorType& value, \ bool deterministic, \ - uint8* output); \ + uint8* target); \ static inline uint8* WriteToArray(int field, \ const MapEntryAccessorType& value, \ - uint8* output) { \ - return InternalWriteToArray(field, value, false, output); \ + uint8* target) { \ + return InternalWriteToArray(field, value, false, target); \ } \ static inline const MapEntryAccessorType& GetExternalReference( \ const TypeOnMemory& value); \ @@ -374,9 +374,9 @@ template inline uint8* MapTypeHandler::InternalWriteToArray( int field, const MapEntryAccessorType& value, bool deterministic, - uint8* output) { + uint8* target) { return WireFormatLite::InternalWriteMessageToArray(field, value, - deterministic, output); + deterministic, target); } #define WRITE_METHOD(FieldType, DeclaredType) \ @@ -390,8 +390,8 @@ MapTypeHandler::InternalWriteToArray( inline uint8* \ MapTypeHandler::InternalWriteToArray( \ - int field, const MapEntryAccessorType& value, bool, uint8* output) { \ - return WireFormatLite::Write##DeclaredType##ToArray(field, value, output); \ + int field, const MapEntryAccessorType& value, bool, uint8* target) { \ + return WireFormatLite::Write##DeclaredType##ToArray(field, value, target); \ } WRITE_METHOD(STRING , String) diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index d62ca79c..f18077dd 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -62,8 +62,6 @@ namespace protobuf { using internal::WireFormat; using internal::ReflectionOps; -Message::~Message() {} - void Message::MergeFrom(const Message& from) { const Descriptor* descriptor = GetDescriptor(); GOOGLE_CHECK_EQ(from.GetDescriptor(), descriptor) diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index dcdffe1c..9705e97e 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -179,7 +179,7 @@ struct Metadata { class LIBPROTOBUF_EXPORT Message : public MessageLite { public: inline Message() {} - virtual ~Message(); + virtual ~Message() {} // Basic Operations ------------------------------------------------ diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 3913be1b..ba56db95 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -46,8 +46,6 @@ namespace google { namespace protobuf { -MessageLite::~MessageLite() {} - string MessageLite::InitializationErrorString() const { return "(cannot determine missing fields for lite message)"; } diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index f606aeec..2bdfe496 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -81,7 +81,7 @@ namespace internal { class LIBPROTOBUF_EXPORT MessageLite { public: inline MessageLite() {} - virtual ~MessageLite(); + virtual ~MessageLite() {} // Basic Operations ------------------------------------------------ diff --git a/src/google/protobuf/testdata/golden_message_maps b/src/google/protobuf/testdata/golden_message_maps new file mode 100644 index 00000000..c70a4d7c Binary files /dev/null and b/src/google/protobuf/testdata/golden_message_maps differ diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index d49d8588..66b2648b 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -759,6 +759,20 @@ class TextFormat::Parser::ParserImpl { } return true; } + if (TryConsume("[")) { + while (true) { + if (!LookingAt("{") && !LookingAt("<")) { + DO(SkipFieldValue()); + } else { + DO(SkipFieldMessage()); + } + if (TryConsume("]")) { + break; + } + DO(Consume(",")); + } + return true; + } // Possible field values other than string: // 12345 => TYPE_INTEGER // -12345 => TYPE_SYMBOL + TYPE_INTEGER diff --git a/src/google/protobuf/unknown_field_set.cc b/src/google/protobuf/unknown_field_set.cc index d4e383da..8ee99d48 100644 --- a/src/google/protobuf/unknown_field_set.cc +++ b/src/google/protobuf/unknown_field_set.cc @@ -69,28 +69,14 @@ const UnknownFieldSet* UnknownFieldSet::default_instance() { return default_unknown_field_set_instance_; } -UnknownFieldSet::UnknownFieldSet() - : fields_(NULL) {} - -UnknownFieldSet::~UnknownFieldSet() { - Clear(); - delete fields_; -} - void UnknownFieldSet::ClearFallback() { - if (fields_ != NULL) { - for (int i = 0; i < fields_->size(); i++) { - (*fields_)[i].Delete(); - } - delete fields_; - fields_ = NULL; - } -} - -void UnknownFieldSet::ClearAndFreeMemory() { - if (fields_ != NULL) { - Clear(); - } + GOOGLE_DCHECK(fields_ != NULL && fields_->size() > 0); + int n = fields_->size(); + do { + (*fields_)[--n].Delete(); + } while (n > 0); + delete fields_; + fields_ = NULL; } void UnknownFieldSet::InternalMergeFrom(const UnknownFieldSet& other) { diff --git a/src/google/protobuf/unknown_field_set.h b/src/google/protobuf/unknown_field_set.h index 612a942a..aa752916 100644 --- a/src/google/protobuf/unknown_field_set.h +++ b/src/google/protobuf/unknown_field_set.h @@ -241,8 +241,14 @@ class LIBPROTOBUF_EXPORT UnknownField { // =================================================================== // inline implementations +inline UnknownFieldSet::UnknownFieldSet() : fields_(NULL) {} + +inline UnknownFieldSet::~UnknownFieldSet() { Clear(); } + +inline void UnknownFieldSet::ClearAndFreeMemory() { Clear(); } + inline void UnknownFieldSet::Clear() { - if (fields_) { + if (fields_ != NULL) { ClearFallback(); } } diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.cc b/src/google/protobuf/util/internal/default_value_objectwriter.cc index 21d7a2e4..1e8dab70 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter.cc @@ -64,6 +64,7 @@ DefaultValueObjectWriter::DefaultValueObjectWriter( type_(type), current_(NULL), root_(NULL), + suppress_empty_list_(false), field_scrub_callback_(NULL), ow_(ow) {} @@ -184,12 +185,10 @@ void DefaultValueObjectWriter::RegisterFieldScrubCallBack( field_scrub_callback_.reset(field_scrub_callback.release()); } -DefaultValueObjectWriter::Node::Node(const string& name, - const google::protobuf::Type* type, - NodeKind kind, const DataPiece& data, - bool is_placeholder, - const vector& path, - FieldScrubCallBack* field_scrub_callback) +DefaultValueObjectWriter::Node::Node( + const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, const vector& path, + bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback) : name_(name), type_(type), kind_(kind), @@ -197,6 +196,7 @@ DefaultValueObjectWriter::Node::Node(const string& name, data_(data), is_placeholder_(is_placeholder), path_(path), + suppress_empty_list_(suppress_empty_list), field_scrub_callback_(field_scrub_callback) {} DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild( @@ -230,6 +230,9 @@ void DefaultValueObjectWriter::Node::WriteTo(ObjectWriter* ow) { // Write out lists. If we didn't have any list in response, write out empty // list. if (kind_ == LIST) { + // Suppress empty lists if requested. + if (suppress_empty_list_ && is_placeholder_) return; + ow->StartList(name_); WriteChildren(ow); ow->EndList(); @@ -366,7 +369,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren( field.json_name(), field_type, kind, kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo) : DataPiece::NullData(), - true, path, field_scrub_callback_)); + true, path, suppress_empty_list_, field_scrub_callback_)); new_children.push_back(child.release()); } // Adds all leftover nodes in children_ to the beginning of new_child. @@ -462,7 +465,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject( if (current_ == NULL) { vector path; root_.reset(new Node(name.ToString(), &type_, OBJECT, DataPiece::NullData(), - false, path, field_scrub_callback_.get())); + false, path, suppress_empty_list_, + field_scrub_callback_.get())); root_->PopulateChildren(typeinfo_); current_ = root_.get(); return this; @@ -478,7 +482,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject( : NULL), OBJECT, DataPiece::NullData(), false, child == NULL ? current_->path() : child->path(), - field_scrub_callback_.get())); + suppress_empty_list_, field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } @@ -509,7 +513,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList( if (current_ == NULL) { vector path; root_.reset(new Node(name.ToString(), &type_, LIST, DataPiece::NullData(), - false, path, field_scrub_callback_.get())); + false, path, suppress_empty_list_, + field_scrub_callback_.get())); current_ = root_.get(); return this; } @@ -519,7 +524,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList( google::protobuf::scoped_ptr node( new Node(name.ToString(), NULL, LIST, DataPiece::NullData(), false, child == NULL ? current_->path() : child->path(), - field_scrub_callback_.get())); + suppress_empty_list_, field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } @@ -577,7 +582,7 @@ void DefaultValueObjectWriter::RenderDataPiece(StringPiece name, google::protobuf::scoped_ptr node( new Node(name.ToString(), NULL, PRIMITIVE, data, false, child == NULL ? current_->path() : child->path(), - field_scrub_callback_.get())); + suppress_empty_list_, field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } else { diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.h b/src/google/protobuf/util/internal/default_value_objectwriter.h index 1d85bed8..5f3b25f3 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.h +++ b/src/google/protobuf/util/internal/default_value_objectwriter.h @@ -122,6 +122,10 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // field_scrub_callback pointer is also transferred to this class void RegisterFieldScrubCallBack(FieldScrubCallBackPtr field_scrub_callback); + // If set to true, empty lists are suppressed from output when default values + // are written. + void set_suppress_empty_list(bool value) { suppress_empty_list_ = value; } + private: enum NodeKind { PRIMITIVE = 0, @@ -136,7 +140,7 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { public: Node(const string& name, const google::protobuf::Type* type, NodeKind kind, const DataPiece& data, bool is_placeholder, const vector& path, - FieldScrubCallBack* field_scrub_callback); + bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback); virtual ~Node() { for (int i = 0; i < children_.size(); ++i) { delete children_[i]; @@ -212,6 +216,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // Path of the field of this node std::vector path_; + // Whether to suppress empty list output. + bool suppress_empty_list_; + // Pointer to function for determining whether a field needs to be scrubbed // or not. This callback is owned by the creator of this node. FieldScrubCallBack* field_scrub_callback_; @@ -257,6 +264,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // The stack to hold the path of Nodes from current_ to root_; std::stack stack_; + // Whether to suppress output of empty lists. + bool suppress_empty_list_; + // Unique Pointer to function for determining whether a field needs to be // scrubbed or not. FieldScrubCallBackPtr field_scrub_callback_; diff --git a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc index 8254c0fa..e1dd697a 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc @@ -149,6 +149,39 @@ TEST_P(DefaultValueObjectWriterTest, ShouldRetainUnknownField) { } +class DefaultValueObjectWriterSuppressListTest + : public BaseDefaultValueObjectWriterTest { + protected: + DefaultValueObjectWriterSuppressListTest() + : BaseDefaultValueObjectWriterTest(DefaultValueTest::descriptor()) { + testing_->set_suppress_empty_list(true); + } + ~DefaultValueObjectWriterSuppressListTest() {} +}; + +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + DefaultValueObjectWriterSuppressListTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + +TEST_P(DefaultValueObjectWriterSuppressListTest, Empty) { + // Set expectation. Emtpy lists should be suppressed. + expects_.StartObject("") + ->RenderDouble("doubleValue", 0.0) + ->RenderFloat("floatValue", 0.0) + ->RenderInt64("int64Value", 0) + ->RenderUint64("uint64Value", 0) + ->RenderInt32("int32Value", 0) + ->RenderUint32("uint32Value", 0) + ->RenderBool("boolValue", false) + ->RenderString("stringValue", "") + ->RenderBytes("bytesValue", "") + ->RenderString("enumValue", "ENUM_FIRST") + ->EndObject(); + + // Actual testing + testing_->StartObject("")->EndObject(); +} } // namespace testing } // namespace converter } // namespace util diff --git a/src/google/protobuf/util/internal/proto_writer.cc b/src/google/protobuf/util/internal/proto_writer.cc index 7a1a6cbd..0c38aeb9 100644 --- a/src/google/protobuf/util/internal/proto_writer.cc +++ b/src/google/protobuf/util/internal/proto_writer.cc @@ -298,7 +298,9 @@ ProtoWriter::ProtoElement::ProtoElement(const TypeInfo* typeinfo, proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3), type_(type), size_index_(-1), - array_index_(-1) { + array_index_(-1), + // oneof_indices_ values are 1-indexed (0 means not present). + oneof_indices_(type.oneofs_size() + 1) { if (!proto3_) { required_fields_ = GetRequiredFields(type_); } @@ -312,13 +314,15 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent, ow_(this->parent()->ow_), parent_field_(field), typeinfo_(this->parent()->typeinfo_), - proto3_(this->parent()->proto3_), + proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3), type_(type), size_index_( !is_list && field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE ? ow_->size_insert_.size() : -1), - array_index_(is_list ? 0 : -1) { + array_index_(is_list ? 0 : -1), + // oneof_indices_ values are 1-indexed (0 means not present). + oneof_indices_(type_.oneofs_size() + 1) { if (!is_list) { if (ow_->IsRepeated(*field)) { // Update array_index_ if it is an explicit list. @@ -411,11 +415,11 @@ string ProtoWriter::ProtoElement::ToString() const { } bool ProtoWriter::ProtoElement::IsOneofIndexTaken(int32 index) { - return ContainsKey(oneof_indices_, index); + return oneof_indices_[index]; } void ProtoWriter::ProtoElement::TakeOneofIndex(int32 index) { - InsertIfNotPresent(&oneof_indices_, index); + oneof_indices_[index] = true; } void ProtoWriter::InvalidName(StringPiece unknown_name, StringPiece message) { @@ -573,10 +577,19 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField( // Pushing a ProtoElement and then pop it off at the end for 2 purposes: // error location reporting and required field accounting. - element_.reset(new ProtoElement(element_.release(), &field, type, false)); + // + // For proto3, since there is no required field tracking, we only need to push + // ProtoElement for error cases. + if (!element_->proto3()) { + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + } if (field.kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN || field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { + // Push a ProtoElement for location reporting purposes. + if (element_->proto3()) { + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + } InvalidValue(field.type_url().empty() ? google::protobuf::Field_Kind_Name(field.kind()) : field.type_url(), @@ -657,11 +670,18 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField( } if (!status.ok()) { + // Push a ProtoElement for location reporting purposes. + if (element_->proto3()) { + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + } InvalidValue(google::protobuf::Field_Kind_Name(field.kind()), status.error_message()); + element_.reset(element()->pop()); + return this; } - element_.reset(element()->pop()); + if (!element_->proto3()) element_.reset(element()->pop()); + return this; } diff --git a/src/google/protobuf/util/internal/proto_writer.h b/src/google/protobuf/util/internal/proto_writer.h index 8b7c6c34..7f1108ab 100644 --- a/src/google/protobuf/util/internal/proto_writer.h +++ b/src/google/protobuf/util/internal/proto_writer.h @@ -32,8 +32,8 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_PROTO_WRITER_H__ #include -#include #include +#include #include #include @@ -45,6 +45,7 @@ #include #include #include +#include namespace google { namespace protobuf { @@ -191,6 +192,8 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // generate an error. void TakeOneofIndex(int32 index); + bool proto3() { return proto3_; } + private: // Used for access to variables of the enclosing instance of // ProtoWriter. @@ -203,7 +206,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // TypeInfo to lookup types. const TypeInfo* typeinfo_; - // Whether the root type is a proto3 or not. + // Whether the type_ is proto3 or not. bool proto3_; // Additional variables if this element is a message: @@ -221,7 +224,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // Set of oneof indices already seen for the type_. Used to validate // incoming messages so no more than one oneof is set. - hash_set oneof_indices_; + std::vector oneof_indices_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement); }; diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc index 6d45a4f9..d7ac2dba 100644 --- a/src/google/protobuf/util/json_util.cc +++ b/src/google/protobuf/util/json_util.cc @@ -177,6 +177,66 @@ util::Status JsonToBinaryString(TypeResolver* resolver, resolver, type_url, &input_stream, &output_stream, options); } +namespace { +const char* kTypeUrlPrefix = "type.googleapis.com"; +TypeResolver* generated_type_resolver_ = NULL; +GOOGLE_PROTOBUF_DECLARE_ONCE(generated_type_resolver_init_); + +string GetTypeUrl(const Message& message) { + return string(kTypeUrlPrefix) + "/" + message.GetDescriptor()->full_name(); +} + +void DeleteGeneratedTypeResolver() { delete generated_type_resolver_; } + +void InitGeneratedTypeResolver() { + generated_type_resolver_ = NewTypeResolverForDescriptorPool( + kTypeUrlPrefix, DescriptorPool::generated_pool()); + ::google::protobuf::internal::OnShutdown(&DeleteGeneratedTypeResolver); +} + +TypeResolver* GetGeneratedTypeResolver() { + ::google::protobuf::GoogleOnceInit(&generated_type_resolver_init_, &InitGeneratedTypeResolver); + return generated_type_resolver_; +} +} // namespace + +util::Status MessageToJsonString(const Message& message, string* output, + const JsonOptions& options) { + const DescriptorPool* pool = message.GetDescriptor()->file()->pool(); + TypeResolver* resolver = + pool == DescriptorPool::generated_pool() + ? GetGeneratedTypeResolver() + : NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool); + util::Status result = + BinaryToJsonString(resolver, GetTypeUrl(message), + message.SerializeAsString(), output, options); + if (pool != DescriptorPool::generated_pool()) { + delete resolver; + } + return result; +} + +util::Status JsonStringToMessage(const string& input, Message* message, + const JsonParseOptions& options) { + const DescriptorPool* pool = message->GetDescriptor()->file()->pool(); + TypeResolver* resolver = + pool == DescriptorPool::generated_pool() + ? GetGeneratedTypeResolver() + : NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool); + string binary; + util::Status result = JsonToBinaryString( + resolver, GetTypeUrl(*message), input, &binary, options); + if (result.ok() && !message->ParseFromString(binary)) { + result = + util::Status(util::error::INVALID_ARGUMENT, + "JSON transcoder produced invalid protobuf output."); + } + if (pool != DescriptorPool::generated_pool()) { + delete resolver; + } + return result; +} + } // namespace util } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/util/json_util.h b/src/google/protobuf/util/json_util.h index b4c2579b..170ae91b 100644 --- a/src/google/protobuf/util/json_util.h +++ b/src/google/protobuf/util/json_util.h @@ -33,6 +33,7 @@ #ifndef GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__ #define GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__ +#include #include #include @@ -69,6 +70,30 @@ struct JsonPrintOptions { // DEPRECATED. Use JsonPrintOptions instead. typedef JsonPrintOptions JsonOptions; +// Converts from protobuf message to JSON. This is a simple wrapper of +// BinaryToJsonString(). It will use the DescriptorPool of the passed-in +// message to resolve Any types. +util::Status MessageToJsonString(const Message& message, + string* output, + const JsonOptions& options); + +inline util::Status MessageToJsonString(const Message& message, + string* output) { + return MessageToJsonString(message, output, JsonOptions()); +} + +// Converts from JSON to protobuf message. This is a simple wrapper of +// JsonStringToBinary(). It will use the DescriptorPool of the passed-in +// message to resolve Any types. +util::Status JsonStringToMessage(const string& input, + Message* message, + const JsonParseOptions& options); + +inline util::Status JsonStringToMessage(const string& input, + Message* message) { + return JsonStringToMessage(input, message, JsonParseOptions()); +} + // Converts protobuf binary data to JSON. // The conversion will fail if: // 1. TypeResolver fails to resolve a type. diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index c7d5c59e..dacac5e0 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -34,6 +34,8 @@ #include #include +#include +#include #include #include #include @@ -63,28 +65,21 @@ static string GetTypeUrl(const Descriptor* message) { class JsonUtilTest : public testing::Test { protected: JsonUtilTest() { - resolver_.reset(NewTypeResolverForDescriptorPool( - kTypeUrlPrefix, DescriptorPool::generated_pool())); } string ToJson(const Message& message, const JsonPrintOptions& options) { string result; - GOOGLE_CHECK_OK(BinaryToJsonString(resolver_.get(), - GetTypeUrl(message.GetDescriptor()), - message.SerializeAsString(), &result, options)); + GOOGLE_CHECK_OK(MessageToJsonString(message, &result, options)); return result; } bool FromJson(const string& json, Message* message, const JsonParseOptions& options) { - string binary; - if (!JsonToBinaryString(resolver_.get(), - GetTypeUrl(message->GetDescriptor()), json, &binary, - options) - .ok()) { - return false; - } - return message->ParseFromString(binary); + return JsonStringToMessage(json, message, options).ok(); + } + + bool FromJson(const string& json, Message* message) { + return FromJson(json, message, JsonParseOptions()); } google::protobuf::scoped_ptr resolver_; @@ -189,6 +184,45 @@ TEST_F(JsonUtilTest, TestParseErrors) { EXPECT_FALSE(FromJson("{\"int32Value\":2147483648}", &m, options)); } +TEST_F(JsonUtilTest, TestDynamicMessage) { + // Some random message but good enough to test the wrapper functions. + string input = + "{\n" + " \"int32Value\": 1024,\n" + " \"repeatedInt32Value\": [1, 2],\n" + " \"messageValue\": {\n" + " \"value\": 2048\n" + " },\n" + " \"repeatedMessageValue\": [\n" + " {\"value\": 40}, {\"value\": 96}\n" + " ]\n" + "}\n"; + + // Create a new DescriptorPool with the same protos as the generated one. + DescriptorPoolDatabase database(*DescriptorPool::generated_pool()); + DescriptorPool pool(&database); + // A dynamic version of the test proto. + DynamicMessageFactory factory; + google::protobuf::scoped_ptr message(factory.GetPrototype( + pool.FindMessageTypeByName("proto3.TestMessage"))->New()); + EXPECT_TRUE(FromJson(input, message.get())); + + // Convert to generated message for easy inspection. + TestMessage generated; + EXPECT_TRUE(generated.ParseFromString(message->SerializeAsString())); + EXPECT_EQ(1024, generated.int32_value()); + ASSERT_EQ(2, generated.repeated_int32_value_size()); + EXPECT_EQ(1, generated.repeated_int32_value(0)); + EXPECT_EQ(2, generated.repeated_int32_value(1)); + EXPECT_EQ(2048, generated.message_value().value()); + ASSERT_EQ(2, generated.repeated_message_value_size()); + EXPECT_EQ(40, generated.repeated_message_value(0).value()); + EXPECT_EQ(96, generated.repeated_message_value(1).value()); + + JsonOptions options; + EXPECT_EQ(ToJson(generated, options), ToJson(*message, options)); +} + typedef pair Segment; // A ZeroCopyOutputStream that writes to multiple buffers. class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream { diff --git a/src/google/protobuf/wire_format_lite.cc b/src/google/protobuf/wire_format_lite.cc index f2517074..05cc0854 100644 --- a/src/google/protobuf/wire_format_lite.cc +++ b/src/google/protobuf/wire_format_lite.cc @@ -466,7 +466,8 @@ void WireFormatLite::WriteGroupMaybeToArray(int field_number, const int size = value.GetCachedSize(); uint8* target = output->GetDirectBufferForNBytesAndAdvance(size); if (target != NULL) { - uint8* end = value.SerializeWithCachedSizesToArray(target); + uint8* end = value.InternalSerializeWithCachedSizesToArray( + output->IsSerializationDeterminstic(), target); GOOGLE_DCHECK_EQ(end - target, size); } else { value.SerializeWithCachedSizes(output); @@ -482,7 +483,8 @@ void WireFormatLite::WriteMessageMaybeToArray(int field_number, output->WriteVarint32(size); uint8* target = output->GetDirectBufferForNBytesAndAdvance(size); if (target != NULL) { - uint8* end = value.SerializeWithCachedSizesToArray(target); + uint8* end = value.InternalSerializeWithCachedSizesToArray( + output->IsSerializationDeterminstic(), target); GOOGLE_DCHECK_EQ(end - target, size); } else { value.SerializeWithCachedSizes(output);