Integrate from internal code base.

cmake/examples.cmake

@@ -1,57 +1,57 @@
-if(protobuf_VERBOSE)
+if(protobuf_VERBOSE)
-  message(STATUS "Protocol Buffers Examples Configuring...")
+  message(STATUS "Protocol Buffers Examples Configuring...")
-endif()
+endif()
-
+
-get_filename_component(examples_dir "../examples" ABSOLUTE)
+get_filename_component(examples_dir "../examples" ABSOLUTE)
-
+
-if(protobuf_VERBOSE)
+if(protobuf_VERBOSE)
-  message(STATUS "Protocol Buffers Examples Configuring done")
+  message(STATUS "Protocol Buffers Examples Configuring done")
-endif()
+endif()
-include(ExternalProject)
+include(ExternalProject)
-
+
-# Internal utility function: Create a custom target representing a build of examples with custom options.
+# Internal utility function: Create a custom target representing a build of examples with custom options.
-function(add_examples_build NAME)
+function(add_examples_build NAME)
-
+
-  ExternalProject_Add(${NAME}
+  ExternalProject_Add(${NAME}
-    PREFIX ${NAME}
+    PREFIX ${NAME}
-    SOURCE_DIR "${examples_dir}"
+    SOURCE_DIR "${examples_dir}"
-    BINARY_DIR ${NAME}
+    BINARY_DIR ${NAME}
-    STAMP_DIR ${NAME}/logs
+    STAMP_DIR ${NAME}/logs
-    INSTALL_COMMAND "" #Skip
+    INSTALL_COMMAND "" #Skip
-    LOG_CONFIGURE 1
+    LOG_CONFIGURE 1
-    CMAKE_CACHE_ARGS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}"
+    CMAKE_CACHE_ARGS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}"
-                     "-Dprotobuf_VERBOSE:BOOL=${protobuf_VERBOSE}"
+                     "-Dprotobuf_VERBOSE:BOOL=${protobuf_VERBOSE}"
-                     ${ARGN}
+                     ${ARGN}
-  )
+  )
-  set_property(TARGET ${NAME} PROPERTY FOLDER "Examples")
+  set_property(TARGET ${NAME} PROPERTY FOLDER "Examples")
-  set_property(TARGET ${NAME} PROPERTY EXCLUDE_FROM_ALL TRUE)
+  set_property(TARGET ${NAME} PROPERTY EXCLUDE_FROM_ALL TRUE)
-endfunction()
+endfunction()
-
+
-# Add examples as an external project.
+# Add examples as an external project.
-# sub_directory cannot be used because the find_package(protobuf) call would cause failures with redefined targets.
+# sub_directory cannot be used because the find_package(protobuf) call would cause failures with redefined targets.
-add_examples_build(examples "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}")
+add_examples_build(examples "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}")
-add_dependencies(examples libprotobuf protoc)
+add_dependencies(examples libprotobuf protoc)
-
+
-option(protobuf_BUILD_EXAMPLES_MULTITEST "Build Examples in multiple configurations. Useful for testing." OFF)
+option(protobuf_BUILD_EXAMPLES_MULTITEST "Build Examples in multiple configurations. Useful for testing." OFF)
-mark_as_advanced(protobuf_BUILD_EXAMPLES_MULTITEST)
+mark_as_advanced(protobuf_BUILD_EXAMPLES_MULTITEST)
-if(protobuf_BUILD_EXAMPLES_MULTITEST)
+if(protobuf_BUILD_EXAMPLES_MULTITEST)
-  set_property(GLOBAL PROPERTY USE_FOLDERS ON)
+  set_property(GLOBAL PROPERTY USE_FOLDERS ON)
-
+
-  #Build using the legacy compatibility module.
+  #Build using the legacy compatibility module.
-  add_examples_build(examples-legacy
+  add_examples_build(examples-legacy
-    "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}"
+    "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}"
-    "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
+    "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
-  )
+  )
-  add_dependencies(examples-legacy libprotobuf protoc)
+  add_dependencies(examples-legacy libprotobuf protoc)
-
+
-  #Build using the installed library.
+  #Build using the installed library.
-  add_examples_build(examples-installed
+  add_examples_build(examples-installed
-    "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
+    "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
-  )
+  )
-
+
-  #Build using the installed library in legacy compatibility mode.
+  #Build using the installed library in legacy compatibility mode.
-  add_examples_build(examples-installed-legacy
+  add_examples_build(examples-installed-legacy
-    "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
+    "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
-    "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
+    "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
-  )
+  )
-endif()
+endif()

cmake/protobuf-options.cmake

@@ -1,7 +1,7 @@
-# Verbose output
+# Verbose output
-option(protobuf_VERBOSE "Enable for verbose output" OFF)
+option(protobuf_VERBOSE "Enable for verbose output" OFF)
-mark_as_advanced(protobuf_VERBOSE)
+mark_as_advanced(protobuf_VERBOSE)
-
+
-# FindProtobuf module compatibel
+# FindProtobuf module compatibel
-option(protobuf_MODULE_COMPATIBLE "CMake build-in FindProtobuf.cmake module compatible" OFF)
+option(protobuf_MODULE_COMPATIBLE "CMake build-in FindProtobuf.cmake module compatible" OFF)
-mark_as_advanced(protobuf_MODULE_COMPATIBLE)
+mark_as_advanced(protobuf_MODULE_COMPATIBLE)

examples/CMakeLists.txt

@@ -1,63 +1,63 @@
-# Minimum CMake required
+# Minimum CMake required
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8.12)
-
+
-# Project
+# Project
-project(protobuf-examples)
+project(protobuf-examples)
-
+
-# Find required protobuf package
+# Find required protobuf package
-find_package(protobuf CONFIG REQUIRED)
+find_package(protobuf CONFIG REQUIRED)
-
+
-if(protobuf_VERBOSE)
+if(protobuf_VERBOSE)
-  message(STATUS "Using Protocol Buffers ${Protobuf_VERSION}")
+  message(STATUS "Using Protocol Buffers ${Protobuf_VERSION}")
-endif()
+endif()
-
+
-set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
-
+
-# http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F
+# http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F
-if(MSVC AND protobuf_MSVC_STATIC_RUNTIME)
+if(MSVC AND protobuf_MSVC_STATIC_RUNTIME)
-  foreach(flag_var
+  foreach(flag_var
-      CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
+      CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
-      CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
+      CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
-    if(${flag_var} MATCHES "/MD")
+    if(${flag_var} MATCHES "/MD")
-      string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
+      string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
-    endif(${flag_var} MATCHES "/MD")
+    endif(${flag_var} MATCHES "/MD")
-  endforeach()
+  endforeach()
-endif()
+endif()
-
+
-foreach(example add_person list_people)
+foreach(example add_person list_people)
-  set(${example}_SRCS ${example}.cc)
+  set(${example}_SRCS ${example}.cc)
-  set(${example}_PROTOS addressbook.proto)
+  set(${example}_PROTOS addressbook.proto)
-
+
-  #Code Generation
+  #Code Generation
-  if(protobuf_MODULE_COMPATIBLE) #Legacy Support
+  if(protobuf_MODULE_COMPATIBLE) #Legacy Support
-    protobuf_generate_cpp(${example}_PROTO_SRCS ${example}_PROTO_HDRS ${${example}_PROTOS})
+    protobuf_generate_cpp(${example}_PROTO_SRCS ${example}_PROTO_HDRS ${${example}_PROTOS})
-    list(APPEND ${example}_SRCS ${${example}_PROTO_SRCS} ${${example}_PROTO_HDRS})
+    list(APPEND ${example}_SRCS ${${example}_PROTO_SRCS} ${${example}_PROTO_HDRS})
-  else()
+  else()
-
+
-    foreach(proto_file ${${example}_PROTOS})
+    foreach(proto_file ${${example}_PROTOS})
-      get_filename_component(proto_file_abs ${proto_file} ABSOLUTE)
+      get_filename_component(proto_file_abs ${proto_file} ABSOLUTE)
-      get_filename_component(basename ${proto_file} NAME_WE)
+      get_filename_component(basename ${proto_file} NAME_WE)
-      set(generated_files ${basename}.pb.cc ${basename}.pb.h)
+      set(generated_files ${basename}.pb.cc ${basename}.pb.h)
-      list(APPEND ${example}_SRCS ${generated_files})
+      list(APPEND ${example}_SRCS ${generated_files})
-
+
-      add_custom_command(
+      add_custom_command(
-        OUTPUT ${generated_files}
+        OUTPUT ${generated_files}
-        COMMAND protobuf::protoc
+        COMMAND protobuf::protoc
-        ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${CMAKE_CURRENT_SOURCE_DIR} ${proto_file_abs}
+        ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${CMAKE_CURRENT_SOURCE_DIR} ${proto_file_abs}
-        COMMENT "Generating ${generated_files} from ${proto_file}"
+        COMMENT "Generating ${generated_files} from ${proto_file}"
-        VERBATIM
+        VERBATIM
-      )
+      )
-    endforeach()
+    endforeach()
-  endif()
+  endif()
-
+
-  #Executable setup
+  #Executable setup
-  set(executable_name ${example}_cpp)
+  set(executable_name ${example}_cpp)
-  add_executable(${executable_name} ${${example}_SRCS} ${${example}_PROTOS})
+  add_executable(${executable_name} ${${example}_SRCS} ${${example}_PROTOS})
-  if(protobuf_MODULE_COMPATIBLE) #Legacy mode
+  if(protobuf_MODULE_COMPATIBLE) #Legacy mode
-    target_include_directories(${executable_name} PUBLIC ${PROTOBUF_INCLUDE_DIRS})
+    target_include_directories(${executable_name} PUBLIC ${PROTOBUF_INCLUDE_DIRS})
-    target_link_libraries(${executable_name} ${PROTOBUF_LIBRARIES})
+    target_link_libraries(${executable_name} ${PROTOBUF_LIBRARIES})
-  else()
+  else()
-    target_link_libraries(${executable_name} protobuf::libprotobuf)
+    target_link_libraries(${executable_name} protobuf::libprotobuf)
-  endif()
+  endif()
-
+
-endforeach()
+endforeach()

java/core/src/main/java/com/google/protobuf/CodedOutputStream.java

@@ -144,6 +144,44 @@ public abstract class CodedOutputStream extends ByteOutput {
     return new NioEncoder(byteBuffer);
   }
 
+  /**
+   * Configures serialization to be deterministic.
+   *
+   * <p>The deterministic serialization guarantees that for a given binary, equal (defined by the
+   * {@code equals()} methods in protos) messages will always be serialized to the same bytes. This
+   * implies:
+   *
+   * <ul>
+   * <li>repeated serialization of a message will return the same bytes
+   * <li>different processes of the same binary (which may be executing on different machines) will
+   *     serialize equal messages to the same bytes.
+   * </ul>
+   *
+   * <p>Note the deterministic serialization is NOT canonical across languages; it is also unstable
+   * across different builds with schema changes due to unknown fields. Users who need canonical
+   * serialization, e.g. persistent storage in a canonical form, fingerprinting, etc, should define
+   * their own canonicalization specification and implement the serializer using reflection APIs
+   * rather than relying on this API.
+   *
+   * <p> Once set, the serializer will:  (Note this is an implementation detail and may subject to
+   * change in the future)
+   *
+   * <ul>
+   * <li> sort map entries by keys in lexicographical order or numerical order. Note: For string
+   *     keys, the order is based on comparing the Unicode value of each character in the strings.
+   *     The order may be different from the deterministic serialization in other languages where
+   *     maps are sorted on the lexicographical order of the UTF8 encoded keys.
+   * </ul>
+   */
+  public final void useDeterministicSerialization() {
+    serializationDeterministic = true;
+  }
+
+  boolean isSerializationDeterministic() {
+    return serializationDeterministic;
+  }
+  private boolean serializationDeterministic;
+
   /**
    * Create a new {@code CodedOutputStream} that writes to the given {@link ByteBuffer}.
    *

java/core/src/main/java/com/google/protobuf/DynamicMessage.java

@@ -526,6 +526,14 @@ public final class DynamicMessage extends AbstractMessage {
           fields.clearField(oldField);
         }
         oneofCases[index] = field;
+      } else if (field.getFile().getSyntax() == Descriptors.FileDescriptor.Syntax.PROTO3) {
+        if (!field.isRepeated()
+            && field.getJavaType() != FieldDescriptor.JavaType.MESSAGE
+            && value.equals(field.getDefaultValue())) {
+          // In proto3, setting a field to its default value is equivalent to clearing the field.
+          fields.clearField(field);
+          return this;
+        }
       }
       fields.setField(field, value);
       return this;

java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java

@@ -31,6 +31,8 @@
 package com.google.protobuf;
 
 import com.google.protobuf.Descriptors.Descriptor;
+import com.google.protobuf.Descriptors.EnumDescriptor;
+import com.google.protobuf.Descriptors.EnumValueDescriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.FieldPresenceTestProto.TestAllTypes;
 import com.google.protobuf.FieldPresenceTestProto.TestOptionalFieldsOnly;
@@ -253,6 +255,54 @@ public class FieldPresenceTest extends TestCase {
     assertEquals(4, message.getAllFields().size());
   }
 
+  public void testFieldPresenceDynamicMessage() {
+    Descriptor descriptor = TestAllTypes.getDescriptor();
+    FieldDescriptor optionalInt32Field = descriptor.findFieldByName("optional_int32");
+    FieldDescriptor optionalStringField = descriptor.findFieldByName("optional_string");
+    FieldDescriptor optionalBytesField = descriptor.findFieldByName("optional_bytes");
+    FieldDescriptor optionalNestedEnumField = descriptor.findFieldByName("optional_nested_enum");
+    EnumDescriptor enumDescriptor = optionalNestedEnumField.getEnumType();
+    EnumValueDescriptor defaultEnumValueDescriptor = enumDescriptor.getValues().get(0);
+    EnumValueDescriptor nonDefaultEnumValueDescriptor = enumDescriptor.getValues().get(1);
+
+    DynamicMessage defaultInstance = DynamicMessage.getDefaultInstance(descriptor);
+    // Field not present.
+    DynamicMessage message = defaultInstance.newBuilderForType().build();
+    assertFalse(message.hasField(optionalInt32Field));
+    assertFalse(message.hasField(optionalStringField));
+    assertFalse(message.hasField(optionalBytesField));
+    assertFalse(message.hasField(optionalNestedEnumField));
+    assertEquals(0, message.getAllFields().size());
+
+    // Field set to non-default value is seen as present.
+    message =
+        defaultInstance
+            .newBuilderForType()
+            .setField(optionalInt32Field, 1)
+            .setField(optionalStringField, "x")
+            .setField(optionalBytesField, ByteString.copyFromUtf8("y"))
+            .setField(optionalNestedEnumField, nonDefaultEnumValueDescriptor)
+            .build();
+    assertTrue(message.hasField(optionalInt32Field));
+    assertTrue(message.hasField(optionalStringField));
+    assertTrue(message.hasField(optionalBytesField));
+    assertTrue(message.hasField(optionalNestedEnumField));
+    assertEquals(4, message.getAllFields().size());
+
+    // Field set to default value is seen as not present.
+    message = message.toBuilder()
+            .setField(optionalInt32Field, 0)
+            .setField(optionalStringField, "")
+            .setField(optionalBytesField, ByteString.EMPTY)
+            .setField(optionalNestedEnumField, defaultEnumValueDescriptor)
+            .build();
+    assertFalse(message.hasField(optionalInt32Field));
+    assertFalse(message.hasField(optionalStringField));
+    assertFalse(message.hasField(optionalBytesField));
+    assertFalse(message.hasField(optionalNestedEnumField));
+    assertEquals(0, message.getAllFields().size());
+  }
+
   public void testMessageField() {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     assertFalse(builder.hasOptionalNestedMessage());

java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java

@@ -35,6 +35,7 @@ import static java.util.Arrays.asList;
 import junit.framework.TestCase;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
 import java.util.List;
@@ -233,7 +234,7 @@ public class LazyStringArrayListTest extends TestCase {
     }
     
     try {
-      list.addAllByteArray(asList(BYTE_STRING_A.toByteArray()));
+      list.addAllByteArray(Collections.singletonList(BYTE_STRING_A.toByteArray()));
       fail();
     } catch (UnsupportedOperationException e) {
       // expected

java/util/src/main/java/com/google/protobuf/util/JsonFormat.java

@@ -116,7 +116,8 @@ public class JsonFormat {
     private Printer(
         TypeRegistry registry,
         boolean includingDefaultValueFields,
-        boolean preservingProtoFieldNames, boolean omittingInsignificantWhitespace) {
+        boolean preservingProtoFieldNames,
+        boolean omittingInsignificantWhitespace) {
       this.registry = registry;
       this.includingDefaultValueFields = includingDefaultValueFields;
       this.preservingProtoFieldNames = preservingProtoFieldNames;
@@ -133,7 +134,11 @@ public class JsonFormat {
       if (this.registry != TypeRegistry.getEmptyTypeRegistry()) {
         throw new IllegalArgumentException("Only one registry is allowed.");
       }
-      return new Printer(registry, includingDefaultValueFields, preservingProtoFieldNames, omittingInsignificantWhitespace);
+      return new Printer(
+          registry,
+          includingDefaultValueFields,
+          preservingProtoFieldNames,
+          omittingInsignificantWhitespace);
     }
 
     /**
@@ -143,7 +148,8 @@ public class JsonFormat {
      * {@link Printer}.
      */
     public Printer includingDefaultValueFields() {
-      return new Printer(registry, true, preservingProtoFieldNames, omittingInsignificantWhitespace);
+      return new Printer(
+          registry, true, preservingProtoFieldNames, omittingInsignificantWhitespace);
     }
 
     /**
@@ -153,7 +159,8 @@ public class JsonFormat {
      * current {@link Printer}.
      */
     public Printer preservingProtoFieldNames() {
-      return new Printer(registry, includingDefaultValueFields, true, omittingInsignificantWhitespace);
+      return new Printer(
+          registry, includingDefaultValueFields, true, omittingInsignificantWhitespace);
     }
 
 
@@ -172,7 +179,7 @@ public class JsonFormat {
      * See <a href="https://tools.ietf.org/html/rfc7159">https://tools.ietf.org/html/rfc7159</a>
      * current {@link Printer}.
      */
-    public Printer omittingInsignificantWhitespace(){
+    public Printer omittingInsignificantWhitespace() {
       return new Printer(registry, includingDefaultValueFields, preservingProtoFieldNames, true);
     }
 
@@ -186,7 +193,12 @@ public class JsonFormat {
     public void appendTo(MessageOrBuilder message, Appendable output) throws IOException {
       // TODO(xiaofeng): Investigate the allocation overhead and optimize for
       // mobile.
-      new PrinterImpl(registry, includingDefaultValueFields, preservingProtoFieldNames, output, omittingInsignificantWhitespace)
+      new PrinterImpl(
+              registry,
+              includingDefaultValueFields,
+              preservingProtoFieldNames,
+              output,
+              omittingInsignificantWhitespace)
           .print(message);
     }
 
@@ -379,18 +391,18 @@ public class JsonFormat {
    */
   interface TextGenerator {
     void indent();
+
     void outdent();
+
     void print(final CharSequence text) throws IOException;
   }
 
-
   /**
    * Format the json without indentation
    */
-  private static final class CompactTextGenerator implements TextGenerator{
+  private static final class CompactTextGenerator implements TextGenerator {
     private final Appendable output;
 
-
     private CompactTextGenerator(final Appendable output) {
       this.output = output;
     }
@@ -411,12 +423,11 @@ public class JsonFormat {
     public void print(final CharSequence text) throws IOException {
       output.append(text);
     }
-
   }
   /**
    * A TextGenerator adds indentation when writing formatted text.
    */
-  private static final class PrettyTextGenerator implements TextGenerator{
+  private static final class PrettyTextGenerator implements TextGenerator {
     private final Appendable output;
     private final StringBuilder indent = new StringBuilder();
     private boolean atStartOfLine = true;
@@ -496,7 +507,8 @@ public class JsonFormat {
         TypeRegistry registry,
         boolean includingDefaultValueFields,
         boolean preservingProtoFieldNames,
-        Appendable jsonOutput, boolean omittingInsignificantWhitespace) {
+        Appendable jsonOutput,
+        boolean omittingInsignificantWhitespace) {
       this.registry = registry;
       this.includingDefaultValueFields = includingDefaultValueFields;
       this.preservingProtoFieldNames = preservingProtoFieldNames;
@@ -734,9 +746,7 @@ public class JsonFormat {
     }
 
     /** Prints a regular message with an optional type URL. */
-
+    private void print(MessageOrBuilder message, String typeUrl) throws IOException {
-    private void print(MessageOrBuilder message, String typeUrl)
-        throws IOException {
       generator.print("{" + blankOrNewLine);
       generator.indent();
 

java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java

@@ -140,7 +140,7 @@ public class JsonFormatTest extends TestCase {
   private String toJsonString(Message message) throws IOException {
     return JsonFormat.printer().print(message);
   }
-  private String toCompactJsonString(Message message) throws IOException{
+  private String toCompactJsonString(Message message) throws IOException {
     return JsonFormat.printer().omittingInsignificantWhitespace().print(message);
   }
 
@@ -1172,7 +1172,9 @@ public class JsonFormatTest extends TestCase {
 
   public void testOmittingInsignificantWhiteSpace() throws Exception {
     TestAllTypes message = TestAllTypes.newBuilder().setOptionalInt32(12345).build();
-    assertEquals("{" + "\"optionalInt32\":12345" + "}", JsonFormat.printer().omittingInsignificantWhitespace().print(message));
+    assertEquals(
+        "{" + "\"optionalInt32\":12345" + "}",
+        JsonFormat.printer().omittingInsignificantWhitespace().print(message));
     TestAllTypes message1 = TestAllTypes.getDefaultInstance();
     assertEquals("{}", JsonFormat.printer().omittingInsignificantWhitespace().print(message1));
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
@@ -1224,4 +1226,20 @@ public class JsonFormatTest extends TestCase {
         toCompactJsonString(message2));
   }
 
+  // Regression test for b/29892357
+  public void testEmptyWrapperTypesInAny() throws Exception {
+    JsonFormat.TypeRegistry registry =
+        JsonFormat.TypeRegistry.newBuilder().add(TestAllTypes.getDescriptor()).build();
+    JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(registry);
+
+    Any.Builder builder = Any.newBuilder();
+    parser.merge(
+        "{\n"
+            + "  \"@type\": \"type.googleapis.com/google.protobuf.BoolValue\",\n"
+            + "  \"value\": false\n"
+            + "}\n",
+        builder);
+    Any any = builder.build();
+    assertEquals(0, any.getValue().size());
+  }
 }

js/message_test.js

@@ -215,6 +215,10 @@ describe('Message test suite', function() {
     assertEquals(true, response.getBoolField());
     assertEquals(11, response.getIntField());
     assertEquals(13, response.getEnumField());
+    assertFalse(response.hasStringField());
+    assertFalse(response.hasBoolField());
+    assertFalse(response.hasIntField());
+    assertFalse(response.hasEnumField());
 
     // Test with null values, as would be returned by a JSON serializer.
     response = makeDefault([null, null, null, null]);
@@ -222,6 +226,10 @@ describe('Message test suite', function() {
     assertEquals(true, response.getBoolField());
     assertEquals(11, response.getIntField());
     assertEquals(13, response.getEnumField());
+    assertFalse(response.hasStringField());
+    assertFalse(response.hasBoolField());
+    assertFalse(response.hasIntField());
+    assertFalse(response.hasEnumField());
 
     // Test with false-like values.
     response = makeDefault(['', false, 0, 0]);
@@ -229,6 +237,10 @@ describe('Message test suite', function() {
     assertEquals(false, response.getBoolField());
     assertEquals(true, response.getIntField() == 0);
     assertEquals(true, response.getEnumField() == 0);
+    assertTrue(response.hasStringField());
+    assertTrue(response.hasBoolField());
+    assertTrue(response.hasIntField());
+    assertTrue(response.hasEnumField());
 
     // Test that clearing the values reverts them to the default state.
     response = makeDefault(['blah', false, 111, 77]);
@@ -238,6 +250,10 @@ describe('Message test suite', function() {
     assertEquals(true, response.getBoolField());
     assertEquals(11, response.getIntField());
     assertEquals(13, response.getEnumField());
+    assertFalse(response.hasStringField());
+    assertFalse(response.hasBoolField());
+    assertFalse(response.hasIntField());
+    assertFalse(response.hasEnumField());
 
     // Test that setFoo(null) clears the values.
     response = makeDefault(['blah', false, 111, 77]);
@@ -247,6 +263,10 @@ describe('Message test suite', function() {
     assertEquals(true, response.getBoolField());
     assertEquals(11, response.getIntField());
     assertEquals(13, response.getEnumField());
+    assertFalse(response.hasStringField());
+    assertFalse(response.hasBoolField());
+    assertFalse(response.hasIntField());
+    assertFalse(response.hasEnumField());
   });
 
   it('testMessageRegistration', function() {
@@ -269,6 +289,8 @@ describe('Message test suite', function() {
     assertUndefined(foo.getAString());
     assertUndefined(foo.getABool());
     assertUndefined(foo.getANestedMessage());
+    assertFalse(foo.hasAString());
+    assertFalse(foo.hasABool());
     assertObjectEquals([], foo.getARepeatedMessageList());
     assertObjectEquals([], foo.getARepeatedStringList());
     // NOTE: We want the missing fields in 'expected' to be undefined,
@@ -291,6 +313,8 @@ describe('Message test suite', function() {
     assertNull(foo.getAString());
     assertNull(foo.getABool());
     assertNull(foo.getANestedMessage());
+    assertFalse(foo.hasAString());
+    assertFalse(foo.hasABool());
     assertObjectEquals([], foo.getARepeatedMessageList());
     assertObjectEquals([], foo.getARepeatedStringList());
     assertObjectEquals([null, null, null, [], []], foo.toArray());
@@ -307,6 +331,8 @@ describe('Message test suite', function() {
     assertUndefined(foo.getAString());
     assertUndefined(foo.getABool());
     assertUndefined(foo.getANestedMessage());
+    assertFalse(foo.hasAString());
+    assertFalse(foo.hasABool());
     assertObjectEquals([], foo.getARepeatedMessageList());
     assertObjectEquals([], foo.getARepeatedStringList());
     expected = [,,, [], []];
@@ -800,14 +826,20 @@ describe('Message test suite', function() {
     var message = new proto.jspb.test.TestMessageWithOneof;
     assertUndefined(message.getPone());
     assertUndefined(message.getPthree());
+    assertFalse(message.hasPone());
+    assertFalse(message.hasPthree());
 
     message.setPone('hi');
     assertEquals('hi', message.getPone());
     assertUndefined(message.getPthree());
+    assertTrue(message.hasPone());
+    assertFalse(message.hasPthree());
 
     message.setPthree('bye');
     assertUndefined(message.getPone());
     assertEquals('bye', message.getPthree());
+    assertFalse(message.hasPone());
+    assertTrue(message.hasPthree());
   });
 
   it('testSettingOneofFieldDoesNotClearFieldsFromOtherUnions', function() {
@@ -816,17 +848,23 @@ describe('Message test suite', function() {
     assertUndefined(message.getPone());
     assertUndefined(message.getPthree());
     assertUndefined(message.getRone());
+    assertFalse(message.hasPone());
+    assertFalse(message.hasPthree());
 
     message.setPone('hi');
     message.setRone(other);
     assertEquals('hi', message.getPone());
     assertUndefined(message.getPthree());
     assertEquals(other, message.getRone());
+    assertTrue(message.hasPone());
+    assertFalse(message.hasPthree());
 
     message.setPthree('bye');
     assertUndefined(message.getPone());
     assertEquals('bye', message.getPthree());
     assertEquals(other, message.getRone());
+    assertFalse(message.hasPone());
+    assertTrue(message.hasPthree());
   });
 
   it('testUnsetsOneofCaseWhenFieldIsCleared', function() {
@@ -884,6 +922,8 @@ describe('Message test suite', function() {
     var message = new proto.jspb.test.TestMessageWithOneof;
     assertUndefined(message.getBone());
     assertEquals(1234, message.getBtwo());
+    assertFalse(message.hasBone());
+    assertFalse(message.hasBtwo());
     assertEquals(
         proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase
             .DEFAULT_ONEOF_B_NOT_SET,
@@ -892,12 +932,16 @@ describe('Message test suite', function() {
     message.setBone(2);
     assertEquals(2, message.getBone());
     assertEquals(1234, message.getBtwo());
+    assertTrue(message.hasBone());
+    assertFalse(message.hasBtwo());
     assertEquals(
         proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BONE,
         message.getDefaultOneofBCase());
 
     message.setBtwo(3);
     assertUndefined(message.getBone());
+    assertFalse(message.hasBone());
+    assertTrue(message.hasBtwo());
     assertEquals(3, message.getBtwo());
     assertEquals(
         proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BTWO,
@@ -905,6 +949,8 @@ describe('Message test suite', function() {
 
     message.clearBtwo();
     assertUndefined(message.getBone());
+    assertFalse(message.hasBone());
+    assertFalse(message.hasBtwo());
     assertEquals(1234, message.getBtwo());
     assertEquals(
         proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase

js/proto3_test.js

@@ -225,12 +225,18 @@ describe('proto3Test', function() {
     assertEquals(msg.getOneofForeignMessage(), undefined);
     assertEquals(msg.getOneofString(), undefined);
     assertEquals(msg.getOneofBytes(), undefined);
+    assertFalse(msg.hasOneofUint32());
+    assertFalse(msg.hasOneofString());
+    assertFalse(msg.hasOneofBytes());
 
     msg.setOneofUint32(42);
     assertEquals(msg.getOneofUint32(), 42);
     assertEquals(msg.getOneofForeignMessage(), undefined);
     assertEquals(msg.getOneofString(), undefined);
     assertEquals(msg.getOneofBytes(), undefined);
+    assertTrue(msg.hasOneofUint32());
+    assertFalse(msg.hasOneofString());
+    assertFalse(msg.hasOneofBytes());
 
 
     var submsg = new proto.jspb.test.ForeignMessage();
@@ -239,12 +245,18 @@ describe('proto3Test', function() {
     assertEquals(msg.getOneofForeignMessage(), submsg);
     assertEquals(msg.getOneofString(), undefined);
     assertEquals(msg.getOneofBytes(), undefined);
+    assertFalse(msg.hasOneofUint32());
+    assertFalse(msg.hasOneofString());
+    assertFalse(msg.hasOneofBytes());
 
     msg.setOneofString('hello');
     assertEquals(msg.getOneofUint32(), undefined);
     assertEquals(msg.getOneofForeignMessage(), undefined);
     assertEquals(msg.getOneofString(), 'hello');
     assertEquals(msg.getOneofBytes(), undefined);
+    assertFalse(msg.hasOneofUint32());
+    assertTrue(msg.hasOneofString());
+    assertFalse(msg.hasOneofBytes());
 
     msg.setOneofBytes(goog.crypt.base64.encodeString('\u00FF\u00FF'));
     assertEquals(msg.getOneofUint32(), undefined);
@@ -252,6 +264,9 @@ describe('proto3Test', function() {
     assertEquals(msg.getOneofString(), undefined);
     assertEquals(msg.getOneofBytes_asB64(),
         goog.crypt.base64.encodeString('\u00FF\u00FF'));
+    assertFalse(msg.hasOneofUint32());
+    assertFalse(msg.hasOneofString());
+    assertTrue(msg.hasOneofBytes());
   });
 
 

js/test.proto

@@ -233,3 +233,4 @@ message TestEndsWithBytes {
   optional int32 value = 1;
   optional bytes data = 2;
 }
+

python/google/protobuf/internal/json_format_test.py

@@ -252,10 +252,7 @@ class JsonFormatTest(JsonFormatBase):
     message = json_format_proto3_pb2.TestMessage()
     json_format.Parse('{"stringValue": "\\uD83D\\uDE01"}', message)
     self.assertEqual(message.string_value,
-                     b'\xF0\x9F\x98\x81'.decode("utf-8", "strict"))
+                     b'\xF0\x9F\x98\x81'.decode('utf-8', 'strict'))
-
-    # TODO: add test that UTF-8 encoded surrogate code points are rejected.
-    # UTF-8 does not allow them.
 
     # Error case: unpaired high surrogate.
     self.CheckError(
@@ -267,7 +264,6 @@ class JsonFormatTest(JsonFormatBase):
         '{"stringValue": "\\uDE01"}',
         r'Invalid \\uXXXX escape|Unpaired.*surrogate')
 
-
   def testTimestampMessage(self):
     message = json_format_proto3_pb2.TestTimestamp()
     message.value.seconds = 0

python/google/protobuf/internal/python_message.py

@@ -76,7 +76,6 @@ from google.protobuf.internal import well_known_types
 from google.protobuf.internal import wire_format
 from google.protobuf import descriptor as descriptor_mod
 from google.protobuf import message as message_mod
-from google.protobuf import symbol_database
 from google.protobuf import text_format
 
 _FieldDescriptor = descriptor_mod.FieldDescriptor
@@ -98,16 +97,12 @@ class GeneratedProtocolMessageType(type):
   classes at runtime, as in this example:
 
   mydescriptor = Descriptor(.....)
-  class MyProtoClass(Message):
+  factory = symbol_database.Default()
-    __metaclass__ = GeneratedProtocolMessageType
+  factory.pool.AddDescriptor(mydescriptor)
-    DESCRIPTOR = mydescriptor
+  MyProtoClass = factory.GetPrototype(mydescriptor)
   myproto_instance = MyProtoClass()
   myproto.foo_field = 23
   ...
-
-  The above example will not work for nested types. If you wish to include them,
-  use reflection.MakeClass() instead of manually instantiating the class in
-  order to create the appropriate class structure.
   """
 
   # Must be consistent with the protocol-compiler code in
@@ -926,26 +921,33 @@ def _InternalUnpackAny(msg):
   Returns:
     The unpacked message.
   """
+  # TODO(amauryfa): Don't use the factory of generated messages.
+  # To make Any work with custom factories, use the message factory of the
+  # parent message.
+  # pylint: disable=g-import-not-at-top
+  from google.protobuf import symbol_database
+  factory = symbol_database.Default()
+
   type_url = msg.type_url
-  db = symbol_database.Default()
 
   if not type_url:
     return None
 
   # TODO(haberman): For now we just strip the hostname.  Better logic will be
   # required.
-  type_name = type_url.split("/")[-1]
+  type_name = type_url.split('/')[-1]
-  descriptor = db.pool.FindMessageTypeByName(type_name)
+  descriptor = factory.pool.FindMessageTypeByName(type_name)
 
   if descriptor is None:
     return None
 
-  message_class = db.GetPrototype(descriptor)
+  message_class = factory.GetPrototype(descriptor)
   message = message_class()
 
   message.ParseFromString(msg.value)
   return message
 
+
 def _AddEqualsMethod(message_descriptor, cls):
   """Helper for _AddMessageMethods()."""
   def __eq__(self, other):

python/google/protobuf/internal/reflection_test.py

@@ -972,6 +972,7 @@ class ReflectionTest(unittest.TestCase):
     proto.repeated_nested_message.add(bb=23)
     self.assertEqual(1, len(proto.repeated_nested_message))
     self.assertEqual(23, proto.repeated_nested_message[0].bb)
+    self.assertRaises(TypeError, proto.repeated_nested_message.add, 23)
 
   def testRepeatedCompositeRemove(self):
     proto = unittest_pb2.TestAllTypes()

python/google/protobuf/internal/symbol_database_test.py

@@ -39,26 +39,28 @@ except ImportError:
 
 from google.protobuf import unittest_pb2
 from google.protobuf import descriptor
+from google.protobuf import descriptor_pool
 from google.protobuf import symbol_database
 
+
 class SymbolDatabaseTest(unittest.TestCase):
 
   def _Database(self):
-    # TODO(b/17734095): Remove this difference when the C++ implementation
-    # supports multiple databases.
     if descriptor._USE_C_DESCRIPTORS:
-      return symbol_database.Default()
+      # The C++ implementation does not allow mixing descriptors from
+      # different pools.
+      db = symbol_database.SymbolDatabase(pool=descriptor_pool.Default())
     else:
       db = symbol_database.SymbolDatabase()
-      # Register representative types from unittest_pb2.
+    # Register representative types from unittest_pb2.
-      db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR)
+    db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR)
-      db.RegisterMessage(unittest_pb2.TestAllTypes)
+    db.RegisterMessage(unittest_pb2.TestAllTypes)
-      db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage)
+    db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage)
-      db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup)
+    db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup)
-      db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup)
+    db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup)
-      db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR)
+    db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR)
-      db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR)
+    db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR)
-      return db
+    return db
 
   def testGetPrototype(self):
     instance = self._Database().GetPrototype(

python/google/protobuf/pyext/cpp_message.py

@@ -48,9 +48,9 @@ class GeneratedProtocolMessageType(_message.MessageMeta):
   classes at runtime, as in this example:
 
   mydescriptor = Descriptor(.....)
-  class MyProtoClass(Message):
+  factory = symbol_database.Default()
-    __metaclass__ = GeneratedProtocolMessageType
+  factory.pool.AddDescriptor(mydescriptor)
-    DESCRIPTOR = mydescriptor
+  MyProtoClass = factory.GetPrototype(mydescriptor)
   myproto_instance = MyProtoClass()
   myproto.foo_field = 23
   ...

python/google/protobuf/pyext/map_container.cc

@@ -348,9 +348,10 @@ PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
 }
 
 // Initializes the underlying Message object of "to" so it becomes a new parent
-// repeated scalar, and copies all the values from "from" to it. A child scalar
+// map container, and copies all the values from "from" to it. A child map
 // container can be released by passing it as both from and to (e.g. making it
 // the recipient of the new parent message and copying the values from itself).
+// In fact, this is the only supported use at the moment.
 static int InitializeAndCopyToParentContainer(MapContainer* from,
                                               MapContainer* to) {
   // For now we require from == to, re-evaluate if we want to support deep copy

python/google/protobuf/pyext/message.cc

@@ -1041,7 +1041,12 @@ int InternalDeleteRepeatedField(
 }
 
 // Initializes fields of a message. Used in constructors.
-int InitAttributes(CMessage* self, PyObject* kwargs) {
+int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) {
+  if (args != NULL && PyTuple_Size(args) != 0) {
+    PyErr_SetString(PyExc_TypeError, "No positional arguments allowed");
+    return -1;
+  }
+
   if (kwargs == NULL) {
     return 0;
   }
@@ -1167,7 +1172,7 @@ int InitAttributes(CMessage* self, PyObject* kwargs) {
       }
       CMessage* cmessage = reinterpret_cast<CMessage*>(message.get());
       if (PyDict_Check(value)) {
-        if (InitAttributes(cmessage, value) < 0) {
+        if (InitAttributes(cmessage, NULL, value) < 0) {
           return -1;
         }
       } else {
@@ -1245,12 +1250,7 @@ static PyObject* New(PyTypeObject* cls,
 // The __init__ method of Message classes.
 // It initializes fields from keywords passed to the constructor.
 static int Init(CMessage* self, PyObject* args, PyObject* kwargs) {
-  if (PyTuple_Size(args) != 0) {
+  return InitAttributes(self, args, kwargs);
-    PyErr_SetString(PyExc_TypeError, "No positional arguments allowed");
-    return -1;
-  }
-
-  return InitAttributes(self, kwargs);
 }
 
 // ---------------------------------------------------------------------

python/google/protobuf/pyext/message.h

@@ -237,7 +237,9 @@ PyObject* HasFieldByDescriptor(
 PyObject* HasField(CMessage* self, PyObject* arg);
 
 // Initializes values of fields on a newly constructed message.
-int InitAttributes(CMessage* self, PyObject* kwargs);
+// Note that positional arguments are disallowed: 'args' must be NULL or the
+// empty tuple.
+int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs);
 
 PyObject* MergeFrom(CMessage* self, PyObject* arg);
 

python/google/protobuf/pyext/repeated_composite_container.cc

@@ -146,7 +146,7 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self,
   cmsg->owner = self->owner;
   cmsg->message = sub_message;
   cmsg->parent = self->parent;
-  if (cmessage::InitAttributes(cmsg, kwargs) < 0) {
+  if (cmessage::InitAttributes(cmsg, args, kwargs) < 0) {
     Py_DECREF(cmsg);
     return NULL;
   }
@@ -166,7 +166,7 @@ static PyObject* AddToReleased(RepeatedCompositeContainer* self,
 
   // Create a new Message detached from the rest.
   PyObject* py_cmsg = PyEval_CallObjectWithKeywords(
-      self->child_message_class->AsPyObject(), NULL, kwargs);
+      self->child_message_class->AsPyObject(), args, kwargs);
   if (py_cmsg == NULL)
     return NULL;
 

python/google/protobuf/reflection.py

@@ -58,13 +58,7 @@ else:
   from google.protobuf.internal import python_message as message_impl
 
 # The type of all Message classes.
-# Part of the public interface.
+# Part of the public interface, but normally only used by message factories.
-#
-# Used by generated files, but clients can also use it at runtime:
-#   mydescriptor = pool.FindDescriptor(.....)
-#   class MyProtoClass(Message):
-#     __metaclass__ = GeneratedProtocolMessageType
-#     DESCRIPTOR = mydescriptor
 GeneratedProtocolMessageType = message_impl.GeneratedProtocolMessageType
 
 

python/google/protobuf/symbol_database.py

@@ -30,11 +30,9 @@
 
 """A database of Python protocol buffer generated symbols.
 
-SymbolDatabase makes it easy to create new instances of a registered type, given
+SymbolDatabase is the MessageFactory for messages generated at compile time,
-only the type's protocol buffer symbol name. Once all symbols are registered,
+and makes it easy to create new instances of a registered type, given only the
-they can be accessed using either the MessageFactory interface which
+type's protocol buffer symbol name.
-SymbolDatabase exposes, or the DescriptorPool interface of the underlying
-pool.
 
 Example usage:
 
@@ -61,27 +59,17 @@ Example usage:
 
 
 from google.protobuf import descriptor_pool
+from google.protobuf import message_factory
 
 
-class SymbolDatabase(object):
+class SymbolDatabase(message_factory.MessageFactory):
-  """A database of Python generated symbols.
+  """A database of Python generated symbols."""
-
-  SymbolDatabase also models message_factory.MessageFactory.
-
-  The symbol database can be used to keep a global registry of all protocol
-  buffer types used within a program.
-  """
-
-  def __init__(self, pool=None):
-    """Constructor."""
-
-    self._symbols = {}
-    self._symbols_by_file = {}
-    self.pool = pool or descriptor_pool.Default()
 
   def RegisterMessage(self, message):
     """Registers the given message type in the local database.
 
+    Calls to GetSymbol() and GetMessages() will return messages registered here.
+
     Args:
       message: a message.Message, to be registered.
 
@@ -90,10 +78,7 @@ class SymbolDatabase(object):
     """
 
     desc = message.DESCRIPTOR
-    self._symbols[desc.full_name] = message
+    self._classes[desc.full_name] = message
-    if desc.file.name not in self._symbols_by_file:
-      self._symbols_by_file[desc.file.name] = {}
-    self._symbols_by_file[desc.file.name][desc.full_name] = message
     self.pool.AddDescriptor(desc)
     return message
 
@@ -136,47 +121,46 @@ class SymbolDatabase(object):
       KeyError: if the symbol could not be found.
     """
 
-    return self._symbols[symbol]
+    return self._classes[symbol]
-
-  def GetPrototype(self, descriptor):
-    """Builds a proto2 message class based on the passed in descriptor.
-
-    Passing a descriptor with a fully qualified name matching a previous
-    invocation will cause the same class to be returned.
-
-    Args:
-      descriptor: The descriptor to build from.
-
-    Returns:
-      A class describing the passed in descriptor.
-    """
-
-    return self.GetSymbol(descriptor.full_name)
 
   def GetMessages(self, files):
-    """Gets all the messages from a specified file.
+    # TODO(amauryfa): Fix the differences with MessageFactory.
-
+    """Gets all registered messages from a specified file.
-    This will find and resolve dependencies, failing if they are not registered
-    in the symbol database.
 
+    Only messages already created and registered will be returned; (this is the
+    case for imported _pb2 modules)
+    But unlike MessageFactory, this version also returns nested messages.
 
     Args:
       files: The file names to extract messages from.
 
     Returns:
-      A dictionary mapping proto names to the message classes. This will include
+      A dictionary mapping proto names to the message classes.
-      any dependent messages as well as any messages defined in the same file as
-      a specified message.
 
     Raises:
       KeyError: if a file could not be found.
     """
 
+    def _GetAllMessageNames(desc):
+      """Walk a message Descriptor and recursively yields all message names."""
+      yield desc.full_name
+      for msg_desc in desc.nested_types:
+        for full_name in _GetAllMessageNames(msg_desc):
+          yield full_name
+
     result = {}
-    for f in files:
+    for file_name in files:
-      result.update(self._symbols_by_file[f])
+      file_desc = self.pool.FindFileByName(file_name)
+      for msg_desc in file_desc.message_types_by_name.values():
+        for full_name in _GetAllMessageNames(msg_desc):
+          try:
+            result[full_name] = self._classes[full_name]
+          except KeyError:
+            # This descriptor has no registered class, skip it.
+            pass
     return result
 
+
 _DEFAULT = SymbolDatabase(pool=descriptor_pool.Default())
 
 

src/google/protobuf/compiler/cpp/cpp_file.cc

@@ -336,19 +336,6 @@ void FileGenerator::GenerateSource(io::Printer* printer) {
 
   // Generate classes.
   for (int i = 0; i < file_->message_type_count(); i++) {
-    if (i == 0 && HasGeneratedMethods(file_, options_)) {
-      printer->Print(
-          "\n"
-          "namespace {\n"
-          "\n"
-          "static void MergeFromFail(int line) GOOGLE_ATTRIBUTE_COLD;\n"
-          "static void MergeFromFail(int line) {\n"
-          "  GOOGLE_CHECK(false) << __FILE__ << \":\" << line;\n"
-          "}\n"
-          "\n"
-          "}  // namespace\n"
-          "\n");
-    }
     printer->Print("\n");
     printer->Print(kThickSeparator);
     printer->Print("\n");
@@ -464,9 +451,10 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) {
   // and we only use AddDescriptors() to allocate default instances.
   if (HasDescriptorMethods(file_, options_)) {
     printer->Print(
-      "\n"
+        "\n"
-      "void $assigndescriptorsname$() {\n",
+        "void $assigndescriptorsname$() GOOGLE_ATTRIBUTE_COLD;\n"
-      "assigndescriptorsname", GlobalAssignDescriptorsName(file_->name()));
+        "void $assigndescriptorsname$() {\n",
+        "assigndescriptorsname", GlobalAssignDescriptorsName(file_->name()));
     printer->Indent();
 
     // Make sure the file has found its way into the pool.  If a descriptor
@@ -525,8 +513,9 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) {
     // protobuf_RegisterTypes():  Calls
     // MessageFactory::InternalRegisterGeneratedType() for each message type.
     printer->Print(
-      "void protobuf_RegisterTypes(const ::std::string&) {\n"
+        "void protobuf_RegisterTypes(const ::std::string&) GOOGLE_ATTRIBUTE_COLD;\n"
-      "  protobuf_AssignDescriptorsOnce();\n");
+        "void protobuf_RegisterTypes(const ::std::string&) {\n"
+        "  protobuf_AssignDescriptorsOnce();\n");
     printer->Indent();
 
     for (int i = 0; i < file_->message_type_count(); i++) {
@@ -566,6 +555,7 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) {
       // Note that we don't need any special synchronization in the following
       // code
       // because it is called at static init time before any threads exist.
+      "void $adddescriptorsname$() GOOGLE_ATTRIBUTE_COLD;\n"
       "void $adddescriptorsname$() {\n"
       "  static bool already_here = false;\n"
       "  if (already_here) return;\n"

src/google/protobuf/compiler/cpp/cpp_map_field.cc

@@ -251,117 +251,148 @@ GenerateMergeFromCodedStream(io::Printer* printer) const {
   }
 }
 
+static void GenerateSerializationLoop(io::Printer* printer,
+                                      const map<string, string>& variables,
+                                      bool supports_arenas,
+                                      const string& utf8_check,
+                                      const string& loop_header,
+                                      const string& ptr,
+                                      bool loop_via_iterators) {
+  printer->Print(variables,
+      StrCat("::google::protobuf::scoped_ptr<$map_classname$> entry;\n",
+             loop_header, " {\n").c_str());
+  printer->Indent();
+
+  printer->Print(variables, StrCat(
+      "entry.reset($name$_.New$wrapper$(\n"
+      "    ", ptr, "->first, ", ptr, "->second));\n"
+      "$write_entry$;\n").c_str());
+
+  // If entry is allocated by arena, its desctructor should be avoided.
+  if (supports_arenas) {
+    printer->Print(
+        "if (entry->GetArena() != NULL) {\n"
+        "  entry.release();\n"
+        "}\n");
+  }
+
+  if (!utf8_check.empty()) {
+    // If loop_via_iterators is true then ptr is actually an iterator, and we
+    // create a pointer by prefixing it with "&*".
+    printer->Print(
+        StrCat(utf8_check, "(", (loop_via_iterators ? "&*" : ""), ptr, ");\n")
+            .c_str());
+  }
+
+  printer->Outdent();
+  printer->Print(
+      "}\n");
+}
+
 void MapFieldGenerator::
 GenerateSerializeWithCachedSizes(io::Printer* printer) const {
-  printer->Print(variables_,
+  map<string, string> variables(variables_);
-      "{\n"
+  variables["write_entry"] = "::google::protobuf::internal::WireFormatLite::Write" +
-      "  ::google::protobuf::scoped_ptr<$map_classname$> entry;\n"
+                             variables["stream_writer"] + "(\n            " +
-      "  for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
+                             variables["number"] + ", *entry, output)";
-      "      it = this->$name$().begin();\n"
+  variables["deterministic"] = "output->IsSerializationDeterminstic()";
-      "      it != this->$name$().end(); ++it) {\n");
+  GenerateSerializeWithCachedSizes(printer, variables);
-
-  // If entry is allocated by arena, its desctructor should be avoided.
-  if (SupportsArenas(descriptor_)) {
-    printer->Print(variables_,
-        "    if (entry.get() != NULL && entry->GetArena() != NULL) {\n"
-        "      entry.release();\n"
-        "    }\n");
-  }
-
-  printer->Print(variables_,
-      "    entry.reset($name$_.New$wrapper$(it->first, it->second));\n"
-      "    ::google::protobuf::internal::WireFormatLite::Write$stream_writer$(\n"
-      "        $number$, *entry, output);\n");
-
-  printer->Indent();
-  printer->Indent();
-
-  const FieldDescriptor* key_field =
-      descriptor_->message_type()->FindFieldByName("key");
-  const FieldDescriptor* value_field =
-      descriptor_->message_type()->FindFieldByName("value");
-  if (key_field->type() == FieldDescriptor::TYPE_STRING) {
-    GenerateUtf8CheckCodeForString(key_field, options_, false, variables_,
-                                   "it->first.data(), it->first.length(),\n",
-                                   printer);
-  }
-  if (value_field->type() == FieldDescriptor::TYPE_STRING) {
-    GenerateUtf8CheckCodeForString(value_field, options_, false, variables_,
-                                   "it->second.data(), it->second.length(),\n",
-                                   printer);
-  }
-
-  printer->Outdent();
-  printer->Outdent();
-
-  printer->Print(
-      "  }\n");
-
-  // If entry is allocated by arena, its desctructor should be avoided.
-  if (SupportsArenas(descriptor_)) {
-    printer->Print(variables_,
-        "  if (entry.get() != NULL && entry->GetArena() != NULL) {\n"
-        "    entry.release();\n"
-        "  }\n");
-  }
-
-  printer->Print("}\n");
 }
 
 void MapFieldGenerator::
 GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const {
-  printer->Print(variables_,
+  map<string, string> variables(variables_);
-      "{\n"
+  variables["write_entry"] =
-      "  ::google::protobuf::scoped_ptr<$map_classname$> entry;\n"
+      "target = ::google::protobuf::internal::WireFormatLite::\n"
-      "  for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
+      "                   InternalWrite" + variables["declared_type"] +
-      "      it = this->$name$().begin();\n"
+      "NoVirtualToArray(\n                       " + variables["number"] +
-      "      it != this->$name$().end(); ++it) {\n");
+      ", *entry, deterministic, target);\n";
-
+  variables["deterministic"] = "deterministic";
-  // If entry is allocated by arena, its desctructor should be avoided.
+  GenerateSerializeWithCachedSizes(printer, variables);
-  if (SupportsArenas(descriptor_)) {
+}
-    printer->Print(variables_,
-        "    if (entry.get() != NULL && entry->GetArena() != NULL) {\n"
-        "      entry.release();\n"
-        "    }\n");
-  }
-
-  printer->Print(variables_,
-      "    entry.reset($name$_.New$wrapper$(it->first, it->second));\n"
-      "    target = ::google::protobuf::internal::WireFormatLite::\n"
-      "        InternalWrite$declared_type$NoVirtualToArray(\n"
-      "            $number$, *entry, false, target);\n");
 
+void MapFieldGenerator::GenerateSerializeWithCachedSizes(
+    io::Printer* printer, const map<string, string>& variables) const {
+  printer->Print(variables,
+      "if (!this->$name$().empty()) {\n");
   printer->Indent();
-  printer->Indent();
-
   const FieldDescriptor* key_field =
       descriptor_->message_type()->FindFieldByName("key");
   const FieldDescriptor* value_field =
       descriptor_->message_type()->FindFieldByName("value");
-  if (key_field->type() == FieldDescriptor::TYPE_STRING) {
+  const bool string_key = key_field->type() == FieldDescriptor::TYPE_STRING;
-    GenerateUtf8CheckCodeForString(key_field, options_, false, variables_,
+  const bool string_value = value_field->type() == FieldDescriptor::TYPE_STRING;
-                                   "it->first.data(), it->first.length(),\n",
+
-                                   printer);
+  printer->Print(variables,
+      "typedef ::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_pointer\n"
+      "    ConstPtr;\n");
+  if (string_key) {
+    printer->Print(variables,
+        "typedef ConstPtr SortItem;\n"
+        "typedef ::google::protobuf::internal::"
+        "CompareByDerefFirst<SortItem> Less;\n");
+  } else {
+    printer->Print(variables,
+        "typedef ::google::protobuf::internal::SortItem< $key_cpp$, ConstPtr > "
+        "SortItem;\n"
+        "typedef ::google::protobuf::internal::CompareByFirstField<SortItem> Less;\n");
   }
-  if (value_field->type() == FieldDescriptor::TYPE_STRING) {
+  string utf8_check;
-    GenerateUtf8CheckCodeForString(value_field, options_, false, variables_,
+  if (string_key || string_value) {
-                                   "it->second.data(), it->second.length(),\n",
+    printer->Print(
-                                   printer);
+        "struct Utf8Check {\n"
+        "  static void Check(ConstPtr p) {\n");
+    printer->Indent();
+    printer->Indent();
+    if (string_key) {
+      GenerateUtf8CheckCodeForString(key_field, options_, false, variables,
+                                     "p->first.data(), p->first.length(),\n",
+                                     printer);
+    }
+    if (string_value) {
+      GenerateUtf8CheckCodeForString(value_field, options_, false, variables,
+                                     "p->second.data(), p->second.length(),\n",
+                                     printer);
+    }
+    printer->Outdent();
+    printer->Outdent();
+    printer->Print(
+        "  }\n"
+        "};\n");
+    utf8_check = "Utf8Check::Check";
   }
 
-  printer->Outdent();
+  printer->Print(variables,
+      "\n"
+      "if ($deterministic$ &&\n"
+      "    this->$name$().size() > 1) {\n"
+      "  ::google::protobuf::scoped_array<SortItem> items(\n"
+      "      new SortItem[this->$name$().size()]);\n"
+      "  typedef ::google::protobuf::Map< $key_cpp$, $val_cpp$ >::size_type size_type;\n"
+      "  size_type n = 0;\n"
+      "  for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
+      "      it = this->$name$().begin();\n"
+      "      it != this->$name$().end(); ++it, ++n) {\n"
+      "    items[n] = SortItem(&*it);\n"
+      "  }\n"
+      "  ::std::sort(&items[0], &items[n], Less());\n");
+  printer->Indent();
+  GenerateSerializationLoop(printer, variables, SupportsArenas(descriptor_),
+                            utf8_check, "for (size_type i = 0; i < n; i++)",
+                            string_key ? "items[i]" : "items[i].second", false);
   printer->Outdent();
   printer->Print(
-      "  }\n");
+      "} else {\n");
-
+  printer->Indent();
-  // If entry is allocated by arena, its desctructor should be avoided.
+  GenerateSerializationLoop(
-  if (SupportsArenas(descriptor_)) {
+      printer, variables, SupportsArenas(descriptor_), utf8_check,
-    printer->Print(variables_,
+      "for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
-        "  if (entry.get() != NULL && entry->GetArena() != NULL) {\n"
+      "    it = this->$name$().begin();\n"
-        "    entry.release();\n"
+      "    it != this->$name$().end(); ++it)",
-        "  }\n");
+      "it", true);
-  }
+  printer->Outdent();
-
+  printer->Print("}\n");
+  printer->Outdent();
   printer->Print("}\n");
 }
 

src/google/protobuf/compiler/cpp/cpp_map_field.h

@@ -61,6 +61,10 @@ class MapFieldGenerator : public FieldGenerator {
   void GenerateByteSize(io::Printer* printer) const;
 
  private:
+  // A helper for GenerateSerializeWithCachedSizes{,ToArray}.
+  void GenerateSerializeWithCachedSizes(
+      io::Printer* printer, const map<string, string>& variables) const;
+
   const FieldDescriptor* descriptor_;
   const bool dependent_field_;
   map<string, string> variables_;

src/google/protobuf/compiler/cpp/cpp_message.cc

@@ -2715,7 +2715,9 @@ GenerateMergeFrom(io::Printer* printer) {
         "void $classname$::MergeFrom(const ::google::protobuf::Message& from) {\n"
         "// @@protoc_insertion_point(generalized_merge_from_start:"
         "$full_name$)\n"
-        "  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);\n",
+        "  if (GOOGLE_PREDICT_FALSE(&from == this)) {\n"
+        "    ::google::protobuf::internal::MergeFromFail(__FILE__, __LINE__);\n"
+        "  }\n",
         "classname", classname_, "full_name", descriptor_->full_name());
     printer->Indent();
 
@@ -2756,7 +2758,9 @@ GenerateMergeFrom(io::Printer* printer) {
       "void $classname$::MergeFrom(const $classname$& from) {\n"
       "// @@protoc_insertion_point(class_specific_merge_from_start:"
       "$full_name$)\n"
-      "  if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);\n",
+      "  if (GOOGLE_PREDICT_FALSE(&from == this)) {\n"
+      "    ::google::protobuf::internal::MergeFromFail(__FILE__, __LINE__);\n"
+      "  }\n",
       "classname", classname_, "full_name", descriptor_->full_name());
   printer->Indent();
 

src/google/protobuf/compiler/java/java_file.cc

@@ -266,9 +266,7 @@ void FileGenerator::Generate(io::Printer* printer) {
 
   printer->Print(
     "public static void registerAllExtensions(\n"
-    "    com.google.protobuf.ExtensionRegistry$lite$ registry) {\n",
+    "    com.google.protobuf.ExtensionRegistryLite registry) {\n");
-    "lite",
-    HasDescriptorMethods(file_, context_->EnforceLite()) ? "" : "Lite");
 
   printer->Indent();
 
@@ -283,6 +281,20 @@ void FileGenerator::Generate(io::Printer* printer) {
   printer->Outdent();
   printer->Print(
     "}\n");
+  if (HasDescriptorMethods(file_, context_->EnforceLite())) {
+    // Overload registerAllExtensions for the non-lite usage to
+    // redundantly maintain the original signature (this is
+    // redundant because ExtensionRegistryLite now invokes
+    // ExtensionRegistry in the non-lite usage). Intent is
+    // to remove this in the future.
+    printer->Print(
+      "\n"
+      "public static void registerAllExtensions(\n"
+      "    com.google.protobuf.ExtensionRegistry registry) {\n"
+      "  registerAllExtensions(\n"
+      "      (com.google.protobuf.ExtensionRegistryLite) registry);\n"
+      "}\n");
+  }
 
   // -----------------------------------------------------------------
 

src/google/protobuf/compiler/java/java_generator.cc

@@ -79,8 +79,6 @@ bool JavaGenerator::Generate(const FileDescriptor* file,
       file_options.generate_mutable_code = true;
     } else if (options[i].first == "shared") {
       file_options.generate_shared_code = true;
-    } else if (options[i].first == "lite") {
-      file_options.enforce_lite = true;
     } else if (options[i].first == "annotate_code") {
       file_options.annotate_code = true;
     } else if (options[i].first == "annotation_list_file") {

src/google/protobuf/compiler/js/js_generator.cc

@@ -2031,9 +2031,8 @@ void Generator::GenerateClassFieldToObject(const GeneratorOptions& options,
                      "getter", JSGetterName(options, field, BYTES_B64));
     } else {
       if (field->has_default_value()) {
-        printer->Print("jspb.Message.getField(msg, $index$) == null ? "
+        printer->Print("!msg.has$name$() ? $defaultValue$ : ",
-                       "$defaultValue$ : ",
+                       "name", JSGetterName(options, field),
-                       "index", JSFieldIndex(field),
                        "defaultValue", JSFieldDefault(field));
       }
       if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT ||
@@ -2408,9 +2407,8 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
                      "default", Proto3PrimitiveFieldDefault(field));
     } else {
       if (field->has_default_value()) {
-        printer->Print("jspb.Message.getField(this, $index$) == null ? "
+        printer->Print("!this.has$name$() ? $defaultValue$ : ",
-                       "$defaultValue$ : ",
+                       "name", JSGetterName(options, field),
-                       "index", JSFieldIndex(field),
                        "defaultValue", JSFieldDefault(field));
       }
       if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT ||
@@ -2515,6 +2513,20 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
           "\n",
           "clearedvalue", (field->is_repeated() ? "[]" : "undefined"),
           "returnvalue", JSReturnClause(field));
+
+      printer->Print(
+          "/**\n"
+          " * Returns whether this field is set.\n"
+          " * @return{!boolean}\n"
+          " */\n"
+          "$class$.prototype.has$name$ = function() {\n"
+          "  return jspb.Message.getField(this, $index$) != null;\n"
+          "};\n"
+          "\n"
+          "\n",
+          "class", GetPath(options, field->containing_type()),
+          "name", JSGetterName(options, field),
+          "index", JSFieldIndex(field));
     }
   }
 }

src/google/protobuf/compiler/python/python_generator.cc

@@ -44,6 +44,7 @@
 // performance-minded Python code leverage the fast C++ implementation
 // directly.
 
+#include <algorithm>
 #include <google/protobuf/stubs/hash.h>
 #include <limits>
 #include <map>
@@ -107,20 +108,25 @@ string ModuleAlias(const string& filename) {
   return module_name;
 }
 
+// Keywords reserved by the Python language.
+const char* const kKeywords[] = {
+    "False",   "None",     "True",     "and",    "as",    "assert", "break",
+    "class",   "continue", "def",      "del",    "elif",  "else",   "except",
+    "finally", "for",      "from",     "global", "if",    "import", "in",
+    "is",      "lambda",   "nonlocal", "not",    "or",    "pass",   "raise",
+    "return",  "try",      "while",    "with",   "yield",
+};
+const char* const* kKeywordsEnd =
+    kKeywords + (sizeof(kKeywords) / sizeof(kKeywords[0]));
 
-// Returns an import statement of form "from X.Y.Z import T" for the given
+bool ContainsPythonKeyword(const string& module_name) {
-// .proto filename.
+  vector<string> tokens = Split(module_name, ".");
-string ModuleImportStatement(const string& filename) {
+  for (int i = 0; i < tokens.size(); ++i) {
-  string module_name = ModuleName(filename);
+    if (std::find(kKeywords, kKeywordsEnd, tokens[i]) != kKeywordsEnd) {
-  int last_dot_pos = module_name.rfind('.');
+      return true;
-  if (last_dot_pos == string::npos) {
+    }
-    // NOTE(petya): this is not tested as it would require a protocol buffer
-    // outside of any package, and I don't think that is easily achievable.
-    return "import " + module_name;
-  } else {
-    return "from " + module_name.substr(0, last_dot_pos) + " import " +
-        module_name.substr(last_dot_pos + 1);
   }
+  return false;
 }
 
 
@@ -359,10 +365,32 @@ bool Generator::Generate(const FileDescriptor* file,
 void Generator::PrintImports() const {
   for (int i = 0; i < file_->dependency_count(); ++i) {
     const string& filename = file_->dependency(i)->name();
-    string import_statement = ModuleImportStatement(filename);
+
+    string module_name = ModuleName(filename);
     string module_alias = ModuleAlias(filename);
-    printer_->Print("$statement$ as $alias$\n", "statement",
+    if (ContainsPythonKeyword(module_name)) {
-                    import_statement, "alias", module_alias);
+      // If the module path contains a Python keyword, we have to quote the
+      // module name and import it using importlib. Otherwise the usual kind of
+      // import statement would result in a syntax error from the presence of
+      // the keyword.
+      printer_->Print("import importlib\n");
+      printer_->Print("$alias$ = importlib.import_module('$name$')\n", "alias",
+                      module_alias, "name", module_name);
+    } else {
+      int last_dot_pos = module_name.rfind('.');
+      string import_statement;
+      if (last_dot_pos == string::npos) {
+        // NOTE(petya): this is not tested as it would require a protocol buffer
+        // outside of any package, and I don't think that is easily achievable.
+        import_statement = "import " + module_name;
+      } else {
+        import_statement = "from " + module_name.substr(0, last_dot_pos) +
+                           " import " + module_name.substr(last_dot_pos + 1);
+      }
+      printer_->Print("$statement$ as $alias$\n", "statement", import_statement,
+                      "alias", module_alias);
+    }
+
     CopyPublicDependenciesAliases(module_alias, file_->dependency(i));
   }
   printer_->Print("\n");

src/google/protobuf/compiler/python/python_plugin_unittest.cc

@@ -46,6 +46,7 @@
 
 #include <google/protobuf/testing/file.h>
 #include <google/protobuf/testing/file.h>
+#include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/testing/googletest.h>
 #include <gtest/gtest.h>
 
@@ -115,6 +116,53 @@ TEST(PythonPluginTest, PluginTest) {
   EXPECT_EQ(0, cli.Run(5, argv));
 }
 
+// This test verifies that the generated Python output uses regular imports (as
+// opposed to importlib) in the usual case where the .proto file paths do not
+// not contain any Python keywords.
+TEST(PythonPluginTest, ImportTest) {
+  // Create files test1.proto and test2.proto with the former importing the
+  // latter.
+  GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/test1.proto",
+                             "syntax = \"proto3\";\n"
+                             "package foo;\n"
+                             "import \"test2.proto\";"
+                             "message Message1 {\n"
+                             "  Message2 message_2 = 1;\n"
+                             "}\n",
+                             true));
+  GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/test2.proto",
+                             "syntax = \"proto3\";\n"
+                             "package foo;\n"
+                             "message Message2 {}\n",
+                             true));
+
+  google::protobuf::compiler::CommandLineInterface cli;
+  cli.SetInputsAreProtoPathRelative(true);
+  python::Generator python_generator;
+  cli.RegisterGenerator("--python_out", &python_generator, "");
+  string proto_path = "-I" + TestTempDir();
+  string python_out = "--python_out=" + TestTempDir();
+  const char* argv[] = {"protoc", proto_path.c_str(), "-I.", python_out.c_str(),
+                        "test1.proto"};
+  ASSERT_EQ(0, cli.Run(5, argv));
+
+  // Loop over the lines of the generated code and verify that we find an
+  // ordinary Python import but do not find the string "importlib".
+  string output;
+  GOOGLE_CHECK_OK(File::GetContents(TestTempDir() + "/test1_pb2.py", &output,
+                             true));
+  std::vector<string> lines = Split(output, "\n");
+  string expected_import = "import test2_pb2";
+  bool found_expected_import = false;
+  for (int i = 0; i < lines.size(); ++i) {
+    if (lines[i].find(expected_import) != string::npos) {
+      found_expected_import = true;
+    }
+    EXPECT_EQ(string::npos, lines[i].find("importlib"));
+  }
+  EXPECT_TRUE(found_expected_import);
+}
+
 }  // namespace
 }  // namespace python
 }  // namespace compiler

src/google/protobuf/descriptor.proto

@@ -45,6 +45,7 @@ option java_package = "com.google.protobuf";
 option java_outer_classname = "DescriptorProtos";
 option csharp_namespace = "Google.Protobuf.Reflection";
 option objc_class_prefix = "GPB";
+option java_generate_equals_and_hash = true;
 
 // descriptor.proto must be optimized for speed because reflection-based
 // algorithms don't work during bootstrapping.

src/google/protobuf/extension_set_heavy.cc

@@ -341,7 +341,7 @@ bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input,
 
 int ExtensionSet::SpaceUsedExcludingSelf() const {
   int total_size =
-      extensions_.size() * sizeof(map<int, Extension>::value_type);
+      extensions_.size() * sizeof(ExtensionMap::value_type);
   for (ExtensionMap::const_iterator iter = extensions_.begin(),
        end = extensions_.end();
        iter != end;

src/google/protobuf/generated_message_util.cc

@@ -73,6 +73,12 @@ int StringSpaceUsedExcludingSelf(const string& str) {
 
 
 
+void MergeFromFail(const char* file, int line) {
+  GOOGLE_CHECK(false) << file << ":" << line;
+  // Open-source GOOGLE_CHECK(false) is not NORETURN.
+  exit(1);
+}
+
 }  // namespace internal
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/generated_message_util.h

@@ -41,8 +41,8 @@
 #include <assert.h>
 #include <string>
 
-#include <google/protobuf/stubs/once.h>
 #include <google/protobuf/stubs/common.h>
+#include <google/protobuf/stubs/once.h>
 
 namespace google {
 
@@ -115,6 +115,10 @@ class ArenaString;
 ArenaString* ReadArenaString(::google::protobuf::io::CodedInputStream* input,
                              ::google::protobuf::Arena* arena);
 
+// Helper function to crash on merge failure.
+// Moved out of generated code to reduce binary size.
+void MergeFromFail(const char* file, int line) GOOGLE_ATTRIBUTE_NORETURN;
+
 }  // namespace internal
 }  // namespace protobuf
 

src/google/protobuf/io/coded_stream.cc

@@ -649,13 +649,16 @@ bool CodedInputStream::Refresh() {
 
 // CodedOutputStream =================================================
 
+bool CodedOutputStream::default_serialization_deterministic_ = false;
+
 CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output)
   : output_(output),
     buffer_(NULL),
     buffer_size_(0),
     total_bytes_(0),
     had_error_(false),
-    aliasing_enabled_(false) {
+    aliasing_enabled_(false),
+    serialization_deterministic_is_overridden_(false) {
   // Eagerly Refresh() so buffer space is immediately available.
   Refresh();
   // The Refresh() may have failed. If the client doesn't write any data,
@@ -671,7 +674,8 @@ CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output,
     buffer_size_(0),
     total_bytes_(0),
     had_error_(false),
-    aliasing_enabled_(false) {
+    aliasing_enabled_(false),
+    serialization_deterministic_is_overridden_(false) {
   if (do_eager_refresh) {
     // Eagerly Refresh() so buffer space is immediately available.
     Refresh();

src/google/protobuf/io/coded_stream.h

@@ -813,6 +813,44 @@ class LIBPROTOBUF_EXPORT CodedOutputStream {
   // created.
   bool HadError() const { return had_error_; }
 
+  // Deterministic serialization, if requested, guarantees that for a given
+  // binary, equal messages will always be serialized to the same bytes. This
+  // implies:
+  //   . repeated serialization of a message will return the same bytes
+  //   . different processes of the same binary (which may be executing on
+  //     different machines) will serialize equal messages to the same bytes.
+  //
+  // Note the deterministic serialization is NOT canonical across languages; it
+  // is also unstable across different builds with schema changes due to unknown
+  // fields. Users who need canonical serialization, e.g., persistent storage in
+  // a canonical form, fingerprinting, etc., should define their own
+  // canonicalization specification and implement the serializer using
+  // reflection APIs rather than relying on this API.
+  //
+  // If determinisitc serialization is requested, the serializer will
+  // sort map entries by keys in lexicographical order or numerical order.
+  // (This is an implementation detail and may subject to change.)
+  //
+  // There are two ways to determine whether serialization should be
+  // deterministic for this CodedOutputStream.  If SetSerializationDeterministic
+  // has not yet been called, then the default comes from the global default,
+  // which is false, until SetDefaultSerializationDeterministic has been called.
+  // Otherwise, SetSerializationDeterministic has been called, and the last
+  // value passed to it is all that matters.
+  void SetSerializationDeterministic(bool value) {
+    serialization_deterministic_is_overridden_ = true;
+    serialization_deterministic_override_ = value;
+  }
+  // See above.  Also, note that users of this CodedOutputStream may need to
+  // call IsSerializationDeterminstic() to serialize in the intended way.  This
+  // CodedOutputStream cannot enforce a desire for deterministic serialization
+  // by itself.
+  bool IsSerializationDeterminstic() const {
+    return serialization_deterministic_is_overridden_ ?
+        serialization_deterministic_override_ :
+        default_serialization_deterministic_;
+  }
+
  private:
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedOutputStream);
 
@@ -822,6 +860,10 @@ class LIBPROTOBUF_EXPORT CodedOutputStream {
   int total_bytes_;  // Sum of sizes of all buffers seen so far.
   bool had_error_;   // Whether an error occurred during output.
   bool aliasing_enabled_;  // See EnableAliasing().
+  // See SetSerializationDeterministic() regarding these three fields.
+  bool serialization_deterministic_is_overridden_;
+  bool serialization_deterministic_override_;
+  static bool default_serialization_deterministic_;
 
   // Advance the buffer by a given number of bytes.
   void Advance(int amount);
@@ -849,6 +891,11 @@ class LIBPROTOBUF_EXPORT CodedOutputStream {
       uint64 value, uint8* target);
 
   static int VarintSize32Fallback(uint32 value);
+
+  // See above.  Other projects may use "friend" to allow them to call this.
+  static void SetDefaultSerializationDeterministic() {
+    default_serialization_deterministic_ = true;
+  }
 };
 
 // inline methods ====================================================

src/google/protobuf/map_entry_lite.h

@@ -535,6 +535,32 @@ class MapEntryLite : public MessageLite {
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MapEntryLite);
 };
 
+// Helpers for deterministic serialization =============================
+
+// This struct can be used with any generic sorting algorithm.  If the Key
+// type is relatively small and easy to copy then copying Keys into an
+// array of SortItems can be beneficial.  Then all the data the sorting
+// algorithm needs to touch is in that one array.
+template <typename Key, typename PtrToKeyValuePair> struct SortItem {
+  SortItem() {}
+  explicit SortItem(PtrToKeyValuePair p) : first(p->first), second(p) {}
+
+  Key first;
+  PtrToKeyValuePair second;
+};
+
+template <typename T> struct CompareByFirstField {
+  bool operator()(const T& a, const T& b) const {
+    return a.first < b.first;
+  }
+};
+
+template <typename T> struct CompareByDerefFirst {
+  bool operator()(const T& a, const T& b) const {
+    return a->first < b->first;
+  }
+};
+
 }  // namespace internal
 }  // namespace protobuf
 

src/google/protobuf/map_proto2_unittest.proto

@@ -64,3 +64,23 @@ message TestEnumMapPlusExtra {
 message TestImportEnumMap {
   map<int32, protobuf_unittest_import.ImportEnumForMap> import_enum_amp = 1;
 }
+
+message TestIntIntMap {
+  map<int32, int32> m = 1;
+}
+
+// Test all key types: string, plus the non-floating-point scalars.
+message TestMaps {
+  map<int32, TestIntIntMap> m_int32 = 1;
+  map<int64, TestIntIntMap> m_int64 = 2;
+  map<uint32, TestIntIntMap> m_uint32 = 3;
+  map<uint64, TestIntIntMap> m_uint64 = 4;
+  map<sint32, TestIntIntMap> m_sint32 = 5;
+  map<sint64, TestIntIntMap> m_sint64 = 6;
+  map<fixed32, TestIntIntMap> m_fixed32 = 7;
+  map<fixed64, TestIntIntMap> m_fixed64 = 8;
+  map<sfixed32, TestIntIntMap> m_sfixed32 = 9;
+  map<sfixed64, TestIntIntMap> m_sfixed64 = 10;
+  map<bool, TestIntIntMap> m_bool = 11;
+  map<string, TestIntIntMap> m_string = 12;
+}

src/google/protobuf/map_test.cc

@@ -74,6 +74,7 @@
 #include <google/protobuf/io/tokenizer.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <google/protobuf/util/time_util.h>
+#include <google/protobuf/util/message_differencer.h>
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/stubs/substitute.h>
 #include <gmock/gmock.h>
@@ -2869,6 +2870,82 @@ TEST(WireFormatForMapFieldTest, MapParseHelpers) {
   }
 }
 
+// Deterministic Serialization Test ==========================================
+
+template <typename T>
+static string DeterministicSerialization(const T& t) {
+  const int size = t.ByteSize();
+  string result(size, '\0');
+  io::ArrayOutputStream array_stream(string_as_array(&result), size);
+  io::CodedOutputStream output_stream(&array_stream);
+  output_stream.SetSerializationDeterministic(true);
+  t.SerializeWithCachedSizes(&output_stream);
+  EXPECT_FALSE(output_stream.HadError());
+  EXPECT_EQ(size, output_stream.ByteCount());
+  return result;
+}
+
+// Helper to test the serialization of the first arg against a golden file.
+static void TestDeterministicSerialization(const protobuf_unittest::TestMaps& t,
+                                           const string& filename) {
+  string expected;
+  GOOGLE_CHECK_OK(File::GetContents(
+      TestSourceDir() + "/google/protobuf/testdata/" + filename,
+      &expected, true));
+  const string actual = DeterministicSerialization(t);
+  EXPECT_EQ(expected, actual);
+  protobuf_unittest::TestMaps u;
+  EXPECT_TRUE(u.ParseFromString(actual));
+  EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(u, t));
+}
+
+// Helper for MapSerializationTest.  Return a 7-bit ASCII string.
+static string ConstructKey(uint64 n) {
+  string s(n % static_cast<uint64>(9), '\0');
+  if (s.empty()) {
+    return StrCat(n);
+  } else {
+    while (n != 0) {
+      s[n % s.size()] = (n >> 10) & 0x7f;
+      n /= 888;
+    }
+    return s;
+  }
+}
+
+TEST(MapSerializationTest, Deterministic) {
+  const int kIters = 25;
+  protobuf_unittest::TestMaps t;
+  protobuf_unittest::TestIntIntMap inner;
+  (*inner.mutable_m())[0] = (*inner.mutable_m())[10] =
+      (*inner.mutable_m())[-200] = 0;
+  uint64 frog = 9;
+  const uint64 multiplier = 0xa29cd16f;
+  for (int i = 0; i < kIters; i++) {
+    const int32 i32 = static_cast<int32>(frog & 0xffffffff);
+    const uint32 u32 = static_cast<uint32>(i32) * 91919;
+    const int64 i64 = static_cast<int64>(frog);
+    const uint64 u64 = frog * static_cast<uint64>(187321);
+    const bool b = i32 > 0;
+    const string s = ConstructKey(frog);
+    (*inner.mutable_m())[i] = i32;
+    (*t.mutable_m_int32())[i32] = (*t.mutable_m_sint32())[i32] =
+        (*t.mutable_m_sfixed32())[i32] = inner;
+    (*t.mutable_m_uint32())[u32] = (*t.mutable_m_fixed32())[u32] = inner;
+    (*t.mutable_m_int64())[i64] = (*t.mutable_m_sint64())[i64] =
+        (*t.mutable_m_sfixed64())[i64] = inner;
+    (*t.mutable_m_uint64())[u64] = (*t.mutable_m_fixed64())[u64] = inner;
+    (*t.mutable_m_bool())[b] = inner;
+    (*t.mutable_m_string())[s] = inner;
+    (*t.mutable_m_string())[s + string(1 << (u32 % static_cast<uint32>(9)),
+                                       b)] = inner;
+    inner.mutable_m()->erase(i);
+    frog = frog * multiplier + i;
+    frog ^= (frog >> 41);
+  }
+  TestDeterministicSerialization(t, "golden_message_maps");
+}
+
 // Text Format Test =================================================
 
 TEST(TextFormatMapTest, SerializeAndParse) {

src/google/protobuf/map_type_handler.h

@@ -166,10 +166,10 @@ class MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type> {
                            io::CodedOutputStream* output);
   static inline uint8* InternalWriteToArray(int field,
                                             const MapEntryAccessorType& value,
-                                            bool deterministic, uint8* output);
+                                            bool deterministic, uint8* target);
   static inline uint8* WriteToArray(int field,
                                     const MapEntryAccessorType& value,
-                                    uint8* output);
+                                    uint8* target);
 
   // Functions to manipulate data on memory. ========================
   static inline const Type& GetExternalReference(const Type* value);
@@ -227,11 +227,11 @@ class MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type> {
         int field,                                                            \
         const MapEntryAccessorType& value,                                    \
         bool deterministic,                                                   \
-        uint8* output);                                                       \
+        uint8* target);                                                       \
     static inline uint8* WriteToArray(int field,                              \
                                       const MapEntryAccessorType& value,      \
-                                      uint8* output) {                        \
+                                      uint8* target) {                        \
-      return InternalWriteToArray(field, value, false, output);               \
+      return InternalWriteToArray(field, value, false, target);               \
     }                                                                         \
     static inline const MapEntryAccessorType& GetExternalReference(           \
         const TypeOnMemory& value);                                           \
@@ -374,9 +374,9 @@ template <typename Type>
 inline uint8*
 MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type>::InternalWriteToArray(
     int field, const MapEntryAccessorType& value, bool deterministic,
-    uint8* output) {
+    uint8* target) {
   return WireFormatLite::InternalWriteMessageToArray(field, value,
-                                                     deterministic, output);
+                                                     deterministic, target);
 }
 
 #define WRITE_METHOD(FieldType, DeclaredType)                                  \
@@ -390,8 +390,8 @@ MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type>::InternalWriteToArray(
   inline uint8*                                                                \
   MapTypeHandler<WireFormatLite::TYPE_##FieldType,                             \
                  Type>::InternalWriteToArray(                                  \
-      int field, const MapEntryAccessorType& value, bool, uint8* output) {     \
+      int field, const MapEntryAccessorType& value, bool, uint8* target) {     \
-    return WireFormatLite::Write##DeclaredType##ToArray(field, value, output); \
+    return WireFormatLite::Write##DeclaredType##ToArray(field, value, target); \
   }
 
 WRITE_METHOD(STRING  , String)

src/google/protobuf/message.cc

@@ -62,8 +62,6 @@ namespace protobuf {
 using internal::WireFormat;
 using internal::ReflectionOps;
 
-Message::~Message() {}
-
 void Message::MergeFrom(const Message& from) {
   const Descriptor* descriptor = GetDescriptor();
   GOOGLE_CHECK_EQ(from.GetDescriptor(), descriptor)

src/google/protobuf/message.h

@@ -179,7 +179,7 @@ struct Metadata {
 class LIBPROTOBUF_EXPORT Message : public MessageLite {
  public:
   inline Message() {}
-  virtual ~Message();
+  virtual ~Message() {}
 
   // Basic Operations ------------------------------------------------
 

src/google/protobuf/message_lite.cc

@@ -46,8 +46,6 @@
 namespace google {
 namespace protobuf {
 
-MessageLite::~MessageLite() {}
-
 string MessageLite::InitializationErrorString() const {
   return "(cannot determine missing fields for lite message)";
 }

src/google/protobuf/message_lite.h

@@ -81,7 +81,7 @@ namespace internal {
 class LIBPROTOBUF_EXPORT MessageLite {
  public:
   inline MessageLite() {}
-  virtual ~MessageLite();
+  virtual ~MessageLite() {}
 
   // Basic Operations ------------------------------------------------
 

src/google/protobuf/text_format.cc

@@ -759,6 +759,20 @@ class TextFormat::Parser::ParserImpl {
       }
       return true;
     }
+    if (TryConsume("[")) {
+      while (true) {
+        if (!LookingAt("{") && !LookingAt("<")) {
+          DO(SkipFieldValue());
+        } else {
+          DO(SkipFieldMessage());
+        }
+        if (TryConsume("]")) {
+          break;
+        }
+        DO(Consume(","));
+      }
+      return true;
+    }
     // Possible field values other than string:
     //   12345        => TYPE_INTEGER
     //   -12345       => TYPE_SYMBOL + TYPE_INTEGER

src/google/protobuf/unknown_field_set.cc

@@ -69,28 +69,14 @@ const UnknownFieldSet* UnknownFieldSet::default_instance() {
   return default_unknown_field_set_instance_;
 }
 
-UnknownFieldSet::UnknownFieldSet()
-    : fields_(NULL) {}
-
-UnknownFieldSet::~UnknownFieldSet() {
-  Clear();
-  delete fields_;
-}
-
 void UnknownFieldSet::ClearFallback() {
-  if (fields_ != NULL) {
+  GOOGLE_DCHECK(fields_ != NULL && fields_->size() > 0);
-    for (int i = 0; i < fields_->size(); i++) {
+  int n = fields_->size();
-      (*fields_)[i].Delete();
+  do {
-    }
+    (*fields_)[--n].Delete();
-    delete fields_;
+  } while (n > 0);
-    fields_ = NULL;
+  delete fields_;
-  }
+  fields_ = NULL;
-}
-
-void UnknownFieldSet::ClearAndFreeMemory() {
-  if (fields_ != NULL) {
-    Clear();
-  }
 }
 
 void UnknownFieldSet::InternalMergeFrom(const UnknownFieldSet& other) {

src/google/protobuf/unknown_field_set.h

@@ -241,8 +241,14 @@ class LIBPROTOBUF_EXPORT UnknownField {
 // ===================================================================
 // inline implementations
 
+inline UnknownFieldSet::UnknownFieldSet() : fields_(NULL) {}
+
+inline UnknownFieldSet::~UnknownFieldSet() { Clear(); }
+
+inline void UnknownFieldSet::ClearAndFreeMemory() { Clear(); }
+
 inline void UnknownFieldSet::Clear() {
-  if (fields_) {
+  if (fields_ != NULL) {
     ClearFallback();
   }
 }

src/google/protobuf/util/internal/default_value_objectwriter.cc

@@ -64,6 +64,7 @@ DefaultValueObjectWriter::DefaultValueObjectWriter(
       type_(type),
       current_(NULL),
       root_(NULL),
+      suppress_empty_list_(false),
       field_scrub_callback_(NULL),
       ow_(ow) {}
 
@@ -184,12 +185,10 @@ void DefaultValueObjectWriter::RegisterFieldScrubCallBack(
   field_scrub_callback_.reset(field_scrub_callback.release());
 }
 
-DefaultValueObjectWriter::Node::Node(const string& name,
+DefaultValueObjectWriter::Node::Node(
-                                     const google::protobuf::Type* type,
+    const string& name, const google::protobuf::Type* type, NodeKind kind,
-                                     NodeKind kind, const DataPiece& data,
+    const DataPiece& data, bool is_placeholder, const vector<string>& path,
-                                     bool is_placeholder,
+    bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback)
-                                     const vector<string>& path,
-                                     FieldScrubCallBack* field_scrub_callback)
     : name_(name),
       type_(type),
       kind_(kind),
@@ -197,6 +196,7 @@ DefaultValueObjectWriter::Node::Node(const string& name,
       data_(data),
       is_placeholder_(is_placeholder),
       path_(path),
+      suppress_empty_list_(suppress_empty_list),
       field_scrub_callback_(field_scrub_callback) {}
 
 DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild(
@@ -230,6 +230,9 @@ void DefaultValueObjectWriter::Node::WriteTo(ObjectWriter* ow) {
   // Write out lists. If we didn't have any list in response, write out empty
   // list.
   if (kind_ == LIST) {
+    // Suppress empty lists if requested.
+    if (suppress_empty_list_ && is_placeholder_) return;
+
     ow->StartList(name_);
     WriteChildren(ow);
     ow->EndList();
@@ -366,7 +369,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren(
         field.json_name(), field_type, kind,
         kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo)
                           : DataPiece::NullData(),
-        true, path, field_scrub_callback_));
+        true, path, suppress_empty_list_, field_scrub_callback_));
     new_children.push_back(child.release());
   }
   // Adds all leftover nodes in children_ to the beginning of new_child.
@@ -462,7 +465,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject(
   if (current_ == NULL) {
     vector<string> path;
     root_.reset(new Node(name.ToString(), &type_, OBJECT, DataPiece::NullData(),
-                         false, path, field_scrub_callback_.get()));
+                         false, path, suppress_empty_list_,
+                         field_scrub_callback_.get()));
     root_->PopulateChildren(typeinfo_);
     current_ = root_.get();
     return this;
@@ -478,7 +482,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject(
                               : NULL),
         OBJECT, DataPiece::NullData(), false,
         child == NULL ? current_->path() : child->path(),
-        field_scrub_callback_.get()));
+        suppress_empty_list_, field_scrub_callback_.get()));
     child = node.get();
     current_->AddChild(node.release());
   }
@@ -509,7 +513,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList(
   if (current_ == NULL) {
     vector<string> path;
     root_.reset(new Node(name.ToString(), &type_, LIST, DataPiece::NullData(),
-                         false, path, field_scrub_callback_.get()));
+                         false, path, suppress_empty_list_,
+                         field_scrub_callback_.get()));
     current_ = root_.get();
     return this;
   }
@@ -519,7 +524,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList(
     google::protobuf::scoped_ptr<Node> node(
         new Node(name.ToString(), NULL, LIST, DataPiece::NullData(), false,
                  child == NULL ? current_->path() : child->path(),
-                 field_scrub_callback_.get()));
+                 suppress_empty_list_, field_scrub_callback_.get()));
     child = node.get();
     current_->AddChild(node.release());
   }
@@ -577,7 +582,7 @@ void DefaultValueObjectWriter::RenderDataPiece(StringPiece name,
     google::protobuf::scoped_ptr<Node> node(
         new Node(name.ToString(), NULL, PRIMITIVE, data, false,
                  child == NULL ? current_->path() : child->path(),
-                 field_scrub_callback_.get()));
+                 suppress_empty_list_, field_scrub_callback_.get()));
     child = node.get();
     current_->AddChild(node.release());
   } else {

src/google/protobuf/util/internal/default_value_objectwriter.h

@@ -122,6 +122,10 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
   // field_scrub_callback pointer is also transferred to this class
   void RegisterFieldScrubCallBack(FieldScrubCallBackPtr field_scrub_callback);
 
+  // If set to true, empty lists are suppressed from output when default values
+  // are written.
+  void set_suppress_empty_list(bool value) { suppress_empty_list_ = value; }
+
  private:
   enum NodeKind {
     PRIMITIVE = 0,
@@ -136,7 +140,7 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
    public:
     Node(const string& name, const google::protobuf::Type* type, NodeKind kind,
          const DataPiece& data, bool is_placeholder, const vector<string>& path,
-         FieldScrubCallBack* field_scrub_callback);
+         bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback);
     virtual ~Node() {
       for (int i = 0; i < children_.size(); ++i) {
         delete children_[i];
@@ -212,6 +216,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
     // Path of the field of this node
     std::vector<string> path_;
 
+    // Whether to suppress empty list output.
+    bool suppress_empty_list_;
+
     // Pointer to function for determining whether a field needs to be scrubbed
     // or not. This callback is owned by the creator of this node.
     FieldScrubCallBack* field_scrub_callback_;
@@ -257,6 +264,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
   // The stack to hold the path of Nodes from current_ to root_;
   std::stack<Node*> stack_;
 
+  // Whether to suppress output of empty lists.
+  bool suppress_empty_list_;
+
   // Unique Pointer to function for determining whether a field needs to be
   // scrubbed or not.
   FieldScrubCallBackPtr field_scrub_callback_;

src/google/protobuf/util/internal/default_value_objectwriter_test.cc

@@ -149,6 +149,39 @@ TEST_P(DefaultValueObjectWriterTest, ShouldRetainUnknownField) {
 }
 
 
+class DefaultValueObjectWriterSuppressListTest
+    : public BaseDefaultValueObjectWriterTest {
+ protected:
+  DefaultValueObjectWriterSuppressListTest()
+      : BaseDefaultValueObjectWriterTest(DefaultValueTest::descriptor()) {
+    testing_->set_suppress_empty_list(true);
+  }
+  ~DefaultValueObjectWriterSuppressListTest() {}
+};
+
+INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest,
+                        DefaultValueObjectWriterSuppressListTest,
+                        ::testing::Values(
+                            testing::USE_TYPE_RESOLVER));
+
+TEST_P(DefaultValueObjectWriterSuppressListTest, Empty) {
+  // Set expectation. Emtpy lists should be suppressed.
+  expects_.StartObject("")
+      ->RenderDouble("doubleValue", 0.0)
+      ->RenderFloat("floatValue", 0.0)
+      ->RenderInt64("int64Value", 0)
+      ->RenderUint64("uint64Value", 0)
+      ->RenderInt32("int32Value", 0)
+      ->RenderUint32("uint32Value", 0)
+      ->RenderBool("boolValue", false)
+      ->RenderString("stringValue", "")
+      ->RenderBytes("bytesValue", "")
+      ->RenderString("enumValue", "ENUM_FIRST")
+      ->EndObject();
+
+  // Actual testing
+  testing_->StartObject("")->EndObject();
+}
 }  // namespace testing
 }  // namespace converter
 }  // namespace util

src/google/protobuf/util/internal/proto_writer.cc

@@ -298,7 +298,9 @@ ProtoWriter::ProtoElement::ProtoElement(const TypeInfo* typeinfo,
       proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3),
       type_(type),
       size_index_(-1),
-      array_index_(-1) {
+      array_index_(-1),
+      // oneof_indices_ values are 1-indexed (0 means not present).
+      oneof_indices_(type.oneofs_size() + 1) {
   if (!proto3_) {
     required_fields_ = GetRequiredFields(type_);
   }
@@ -312,13 +314,15 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent,
       ow_(this->parent()->ow_),
       parent_field_(field),
       typeinfo_(this->parent()->typeinfo_),
-      proto3_(this->parent()->proto3_),
+      proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3),
       type_(type),
       size_index_(
           !is_list && field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE
               ? ow_->size_insert_.size()
               : -1),
-      array_index_(is_list ? 0 : -1) {
+      array_index_(is_list ? 0 : -1),
+      // oneof_indices_ values are 1-indexed (0 means not present).
+      oneof_indices_(type_.oneofs_size() + 1) {
   if (!is_list) {
     if (ow_->IsRepeated(*field)) {
       // Update array_index_ if it is an explicit list.
@@ -411,11 +415,11 @@ string ProtoWriter::ProtoElement::ToString() const {
 }
 
 bool ProtoWriter::ProtoElement::IsOneofIndexTaken(int32 index) {
-  return ContainsKey(oneof_indices_, index);
+  return oneof_indices_[index];
 }
 
 void ProtoWriter::ProtoElement::TakeOneofIndex(int32 index) {
-  InsertIfNotPresent(&oneof_indices_, index);
+  oneof_indices_[index] = true;
 }
 
 void ProtoWriter::InvalidName(StringPiece unknown_name, StringPiece message) {
@@ -573,10 +577,19 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField(
 
   // Pushing a ProtoElement and then pop it off at the end for 2 purposes:
   // error location reporting and required field accounting.
-  element_.reset(new ProtoElement(element_.release(), &field, type, false));
+  //
+  // For proto3, since there is no required field tracking, we only need to push
+  // ProtoElement for error cases.
+  if (!element_->proto3()) {
+    element_.reset(new ProtoElement(element_.release(), &field, type, false));
+  }
 
   if (field.kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN ||
       field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) {
+    // Push a ProtoElement for location reporting purposes.
+    if (element_->proto3()) {
+      element_.reset(new ProtoElement(element_.release(), &field, type, false));
+    }
     InvalidValue(field.type_url().empty()
                      ? google::protobuf::Field_Kind_Name(field.kind())
                      : field.type_url(),
@@ -657,11 +670,18 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField(
   }
 
   if (!status.ok()) {
+    // Push a ProtoElement for location reporting purposes.
+    if (element_->proto3()) {
+      element_.reset(new ProtoElement(element_.release(), &field, type, false));
+    }
     InvalidValue(google::protobuf::Field_Kind_Name(field.kind()),
                  status.error_message());
+    element_.reset(element()->pop());
+    return this;
   }
 
-  element_.reset(element()->pop());
+  if (!element_->proto3()) element_.reset(element()->pop());
+
   return this;
 }
 

src/google/protobuf/util/internal/proto_writer.h

@@ -32,8 +32,8 @@
 #define GOOGLE_PROTOBUF_UTIL_CONVERTER_PROTO_WRITER_H__
 
 #include <deque>
-#include <google/protobuf/stubs/hash.h>
 #include <string>
+#include <vector>
 
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/io/coded_stream.h>
@@ -45,6 +45,7 @@
 #include <google/protobuf/util/internal/structured_objectwriter.h>
 #include <google/protobuf/util/type_resolver.h>
 #include <google/protobuf/stubs/bytestream.h>
+#include <google/protobuf/stubs/hash.h>
 
 namespace google {
 namespace protobuf {
@@ -191,6 +192,8 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
     // generate an error.
     void TakeOneofIndex(int32 index);
 
+    bool proto3() { return proto3_; }
+
    private:
     // Used for access to variables of the enclosing instance of
     // ProtoWriter.
@@ -203,7 +206,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
     // TypeInfo to lookup types.
     const TypeInfo* typeinfo_;
 
-    // Whether the root type is a proto3 or not.
+    // Whether the type_ is proto3 or not.
     bool proto3_;
 
     // Additional variables if this element is a message:
@@ -221,7 +224,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
 
     // Set of oneof indices already seen for the type_. Used to validate
     // incoming messages so no more than one oneof is set.
-    hash_set<int32> oneof_indices_;
+    std::vector<bool> oneof_indices_;
 
     GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement);
   };

src/google/protobuf/util/json_util.cc

@@ -177,6 +177,66 @@ util::Status JsonToBinaryString(TypeResolver* resolver,
       resolver, type_url, &input_stream, &output_stream, options);
 }
 
+namespace {
+const char* kTypeUrlPrefix = "type.googleapis.com";
+TypeResolver* generated_type_resolver_ = NULL;
+GOOGLE_PROTOBUF_DECLARE_ONCE(generated_type_resolver_init_);
+
+string GetTypeUrl(const Message& message) {
+  return string(kTypeUrlPrefix) + "/" + message.GetDescriptor()->full_name();
+}
+
+void DeleteGeneratedTypeResolver() { delete generated_type_resolver_; }
+
+void InitGeneratedTypeResolver() {
+  generated_type_resolver_ = NewTypeResolverForDescriptorPool(
+      kTypeUrlPrefix, DescriptorPool::generated_pool());
+  ::google::protobuf::internal::OnShutdown(&DeleteGeneratedTypeResolver);
+}
+
+TypeResolver* GetGeneratedTypeResolver() {
+  ::google::protobuf::GoogleOnceInit(&generated_type_resolver_init_, &InitGeneratedTypeResolver);
+  return generated_type_resolver_;
+}
+}  // namespace
+
+util::Status MessageToJsonString(const Message& message, string* output,
+                                   const JsonOptions& options) {
+  const DescriptorPool* pool = message.GetDescriptor()->file()->pool();
+  TypeResolver* resolver =
+      pool == DescriptorPool::generated_pool()
+          ? GetGeneratedTypeResolver()
+          : NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool);
+  util::Status result =
+      BinaryToJsonString(resolver, GetTypeUrl(message),
+                         message.SerializeAsString(), output, options);
+  if (pool != DescriptorPool::generated_pool()) {
+    delete resolver;
+  }
+  return result;
+}
+
+util::Status JsonStringToMessage(const string& input, Message* message,
+                                   const JsonParseOptions& options) {
+  const DescriptorPool* pool = message->GetDescriptor()->file()->pool();
+  TypeResolver* resolver =
+      pool == DescriptorPool::generated_pool()
+          ? GetGeneratedTypeResolver()
+          : NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool);
+  string binary;
+  util::Status result = JsonToBinaryString(
+      resolver, GetTypeUrl(*message), input, &binary, options);
+  if (result.ok() && !message->ParseFromString(binary)) {
+    result =
+        util::Status(util::error::INVALID_ARGUMENT,
+                       "JSON transcoder produced invalid protobuf output.");
+  }
+  if (pool != DescriptorPool::generated_pool()) {
+    delete resolver;
+  }
+  return result;
+}
+
 }  // namespace util
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/util/json_util.h

@@ -33,6 +33,7 @@
 #ifndef GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__
 #define GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__
 
+#include <google/protobuf/message.h>
 #include <google/protobuf/util/type_resolver.h>
 #include <google/protobuf/stubs/bytestream.h>
 
@@ -69,6 +70,30 @@ struct JsonPrintOptions {
 // DEPRECATED. Use JsonPrintOptions instead.
 typedef JsonPrintOptions JsonOptions;
 
+// Converts from protobuf message to JSON. This is a simple wrapper of
+// BinaryToJsonString(). It will use the DescriptorPool of the passed-in
+// message to resolve Any types.
+util::Status MessageToJsonString(const Message& message,
+                                   string* output,
+                                   const JsonOptions& options);
+
+inline util::Status MessageToJsonString(const Message& message,
+                                          string* output) {
+  return MessageToJsonString(message, output, JsonOptions());
+}
+
+// Converts from JSON to protobuf message. This is a simple wrapper of
+// JsonStringToBinary(). It will use the DescriptorPool of the passed-in
+// message to resolve Any types.
+util::Status JsonStringToMessage(const string& input,
+                                   Message* message,
+                                   const JsonParseOptions& options);
+
+inline util::Status JsonStringToMessage(const string& input,
+                                          Message* message) {
+  return JsonStringToMessage(input, message, JsonParseOptions());
+}
+
 // Converts protobuf binary data to JSON.
 // The conversion will fail if:
 //   1. TypeResolver fails to resolve a type.

src/google/protobuf/util/json_util_test.cc

@@ -34,6 +34,8 @@
 #include <string>
 
 #include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/descriptor_database.h>
+#include <google/protobuf/dynamic_message.h>
 #include <google/protobuf/util/json_format_proto3.pb.h>
 #include <google/protobuf/util/type_resolver.h>
 #include <google/protobuf/util/type_resolver_util.h>
@@ -63,28 +65,21 @@ static string GetTypeUrl(const Descriptor* message) {
 class JsonUtilTest : public testing::Test {
  protected:
   JsonUtilTest() {
-    resolver_.reset(NewTypeResolverForDescriptorPool(
-        kTypeUrlPrefix, DescriptorPool::generated_pool()));
   }
 
   string ToJson(const Message& message, const JsonPrintOptions& options) {
     string result;
-    GOOGLE_CHECK_OK(BinaryToJsonString(resolver_.get(),
+    GOOGLE_CHECK_OK(MessageToJsonString(message, &result, options));
-                                GetTypeUrl(message.GetDescriptor()),
-                                message.SerializeAsString(), &result, options));
     return result;
   }
 
   bool FromJson(const string& json, Message* message,
                 const JsonParseOptions& options) {
-    string binary;
+    return JsonStringToMessage(json, message, options).ok();
-    if (!JsonToBinaryString(resolver_.get(),
+  }
-                            GetTypeUrl(message->GetDescriptor()), json, &binary,
+
-                            options)
+  bool FromJson(const string& json, Message* message) {
-             .ok()) {
+    return FromJson(json, message, JsonParseOptions());
-      return false;
-    }
-    return message->ParseFromString(binary);
   }
 
   google::protobuf::scoped_ptr<TypeResolver> resolver_;
@@ -189,6 +184,45 @@ TEST_F(JsonUtilTest, TestParseErrors) {
   EXPECT_FALSE(FromJson("{\"int32Value\":2147483648}", &m, options));
 }
 
+TEST_F(JsonUtilTest, TestDynamicMessage) {
+  // Some random message but good enough to test the wrapper functions.
+  string input =
+      "{\n"
+      "  \"int32Value\": 1024,\n"
+      "  \"repeatedInt32Value\": [1, 2],\n"
+      "  \"messageValue\": {\n"
+      "    \"value\": 2048\n"
+      "  },\n"
+      "  \"repeatedMessageValue\": [\n"
+      "    {\"value\": 40}, {\"value\": 96}\n"
+      "  ]\n"
+      "}\n";
+
+  // Create a new DescriptorPool with the same protos as the generated one.
+  DescriptorPoolDatabase database(*DescriptorPool::generated_pool());
+  DescriptorPool pool(&database);
+  // A dynamic version of the test proto.
+  DynamicMessageFactory factory;
+  google::protobuf::scoped_ptr<Message> message(factory.GetPrototype(
+      pool.FindMessageTypeByName("proto3.TestMessage"))->New());
+  EXPECT_TRUE(FromJson(input, message.get()));
+
+  // Convert to generated message for easy inspection.
+  TestMessage generated;
+  EXPECT_TRUE(generated.ParseFromString(message->SerializeAsString()));
+  EXPECT_EQ(1024, generated.int32_value());
+  ASSERT_EQ(2, generated.repeated_int32_value_size());
+  EXPECT_EQ(1, generated.repeated_int32_value(0));
+  EXPECT_EQ(2, generated.repeated_int32_value(1));
+  EXPECT_EQ(2048, generated.message_value().value());
+  ASSERT_EQ(2, generated.repeated_message_value_size());
+  EXPECT_EQ(40, generated.repeated_message_value(0).value());
+  EXPECT_EQ(96, generated.repeated_message_value(1).value());
+
+  JsonOptions options;
+  EXPECT_EQ(ToJson(generated, options), ToJson(*message, options));
+}
+
 typedef pair<char*, int> Segment;
 // A ZeroCopyOutputStream that writes to multiple buffers.
 class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream {

src/google/protobuf/wire_format_lite.cc

@@ -466,7 +466,8 @@ void WireFormatLite::WriteGroupMaybeToArray(int field_number,
   const int size = value.GetCachedSize();
   uint8* target = output->GetDirectBufferForNBytesAndAdvance(size);
   if (target != NULL) {
-    uint8* end = value.SerializeWithCachedSizesToArray(target);
+    uint8* end = value.InternalSerializeWithCachedSizesToArray(
+        output->IsSerializationDeterminstic(), target);
     GOOGLE_DCHECK_EQ(end - target, size);
   } else {
     value.SerializeWithCachedSizes(output);
@@ -482,7 +483,8 @@ void WireFormatLite::WriteMessageMaybeToArray(int field_number,
   output->WriteVarint32(size);
   uint8* target = output->GetDirectBufferForNBytesAndAdvance(size);
   if (target != NULL) {
-    uint8* end = value.SerializeWithCachedSizesToArray(target);
+    uint8* end = value.InternalSerializeWithCachedSizesToArray(
+        output->IsSerializationDeterminstic(), target);
     GOOGLE_DCHECK_EQ(end - target, size);
   } else {
     value.SerializeWithCachedSizes(output);