organicmaps/protobuf
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

Down integrate from Google internal.

Browse source 
Change-Id: I34d301133eea9c6f3a822c47d1f91e136fd33145
This commit is contained in:
Jisi Liu 2015-02-25 16:39:11 -08:00
parent 581be24606
commit ada6556785
39 changed files with 5816 additions and 2144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
python/README.txt
View file

@ -48,8 +48,9 @@ Installation
$ python setup.py build $ python setup.py build
$ python setup.py google_test $ python setup.py google_test
If you want to test c++ implementation, run: If you want to build/test c++ implementation, run:
$ python setup.py test --cpp_implementation $ python setup.py build --cpp_implementation
$ python setup.py google_test --cpp_implementation
If some tests fail, this library may not work correctly on your If some tests fail, this library may not work correctly on your
system. Continue at your own risk. system. Continue at your own risk.

158
python/google/protobuf/descriptor.py
View file

@ -41,15 +41,13 @@ __author__ = 'robinson@google.com (Will Robinson)'
from google.protobuf.internal import api_implementation from google.protobuf.internal import api_implementation
_USE_C_DESCRIPTORS = False
if api_implementation.Type() == 'cpp': if api_implementation.Type() == 'cpp':
# Used by MakeDescriptor in cpp mode # Used by MakeDescriptor in cpp mode
import os import os
import uuid import uuid
from google.protobuf.pyext import _message
if api_implementation.Version() == 2: _USE_C_DESCRIPTORS = getattr(_message, '_USE_C_DESCRIPTORS', False)
from google.protobuf.pyext import _message
else:
from google.protobuf.internal import cpp_message
class Error(Exception): class Error(Exception):
@ -60,12 +58,29 @@ class TypeTransformationError(Error):
"""Error transforming between python proto type and corresponding C++ type.""" """Error transforming between python proto type and corresponding C++ type."""
if _USE_C_DESCRIPTORS:
# This metaclass allows to override the behavior of code like
# isinstance(my_descriptor, FieldDescriptor)
# and make it return True when the descriptor is an instance of the extension
# type written in C++.
class DescriptorMetaclass(type):
def __instancecheck__(cls, obj):
if super(DescriptorMetaclass, cls).__instancecheck__(obj):
return True
if isinstance(obj, cls._C_DESCRIPTOR_CLASS):
return True
return False
else:
# The standard metaclass; nothing changes.
DescriptorMetaclass = type
class DescriptorBase(object): class DescriptorBase(object):
"""Descriptors base class. """Descriptors base class.
This class is the base of all descriptor classes. It provides common options This class is the base of all descriptor classes. It provides common options
related functionaility. related functionality.
Attributes: Attributes:
has_options: True if the descriptor has non-default options. Usually it has_options: True if the descriptor has non-default options. Usually it
@ -75,6 +90,12 @@ class DescriptorBase(object):
avoid some bootstrapping issues. avoid some bootstrapping issues.
""" """
__metaclass__ = DescriptorMetaclass
if _USE_C_DESCRIPTORS:
# The class, or tuple of classes, that are considered as "virtual
# subclasses" of this descriptor class.
_C_DESCRIPTOR_CLASS = ()
def __init__(self, options, options_class_name): def __init__(self, options, options_class_name):
"""Initialize the descriptor given its options message and the name of the """Initialize the descriptor given its options message and the name of the
class of the options message. The name of the class is required in case class of the options message. The name of the class is required in case
@ -235,13 +256,25 @@ class Descriptor(_NestedDescriptorBase):
file: (FileDescriptor) Reference to file descriptor. file: (FileDescriptor) Reference to file descriptor.
""" """
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.Descriptor
def __new__(cls, name, full_name, filename, containing_type, fields,
nested_types, enum_types, extensions, options=None,
is_extendable=True, extension_ranges=None, oneofs=None,
file=None, serialized_start=None, serialized_end=None,
syntax=None):
_message.Message._CheckCalledFromGeneratedFile()
return _message.Message._GetMessageDescriptor(full_name)
# NOTE(tmarek): The file argument redefining a builtin is nothing we can # NOTE(tmarek): The file argument redefining a builtin is nothing we can
# fix right now since we don't know how many clients already rely on the # fix right now since we don't know how many clients already rely on the
# name of the argument. # name of the argument.
def __init__(self, name, full_name, filename, containing_type, fields, def __init__(self, name, full_name, filename, containing_type, fields,
nested_types, enum_types, extensions, options=None, nested_types, enum_types, extensions, options=None,
is_extendable=True, extension_ranges=None, oneofs=None, is_extendable=True, extension_ranges=None, oneofs=None,
file=None, serialized_start=None, serialized_end=None): # pylint:disable=redefined-builtin file=None, serialized_start=None, serialized_end=None,
syntax=None): # pylint:disable=redefined-builtin
"""Arguments to __init__() are as described in the description """Arguments to __init__() are as described in the description
of Descriptor fields above. of Descriptor fields above.
@ -286,6 +319,7 @@ class Descriptor(_NestedDescriptorBase):
self.oneofs_by_name = dict((o.name, o) for o in self.oneofs) self.oneofs_by_name = dict((o.name, o) for o in self.oneofs)
for oneof in self.oneofs: for oneof in self.oneofs:
oneof.containing_type = self oneof.containing_type = self
self.syntax = syntax or "proto2"
def EnumValueName(self, enum, value): def EnumValueName(self, enum, value):
"""Returns the string name of an enum value. """Returns the string name of an enum value.
@ -452,6 +486,19 @@ class FieldDescriptor(DescriptorBase):
FIRST_RESERVED_FIELD_NUMBER = 19000 FIRST_RESERVED_FIELD_NUMBER = 19000
LAST_RESERVED_FIELD_NUMBER = 19999 LAST_RESERVED_FIELD_NUMBER = 19999
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.FieldDescriptor
def __new__(cls, name, full_name, index, number, type, cpp_type, label,
default_value, message_type, enum_type, containing_type,
is_extension, extension_scope, options=None,
has_default_value=True, containing_oneof=None):
_message.Message._CheckCalledFromGeneratedFile()
if is_extension:
return _message.Message._GetExtensionDescriptor(full_name)
else:
return _message.Message._GetFieldDescriptor(full_name)
def __init__(self, name, full_name, index, number, type, cpp_type, label, def __init__(self, name, full_name, index, number, type, cpp_type, label,
default_value, message_type, enum_type, containing_type, default_value, message_type, enum_type, containing_type,
is_extension, extension_scope, options=None, is_extension, extension_scope, options=None,
@ -481,20 +528,14 @@ class FieldDescriptor(DescriptorBase):
self.containing_oneof = containing_oneof self.containing_oneof = containing_oneof
if api_implementation.Type() == 'cpp': if api_implementation.Type() == 'cpp':
if is_extension: if is_extension:
if api_implementation.Version() == 2: # pylint: disable=protected-access
# pylint: disable=protected-access self._cdescriptor = (
self._cdescriptor = ( _message.Message._GetExtensionDescriptor(full_name))
_message.Message._GetExtensionDescriptor(full_name)) # pylint: enable=protected-access
# pylint: enable=protected-access
else:
self._cdescriptor = cpp_message.GetExtensionDescriptor(full_name)
else: else:
if api_implementation.Version() == 2: # pylint: disable=protected-access
# pylint: disable=protected-access self._cdescriptor = _message.Message._GetFieldDescriptor(full_name)
self._cdescriptor = _message.Message._GetFieldDescriptor(full_name) # pylint: enable=protected-access
# pylint: enable=protected-access
else:
self._cdescriptor = cpp_message.GetFieldDescriptor(full_name)
else: else:
self._cdescriptor = None self._cdescriptor = None
@ -544,6 +585,15 @@ class EnumDescriptor(_NestedDescriptorBase):
None to use default enum options. None to use default enum options.
""" """
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.EnumDescriptor
def __new__(cls, name, full_name, filename, values,
containing_type=None, options=None, file=None,
serialized_start=None, serialized_end=None):
_message.Message._CheckCalledFromGeneratedFile()
return _message.Message._GetEnumDescriptor(full_name)
def __init__(self, name, full_name, filename, values, def __init__(self, name, full_name, filename, values,
containing_type=None, options=None, file=None, containing_type=None, options=None, file=None,
serialized_start=None, serialized_end=None): serialized_start=None, serialized_end=None):
@ -588,6 +638,17 @@ class EnumValueDescriptor(DescriptorBase):
None to use default enum value options options. None to use default enum value options options.
""" """
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.EnumValueDescriptor
def __new__(cls, name, index, number, type=None, options=None):
_message.Message._CheckCalledFromGeneratedFile()
# There is no way we can build a complete EnumValueDescriptor with the
# given parameters (the name of the Enum is not known, for example).
# Fortunately generated files just pass it to the EnumDescriptor()
# constructor, which will ignore it, so returning None is good enough.
return None
def __init__(self, name, index, number, type=None, options=None): def __init__(self, name, index, number, type=None, options=None):
"""Arguments are as described in the attribute description above.""" """Arguments are as described in the attribute description above."""
super(EnumValueDescriptor, self).__init__(options, 'EnumValueOptions') super(EnumValueDescriptor, self).__init__(options, 'EnumValueOptions')
@ -611,6 +672,13 @@ class OneofDescriptor(object):
oneof can contain. oneof can contain.
""" """
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.OneofDescriptor
def __new__(cls, name, full_name, index, containing_type, fields):
_message.Message._CheckCalledFromGeneratedFile()
return _message.Message._GetOneofDescriptor(full_name)
def __init__(self, name, full_name, index, containing_type, fields): def __init__(self, name, full_name, index, containing_type, fields):
"""Arguments are as described in the attribute description above.""" """Arguments are as described in the attribute description above."""
self.name = name self.name = name
@ -704,6 +772,7 @@ class FileDescriptor(DescriptorBase):
name: name of file, relative to root of source tree. name: name of file, relative to root of source tree.
package: name of the package package: name of the package
syntax: string indicating syntax of the file (can be "proto2" or "proto3")
serialized_pb: (str) Byte string of serialized serialized_pb: (str) Byte string of serialized
descriptor_pb2.FileDescriptorProto. descriptor_pb2.FileDescriptorProto.
dependencies: List of other FileDescriptors this FileDescriptor depends on. dependencies: List of other FileDescriptors this FileDescriptor depends on.
@ -712,14 +781,31 @@ class FileDescriptor(DescriptorBase):
extensions_by_name: Dict of extension names and their descriptors. extensions_by_name: Dict of extension names and their descriptors.
""" """
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.FileDescriptor
def __new__(cls, name, package, options=None, serialized_pb=None,
dependencies=None, syntax=None):
# FileDescriptor() is called from various places, not only from generated
# files, to register dynamic proto files and messages.
# TODO(amauryfa): Expose BuildFile() as a public function and make this
# constructor an implementation detail.
if serialized_pb:
# pylint: disable=protected-access2
return _message.Message._BuildFile(serialized_pb)
# pylint: enable=protected-access
else:
return super(FileDescriptor, cls).__new__(cls)
def __init__(self, name, package, options=None, serialized_pb=None, def __init__(self, name, package, options=None, serialized_pb=None,
dependencies=None): dependencies=None, syntax=None):
"""Constructor.""" """Constructor."""
super(FileDescriptor, self).__init__(options, 'FileOptions') super(FileDescriptor, self).__init__(options, 'FileOptions')
self.message_types_by_name = {} self.message_types_by_name = {}
self.name = name self.name = name
self.package = package self.package = package
self.syntax = syntax or "proto2"
self.serialized_pb = serialized_pb self.serialized_pb = serialized_pb
self.enum_types_by_name = {} self.enum_types_by_name = {}
@ -728,12 +814,9 @@ class FileDescriptor(DescriptorBase):
if (api_implementation.Type() == 'cpp' and if (api_implementation.Type() == 'cpp' and
self.serialized_pb is not None): self.serialized_pb is not None):
if api_implementation.Version() == 2: # pylint: disable=protected-access
# pylint: disable=protected-access _message.Message._BuildFile(self.serialized_pb)
_message.Message._BuildFile(self.serialized_pb) # pylint: enable=protected-access
# pylint: enable=protected-access
else:
cpp_message.BuildFile(self.serialized_pb)
def CopyToProto(self, proto): def CopyToProto(self, proto):
"""Copies this to a descriptor_pb2.FileDescriptorProto. """Copies this to a descriptor_pb2.FileDescriptorProto.
@ -754,7 +837,8 @@ def _ParseOptions(message, string):
return message return message
def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True): def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
syntax=None):
"""Make a protobuf Descriptor given a DescriptorProto protobuf. """Make a protobuf Descriptor given a DescriptorProto protobuf.
Handles nested descriptors. Note that this is limited to the scope of defining Handles nested descriptors. Note that this is limited to the scope of defining
@ -766,6 +850,8 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
package: Optional package name for the new message Descriptor (string). package: Optional package name for the new message Descriptor (string).
build_file_if_cpp: Update the C++ descriptor pool if api matches. build_file_if_cpp: Update the C++ descriptor pool if api matches.
Set to False on recursion, so no duplicates are created. Set to False on recursion, so no duplicates are created.
syntax: The syntax/semantics that should be used. Set to "proto3" to get
proto3 field presence semantics.
Returns: Returns:
A Descriptor for protobuf messages. A Descriptor for protobuf messages.
""" """
@ -791,12 +877,13 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
else: else:
file_descriptor_proto.name = proto_name + '.proto' file_descriptor_proto.name = proto_name + '.proto'
if api_implementation.Version() == 2: # pylint: disable=protected-access
# pylint: disable=protected-access result = _message.Message._BuildFile(
_message.Message._BuildFile(file_descriptor_proto.SerializeToString()) file_descriptor_proto.SerializeToString())
# pylint: enable=protected-access # pylint: enable=protected-access
else:
cpp_message.BuildFile(file_descriptor_proto.SerializeToString()) if _USE_C_DESCRIPTORS:
return result.message_types_by_name[desc_proto.name]
full_message_name = [desc_proto.name] full_message_name = [desc_proto.name]
if package: full_message_name.insert(0, package) if package: full_message_name.insert(0, package)
@ -819,7 +906,8 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
# used by fields in the message, so no loops are possible here. # used by fields in the message, so no loops are possible here.
nested_desc = MakeDescriptor(nested_proto, nested_desc = MakeDescriptor(nested_proto,
package='.'.join(full_message_name), package='.'.join(full_message_name),
build_file_if_cpp=False) build_file_if_cpp=False,
syntax=syntax)
nested_types[full_name] = nested_desc nested_types[full_name] = nested_desc
fields = [] fields = []

114
python/google/protobuf/descriptor_pool.py
View file

@ -64,6 +64,9 @@ from google.protobuf import descriptor_database
from google.protobuf import text_encoding from google.protobuf import text_encoding
_USE_C_DESCRIPTORS = descriptor._USE_C_DESCRIPTORS
def _NormalizeFullyQualifiedName(name): def _NormalizeFullyQualifiedName(name):
"""Remove leading period from fully-qualified type name. """Remove leading period from fully-qualified type name.
@ -271,58 +274,81 @@ class DescriptorPool(object):
file_descriptor = descriptor.FileDescriptor( file_descriptor = descriptor.FileDescriptor(
name=file_proto.name, name=file_proto.name,
package=file_proto.package, package=file_proto.package,
syntax=file_proto.syntax,
options=file_proto.options, options=file_proto.options,
serialized_pb=file_proto.SerializeToString(), serialized_pb=file_proto.SerializeToString(),
dependencies=direct_deps) dependencies=direct_deps)
scope = {} if _USE_C_DESCRIPTORS:
# When using C++ descriptors, all objects defined in the file were added
# This loop extracts all the message and enum types from all the # to the C++ database when the FileDescriptor was built above.
# dependencoes of the file_proto. This is necessary to create the # Just add them to this descriptor pool.
# scope of available message types when defining the passed in def _AddMessageDescriptor(message_desc):
# file proto. self._descriptors[message_desc.full_name] = message_desc
for dependency in built_deps: for nested in message_desc.nested_types:
scope.update(self._ExtractSymbols( _AddMessageDescriptor(nested)
dependency.message_types_by_name.values())) for enum_type in message_desc.enum_types:
scope.update((_PrefixWithDot(enum.full_name), enum) _AddEnumDescriptor(enum_type)
for enum in dependency.enum_types_by_name.values()) def _AddEnumDescriptor(enum_desc):
self._enum_descriptors[enum_desc.full_name] = enum_desc
for message_type in file_proto.message_type: for message_type in file_descriptor.message_types_by_name.values():
message_desc = self._ConvertMessageDescriptor( _AddMessageDescriptor(message_type)
message_type, file_proto.package, file_descriptor, scope) for enum_type in file_descriptor.enum_types_by_name.values():
file_descriptor.message_types_by_name[message_desc.name] = message_desc _AddEnumDescriptor(enum_type)
for enum_type in file_proto.enum_type:
file_descriptor.enum_types_by_name[enum_type.name] = (
self._ConvertEnumDescriptor(enum_type, file_proto.package,
file_descriptor, None, scope))
for index, extension_proto in enumerate(file_proto.extension):
extension_desc = self.MakeFieldDescriptor(
extension_proto, file_proto.package, index, is_extension=True)
extension_desc.containing_type = self._GetTypeFromScope(
file_descriptor.package, extension_proto.extendee, scope)
self.SetFieldType(extension_proto, extension_desc,
file_descriptor.package, scope)
file_descriptor.extensions_by_name[extension_desc.name] = extension_desc
for desc_proto in file_proto.message_type:
self.SetAllFieldTypes(file_proto.package, desc_proto, scope)
if file_proto.package:
desc_proto_prefix = _PrefixWithDot(file_proto.package)
else: else:
desc_proto_prefix = '' scope = {}
# This loop extracts all the message and enum types from all the
# dependencies of the file_proto. This is necessary to create the
# scope of available message types when defining the passed in
# file proto.
for dependency in built_deps:
scope.update(self._ExtractSymbols(
dependency.message_types_by_name.values()))
scope.update((_PrefixWithDot(enum.full_name), enum)
for enum in dependency.enum_types_by_name.values())
for message_type in file_proto.message_type:
message_desc = self._ConvertMessageDescriptor(
message_type, file_proto.package, file_descriptor, scope,
file_proto.syntax)
file_descriptor.message_types_by_name[message_desc.name] = (
message_desc)
for enum_type in file_proto.enum_type:
file_descriptor.enum_types_by_name[enum_type.name] = (
self._ConvertEnumDescriptor(enum_type, file_proto.package,
file_descriptor, None, scope))
for index, extension_proto in enumerate(file_proto.extension):
extension_desc = self.MakeFieldDescriptor(
extension_proto, file_proto.package, index, is_extension=True)
extension_desc.containing_type = self._GetTypeFromScope(
file_descriptor.package, extension_proto.extendee, scope)
self.SetFieldType(extension_proto, extension_desc,
file_descriptor.package, scope)
file_descriptor.extensions_by_name[extension_desc.name] = (
extension_desc)
for desc_proto in file_proto.message_type:
self.SetAllFieldTypes(file_proto.package, desc_proto, scope)
if file_proto.package:
desc_proto_prefix = _PrefixWithDot(file_proto.package)
else:
desc_proto_prefix = ''
for desc_proto in file_proto.message_type:
desc = self._GetTypeFromScope(
desc_proto_prefix, desc_proto.name, scope)
file_descriptor.message_types_by_name[desc_proto.name] = desc
for desc_proto in file_proto.message_type:
desc = self._GetTypeFromScope(desc_proto_prefix, desc_proto.name, scope)
file_descriptor.message_types_by_name[desc_proto.name] = desc
self.Add(file_proto) self.Add(file_proto)
self._file_descriptors[file_proto.name] = file_descriptor self._file_descriptors[file_proto.name] = file_descriptor
return self._file_descriptors[file_proto.name] return self._file_descriptors[file_proto.name]
def _ConvertMessageDescriptor(self, desc_proto, package=None, file_desc=None, def _ConvertMessageDescriptor(self, desc_proto, package=None, file_desc=None,
scope=None): scope=None, syntax=None):
"""Adds the proto to the pool in the specified package. """Adds the proto to the pool in the specified package.
Args: Args:
@ -349,7 +375,8 @@ class DescriptorPool(object):
scope = {} scope = {}
nested = [ nested = [
self._ConvertMessageDescriptor(nested, desc_name, file_desc, scope) self._ConvertMessageDescriptor(
nested, desc_name, file_desc, scope, syntax)
for nested in desc_proto.nested_type] for nested in desc_proto.nested_type]
enums = [ enums = [
self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, scope) self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, scope)
@ -383,7 +410,8 @@ class DescriptorPool(object):
extension_ranges=extension_ranges, extension_ranges=extension_ranges,
file=file_desc, file=file_desc,
serialized_start=None, serialized_start=None,
serialized_end=None) serialized_end=None,
syntax=syntax)
for nested in desc.nested_types: for nested in desc.nested_types:
nested.containing_type = desc nested.containing_type = desc
for enum in desc.enum_types: for enum in desc.enum_types:

14
python/google/protobuf/internal/api_implementation.cc
View file

@ -50,10 +50,7 @@ namespace python {
// and // and
// PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2 // PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2
#ifdef PYTHON_PROTO2_CPP_IMPL_V1 #ifdef PYTHON_PROTO2_CPP_IMPL_V1
#if PY_MAJOR_VERSION >= 3 #error "PYTHON_PROTO2_CPP_IMPL_V1 is no longer supported."
#error "PYTHON_PROTO2_CPP_IMPL_V1 is not supported under Python 3."
#endif
static int kImplVersion = 1;
#else #else
#ifdef PYTHON_PROTO2_CPP_IMPL_V2 #ifdef PYTHON_PROTO2_CPP_IMPL_V2
static int kImplVersion = 2; static int kImplVersion = 2;
@ -62,14 +59,7 @@ static int kImplVersion = 2;
static int kImplVersion = 0; static int kImplVersion = 0;
#else #else
// The defaults are set here. Python 3 uses the fast C++ APIv2 by default. static int kImplVersion = -1; // -1 means "Unspecified by compiler flags".
// Python 2 still uses the Python version by default until some compatibility
// issues can be worked around.
#if PY_MAJOR_VERSION >= 3
static int kImplVersion = 2;
#else
static int kImplVersion = 0;
#endif
#endif // PYTHON_PROTO2_PYTHON_IMPL #endif // PYTHON_PROTO2_PYTHON_IMPL
#endif // PYTHON_PROTO2_CPP_IMPL_V2 #endif // PYTHON_PROTO2_CPP_IMPL_V2

38
python/google/protobuf/internal/api_implementation.py
View file

@ -40,14 +40,33 @@ try:
# The compile-time constants in the _api_implementation module can be used to # The compile-time constants in the _api_implementation module can be used to
# switch to a certain implementation of the Python API at build time. # switch to a certain implementation of the Python API at build time.
_api_version = _api_implementation.api_version _api_version = _api_implementation.api_version
del _api_implementation _proto_extension_modules_exist_in_build = True
except ImportError: except ImportError:
_api_version = 0 _api_version = -1 # Unspecified by compiler flags.
_proto_extension_modules_exist_in_build = False
if _api_version == 1:
raise ValueError('api_version=1 is no longer supported.')
if _api_version < 0: # Still unspecified?
try:
# The presence of this module in a build allows the proto implementation to
# be upgraded merely via build deps rather than a compiler flag or the
# runtime environment variable.
# pylint: disable=g-import-not-at-top
from google.protobuf import _use_fast_cpp_protos
# Work around a known issue in the classic bootstrap .par import hook.
if not _use_fast_cpp_protos:
raise ImportError('_use_fast_cpp_protos import succeeded but was None')
del _use_fast_cpp_protos
_api_version = 2
except ImportError:
if _proto_extension_modules_exist_in_build:
if sys.version_info[0] >= 3: # Python 3 defaults to C++ impl v2.
_api_version = 2
# TODO(b/17427486): Make Python 2 default to C++ impl v2.
_default_implementation_type = ( _default_implementation_type = (
'python' if _api_version == 0 else 'cpp') 'python' if _api_version <= 0 else 'cpp')
_default_version_str = (
'1' if _api_version <= 1 else '2')
# This environment variable can be used to switch to a certain implementation # This environment variable can be used to switch to a certain implementation
# of the Python API, overriding the compile-time constants in the # of the Python API, overriding the compile-time constants in the
@ -64,13 +83,12 @@ if _implementation_type != 'python':
# _api_implementation module. Right now only 1 and 2 are valid values. Any other # _api_implementation module. Right now only 1 and 2 are valid values. Any other
# value will be ignored. # value will be ignored.
_implementation_version_str = os.getenv( _implementation_version_str = os.getenv(
'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', '2')
_default_version_str)
if _implementation_version_str not in ('1', '2'): if _implementation_version_str != '2':
raise ValueError( raise ValueError(
"unsupported PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION: '" + 'unsupported PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION: "' +
_implementation_version_str + "' (supported versions: 1, 2)" _implementation_version_str + '" (supported versions: 2)'
) )
_implementation_version = int(_implementation_version_str) _implementation_version = int(_implementation_version_str)

63
python/google/protobuf/internal/api_implementation_default_test.py
View file

@ -1,63 +0,0 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Test that the api_implementation defaults are what we expect."""
import os
import sys
# Clear environment implementation settings before the google3 imports.
os.environ.pop('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', None)
os.environ.pop('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', None)
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
class ApiImplementationDefaultTest(basetest.TestCase):
if sys.version_info.major <= 2:
def testThatPythonIsTheDefault(self):
"""If -DPYTHON_PROTO_*IMPL* was given at build time, this may fail."""
self.assertEqual('python', api_implementation.Type())
else:
def testThatCppApiV2IsTheDefault(self):
"""If -DPYTHON_PROTO_*IMPL* was given at build time, this may fail."""
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
if __name__ == '__main__':
basetest.main()

37
python/google/protobuf/internal/containers.py
View file

@ -41,7 +41,6 @@ are:
__author__ = 'petar@google.com (Petar Petrov)' __author__ = 'petar@google.com (Petar Petrov)'
class BaseContainer(object): class BaseContainer(object):
"""Base container class.""" """Base container class."""
@ -119,15 +118,23 @@ class RepeatedScalarFieldContainer(BaseContainer):
self._message_listener.Modified() self._message_listener.Modified()
def extend(self, elem_seq): def extend(self, elem_seq):
"""Extends by appending the given sequence. Similar to list.extend().""" """Extends by appending the given iterable. Similar to list.extend()."""
if not elem_seq:
return
new_values = [] if elem_seq is None:
for elem in elem_seq: return
new_values.append(self._type_checker.CheckValue(elem)) try:
self._values.extend(new_values) elem_seq_iter = iter(elem_seq)
self._message_listener.Modified() except TypeError:
if not elem_seq:
# silently ignore falsy inputs :-/.
# TODO(ptucker): Deprecate this behavior. b/18413862
return
raise
new_values = [self._type_checker.CheckValue(elem) for elem in elem_seq_iter]
if new_values:
self._values.extend(new_values)
self._message_listener.Modified()
def MergeFrom(self, other): def MergeFrom(self, other):
"""Appends the contents of another repeated field of the same type to this """Appends the contents of another repeated field of the same type to this
@ -141,6 +148,12 @@ class RepeatedScalarFieldContainer(BaseContainer):
self._values.remove(elem) self._values.remove(elem)
self._message_listener.Modified() self._message_listener.Modified()
def pop(self, key=-1):
"""Removes and returns an item at a given index. Similar to list.pop()."""
value = self._values[key]
self.__delitem__(key)
return value
def __setitem__(self, key, value): def __setitem__(self, key, value):
"""Sets the item on the specified position.""" """Sets the item on the specified position."""
if isinstance(key, slice): # PY3 if isinstance(key, slice): # PY3
@ -245,6 +258,12 @@ class RepeatedCompositeFieldContainer(BaseContainer):
self._values.remove(elem) self._values.remove(elem)
self._message_listener.Modified() self._message_listener.Modified()
def pop(self, key=-1):
"""Removes and returns an item at a given index. Similar to list.pop()."""
value = self._values[key]
self.__delitem__(key)
return value
def __getslice__(self, start, stop): def __getslice__(self, start, stop):
"""Retrieves the subset of items from between the specified indices.""" """Retrieves the subset of items from between the specified indices."""
return self._values[start:stop] return self._values[start:stop]

3
python/google/protobuf/internal/decoder.py
View file

@ -621,9 +621,6 @@ def MessageDecoder(field_number, is_repeated, is_packed, key, new_default):
if value is None: if value is None:
value = field_dict.setdefault(key, new_default(message)) value = field_dict.setdefault(key, new_default(message))
while 1: while 1:
value = field_dict.get(key)
if value is None:
value = field_dict.setdefault(key, new_default(message))
# Read length. # Read length.
(size, pos) = local_DecodeVarint(buffer, pos) (size, pos) = local_DecodeVarint(buffer, pos)
new_pos = pos + size new_pos = pos + size

176
python/google/protobuf/internal/descriptor_test.py
View file

@ -34,12 +34,16 @@
__author__ = 'robinson@google.com (Will Robinson)' __author__ = 'robinson@google.com (Will Robinson)'
import sys
from google.apputils import basetest from google.apputils import basetest
from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_custom_options_pb2
from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_pb2 from google.protobuf import unittest_pb2
from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pb2
from google.protobuf.internal import api_implementation
from google.protobuf import descriptor from google.protobuf import descriptor
from google.protobuf import symbol_database
from google.protobuf import text_format from google.protobuf import text_format
@ -51,41 +55,28 @@ name: 'TestEmptyMessage'
class DescriptorTest(basetest.TestCase): class DescriptorTest(basetest.TestCase):
def setUp(self): def setUp(self):
self.my_file = descriptor.FileDescriptor( file_proto = descriptor_pb2.FileDescriptorProto(
name='some/filename/some.proto', name='some/filename/some.proto',
package='protobuf_unittest' package='protobuf_unittest')
) message_proto = file_proto.message_type.add(
self.my_enum = descriptor.EnumDescriptor( name='NestedMessage')
name='ForeignEnum', message_proto.field.add(
full_name='protobuf_unittest.ForeignEnum', name='bb',
filename=None, number=1,
file=self.my_file, type=descriptor_pb2.FieldDescriptorProto.TYPE_INT32,
values=[ label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL)
descriptor.EnumValueDescriptor(name='FOREIGN_FOO', index=0, number=4), enum_proto = message_proto.enum_type.add(
descriptor.EnumValueDescriptor(name='FOREIGN_BAR', index=1, number=5), name='ForeignEnum')
descriptor.EnumValueDescriptor(name='FOREIGN_BAZ', index=2, number=6), enum_proto.value.add(name='FOREIGN_FOO', number=4)
]) enum_proto.value.add(name='FOREIGN_BAR', number=5)
self.my_message = descriptor.Descriptor( enum_proto.value.add(name='FOREIGN_BAZ', number=6)
name='NestedMessage',
full_name='protobuf_unittest.TestAllTypes.NestedMessage', descriptor_pool = symbol_database.Default().pool
filename=None, descriptor_pool.Add(file_proto)
file=self.my_file, self.my_file = descriptor_pool.FindFileByName(file_proto.name)
containing_type=None, self.my_message = self.my_file.message_types_by_name[message_proto.name]
fields=[ self.my_enum = self.my_message.enum_types_by_name[enum_proto.name]
descriptor.FieldDescriptor(
name='bb',
full_name='protobuf_unittest.TestAllTypes.NestedMessage.bb',
index=0, number=1,
type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None),
],
nested_types=[],
enum_types=[
self.my_enum,
],
extensions=[])
self.my_method = descriptor.MethodDescriptor( self.my_method = descriptor.MethodDescriptor(
name='Bar', name='Bar',
full_name='protobuf_unittest.TestService.Bar', full_name='protobuf_unittest.TestService.Bar',
@ -173,6 +164,11 @@ class DescriptorTest(basetest.TestCase):
self.assertEqual(unittest_custom_options_pb2.METHODOPT1_VAL2, self.assertEqual(unittest_custom_options_pb2.METHODOPT1_VAL2,
method_options.Extensions[method_opt1]) method_options.Extensions[method_opt1])
message_descriptor = (
unittest_custom_options_pb2.DummyMessageContainingEnum.DESCRIPTOR)
self.assertTrue(file_descriptor.has_options)
self.assertFalse(message_descriptor.has_options)
def testDifferentCustomOptionTypes(self): def testDifferentCustomOptionTypes(self):
kint32min = -2**31 kint32min = -2**31
kint64min = -2**63 kint64min = -2**63
@ -394,6 +390,108 @@ class DescriptorTest(basetest.TestCase):
self.assertEqual(self.my_file.name, 'some/filename/some.proto') self.assertEqual(self.my_file.name, 'some/filename/some.proto')
self.assertEqual(self.my_file.package, 'protobuf_unittest') self.assertEqual(self.my_file.package, 'protobuf_unittest')
@basetest.unittest.skipIf(
api_implementation.Type() != 'cpp' or api_implementation.Version() != 2,
'Immutability of descriptors is only enforced in v2 implementation')
def testImmutableCppDescriptor(self):
message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
with self.assertRaises(AttributeError):
message_descriptor.fields_by_name = None
with self.assertRaises(TypeError):
message_descriptor.fields_by_name['Another'] = None
with self.assertRaises(TypeError):
message_descriptor.fields.append(None)
class GeneratedDescriptorTest(basetest.TestCase):
"""Tests for the properties of descriptors in generated code."""
def CheckMessageDescriptor(self, message_descriptor):
# Basic properties
self.assertEqual(message_descriptor.name, 'TestAllTypes')
self.assertEqual(message_descriptor.full_name,
'protobuf_unittest.TestAllTypes')
# Test equality and hashability
self.assertEqual(message_descriptor, message_descriptor)
self.assertEqual(message_descriptor.fields[0].containing_type,
message_descriptor)
self.assertIn(message_descriptor, [message_descriptor])
self.assertIn(message_descriptor, {message_descriptor: None})
# Test field containers
self.CheckDescriptorSequence(message_descriptor.fields)
self.CheckDescriptorMapping(message_descriptor.fields_by_name)
self.CheckDescriptorMapping(message_descriptor.fields_by_number)
def CheckFieldDescriptor(self, field_descriptor):
# Basic properties
self.assertEqual(field_descriptor.name, 'optional_int32')
self.assertEqual(field_descriptor.full_name,
'protobuf_unittest.TestAllTypes.optional_int32')
self.assertEqual(field_descriptor.containing_type.name, 'TestAllTypes')
# Test equality and hashability
self.assertEqual(field_descriptor, field_descriptor)
self.assertEqual(
field_descriptor.containing_type.fields_by_name['optional_int32'],
field_descriptor)
self.assertIn(field_descriptor, [field_descriptor])
self.assertIn(field_descriptor, {field_descriptor: None})
def CheckDescriptorSequence(self, sequence):
# Verifies that a property like 'messageDescriptor.fields' has all the
# properties of an immutable abc.Sequence.
self.assertGreater(len(sequence), 0) # Sized
self.assertEqual(len(sequence), len(list(sequence))) # Iterable
item = sequence[0]
self.assertEqual(item, sequence[0])
self.assertIn(item, sequence) # Container
self.assertEqual(sequence.index(item), 0)
self.assertEqual(sequence.count(item), 1)
reversed_iterator = reversed(sequence)
self.assertEqual(list(reversed_iterator), list(sequence)[::-1])
self.assertRaises(StopIteration, next, reversed_iterator)
def CheckDescriptorMapping(self, mapping):
# Verifies that a property like 'messageDescriptor.fields' has all the
# properties of an immutable abc.Mapping.
self.assertGreater(len(mapping), 0) # Sized
self.assertEqual(len(mapping), len(list(mapping))) # Iterable
if sys.version_info.major >= 3:
key, item = next(iter(mapping.items()))
else:
key, item = mapping.items()[0]
self.assertIn(key, mapping) # Container
self.assertEqual(mapping.get(key), item)
# keys(), iterkeys() &co
item = (next(iter(mapping.keys())), next(iter(mapping.values())))
self.assertEqual(item, next(iter(mapping.items())))
if sys.version_info.major < 3:
def CheckItems(seq, iterator):
self.assertEqual(next(iterator), seq[0])
self.assertEqual(list(iterator), seq[1:])
CheckItems(mapping.keys(), mapping.iterkeys())
CheckItems(mapping.values(), mapping.itervalues())
CheckItems(mapping.items(), mapping.iteritems())
def testDescriptor(self):
message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
self.CheckMessageDescriptor(message_descriptor)
field_descriptor = message_descriptor.fields_by_name['optional_int32']
self.CheckFieldDescriptor(field_descriptor)
def testCppDescriptorContainer(self):
# Check that the collection is still valid even if the parent disappeared.
enum = unittest_pb2.TestAllTypes.DESCRIPTOR.enum_types_by_name['NestedEnum']
values = enum.values
del enum
self.assertEqual('FOO', values[0].name)
def testCppDescriptorContainer_Iterator(self):
# Same test with the iterator
enum = unittest_pb2.TestAllTypes.DESCRIPTOR.enum_types_by_name['NestedEnum']
values_iter = iter(enum.values)
del enum
self.assertEqual('FOO', next(values_iter).name)
class DescriptorCopyToProtoTest(basetest.TestCase): class DescriptorCopyToProtoTest(basetest.TestCase):
"""Tests for CopyTo functions of Descriptor.""" """Tests for CopyTo functions of Descriptor."""
@ -588,10 +686,12 @@ class DescriptorCopyToProtoTest(basetest.TestCase):
output_type: '.protobuf_unittest.BarResponse' output_type: '.protobuf_unittest.BarResponse'
> >
""" """
self._InternalTestCopyToProto( # TODO(rocking): enable this test after the proto descriptor change is
unittest_pb2.TestService.DESCRIPTOR, # checked in.
descriptor_pb2.ServiceDescriptorProto, #self._InternalTestCopyToProto(
TEST_SERVICE_ASCII) # unittest_pb2.TestService.DESCRIPTOR,
# descriptor_pb2.ServiceDescriptorProto,
# TEST_SERVICE_ASCII)
class MakeDescriptorTest(basetest.TestCase): class MakeDescriptorTest(basetest.TestCase):

832
python/google/protobuf/internal/message_test.py
View file

File diff suppressed because it is too large Load diff

81
python/google/protobuf/internal/python_message.py
View file

@ -219,12 +219,20 @@ def _AttachFieldHelpers(cls, field_descriptor):
def AddDecoder(wiretype, is_packed): def AddDecoder(wiretype, is_packed):
tag_bytes = encoder.TagBytes(field_descriptor.number, wiretype) tag_bytes = encoder.TagBytes(field_descriptor.number, wiretype)
cls._decoders_by_tag[tag_bytes] = ( decode_type = field_descriptor.type
type_checkers.TYPE_TO_DECODER[field_descriptor.type]( if (decode_type == _FieldDescriptor.TYPE_ENUM and
field_descriptor.number, is_repeated, is_packed, type_checkers.SupportsOpenEnums(field_descriptor)):
field_descriptor, field_descriptor._default_constructor), decode_type = _FieldDescriptor.TYPE_INT32
field_descriptor if field_descriptor.containing_oneof is not None
else None) oneof_descriptor = None
if field_descriptor.containing_oneof is not None:
oneof_descriptor = field_descriptor
field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
field_descriptor.number, is_repeated, is_packed,
field_descriptor, field_descriptor._default_constructor)
cls._decoders_by_tag[tag_bytes] = (field_decoder, oneof_descriptor)
AddDecoder(type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type], AddDecoder(type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type],
False) False)
@ -296,6 +304,8 @@ def _DefaultValueConstructorForField(field):
def MakeSubMessageDefault(message): def MakeSubMessageDefault(message):
result = message_type._concrete_class() result = message_type._concrete_class()
result._SetListener(message._listener_for_children) result._SetListener(message._listener_for_children)
if field.containing_oneof:
message._UpdateOneofState(field)
return result return result
return MakeSubMessageDefault return MakeSubMessageDefault
@ -476,6 +486,7 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
type_checker = type_checkers.GetTypeChecker(field) type_checker = type_checkers.GetTypeChecker(field)
default_value = field.default_value default_value = field.default_value
valid_values = set() valid_values = set()
is_proto3 = field.containing_type.syntax == "proto3"
def getter(self): def getter(self):
# TODO(protobuf-team): This may be broken since there may not be # TODO(protobuf-team): This may be broken since there may not be
@ -483,15 +494,24 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
return self._fields.get(field, default_value) return self._fields.get(field, default_value)
getter.__module__ = None getter.__module__ = None
getter.__doc__ = 'Getter for %s.' % proto_field_name getter.__doc__ = 'Getter for %s.' % proto_field_name
clear_when_set_to_default = is_proto3 and not field.containing_oneof
def field_setter(self, new_value): def field_setter(self, new_value):
# pylint: disable=protected-access # pylint: disable=protected-access
self._fields[field] = type_checker.CheckValue(new_value) # Testing the value for truthiness captures all of the proto3 defaults
# (0, 0.0, enum 0, and False).
new_value = type_checker.CheckValue(new_value)
if clear_when_set_to_default and not new_value:
self._fields.pop(field, None)
else:
self._fields[field] = new_value
# Check _cached_byte_size_dirty inline to improve performance, since scalar # Check _cached_byte_size_dirty inline to improve performance, since scalar
# setters are called frequently. # setters are called frequently.
if not self._cached_byte_size_dirty: if not self._cached_byte_size_dirty:
self._Modified() self._Modified()
if field.containing_oneof is not None: if field.containing_oneof:
def setter(self, new_value): def setter(self, new_value):
field_setter(self, new_value) field_setter(self, new_value)
self._UpdateOneofState(field) self._UpdateOneofState(field)
@ -624,24 +644,35 @@ def _AddListFieldsMethod(message_descriptor, cls):
cls.ListFields = ListFields cls.ListFields = ListFields
_Proto3HasError = 'Protocol message has no non-repeated submessage field "%s"'
_Proto2HasError = 'Protocol message has no non-repeated field "%s"'
def _AddHasFieldMethod(message_descriptor, cls): def _AddHasFieldMethod(message_descriptor, cls):
"""Helper for _AddMessageMethods().""" """Helper for _AddMessageMethods()."""
singular_fields = {} is_proto3 = (message_descriptor.syntax == "proto3")
error_msg = _Proto3HasError if is_proto3 else _Proto2HasError
hassable_fields = {}
for field in message_descriptor.fields: for field in message_descriptor.fields:
if field.label != _FieldDescriptor.LABEL_REPEATED: if field.label == _FieldDescriptor.LABEL_REPEATED:
singular_fields[field.name] = field continue
# Fields inside oneofs are never repeated (enforced by the compiler). # For proto3, only submessages and fields inside a oneof have presence.
for field in message_descriptor.oneofs: if (is_proto3 and field.cpp_type != _FieldDescriptor.CPPTYPE_MESSAGE and
singular_fields[field.name] = field not field.containing_oneof):
continue
hassable_fields[field.name] = field
if not is_proto3:
# Fields inside oneofs are never repeated (enforced by the compiler).
for oneof in message_descriptor.oneofs:
hassable_fields[oneof.name] = oneof
def HasField(self, field_name): def HasField(self, field_name):
try: try:
field = singular_fields[field_name] field = hassable_fields[field_name]
except KeyError: except KeyError:
raise ValueError( raise ValueError(error_msg % field_name)
'Protocol message has no singular "%s" field.' % field_name)
if isinstance(field, descriptor_mod.OneofDescriptor): if isinstance(field, descriptor_mod.OneofDescriptor):
try: try:
@ -871,6 +902,7 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
local_ReadTag = decoder.ReadTag local_ReadTag = decoder.ReadTag
local_SkipField = decoder.SkipField local_SkipField = decoder.SkipField
decoders_by_tag = cls._decoders_by_tag decoders_by_tag = cls._decoders_by_tag
is_proto3 = message_descriptor.syntax == "proto3"
def InternalParse(self, buffer, pos, end): def InternalParse(self, buffer, pos, end):
self._Modified() self._Modified()
@ -884,9 +916,11 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
new_pos = local_SkipField(buffer, new_pos, end, tag_bytes) new_pos = local_SkipField(buffer, new_pos, end, tag_bytes)
if new_pos == -1: if new_pos == -1:
return pos return pos
if not unknown_field_list: if not is_proto3:
unknown_field_list = self._unknown_fields = [] if not unknown_field_list:
unknown_field_list.append((tag_bytes, buffer[value_start_pos:new_pos])) unknown_field_list = self._unknown_fields = []
unknown_field_list.append(
(tag_bytes, buffer[value_start_pos:new_pos]))
pos = new_pos pos = new_pos
else: else:
pos = field_decoder(buffer, new_pos, end, self, field_dict) pos = field_decoder(buffer, new_pos, end, self, field_dict)
@ -1008,6 +1042,8 @@ def _AddMergeFromMethod(cls):
# Construct a new object to represent this field. # Construct a new object to represent this field.
field_value = field._default_constructor(self) field_value = field._default_constructor(self)
fields[field] = field_value fields[field] = field_value
if field.containing_oneof:
self._UpdateOneofState(field)
field_value.MergeFrom(value) field_value.MergeFrom(value)
else: else:
self._fields[field] = value self._fields[field] = value
@ -1252,11 +1288,10 @@ class _ExtensionDict(object):
# It's slightly wasteful to lookup the type checker each time, # It's slightly wasteful to lookup the type checker each time,
# but we expect this to be a vanishingly uncommon case anyway. # but we expect this to be a vanishingly uncommon case anyway.
type_checker = type_checkers.GetTypeChecker( type_checker = type_checkers.GetTypeChecker(extension_handle)
extension_handle)
# pylint: disable=protected-access # pylint: disable=protected-access
self._extended_message._fields[extension_handle] = ( self._extended_message._fields[extension_handle] = (
type_checker.CheckValue(value)) type_checker.CheckValue(value))
self._extended_message._Modified() self._extended_message._Modified()
def _FindExtensionByName(self, name): def _FindExtensionByName(self, name):

21
python/google/protobuf/internal/reflection_test.py
View file

@ -1792,6 +1792,27 @@ class ReflectionTest(basetest.TestCase):
# Just check the default value. # Just check the default value.
self.assertEqual(57, msg.inner.value) self.assertEqual(57, msg.inner.value)
@basetest.unittest.skipIf(
api_implementation.Type() != 'cpp' or api_implementation.Version() != 2,
'CPPv2-specific test')
def testBadArguments(self):
# Some of these assertions used to segfault.
from google.protobuf.pyext import _message
self.assertRaises(TypeError, _message.Message._GetFieldDescriptor, 3)
self.assertRaises(TypeError, _message.Message._GetExtensionDescriptor, 42)
self.assertRaises(TypeError,
unittest_pb2.TestAllTypes().__getattribute__, 42)
@basetest.unittest.skipIf(
api_implementation.Type() != 'cpp' or api_implementation.Version() != 2,
'CPPv2-specific test')
def testRosyHack(self):
from google.protobuf.pyext import _message
from google3.gdata.rosy.proto import core_api2_pb2
from google3.gdata.rosy.proto import core_pb2
self.assertEqual(_message.Message, core_pb2.PageSelection.__base__)
self.assertEqual(_message.Message, core_api2_pb2.PageSelection.__base__)
# Since we had so many tests for protocol buffer equality, we broke these out # Since we had so many tests for protocol buffer equality, we broke these out
# into separate TestCase classes. # into separate TestCase classes.

160
python/google/protobuf/internal/test_util.py
View file

@ -40,13 +40,19 @@ import os.path
from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_pb2 from google.protobuf import unittest_pb2
from google.protobuf import descriptor_pb2
# Tests whether the given TestAllTypes message is proto2 or not.
# This is used to gate several fields/features that only exist
# for the proto2 version of the message.
def IsProto2(message):
return message.DESCRIPTOR.syntax == "proto2"
def SetAllNonLazyFields(message): def SetAllNonLazyFields(message):
"""Sets every non-lazy field in the message to a unique value. """Sets every non-lazy field in the message to a unique value.
Args: Args:
message: A unittest_pb2.TestAllTypes instance. message: A TestAllTypes instance.
""" """
# #
@ -77,7 +83,8 @@ def SetAllNonLazyFields(message):
message.optional_nested_enum = unittest_pb2.TestAllTypes.BAZ message.optional_nested_enum = unittest_pb2.TestAllTypes.BAZ
message.optional_foreign_enum = unittest_pb2.FOREIGN_BAZ message.optional_foreign_enum = unittest_pb2.FOREIGN_BAZ
message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ if IsProto2(message):
message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ
message.optional_string_piece = u'124' message.optional_string_piece = u'124'
message.optional_cord = u'125' message.optional_cord = u'125'
@ -110,7 +117,8 @@ def SetAllNonLazyFields(message):
message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAR) message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAR)
message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAR) message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAR)
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR) if IsProto2(message):
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR)
message.repeated_string_piece.append(u'224') message.repeated_string_piece.append(u'224')
message.repeated_cord.append(u'225') message.repeated_cord.append(u'225')
@ -140,7 +148,8 @@ def SetAllNonLazyFields(message):
message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAZ) message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAZ)
message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ) message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ)
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ) if IsProto2(message):
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ)
message.repeated_string_piece.append(u'324') message.repeated_string_piece.append(u'324')
message.repeated_cord.append(u'325') message.repeated_cord.append(u'325')
@ -149,28 +158,29 @@ def SetAllNonLazyFields(message):
# Fields that have defaults. # Fields that have defaults.
# #
message.default_int32 = 401 if IsProto2(message):
message.default_int64 = 402 message.default_int32 = 401
message.default_uint32 = 403 message.default_int64 = 402
message.default_uint64 = 404 message.default_uint32 = 403
message.default_sint32 = 405 message.default_uint64 = 404
message.default_sint64 = 406 message.default_sint32 = 405
message.default_fixed32 = 407 message.default_sint64 = 406
message.default_fixed64 = 408 message.default_fixed32 = 407
message.default_sfixed32 = 409 message.default_fixed64 = 408
message.default_sfixed64 = 410 message.default_sfixed32 = 409
message.default_float = 411 message.default_sfixed64 = 410
message.default_double = 412 message.default_float = 411
message.default_bool = False message.default_double = 412
message.default_string = '415' message.default_bool = False
message.default_bytes = b'416' message.default_string = '415'
message.default_bytes = b'416'
message.default_nested_enum = unittest_pb2.TestAllTypes.FOO message.default_nested_enum = unittest_pb2.TestAllTypes.FOO
message.default_foreign_enum = unittest_pb2.FOREIGN_FOO message.default_foreign_enum = unittest_pb2.FOREIGN_FOO
message.default_import_enum = unittest_import_pb2.IMPORT_FOO message.default_import_enum = unittest_import_pb2.IMPORT_FOO
message.default_string_piece = '424' message.default_string_piece = '424'
message.default_cord = '425' message.default_cord = '425'
message.oneof_uint32 = 601 message.oneof_uint32 = 601
message.oneof_nested_message.bb = 602 message.oneof_nested_message.bb = 602
@ -398,7 +408,8 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertTrue(message.HasField('optional_nested_enum')) test_case.assertTrue(message.HasField('optional_nested_enum'))
test_case.assertTrue(message.HasField('optional_foreign_enum')) test_case.assertTrue(message.HasField('optional_foreign_enum'))
test_case.assertTrue(message.HasField('optional_import_enum')) if IsProto2(message):
test_case.assertTrue(message.HasField('optional_import_enum'))
test_case.assertTrue(message.HasField('optional_string_piece')) test_case.assertTrue(message.HasField('optional_string_piece'))
test_case.assertTrue(message.HasField('optional_cord')) test_case.assertTrue(message.HasField('optional_cord'))
@ -430,8 +441,9 @@ def ExpectAllFieldsSet(test_case, message):
message.optional_nested_enum) message.optional_nested_enum)
test_case.assertEqual(unittest_pb2.FOREIGN_BAZ, test_case.assertEqual(unittest_pb2.FOREIGN_BAZ,
message.optional_foreign_enum) message.optional_foreign_enum)
test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ, if IsProto2(message):
message.optional_import_enum) test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
message.optional_import_enum)
# ----------------------------------------------------------------- # -----------------------------------------------------------------
@ -457,7 +469,8 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(2, len(message.repeated_import_message)) test_case.assertEqual(2, len(message.repeated_import_message))
test_case.assertEqual(2, len(message.repeated_nested_enum)) test_case.assertEqual(2, len(message.repeated_nested_enum))
test_case.assertEqual(2, len(message.repeated_foreign_enum)) test_case.assertEqual(2, len(message.repeated_foreign_enum))
test_case.assertEqual(2, len(message.repeated_import_enum)) if IsProto2(message):
test_case.assertEqual(2, len(message.repeated_import_enum))
test_case.assertEqual(2, len(message.repeated_string_piece)) test_case.assertEqual(2, len(message.repeated_string_piece))
test_case.assertEqual(2, len(message.repeated_cord)) test_case.assertEqual(2, len(message.repeated_cord))
@ -488,8 +501,9 @@ def ExpectAllFieldsSet(test_case, message):
message.repeated_nested_enum[0]) message.repeated_nested_enum[0])
test_case.assertEqual(unittest_pb2.FOREIGN_BAR, test_case.assertEqual(unittest_pb2.FOREIGN_BAR,
message.repeated_foreign_enum[0]) message.repeated_foreign_enum[0])
test_case.assertEqual(unittest_import_pb2.IMPORT_BAR, if IsProto2(message):
message.repeated_import_enum[0]) test_case.assertEqual(unittest_import_pb2.IMPORT_BAR,
message.repeated_import_enum[0])
test_case.assertEqual(301, message.repeated_int32[1]) test_case.assertEqual(301, message.repeated_int32[1])
test_case.assertEqual(302, message.repeated_int64[1]) test_case.assertEqual(302, message.repeated_int64[1])
@ -517,53 +531,55 @@ def ExpectAllFieldsSet(test_case, message):
message.repeated_nested_enum[1]) message.repeated_nested_enum[1])
test_case.assertEqual(unittest_pb2.FOREIGN_BAZ, test_case.assertEqual(unittest_pb2.FOREIGN_BAZ,
message.repeated_foreign_enum[1]) message.repeated_foreign_enum[1])
test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ, if IsProto2(message):
message.repeated_import_enum[1]) test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
message.repeated_import_enum[1])
# ----------------------------------------------------------------- # -----------------------------------------------------------------
test_case.assertTrue(message.HasField('default_int32')) if IsProto2(message):
test_case.assertTrue(message.HasField('default_int64')) test_case.assertTrue(message.HasField('default_int32'))
test_case.assertTrue(message.HasField('default_uint32')) test_case.assertTrue(message.HasField('default_int64'))
test_case.assertTrue(message.HasField('default_uint64')) test_case.assertTrue(message.HasField('default_uint32'))
test_case.assertTrue(message.HasField('default_sint32')) test_case.assertTrue(message.HasField('default_uint64'))
test_case.assertTrue(message.HasField('default_sint64')) test_case.assertTrue(message.HasField('default_sint32'))
test_case.assertTrue(message.HasField('default_fixed32')) test_case.assertTrue(message.HasField('default_sint64'))
test_case.assertTrue(message.HasField('default_fixed64')) test_case.assertTrue(message.HasField('default_fixed32'))
test_case.assertTrue(message.HasField('default_sfixed32')) test_case.assertTrue(message.HasField('default_fixed64'))
test_case.assertTrue(message.HasField('default_sfixed64')) test_case.assertTrue(message.HasField('default_sfixed32'))
test_case.assertTrue(message.HasField('default_float')) test_case.assertTrue(message.HasField('default_sfixed64'))
test_case.assertTrue(message.HasField('default_double')) test_case.assertTrue(message.HasField('default_float'))
test_case.assertTrue(message.HasField('default_bool')) test_case.assertTrue(message.HasField('default_double'))
test_case.assertTrue(message.HasField('default_string')) test_case.assertTrue(message.HasField('default_bool'))
test_case.assertTrue(message.HasField('default_bytes')) test_case.assertTrue(message.HasField('default_string'))
test_case.assertTrue(message.HasField('default_bytes'))
test_case.assertTrue(message.HasField('default_nested_enum')) test_case.assertTrue(message.HasField('default_nested_enum'))
test_case.assertTrue(message.HasField('default_foreign_enum')) test_case.assertTrue(message.HasField('default_foreign_enum'))
test_case.assertTrue(message.HasField('default_import_enum')) test_case.assertTrue(message.HasField('default_import_enum'))
test_case.assertEqual(401, message.default_int32) test_case.assertEqual(401, message.default_int32)
test_case.assertEqual(402, message.default_int64) test_case.assertEqual(402, message.default_int64)
test_case.assertEqual(403, message.default_uint32) test_case.assertEqual(403, message.default_uint32)
test_case.assertEqual(404, message.default_uint64) test_case.assertEqual(404, message.default_uint64)
test_case.assertEqual(405, message.default_sint32) test_case.assertEqual(405, message.default_sint32)
test_case.assertEqual(406, message.default_sint64) test_case.assertEqual(406, message.default_sint64)
test_case.assertEqual(407, message.default_fixed32) test_case.assertEqual(407, message.default_fixed32)
test_case.assertEqual(408, message.default_fixed64) test_case.assertEqual(408, message.default_fixed64)
test_case.assertEqual(409, message.default_sfixed32) test_case.assertEqual(409, message.default_sfixed32)
test_case.assertEqual(410, message.default_sfixed64) test_case.assertEqual(410, message.default_sfixed64)
test_case.assertEqual(411, message.default_float) test_case.assertEqual(411, message.default_float)
test_case.assertEqual(412, message.default_double) test_case.assertEqual(412, message.default_double)
test_case.assertEqual(False, message.default_bool) test_case.assertEqual(False, message.default_bool)
test_case.assertEqual('415', message.default_string) test_case.assertEqual('415', message.default_string)
test_case.assertEqual(b'416', message.default_bytes) test_case.assertEqual(b'416', message.default_bytes)
test_case.assertEqual(unittest_pb2.TestAllTypes.FOO, test_case.assertEqual(unittest_pb2.TestAllTypes.FOO,
message.default_nested_enum) message.default_nested_enum)
test_case.assertEqual(unittest_pb2.FOREIGN_FOO, test_case.assertEqual(unittest_pb2.FOREIGN_FOO,
message.default_foreign_enum) message.default_foreign_enum)
test_case.assertEqual(unittest_import_pb2.IMPORT_FOO, test_case.assertEqual(unittest_import_pb2.IMPORT_FOO,
message.default_import_enum) message.default_import_enum)
def GoldenFile(filename): def GoldenFile(filename):
@ -594,7 +610,7 @@ def SetAllPackedFields(message):
"""Sets every field in the message to a unique value. """Sets every field in the message to a unique value.
Args: Args:
message: A unittest_pb2.TestPackedTypes instance. message: A TestPackedTypes instance.
""" """
message.packed_int32.extend([601, 701]) message.packed_int32.extend([601, 701])
message.packed_int64.extend([602, 702]) message.packed_int64.extend([602, 702])

557
python/google/protobuf/internal/text_format_test.py
View file

@ -37,13 +37,17 @@ __author__ = 'kenton@google.com (Kenton Varda)'
import re import re
from google.apputils import basetest from google.apputils import basetest
from google.protobuf import text_format from google.apputils.pybase import parameterized
from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_pb2
from google.protobuf import unittest_proto3_arena_pb2
from google.protobuf.internal import api_implementation from google.protobuf.internal import api_implementation
from google.protobuf.internal import test_util from google.protobuf.internal import test_util
from google.protobuf import unittest_pb2 from google.protobuf import text_format
from google.protobuf import unittest_mset_pb2
class TextFormatTest(basetest.TestCase): # Base class with some common functionality.
class TextFormatBase(basetest.TestCase):
def ReadGolden(self, golden_filename): def ReadGolden(self, golden_filename):
with test_util.GoldenFile(golden_filename) as f: with test_util.GoldenFile(golden_filename) as f:
@ -57,73 +61,24 @@ class TextFormatTest(basetest.TestCase):
def CompareToGoldenText(self, text, golden_text): def CompareToGoldenText(self, text, golden_text):
self.assertMultiLineEqual(text, golden_text) self.assertMultiLineEqual(text, golden_text)
def testPrintAllFields(self): def RemoveRedundantZeros(self, text):
message = unittest_pb2.TestAllTypes() # Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove
test_util.SetAllFields(message) # these zeros in order to match the golden file.
self.CompareToGoldenFile( text = text.replace('e+0','e+').replace('e+0','e+') \
self.RemoveRedundantZeros(text_format.MessageToString(message)), .replace('e-0','e-').replace('e-0','e-')
'text_format_unittest_data_oneof_implemented.txt') # Floating point fields are printed with .0 suffix even if they are
# actualy integer numbers.
text = re.compile('\.0$', re.MULTILINE).sub('', text)
return text
def testPrintInIndexOrder(self):
message = unittest_pb2.TestFieldOrderings()
message.my_string = '115'
message.my_int = 101
message.my_float = 111
message.optional_nested_message.oo = 0
message.optional_nested_message.bb = 1
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message, use_index_order=True)),
'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n'
'optional_nested_message {\n oo: 0\n bb: 1\n}\n')
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message)),
'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n'
'optional_nested_message {\n bb: 1\n oo: 0\n}\n')
def testPrintAllExtensions(self): @parameterized.Parameters(
message = unittest_pb2.TestAllExtensions() (unittest_pb2),
test_util.SetAllExtensions(message) (unittest_proto3_arena_pb2))
self.CompareToGoldenFile( class TextFormatTest(TextFormatBase):
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_extensions_data.txt')
def testPrintAllFieldsPointy(self): def testPrintExotic(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
test_util.SetAllFields(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(
text_format.MessageToString(message, pointy_brackets=True)),
'text_format_unittest_data_pointy_oneof.txt')
def testPrintAllExtensionsPointy(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(
message, pointy_brackets=True)),
'text_format_unittest_extensions_data_pointy.txt')
def testPrintMessageSet(self):
message = unittest_mset_pb2.TestMessageSetContainer()
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(
text_format.MessageToString(message),
'message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
def testPrintExotic(self):
message = unittest_pb2.TestAllTypes()
message.repeated_int64.append(-9223372036854775808) message.repeated_int64.append(-9223372036854775808)
message.repeated_uint64.append(18446744073709551615) message.repeated_uint64.append(18446744073709551615)
message.repeated_double.append(123.456) message.repeated_double.append(123.456)
@ -142,61 +97,44 @@ class TextFormatTest(basetest.TestCase):
' "\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n' ' "\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n'
'repeated_string: "\\303\\274\\352\\234\\237"\n') 'repeated_string: "\\303\\274\\352\\234\\237"\n')
def testPrintExoticUnicodeSubclass(self): def testPrintExoticUnicodeSubclass(self, message_module):
class UnicodeSub(unicode): class UnicodeSub(unicode):
pass pass
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
message.repeated_string.append(UnicodeSub(u'\u00fc\ua71f')) message.repeated_string.append(UnicodeSub(u'\u00fc\ua71f'))
self.CompareToGoldenText( self.CompareToGoldenText(
text_format.MessageToString(message), text_format.MessageToString(message),
'repeated_string: "\\303\\274\\352\\234\\237"\n') 'repeated_string: "\\303\\274\\352\\234\\237"\n')
def testPrintNestedMessageAsOneLine(self): def testPrintNestedMessageAsOneLine(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
msg = message.repeated_nested_message.add() msg = message.repeated_nested_message.add()
msg.bb = 42 msg.bb = 42
self.CompareToGoldenText( self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True), text_format.MessageToString(message, as_one_line=True),
'repeated_nested_message { bb: 42 }') 'repeated_nested_message { bb: 42 }')
def testPrintRepeatedFieldsAsOneLine(self): def testPrintRepeatedFieldsAsOneLine(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
message.repeated_int32.append(1) message.repeated_int32.append(1)
message.repeated_int32.append(1) message.repeated_int32.append(1)
message.repeated_int32.append(3) message.repeated_int32.append(3)
message.repeated_string.append("Google") message.repeated_string.append('Google')
message.repeated_string.append("Zurich") message.repeated_string.append('Zurich')
self.CompareToGoldenText( self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True), text_format.MessageToString(message, as_one_line=True),
'repeated_int32: 1 repeated_int32: 1 repeated_int32: 3 ' 'repeated_int32: 1 repeated_int32: 1 repeated_int32: 3 '
'repeated_string: "Google" repeated_string: "Zurich"') 'repeated_string: "Google" repeated_string: "Zurich"')
def testPrintNestedNewLineInStringAsOneLine(self): def testPrintNestedNewLineInStringAsOneLine(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
message.optional_string = "a\nnew\nline" message.optional_string = 'a\nnew\nline'
self.CompareToGoldenText( self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True), text_format.MessageToString(message, as_one_line=True),
'optional_string: "a\\nnew\\nline"') 'optional_string: "a\\nnew\\nline"')
def testPrintMessageSetAsOneLine(self): def testPrintExoticAsOneLine(self, message_module):
message = unittest_mset_pb2.TestMessageSetContainer() message = message_module.TestAllTypes()
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
'message_set {'
' [protobuf_unittest.TestMessageSetExtension1] {'
' i: 23'
' }'
' [protobuf_unittest.TestMessageSetExtension2] {'
' str: \"foo\"'
' }'
' }')
def testPrintExoticAsOneLine(self):
message = unittest_pb2.TestAllTypes()
message.repeated_int64.append(-9223372036854775808) message.repeated_int64.append(-9223372036854775808)
message.repeated_uint64.append(18446744073709551615) message.repeated_uint64.append(18446744073709551615)
message.repeated_double.append(123.456) message.repeated_double.append(123.456)
@ -216,8 +154,8 @@ class TextFormatTest(basetest.TestCase):
'"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""' '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""'
' repeated_string: "\\303\\274\\352\\234\\237"') ' repeated_string: "\\303\\274\\352\\234\\237"')
def testRoundTripExoticAsOneLine(self): def testRoundTripExoticAsOneLine(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
message.repeated_int64.append(-9223372036854775808) message.repeated_int64.append(-9223372036854775808)
message.repeated_uint64.append(18446744073709551615) message.repeated_uint64.append(18446744073709551615)
message.repeated_double.append(123.456) message.repeated_double.append(123.456)
@ -229,7 +167,7 @@ class TextFormatTest(basetest.TestCase):
# Test as_utf8 = False. # Test as_utf8 = False.
wire_text = text_format.MessageToString( wire_text = text_format.MessageToString(
message, as_one_line=True, as_utf8=False) message, as_one_line=True, as_utf8=False)
parsed_message = unittest_pb2.TestAllTypes() parsed_message = message_module.TestAllTypes()
r = text_format.Parse(wire_text, parsed_message) r = text_format.Parse(wire_text, parsed_message)
self.assertIs(r, parsed_message) self.assertIs(r, parsed_message)
self.assertEquals(message, parsed_message) self.assertEquals(message, parsed_message)
@ -237,25 +175,25 @@ class TextFormatTest(basetest.TestCase):
# Test as_utf8 = True. # Test as_utf8 = True.
wire_text = text_format.MessageToString( wire_text = text_format.MessageToString(
message, as_one_line=True, as_utf8=True) message, as_one_line=True, as_utf8=True)
parsed_message = unittest_pb2.TestAllTypes() parsed_message = message_module.TestAllTypes()
r = text_format.Parse(wire_text, parsed_message) r = text_format.Parse(wire_text, parsed_message)
self.assertIs(r, parsed_message) self.assertIs(r, parsed_message)
self.assertEquals(message, parsed_message, self.assertEquals(message, parsed_message,
'\n%s != %s' % (message, parsed_message)) '\n%s != %s' % (message, parsed_message))
def testPrintRawUtf8String(self): def testPrintRawUtf8String(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
message.repeated_string.append(u'\u00fc\ua71f') message.repeated_string.append(u'\u00fc\ua71f')
text = text_format.MessageToString(message, as_utf8=True) text = text_format.MessageToString(message, as_utf8=True)
self.CompareToGoldenText(text, 'repeated_string: "\303\274\352\234\237"\n') self.CompareToGoldenText(text, 'repeated_string: "\303\274\352\234\237"\n')
parsed_message = unittest_pb2.TestAllTypes() parsed_message = message_module.TestAllTypes()
text_format.Parse(text, parsed_message) text_format.Parse(text, parsed_message)
self.assertEquals(message, parsed_message, self.assertEquals(message, parsed_message,
'\n%s != %s' % (message, parsed_message)) '\n%s != %s' % (message, parsed_message))
def testPrintFloatFormat(self): def testPrintFloatFormat(self, message_module):
# Check that float_format argument is passed to sub-message formatting. # Check that float_format argument is passed to sub-message formatting.
message = unittest_pb2.NestedTestAllTypes() message = message_module.NestedTestAllTypes()
# We use 1.25 as it is a round number in binary. The proto 32-bit float # We use 1.25 as it is a round number in binary. The proto 32-bit float
# will not gain additional imprecise digits as a 64-bit Python float and # will not gain additional imprecise digits as a 64-bit Python float and
# show up in its str. 32-bit 1.2 is noisy when extended to 64-bit: # show up in its str. 32-bit 1.2 is noisy when extended to 64-bit:
@ -285,85 +223,24 @@ class TextFormatTest(basetest.TestCase):
self.RemoveRedundantZeros(text_message), self.RemoveRedundantZeros(text_message),
'payload {{ {} {} {} {} }}'.format(*formatted_fields)) 'payload {{ {} {} {} {} }}'.format(*formatted_fields))
def testMessageToString(self): def testMessageToString(self, message_module):
message = unittest_pb2.ForeignMessage() message = message_module.ForeignMessage()
message.c = 123 message.c = 123
self.assertEqual('c: 123\n', str(message)) self.assertEqual('c: 123\n', str(message))
def RemoveRedundantZeros(self, text): def testParseAllFields(self, message_module):
# Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove message = message_module.TestAllTypes()
# these zeros in order to match the golden file.
text = text.replace('e+0','e+').replace('e+0','e+') \
.replace('e-0','e-').replace('e-0','e-')
# Floating point fields are printed with .0 suffix even if they are
# actualy integer numbers.
text = re.compile('\.0$', re.MULTILINE).sub('', text)
return text
def testParseGolden(self):
golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt'))
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.Parse(golden_text, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEquals(message, parsed_message)
def testParseGoldenExtensions(self):
golden_text = '\n'.join(self.ReadGolden(
'text_format_unittest_extensions_data.txt'))
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Parse(golden_text, parsed_message)
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.assertEquals(message, parsed_message)
def testParseAllFields(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message) test_util.SetAllFields(message)
ascii_text = text_format.MessageToString(message) ascii_text = text_format.MessageToString(message)
parsed_message = unittest_pb2.TestAllTypes() parsed_message = message_module.TestAllTypes()
text_format.Parse(ascii_text, parsed_message) text_format.Parse(ascii_text, parsed_message)
self.assertEqual(message, parsed_message) self.assertEqual(message, parsed_message)
test_util.ExpectAllFieldsSet(self, message) if message_module is unittest_pb2:
test_util.ExpectAllFieldsSet(self, message)
def testParseAllExtensions(self): def testParseExotic(self, message_module):
message = unittest_pb2.TestAllExtensions() message = message_module.TestAllTypes()
test_util.SetAllExtensions(message)
ascii_text = text_format.MessageToString(message)
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Parse(ascii_text, parsed_message)
self.assertEqual(message, parsed_message)
def testParseMessageSet(self):
message = unittest_pb2.TestAllTypes()
text = ('repeated_uint64: 1\n'
'repeated_uint64: 2\n')
text_format.Parse(text, message)
self.assertEqual(1, message.repeated_uint64[0])
self.assertEqual(2, message.repeated_uint64[1])
message = unittest_mset_pb2.TestMessageSetContainer()
text = ('message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
text_format.Parse(text, message)
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
self.assertEquals(23, message.message_set.Extensions[ext1].i)
self.assertEquals('foo', message.message_set.Extensions[ext2].str)
def testParseExotic(self):
message = unittest_pb2.TestAllTypes()
text = ('repeated_int64: -9223372036854775808\n' text = ('repeated_int64: -9223372036854775808\n'
'repeated_uint64: 18446744073709551615\n' 'repeated_uint64: 18446744073709551615\n'
'repeated_double: 123.456\n' 'repeated_double: 123.456\n'
@ -388,8 +265,8 @@ class TextFormatTest(basetest.TestCase):
self.assertEqual(u'\u00fc\ua71f', message.repeated_string[2]) self.assertEqual(u'\u00fc\ua71f', message.repeated_string[2])
self.assertEqual(u'\u00fc', message.repeated_string[3]) self.assertEqual(u'\u00fc', message.repeated_string[3])
def testParseTrailingCommas(self): def testParseTrailingCommas(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = ('repeated_int64: 100;\n' text = ('repeated_int64: 100;\n'
'repeated_int64: 200;\n' 'repeated_int64: 200;\n'
'repeated_int64: 300,\n' 'repeated_int64: 300,\n'
@ -403,51 +280,37 @@ class TextFormatTest(basetest.TestCase):
self.assertEqual(u'one', message.repeated_string[0]) self.assertEqual(u'one', message.repeated_string[0])
self.assertEqual(u'two', message.repeated_string[1]) self.assertEqual(u'two', message.repeated_string[1])
def testParseEmptyText(self): def testParseEmptyText(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = '' text = ''
text_format.Parse(text, message) text_format.Parse(text, message)
self.assertEquals(unittest_pb2.TestAllTypes(), message) self.assertEquals(message_module.TestAllTypes(), message)
def testParseInvalidUtf8(self): def testParseInvalidUtf8(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = 'repeated_string: "\\xc3\\xc3"' text = 'repeated_string: "\\xc3\\xc3"'
self.assertRaises(text_format.ParseError, text_format.Parse, text, message) self.assertRaises(text_format.ParseError, text_format.Parse, text, message)
def testParseSingleWord(self): def testParseSingleWord(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = 'foo' text = 'foo'
self.assertRaisesWithLiteralMatch( self.assertRaisesRegexp(
text_format.ParseError, text_format.ParseError,
('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named ' (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
'"foo".'), r'"foo".'),
text_format.Parse, text, message) text_format.Parse, text, message)
def testParseUnknownField(self): def testParseUnknownField(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = 'unknown_field: 8\n' text = 'unknown_field: 8\n'
self.assertRaisesWithLiteralMatch( self.assertRaisesRegexp(
text_format.ParseError, text_format.ParseError,
('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named ' (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
'"unknown_field".'), r'"unknown_field".'),
text_format.Parse, text, message) text_format.Parse, text, message)
def testParseBadExtension(self): def testParseGroupNotClosed(self, message_module):
message = unittest_pb2.TestAllExtensions() message = message_module.TestAllTypes()
text = '[unknown_extension]: 8\n'
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
'1:2 : Extension "unknown_extension" not registered.',
text_format.Parse, text, message)
message = unittest_pb2.TestAllTypes()
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
'extensions.'),
text_format.Parse, text, message)
def testParseGroupNotClosed(self):
message = unittest_pb2.TestAllTypes()
text = 'RepeatedGroup: <' text = 'RepeatedGroup: <'
self.assertRaisesWithLiteralMatch( self.assertRaisesWithLiteralMatch(
text_format.ParseError, '1:16 : Expected ">".', text_format.ParseError, '1:16 : Expected ">".',
@ -458,46 +321,46 @@ class TextFormatTest(basetest.TestCase):
text_format.ParseError, '1:16 : Expected "}".', text_format.ParseError, '1:16 : Expected "}".',
text_format.Parse, text, message) text_format.Parse, text, message)
def testParseEmptyGroup(self): def testParseEmptyGroup(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = 'OptionalGroup: {}' text = 'OptionalGroup: {}'
text_format.Parse(text, message) text_format.Parse(text, message)
self.assertTrue(message.HasField('optionalgroup')) self.assertTrue(message.HasField('optionalgroup'))
message.Clear() message.Clear()
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = 'OptionalGroup: <>' text = 'OptionalGroup: <>'
text_format.Parse(text, message) text_format.Parse(text, message)
self.assertTrue(message.HasField('optionalgroup')) self.assertTrue(message.HasField('optionalgroup'))
def testParseBadEnumValue(self): def testParseBadEnumValue(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = 'optional_nested_enum: BARR' text = 'optional_nested_enum: BARR'
self.assertRaisesWithLiteralMatch( self.assertRaisesRegexp(
text_format.ParseError, text_format.ParseError,
('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" ' (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
'has no value named BARR.'), r'has no value named BARR.'),
text_format.Parse, text, message) text_format.Parse, text, message)
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = 'optional_nested_enum: 100' text = 'optional_nested_enum: 100'
self.assertRaisesWithLiteralMatch( self.assertRaisesRegexp(
text_format.ParseError, text_format.ParseError,
('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" ' (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
'has no value with number 100.'), r'has no value with number 100.'),
text_format.Parse, text, message) text_format.Parse, text, message)
def testParseBadIntValue(self): def testParseBadIntValue(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = 'optional_int32: bork' text = 'optional_int32: bork'
self.assertRaisesWithLiteralMatch( self.assertRaisesWithLiteralMatch(
text_format.ParseError, text_format.ParseError,
('1:17 : Couldn\'t parse integer: bork'), ('1:17 : Couldn\'t parse integer: bork'),
text_format.Parse, text, message) text_format.Parse, text, message)
def testParseStringFieldUnescape(self): def testParseStringFieldUnescape(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = r'''repeated_string: "\xf\x62" text = r'''repeated_string: "\xf\x62"
repeated_string: "\\xf\\x62" repeated_string: "\\xf\\x62"
repeated_string: "\\\xf\\\x62" repeated_string: "\\\xf\\\x62"
@ -516,40 +379,205 @@ class TextFormatTest(basetest.TestCase):
message.repeated_string[4]) message.repeated_string[4])
self.assertEqual(SLASH + 'x20', message.repeated_string[5]) self.assertEqual(SLASH + 'x20', message.repeated_string[5])
def testMergeDuplicateScalars(self): def testMergeDuplicateScalars(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = ('optional_int32: 42 ' text = ('optional_int32: 42 '
'optional_int32: 67') 'optional_int32: 67')
r = text_format.Merge(text, message) r = text_format.Merge(text, message)
self.assertIs(r, message) self.assertIs(r, message)
self.assertEqual(67, message.optional_int32) self.assertEqual(67, message.optional_int32)
def testParseDuplicateScalars(self): def testMergeDuplicateNestedMessageScalars(self, message_module):
message = unittest_pb2.TestAllTypes() message = message_module.TestAllTypes()
text = ('optional_int32: 42 '
'optional_int32: 67')
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:36 : Message type "protobuf_unittest.TestAllTypes" should not '
'have multiple "optional_int32" fields.'),
text_format.Parse, text, message)
def testMergeDuplicateNestedMessageScalars(self):
message = unittest_pb2.TestAllTypes()
text = ('optional_nested_message { bb: 1 } ' text = ('optional_nested_message { bb: 1 } '
'optional_nested_message { bb: 2 }') 'optional_nested_message { bb: 2 }')
r = text_format.Merge(text, message) r = text_format.Merge(text, message)
self.assertTrue(r is message) self.assertTrue(r is message)
self.assertEqual(2, message.optional_nested_message.bb) self.assertEqual(2, message.optional_nested_message.bb)
def testParseDuplicateNestedMessageScalars(self): def testParseOneof(self, message_module):
m = message_module.TestAllTypes()
m.oneof_uint32 = 11
m2 = message_module.TestAllTypes()
text_format.Parse(text_format.MessageToString(m), m2)
self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field'))
# These are tests that aren't fundamentally specific to proto2, but are at
# the moment because of differences between the proto2 and proto3 test schemas.
# Ideally the schemas would be made more similar so these tests could pass.
class OnlyWorksWithProto2RightNowTests(TextFormatBase):
def testParseGolden(self):
golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt'))
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.Parse(golden_text, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes() message = unittest_pb2.TestAllTypes()
text = ('optional_nested_message { bb: 1 } ' test_util.SetAllFields(message)
'optional_nested_message { bb: 2 }') self.assertEquals(message, parsed_message)
def testPrintAllFields(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_data_oneof_implemented.txt')
def testPrintAllFieldsPointy(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(
text_format.MessageToString(message, pointy_brackets=True)),
'text_format_unittest_data_pointy_oneof.txt')
def testPrintInIndexOrder(self):
message = unittest_pb2.TestFieldOrderings()
message.my_string = '115'
message.my_int = 101
message.my_float = 111
message.optional_nested_message.oo = 0
message.optional_nested_message.bb = 1
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message, use_index_order=True)),
'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n'
'optional_nested_message {\n oo: 0\n bb: 1\n}\n')
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message)),
'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n'
'optional_nested_message {\n bb: 1\n oo: 0\n}\n')
def testMergeLinesGolden(self):
opened = self.ReadGolden('text_format_unittest_data.txt')
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.MergeLines(opened, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEqual(message, parsed_message)
def testParseLinesGolden(self):
opened = self.ReadGolden('text_format_unittest_data.txt')
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.ParseLines(opened, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEquals(message, parsed_message)
# Tests of proto2-only features (MessageSet and extensions).
class Proto2Tests(TextFormatBase):
def testPrintMessageSet(self):
message = unittest_mset_pb2.TestMessageSetContainer()
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(
text_format.MessageToString(message),
'message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
def testPrintMessageSetAsOneLine(self):
message = unittest_mset_pb2.TestMessageSetContainer()
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
'message_set {'
' [protobuf_unittest.TestMessageSetExtension1] {'
' i: 23'
' }'
' [protobuf_unittest.TestMessageSetExtension2] {'
' str: \"foo\"'
' }'
' }')
def testParseMessageSet(self):
message = unittest_pb2.TestAllTypes()
text = ('repeated_uint64: 1\n'
'repeated_uint64: 2\n')
text_format.Parse(text, message)
self.assertEqual(1, message.repeated_uint64[0])
self.assertEqual(2, message.repeated_uint64[1])
message = unittest_mset_pb2.TestMessageSetContainer()
text = ('message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
text_format.Parse(text, message)
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
self.assertEquals(23, message.message_set.Extensions[ext1].i)
self.assertEquals('foo', message.message_set.Extensions[ext2].str)
def testPrintAllExtensions(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_extensions_data.txt')
def testPrintAllExtensionsPointy(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(
message, pointy_brackets=True)),
'text_format_unittest_extensions_data_pointy.txt')
def testParseGoldenExtensions(self):
golden_text = '\n'.join(self.ReadGolden(
'text_format_unittest_extensions_data.txt'))
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Parse(golden_text, parsed_message)
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.assertEquals(message, parsed_message)
def testParseAllExtensions(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
ascii_text = text_format.MessageToString(message)
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Parse(ascii_text, parsed_message)
self.assertEqual(message, parsed_message)
def testParseBadExtension(self):
message = unittest_pb2.TestAllExtensions()
text = '[unknown_extension]: 8\n'
self.assertRaisesWithLiteralMatch( self.assertRaisesWithLiteralMatch(
text_format.ParseError, text_format.ParseError,
('1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" ' '1:2 : Extension "unknown_extension" not registered.',
'should not have multiple "bb" fields.'), text_format.Parse, text, message)
message = unittest_pb2.TestAllTypes()
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
'extensions.'),
text_format.Parse, text, message) text_format.Parse, text, message)
def testMergeDuplicateExtensionScalars(self): def testMergeDuplicateExtensionScalars(self):
@ -572,32 +600,25 @@ class TextFormatTest(basetest.TestCase):
'"protobuf_unittest.optional_int32_extension" extensions.'), '"protobuf_unittest.optional_int32_extension" extensions.'),
text_format.Parse, text, message) text_format.Parse, text, message)
def testParseLinesGolden(self): def testParseDuplicateNestedMessageScalars(self):
opened = self.ReadGolden('text_format_unittest_data.txt')
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.ParseLines(opened, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes() message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message) text = ('optional_nested_message { bb: 1 } '
self.assertEquals(message, parsed_message) 'optional_nested_message { bb: 2 }')
self.assertRaisesWithLiteralMatch(
def testMergeLinesGolden(self): text_format.ParseError,
opened = self.ReadGolden('text_format_unittest_data.txt') ('1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" '
parsed_message = unittest_pb2.TestAllTypes() 'should not have multiple "bb" fields.'),
r = text_format.MergeLines(opened, parsed_message) text_format.Parse, text, message)
self.assertIs(r, parsed_message)
def testParseDuplicateScalars(self):
message = unittest_pb2.TestAllTypes() message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message) text = ('optional_int32: 42 '
self.assertEqual(message, parsed_message) 'optional_int32: 67')
self.assertRaisesWithLiteralMatch(
def testParseOneof(self): text_format.ParseError,
m = unittest_pb2.TestAllTypes() ('1:36 : Message type "protobuf_unittest.TestAllTypes" should not '
m.oneof_uint32 = 11 'have multiple "optional_int32" fields.'),
m2 = unittest_pb2.TestAllTypes() text_format.Parse, text, message)
text_format.Parse(text_format.MessageToString(m), m2)
self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field'))
class TokenizerTest(basetest.TestCase): class TokenizerTest(basetest.TestCase):

8
python/google/protobuf/internal/type_checkers.py
View file

@ -59,6 +59,8 @@ from google.protobuf import descriptor
_FieldDescriptor = descriptor.FieldDescriptor _FieldDescriptor = descriptor.FieldDescriptor
def SupportsOpenEnums(field_descriptor):
return field_descriptor.containing_type.syntax == "proto3"
def GetTypeChecker(field): def GetTypeChecker(field):
"""Returns a type checker for a message field of the specified types. """Returns a type checker for a message field of the specified types.
@ -74,7 +76,11 @@ def GetTypeChecker(field):
field.type == _FieldDescriptor.TYPE_STRING): field.type == _FieldDescriptor.TYPE_STRING):
return UnicodeValueChecker() return UnicodeValueChecker()
if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM: if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM:
return EnumValueChecker(field.enum_type) if SupportsOpenEnums(field):
# When open enums are supported, any int32 can be assigned.
return _VALUE_CHECKERS[_FieldDescriptor.CPPTYPE_INT32]
else:
return EnumValueChecker(field.enum_type)
return _VALUE_CHECKERS[field.cpp_type] return _VALUE_CHECKERS[field.cpp_type]

120
python/google/protobuf/internal/unknown_fields_test.py
View file

@ -38,6 +38,7 @@ __author__ = 'bohdank@google.com (Bohdan Koval)'
from google.apputils import basetest from google.apputils import basetest
from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_pb2 from google.protobuf import unittest_pb2
from google.protobuf import unittest_proto3_arena_pb2
from google.protobuf.internal import api_implementation from google.protobuf.internal import api_implementation
from google.protobuf.internal import encoder from google.protobuf.internal import encoder
from google.protobuf.internal import missing_enum_values_pb2 from google.protobuf.internal import missing_enum_values_pb2
@ -45,10 +46,81 @@ from google.protobuf.internal import test_util
from google.protobuf.internal import type_checkers from google.protobuf.internal import type_checkers
class UnknownFieldsTest(basetest.TestCase):
def setUp(self):
self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
self.all_fields = unittest_pb2.TestAllTypes()
test_util.SetAllFields(self.all_fields)
self.all_fields_data = self.all_fields.SerializeToString()
self.empty_message = unittest_pb2.TestEmptyMessage()
self.empty_message.ParseFromString(self.all_fields_data)
def testSerialize(self):
data = self.empty_message.SerializeToString()
# Don't use assertEqual because we don't want to dump raw binary data to
# stdout.
self.assertTrue(data == self.all_fields_data)
def testSerializeProto3(self):
# Verify that proto3 doesn't preserve unknown fields.
message = unittest_proto3_arena_pb2.TestEmptyMessage()
message.ParseFromString(self.all_fields_data)
self.assertEqual(0, len(message.SerializeToString()))
def testByteSize(self):
self.assertEqual(self.all_fields.ByteSize(), self.empty_message.ByteSize())
def testListFields(self):
# Make sure ListFields doesn't return unknown fields.
self.assertEqual(0, len(self.empty_message.ListFields()))
def testSerializeMessageSetWireFormatUnknownExtension(self):
# Create a message using the message set wire format with an unknown
# message.
raw = unittest_mset_pb2.RawMessageSet()
# Add an unknown extension.
item = raw.item.add()
item.type_id = 1545009
message1 = unittest_mset_pb2.TestMessageSetExtension1()
message1.i = 12345
item.message = message1.SerializeToString()
serialized = raw.SerializeToString()
# Parse message using the message set wire format.
proto = unittest_mset_pb2.TestMessageSet()
proto.MergeFromString(serialized)
# Verify that the unknown extension is serialized unchanged
reserialized = proto.SerializeToString()
new_raw = unittest_mset_pb2.RawMessageSet()
new_raw.MergeFromString(reserialized)
self.assertEqual(raw, new_raw)
# C++ implementation for proto2 does not currently take into account unknown
# fields when checking equality.
#
# TODO(haberman): fix this.
@basetest.unittest.skipIf(
api_implementation.Type() == 'cpp' and api_implementation.Version() == 2,
'C++ implementation does not expose unknown fields to Python')
def testEquals(self):
message = unittest_pb2.TestEmptyMessage()
message.ParseFromString(self.all_fields_data)
self.assertEqual(self.empty_message, message)
self.all_fields.ClearField('optional_string')
message.ParseFromString(self.all_fields.SerializeToString())
self.assertNotEqual(self.empty_message, message)
@basetest.unittest.skipIf( @basetest.unittest.skipIf(
api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, api_implementation.Type() == 'cpp' and api_implementation.Version() == 2,
'C++ implementation does not expose unknown fields to Python') 'C++ implementation does not expose unknown fields to Python')
class UnknownFieldsTest(basetest.TestCase): class UnknownFieldsAccessorsTest(basetest.TestCase):
def setUp(self): def setUp(self):
self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
@ -98,13 +170,6 @@ class UnknownFieldsTest(basetest.TestCase):
value = self.GetField('optionalgroup') value = self.GetField('optionalgroup')
self.assertEqual(self.all_fields.optionalgroup, value) self.assertEqual(self.all_fields.optionalgroup, value)
def testSerialize(self):
data = self.empty_message.SerializeToString()
# Don't use assertEqual because we don't want to dump raw binary data to
# stdout.
self.assertTrue(data == self.all_fields_data)
def testCopyFrom(self): def testCopyFrom(self):
message = unittest_pb2.TestEmptyMessage() message = unittest_pb2.TestEmptyMessage()
message.CopyFrom(self.empty_message) message.CopyFrom(self.empty_message)
@ -132,51 +197,12 @@ class UnknownFieldsTest(basetest.TestCase):
self.empty_message.Clear() self.empty_message.Clear()
self.assertEqual(0, len(self.empty_message._unknown_fields)) self.assertEqual(0, len(self.empty_message._unknown_fields))
def testByteSize(self):
self.assertEqual(self.all_fields.ByteSize(), self.empty_message.ByteSize())
def testUnknownExtensions(self): def testUnknownExtensions(self):
message = unittest_pb2.TestEmptyMessageWithExtensions() message = unittest_pb2.TestEmptyMessageWithExtensions()
message.ParseFromString(self.all_fields_data) message.ParseFromString(self.all_fields_data)
self.assertEqual(self.empty_message._unknown_fields, self.assertEqual(self.empty_message._unknown_fields,
message._unknown_fields) message._unknown_fields)
def testListFields(self):
# Make sure ListFields doesn't return unknown fields.
self.assertEqual(0, len(self.empty_message.ListFields()))
def testSerializeMessageSetWireFormatUnknownExtension(self):
# Create a message using the message set wire format with an unknown
# message.
raw = unittest_mset_pb2.RawMessageSet()
# Add an unknown extension.
item = raw.item.add()
item.type_id = 1545009
message1 = unittest_mset_pb2.TestMessageSetExtension1()
message1.i = 12345
item.message = message1.SerializeToString()
serialized = raw.SerializeToString()
# Parse message using the message set wire format.
proto = unittest_mset_pb2.TestMessageSet()
proto.MergeFromString(serialized)
# Verify that the unknown extension is serialized unchanged
reserialized = proto.SerializeToString()
new_raw = unittest_mset_pb2.RawMessageSet()
new_raw.MergeFromString(reserialized)
self.assertEqual(raw, new_raw)
def testEquals(self):
message = unittest_pb2.TestEmptyMessage()
message.ParseFromString(self.all_fields_data)
self.assertEqual(self.empty_message, message)
self.all_fields.ClearField('optional_string')
message.ParseFromString(self.all_fields.SerializeToString())
self.assertNotEqual(self.empty_message, message)
@basetest.unittest.skipIf( @basetest.unittest.skipIf(

1564
python/google/protobuf/pyext/descriptor.cc
View file

File diff suppressed because it is too large Load diff

118
python/google/protobuf/pyext/descriptor.h
View file

@ -34,105 +34,55 @@
#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__
#include <Python.h> #include <Python.h>
#include <structmember.h>
#include <google/protobuf/stubs/hash.h>
#include <google/protobuf/descriptor.h> #include <google/protobuf/descriptor.h>
#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
typedef int Py_ssize_t;
#define PY_SSIZE_T_MAX INT_MAX
#define PY_SSIZE_T_MIN INT_MIN
#endif
namespace google { namespace google {
namespace protobuf { namespace protobuf {
namespace python { namespace python {
typedef struct CMessageDescriptor { extern PyTypeObject PyMessageDescriptor_Type;
PyObject_HEAD extern PyTypeObject PyFieldDescriptor_Type;
extern PyTypeObject PyEnumDescriptor_Type;
extern PyTypeObject PyEnumValueDescriptor_Type;
extern PyTypeObject PyFileDescriptor_Type;
extern PyTypeObject PyOneofDescriptor_Type;
// The proto2 descriptor that this object represents. // Return a new reference to a Descriptor object.
const google::protobuf::Descriptor* descriptor; // The C++ pointer is usually borrowed from the global DescriptorPool.
} CMessageDescriptor; // In any case, it must stay alive as long as the Python object.
PyObject* PyMessageDescriptor_New(const Descriptor* descriptor);
PyObject* PyFieldDescriptor_New(const FieldDescriptor* descriptor);
PyObject* PyEnumDescriptor_New(const EnumDescriptor* descriptor);
PyObject* PyEnumValueDescriptor_New(const EnumValueDescriptor* descriptor);
PyObject* PyOneofDescriptor_New(const OneofDescriptor* descriptor);
PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor);
// Alternate constructor of PyFileDescriptor, used when we already have a
// serialized FileDescriptorProto that can be cached.
// Returns a new reference.
PyObject* PyFileDescriptor_NewWithPb(const FileDescriptor* file_descriptor,
PyObject* serialized_pb);
typedef struct CFieldDescriptor { // Return the C++ descriptor pointer.
PyObject_HEAD // This function checks the parameter type; on error, return NULL with a Python
// The proto2 descriptor that this object represents.
const google::protobuf::FieldDescriptor* descriptor;
} CFieldDescriptor;
// Wraps operations to the global DescriptorPool which contains information
// about all messages and fields.
//
// There is normally one pool per process. We make it a Python object only
// because it contains many Python references.
// TODO(amauryfa): See whether such objects can appear in reference cycles, and
// consider adding support for the cyclic GC.
//
// "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool
// namespace.
typedef struct PyDescriptorPool {
PyObject_HEAD
google::protobuf::DescriptorPool* pool;
// Make our own mapping to retrieve Python classes from C++ descriptors.
//
// Descriptor pointers stored here are owned by the DescriptorPool above.
// Python references to classes are owned by this PyDescriptorPool.
typedef hash_map<const Descriptor *, PyObject *> ClassesByMessageMap;
ClassesByMessageMap *classes_by_descriptor;
} PyDescriptorPool;
extern PyTypeObject CMessageDescriptor_Type;
extern PyTypeObject CFieldDescriptor_Type;
extern PyTypeObject PyDescriptorPool_Type;
namespace cdescriptor_pool {
// Builds a new DescriptorPool. Normally called only once per process.
PyDescriptorPool* NewDescriptorPool();
// Looks up a message by name.
// Returns a message Descriptor, or NULL if not found.
const google::protobuf::Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
const string& name);
// Registers a new Python class for the given message descriptor.
// Returns the message Descriptor.
// On error, returns NULL with a Python exception set.
const google::protobuf::Descriptor* RegisterMessageClass(
PyDescriptorPool* self, PyObject *message_class, PyObject *descriptor);
// Retrieves the Python class registered with the given message descriptor.
//
// Returns a *borrowed* reference if found, otherwise returns NULL with an
// exception set. // exception set.
PyObject *GetMessageClass(PyDescriptorPool* self, const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj);
const Descriptor *message_descriptor); const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj);
// Looks up a field by name. Returns a CDescriptor corresponding to // Returns the raw C++ pointer.
// the field on success, or NULL on failure. const void* PyDescriptor_AsVoidPtr(PyObject* obj);
// Check that the calling Python code is the global scope of a _pb2.py module.
// This function is used to support the current code generated by the proto
// compiler, which insists on modifying descriptors after they have been
// created.
// //
// Returns a new reference. // stacklevel indicates which Python frame should be the _pb2.py module.
PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name);
// Looks up an extension by name. Returns a CDescriptor corresponding
// to the field on success, or NULL on failure.
// //
// Returns a new reference. // Don't use this function outside descriptor classes.
PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg); bool _CalledFromGeneratedFile(int stacklevel);
} // namespace cdescriptor_pool
PyObject* Python_BuildFile(PyObject* ignored, PyObject* args);
bool InitDescriptor(); bool InitDescriptor();
} // namespace python } // namespace python

1564
python/google/protobuf/pyext/descriptor_containers.cc Normal file
View file

File diff suppressed because it is too large Load diff

95
python/google/protobuf/pyext/descriptor_containers.h Normal file
View file

@ -0,0 +1,95 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Mappings and Sequences of descriptors.
// They implement containers like fields_by_name, EnumDescriptor.values...
// See descriptor_containers.cc for more description.
#include <Python.h>
namespace google {
namespace protobuf {
class Descriptor;
class FileDescriptor;
class EnumDescriptor;
class OneofDescriptor;
namespace python {
// Initialize the various types and objects.
bool InitDescriptorMappingTypes();
// Each function below returns a Mapping, or a Sequence of descriptors.
// They all return a new reference.
namespace message_descriptor {
PyObject* NewMessageFieldsByName(const Descriptor* descriptor);
PyObject* NewMessageFieldsByNumber(const Descriptor* descriptor);
PyObject* NewMessageFieldsSeq(const Descriptor* descriptor);
PyObject* NewMessageNestedTypesSeq(const Descriptor* descriptor);
PyObject* NewMessageNestedTypesByName(const Descriptor* descriptor);
PyObject* NewMessageEnumsByName(const Descriptor* descriptor);
PyObject* NewMessageEnumsSeq(const Descriptor* descriptor);
PyObject* NewMessageEnumValuesByName(const Descriptor* descriptor);
PyObject* NewMessageExtensionsByName(const Descriptor* descriptor);
PyObject* NewMessageExtensionsSeq(const Descriptor* descriptor);
PyObject* NewMessageOneofsByName(const Descriptor* descriptor);
PyObject* NewMessageOneofsSeq(const Descriptor* descriptor);
} // namespace message_descriptor
namespace enum_descriptor {
PyObject* NewEnumValuesByName(const EnumDescriptor* descriptor);
PyObject* NewEnumValuesByNumber(const EnumDescriptor* descriptor);
PyObject* NewEnumValuesSeq(const EnumDescriptor* descriptor);
} // namespace enum_descriptor
namespace oneof_descriptor {
PyObject* NewOneofFieldsSeq(const OneofDescriptor* descriptor);
} // namespace oneof_descriptor
namespace file_descriptor {
PyObject* NewFileMessageTypesByName(const FileDescriptor* descriptor);
PyObject* NewFileEnumTypesByName(const FileDescriptor* descriptor);
PyObject* NewFileExtensionsByName(const FileDescriptor* descriptor);
PyObject* NewFileDependencies(const FileDescriptor* descriptor);
PyObject* NewFilePublicDependencies(const FileDescriptor* descriptor);
} // namespace file_descriptor
} // namespace python
} // namespace protobuf
} // namespace google

58
python/google/protobuf/pyext/descriptor_cpp2_test.py
View file

@ -1,58 +0,0 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for google.protobuf.pyext behavior."""
__author__ = 'anuraag@google.com (Anuraag Agrawal)'
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
# We must set the implementation version above before the google3 imports.
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
# Run all tests from the original module by putting them in our namespace.
# pylint: disable=wildcard-import
from google.protobuf.internal.descriptor_test import *
class ConfirmCppApi2Test(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
if __name__ == '__main__':
basetest.main()

370
python/google/protobuf/pyext/descriptor_pool.cc Normal file
View file

@ -0,0 +1,370 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Implements the DescriptorPool, which collects all descriptors.
#include <Python.h>
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/pyext/descriptor_pool.h>
#include <google/protobuf/pyext/descriptor.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#define C(str) const_cast<char*>(str)
#if PY_MAJOR_VERSION >= 3
#define PyString_FromStringAndSize PyUnicode_FromStringAndSize
#if PY_VERSION_HEX < 0x03030000
#error "Python 3.0 - 3.2 are not supported."
#endif
#define PyString_AsStringAndSize(ob, charpp, sizep) \
(PyUnicode_Check(ob)? \
((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \
PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
#endif
namespace google {
namespace protobuf {
namespace python {
namespace cdescriptor_pool {
PyDescriptorPool* NewDescriptorPool() {
PyDescriptorPool* cdescriptor_pool = PyObject_New(
PyDescriptorPool, &PyDescriptorPool_Type);
if (cdescriptor_pool == NULL) {
return NULL;
}
// Build a DescriptorPool for messages only declared in Python libraries.
// generated_pool() contains all messages linked in C++ libraries, and is used
// as underlay.
cdescriptor_pool->pool = new DescriptorPool(DescriptorPool::generated_pool());
// TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same
// storage.
cdescriptor_pool->classes_by_descriptor =
new PyDescriptorPool::ClassesByMessageMap();
cdescriptor_pool->interned_descriptors =
new hash_map<const void*, PyObject *>();
cdescriptor_pool->descriptor_options =
new hash_map<const void*, PyObject *>();
return cdescriptor_pool;
}
static void Dealloc(PyDescriptorPool* self) {
typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator;
for (iterator it = self->classes_by_descriptor->begin();
it != self->classes_by_descriptor->end(); ++it) {
Py_DECREF(it->second);
}
delete self->classes_by_descriptor;
delete self->interned_descriptors; // its references were borrowed.
for (hash_map<const void*, PyObject*>::iterator it =
self->descriptor_options->begin();
it != self->descriptor_options->end(); ++it) {
Py_DECREF(it->second);
}
delete self->descriptor_options;
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const Descriptor* message_descriptor =
self->pool->FindMessageTypeByName(string(name, name_size));
if (message_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find message %.200s", name);
return NULL;
}
return PyMessageDescriptor_New(message_descriptor);
}
// Add a message class to our database.
const Descriptor* RegisterMessageClass(
PyDescriptorPool* self, PyObject *message_class, PyObject* descriptor) {
ScopedPyObjectPtr full_message_name(
PyObject_GetAttrString(descriptor, "full_name"));
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(full_message_name, &name, &name_size) < 0) {
return NULL;
}
const Descriptor *message_descriptor =
self->pool->FindMessageTypeByName(string(name, name_size));
if (!message_descriptor) {
PyErr_Format(PyExc_TypeError, "Could not find C++ descriptor for '%s'",
name);
return NULL;
}
Py_INCREF(message_class);
typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator;
std::pair<iterator, bool> ret = self->classes_by_descriptor->insert(
std::make_pair(message_descriptor, message_class));
if (!ret.second) {
// Update case: DECREF the previous value.
Py_DECREF(ret.first->second);
ret.first->second = message_class;
}
return message_descriptor;
}
// Retrieve the message class added to our database.
PyObject *GetMessageClass(PyDescriptorPool* self,
const Descriptor *message_descriptor) {
typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator;
iterator ret = self->classes_by_descriptor->find(message_descriptor);
if (ret == self->classes_by_descriptor->end()) {
PyErr_Format(PyExc_TypeError, "No message class registered for '%s'",
message_descriptor->full_name().c_str());
return NULL;
} else {
return ret->second;
}
}
PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const FieldDescriptor* field_descriptor =
self->pool->FindFieldByName(string(name, name_size));
if (field_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s",
name);
return NULL;
}
return PyFieldDescriptor_New(field_descriptor);
}
PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const FieldDescriptor* field_descriptor =
self->pool->FindExtensionByName(string(name, name_size));
if (field_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", name);
return NULL;
}
return PyFieldDescriptor_New(field_descriptor);
}
PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const EnumDescriptor* enum_descriptor =
self->pool->FindEnumTypeByName(string(name, name_size));
if (enum_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find enum %.200s", name);
return NULL;
}
return PyEnumDescriptor_New(enum_descriptor);
}
PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const OneofDescriptor* oneof_descriptor =
self->pool->FindOneofByName(string(name, name_size));
if (oneof_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find oneof %.200s", name);
return NULL;
}
return PyOneofDescriptor_New(oneof_descriptor);
}
static PyMethodDef Methods[] = {
{ C("FindFieldByName"),
(PyCFunction)FindFieldByName,
METH_O,
C("Searches for a field descriptor by full name.") },
{ C("FindExtensionByName"),
(PyCFunction)FindExtensionByName,
METH_O,
C("Searches for extension descriptor by full name.") },
{NULL}
};
} // namespace cdescriptor_pool
PyTypeObject PyDescriptorPool_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
C("google.protobuf.internal."
"_message.DescriptorPool"), // tp_name
sizeof(PyDescriptorPool), // tp_basicsize
0, // tp_itemsize
(destructor)cdescriptor_pool::Dealloc, // tp_dealloc
0, // tp_print
0, // tp_getattr
0, // tp_setattr
0, // tp_compare
0, // tp_repr
0, // tp_as_number
0, // tp_as_sequence
0, // tp_as_mapping
0, // tp_hash
0, // tp_call
0, // tp_str
0, // tp_getattro
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
C("A Descriptor Pool"), // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
0, // tp_weaklistoffset
0, // tp_iter
0, // tp_iternext
cdescriptor_pool::Methods, // tp_methods
0, // tp_members
0, // tp_getset
0, // tp_base
0, // tp_dict
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
0, // tp_init
0, // tp_alloc
0, // tp_new
PyObject_Del, // tp_free
};
// The code below loads new Descriptors from a serialized FileDescriptorProto.
// Collects errors that occur during proto file building to allow them to be
// propagated in the python exception instead of only living in ERROR logs.
class BuildFileErrorCollector : public DescriptorPool::ErrorCollector {
public:
BuildFileErrorCollector() : error_message(""), had_errors(false) {}
void AddError(const string& filename, const string& element_name,
const Message* descriptor, ErrorLocation location,
const string& message) {
// Replicates the logging behavior that happens in the C++ implementation
// when an error collector is not passed in.
if (!had_errors) {
error_message +=
("Invalid proto descriptor for file \"" + filename + "\":\n");
}
// As this only happens on failure and will result in the program not
// running at all, no effort is made to optimize this string manipulation.
error_message += (" " + element_name + ": " + message + "\n");
}
string error_message;
bool had_errors;
};
PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) {
char* message_type;
Py_ssize_t message_len;
if (PyBytes_AsStringAndSize(serialized_pb, &message_type, &message_len) < 0) {
return NULL;
}
FileDescriptorProto file_proto;
if (!file_proto.ParseFromArray(message_type, message_len)) {
PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!");
return NULL;
}
// If the file was already part of a C++ library, all its descriptors are in
// the underlying pool. No need to do anything else.
const FileDescriptor* generated_file =
DescriptorPool::generated_pool()->FindFileByName(file_proto.name());
if (generated_file != NULL) {
return PyFileDescriptor_NewWithPb(generated_file, serialized_pb);
}
BuildFileErrorCollector error_collector;
const FileDescriptor* descriptor =
GetDescriptorPool()->pool->BuildFileCollectingErrors(file_proto,
&error_collector);
if (descriptor == NULL) {
PyErr_Format(PyExc_TypeError,
"Couldn't build proto file into descriptor pool!\n%s",
error_collector.error_message.c_str());
return NULL;
}
return PyFileDescriptor_NewWithPb(descriptor, serialized_pb);
}
static PyDescriptorPool* global_cdescriptor_pool = NULL;
bool InitDescriptorPool() {
if (PyType_Ready(&PyDescriptorPool_Type) < 0)
return false;
global_cdescriptor_pool = cdescriptor_pool::NewDescriptorPool();
if (global_cdescriptor_pool == NULL) {
return false;
}
return true;
}
PyDescriptorPool* GetDescriptorPool() {
return global_cdescriptor_pool;
}
} // namespace python
} // namespace protobuf
} // namespace google

152
python/google/protobuf/pyext/descriptor_pool.h Normal file
View file

@ -0,0 +1,152 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__
#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__
#include <Python.h>
#include <google/protobuf/stubs/hash.h>
#include <google/protobuf/descriptor.h>
namespace google {
namespace protobuf {
namespace python {
// Wraps operations to the global DescriptorPool which contains information
// about all messages and fields.
//
// There is normally one pool per process. We make it a Python object only
// because it contains many Python references.
// TODO(amauryfa): See whether such objects can appear in reference cycles, and
// consider adding support for the cyclic GC.
//
// "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool
// namespace.
typedef struct PyDescriptorPool {
PyObject_HEAD
DescriptorPool* pool;
// Make our own mapping to retrieve Python classes from C++ descriptors.
//
// Descriptor pointers stored here are owned by the DescriptorPool above.
// Python references to classes are owned by this PyDescriptorPool.
typedef hash_map<const Descriptor*, PyObject*> ClassesByMessageMap;
ClassesByMessageMap* classes_by_descriptor;
// Store interned descriptors, so that the same C++ descriptor yields the same
// Python object. Objects are not immortal: this map does not own the
// references, and items are deleted when the last reference to the object is
// released.
// This is enough to support the "is" operator on live objects.
// All descriptors are stored here.
hash_map<const void*, PyObject*>* interned_descriptors;
// Cache the options for any kind of descriptor.
// Descriptor pointers are owned by the DescriptorPool above.
// Python objects are owned by the map.
hash_map<const void*, PyObject*>* descriptor_options;
} PyDescriptorPool;
extern PyTypeObject PyDescriptorPool_Type;
namespace cdescriptor_pool {
// Builds a new DescriptorPool. Normally called only once per process.
PyDescriptorPool* NewDescriptorPool();
// Looks up a message by name.
// Returns a message Descriptor, or NULL if not found.
const Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
const string& name);
// Registers a new Python class for the given message descriptor.
// Returns the message Descriptor.
// On error, returns NULL with a Python exception set.
const Descriptor* RegisterMessageClass(
PyDescriptorPool* self, PyObject* message_class, PyObject* descriptor);
// Retrieves the Python class registered with the given message descriptor.
//
// Returns a *borrowed* reference if found, otherwise returns NULL with an
// exception set.
PyObject* GetMessageClass(PyDescriptorPool* self,
const Descriptor* message_descriptor);
// Looks up a message by name. Returns a PyMessageDescriptor corresponding to
// the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* name);
// Looks up a field by name. Returns a PyFieldDescriptor corresponding to
// the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name);
// Looks up an extension by name. Returns a PyFieldDescriptor corresponding
// to the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg);
// Looks up an enum type by name. Returns a PyEnumDescriptor corresponding
// to the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg);
// Looks up a oneof by name. Returns a COneofDescriptor corresponding
// to the oneof on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg);
} // namespace cdescriptor_pool
// Implement the Python "_BuildFile" method, it takes a serialized
// FileDescriptorProto, and adds it to the C++ DescriptorPool.
// It returns a new FileDescriptor object, or NULL when an exception is raised.
PyObject* Python_BuildFile(PyObject* ignored, PyObject* args);
// Retrieve the global descriptor pool owned by the _message module.
PyDescriptorPool* GetDescriptorPool();
// Initialize objects used by this module.
bool InitDescriptorPool();
} // namespace python
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__

25
python/google/protobuf/pyext/extension_dict.cc
View file

@ -38,6 +38,7 @@
#include <google/protobuf/dynamic_message.h> #include <google/protobuf/dynamic_message.h>
#include <google/protobuf/message.h> #include <google/protobuf/message.h>
#include <google/protobuf/pyext/descriptor.h> #include <google/protobuf/pyext/descriptor.h>
#include <google/protobuf/pyext/descriptor_pool.h>
#include <google/protobuf/pyext/message.h> #include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/repeated_composite_container.h> #include <google/protobuf/pyext/repeated_composite_container.h>
#include <google/protobuf/pyext/repeated_scalar_container.h> #include <google/protobuf/pyext/repeated_scalar_container.h>
@ -48,13 +49,11 @@ namespace google {
namespace protobuf { namespace protobuf {
namespace python { namespace python {
extern google::protobuf::DynamicMessageFactory* global_message_factory;
namespace extension_dict { namespace extension_dict {
// TODO(tibell): Always use self->message for clarity, just like in // TODO(tibell): Always use self->message for clarity, just like in
// RepeatedCompositeContainer. // RepeatedCompositeContainer.
static google::protobuf::Message* GetMessage(ExtensionDict* self) { static Message* GetMessage(ExtensionDict* self) {
if (self->parent != NULL) { if (self->parent != NULL) {
return self->parent->message; return self->parent->message;
} else { } else {
@ -73,10 +72,9 @@ PyObject* len(ExtensionDict* self) {
// TODO(tibell): Use VisitCompositeField. // TODO(tibell): Use VisitCompositeField.
int ReleaseExtension(ExtensionDict* self, int ReleaseExtension(ExtensionDict* self,
PyObject* extension, PyObject* extension,
const google::protobuf::FieldDescriptor* descriptor) { const FieldDescriptor* descriptor) {
if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
if (descriptor->cpp_type() == if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
if (repeated_composite_container::Release( if (repeated_composite_container::Release(
reinterpret_cast<RepeatedCompositeContainer*>( reinterpret_cast<RepeatedCompositeContainer*>(
extension)) < 0) { extension)) < 0) {
@ -89,8 +87,7 @@ int ReleaseExtension(ExtensionDict* self,
return -1; return -1;
} }
} }
} else if (descriptor->cpp_type() == } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
if (cmessage::ReleaseSubMessage( if (cmessage::ReleaseSubMessage(
GetMessage(self), descriptor, GetMessage(self), descriptor,
reinterpret_cast<CMessage*>(extension)) < 0) { reinterpret_cast<CMessage*>(extension)) < 0) {
@ -102,8 +99,7 @@ int ReleaseExtension(ExtensionDict* self,
} }
PyObject* subscript(ExtensionDict* self, PyObject* key) { PyObject* subscript(ExtensionDict* self, PyObject* key) {
const google::protobuf::FieldDescriptor* descriptor = const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key);
cmessage::GetExtensionDescriptor(key);
if (descriptor == NULL) { if (descriptor == NULL) {
return NULL; return NULL;
} }
@ -162,8 +158,7 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
} }
int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
const google::protobuf::FieldDescriptor* descriptor = const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key);
cmessage::GetExtensionDescriptor(key);
if (descriptor == NULL) { if (descriptor == NULL) {
return -1; return -1;
} }
@ -187,7 +182,7 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
} }
PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) { PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
const google::protobuf::FieldDescriptor* descriptor = const FieldDescriptor* descriptor =
cmessage::GetExtensionDescriptor(extension); cmessage::GetExtensionDescriptor(extension);
if (descriptor == NULL) { if (descriptor == NULL) {
return NULL; return NULL;
@ -208,7 +203,7 @@ PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
} }
PyObject* HasExtension(ExtensionDict* self, PyObject* extension) { PyObject* HasExtension(ExtensionDict* self, PyObject* extension) {
const google::protobuf::FieldDescriptor* descriptor = const FieldDescriptor* descriptor =
cmessage::GetExtensionDescriptor(extension); cmessage::GetExtensionDescriptor(extension);
if (descriptor == NULL) { if (descriptor == NULL) {
return NULL; return NULL;

3
python/google/protobuf/pyext/extension_dict.h
View file

@ -41,7 +41,6 @@
#include <google/protobuf/stubs/shared_ptr.h> #include <google/protobuf/stubs/shared_ptr.h>
#endif #endif
namespace google { namespace google {
namespace protobuf { namespace protobuf {
@ -94,7 +93,7 @@ PyObject* len(ExtensionDict* self);
// Returns 0 on success, -1 on failure. // Returns 0 on success, -1 on failure.
int ReleaseExtension(ExtensionDict* self, int ReleaseExtension(ExtensionDict* self,
PyObject* extension, PyObject* extension,
const google::protobuf::FieldDescriptor* descriptor); const FieldDescriptor* descriptor);
// Gets an extension from the dict for the given extension descriptor. // Gets an extension from the dict for the given extension descriptor.
// //

1052
python/google/protobuf/pyext/message.cc
View file

File diff suppressed because it is too large Load diff

50
python/google/protobuf/pyext/message.h
View file

@ -42,7 +42,6 @@
#endif #endif
#include <string> #include <string>
namespace google { namespace google {
namespace protobuf { namespace protobuf {
@ -50,12 +49,12 @@ class Message;
class Reflection; class Reflection;
class FieldDescriptor; class FieldDescriptor;
class Descriptor; class Descriptor;
class DynamicMessageFactory;
using internal::shared_ptr; using internal::shared_ptr;
namespace python { namespace python {
struct PyDescriptorPool;
struct ExtensionDict; struct ExtensionDict;
typedef struct CMessage { typedef struct CMessage {
@ -84,7 +83,7 @@ typedef struct CMessage {
// Used together with the parent's message when making a default message // Used together with the parent's message when making a default message
// instance mutable. // instance mutable.
// The pointer is owned by the global DescriptorPool. // The pointer is owned by the global DescriptorPool.
const google::protobuf::FieldDescriptor* parent_field_descriptor; const FieldDescriptor* parent_field_descriptor;
// Pointer to the C++ Message object for this CMessage. The // Pointer to the C++ Message object for this CMessage. The
// CMessage does not own this pointer. // CMessage does not own this pointer.
@ -115,27 +114,26 @@ namespace cmessage {
// Internal function to create a new empty Message Python object, but with empty // Internal function to create a new empty Message Python object, but with empty
// pointers to the C++ objects. // pointers to the C++ objects.
// The caller must fill self->message, self->owner and eventually self->parent. // The caller must fill self->message, self->owner and eventually self->parent.
CMessage* NewEmptyMessage(PyObject* type, CMessage* NewEmptyMessage(PyObject* type, const Descriptor* descriptor);
const google::protobuf::Descriptor* descriptor);
// Release a submessage from its proto tree, making it a new top-level messgae. // Release a submessage from its proto tree, making it a new top-level messgae.
// A new message will be created if this is a read-only default instance. // A new message will be created if this is a read-only default instance.
// //
// Corresponds to reflection api method ReleaseMessage. // Corresponds to reflection api method ReleaseMessage.
int ReleaseSubMessage(google::protobuf::Message* message, int ReleaseSubMessage(Message* message,
const google::protobuf::FieldDescriptor* field_descriptor, const FieldDescriptor* field_descriptor,
CMessage* child_cmessage); CMessage* child_cmessage);
// Retrieves the C++ descriptor of a Python Extension descriptor. // Retrieves the C++ descriptor of a Python Extension descriptor.
// On error, return NULL with an exception set. // On error, return NULL with an exception set.
const google::protobuf::FieldDescriptor* GetExtensionDescriptor(PyObject* extension); const FieldDescriptor* GetExtensionDescriptor(PyObject* extension);
// Initializes a new CMessage instance for a submessage. Only called once per // Initializes a new CMessage instance for a submessage. Only called once per
// submessage as the result is cached in composite_fields. // submessage as the result is cached in composite_fields.
// //
// Corresponds to reflection api method GetMessage. // Corresponds to reflection api method GetMessage.
PyObject* InternalGetSubMessage( PyObject* InternalGetSubMessage(
CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor); CMessage* self, const FieldDescriptor* field_descriptor);
// Deletes a range of C++ submessages in a repeated field (following a // Deletes a range of C++ submessages in a repeated field (following a
// removal in a RepeatedCompositeContainer). // removal in a RepeatedCompositeContainer).
@ -146,20 +144,20 @@ PyObject* InternalGetSubMessage(
// by slice will be removed from cmessage_list by this function. // by slice will be removed from cmessage_list by this function.
// //
// Corresponds to reflection api method RemoveLast. // Corresponds to reflection api method RemoveLast.
int InternalDeleteRepeatedField(google::protobuf::Message* message, int InternalDeleteRepeatedField(Message* message,
const google::protobuf::FieldDescriptor* field_descriptor, const FieldDescriptor* field_descriptor,
PyObject* slice, PyObject* cmessage_list); PyObject* slice, PyObject* cmessage_list);
// Sets the specified scalar value to the message. // Sets the specified scalar value to the message.
int InternalSetScalar(CMessage* self, int InternalSetScalar(CMessage* self,
const google::protobuf::FieldDescriptor* field_descriptor, const FieldDescriptor* field_descriptor,
PyObject* value); PyObject* value);
// Retrieves the specified scalar value from the message. // Retrieves the specified scalar value from the message.
// //
// Returns a new python reference. // Returns a new python reference.
PyObject* InternalGetScalar(CMessage* self, PyObject* InternalGetScalar(CMessage* self,
const google::protobuf::FieldDescriptor* field_descriptor); const FieldDescriptor* field_descriptor);
// Clears the message, removing all contained data. Extension dictionary and // Clears the message, removing all contained data. Extension dictionary and
// submessages are released first if there are remaining external references. // submessages are released first if there are remaining external references.
@ -175,8 +173,7 @@ PyObject* Clear(CMessage* self);
// //
// Corresponds to reflection api method ClearField. // Corresponds to reflection api method ClearField.
PyObject* ClearFieldByDescriptor( PyObject* ClearFieldByDescriptor(
CMessage* self, CMessage* self, const FieldDescriptor* descriptor);
const google::protobuf::FieldDescriptor* descriptor);
// Clears the data for the given field name. The message is released if there // Clears the data for the given field name. The message is released if there
// are any external references. // are any external references.
@ -189,7 +186,7 @@ PyObject* ClearField(CMessage* self, PyObject* arg);
// //
// Corresponds to reflection api method HasField // Corresponds to reflection api method HasField
PyObject* HasFieldByDescriptor( PyObject* HasFieldByDescriptor(
CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor); CMessage* self, const FieldDescriptor* field_descriptor);
// Checks if the message has the named field. // Checks if the message has the named field.
// //
@ -220,18 +217,16 @@ int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner);
int AssureWritable(CMessage* self); int AssureWritable(CMessage* self);
DynamicMessageFactory* GetMessageFactory();
} // namespace cmessage } // namespace cmessage
// Retrieve the global descriptor pool owned by the _message module.
PyDescriptorPool* GetDescriptorPool();
/* Is 64bit */ /* Is 64bit */
#define IS_64BIT (SIZEOF_LONG == 8) #define IS_64BIT (SIZEOF_LONG == 8)
#define FIELD_IS_REPEATED(field_descriptor) \ #define FIELD_IS_REPEATED(field_descriptor) \
((field_descriptor)->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) ((field_descriptor)->label() == FieldDescriptor::LABEL_REPEATED)
#define GOOGLE_CHECK_GET_INT32(arg, value, err) \ #define GOOGLE_CHECK_GET_INT32(arg, value, err) \
int32 value; \ int32 value; \
@ -294,18 +289,17 @@ bool CheckAndGetDouble(PyObject* arg, double* value);
bool CheckAndGetFloat(PyObject* arg, float* value); bool CheckAndGetFloat(PyObject* arg, float* value);
bool CheckAndGetBool(PyObject* arg, bool* value); bool CheckAndGetBool(PyObject* arg, bool* value);
bool CheckAndSetString( bool CheckAndSetString(
PyObject* arg, google::protobuf::Message* message, PyObject* arg, Message* message,
const google::protobuf::FieldDescriptor* descriptor, const FieldDescriptor* descriptor,
const google::protobuf::Reflection* reflection, const Reflection* reflection,
bool append, bool append,
int index); int index);
PyObject* ToStringObject( PyObject* ToStringObject(const FieldDescriptor* descriptor, string value);
const google::protobuf::FieldDescriptor* descriptor, string value);
// Check if the passed field descriptor belongs to the given message. // Check if the passed field descriptor belongs to the given message.
// If not, return false and set a Python exception (a KeyError) // If not, return false and set a Python exception (a KeyError)
bool CheckFieldBelongsToMessage(const google::protobuf::FieldDescriptor* field_descriptor, bool CheckFieldBelongsToMessage(const FieldDescriptor* field_descriptor,
const google::protobuf::Message* message); const Message* message);
extern PyObject* PickleError_class; extern PyObject* PickleError_class;

56
python/google/protobuf/pyext/message_factory_cpp2_test.py
View file

@ -1,56 +0,0 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for google.protobuf.message_factory."""
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
# We must set the implementation version above before the google3 imports.
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
# Run all tests from the original module by putting them in our namespace.
# pylint: disable=wildcard-import
from google.protobuf.internal.message_factory_test import *
class ConfirmCppApi2Test(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
if __name__ == '__main__':
basetest.main()

94
python/google/protobuf/pyext/reflection_cpp2_generated_test.py
View file

@ -1,94 +0,0 @@
#! /usr/bin/python
# -*- coding: utf-8 -*-
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Unittest for reflection.py, which tests the generated C++ implementation."""
__author__ = 'jasonh@google.com (Jason Hsueh)'
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
from google.apputils import basetest
from google.protobuf.internal import api_implementation
from google.protobuf.internal import more_extensions_dynamic_pb2
from google.protobuf.internal import more_extensions_pb2
from google.protobuf.internal.reflection_test import *
class ReflectionCppTest(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
def testExtensionOfGeneratedTypeInDynamicFile(self):
"""Tests that a file built dynamically can extend a generated C++ type.
The C++ implementation uses a DescriptorPool that has the generated
DescriptorPool as an underlay. Typically, a type can only find
extensions in its own pool. With the python C-extension, the generated C++
extendee may be available, but not the extension. This tests that the
C-extension implements the correct special handling to make such extensions
available.
"""
pb1 = more_extensions_pb2.ExtendedMessage()
# Test that basic accessors work.
self.assertFalse(
pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension))
self.assertFalse(
pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension))
pb1.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension] = 17
pb1.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a = 24
self.assertTrue(
pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension))
self.assertTrue(
pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension))
# Now serialize the data and parse to a new message.
pb2 = more_extensions_pb2.ExtendedMessage()
pb2.MergeFromString(pb1.SerializeToString())
self.assertTrue(
pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension))
self.assertTrue(
pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension))
self.assertEqual(
17, pb2.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension])
self.assertEqual(
24,
pb2.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a)
if __name__ == '__main__':
basetest.main()

53
python/google/protobuf/pyext/repeated_composite_container.cc
View file

@ -56,8 +56,6 @@ namespace google {
namespace protobuf { namespace protobuf {
namespace python { namespace python {
extern google::protobuf::DynamicMessageFactory* global_message_factory;
namespace repeated_composite_container { namespace repeated_composite_container {
// TODO(tibell): We might also want to check: // TODO(tibell): We might also want to check:
@ -120,9 +118,9 @@ static int InternalQuickSort(RepeatedCompositeContainer* self,
GOOGLE_CHECK_ATTACHED(self); GOOGLE_CHECK_ATTACHED(self);
google::protobuf::Message* message = self->message; Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection(); const Reflection* reflection = message->GetReflection();
const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor; const FieldDescriptor* descriptor = self->parent_field_descriptor;
Py_ssize_t left; Py_ssize_t left;
Py_ssize_t right; Py_ssize_t right;
@ -199,7 +197,7 @@ static int InternalQuickSort(RepeatedCompositeContainer* self,
// len() // len()
static Py_ssize_t Length(RepeatedCompositeContainer* self) { static Py_ssize_t Length(RepeatedCompositeContainer* self) {
google::protobuf::Message* message = self->message; Message* message = self->message;
if (message != NULL) { if (message != NULL) {
return message->GetReflection()->FieldSize(*message, return message->GetReflection()->FieldSize(*message,
self->parent_field_descriptor); self->parent_field_descriptor);
@ -221,8 +219,8 @@ static int UpdateChildMessages(RepeatedCompositeContainer* self) {
// be removed in such a way so there's no need to worry about that. // be removed in such a way so there's no need to worry about that.
Py_ssize_t message_length = Length(self); Py_ssize_t message_length = Length(self);
Py_ssize_t child_length = PyList_GET_SIZE(self->child_messages); Py_ssize_t child_length = PyList_GET_SIZE(self->child_messages);
google::protobuf::Message* message = self->message; Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection(); const Reflection* reflection = message->GetReflection();
for (Py_ssize_t i = child_length; i < message_length; ++i) { for (Py_ssize_t i = child_length; i < message_length; ++i) {
const Message& sub_message = reflection->GetRepeatedMessage( const Message& sub_message = reflection->GetRepeatedMessage(
*(self->message), self->parent_field_descriptor, i); *(self->message), self->parent_field_descriptor, i);
@ -233,7 +231,7 @@ static int UpdateChildMessages(RepeatedCompositeContainer* self) {
return -1; return -1;
} }
cmsg->owner = self->owner; cmsg->owner = self->owner;
cmsg->message = const_cast<google::protobuf::Message*>(&sub_message); cmsg->message = const_cast<Message*>(&sub_message);
cmsg->parent = self->parent; cmsg->parent = self->parent;
if (PyList_Append(self->child_messages, py_cmsg) < 0) { if (PyList_Append(self->child_messages, py_cmsg) < 0) {
return -1; return -1;
@ -255,8 +253,8 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self,
} }
if (cmessage::AssureWritable(self->parent) == -1) if (cmessage::AssureWritable(self->parent) == -1)
return NULL; return NULL;
google::protobuf::Message* message = self->message; Message* message = self->message;
google::protobuf::Message* sub_message = Message* sub_message =
message->GetReflection()->AddMessage(message, message->GetReflection()->AddMessage(message,
self->parent_field_descriptor); self->parent_field_descriptor);
CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init, CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
@ -482,9 +480,9 @@ static PyObject* SortAttached(RepeatedCompositeContainer* self,
// Finally reverse the result if requested. // Finally reverse the result if requested.
if (reverse) { if (reverse) {
google::protobuf::Message* message = self->message; Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection(); const Reflection* reflection = message->GetReflection();
const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor; const FieldDescriptor* descriptor = self->parent_field_descriptor;
// Reverse the Message array. // Reverse the Message array.
for (int i = 0; i < length / 2; ++i) for (int i = 0; i < length / 2; ++i)
@ -554,6 +552,26 @@ static PyObject* Item(RepeatedCompositeContainer* self, Py_ssize_t index) {
return item; return item;
} }
static PyObject* Pop(RepeatedCompositeContainer* self,
PyObject* args) {
Py_ssize_t index = -1;
if (!PyArg_ParseTuple(args, "|n", &index)) {
return NULL;
}
PyObject* item = Item(self, index);
if (item == NULL) {
PyErr_Format(PyExc_IndexError,
"list index (%zd) out of range",
index);
return NULL;
}
ScopedPyObjectPtr py_index(PyLong_FromSsize_t(index));
if (AssignSubscript(self, py_index, NULL) < 0) {
return NULL;
}
return item;
}
// The caller takes ownership of the returned Message. // The caller takes ownership of the returned Message.
Message* ReleaseLast(const FieldDescriptor* field, Message* ReleaseLast(const FieldDescriptor* field,
const Descriptor* type, const Descriptor* type,
@ -571,7 +589,8 @@ Message* ReleaseLast(const FieldDescriptor* field,
// the latter points to the default instance via a const_cast<>, so we // the latter points to the default instance via a const_cast<>, so we
// have to reset it to a new mutable object since we are taking ownership. // have to reset it to a new mutable object since we are taking ownership.
if (released_message == NULL) { if (released_message == NULL) {
const Message* prototype = global_message_factory->GetPrototype(type); const Message* prototype =
cmessage::GetMessageFactory()->GetPrototype(type);
GOOGLE_CHECK_NOTNULL(prototype); GOOGLE_CHECK_NOTNULL(prototype);
return prototype->New(); return prototype->New();
} else { } else {
@ -646,7 +665,7 @@ int SetOwner(RepeatedCompositeContainer* self,
// The private constructor of RepeatedCompositeContainer objects. // The private constructor of RepeatedCompositeContainer objects.
PyObject *NewContainer( PyObject *NewContainer(
CMessage* parent, CMessage* parent,
const google::protobuf::FieldDescriptor* parent_field_descriptor, const FieldDescriptor* parent_field_descriptor,
PyObject *concrete_class) { PyObject *concrete_class) {
if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
return NULL; return NULL;
@ -698,6 +717,8 @@ static PyMethodDef Methods[] = {
"Adds an object to the repeated container." }, "Adds an object to the repeated container." },
{ "extend", (PyCFunction) Extend, METH_O, { "extend", (PyCFunction) Extend, METH_O,
"Adds objects to the repeated container." }, "Adds objects to the repeated container." },
{ "pop", (PyCFunction)Pop, METH_VARARGS,
"Removes an object from the repeated container and returns it." },
{ "remove", (PyCFunction) Remove, METH_O, { "remove", (PyCFunction) Remove, METH_O,
"Removes an object from the repeated container." }, "Removes an object from the repeated container." },
{ "sort", (PyCFunction) Sort, METH_VARARGS | METH_KEYWORDS, { "sort", (PyCFunction) Sort, METH_VARARGS | METH_KEYWORDS,

8
python/google/protobuf/pyext/repeated_composite_container.h
View file

@ -43,7 +43,6 @@
#include <string> #include <string>
#include <vector> #include <vector>
namespace google { namespace google {
namespace protobuf { namespace protobuf {
@ -82,7 +81,7 @@ typedef struct RepeatedCompositeContainer {
// A descriptor used to modify the underlying 'message'. // A descriptor used to modify the underlying 'message'.
// The pointer is owned by the global DescriptorPool. // The pointer is owned by the global DescriptorPool.
const google::protobuf::FieldDescriptor* parent_field_descriptor; const FieldDescriptor* parent_field_descriptor;
// Pointer to the C++ Message that contains this container. The // Pointer to the C++ Message that contains this container. The
// RepeatedCompositeContainer does not own this pointer. // RepeatedCompositeContainer does not own this pointer.
@ -106,7 +105,7 @@ namespace repeated_composite_container {
// field descriptor. // field descriptor.
PyObject *NewContainer( PyObject *NewContainer(
CMessage* parent, CMessage* parent,
const google::protobuf::FieldDescriptor* parent_field_descriptor, const FieldDescriptor* parent_field_descriptor,
PyObject *concrete_class); PyObject *concrete_class);
// Returns the number of items in this repeated composite container. // Returns the number of items in this repeated composite container.
@ -150,8 +149,7 @@ int AssignSubscript(RepeatedCompositeContainer* self,
// Releases the messages in the container to the given message. // Releases the messages in the container to the given message.
// //
// Returns 0 on success, -1 on failure. // Returns 0 on success, -1 on failure.
int ReleaseToMessage(RepeatedCompositeContainer* self, int ReleaseToMessage(RepeatedCompositeContainer* self, Message* new_message);
google::protobuf::Message* new_message);
// Releases the messages in the container to a new message. // Releases the messages in the container to a new message.
// //

183
python/google/protobuf/pyext/repeated_scalar_container.cc
View file

@ -60,8 +60,6 @@ namespace google {
namespace protobuf { namespace protobuf {
namespace python { namespace python {
extern google::protobuf::DynamicMessageFactory* global_message_factory;
namespace repeated_scalar_container { namespace repeated_scalar_container {
static int InternalAssignRepeatedField( static int InternalAssignRepeatedField(
@ -78,7 +76,7 @@ static int InternalAssignRepeatedField(
} }
static Py_ssize_t Len(RepeatedScalarContainer* self) { static Py_ssize_t Len(RepeatedScalarContainer* self) {
google::protobuf::Message* message = self->message; Message* message = self->message;
return message->GetReflection()->FieldSize(*message, return message->GetReflection()->FieldSize(*message,
self->parent_field_descriptor); self->parent_field_descriptor);
} }
@ -87,11 +85,10 @@ static int AssignItem(RepeatedScalarContainer* self,
Py_ssize_t index, Py_ssize_t index,
PyObject* arg) { PyObject* arg) {
cmessage::AssureWritable(self->parent); cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message; Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor = const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
self->parent_field_descriptor;
const google::protobuf::Reflection* reflection = message->GetReflection(); const Reflection* reflection = message->GetReflection();
int field_size = reflection->FieldSize(*message, field_descriptor); int field_size = reflection->FieldSize(*message, field_descriptor);
if (index < 0) { if (index < 0) {
index = field_size + index; index = field_size + index;
@ -115,64 +112,68 @@ static int AssignItem(RepeatedScalarContainer* self,
} }
switch (field_descriptor->cpp_type()) { switch (field_descriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { case FieldDescriptor::CPPTYPE_INT32: {
GOOGLE_CHECK_GET_INT32(arg, value, -1); GOOGLE_CHECK_GET_INT32(arg, value, -1);
reflection->SetRepeatedInt32(message, field_descriptor, index, value); reflection->SetRepeatedInt32(message, field_descriptor, index, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { case FieldDescriptor::CPPTYPE_INT64: {
GOOGLE_CHECK_GET_INT64(arg, value, -1); GOOGLE_CHECK_GET_INT64(arg, value, -1);
reflection->SetRepeatedInt64(message, field_descriptor, index, value); reflection->SetRepeatedInt64(message, field_descriptor, index, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { case FieldDescriptor::CPPTYPE_UINT32: {
GOOGLE_CHECK_GET_UINT32(arg, value, -1); GOOGLE_CHECK_GET_UINT32(arg, value, -1);
reflection->SetRepeatedUInt32(message, field_descriptor, index, value); reflection->SetRepeatedUInt32(message, field_descriptor, index, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { case FieldDescriptor::CPPTYPE_UINT64: {
GOOGLE_CHECK_GET_UINT64(arg, value, -1); GOOGLE_CHECK_GET_UINT64(arg, value, -1);
reflection->SetRepeatedUInt64(message, field_descriptor, index, value); reflection->SetRepeatedUInt64(message, field_descriptor, index, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { case FieldDescriptor::CPPTYPE_FLOAT: {
GOOGLE_CHECK_GET_FLOAT(arg, value, -1); GOOGLE_CHECK_GET_FLOAT(arg, value, -1);
reflection->SetRepeatedFloat(message, field_descriptor, index, value); reflection->SetRepeatedFloat(message, field_descriptor, index, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { case FieldDescriptor::CPPTYPE_DOUBLE: {
GOOGLE_CHECK_GET_DOUBLE(arg, value, -1); GOOGLE_CHECK_GET_DOUBLE(arg, value, -1);
reflection->SetRepeatedDouble(message, field_descriptor, index, value); reflection->SetRepeatedDouble(message, field_descriptor, index, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { case FieldDescriptor::CPPTYPE_BOOL: {
GOOGLE_CHECK_GET_BOOL(arg, value, -1); GOOGLE_CHECK_GET_BOOL(arg, value, -1);
reflection->SetRepeatedBool(message, field_descriptor, index, value); reflection->SetRepeatedBool(message, field_descriptor, index, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { case FieldDescriptor::CPPTYPE_STRING: {
if (!CheckAndSetString( if (!CheckAndSetString(
arg, message, field_descriptor, reflection, false, index)) { arg, message, field_descriptor, reflection, false, index)) {
return -1; return -1;
} }
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { case FieldDescriptor::CPPTYPE_ENUM: {
GOOGLE_CHECK_GET_INT32(arg, value, -1); GOOGLE_CHECK_GET_INT32(arg, value, -1);
const google::protobuf::EnumDescriptor* enum_descriptor = if (reflection->SupportsUnknownEnumValues()) {
field_descriptor->enum_type(); reflection->SetRepeatedEnumValue(message, field_descriptor, index,
const google::protobuf::EnumValueDescriptor* enum_value = value);
enum_descriptor->FindValueByNumber(value);
if (enum_value != NULL) {
reflection->SetRepeatedEnum(message, field_descriptor, index,
enum_value);
} else { } else {
ScopedPyObjectPtr s(PyObject_Str(arg)); const EnumDescriptor* enum_descriptor = field_descriptor->enum_type();
if (s != NULL) { const EnumValueDescriptor* enum_value =
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s", enum_descriptor->FindValueByNumber(value);
PyString_AsString(s)); if (enum_value != NULL) {
reflection->SetRepeatedEnum(message, field_descriptor, index,
enum_value);
} else {
ScopedPyObjectPtr s(PyObject_Str(arg));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
PyString_AsString(s));
}
return -1;
} }
return -1;
} }
break; break;
} }
@ -186,10 +187,9 @@ static int AssignItem(RepeatedScalarContainer* self,
} }
static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) { static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
google::protobuf::Message* message = self->message; Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor = const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
self->parent_field_descriptor; const Reflection* reflection = message->GetReflection();
const google::protobuf::Reflection* reflection = message->GetReflection();
int field_size = reflection->FieldSize(*message, field_descriptor); int field_size = reflection->FieldSize(*message, field_descriptor);
if (index < 0) { if (index < 0) {
@ -197,80 +197,80 @@ static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
} }
if (index < 0 || index >= field_size) { if (index < 0 || index >= field_size) {
PyErr_Format(PyExc_IndexError, PyErr_Format(PyExc_IndexError,
"list assignment index (%d) out of range", "list index (%zd) out of range",
static_cast<int>(index)); index);
return NULL; return NULL;
} }
PyObject* result = NULL; PyObject* result = NULL;
switch (field_descriptor->cpp_type()) { switch (field_descriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { case FieldDescriptor::CPPTYPE_INT32: {
int32 value = reflection->GetRepeatedInt32( int32 value = reflection->GetRepeatedInt32(
*message, field_descriptor, index); *message, field_descriptor, index);
result = PyInt_FromLong(value); result = PyInt_FromLong(value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { case FieldDescriptor::CPPTYPE_INT64: {
int64 value = reflection->GetRepeatedInt64( int64 value = reflection->GetRepeatedInt64(
*message, field_descriptor, index); *message, field_descriptor, index);
result = PyLong_FromLongLong(value); result = PyLong_FromLongLong(value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { case FieldDescriptor::CPPTYPE_UINT32: {
uint32 value = reflection->GetRepeatedUInt32( uint32 value = reflection->GetRepeatedUInt32(
*message, field_descriptor, index); *message, field_descriptor, index);
result = PyLong_FromLongLong(value); result = PyLong_FromLongLong(value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { case FieldDescriptor::CPPTYPE_UINT64: {
uint64 value = reflection->GetRepeatedUInt64( uint64 value = reflection->GetRepeatedUInt64(
*message, field_descriptor, index); *message, field_descriptor, index);
result = PyLong_FromUnsignedLongLong(value); result = PyLong_FromUnsignedLongLong(value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { case FieldDescriptor::CPPTYPE_FLOAT: {
float value = reflection->GetRepeatedFloat( float value = reflection->GetRepeatedFloat(
*message, field_descriptor, index); *message, field_descriptor, index);
result = PyFloat_FromDouble(value); result = PyFloat_FromDouble(value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { case FieldDescriptor::CPPTYPE_DOUBLE: {
double value = reflection->GetRepeatedDouble( double value = reflection->GetRepeatedDouble(
*message, field_descriptor, index); *message, field_descriptor, index);
result = PyFloat_FromDouble(value); result = PyFloat_FromDouble(value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { case FieldDescriptor::CPPTYPE_BOOL: {
bool value = reflection->GetRepeatedBool( bool value = reflection->GetRepeatedBool(
*message, field_descriptor, index); *message, field_descriptor, index);
result = PyBool_FromLong(value ? 1 : 0); result = PyBool_FromLong(value ? 1 : 0);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { case FieldDescriptor::CPPTYPE_ENUM: {
const google::protobuf::EnumValueDescriptor* enum_value = const EnumValueDescriptor* enum_value =
message->GetReflection()->GetRepeatedEnum( message->GetReflection()->GetRepeatedEnum(
*message, field_descriptor, index); *message, field_descriptor, index);
result = PyInt_FromLong(enum_value->number()); result = PyInt_FromLong(enum_value->number());
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { case FieldDescriptor::CPPTYPE_STRING: {
string value = reflection->GetRepeatedString( string value = reflection->GetRepeatedString(
*message, field_descriptor, index); *message, field_descriptor, index);
result = ToStringObject(field_descriptor, value); result = ToStringObject(field_descriptor, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { case FieldDescriptor::CPPTYPE_MESSAGE: {
PyObject* py_cmsg = PyObject_CallObject(reinterpret_cast<PyObject*>( PyObject* py_cmsg = PyObject_CallObject(reinterpret_cast<PyObject*>(
&CMessage_Type), NULL); &CMessage_Type), NULL);
if (py_cmsg == NULL) { if (py_cmsg == NULL) {
return NULL; return NULL;
} }
CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg); CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
const google::protobuf::Message& msg = reflection->GetRepeatedMessage( const Message& msg = reflection->GetRepeatedMessage(
*message, field_descriptor, index); *message, field_descriptor, index);
cmsg->owner = self->owner; cmsg->owner = self->owner;
cmsg->parent = self->parent; cmsg->parent = self->parent;
cmsg->message = const_cast<google::protobuf::Message*>(&msg); cmsg->message = const_cast<Message*>(&msg);
cmsg->read_only = false; cmsg->read_only = false;
result = reinterpret_cast<PyObject*>(py_cmsg); result = reinterpret_cast<PyObject*>(py_cmsg);
break; break;
@ -351,69 +351,71 @@ static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) {
PyObject* Append(RepeatedScalarContainer* self, PyObject* item) { PyObject* Append(RepeatedScalarContainer* self, PyObject* item) {
cmessage::AssureWritable(self->parent); cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message; Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor = const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
self->parent_field_descriptor;
const google::protobuf::Reflection* reflection = message->GetReflection(); const Reflection* reflection = message->GetReflection();
switch (field_descriptor->cpp_type()) { switch (field_descriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { case FieldDescriptor::CPPTYPE_INT32: {
GOOGLE_CHECK_GET_INT32(item, value, NULL); GOOGLE_CHECK_GET_INT32(item, value, NULL);
reflection->AddInt32(message, field_descriptor, value); reflection->AddInt32(message, field_descriptor, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { case FieldDescriptor::CPPTYPE_INT64: {
GOOGLE_CHECK_GET_INT64(item, value, NULL); GOOGLE_CHECK_GET_INT64(item, value, NULL);
reflection->AddInt64(message, field_descriptor, value); reflection->AddInt64(message, field_descriptor, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { case FieldDescriptor::CPPTYPE_UINT32: {
GOOGLE_CHECK_GET_UINT32(item, value, NULL); GOOGLE_CHECK_GET_UINT32(item, value, NULL);
reflection->AddUInt32(message, field_descriptor, value); reflection->AddUInt32(message, field_descriptor, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { case FieldDescriptor::CPPTYPE_UINT64: {
GOOGLE_CHECK_GET_UINT64(item, value, NULL); GOOGLE_CHECK_GET_UINT64(item, value, NULL);
reflection->AddUInt64(message, field_descriptor, value); reflection->AddUInt64(message, field_descriptor, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { case FieldDescriptor::CPPTYPE_FLOAT: {
GOOGLE_CHECK_GET_FLOAT(item, value, NULL); GOOGLE_CHECK_GET_FLOAT(item, value, NULL);
reflection->AddFloat(message, field_descriptor, value); reflection->AddFloat(message, field_descriptor, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { case FieldDescriptor::CPPTYPE_DOUBLE: {
GOOGLE_CHECK_GET_DOUBLE(item, value, NULL); GOOGLE_CHECK_GET_DOUBLE(item, value, NULL);
reflection->AddDouble(message, field_descriptor, value); reflection->AddDouble(message, field_descriptor, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { case FieldDescriptor::CPPTYPE_BOOL: {
GOOGLE_CHECK_GET_BOOL(item, value, NULL); GOOGLE_CHECK_GET_BOOL(item, value, NULL);
reflection->AddBool(message, field_descriptor, value); reflection->AddBool(message, field_descriptor, value);
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { case FieldDescriptor::CPPTYPE_STRING: {
if (!CheckAndSetString( if (!CheckAndSetString(
item, message, field_descriptor, reflection, true, -1)) { item, message, field_descriptor, reflection, true, -1)) {
return NULL; return NULL;
} }
break; break;
} }
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { case FieldDescriptor::CPPTYPE_ENUM: {
GOOGLE_CHECK_GET_INT32(item, value, NULL); GOOGLE_CHECK_GET_INT32(item, value, NULL);
const google::protobuf::EnumDescriptor* enum_descriptor = if (reflection->SupportsUnknownEnumValues()) {
field_descriptor->enum_type(); reflection->AddEnumValue(message, field_descriptor, value);
const google::protobuf::EnumValueDescriptor* enum_value =
enum_descriptor->FindValueByNumber(value);
if (enum_value != NULL) {
reflection->AddEnum(message, field_descriptor, enum_value);
} else { } else {
ScopedPyObjectPtr s(PyObject_Str(item)); const EnumDescriptor* enum_descriptor = field_descriptor->enum_type();
if (s != NULL) { const EnumValueDescriptor* enum_value =
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s", enum_descriptor->FindValueByNumber(value);
PyString_AsString(s)); if (enum_value != NULL) {
reflection->AddEnum(message, field_descriptor, enum_value);
} else {
ScopedPyObjectPtr s(PyObject_Str(item));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
PyString_AsString(s));
}
return NULL;
} }
return NULL;
} }
break; break;
} }
@ -438,8 +440,8 @@ static int AssSubscript(RepeatedScalarContainer* self,
bool create_list = false; bool create_list = false;
cmessage::AssureWritable(self->parent); cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message; Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor = const FieldDescriptor* field_descriptor =
self->parent_field_descriptor; self->parent_field_descriptor;
#if PY_MAJOR_VERSION < 3 #if PY_MAJOR_VERSION < 3
@ -450,7 +452,7 @@ static int AssSubscript(RepeatedScalarContainer* self,
if (PyLong_Check(slice)) { if (PyLong_Check(slice)) {
from = to = PyLong_AsLong(slice); from = to = PyLong_AsLong(slice);
} else if (PySlice_Check(slice)) { } else if (PySlice_Check(slice)) {
const google::protobuf::Reflection* reflection = message->GetReflection(); const Reflection* reflection = message->GetReflection();
length = reflection->FieldSize(*message, field_descriptor); length = reflection->FieldSize(*message, field_descriptor);
#if PY_MAJOR_VERSION >= 3 #if PY_MAJOR_VERSION >= 3
if (PySlice_GetIndicesEx(slice, if (PySlice_GetIndicesEx(slice,
@ -492,9 +494,15 @@ static int AssSubscript(RepeatedScalarContainer* self,
PyObject* Extend(RepeatedScalarContainer* self, PyObject* value) { PyObject* Extend(RepeatedScalarContainer* self, PyObject* value) {
cmessage::AssureWritable(self->parent); cmessage::AssureWritable(self->parent);
if (PyObject_Not(value)) {
// TODO(ptucker): Deprecate this behavior. b/18413862
if (value == Py_None) {
Py_RETURN_NONE; Py_RETURN_NONE;
} }
if ((Py_TYPE(value)->tp_as_sequence == NULL) && PyObject_Not(value)) {
Py_RETURN_NONE;
}
ScopedPyObjectPtr iter(PyObject_GetIter(value)); ScopedPyObjectPtr iter(PyObject_GetIter(value));
if (iter == NULL) { if (iter == NULL) {
PyErr_SetString(PyExc_TypeError, "Value must be iterable"); PyErr_SetString(PyExc_TypeError, "Value must be iterable");
@ -627,9 +635,28 @@ static PyObject* Sort(RepeatedScalarContainer* self,
Py_RETURN_NONE; Py_RETURN_NONE;
} }
static PyObject* Pop(RepeatedScalarContainer* self,
PyObject* args) {
Py_ssize_t index = -1;
if (!PyArg_ParseTuple(args, "|n", &index)) {
return NULL;
}
PyObject* item = Item(self, index);
if (item == NULL) {
PyErr_Format(PyExc_IndexError,
"list index (%zd) out of range",
index);
return NULL;
}
if (AssignItem(self, index, NULL) < 0) {
return NULL;
}
return item;
}
// The private constructor of RepeatedScalarContainer objects. // The private constructor of RepeatedScalarContainer objects.
PyObject *NewContainer( PyObject *NewContainer(
CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) { CMessage* parent, const FieldDescriptor* parent_field_descriptor) {
if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
return NULL; return NULL;
} }
@ -663,7 +690,7 @@ static int InitializeAndCopyToParentContainer(
if (values == NULL) { if (values == NULL) {
return -1; return -1;
} }
google::protobuf::Message* new_message = global_message_factory->GetPrototype( Message* new_message = cmessage::GetMessageFactory()->GetPrototype(
from->message->GetDescriptor())->New(); from->message->GetDescriptor())->New();
to->parent = NULL; to->parent = NULL;
to->parent_field_descriptor = from->parent_field_descriptor; to->parent_field_descriptor = from->parent_field_descriptor;
@ -729,6 +756,8 @@ static PyMethodDef Methods[] = {
"Appends objects to the repeated container." }, "Appends objects to the repeated container." },
{ "insert", (PyCFunction)Insert, METH_VARARGS, { "insert", (PyCFunction)Insert, METH_VARARGS,
"Appends objects to the repeated container." }, "Appends objects to the repeated container." },
{ "pop", (PyCFunction)Pop, METH_VARARGS,
"Removes an object from the repeated container and returns it." },
{ "remove", (PyCFunction)Remove, METH_O, { "remove", (PyCFunction)Remove, METH_O,
"Removes an object from the repeated container." }, "Removes an object from the repeated container." },
{ "sort", (PyCFunction)Sort, METH_VARARGS | METH_KEYWORDS, { "sort", (PyCFunction)Sort, METH_VARARGS | METH_KEYWORDS,

4
python/google/protobuf/pyext/repeated_scalar_container.h
View file

@ -77,7 +77,7 @@ typedef struct RepeatedScalarContainer {
// field. Used together with the parent's message when making a // field. Used together with the parent's message when making a
// default message instance mutable. // default message instance mutable.
// The pointer is owned by the global DescriptorPool. // The pointer is owned by the global DescriptorPool.
const google::protobuf::FieldDescriptor* parent_field_descriptor; const FieldDescriptor* parent_field_descriptor;
} RepeatedScalarContainer; } RepeatedScalarContainer;
extern PyTypeObject RepeatedScalarContainer_Type; extern PyTypeObject RepeatedScalarContainer_Type;
@ -87,7 +87,7 @@ namespace repeated_scalar_container {
// Builds a RepeatedScalarContainer object, from a parent message and a // Builds a RepeatedScalarContainer object, from a parent message and a
// field descriptor. // field descriptor.
extern PyObject *NewContainer( extern PyObject *NewContainer(
CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor); CMessage* parent, const FieldDescriptor* parent_field_descriptor);
// Appends the scalar 'item' to the end of the container 'self'. // Appends the scalar 'item' to the end of the container 'self'.
// //

18
python/google/protobuf/reflection.py
View file

@ -56,18 +56,12 @@ _FieldDescriptor = descriptor_mod.FieldDescriptor
if api_implementation.Type() == 'cpp': if api_implementation.Type() == 'cpp':
if api_implementation.Version() == 2: from google.protobuf.pyext import cpp_message as message_impl
from google.protobuf.pyext import cpp_message
_NewMessage = cpp_message.NewMessage
_InitMessage = cpp_message.InitMessage
else:
from google.protobuf.internal import cpp_message
_NewMessage = cpp_message.NewMessage
_InitMessage = cpp_message.InitMessage
else: else:
from google.protobuf.internal import python_message from google.protobuf.internal import python_message as message_impl
_NewMessage = python_message.NewMessage
_InitMessage = python_message.InitMessage _NewMessage = message_impl.NewMessage
_InitMessage = message_impl.InitMessage
class GeneratedProtocolMessageType(type): class GeneratedProtocolMessageType(type):
@ -127,7 +121,6 @@ class GeneratedProtocolMessageType(type):
superclass = super(GeneratedProtocolMessageType, cls) superclass = super(GeneratedProtocolMessageType, cls)
new_class = superclass.__new__(cls, name, bases, dictionary) new_class = superclass.__new__(cls, name, bases, dictionary)
setattr(descriptor, '_concrete_class', new_class)
return new_class return new_class
def __init__(cls, name, bases, dictionary): def __init__(cls, name, bases, dictionary):
@ -151,6 +144,7 @@ class GeneratedProtocolMessageType(type):
_InitMessage(descriptor, cls) _InitMessage(descriptor, cls)
superclass = super(GeneratedProtocolMessageType, cls) superclass = super(GeneratedProtocolMessageType, cls)
superclass.__init__(name, bases, dictionary) superclass.__init__(name, bases, dictionary)
setattr(descriptor, '_concrete_class', cls)
def ParseMessage(descriptor, byte_str): def ParseMessage(descriptor, byte_str):

5
python/google/protobuf/text_format.py
View file

@ -319,6 +319,11 @@ def _MergeField(tokenizer, message, allow_multiple_scalars):
ParseError: In case of ASCII parsing problems. ParseError: In case of ASCII parsing problems.
""" """
message_descriptor = message.DESCRIPTOR message_descriptor = message.DESCRIPTOR
if (hasattr(message_descriptor, 'syntax') and
message_descriptor.syntax == 'proto3'):
# Proto3 doesn't represent presence so we can't test if multiple
# scalars have occurred. We have to allow them.
allow_multiple_scalars = True
if tokenizer.TryConsume('['): if tokenizer.TryConsume('['):
name = [tokenizer.ConsumeIdentifier()] name = [tokenizer.ConsumeIdentifier()]
while tokenizer.TryConsume('.'): while tokenizer.TryConsume('.'):

22
python/setup.py
View file

@ -77,6 +77,7 @@ def GenerateUnittestProtos():
generate_proto("../src/google/protobuf/unittest_import_public.proto") generate_proto("../src/google/protobuf/unittest_import_public.proto")
generate_proto("../src/google/protobuf/unittest_mset.proto") generate_proto("../src/google/protobuf/unittest_mset.proto")
generate_proto("../src/google/protobuf/unittest_no_generic_services.proto") generate_proto("../src/google/protobuf/unittest_no_generic_services.proto")
generate_proto("../src/google/protobuf/unittest_proto3_arena.proto")
generate_proto("google/protobuf/internal/descriptor_pool_test1.proto") generate_proto("google/protobuf/internal/descriptor_pool_test1.proto")
generate_proto("google/protobuf/internal/descriptor_pool_test2.proto") generate_proto("google/protobuf/internal/descriptor_pool_test2.proto")
generate_proto("google/protobuf/internal/test_bad_identifiers.proto") generate_proto("google/protobuf/internal/test_bad_identifiers.proto")
@ -90,23 +91,6 @@ def GenerateUnittestProtos():
generate_proto("google/protobuf/internal/import_test_package/outer.proto") generate_proto("google/protobuf/internal/import_test_package/outer.proto")
generate_proto("google/protobuf/pyext/python.proto") generate_proto("google/protobuf/pyext/python.proto")
def MakeTestSuite():
# Test C++ implementation
import unittest
import google.protobuf.pyext.descriptor_cpp2_test as descriptor_cpp2_test
import google.protobuf.pyext.message_factory_cpp2_test \
as message_factory_cpp2_test
import google.protobuf.pyext.reflection_cpp2_generated_test \
as reflection_cpp2_generated_test
loader = unittest.defaultTestLoader
suite = unittest.TestSuite()
for test in [ descriptor_cpp2_test,
message_factory_cpp2_test,
reflection_cpp2_generated_test]:
suite.addTest(loader.loadTestsFromModule(test))
return suite
class clean(_clean): class clean(_clean):
def run(self): def run(self):
# Delete generated files in the code tree. # Delete generated files in the code tree.
@ -152,6 +136,8 @@ if __name__ == '__main__':
ext_module_list.append(Extension( ext_module_list.append(Extension(
"google.protobuf.pyext._message", "google.protobuf.pyext._message",
[ "google/protobuf/pyext/descriptor.cc", [ "google/protobuf/pyext/descriptor.cc",
"google/protobuf/pyext/descriptor_containers.cc",
"google/protobuf/pyext/descriptor_pool.cc",
"google/protobuf/pyext/message.cc", "google/protobuf/pyext/message.cc",
"google/protobuf/pyext/extension_dict.cc", "google/protobuf/pyext/extension_dict.cc",
"google/protobuf/pyext/repeated_scalar_container.cc", "google/protobuf/pyext/repeated_scalar_container.cc",
@ -161,12 +147,12 @@ if __name__ == '__main__':
libraries = [ "protobuf" ], libraries = [ "protobuf" ],
library_dirs = [ '../src/.libs' ], library_dirs = [ '../src/.libs' ],
)) ))
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
setup(name = 'protobuf', setup(name = 'protobuf',
version = '3.0.0-alpha-2', version = '3.0.0-alpha-2',
packages = [ 'google' ], packages = [ 'google' ],
namespace_packages = [ 'google' ], namespace_packages = [ 'google' ],
test_suite = 'setup.MakeTestSuite',
google_test_dir = "google/protobuf/internal", google_test_dir = "google/protobuf/internal",
# Must list modules explicitly so that we don't install tests. # Must list modules explicitly so that we don't install tests.
py_modules = [ py_modules = [

25
src/google/protobuf/compiler/python/python_generator.cc
View file

@ -273,6 +273,19 @@ string StringifyDefaultValue(const FieldDescriptor& field) {
return ""; return "";
} }
string StringifySyntax(FileDescriptor::Syntax syntax) {
switch (syntax) {
case FileDescriptor::SYNTAX_PROTO2:
return "proto2";
case FileDescriptor::SYNTAX_PROTO3:
return "proto3";
case FileDescriptor::SYNTAX_UNKNOWN:
default:
GOOGLE_LOG(FATAL) << "Unsupported syntax; this generator only supports proto2 "
"and proto3 syntax.";
return "";
}
}
} // namespace } // namespace
@ -367,10 +380,12 @@ void Generator::PrintFileDescriptor() const {
m["descriptor_name"] = kDescriptorKey; m["descriptor_name"] = kDescriptorKey;
m["name"] = file_->name(); m["name"] = file_->name();
m["package"] = file_->package(); m["package"] = file_->package();
m["syntax"] = StringifySyntax(file_->syntax());
const char file_descriptor_template[] = const char file_descriptor_template[] =
"$descriptor_name$ = _descriptor.FileDescriptor(\n" "$descriptor_name$ = _descriptor.FileDescriptor(\n"
" name='$name$',\n" " name='$name$',\n"
" package='$package$',\n"; " package='$package$',\n"
" syntax='$syntax$',\n";
printer_->Print(m, file_descriptor_template); printer_->Print(m, file_descriptor_template);
printer_->Indent(); printer_->Indent();
printer_->Print( printer_->Print(
@ -414,7 +429,7 @@ void Generator::PrintTopLevelEnums() const {
for (int j = 0; j < enum_descriptor.value_count(); ++j) { for (int j = 0; j < enum_descriptor.value_count(); ++j) {
const EnumValueDescriptor& value_descriptor = *enum_descriptor.value(j); const EnumValueDescriptor& value_descriptor = *enum_descriptor.value(j);
top_level_enum_values.push_back( top_level_enum_values.push_back(
make_pair(value_descriptor.name(), value_descriptor.number())); std::make_pair(value_descriptor.name(), value_descriptor.number()));
} }
} }
@ -665,10 +680,12 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const {
message_descriptor.options().SerializeToString(&options_string); message_descriptor.options().SerializeToString(&options_string);
printer_->Print( printer_->Print(
"options=$options_value$,\n" "options=$options_value$,\n"
"is_extendable=$extendable$", "is_extendable=$extendable$,\n"
"syntax='$syntax$'",
"options_value", OptionsValue("MessageOptions", options_string), "options_value", OptionsValue("MessageOptions", options_string),
"extendable", message_descriptor.extension_range_count() > 0 ? "extendable", message_descriptor.extension_range_count() > 0 ?
"True" : "False"); "True" : "False",
"syntax", StringifySyntax(message_descriptor.file()->syntax()));
printer_->Print(",\n"); printer_->Print(",\n");
// Extension ranges // Extension ranges

BIN
src/google/protobuf/testdata/golden_message_proto3 vendored Normal file
View file

Binary file not shown.

54
src/google/protobuf/unittest_proto3_arena.proto
View file

@ -47,9 +47,10 @@ message TestAllTypes {
} }
enum NestedEnum { enum NestedEnum {
FOO = 0; ZERO = 0;
BAR = 1; FOO = 1;
BAZ = 2; BAR = 2;
BAZ = 3;
NEG = -1; // Intentionally negative. NEG = -1; // Intentionally negative.
} }
@ -81,6 +82,11 @@ message TestAllTypes {
optional NestedEnum optional_nested_enum = 21; optional NestedEnum optional_nested_enum = 21;
optional ForeignEnum optional_foreign_enum = 22; optional ForeignEnum optional_foreign_enum = 22;
// Omitted (compared to unittest.proto) because proto2 enums are not allowed
// inside proto2 messages.
//
// optional protobuf_unittest_import.ImportEnum optional_import_enum = 23;
optional string optional_string_piece = 24 [ctype=STRING_PIECE]; optional string optional_string_piece = 24 [ctype=STRING_PIECE];
optional string optional_cord = 25 [ctype=CORD]; optional string optional_cord = 25 [ctype=CORD];
@ -118,6 +124,11 @@ message TestAllTypes {
repeated NestedEnum repeated_nested_enum = 51; repeated NestedEnum repeated_nested_enum = 51;
repeated ForeignEnum repeated_foreign_enum = 52; repeated ForeignEnum repeated_foreign_enum = 52;
// Omitted (compared to unittest.proto) because proto2 enums are not allowed
// inside proto2 messages.
//
// repeated protobuf_unittest_import.ImportEnum repeated_import_enum = 53;
repeated string repeated_string_piece = 54 [ctype=STRING_PIECE]; repeated string repeated_string_piece = 54 [ctype=STRING_PIECE];
repeated string repeated_cord = 55 [ctype=CORD]; repeated string repeated_cord = 55 [ctype=CORD];
@ -131,6 +142,31 @@ message TestAllTypes {
} }
} }
// Test messages for packed fields
message TestPackedTypes {
repeated int32 packed_int32 = 90 [packed = true];
repeated int64 packed_int64 = 91 [packed = true];
repeated uint32 packed_uint32 = 92 [packed = true];
repeated uint64 packed_uint64 = 93 [packed = true];
repeated sint32 packed_sint32 = 94 [packed = true];
repeated sint64 packed_sint64 = 95 [packed = true];
repeated fixed32 packed_fixed32 = 96 [packed = true];
repeated fixed64 packed_fixed64 = 97 [packed = true];
repeated sfixed32 packed_sfixed32 = 98 [packed = true];
repeated sfixed64 packed_sfixed64 = 99 [packed = true];
repeated float packed_float = 100 [packed = true];
repeated double packed_double = 101 [packed = true];
repeated bool packed_bool = 102 [packed = true];
repeated ForeignEnum packed_enum = 103 [packed = true];
}
// This proto includes a recusively nested message.
message NestedTestAllTypes {
NestedTestAllTypes child = 1;
TestAllTypes payload = 2;
}
// Define these after TestAllTypes to make sure the compiler can handle // Define these after TestAllTypes to make sure the compiler can handle
// that. // that.
message ForeignMessage { message ForeignMessage {
@ -138,7 +174,13 @@ message ForeignMessage {
} }
enum ForeignEnum { enum ForeignEnum {
FOREIGN_FOO = 0; FOREIGN_ZERO = 0;
FOREIGN_BAR = 1; FOREIGN_FOO = 4;
FOREIGN_BAZ = 2; FOREIGN_BAR = 5;
FOREIGN_BAZ = 6;
} }
// TestEmptyMessage is used to test behavior of unknown fields.
message TestEmptyMessage {
}