organicmaps/protobuf
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

Merge pull request #221 from google/python_proto3

Browse source 
Proto3 Python changes for v3.0.0-alpha-2
This commit is contained in:
Jisi Liu 2015-02-25 20:35:29 -08:00
commit f8e7a46bc2
43 changed files with 6293 additions and 2158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Makefile.am
View file

@ -197,7 +197,6 @@ javanano_EXTRA_DIST=
python_EXTRA_DIST= \
python/google/protobuf/internal/api_implementation.cc \
python/google/protobuf/internal/api_implementation.py \
python/google/protobuf/internal/api_implementation_default_test.py \
python/google/protobuf/internal/containers.py \
python/google/protobuf/internal/cpp_message.py \
python/google/protobuf/internal/decoder.py \
@ -221,6 +220,7 @@ python_EXTRA_DIST= \
python/google/protobuf/internal/more_extensions.proto \
python/google/protobuf/internal/more_extensions_dynamic.proto \
python/google/protobuf/internal/more_messages.proto \
python/google/protobuf/internal/_parameterized.py \
python/google/protobuf/internal/proto_builder_test.py \
python/google/protobuf/internal/python_message.py \
python/google/protobuf/internal/reflection_test.py \
@ -242,16 +242,17 @@ python_EXTRA_DIST= \
python/google/protobuf/pyext/cpp_message.py \
python/google/protobuf/pyext/descriptor.h \
python/google/protobuf/pyext/descriptor.cc \
python/google/protobuf/pyext/descriptor_cpp2_test.py \
python/google/protobuf/pyext/descriptor_pool.h \
python/google/protobuf/pyext/descriptor_pool.cc \
python/google/protobuf/pyext/descriptor_containers.h \
python/google/protobuf/pyext/descriptor_containers.cc \
python/google/protobuf/pyext/extension_dict.h \
python/google/protobuf/pyext/extension_dict.cc \
python/google/protobuf/pyext/message.h \
python/google/protobuf/pyext/message.cc \
python/google/protobuf/pyext/message_factory_cpp2_test.py \
python/google/protobuf/pyext/proto2_api_test.proto \
python/google/protobuf/pyext/python.proto \
python/google/protobuf/pyext/python_protobuf.h \
python/google/protobuf/pyext/reflection_cpp2_generated_test.py \
python/google/protobuf/pyext/repeated_composite_container.h \
python/google/protobuf/pyext/repeated_composite_container.cc \
python/google/protobuf/pyext/repeated_scalar_container.h \

5
python/README.txt
View file

@ -48,8 +48,9 @@ Installation
$ python setup.py build
$ python setup.py google_test
If you want to test c++ implementation, run:
$ python setup.py test --cpp_implementation
If you want to build/test c++ implementation, run:
$ python setup.py build --cpp_implementation
$ python setup.py google_test --cpp_implementation
If some tests fail, this library may not work correctly on your
system. Continue at your own risk.

158
python/google/protobuf/descriptor.py
View file

@ -41,15 +41,13 @@ __author__ = 'robinson@google.com (Will Robinson)'
from google.protobuf.internal import api_implementation
_USE_C_DESCRIPTORS = False
if api_implementation.Type() == 'cpp':
# Used by MakeDescriptor in cpp mode
import os
import uuid
if api_implementation.Version() == 2:
from google.protobuf.pyext import _message
else:
from google.protobuf.internal import cpp_message
from google.protobuf.pyext import _message
_USE_C_DESCRIPTORS = getattr(_message, '_USE_C_DESCRIPTORS', False)
class Error(Exception):
@ -60,12 +58,29 @@ class TypeTransformationError(Error):
"""Error transforming between python proto type and corresponding C++ type."""
if _USE_C_DESCRIPTORS:
# This metaclass allows to override the behavior of code like
# isinstance(my_descriptor, FieldDescriptor)
# and make it return True when the descriptor is an instance of the extension
# type written in C++.
class DescriptorMetaclass(type):
def __instancecheck__(cls, obj):
if super(DescriptorMetaclass, cls).__instancecheck__(obj):
return True
if isinstance(obj, cls._C_DESCRIPTOR_CLASS):
return True
return False
else:
# The standard metaclass; nothing changes.
DescriptorMetaclass = type
class DescriptorBase(object):
"""Descriptors base class.
This class is the base of all descriptor classes. It provides common options
related functionaility.
related functionality.
Attributes:
has_options: True if the descriptor has non-default options. Usually it
@ -75,6 +90,12 @@ class DescriptorBase(object):
avoid some bootstrapping issues.
"""
__metaclass__ = DescriptorMetaclass
if _USE_C_DESCRIPTORS:
# The class, or tuple of classes, that are considered as "virtual
# subclasses" of this descriptor class.
_C_DESCRIPTOR_CLASS = ()
def __init__(self, options, options_class_name):
"""Initialize the descriptor given its options message and the name of the
class of the options message. The name of the class is required in case
@ -235,13 +256,25 @@ class Descriptor(_NestedDescriptorBase):
file: (FileDescriptor) Reference to file descriptor.
"""
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.Descriptor
def __new__(cls, name, full_name, filename, containing_type, fields,
nested_types, enum_types, extensions, options=None,
is_extendable=True, extension_ranges=None, oneofs=None,
file=None, serialized_start=None, serialized_end=None,
syntax=None):
_message.Message._CheckCalledFromGeneratedFile()
return _message.Message._GetMessageDescriptor(full_name)
# NOTE(tmarek): The file argument redefining a builtin is nothing we can
# fix right now since we don't know how many clients already rely on the
# name of the argument.
def __init__(self, name, full_name, filename, containing_type, fields,
nested_types, enum_types, extensions, options=None,
is_extendable=True, extension_ranges=None, oneofs=None,
file=None, serialized_start=None, serialized_end=None): # pylint:disable=redefined-builtin
file=None, serialized_start=None, serialized_end=None,
syntax=None): # pylint:disable=redefined-builtin
"""Arguments to __init__() are as described in the description
of Descriptor fields above.
@ -286,6 +319,7 @@ class Descriptor(_NestedDescriptorBase):
self.oneofs_by_name = dict((o.name, o) for o in self.oneofs)
for oneof in self.oneofs:
oneof.containing_type = self
self.syntax = syntax or "proto2"
def EnumValueName(self, enum, value):
"""Returns the string name of an enum value.
@ -452,6 +486,19 @@ class FieldDescriptor(DescriptorBase):
FIRST_RESERVED_FIELD_NUMBER = 19000
LAST_RESERVED_FIELD_NUMBER = 19999
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.FieldDescriptor
def __new__(cls, name, full_name, index, number, type, cpp_type, label,
default_value, message_type, enum_type, containing_type,
is_extension, extension_scope, options=None,
has_default_value=True, containing_oneof=None):
_message.Message._CheckCalledFromGeneratedFile()
if is_extension:
return _message.Message._GetExtensionDescriptor(full_name)
else:
return _message.Message._GetFieldDescriptor(full_name)
def __init__(self, name, full_name, index, number, type, cpp_type, label,
default_value, message_type, enum_type, containing_type,
is_extension, extension_scope, options=None,
@ -481,20 +528,14 @@ class FieldDescriptor(DescriptorBase):
self.containing_oneof = containing_oneof
if api_implementation.Type() == 'cpp':
if is_extension:
if api_implementation.Version() == 2:
# pylint: disable=protected-access
self._cdescriptor = (
_message.Message._GetExtensionDescriptor(full_name))
# pylint: enable=protected-access
else:
self._cdescriptor = cpp_message.GetExtensionDescriptor(full_name)
# pylint: disable=protected-access
self._cdescriptor = (
_message.Message._GetExtensionDescriptor(full_name))
# pylint: enable=protected-access
else:
if api_implementation.Version() == 2:
# pylint: disable=protected-access
self._cdescriptor = _message.Message._GetFieldDescriptor(full_name)
# pylint: enable=protected-access
else:
self._cdescriptor = cpp_message.GetFieldDescriptor(full_name)
# pylint: disable=protected-access
self._cdescriptor = _message.Message._GetFieldDescriptor(full_name)
# pylint: enable=protected-access
else:
self._cdescriptor = None
@ -544,6 +585,15 @@ class EnumDescriptor(_NestedDescriptorBase):
None to use default enum options.
"""
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.EnumDescriptor
def __new__(cls, name, full_name, filename, values,
containing_type=None, options=None, file=None,
serialized_start=None, serialized_end=None):
_message.Message._CheckCalledFromGeneratedFile()
return _message.Message._GetEnumDescriptor(full_name)
def __init__(self, name, full_name, filename, values,
containing_type=None, options=None, file=None,
serialized_start=None, serialized_end=None):
@ -588,6 +638,17 @@ class EnumValueDescriptor(DescriptorBase):
None to use default enum value options options.
"""
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.EnumValueDescriptor
def __new__(cls, name, index, number, type=None, options=None):
_message.Message._CheckCalledFromGeneratedFile()
# There is no way we can build a complete EnumValueDescriptor with the
# given parameters (the name of the Enum is not known, for example).
# Fortunately generated files just pass it to the EnumDescriptor()
# constructor, which will ignore it, so returning None is good enough.
return None
def __init__(self, name, index, number, type=None, options=None):
"""Arguments are as described in the attribute description above."""
super(EnumValueDescriptor, self).__init__(options, 'EnumValueOptions')
@ -611,6 +672,13 @@ class OneofDescriptor(object):
oneof can contain.
"""
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.OneofDescriptor
def __new__(cls, name, full_name, index, containing_type, fields):
_message.Message._CheckCalledFromGeneratedFile()
return _message.Message._GetOneofDescriptor(full_name)
def __init__(self, name, full_name, index, containing_type, fields):
"""Arguments are as described in the attribute description above."""
self.name = name
@ -704,6 +772,7 @@ class FileDescriptor(DescriptorBase):
name: name of file, relative to root of source tree.
package: name of the package
syntax: string indicating syntax of the file (can be "proto2" or "proto3")
serialized_pb: (str) Byte string of serialized
descriptor_pb2.FileDescriptorProto.
dependencies: List of other FileDescriptors this FileDescriptor depends on.
@ -712,14 +781,31 @@ class FileDescriptor(DescriptorBase):
extensions_by_name: Dict of extension names and their descriptors.
"""
if _USE_C_DESCRIPTORS:
_C_DESCRIPTOR_CLASS = _message.FileDescriptor
def __new__(cls, name, package, options=None, serialized_pb=None,
dependencies=None, syntax=None):
# FileDescriptor() is called from various places, not only from generated
# files, to register dynamic proto files and messages.
# TODO(amauryfa): Expose BuildFile() as a public function and make this
# constructor an implementation detail.
if serialized_pb:
# pylint: disable=protected-access2
return _message.Message._BuildFile(serialized_pb)
# pylint: enable=protected-access
else:
return super(FileDescriptor, cls).__new__(cls)
def __init__(self, name, package, options=None, serialized_pb=None,
dependencies=None):
dependencies=None, syntax=None):
"""Constructor."""
super(FileDescriptor, self).__init__(options, 'FileOptions')
self.message_types_by_name = {}
self.name = name
self.package = package
self.syntax = syntax or "proto2"
self.serialized_pb = serialized_pb
self.enum_types_by_name = {}
@ -728,12 +814,9 @@ class FileDescriptor(DescriptorBase):
if (api_implementation.Type() == 'cpp' and
self.serialized_pb is not None):
if api_implementation.Version() == 2:
# pylint: disable=protected-access
_message.Message._BuildFile(self.serialized_pb)
# pylint: enable=protected-access
else:
cpp_message.BuildFile(self.serialized_pb)
# pylint: disable=protected-access
_message.Message._BuildFile(self.serialized_pb)
# pylint: enable=protected-access
def CopyToProto(self, proto):
"""Copies this to a descriptor_pb2.FileDescriptorProto.
@ -754,7 +837,8 @@ def _ParseOptions(message, string):
return message
def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
syntax=None):
"""Make a protobuf Descriptor given a DescriptorProto protobuf.
Handles nested descriptors. Note that this is limited to the scope of defining
@ -766,6 +850,8 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
package: Optional package name for the new message Descriptor (string).
build_file_if_cpp: Update the C++ descriptor pool if api matches.
Set to False on recursion, so no duplicates are created.
syntax: The syntax/semantics that should be used. Set to "proto3" to get
proto3 field presence semantics.
Returns:
A Descriptor for protobuf messages.
"""
@ -791,12 +877,13 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
else:
file_descriptor_proto.name = proto_name + '.proto'
if api_implementation.Version() == 2:
# pylint: disable=protected-access
_message.Message._BuildFile(file_descriptor_proto.SerializeToString())
# pylint: enable=protected-access
else:
cpp_message.BuildFile(file_descriptor_proto.SerializeToString())
# pylint: disable=protected-access
result = _message.Message._BuildFile(
file_descriptor_proto.SerializeToString())
# pylint: enable=protected-access
if _USE_C_DESCRIPTORS:
return result.message_types_by_name[desc_proto.name]
full_message_name = [desc_proto.name]
if package: full_message_name.insert(0, package)
@ -819,7 +906,8 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
# used by fields in the message, so no loops are possible here.
nested_desc = MakeDescriptor(nested_proto,
package='.'.join(full_message_name),
build_file_if_cpp=False)
build_file_if_cpp=False,
syntax=syntax)
nested_types[full_name] = nested_desc
fields = []

114
python/google/protobuf/descriptor_pool.py
View file

@ -64,6 +64,9 @@ from google.protobuf import descriptor_database
from google.protobuf import text_encoding
_USE_C_DESCRIPTORS = descriptor._USE_C_DESCRIPTORS
def _NormalizeFullyQualifiedName(name):
"""Remove leading period from fully-qualified type name.
@ -271,58 +274,81 @@ class DescriptorPool(object):
file_descriptor = descriptor.FileDescriptor(
name=file_proto.name,
package=file_proto.package,
syntax=file_proto.syntax,
options=file_proto.options,
serialized_pb=file_proto.SerializeToString(),
dependencies=direct_deps)
scope = {}
# This loop extracts all the message and enum types from all the
# dependencoes of the file_proto. This is necessary to create the
# scope of available message types when defining the passed in
# file proto.
for dependency in built_deps:
scope.update(self._ExtractSymbols(
dependency.message_types_by_name.values()))
scope.update((_PrefixWithDot(enum.full_name), enum)
for enum in dependency.enum_types_by_name.values())
for message_type in file_proto.message_type:
message_desc = self._ConvertMessageDescriptor(
message_type, file_proto.package, file_descriptor, scope)
file_descriptor.message_types_by_name[message_desc.name] = message_desc
for enum_type in file_proto.enum_type:
file_descriptor.enum_types_by_name[enum_type.name] = (
self._ConvertEnumDescriptor(enum_type, file_proto.package,
file_descriptor, None, scope))
for index, extension_proto in enumerate(file_proto.extension):
extension_desc = self.MakeFieldDescriptor(
extension_proto, file_proto.package, index, is_extension=True)
extension_desc.containing_type = self._GetTypeFromScope(
file_descriptor.package, extension_proto.extendee, scope)
self.SetFieldType(extension_proto, extension_desc,
file_descriptor.package, scope)
file_descriptor.extensions_by_name[extension_desc.name] = extension_desc
for desc_proto in file_proto.message_type:
self.SetAllFieldTypes(file_proto.package, desc_proto, scope)
if file_proto.package:
desc_proto_prefix = _PrefixWithDot(file_proto.package)
if _USE_C_DESCRIPTORS:
# When using C++ descriptors, all objects defined in the file were added
# to the C++ database when the FileDescriptor was built above.
# Just add them to this descriptor pool.
def _AddMessageDescriptor(message_desc):
self._descriptors[message_desc.full_name] = message_desc
for nested in message_desc.nested_types:
_AddMessageDescriptor(nested)
for enum_type in message_desc.enum_types:
_AddEnumDescriptor(enum_type)
def _AddEnumDescriptor(enum_desc):
self._enum_descriptors[enum_desc.full_name] = enum_desc
for message_type in file_descriptor.message_types_by_name.values():
_AddMessageDescriptor(message_type)
for enum_type in file_descriptor.enum_types_by_name.values():
_AddEnumDescriptor(enum_type)
else:
desc_proto_prefix = ''
scope = {}
# This loop extracts all the message and enum types from all the
# dependencies of the file_proto. This is necessary to create the
# scope of available message types when defining the passed in
# file proto.
for dependency in built_deps:
scope.update(self._ExtractSymbols(
dependency.message_types_by_name.values()))
scope.update((_PrefixWithDot(enum.full_name), enum)
for enum in dependency.enum_types_by_name.values())
for message_type in file_proto.message_type:
message_desc = self._ConvertMessageDescriptor(
message_type, file_proto.package, file_descriptor, scope,
file_proto.syntax)
file_descriptor.message_types_by_name[message_desc.name] = (
message_desc)
for enum_type in file_proto.enum_type:
file_descriptor.enum_types_by_name[enum_type.name] = (
self._ConvertEnumDescriptor(enum_type, file_proto.package,
file_descriptor, None, scope))
for index, extension_proto in enumerate(file_proto.extension):
extension_desc = self.MakeFieldDescriptor(
extension_proto, file_proto.package, index, is_extension=True)
extension_desc.containing_type = self._GetTypeFromScope(
file_descriptor.package, extension_proto.extendee, scope)
self.SetFieldType(extension_proto, extension_desc,
file_descriptor.package, scope)
file_descriptor.extensions_by_name[extension_desc.name] = (
extension_desc)
for desc_proto in file_proto.message_type:
self.SetAllFieldTypes(file_proto.package, desc_proto, scope)
if file_proto.package:
desc_proto_prefix = _PrefixWithDot(file_proto.package)
else:
desc_proto_prefix = ''
for desc_proto in file_proto.message_type:
desc = self._GetTypeFromScope(
desc_proto_prefix, desc_proto.name, scope)
file_descriptor.message_types_by_name[desc_proto.name] = desc
for desc_proto in file_proto.message_type:
desc = self._GetTypeFromScope(desc_proto_prefix, desc_proto.name, scope)
file_descriptor.message_types_by_name[desc_proto.name] = desc
self.Add(file_proto)
self._file_descriptors[file_proto.name] = file_descriptor
return self._file_descriptors[file_proto.name]
def _ConvertMessageDescriptor(self, desc_proto, package=None, file_desc=None,
scope=None):
scope=None, syntax=None):
"""Adds the proto to the pool in the specified package.
Args:
@ -349,7 +375,8 @@ class DescriptorPool(object):
scope = {}
nested = [
self._ConvertMessageDescriptor(nested, desc_name, file_desc, scope)
self._ConvertMessageDescriptor(
nested, desc_name, file_desc, scope, syntax)
for nested in desc_proto.nested_type]
enums = [
self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, scope)
@ -383,7 +410,8 @@ class DescriptorPool(object):
extension_ranges=extension_ranges,
file=file_desc,
serialized_start=None,
serialized_end=None)
serialized_end=None,
syntax=syntax)
for nested in desc.nested_types:
nested.containing_type = desc
for enum in desc.enum_types:

438
python/google/protobuf/internal/_parameterized.py Executable file
View file

@ -0,0 +1,438 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Adds support for parameterized tests to Python's unittest TestCase class.
A parameterized test is a method in a test case that is invoked with different
argument tuples.
A simple example:
class AdditionExample(parameterized.ParameterizedTestCase):
@parameterized.Parameters(
(1, 2, 3),
(4, 5, 9),
(1, 1, 3))
def testAddition(self, op1, op2, result):
self.assertEquals(result, op1 + op2)
Each invocation is a separate test case and properly isolated just
like a normal test method, with its own setUp/tearDown cycle. In the
example above, there are three separate testcases, one of which will
fail due to an assertion error (1 + 1 != 3).
Parameters for invididual test cases can be tuples (with positional parameters)
or dictionaries (with named parameters):
class AdditionExample(parameterized.ParameterizedTestCase):
@parameterized.Parameters(
{'op1': 1, 'op2': 2, 'result': 3},
{'op1': 4, 'op2': 5, 'result': 9},
)
def testAddition(self, op1, op2, result):
self.assertEquals(result, op1 + op2)
If a parameterized test fails, the error message will show the
original test name (which is modified internally) and the arguments
for the specific invocation, which are part of the string returned by
the shortDescription() method on test cases.
The id method of the test, used internally by the unittest framework,
is also modified to show the arguments. To make sure that test names
stay the same across several invocations, object representations like
>>> class Foo(object):
... pass
>>> repr(Foo())
'<__main__.Foo object at 0x23d8610>'
are turned into '<__main__.Foo>'. For even more descriptive names,
especially in test logs, you can use the NamedParameters decorator. In
this case, only tuples are supported, and the first parameters has to
be a string (or an object that returns an apt name when converted via
str()):
class NamedExample(parameterized.ParameterizedTestCase):
@parameterized.NamedParameters(
('Normal', 'aa', 'aaa', True),
('EmptyPrefix', '', 'abc', True),
('BothEmpty', '', '', True))
def testStartsWith(self, prefix, string, result):
self.assertEquals(result, strings.startswith(prefix))
Named tests also have the benefit that they can be run individually
from the command line:
$ testmodule.py NamedExample.testStartsWithNormal
.
--------------------------------------------------------------------
Ran 1 test in 0.000s
OK
Parameterized Classes
=====================
If invocation arguments are shared across test methods in a single
ParameterizedTestCase class, instead of decorating all test methods
individually, the class itself can be decorated:
@parameterized.Parameters(
(1, 2, 3)
(4, 5, 9))
class ArithmeticTest(parameterized.ParameterizedTestCase):
def testAdd(self, arg1, arg2, result):
self.assertEqual(arg1 + arg2, result)
def testSubtract(self, arg2, arg2, result):
self.assertEqual(result - arg1, arg2)
Inputs from Iterables
=====================
If parameters should be shared across several test cases, or are dynamically
created from other sources, a single non-tuple iterable can be passed into
the decorator. This iterable will be used to obtain the test cases:
class AdditionExample(parameterized.ParameterizedTestCase):
@parameterized.Parameters(
c.op1, c.op2, c.result for c in testcases
)
def testAddition(self, op1, op2, result):
self.assertEquals(result, op1 + op2)
Single-Argument Test Methods
============================
If a test method takes only one argument, the single argument does not need to
be wrapped into a tuple:
class NegativeNumberExample(parameterized.ParameterizedTestCase):
@parameterized.Parameters(
-1, -3, -4, -5
)
def testIsNegative(self, arg):
self.assertTrue(IsNegative(arg))
"""
__author__ = 'tmarek@google.com (Torsten Marek)'
import collections
import functools
import re
import types
import unittest
import uuid
from google.apputils import basetest
ADDR_RE = re.compile(r'\<([a-zA-Z0-9_\-\.]+) object at 0x[a-fA-F0-9]+\>')
_SEPARATOR = uuid.uuid1().hex
_FIRST_ARG = object()
_ARGUMENT_REPR = object()
def _CleanRepr(obj):
return ADDR_RE.sub(r'<\1>', repr(obj))
# Helper function formerly from the unittest module, removed from it in
# Python 2.7.
def _StrClass(cls):
return '%s.%s' % (cls.__module__, cls.__name__)
def _NonStringIterable(obj):
return (isinstance(obj, collections.Iterable) and not
isinstance(obj, basestring))
def _FormatParameterList(testcase_params):
if isinstance(testcase_params, collections.Mapping):
return ', '.join('%s=%s' % (argname, _CleanRepr(value))
for argname, value in testcase_params.iteritems())
elif _NonStringIterable(testcase_params):
return ', '.join(map(_CleanRepr, testcase_params))
else:
return _FormatParameterList((testcase_params,))
class _ParameterizedTestIter(object):
"""Callable and iterable class for producing new test cases."""
def __init__(self, test_method, testcases, naming_type):
"""Returns concrete test functions for a test and a list of parameters.
The naming_type is used to determine the name of the concrete
functions as reported by the unittest framework. If naming_type is
_FIRST_ARG, the testcases must be tuples, and the first element must
have a string representation that is a valid Python identifier.
Args:
test_method: The decorated test method.
testcases: (list of tuple/dict) A list of parameter
tuples/dicts for individual test invocations.
naming_type: The test naming type, either _NAMED or _ARGUMENT_REPR.
"""
self._test_method = test_method
self.testcases = testcases
self._naming_type = naming_type
def __call__(self, *args, **kwargs):
raise RuntimeError('You appear to be running a parameterized test case '
'without having inherited from parameterized.'
'ParameterizedTestCase. This is bad because none of '
'your test cases are actually being run.')
def __iter__(self):
test_method = self._test_method
naming_type = self._naming_type
def MakeBoundParamTest(testcase_params):
@functools.wraps(test_method)
def BoundParamTest(self):
if isinstance(testcase_params, collections.Mapping):
test_method(self, **testcase_params)
elif _NonStringIterable(testcase_params):
test_method(self, *testcase_params)
else:
test_method(self, testcase_params)
if naming_type is _FIRST_ARG:
# Signal the metaclass that the name of the test function is unique
# and descriptive.
BoundParamTest.__x_use_name__ = True
BoundParamTest.__name__ += str(testcase_params[0])
testcase_params = testcase_params[1:]
elif naming_type is _ARGUMENT_REPR:
# __x_extra_id__ is used to pass naming information to the __new__
# method of TestGeneratorMetaclass.
# The metaclass will make sure to create a unique, but nondescriptive
# name for this test.
BoundParamTest.__x_extra_id__ = '(%s)' % (
_FormatParameterList(testcase_params),)
else:
raise RuntimeError('%s is not a valid naming type.' % (naming_type,))
BoundParamTest.__doc__ = '%s(%s)' % (
BoundParamTest.__name__, _FormatParameterList(testcase_params))
if test_method.__doc__:
BoundParamTest.__doc__ += '\n%s' % (test_method.__doc__,)
return BoundParamTest
return (MakeBoundParamTest(c) for c in self.testcases)
def _IsSingletonList(testcases):
"""True iff testcases contains only a single non-tuple element."""
return len(testcases) == 1 and not isinstance(testcases[0], tuple)
def _ModifyClass(class_object, testcases, naming_type):
assert not getattr(class_object, '_id_suffix', None), (
'Cannot add parameters to %s,'
' which already has parameterized methods.' % (class_object,))
class_object._id_suffix = id_suffix = {}
for name, obj in class_object.__dict__.items():
if (name.startswith(unittest.TestLoader.testMethodPrefix)
and isinstance(obj, types.FunctionType)):
delattr(class_object, name)
methods = {}
_UpdateClassDictForParamTestCase(
methods, id_suffix, name,
_ParameterizedTestIter(obj, testcases, naming_type))
for name, meth in methods.iteritems():
setattr(class_object, name, meth)
def _ParameterDecorator(naming_type, testcases):
"""Implementation of the parameterization decorators.
Args:
naming_type: The naming type.
testcases: Testcase parameters.
Returns:
A function for modifying the decorated object.
"""
def _Apply(obj):
if isinstance(obj, type):
_ModifyClass(
obj,
list(testcases) if not isinstance(testcases, collections.Sequence)
else testcases,
naming_type)
return obj
else:
return _ParameterizedTestIter(obj, testcases, naming_type)
if _IsSingletonList(testcases):
assert _NonStringIterable(testcases[0]), (
'Single parameter argument must be a non-string iterable')
testcases = testcases[0]
return _Apply
def Parameters(*testcases):
"""A decorator for creating parameterized tests.
See the module docstring for a usage example.
Args:
*testcases: Parameters for the decorated method, either a single
iterable, or a list of tuples/dicts/objects (for tests
with only one argument).
Returns:
A test generator to be handled by TestGeneratorMetaclass.
"""
return _ParameterDecorator(_ARGUMENT_REPR, testcases)
def NamedParameters(*testcases):
"""A decorator for creating parameterized tests.
See the module docstring for a usage example. The first element of
each parameter tuple should be a string and will be appended to the
name of the test method.
Args:
*testcases: Parameters for the decorated method, either a single
iterable, or a list of tuples.
Returns:
A test generator to be handled by TestGeneratorMetaclass.
"""
return _ParameterDecorator(_FIRST_ARG, testcases)
class TestGeneratorMetaclass(type):
"""Metaclass for test cases with test generators.
A test generator is an iterable in a testcase that produces callables. These
callables must be single-argument methods. These methods are injected into
the class namespace and the original iterable is removed. If the name of the
iterable conforms to the test pattern, the injected methods will be picked
up as tests by the unittest framework.
In general, it is supposed to be used in conjuction with the
Parameters decorator.
"""
def __new__(mcs, class_name, bases, dct):
dct['_id_suffix'] = id_suffix = {}
for name, obj in dct.items():
if (name.startswith(unittest.TestLoader.testMethodPrefix) and
_NonStringIterable(obj)):
iterator = iter(obj)
dct.pop(name)
_UpdateClassDictForParamTestCase(dct, id_suffix, name, iterator)
return type.__new__(mcs, class_name, bases, dct)
def _UpdateClassDictForParamTestCase(dct, id_suffix, name, iterator):
"""Adds individual test cases to a dictionary.
Args:
dct: The target dictionary.
id_suffix: The dictionary for mapping names to test IDs.
name: The original name of the test case.
iterator: The iterator generating the individual test cases.
"""
for idx, func in enumerate(iterator):
assert callable(func), 'Test generators must yield callables, got %r' % (
func,)
if getattr(func, '__x_use_name__', False):
new_name = func.__name__
else:
new_name = '%s%s%d' % (name, _SEPARATOR, idx)
assert new_name not in dct, (
'Name of parameterized test case "%s" not unique' % (new_name,))
dct[new_name] = func
id_suffix[new_name] = getattr(func, '__x_extra_id__', '')
class ParameterizedTestCase(basetest.TestCase):
"""Base class for test cases using the Parameters decorator."""
__metaclass__ = TestGeneratorMetaclass
def _OriginalName(self):
return self._testMethodName.split(_SEPARATOR)[0]
def __str__(self):
return '%s (%s)' % (self._OriginalName(), _StrClass(self.__class__))
def id(self): # pylint: disable=invalid-name
"""Returns the descriptive ID of the test.
This is used internally by the unittesting framework to get a name
for the test to be used in reports.
Returns:
The test id.
"""
return '%s.%s%s' % (_StrClass(self.__class__),
self._OriginalName(),
self._id_suffix.get(self._testMethodName, ''))
def CoopParameterizedTestCase(other_base_class):
"""Returns a new base class with a cooperative metaclass base.
This enables the ParameterizedTestCase to be used in combination
with other base classes that have custom metaclasses, such as
mox.MoxTestBase.
Only works with metaclasses that do not override type.__new__.
Example:
import google3
import mox
from google3.testing.pybase import parameterized
class ExampleTest(parameterized.CoopParameterizedTestCase(mox.MoxTestBase)):
...
Args:
other_base_class: (class) A test case base class.
Returns:
A new class object.
"""
metaclass = type(
'CoopMetaclass',
(other_base_class.__metaclass__,
TestGeneratorMetaclass), {})
return metaclass(
'CoopParameterizedTestCase',
(other_base_class, ParameterizedTestCase), {})

14
python/google/protobuf/internal/api_implementation.cc
View file

@ -50,10 +50,7 @@ namespace python {
// and
// PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2
#ifdef PYTHON_PROTO2_CPP_IMPL_V1
#if PY_MAJOR_VERSION >= 3
#error "PYTHON_PROTO2_CPP_IMPL_V1 is not supported under Python 3."
#endif
static int kImplVersion = 1;
#error "PYTHON_PROTO2_CPP_IMPL_V1 is no longer supported."
#else
#ifdef PYTHON_PROTO2_CPP_IMPL_V2
static int kImplVersion = 2;
@ -62,14 +59,7 @@ static int kImplVersion = 2;
static int kImplVersion = 0;
#else
// The defaults are set here. Python 3 uses the fast C++ APIv2 by default.
// Python 2 still uses the Python version by default until some compatibility
// issues can be worked around.
#if PY_MAJOR_VERSION >= 3
static int kImplVersion = 2;
#else
static int kImplVersion = 0;
#endif
static int kImplVersion = -1; // -1 means "Unspecified by compiler flags".
#endif // PYTHON_PROTO2_PYTHON_IMPL
#endif // PYTHON_PROTO2_CPP_IMPL_V2

38
python/google/protobuf/internal/api_implementation.py
View file

@ -40,14 +40,33 @@ try:
# The compile-time constants in the _api_implementation module can be used to
# switch to a certain implementation of the Python API at build time.
_api_version = _api_implementation.api_version
del _api_implementation
_proto_extension_modules_exist_in_build = True
except ImportError:
_api_version = 0
_api_version = -1 # Unspecified by compiler flags.
_proto_extension_modules_exist_in_build = False
if _api_version == 1:
raise ValueError('api_version=1 is no longer supported.')
if _api_version < 0: # Still unspecified?
try:
# The presence of this module in a build allows the proto implementation to
# be upgraded merely via build deps rather than a compiler flag or the
# runtime environment variable.
# pylint: disable=g-import-not-at-top
from google.protobuf import _use_fast_cpp_protos
# Work around a known issue in the classic bootstrap .par import hook.
if not _use_fast_cpp_protos:
raise ImportError('_use_fast_cpp_protos import succeeded but was None')
del _use_fast_cpp_protos
_api_version = 2
except ImportError:
if _proto_extension_modules_exist_in_build:
if sys.version_info[0] >= 3: # Python 3 defaults to C++ impl v2.
_api_version = 2
# TODO(b/17427486): Make Python 2 default to C++ impl v2.
_default_implementation_type = (
'python' if _api_version == 0 else 'cpp')
_default_version_str = (
'1' if _api_version <= 1 else '2')
'python' if _api_version <= 0 else 'cpp')
# This environment variable can be used to switch to a certain implementation
# of the Python API, overriding the compile-time constants in the
@ -64,13 +83,12 @@ if _implementation_type != 'python':
# _api_implementation module. Right now only 1 and 2 are valid values. Any other
# value will be ignored.
_implementation_version_str = os.getenv(
'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION',
_default_version_str)
'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', '2')
if _implementation_version_str not in ('1', '2'):
if _implementation_version_str != '2':
raise ValueError(
"unsupported PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION: '" +
_implementation_version_str + "' (supported versions: 1, 2)"
'unsupported PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION: "' +
_implementation_version_str + '" (supported versions: 2)'
)
_implementation_version = int(_implementation_version_str)

63
python/google/protobuf/internal/api_implementation_default_test.py
View file

@ -1,63 +0,0 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Test that the api_implementation defaults are what we expect."""
import os
import sys
# Clear environment implementation settings before the google3 imports.
os.environ.pop('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', None)
os.environ.pop('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', None)
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
class ApiImplementationDefaultTest(basetest.TestCase):
if sys.version_info.major <= 2:
def testThatPythonIsTheDefault(self):
"""If -DPYTHON_PROTO_*IMPL* was given at build time, this may fail."""
self.assertEqual('python', api_implementation.Type())
else:
def testThatCppApiV2IsTheDefault(self):
"""If -DPYTHON_PROTO_*IMPL* was given at build time, this may fail."""
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
if __name__ == '__main__':
basetest.main()

37
python/google/protobuf/internal/containers.py
View file

@ -41,7 +41,6 @@ are:
__author__ = 'petar@google.com (Petar Petrov)'
class BaseContainer(object):
"""Base container class."""
@ -119,15 +118,23 @@ class RepeatedScalarFieldContainer(BaseContainer):
self._message_listener.Modified()
def extend(self, elem_seq):
"""Extends by appending the given sequence. Similar to list.extend()."""
if not elem_seq:
return
"""Extends by appending the given iterable. Similar to list.extend()."""
new_values = []
for elem in elem_seq:
new_values.append(self._type_checker.CheckValue(elem))
self._values.extend(new_values)
self._message_listener.Modified()
if elem_seq is None:
return
try:
elem_seq_iter = iter(elem_seq)
except TypeError:
if not elem_seq:
# silently ignore falsy inputs :-/.
# TODO(ptucker): Deprecate this behavior. b/18413862
return
raise
new_values = [self._type_checker.CheckValue(elem) for elem in elem_seq_iter]
if new_values:
self._values.extend(new_values)
self._message_listener.Modified()
def MergeFrom(self, other):
"""Appends the contents of another repeated field of the same type to this
@ -141,6 +148,12 @@ class RepeatedScalarFieldContainer(BaseContainer):
self._values.remove(elem)
self._message_listener.Modified()
def pop(self, key=-1):
"""Removes and returns an item at a given index. Similar to list.pop()."""
value = self._values[key]
self.__delitem__(key)
return value
def __setitem__(self, key, value):
"""Sets the item on the specified position."""
if isinstance(key, slice): # PY3
@ -245,6 +258,12 @@ class RepeatedCompositeFieldContainer(BaseContainer):
self._values.remove(elem)
self._message_listener.Modified()
def pop(self, key=-1):
"""Removes and returns an item at a given index. Similar to list.pop()."""
value = self._values[key]
self.__delitem__(key)
return value
def __getslice__(self, start, stop):
"""Retrieves the subset of items from between the specified indices."""
return self._values[start:stop]

3
python/google/protobuf/internal/decoder.py
View file

@ -621,9 +621,6 @@ def MessageDecoder(field_number, is_repeated, is_packed, key, new_default):
if value is None:
value = field_dict.setdefault(key, new_default(message))
while 1:
value = field_dict.get(key)
if value is None:
value = field_dict.setdefault(key, new_default(message))
# Read length.
(size, pos) = local_DecodeVarint(buffer, pos)
new_pos = pos + size

176
python/google/protobuf/internal/descriptor_test.py
View file

@ -34,12 +34,16 @@
__author__ = 'robinson@google.com (Will Robinson)'
import sys
from google.apputils import basetest
from google.protobuf import unittest_custom_options_pb2
from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_pb2
from google.protobuf import descriptor_pb2
from google.protobuf.internal import api_implementation
from google.protobuf import descriptor
from google.protobuf import symbol_database
from google.protobuf import text_format
@ -51,41 +55,28 @@ name: 'TestEmptyMessage'
class DescriptorTest(basetest.TestCase):
def setUp(self):
self.my_file = descriptor.FileDescriptor(
file_proto = descriptor_pb2.FileDescriptorProto(
name='some/filename/some.proto',
package='protobuf_unittest'
)
self.my_enum = descriptor.EnumDescriptor(
name='ForeignEnum',
full_name='protobuf_unittest.ForeignEnum',
filename=None,
file=self.my_file,
values=[
descriptor.EnumValueDescriptor(name='FOREIGN_FOO', index=0, number=4),
descriptor.EnumValueDescriptor(name='FOREIGN_BAR', index=1, number=5),
descriptor.EnumValueDescriptor(name='FOREIGN_BAZ', index=2, number=6),
])
self.my_message = descriptor.Descriptor(
name='NestedMessage',
full_name='protobuf_unittest.TestAllTypes.NestedMessage',
filename=None,
file=self.my_file,
containing_type=None,
fields=[
descriptor.FieldDescriptor(
name='bb',
full_name='protobuf_unittest.TestAllTypes.NestedMessage.bb',
index=0, number=1,
type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None),
],
nested_types=[],
enum_types=[
self.my_enum,
],
extensions=[])
package='protobuf_unittest')
message_proto = file_proto.message_type.add(
name='NestedMessage')
message_proto.field.add(
name='bb',
number=1,
type=descriptor_pb2.FieldDescriptorProto.TYPE_INT32,
label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL)
enum_proto = message_proto.enum_type.add(
name='ForeignEnum')
enum_proto.value.add(name='FOREIGN_FOO', number=4)
enum_proto.value.add(name='FOREIGN_BAR', number=5)
enum_proto.value.add(name='FOREIGN_BAZ', number=6)
descriptor_pool = symbol_database.Default().pool
descriptor_pool.Add(file_proto)
self.my_file = descriptor_pool.FindFileByName(file_proto.name)
self.my_message = self.my_file.message_types_by_name[message_proto.name]
self.my_enum = self.my_message.enum_types_by_name[enum_proto.name]
self.my_method = descriptor.MethodDescriptor(
name='Bar',
full_name='protobuf_unittest.TestService.Bar',
@ -173,6 +164,11 @@ class DescriptorTest(basetest.TestCase):
self.assertEqual(unittest_custom_options_pb2.METHODOPT1_VAL2,
method_options.Extensions[method_opt1])
message_descriptor = (
unittest_custom_options_pb2.DummyMessageContainingEnum.DESCRIPTOR)
self.assertTrue(file_descriptor.has_options)
self.assertFalse(message_descriptor.has_options)
def testDifferentCustomOptionTypes(self):
kint32min = -2**31
kint64min = -2**63
@ -394,6 +390,108 @@ class DescriptorTest(basetest.TestCase):
self.assertEqual(self.my_file.name, 'some/filename/some.proto')
self.assertEqual(self.my_file.package, 'protobuf_unittest')
@basetest.unittest.skipIf(
api_implementation.Type() != 'cpp' or api_implementation.Version() != 2,
'Immutability of descriptors is only enforced in v2 implementation')
def testImmutableCppDescriptor(self):
message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
with self.assertRaises(AttributeError):
message_descriptor.fields_by_name = None
with self.assertRaises(TypeError):
message_descriptor.fields_by_name['Another'] = None
with self.assertRaises(TypeError):
message_descriptor.fields.append(None)
class GeneratedDescriptorTest(basetest.TestCase):
"""Tests for the properties of descriptors in generated code."""
def CheckMessageDescriptor(self, message_descriptor):
# Basic properties
self.assertEqual(message_descriptor.name, 'TestAllTypes')
self.assertEqual(message_descriptor.full_name,
'protobuf_unittest.TestAllTypes')
# Test equality and hashability
self.assertEqual(message_descriptor, message_descriptor)
self.assertEqual(message_descriptor.fields[0].containing_type,
message_descriptor)
self.assertIn(message_descriptor, [message_descriptor])
self.assertIn(message_descriptor, {message_descriptor: None})
# Test field containers
self.CheckDescriptorSequence(message_descriptor.fields)
self.CheckDescriptorMapping(message_descriptor.fields_by_name)
self.CheckDescriptorMapping(message_descriptor.fields_by_number)
def CheckFieldDescriptor(self, field_descriptor):
# Basic properties
self.assertEqual(field_descriptor.name, 'optional_int32')
self.assertEqual(field_descriptor.full_name,
'protobuf_unittest.TestAllTypes.optional_int32')
self.assertEqual(field_descriptor.containing_type.name, 'TestAllTypes')
# Test equality and hashability
self.assertEqual(field_descriptor, field_descriptor)
self.assertEqual(
field_descriptor.containing_type.fields_by_name['optional_int32'],
field_descriptor)
self.assertIn(field_descriptor, [field_descriptor])
self.assertIn(field_descriptor, {field_descriptor: None})
def CheckDescriptorSequence(self, sequence):
# Verifies that a property like 'messageDescriptor.fields' has all the
# properties of an immutable abc.Sequence.
self.assertGreater(len(sequence), 0) # Sized
self.assertEqual(len(sequence), len(list(sequence))) # Iterable
item = sequence[0]
self.assertEqual(item, sequence[0])
self.assertIn(item, sequence) # Container
self.assertEqual(sequence.index(item), 0)
self.assertEqual(sequence.count(item), 1)
reversed_iterator = reversed(sequence)
self.assertEqual(list(reversed_iterator), list(sequence)[::-1])
self.assertRaises(StopIteration, next, reversed_iterator)
def CheckDescriptorMapping(self, mapping):
# Verifies that a property like 'messageDescriptor.fields' has all the
# properties of an immutable abc.Mapping.
self.assertGreater(len(mapping), 0) # Sized
self.assertEqual(len(mapping), len(list(mapping))) # Iterable
if sys.version_info.major >= 3:
key, item = next(iter(mapping.items()))
else:
key, item = mapping.items()[0]
self.assertIn(key, mapping) # Container
self.assertEqual(mapping.get(key), item)
# keys(), iterkeys() &co
item = (next(iter(mapping.keys())), next(iter(mapping.values())))
self.assertEqual(item, next(iter(mapping.items())))
if sys.version_info.major < 3:
def CheckItems(seq, iterator):
self.assertEqual(next(iterator), seq[0])
self.assertEqual(list(iterator), seq[1:])
CheckItems(mapping.keys(), mapping.iterkeys())
CheckItems(mapping.values(), mapping.itervalues())
CheckItems(mapping.items(), mapping.iteritems())
def testDescriptor(self):
message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
self.CheckMessageDescriptor(message_descriptor)
field_descriptor = message_descriptor.fields_by_name['optional_int32']
self.CheckFieldDescriptor(field_descriptor)
def testCppDescriptorContainer(self):
# Check that the collection is still valid even if the parent disappeared.
enum = unittest_pb2.TestAllTypes.DESCRIPTOR.enum_types_by_name['NestedEnum']
values = enum.values
del enum
self.assertEqual('FOO', values[0].name)
def testCppDescriptorContainer_Iterator(self):
# Same test with the iterator
enum = unittest_pb2.TestAllTypes.DESCRIPTOR.enum_types_by_name['NestedEnum']
values_iter = iter(enum.values)
del enum
self.assertEqual('FOO', next(values_iter).name)
class DescriptorCopyToProtoTest(basetest.TestCase):
"""Tests for CopyTo functions of Descriptor."""
@ -588,10 +686,12 @@ class DescriptorCopyToProtoTest(basetest.TestCase):
output_type: '.protobuf_unittest.BarResponse'
>
"""
self._InternalTestCopyToProto(
unittest_pb2.TestService.DESCRIPTOR,
descriptor_pb2.ServiceDescriptorProto,
TEST_SERVICE_ASCII)
# TODO(rocking): enable this test after the proto descriptor change is
# checked in.
#self._InternalTestCopyToProto(
# unittest_pb2.TestService.DESCRIPTOR,
# descriptor_pb2.ServiceDescriptorProto,
# TEST_SERVICE_ASCII)
class MakeDescriptorTest(basetest.TestCase):

831
python/google/protobuf/internal/message_test.py
View file

File diff suppressed because it is too large Load diff

81
python/google/protobuf/internal/python_message.py
View file

@ -219,12 +219,20 @@ def _AttachFieldHelpers(cls, field_descriptor):
def AddDecoder(wiretype, is_packed):
tag_bytes = encoder.TagBytes(field_descriptor.number, wiretype)
cls._decoders_by_tag[tag_bytes] = (
type_checkers.TYPE_TO_DECODER[field_descriptor.type](
field_descriptor.number, is_repeated, is_packed,
field_descriptor, field_descriptor._default_constructor),
field_descriptor if field_descriptor.containing_oneof is not None
else None)
decode_type = field_descriptor.type
if (decode_type == _FieldDescriptor.TYPE_ENUM and
type_checkers.SupportsOpenEnums(field_descriptor)):
decode_type = _FieldDescriptor.TYPE_INT32
oneof_descriptor = None
if field_descriptor.containing_oneof is not None:
oneof_descriptor = field_descriptor
field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
field_descriptor.number, is_repeated, is_packed,
field_descriptor, field_descriptor._default_constructor)
cls._decoders_by_tag[tag_bytes] = (field_decoder, oneof_descriptor)
AddDecoder(type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type],
False)
@ -296,6 +304,8 @@ def _DefaultValueConstructorForField(field):
def MakeSubMessageDefault(message):
result = message_type._concrete_class()
result._SetListener(message._listener_for_children)
if field.containing_oneof:
message._UpdateOneofState(field)
return result
return MakeSubMessageDefault
@ -476,6 +486,7 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
type_checker = type_checkers.GetTypeChecker(field)
default_value = field.default_value
valid_values = set()
is_proto3 = field.containing_type.syntax == "proto3"
def getter(self):
# TODO(protobuf-team): This may be broken since there may not be
@ -483,15 +494,24 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
return self._fields.get(field, default_value)
getter.__module__ = None
getter.__doc__ = 'Getter for %s.' % proto_field_name
clear_when_set_to_default = is_proto3 and not field.containing_oneof
def field_setter(self, new_value):
# pylint: disable=protected-access
self._fields[field] = type_checker.CheckValue(new_value)
# Testing the value for truthiness captures all of the proto3 defaults
# (0, 0.0, enum 0, and False).
new_value = type_checker.CheckValue(new_value)
if clear_when_set_to_default and not new_value:
self._fields.pop(field, None)
else:
self._fields[field] = new_value
# Check _cached_byte_size_dirty inline to improve performance, since scalar
# setters are called frequently.
if not self._cached_byte_size_dirty:
self._Modified()
if field.containing_oneof is not None:
if field.containing_oneof:
def setter(self, new_value):
field_setter(self, new_value)
self._UpdateOneofState(field)
@ -624,24 +644,35 @@ def _AddListFieldsMethod(message_descriptor, cls):
cls.ListFields = ListFields
_Proto3HasError = 'Protocol message has no non-repeated submessage field "%s"'
_Proto2HasError = 'Protocol message has no non-repeated field "%s"'
def _AddHasFieldMethod(message_descriptor, cls):
"""Helper for _AddMessageMethods()."""
singular_fields = {}
is_proto3 = (message_descriptor.syntax == "proto3")
error_msg = _Proto3HasError if is_proto3 else _Proto2HasError
hassable_fields = {}
for field in message_descriptor.fields:
if field.label != _FieldDescriptor.LABEL_REPEATED:
singular_fields[field.name] = field
# Fields inside oneofs are never repeated (enforced by the compiler).
for field in message_descriptor.oneofs:
singular_fields[field.name] = field
if field.label == _FieldDescriptor.LABEL_REPEATED:
continue
# For proto3, only submessages and fields inside a oneof have presence.
if (is_proto3 and field.cpp_type != _FieldDescriptor.CPPTYPE_MESSAGE and
not field.containing_oneof):
continue
hassable_fields[field.name] = field
if not is_proto3:
# Fields inside oneofs are never repeated (enforced by the compiler).
for oneof in message_descriptor.oneofs:
hassable_fields[oneof.name] = oneof
def HasField(self, field_name):
try:
field = singular_fields[field_name]
field = hassable_fields[field_name]
except KeyError:
raise ValueError(
'Protocol message has no singular "%s" field.' % field_name)
raise ValueError(error_msg % field_name)
if isinstance(field, descriptor_mod.OneofDescriptor):
try:
@ -871,6 +902,7 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
local_ReadTag = decoder.ReadTag
local_SkipField = decoder.SkipField
decoders_by_tag = cls._decoders_by_tag
is_proto3 = message_descriptor.syntax == "proto3"
def InternalParse(self, buffer, pos, end):
self._Modified()
@ -884,9 +916,11 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
new_pos = local_SkipField(buffer, new_pos, end, tag_bytes)
if new_pos == -1:
return pos
if not unknown_field_list:
unknown_field_list = self._unknown_fields = []
unknown_field_list.append((tag_bytes, buffer[value_start_pos:new_pos]))
if not is_proto3:
if not unknown_field_list:
unknown_field_list = self._unknown_fields = []
unknown_field_list.append(
(tag_bytes, buffer[value_start_pos:new_pos]))
pos = new_pos
else:
pos = field_decoder(buffer, new_pos, end, self, field_dict)
@ -1008,6 +1042,8 @@ def _AddMergeFromMethod(cls):
# Construct a new object to represent this field.
field_value = field._default_constructor(self)
fields[field] = field_value
if field.containing_oneof:
self._UpdateOneofState(field)
field_value.MergeFrom(value)
else:
self._fields[field] = value
@ -1252,11 +1288,10 @@ class _ExtensionDict(object):
# It's slightly wasteful to lookup the type checker each time,
# but we expect this to be a vanishingly uncommon case anyway.
type_checker = type_checkers.GetTypeChecker(
extension_handle)
type_checker = type_checkers.GetTypeChecker(extension_handle)
# pylint: disable=protected-access
self._extended_message._fields[extension_handle] = (
type_checker.CheckValue(value))
type_checker.CheckValue(value))
self._extended_message._Modified()
def _FindExtensionByName(self, name):

11
python/google/protobuf/internal/reflection_test.py
View file

@ -1792,6 +1792,17 @@ class ReflectionTest(basetest.TestCase):
# Just check the default value.
self.assertEqual(57, msg.inner.value)
@basetest.unittest.skipIf(
api_implementation.Type() != 'cpp' or api_implementation.Version() != 2,
'CPPv2-specific test')
def testBadArguments(self):
# Some of these assertions used to segfault.
from google.protobuf.pyext import _message
self.assertRaises(TypeError, _message.Message._GetFieldDescriptor, 3)
self.assertRaises(TypeError, _message.Message._GetExtensionDescriptor, 42)
self.assertRaises(TypeError,
unittest_pb2.TestAllTypes().__getattribute__, 42)
# Since we had so many tests for protocol buffer equality, we broke these out
# into separate TestCase classes.

160
python/google/protobuf/internal/test_util.py
View file

@ -40,13 +40,19 @@ import os.path
from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_pb2
from google.protobuf import descriptor_pb2
# Tests whether the given TestAllTypes message is proto2 or not.
# This is used to gate several fields/features that only exist
# for the proto2 version of the message.
def IsProto2(message):
return message.DESCRIPTOR.syntax == "proto2"
def SetAllNonLazyFields(message):
"""Sets every non-lazy field in the message to a unique value.
Args:
message: A unittest_pb2.TestAllTypes instance.
message: A TestAllTypes instance.
"""
#
@ -77,7 +83,8 @@ def SetAllNonLazyFields(message):
message.optional_nested_enum = unittest_pb2.TestAllTypes.BAZ
message.optional_foreign_enum = unittest_pb2.FOREIGN_BAZ
message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ
if IsProto2(message):
message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ
message.optional_string_piece = u'124'
message.optional_cord = u'125'
@ -110,7 +117,8 @@ def SetAllNonLazyFields(message):
message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAR)
message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAR)
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR)
if IsProto2(message):
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR)
message.repeated_string_piece.append(u'224')
message.repeated_cord.append(u'225')
@ -140,7 +148,8 @@ def SetAllNonLazyFields(message):
message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAZ)
message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ)
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ)
if IsProto2(message):
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ)
message.repeated_string_piece.append(u'324')
message.repeated_cord.append(u'325')
@ -149,28 +158,29 @@ def SetAllNonLazyFields(message):
# Fields that have defaults.
#
message.default_int32 = 401
message.default_int64 = 402
message.default_uint32 = 403
message.default_uint64 = 404
message.default_sint32 = 405
message.default_sint64 = 406
message.default_fixed32 = 407
message.default_fixed64 = 408
message.default_sfixed32 = 409
message.default_sfixed64 = 410
message.default_float = 411
message.default_double = 412
message.default_bool = False
message.default_string = '415'
message.default_bytes = b'416'
if IsProto2(message):
message.default_int32 = 401
message.default_int64 = 402
message.default_uint32 = 403
message.default_uint64 = 404
message.default_sint32 = 405
message.default_sint64 = 406
message.default_fixed32 = 407
message.default_fixed64 = 408
message.default_sfixed32 = 409
message.default_sfixed64 = 410
message.default_float = 411
message.default_double = 412
message.default_bool = False
message.default_string = '415'
message.default_bytes = b'416'
message.default_nested_enum = unittest_pb2.TestAllTypes.FOO
message.default_foreign_enum = unittest_pb2.FOREIGN_FOO
message.default_import_enum = unittest_import_pb2.IMPORT_FOO
message.default_nested_enum = unittest_pb2.TestAllTypes.FOO
message.default_foreign_enum = unittest_pb2.FOREIGN_FOO
message.default_import_enum = unittest_import_pb2.IMPORT_FOO
message.default_string_piece = '424'
message.default_cord = '425'
message.default_string_piece = '424'
message.default_cord = '425'
message.oneof_uint32 = 601
message.oneof_nested_message.bb = 602
@ -398,7 +408,8 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertTrue(message.HasField('optional_nested_enum'))
test_case.assertTrue(message.HasField('optional_foreign_enum'))
test_case.assertTrue(message.HasField('optional_import_enum'))
if IsProto2(message):
test_case.assertTrue(message.HasField('optional_import_enum'))
test_case.assertTrue(message.HasField('optional_string_piece'))
test_case.assertTrue(message.HasField('optional_cord'))
@ -430,8 +441,9 @@ def ExpectAllFieldsSet(test_case, message):
message.optional_nested_enum)
test_case.assertEqual(unittest_pb2.FOREIGN_BAZ,
message.optional_foreign_enum)
test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
message.optional_import_enum)
if IsProto2(message):
test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
message.optional_import_enum)
# -----------------------------------------------------------------
@ -457,7 +469,8 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(2, len(message.repeated_import_message))
test_case.assertEqual(2, len(message.repeated_nested_enum))
test_case.assertEqual(2, len(message.repeated_foreign_enum))
test_case.assertEqual(2, len(message.repeated_import_enum))
if IsProto2(message):
test_case.assertEqual(2, len(message.repeated_import_enum))
test_case.assertEqual(2, len(message.repeated_string_piece))
test_case.assertEqual(2, len(message.repeated_cord))
@ -488,8 +501,9 @@ def ExpectAllFieldsSet(test_case, message):
message.repeated_nested_enum[0])
test_case.assertEqual(unittest_pb2.FOREIGN_BAR,
message.repeated_foreign_enum[0])
test_case.assertEqual(unittest_import_pb2.IMPORT_BAR,
message.repeated_import_enum[0])
if IsProto2(message):
test_case.assertEqual(unittest_import_pb2.IMPORT_BAR,
message.repeated_import_enum[0])
test_case.assertEqual(301, message.repeated_int32[1])
test_case.assertEqual(302, message.repeated_int64[1])
@ -517,53 +531,55 @@ def ExpectAllFieldsSet(test_case, message):
message.repeated_nested_enum[1])
test_case.assertEqual(unittest_pb2.FOREIGN_BAZ,
message.repeated_foreign_enum[1])
test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
message.repeated_import_enum[1])
if IsProto2(message):
test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
message.repeated_import_enum[1])
# -----------------------------------------------------------------
test_case.assertTrue(message.HasField('default_int32'))
test_case.assertTrue(message.HasField('default_int64'))
test_case.assertTrue(message.HasField('default_uint32'))
test_case.assertTrue(message.HasField('default_uint64'))
test_case.assertTrue(message.HasField('default_sint32'))
test_case.assertTrue(message.HasField('default_sint64'))
test_case.assertTrue(message.HasField('default_fixed32'))
test_case.assertTrue(message.HasField('default_fixed64'))
test_case.assertTrue(message.HasField('default_sfixed32'))
test_case.assertTrue(message.HasField('default_sfixed64'))
test_case.assertTrue(message.HasField('default_float'))
test_case.assertTrue(message.HasField('default_double'))
test_case.assertTrue(message.HasField('default_bool'))
test_case.assertTrue(message.HasField('default_string'))
test_case.assertTrue(message.HasField('default_bytes'))
if IsProto2(message):
test_case.assertTrue(message.HasField('default_int32'))
test_case.assertTrue(message.HasField('default_int64'))
test_case.assertTrue(message.HasField('default_uint32'))
test_case.assertTrue(message.HasField('default_uint64'))
test_case.assertTrue(message.HasField('default_sint32'))
test_case.assertTrue(message.HasField('default_sint64'))
test_case.assertTrue(message.HasField('default_fixed32'))
test_case.assertTrue(message.HasField('default_fixed64'))
test_case.assertTrue(message.HasField('default_sfixed32'))
test_case.assertTrue(message.HasField('default_sfixed64'))
test_case.assertTrue(message.HasField('default_float'))
test_case.assertTrue(message.HasField('default_double'))
test_case.assertTrue(message.HasField('default_bool'))
test_case.assertTrue(message.HasField('default_string'))
test_case.assertTrue(message.HasField('default_bytes'))
test_case.assertTrue(message.HasField('default_nested_enum'))
test_case.assertTrue(message.HasField('default_foreign_enum'))
test_case.assertTrue(message.HasField('default_import_enum'))
test_case.assertTrue(message.HasField('default_nested_enum'))
test_case.assertTrue(message.HasField('default_foreign_enum'))
test_case.assertTrue(message.HasField('default_import_enum'))
test_case.assertEqual(401, message.default_int32)
test_case.assertEqual(402, message.default_int64)
test_case.assertEqual(403, message.default_uint32)
test_case.assertEqual(404, message.default_uint64)
test_case.assertEqual(405, message.default_sint32)
test_case.assertEqual(406, message.default_sint64)
test_case.assertEqual(407, message.default_fixed32)
test_case.assertEqual(408, message.default_fixed64)
test_case.assertEqual(409, message.default_sfixed32)
test_case.assertEqual(410, message.default_sfixed64)
test_case.assertEqual(411, message.default_float)
test_case.assertEqual(412, message.default_double)
test_case.assertEqual(False, message.default_bool)
test_case.assertEqual('415', message.default_string)
test_case.assertEqual(b'416', message.default_bytes)
test_case.assertEqual(401, message.default_int32)
test_case.assertEqual(402, message.default_int64)
test_case.assertEqual(403, message.default_uint32)
test_case.assertEqual(404, message.default_uint64)
test_case.assertEqual(405, message.default_sint32)
test_case.assertEqual(406, message.default_sint64)
test_case.assertEqual(407, message.default_fixed32)
test_case.assertEqual(408, message.default_fixed64)
test_case.assertEqual(409, message.default_sfixed32)
test_case.assertEqual(410, message.default_sfixed64)
test_case.assertEqual(411, message.default_float)
test_case.assertEqual(412, message.default_double)
test_case.assertEqual(False, message.default_bool)
test_case.assertEqual('415', message.default_string)
test_case.assertEqual(b'416', message.default_bytes)
test_case.assertEqual(unittest_pb2.TestAllTypes.FOO,
message.default_nested_enum)
test_case.assertEqual(unittest_pb2.FOREIGN_FOO,
message.default_foreign_enum)
test_case.assertEqual(unittest_import_pb2.IMPORT_FOO,
message.default_import_enum)
test_case.assertEqual(unittest_pb2.TestAllTypes.FOO,
message.default_nested_enum)
test_case.assertEqual(unittest_pb2.FOREIGN_FOO,
message.default_foreign_enum)
test_case.assertEqual(unittest_import_pb2.IMPORT_FOO,
message.default_import_enum)
def GoldenFile(filename):
@ -594,7 +610,7 @@ def SetAllPackedFields(message):
"""Sets every field in the message to a unique value.
Args:
message: A unittest_pb2.TestPackedTypes instance.
message: A TestPackedTypes instance.
"""
message.packed_int32.extend([601, 701])
message.packed_int64.extend([602, 702])

561
python/google/protobuf/internal/text_format_test.py
View file

@ -37,13 +37,17 @@ __author__ = 'kenton@google.com (Kenton Varda)'
import re
from google.apputils import basetest
from google.protobuf import text_format
from google.protobuf.internal import api_implementation
from google.protobuf.internal import test_util
from google.protobuf import unittest_pb2
from google.protobuf import unittest_mset_pb2
class TextFormatTest(basetest.TestCase):
from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_pb2
from google.protobuf import unittest_proto3_arena_pb2
from google.protobuf.internal import api_implementation
from google.protobuf.internal import _parameterized
from google.protobuf.internal import test_util
from google.protobuf import text_format
# Base class with some common functionality.
class TextFormatBase(basetest.TestCase):
def ReadGolden(self, golden_filename):
with test_util.GoldenFile(golden_filename) as f:
@ -57,73 +61,24 @@ class TextFormatTest(basetest.TestCase):
def CompareToGoldenText(self, text, golden_text):
self.assertMultiLineEqual(text, golden_text)
def testPrintAllFields(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_data_oneof_implemented.txt')
def RemoveRedundantZeros(self, text):
# Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove
# these zeros in order to match the golden file.
text = text.replace('e+0','e+').replace('e+0','e+') \
.replace('e-0','e-').replace('e-0','e-')
# Floating point fields are printed with .0 suffix even if they are
# actualy integer numbers.
text = re.compile('\.0$', re.MULTILINE).sub('', text)
return text
def testPrintInIndexOrder(self):
message = unittest_pb2.TestFieldOrderings()
message.my_string = '115'
message.my_int = 101
message.my_float = 111
message.optional_nested_message.oo = 0
message.optional_nested_message.bb = 1
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message, use_index_order=True)),
'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n'
'optional_nested_message {\n oo: 0\n bb: 1\n}\n')
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message)),
'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n'
'optional_nested_message {\n bb: 1\n oo: 0\n}\n')
def testPrintAllExtensions(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_extensions_data.txt')
@_parameterized.Parameters(
(unittest_pb2),
(unittest_proto3_arena_pb2))
class TextFormatTest(TextFormatBase):
def testPrintAllFieldsPointy(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(
text_format.MessageToString(message, pointy_brackets=True)),
'text_format_unittest_data_pointy_oneof.txt')
def testPrintAllExtensionsPointy(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(
message, pointy_brackets=True)),
'text_format_unittest_extensions_data_pointy.txt')
def testPrintMessageSet(self):
message = unittest_mset_pb2.TestMessageSetContainer()
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(
text_format.MessageToString(message),
'message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
def testPrintExotic(self):
message = unittest_pb2.TestAllTypes()
def testPrintExotic(self, message_module):
message = message_module.TestAllTypes()
message.repeated_int64.append(-9223372036854775808)
message.repeated_uint64.append(18446744073709551615)
message.repeated_double.append(123.456)
@ -142,61 +97,44 @@ class TextFormatTest(basetest.TestCase):
' "\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n'
'repeated_string: "\\303\\274\\352\\234\\237"\n')
def testPrintExoticUnicodeSubclass(self):
def testPrintExoticUnicodeSubclass(self, message_module):
class UnicodeSub(unicode):
pass
message = unittest_pb2.TestAllTypes()
message = message_module.TestAllTypes()
message.repeated_string.append(UnicodeSub(u'\u00fc\ua71f'))
self.CompareToGoldenText(
text_format.MessageToString(message),
'repeated_string: "\\303\\274\\352\\234\\237"\n')
def testPrintNestedMessageAsOneLine(self):
message = unittest_pb2.TestAllTypes()
def testPrintNestedMessageAsOneLine(self, message_module):
message = message_module.TestAllTypes()
msg = message.repeated_nested_message.add()
msg.bb = 42
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
'repeated_nested_message { bb: 42 }')
def testPrintRepeatedFieldsAsOneLine(self):
message = unittest_pb2.TestAllTypes()
def testPrintRepeatedFieldsAsOneLine(self, message_module):
message = message_module.TestAllTypes()
message.repeated_int32.append(1)
message.repeated_int32.append(1)
message.repeated_int32.append(3)
message.repeated_string.append("Google")
message.repeated_string.append("Zurich")
message.repeated_string.append('Google')
message.repeated_string.append('Zurich')
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
'repeated_int32: 1 repeated_int32: 1 repeated_int32: 3 '
'repeated_string: "Google" repeated_string: "Zurich"')
def testPrintNestedNewLineInStringAsOneLine(self):
message = unittest_pb2.TestAllTypes()
message.optional_string = "a\nnew\nline"
def testPrintNestedNewLineInStringAsOneLine(self, message_module):
message = message_module.TestAllTypes()
message.optional_string = 'a\nnew\nline'
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
'optional_string: "a\\nnew\\nline"')
def testPrintMessageSetAsOneLine(self):
message = unittest_mset_pb2.TestMessageSetContainer()
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
'message_set {'
' [protobuf_unittest.TestMessageSetExtension1] {'
' i: 23'
' }'
' [protobuf_unittest.TestMessageSetExtension2] {'
' str: \"foo\"'
' }'
' }')
def testPrintExoticAsOneLine(self):
message = unittest_pb2.TestAllTypes()
def testPrintExoticAsOneLine(self, message_module):
message = message_module.TestAllTypes()
message.repeated_int64.append(-9223372036854775808)
message.repeated_uint64.append(18446744073709551615)
message.repeated_double.append(123.456)
@ -216,8 +154,8 @@ class TextFormatTest(basetest.TestCase):
'"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""'
' repeated_string: "\\303\\274\\352\\234\\237"')
def testRoundTripExoticAsOneLine(self):
message = unittest_pb2.TestAllTypes()
def testRoundTripExoticAsOneLine(self, message_module):
message = message_module.TestAllTypes()
message.repeated_int64.append(-9223372036854775808)
message.repeated_uint64.append(18446744073709551615)
message.repeated_double.append(123.456)
@ -229,7 +167,7 @@ class TextFormatTest(basetest.TestCase):
# Test as_utf8 = False.
wire_text = text_format.MessageToString(
message, as_one_line=True, as_utf8=False)
parsed_message = unittest_pb2.TestAllTypes()
parsed_message = message_module.TestAllTypes()
r = text_format.Parse(wire_text, parsed_message)
self.assertIs(r, parsed_message)
self.assertEquals(message, parsed_message)
@ -237,25 +175,25 @@ class TextFormatTest(basetest.TestCase):
# Test as_utf8 = True.
wire_text = text_format.MessageToString(
message, as_one_line=True, as_utf8=True)
parsed_message = unittest_pb2.TestAllTypes()
parsed_message = message_module.TestAllTypes()
r = text_format.Parse(wire_text, parsed_message)
self.assertIs(r, parsed_message)
self.assertEquals(message, parsed_message,
'\n%s != %s' % (message, parsed_message))
def testPrintRawUtf8String(self):
message = unittest_pb2.TestAllTypes()
def testPrintRawUtf8String(self, message_module):
message = message_module.TestAllTypes()
message.repeated_string.append(u'\u00fc\ua71f')
text = text_format.MessageToString(message, as_utf8=True)
self.CompareToGoldenText(text, 'repeated_string: "\303\274\352\234\237"\n')
parsed_message = unittest_pb2.TestAllTypes()
parsed_message = message_module.TestAllTypes()
text_format.Parse(text, parsed_message)
self.assertEquals(message, parsed_message,
'\n%s != %s' % (message, parsed_message))
def testPrintFloatFormat(self):
def testPrintFloatFormat(self, message_module):
# Check that float_format argument is passed to sub-message formatting.
message = unittest_pb2.NestedTestAllTypes()
message = message_module.NestedTestAllTypes()
# We use 1.25 as it is a round number in binary. The proto 32-bit float
# will not gain additional imprecise digits as a 64-bit Python float and
# show up in its str. 32-bit 1.2 is noisy when extended to 64-bit:
@ -285,85 +223,24 @@ class TextFormatTest(basetest.TestCase):
self.RemoveRedundantZeros(text_message),
'payload {{ {} {} {} {} }}'.format(*formatted_fields))
def testMessageToString(self):
message = unittest_pb2.ForeignMessage()
def testMessageToString(self, message_module):
message = message_module.ForeignMessage()
message.c = 123
self.assertEqual('c: 123\n', str(message))
def RemoveRedundantZeros(self, text):
# Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove
# these zeros in order to match the golden file.
text = text.replace('e+0','e+').replace('e+0','e+') \
.replace('e-0','e-').replace('e-0','e-')
# Floating point fields are printed with .0 suffix even if they are
# actualy integer numbers.
text = re.compile('\.0$', re.MULTILINE).sub('', text)
return text
def testParseGolden(self):
golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt'))
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.Parse(golden_text, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEquals(message, parsed_message)
def testParseGoldenExtensions(self):
golden_text = '\n'.join(self.ReadGolden(
'text_format_unittest_extensions_data.txt'))
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Parse(golden_text, parsed_message)
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.assertEquals(message, parsed_message)
def testParseAllFields(self):
message = unittest_pb2.TestAllTypes()
def testParseAllFields(self, message_module):
message = message_module.TestAllTypes()
test_util.SetAllFields(message)
ascii_text = text_format.MessageToString(message)
parsed_message = unittest_pb2.TestAllTypes()
parsed_message = message_module.TestAllTypes()
text_format.Parse(ascii_text, parsed_message)
self.assertEqual(message, parsed_message)
test_util.ExpectAllFieldsSet(self, message)
if message_module is unittest_pb2:
test_util.ExpectAllFieldsSet(self, message)
def testParseAllExtensions(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
ascii_text = text_format.MessageToString(message)
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Parse(ascii_text, parsed_message)
self.assertEqual(message, parsed_message)
def testParseMessageSet(self):
message = unittest_pb2.TestAllTypes()
text = ('repeated_uint64: 1\n'
'repeated_uint64: 2\n')
text_format.Parse(text, message)
self.assertEqual(1, message.repeated_uint64[0])
self.assertEqual(2, message.repeated_uint64[1])
message = unittest_mset_pb2.TestMessageSetContainer()
text = ('message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
text_format.Parse(text, message)
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
self.assertEquals(23, message.message_set.Extensions[ext1].i)
self.assertEquals('foo', message.message_set.Extensions[ext2].str)
def testParseExotic(self):
message = unittest_pb2.TestAllTypes()
def testParseExotic(self, message_module):
message = message_module.TestAllTypes()
text = ('repeated_int64: -9223372036854775808\n'
'repeated_uint64: 18446744073709551615\n'
'repeated_double: 123.456\n'
@ -388,8 +265,8 @@ class TextFormatTest(basetest.TestCase):
self.assertEqual(u'\u00fc\ua71f', message.repeated_string[2])
self.assertEqual(u'\u00fc', message.repeated_string[3])
def testParseTrailingCommas(self):
message = unittest_pb2.TestAllTypes()
def testParseTrailingCommas(self, message_module):
message = message_module.TestAllTypes()
text = ('repeated_int64: 100;\n'
'repeated_int64: 200;\n'
'repeated_int64: 300,\n'
@ -403,51 +280,37 @@ class TextFormatTest(basetest.TestCase):
self.assertEqual(u'one', message.repeated_string[0])
self.assertEqual(u'two', message.repeated_string[1])
def testParseEmptyText(self):
message = unittest_pb2.TestAllTypes()
def testParseEmptyText(self, message_module):
message = message_module.TestAllTypes()
text = ''
text_format.Parse(text, message)
self.assertEquals(unittest_pb2.TestAllTypes(), message)
self.assertEquals(message_module.TestAllTypes(), message)
def testParseInvalidUtf8(self):
message = unittest_pb2.TestAllTypes()
def testParseInvalidUtf8(self, message_module):
message = message_module.TestAllTypes()
text = 'repeated_string: "\\xc3\\xc3"'
self.assertRaises(text_format.ParseError, text_format.Parse, text, message)
def testParseSingleWord(self):
message = unittest_pb2.TestAllTypes()
def testParseSingleWord(self, message_module):
message = message_module.TestAllTypes()
text = 'foo'
self.assertRaisesWithLiteralMatch(
self.assertRaisesRegexp(
text_format.ParseError,
('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named '
'"foo".'),
(r'1:1 : Message type "\w+.TestAllTypes" has no field named '
r'"foo".'),
text_format.Parse, text, message)
def testParseUnknownField(self):
message = unittest_pb2.TestAllTypes()
def testParseUnknownField(self, message_module):
message = message_module.TestAllTypes()
text = 'unknown_field: 8\n'
self.assertRaisesWithLiteralMatch(
self.assertRaisesRegexp(
text_format.ParseError,
('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named '
'"unknown_field".'),
(r'1:1 : Message type "\w+.TestAllTypes" has no field named '
r'"unknown_field".'),
text_format.Parse, text, message)
def testParseBadExtension(self):
message = unittest_pb2.TestAllExtensions()
text = '[unknown_extension]: 8\n'
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
'1:2 : Extension "unknown_extension" not registered.',
text_format.Parse, text, message)
message = unittest_pb2.TestAllTypes()
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
'extensions.'),
text_format.Parse, text, message)
def testParseGroupNotClosed(self):
message = unittest_pb2.TestAllTypes()
def testParseGroupNotClosed(self, message_module):
message = message_module.TestAllTypes()
text = 'RepeatedGroup: <'
self.assertRaisesWithLiteralMatch(
text_format.ParseError, '1:16 : Expected ">".',
@ -458,46 +321,46 @@ class TextFormatTest(basetest.TestCase):
text_format.ParseError, '1:16 : Expected "}".',
text_format.Parse, text, message)
def testParseEmptyGroup(self):
message = unittest_pb2.TestAllTypes()
def testParseEmptyGroup(self, message_module):
message = message_module.TestAllTypes()
text = 'OptionalGroup: {}'
text_format.Parse(text, message)
self.assertTrue(message.HasField('optionalgroup'))
message.Clear()
message = unittest_pb2.TestAllTypes()
message = message_module.TestAllTypes()
text = 'OptionalGroup: <>'
text_format.Parse(text, message)
self.assertTrue(message.HasField('optionalgroup'))
def testParseBadEnumValue(self):
message = unittest_pb2.TestAllTypes()
def testParseBadEnumValue(self, message_module):
message = message_module.TestAllTypes()
text = 'optional_nested_enum: BARR'
self.assertRaisesWithLiteralMatch(
self.assertRaisesRegexp(
text_format.ParseError,
('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
'has no value named BARR.'),
(r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
r'has no value named BARR.'),
text_format.Parse, text, message)
message = unittest_pb2.TestAllTypes()
message = message_module.TestAllTypes()
text = 'optional_nested_enum: 100'
self.assertRaisesWithLiteralMatch(
self.assertRaisesRegexp(
text_format.ParseError,
('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
'has no value with number 100.'),
(r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
r'has no value with number 100.'),
text_format.Parse, text, message)
def testParseBadIntValue(self):
message = unittest_pb2.TestAllTypes()
def testParseBadIntValue(self, message_module):
message = message_module.TestAllTypes()
text = 'optional_int32: bork'
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:17 : Couldn\'t parse integer: bork'),
text_format.Parse, text, message)
def testParseStringFieldUnescape(self):
message = unittest_pb2.TestAllTypes()
def testParseStringFieldUnescape(self, message_module):
message = message_module.TestAllTypes()
text = r'''repeated_string: "\xf\x62"
repeated_string: "\\xf\\x62"
repeated_string: "\\\xf\\\x62"
@ -516,40 +379,205 @@ class TextFormatTest(basetest.TestCase):
message.repeated_string[4])
self.assertEqual(SLASH + 'x20', message.repeated_string[5])
def testMergeDuplicateScalars(self):
message = unittest_pb2.TestAllTypes()
def testMergeDuplicateScalars(self, message_module):
message = message_module.TestAllTypes()
text = ('optional_int32: 42 '
'optional_int32: 67')
r = text_format.Merge(text, message)
self.assertIs(r, message)
self.assertEqual(67, message.optional_int32)
def testParseDuplicateScalars(self):
message = unittest_pb2.TestAllTypes()
text = ('optional_int32: 42 '
'optional_int32: 67')
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:36 : Message type "protobuf_unittest.TestAllTypes" should not '
'have multiple "optional_int32" fields.'),
text_format.Parse, text, message)
def testMergeDuplicateNestedMessageScalars(self):
message = unittest_pb2.TestAllTypes()
def testMergeDuplicateNestedMessageScalars(self, message_module):
message = message_module.TestAllTypes()
text = ('optional_nested_message { bb: 1 } '
'optional_nested_message { bb: 2 }')
r = text_format.Merge(text, message)
self.assertTrue(r is message)
self.assertEqual(2, message.optional_nested_message.bb)
def testParseDuplicateNestedMessageScalars(self):
def testParseOneof(self, message_module):
m = message_module.TestAllTypes()
m.oneof_uint32 = 11
m2 = message_module.TestAllTypes()
text_format.Parse(text_format.MessageToString(m), m2)
self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field'))
# These are tests that aren't fundamentally specific to proto2, but are at
# the moment because of differences between the proto2 and proto3 test schemas.
# Ideally the schemas would be made more similar so these tests could pass.
class OnlyWorksWithProto2RightNowTests(TextFormatBase):
def testParseGolden(self):
golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt'))
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.Parse(golden_text, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes()
text = ('optional_nested_message { bb: 1 } '
'optional_nested_message { bb: 2 }')
test_util.SetAllFields(message)
self.assertEquals(message, parsed_message)
def testPrintAllFields(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_data_oneof_implemented.txt')
def testPrintAllFieldsPointy(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(
text_format.MessageToString(message, pointy_brackets=True)),
'text_format_unittest_data_pointy_oneof.txt')
def testPrintInIndexOrder(self):
message = unittest_pb2.TestFieldOrderings()
message.my_string = '115'
message.my_int = 101
message.my_float = 111
message.optional_nested_message.oo = 0
message.optional_nested_message.bb = 1
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message, use_index_order=True)),
'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n'
'optional_nested_message {\n oo: 0\n bb: 1\n}\n')
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message)),
'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n'
'optional_nested_message {\n bb: 1\n oo: 0\n}\n')
def testMergeLinesGolden(self):
opened = self.ReadGolden('text_format_unittest_data.txt')
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.MergeLines(opened, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEqual(message, parsed_message)
def testParseLinesGolden(self):
opened = self.ReadGolden('text_format_unittest_data.txt')
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.ParseLines(opened, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEquals(message, parsed_message)
# Tests of proto2-only features (MessageSet and extensions).
class Proto2Tests(TextFormatBase):
def testPrintMessageSet(self):
message = unittest_mset_pb2.TestMessageSetContainer()
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(
text_format.MessageToString(message),
'message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
def testPrintMessageSetAsOneLine(self):
message = unittest_mset_pb2.TestMessageSetContainer()
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
'message_set {'
' [protobuf_unittest.TestMessageSetExtension1] {'
' i: 23'
' }'
' [protobuf_unittest.TestMessageSetExtension2] {'
' str: \"foo\"'
' }'
' }')
def testParseMessageSet(self):
message = unittest_pb2.TestAllTypes()
text = ('repeated_uint64: 1\n'
'repeated_uint64: 2\n')
text_format.Parse(text, message)
self.assertEqual(1, message.repeated_uint64[0])
self.assertEqual(2, message.repeated_uint64[1])
message = unittest_mset_pb2.TestMessageSetContainer()
text = ('message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
text_format.Parse(text, message)
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
self.assertEquals(23, message.message_set.Extensions[ext1].i)
self.assertEquals('foo', message.message_set.Extensions[ext2].str)
def testPrintAllExtensions(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_extensions_data.txt')
def testPrintAllExtensionsPointy(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(
message, pointy_brackets=True)),
'text_format_unittest_extensions_data_pointy.txt')
def testParseGoldenExtensions(self):
golden_text = '\n'.join(self.ReadGolden(
'text_format_unittest_extensions_data.txt'))
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Parse(golden_text, parsed_message)
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.assertEquals(message, parsed_message)
def testParseAllExtensions(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
ascii_text = text_format.MessageToString(message)
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Parse(ascii_text, parsed_message)
self.assertEqual(message, parsed_message)
def testParseBadExtension(self):
message = unittest_pb2.TestAllExtensions()
text = '[unknown_extension]: 8\n'
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" '
'should not have multiple "bb" fields.'),
'1:2 : Extension "unknown_extension" not registered.',
text_format.Parse, text, message)
message = unittest_pb2.TestAllTypes()
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
'extensions.'),
text_format.Parse, text, message)
def testMergeDuplicateExtensionScalars(self):
@ -572,32 +600,25 @@ class TextFormatTest(basetest.TestCase):
'"protobuf_unittest.optional_int32_extension" extensions.'),
text_format.Parse, text, message)
def testParseLinesGolden(self):
opened = self.ReadGolden('text_format_unittest_data.txt')
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.ParseLines(opened, parsed_message)
self.assertIs(r, parsed_message)
def testParseDuplicateNestedMessageScalars(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEquals(message, parsed_message)
def testMergeLinesGolden(self):
opened = self.ReadGolden('text_format_unittest_data.txt')
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.MergeLines(opened, parsed_message)
self.assertIs(r, parsed_message)
text = ('optional_nested_message { bb: 1 } '
'optional_nested_message { bb: 2 }')
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" '
'should not have multiple "bb" fields.'),
text_format.Parse, text, message)
def testParseDuplicateScalars(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEqual(message, parsed_message)
def testParseOneof(self):
m = unittest_pb2.TestAllTypes()
m.oneof_uint32 = 11
m2 = unittest_pb2.TestAllTypes()
text_format.Parse(text_format.MessageToString(m), m2)
self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field'))
text = ('optional_int32: 42 '
'optional_int32: 67')
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:36 : Message type "protobuf_unittest.TestAllTypes" should not '
'have multiple "optional_int32" fields.'),
text_format.Parse, text, message)
class TokenizerTest(basetest.TestCase):

8
python/google/protobuf/internal/type_checkers.py
View file

@ -59,6 +59,8 @@ from google.protobuf import descriptor
_FieldDescriptor = descriptor.FieldDescriptor
def SupportsOpenEnums(field_descriptor):
return field_descriptor.containing_type.syntax == "proto3"
def GetTypeChecker(field):
"""Returns a type checker for a message field of the specified types.
@ -74,7 +76,11 @@ def GetTypeChecker(field):
field.type == _FieldDescriptor.TYPE_STRING):
return UnicodeValueChecker()
if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM:
return EnumValueChecker(field.enum_type)
if SupportsOpenEnums(field):
# When open enums are supported, any int32 can be assigned.
return _VALUE_CHECKERS[_FieldDescriptor.CPPTYPE_INT32]
else:
return EnumValueChecker(field.enum_type)
return _VALUE_CHECKERS[field.cpp_type]

120
python/google/protobuf/internal/unknown_fields_test.py
View file

@ -38,6 +38,7 @@ __author__ = 'bohdank@google.com (Bohdan Koval)'
from google.apputils import basetest
from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_pb2
from google.protobuf import unittest_proto3_arena_pb2
from google.protobuf.internal import api_implementation
from google.protobuf.internal import encoder
from google.protobuf.internal import missing_enum_values_pb2
@ -45,10 +46,81 @@ from google.protobuf.internal import test_util
from google.protobuf.internal import type_checkers
class UnknownFieldsTest(basetest.TestCase):
def setUp(self):
self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
self.all_fields = unittest_pb2.TestAllTypes()
test_util.SetAllFields(self.all_fields)
self.all_fields_data = self.all_fields.SerializeToString()
self.empty_message = unittest_pb2.TestEmptyMessage()
self.empty_message.ParseFromString(self.all_fields_data)
def testSerialize(self):
data = self.empty_message.SerializeToString()
# Don't use assertEqual because we don't want to dump raw binary data to
# stdout.
self.assertTrue(data == self.all_fields_data)
def testSerializeProto3(self):
# Verify that proto3 doesn't preserve unknown fields.
message = unittest_proto3_arena_pb2.TestEmptyMessage()
message.ParseFromString(self.all_fields_data)
self.assertEqual(0, len(message.SerializeToString()))
def testByteSize(self):
self.assertEqual(self.all_fields.ByteSize(), self.empty_message.ByteSize())
def testListFields(self):
# Make sure ListFields doesn't return unknown fields.
self.assertEqual(0, len(self.empty_message.ListFields()))
def testSerializeMessageSetWireFormatUnknownExtension(self):
# Create a message using the message set wire format with an unknown
# message.
raw = unittest_mset_pb2.RawMessageSet()
# Add an unknown extension.
item = raw.item.add()
item.type_id = 1545009
message1 = unittest_mset_pb2.TestMessageSetExtension1()
message1.i = 12345
item.message = message1.SerializeToString()
serialized = raw.SerializeToString()
# Parse message using the message set wire format.
proto = unittest_mset_pb2.TestMessageSet()
proto.MergeFromString(serialized)
# Verify that the unknown extension is serialized unchanged
reserialized = proto.SerializeToString()
new_raw = unittest_mset_pb2.RawMessageSet()
new_raw.MergeFromString(reserialized)
self.assertEqual(raw, new_raw)
# C++ implementation for proto2 does not currently take into account unknown
# fields when checking equality.
#
# TODO(haberman): fix this.
@basetest.unittest.skipIf(
api_implementation.Type() == 'cpp' and api_implementation.Version() == 2,
'C++ implementation does not expose unknown fields to Python')
def testEquals(self):
message = unittest_pb2.TestEmptyMessage()
message.ParseFromString(self.all_fields_data)
self.assertEqual(self.empty_message, message)
self.all_fields.ClearField('optional_string')
message.ParseFromString(self.all_fields.SerializeToString())
self.assertNotEqual(self.empty_message, message)
@basetest.unittest.skipIf(
api_implementation.Type() == 'cpp' and api_implementation.Version() == 2,
'C++ implementation does not expose unknown fields to Python')
class UnknownFieldsTest(basetest.TestCase):
class UnknownFieldsAccessorsTest(basetest.TestCase):
def setUp(self):
self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
@ -98,13 +170,6 @@ class UnknownFieldsTest(basetest.TestCase):
value = self.GetField('optionalgroup')
self.assertEqual(self.all_fields.optionalgroup, value)
def testSerialize(self):
data = self.empty_message.SerializeToString()
# Don't use assertEqual because we don't want to dump raw binary data to
# stdout.
self.assertTrue(data == self.all_fields_data)
def testCopyFrom(self):
message = unittest_pb2.TestEmptyMessage()
message.CopyFrom(self.empty_message)
@ -132,51 +197,12 @@ class UnknownFieldsTest(basetest.TestCase):
self.empty_message.Clear()
self.assertEqual(0, len(self.empty_message._unknown_fields))
def testByteSize(self):
self.assertEqual(self.all_fields.ByteSize(), self.empty_message.ByteSize())
def testUnknownExtensions(self):
message = unittest_pb2.TestEmptyMessageWithExtensions()
message.ParseFromString(self.all_fields_data)
self.assertEqual(self.empty_message._unknown_fields,
message._unknown_fields)
def testListFields(self):
# Make sure ListFields doesn't return unknown fields.
self.assertEqual(0, len(self.empty_message.ListFields()))
def testSerializeMessageSetWireFormatUnknownExtension(self):
# Create a message using the message set wire format with an unknown
# message.
raw = unittest_mset_pb2.RawMessageSet()
# Add an unknown extension.
item = raw.item.add()
item.type_id = 1545009
message1 = unittest_mset_pb2.TestMessageSetExtension1()
message1.i = 12345
item.message = message1.SerializeToString()
serialized = raw.SerializeToString()
# Parse message using the message set wire format.
proto = unittest_mset_pb2.TestMessageSet()
proto.MergeFromString(serialized)
# Verify that the unknown extension is serialized unchanged
reserialized = proto.SerializeToString()
new_raw = unittest_mset_pb2.RawMessageSet()
new_raw.MergeFromString(reserialized)
self.assertEqual(raw, new_raw)
def testEquals(self):
message = unittest_pb2.TestEmptyMessage()
message.ParseFromString(self.all_fields_data)
self.assertEqual(self.empty_message, message)
self.all_fields.ClearField('optional_string')
message.ParseFromString(self.all_fields.SerializeToString())
self.assertNotEqual(self.empty_message, message)
@basetest.unittest.skipIf(

1
python/google/protobuf/message.py
View file

@ -36,7 +36,6 @@
__author__ = 'robinson@google.com (Will Robinson)'
class Error(Exception): pass
class DecodeError(Error): pass
class EncodeError(Error): pass

1579
python/google/protobuf/pyext/descriptor.cc
View file

File diff suppressed because it is too large Load diff

118
python/google/protobuf/pyext/descriptor.h
View file

@ -34,105 +34,55 @@
#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__
#include <Python.h>
#include <structmember.h>
#include <google/protobuf/stubs/hash.h>
#include <google/protobuf/descriptor.h>
#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
typedef int Py_ssize_t;
#define PY_SSIZE_T_MAX INT_MAX
#define PY_SSIZE_T_MIN INT_MIN
#endif
namespace google {
namespace protobuf {
namespace python {
typedef struct CMessageDescriptor {
PyObject_HEAD
extern PyTypeObject PyMessageDescriptor_Type;
extern PyTypeObject PyFieldDescriptor_Type;
extern PyTypeObject PyEnumDescriptor_Type;
extern PyTypeObject PyEnumValueDescriptor_Type;
extern PyTypeObject PyFileDescriptor_Type;
extern PyTypeObject PyOneofDescriptor_Type;
// The proto2 descriptor that this object represents.
const google::protobuf::Descriptor* descriptor;
} CMessageDescriptor;
// Return a new reference to a Descriptor object.
// The C++ pointer is usually borrowed from the global DescriptorPool.
// In any case, it must stay alive as long as the Python object.
PyObject* PyMessageDescriptor_New(const Descriptor* descriptor);
PyObject* PyFieldDescriptor_New(const FieldDescriptor* descriptor);
PyObject* PyEnumDescriptor_New(const EnumDescriptor* descriptor);
PyObject* PyEnumValueDescriptor_New(const EnumValueDescriptor* descriptor);
PyObject* PyOneofDescriptor_New(const OneofDescriptor* descriptor);
PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor);
// Alternate constructor of PyFileDescriptor, used when we already have a
// serialized FileDescriptorProto that can be cached.
// Returns a new reference.
PyObject* PyFileDescriptor_NewWithPb(const FileDescriptor* file_descriptor,
PyObject* serialized_pb);
typedef struct CFieldDescriptor {
PyObject_HEAD
// The proto2 descriptor that this object represents.
const google::protobuf::FieldDescriptor* descriptor;
} CFieldDescriptor;
// Wraps operations to the global DescriptorPool which contains information
// about all messages and fields.
//
// There is normally one pool per process. We make it a Python object only
// because it contains many Python references.
// TODO(amauryfa): See whether such objects can appear in reference cycles, and
// consider adding support for the cyclic GC.
//
// "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool
// namespace.
typedef struct PyDescriptorPool {
PyObject_HEAD
google::protobuf::DescriptorPool* pool;
// Make our own mapping to retrieve Python classes from C++ descriptors.
//
// Descriptor pointers stored here are owned by the DescriptorPool above.
// Python references to classes are owned by this PyDescriptorPool.
typedef hash_map<const Descriptor *, PyObject *> ClassesByMessageMap;
ClassesByMessageMap *classes_by_descriptor;
} PyDescriptorPool;
extern PyTypeObject CMessageDescriptor_Type;
extern PyTypeObject CFieldDescriptor_Type;
extern PyTypeObject PyDescriptorPool_Type;
namespace cdescriptor_pool {
// Builds a new DescriptorPool. Normally called only once per process.
PyDescriptorPool* NewDescriptorPool();
// Looks up a message by name.
// Returns a message Descriptor, or NULL if not found.
const google::protobuf::Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
const string& name);
// Registers a new Python class for the given message descriptor.
// Returns the message Descriptor.
// On error, returns NULL with a Python exception set.
const google::protobuf::Descriptor* RegisterMessageClass(
PyDescriptorPool* self, PyObject *message_class, PyObject *descriptor);
// Retrieves the Python class registered with the given message descriptor.
//
// Returns a *borrowed* reference if found, otherwise returns NULL with an
// Return the C++ descriptor pointer.
// This function checks the parameter type; on error, return NULL with a Python
// exception set.
PyObject *GetMessageClass(PyDescriptorPool* self,
const Descriptor *message_descriptor);
const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj);
const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj);
// Looks up a field by name. Returns a CDescriptor corresponding to
// the field on success, or NULL on failure.
// Returns the raw C++ pointer.
const void* PyDescriptor_AsVoidPtr(PyObject* obj);
// Check that the calling Python code is the global scope of a _pb2.py module.
// This function is used to support the current code generated by the proto
// compiler, which insists on modifying descriptors after they have been
// created.
//
// Returns a new reference.
PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name);
// Looks up an extension by name. Returns a CDescriptor corresponding
// to the field on success, or NULL on failure.
// stacklevel indicates which Python frame should be the _pb2.py module.
//
// Returns a new reference.
PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg);
} // namespace cdescriptor_pool
// Don't use this function outside descriptor classes.
bool _CalledFromGeneratedFile(int stacklevel);
PyObject* Python_BuildFile(PyObject* ignored, PyObject* args);
bool InitDescriptor();
} // namespace python

1564
python/google/protobuf/pyext/descriptor_containers.cc Normal file
View file

File diff suppressed because it is too large Load diff

95
python/google/protobuf/pyext/descriptor_containers.h Normal file
View file

@ -0,0 +1,95 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Mappings and Sequences of descriptors.
// They implement containers like fields_by_name, EnumDescriptor.values...
// See descriptor_containers.cc for more description.
#include <Python.h>
namespace google {
namespace protobuf {
class Descriptor;
class FileDescriptor;
class EnumDescriptor;
class OneofDescriptor;
namespace python {
// Initialize the various types and objects.
bool InitDescriptorMappingTypes();
// Each function below returns a Mapping, or a Sequence of descriptors.
// They all return a new reference.
namespace message_descriptor {
PyObject* NewMessageFieldsByName(const Descriptor* descriptor);
PyObject* NewMessageFieldsByNumber(const Descriptor* descriptor);
PyObject* NewMessageFieldsSeq(const Descriptor* descriptor);
PyObject* NewMessageNestedTypesSeq(const Descriptor* descriptor);
PyObject* NewMessageNestedTypesByName(const Descriptor* descriptor);
PyObject* NewMessageEnumsByName(const Descriptor* descriptor);
PyObject* NewMessageEnumsSeq(const Descriptor* descriptor);
PyObject* NewMessageEnumValuesByName(const Descriptor* descriptor);
PyObject* NewMessageExtensionsByName(const Descriptor* descriptor);
PyObject* NewMessageExtensionsSeq(const Descriptor* descriptor);
PyObject* NewMessageOneofsByName(const Descriptor* descriptor);
PyObject* NewMessageOneofsSeq(const Descriptor* descriptor);
} // namespace message_descriptor
namespace enum_descriptor {
PyObject* NewEnumValuesByName(const EnumDescriptor* descriptor);
PyObject* NewEnumValuesByNumber(const EnumDescriptor* descriptor);
PyObject* NewEnumValuesSeq(const EnumDescriptor* descriptor);
} // namespace enum_descriptor
namespace oneof_descriptor {
PyObject* NewOneofFieldsSeq(const OneofDescriptor* descriptor);
} // namespace oneof_descriptor
namespace file_descriptor {
PyObject* NewFileMessageTypesByName(const FileDescriptor* descriptor);
PyObject* NewFileEnumTypesByName(const FileDescriptor* descriptor);
PyObject* NewFileExtensionsByName(const FileDescriptor* descriptor);
PyObject* NewFileDependencies(const FileDescriptor* descriptor);
PyObject* NewFilePublicDependencies(const FileDescriptor* descriptor);
} // namespace file_descriptor
} // namespace python
} // namespace protobuf
} // namespace google

58
python/google/protobuf/pyext/descriptor_cpp2_test.py
View file

@ -1,58 +0,0 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for google.protobuf.pyext behavior."""
__author__ = 'anuraag@google.com (Anuraag Agrawal)'
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
# We must set the implementation version above before the google3 imports.
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
# Run all tests from the original module by putting them in our namespace.
# pylint: disable=wildcard-import
from google.protobuf.internal.descriptor_test import *
class ConfirmCppApi2Test(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
if __name__ == '__main__':
basetest.main()

370
python/google/protobuf/pyext/descriptor_pool.cc Normal file
View file

@ -0,0 +1,370 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Implements the DescriptorPool, which collects all descriptors.
#include <Python.h>
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/pyext/descriptor_pool.h>
#include <google/protobuf/pyext/descriptor.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#define C(str) const_cast<char*>(str)
#if PY_MAJOR_VERSION >= 3
#define PyString_FromStringAndSize PyUnicode_FromStringAndSize
#if PY_VERSION_HEX < 0x03030000
#error "Python 3.0 - 3.2 are not supported."
#endif
#define PyString_AsStringAndSize(ob, charpp, sizep) \
(PyUnicode_Check(ob)? \
((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \
PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
#endif
namespace google {
namespace protobuf {
namespace python {
namespace cdescriptor_pool {
PyDescriptorPool* NewDescriptorPool() {
PyDescriptorPool* cdescriptor_pool = PyObject_New(
PyDescriptorPool, &PyDescriptorPool_Type);
if (cdescriptor_pool == NULL) {
return NULL;
}
// Build a DescriptorPool for messages only declared in Python libraries.
// generated_pool() contains all messages linked in C++ libraries, and is used
// as underlay.
cdescriptor_pool->pool = new DescriptorPool(DescriptorPool::generated_pool());
// TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same
// storage.
cdescriptor_pool->classes_by_descriptor =
new PyDescriptorPool::ClassesByMessageMap();
cdescriptor_pool->interned_descriptors =
new hash_map<const void*, PyObject *>();
cdescriptor_pool->descriptor_options =
new hash_map<const void*, PyObject *>();
return cdescriptor_pool;
}
static void Dealloc(PyDescriptorPool* self) {
typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator;
for (iterator it = self->classes_by_descriptor->begin();
it != self->classes_by_descriptor->end(); ++it) {
Py_DECREF(it->second);
}
delete self->classes_by_descriptor;
delete self->interned_descriptors; // its references were borrowed.
for (hash_map<const void*, PyObject*>::iterator it =
self->descriptor_options->begin();
it != self->descriptor_options->end(); ++it) {
Py_DECREF(it->second);
}
delete self->descriptor_options;
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const Descriptor* message_descriptor =
self->pool->FindMessageTypeByName(string(name, name_size));
if (message_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find message %.200s", name);
return NULL;
}
return PyMessageDescriptor_New(message_descriptor);
}
// Add a message class to our database.
const Descriptor* RegisterMessageClass(
PyDescriptorPool* self, PyObject *message_class, PyObject* descriptor) {
ScopedPyObjectPtr full_message_name(
PyObject_GetAttrString(descriptor, "full_name"));
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(full_message_name, &name, &name_size) < 0) {
return NULL;
}
const Descriptor *message_descriptor =
self->pool->FindMessageTypeByName(string(name, name_size));
if (!message_descriptor) {
PyErr_Format(PyExc_TypeError, "Could not find C++ descriptor for '%s'",
name);
return NULL;
}
Py_INCREF(message_class);
typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator;
std::pair<iterator, bool> ret = self->classes_by_descriptor->insert(
std::make_pair(message_descriptor, message_class));
if (!ret.second) {
// Update case: DECREF the previous value.
Py_DECREF(ret.first->second);
ret.first->second = message_class;
}
return message_descriptor;
}
// Retrieve the message class added to our database.
PyObject *GetMessageClass(PyDescriptorPool* self,
const Descriptor *message_descriptor) {
typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator;
iterator ret = self->classes_by_descriptor->find(message_descriptor);
if (ret == self->classes_by_descriptor->end()) {
PyErr_Format(PyExc_TypeError, "No message class registered for '%s'",
message_descriptor->full_name().c_str());
return NULL;
} else {
return ret->second;
}
}
PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const FieldDescriptor* field_descriptor =
self->pool->FindFieldByName(string(name, name_size));
if (field_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s",
name);
return NULL;
}
return PyFieldDescriptor_New(field_descriptor);
}
PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const FieldDescriptor* field_descriptor =
self->pool->FindExtensionByName(string(name, name_size));
if (field_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", name);
return NULL;
}
return PyFieldDescriptor_New(field_descriptor);
}
PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const EnumDescriptor* enum_descriptor =
self->pool->FindEnumTypeByName(string(name, name_size));
if (enum_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find enum %.200s", name);
return NULL;
}
return PyEnumDescriptor_New(enum_descriptor);
}
PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
return NULL;
}
const OneofDescriptor* oneof_descriptor =
self->pool->FindOneofByName(string(name, name_size));
if (oneof_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find oneof %.200s", name);
return NULL;
}
return PyOneofDescriptor_New(oneof_descriptor);
}
static PyMethodDef Methods[] = {
{ C("FindFieldByName"),
(PyCFunction)FindFieldByName,
METH_O,
C("Searches for a field descriptor by full name.") },
{ C("FindExtensionByName"),
(PyCFunction)FindExtensionByName,
METH_O,
C("Searches for extension descriptor by full name.") },
{NULL}
};
} // namespace cdescriptor_pool
PyTypeObject PyDescriptorPool_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
C("google.protobuf.internal."
"_message.DescriptorPool"), // tp_name
sizeof(PyDescriptorPool), // tp_basicsize
0, // tp_itemsize
(destructor)cdescriptor_pool::Dealloc, // tp_dealloc
0, // tp_print
0, // tp_getattr
0, // tp_setattr
0, // tp_compare
0, // tp_repr
0, // tp_as_number
0, // tp_as_sequence
0, // tp_as_mapping
0, // tp_hash
0, // tp_call
0, // tp_str
0, // tp_getattro
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
C("A Descriptor Pool"), // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
0, // tp_weaklistoffset
0, // tp_iter
0, // tp_iternext
cdescriptor_pool::Methods, // tp_methods
0, // tp_members
0, // tp_getset
0, // tp_base
0, // tp_dict
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
0, // tp_init
0, // tp_alloc
0, // tp_new
PyObject_Del, // tp_free
};
// The code below loads new Descriptors from a serialized FileDescriptorProto.
// Collects errors that occur during proto file building to allow them to be
// propagated in the python exception instead of only living in ERROR logs.
class BuildFileErrorCollector : public DescriptorPool::ErrorCollector {
public:
BuildFileErrorCollector() : error_message(""), had_errors(false) {}
void AddError(const string& filename, const string& element_name,
const Message* descriptor, ErrorLocation location,
const string& message) {
// Replicates the logging behavior that happens in the C++ implementation
// when an error collector is not passed in.
if (!had_errors) {
error_message +=
("Invalid proto descriptor for file \"" + filename + "\":\n");
}
// As this only happens on failure and will result in the program not
// running at all, no effort is made to optimize this string manipulation.
error_message += (" " + element_name + ": " + message + "\n");
}
string error_message;
bool had_errors;
};
PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) {
char* message_type;
Py_ssize_t message_len;
if (PyBytes_AsStringAndSize(serialized_pb, &message_type, &message_len) < 0) {
return NULL;
}
FileDescriptorProto file_proto;
if (!file_proto.ParseFromArray(message_type, message_len)) {
PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!");
return NULL;
}
// If the file was already part of a C++ library, all its descriptors are in
// the underlying pool. No need to do anything else.
const FileDescriptor* generated_file =
DescriptorPool::generated_pool()->FindFileByName(file_proto.name());
if (generated_file != NULL) {
return PyFileDescriptor_NewWithPb(generated_file, serialized_pb);
}
BuildFileErrorCollector error_collector;
const FileDescriptor* descriptor =
GetDescriptorPool()->pool->BuildFileCollectingErrors(file_proto,
&error_collector);
if (descriptor == NULL) {
PyErr_Format(PyExc_TypeError,
"Couldn't build proto file into descriptor pool!\n%s",
error_collector.error_message.c_str());
return NULL;
}
return PyFileDescriptor_NewWithPb(descriptor, serialized_pb);
}
static PyDescriptorPool* global_cdescriptor_pool = NULL;
bool InitDescriptorPool() {
if (PyType_Ready(&PyDescriptorPool_Type) < 0)
return false;
global_cdescriptor_pool = cdescriptor_pool::NewDescriptorPool();
if (global_cdescriptor_pool == NULL) {
return false;
}
return true;
}
PyDescriptorPool* GetDescriptorPool() {
return global_cdescriptor_pool;
}
} // namespace python
} // namespace protobuf
} // namespace google

152
python/google/protobuf/pyext/descriptor_pool.h Normal file
View file

@ -0,0 +1,152 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__
#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__
#include <Python.h>
#include <google/protobuf/stubs/hash.h>
#include <google/protobuf/descriptor.h>
namespace google {
namespace protobuf {
namespace python {
// Wraps operations to the global DescriptorPool which contains information
// about all messages and fields.
//
// There is normally one pool per process. We make it a Python object only
// because it contains many Python references.
// TODO(amauryfa): See whether such objects can appear in reference cycles, and
// consider adding support for the cyclic GC.
//
// "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool
// namespace.
typedef struct PyDescriptorPool {
PyObject_HEAD
DescriptorPool* pool;
// Make our own mapping to retrieve Python classes from C++ descriptors.
//
// Descriptor pointers stored here are owned by the DescriptorPool above.
// Python references to classes are owned by this PyDescriptorPool.
typedef hash_map<const Descriptor*, PyObject*> ClassesByMessageMap;
ClassesByMessageMap* classes_by_descriptor;
// Store interned descriptors, so that the same C++ descriptor yields the same
// Python object. Objects are not immortal: this map does not own the
// references, and items are deleted when the last reference to the object is
// released.
// This is enough to support the "is" operator on live objects.
// All descriptors are stored here.
hash_map<const void*, PyObject*>* interned_descriptors;
// Cache the options for any kind of descriptor.
// Descriptor pointers are owned by the DescriptorPool above.
// Python objects are owned by the map.
hash_map<const void*, PyObject*>* descriptor_options;
} PyDescriptorPool;
extern PyTypeObject PyDescriptorPool_Type;
namespace cdescriptor_pool {
// Builds a new DescriptorPool. Normally called only once per process.
PyDescriptorPool* NewDescriptorPool();
// Looks up a message by name.
// Returns a message Descriptor, or NULL if not found.
const Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
const string& name);
// Registers a new Python class for the given message descriptor.
// Returns the message Descriptor.
// On error, returns NULL with a Python exception set.
const Descriptor* RegisterMessageClass(
PyDescriptorPool* self, PyObject* message_class, PyObject* descriptor);
// Retrieves the Python class registered with the given message descriptor.
//
// Returns a *borrowed* reference if found, otherwise returns NULL with an
// exception set.
PyObject* GetMessageClass(PyDescriptorPool* self,
const Descriptor* message_descriptor);
// Looks up a message by name. Returns a PyMessageDescriptor corresponding to
// the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* name);
// Looks up a field by name. Returns a PyFieldDescriptor corresponding to
// the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name);
// Looks up an extension by name. Returns a PyFieldDescriptor corresponding
// to the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg);
// Looks up an enum type by name. Returns a PyEnumDescriptor corresponding
// to the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg);
// Looks up a oneof by name. Returns a COneofDescriptor corresponding
// to the oneof on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg);
} // namespace cdescriptor_pool
// Implement the Python "_BuildFile" method, it takes a serialized
// FileDescriptorProto, and adds it to the C++ DescriptorPool.
// It returns a new FileDescriptor object, or NULL when an exception is raised.
PyObject* Python_BuildFile(PyObject* ignored, PyObject* args);
// Retrieve the global descriptor pool owned by the _message module.
PyDescriptorPool* GetDescriptorPool();
// Initialize objects used by this module.
bool InitDescriptorPool();
} // namespace python
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__

25
python/google/protobuf/pyext/extension_dict.cc
View file

@ -38,6 +38,7 @@
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/message.h>
#include <google/protobuf/pyext/descriptor.h>
#include <google/protobuf/pyext/descriptor_pool.h>
#include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/repeated_composite_container.h>
#include <google/protobuf/pyext/repeated_scalar_container.h>
@ -48,13 +49,11 @@ namespace google {
namespace protobuf {
namespace python {
extern google::protobuf::DynamicMessageFactory* global_message_factory;
namespace extension_dict {
// TODO(tibell): Always use self->message for clarity, just like in
// RepeatedCompositeContainer.
static google::protobuf::Message* GetMessage(ExtensionDict* self) {
static Message* GetMessage(ExtensionDict* self) {
if (self->parent != NULL) {
return self->parent->message;
} else {
@ -73,10 +72,9 @@ PyObject* len(ExtensionDict* self) {
// TODO(tibell): Use VisitCompositeField.
int ReleaseExtension(ExtensionDict* self,
PyObject* extension,
const google::protobuf::FieldDescriptor* descriptor) {
if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
if (descriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
const FieldDescriptor* descriptor) {
if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
if (repeated_composite_container::Release(
reinterpret_cast<RepeatedCompositeContainer*>(
extension)) < 0) {
@ -89,8 +87,7 @@ int ReleaseExtension(ExtensionDict* self,
return -1;
}
}
} else if (descriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
} else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
if (cmessage::ReleaseSubMessage(
GetMessage(self), descriptor,
reinterpret_cast<CMessage*>(extension)) < 0) {
@ -102,8 +99,7 @@ int ReleaseExtension(ExtensionDict* self,
}
PyObject* subscript(ExtensionDict* self, PyObject* key) {
const google::protobuf::FieldDescriptor* descriptor =
cmessage::GetExtensionDescriptor(key);
const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key);
if (descriptor == NULL) {
return NULL;
}
@ -162,8 +158,7 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
}
int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
const google::protobuf::FieldDescriptor* descriptor =
cmessage::GetExtensionDescriptor(key);
const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key);
if (descriptor == NULL) {
return -1;
}
@ -187,7 +182,7 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
}
PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
const google::protobuf::FieldDescriptor* descriptor =
const FieldDescriptor* descriptor =
cmessage::GetExtensionDescriptor(extension);
if (descriptor == NULL) {
return NULL;
@ -208,7 +203,7 @@ PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
}
PyObject* HasExtension(ExtensionDict* self, PyObject* extension) {
const google::protobuf::FieldDescriptor* descriptor =
const FieldDescriptor* descriptor =
cmessage::GetExtensionDescriptor(extension);
if (descriptor == NULL) {
return NULL;

3
python/google/protobuf/pyext/extension_dict.h
View file

@ -41,7 +41,6 @@
#include <google/protobuf/stubs/shared_ptr.h>
#endif
namespace google {
namespace protobuf {
@ -94,7 +93,7 @@ PyObject* len(ExtensionDict* self);
// Returns 0 on success, -1 on failure.
int ReleaseExtension(ExtensionDict* self,
PyObject* extension,
const google::protobuf::FieldDescriptor* descriptor);
const FieldDescriptor* descriptor);
// Gets an extension from the dict for the given extension descriptor.
//

1078
python/google/protobuf/pyext/message.cc
View file

File diff suppressed because it is too large Load diff

50
python/google/protobuf/pyext/message.h
View file

@ -42,7 +42,6 @@
#endif
#include <string>
namespace google {
namespace protobuf {
@ -50,12 +49,12 @@ class Message;
class Reflection;
class FieldDescriptor;
class Descriptor;
class DynamicMessageFactory;
using internal::shared_ptr;
namespace python {
struct PyDescriptorPool;
struct ExtensionDict;
typedef struct CMessage {
@ -84,7 +83,7 @@ typedef struct CMessage {
// Used together with the parent's message when making a default message
// instance mutable.
// The pointer is owned by the global DescriptorPool.
const google::protobuf::FieldDescriptor* parent_field_descriptor;
const FieldDescriptor* parent_field_descriptor;
// Pointer to the C++ Message object for this CMessage. The
// CMessage does not own this pointer.
@ -115,27 +114,26 @@ namespace cmessage {
// Internal function to create a new empty Message Python object, but with empty
// pointers to the C++ objects.
// The caller must fill self->message, self->owner and eventually self->parent.
CMessage* NewEmptyMessage(PyObject* type,
const google::protobuf::Descriptor* descriptor);
CMessage* NewEmptyMessage(PyObject* type, const Descriptor* descriptor);
// Release a submessage from its proto tree, making it a new top-level messgae.
// A new message will be created if this is a read-only default instance.
//
// Corresponds to reflection api method ReleaseMessage.
int ReleaseSubMessage(google::protobuf::Message* message,
const google::protobuf::FieldDescriptor* field_descriptor,
int ReleaseSubMessage(Message* message,
const FieldDescriptor* field_descriptor,
CMessage* child_cmessage);
// Retrieves the C++ descriptor of a Python Extension descriptor.
// On error, return NULL with an exception set.
const google::protobuf::FieldDescriptor* GetExtensionDescriptor(PyObject* extension);
const FieldDescriptor* GetExtensionDescriptor(PyObject* extension);
// Initializes a new CMessage instance for a submessage. Only called once per
// submessage as the result is cached in composite_fields.
//
// Corresponds to reflection api method GetMessage.
PyObject* InternalGetSubMessage(
CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor);
CMessage* self, const FieldDescriptor* field_descriptor);
// Deletes a range of C++ submessages in a repeated field (following a
// removal in a RepeatedCompositeContainer).
@ -146,20 +144,20 @@ PyObject* InternalGetSubMessage(
// by slice will be removed from cmessage_list by this function.
//
// Corresponds to reflection api method RemoveLast.
int InternalDeleteRepeatedField(google::protobuf::Message* message,
const google::protobuf::FieldDescriptor* field_descriptor,
int InternalDeleteRepeatedField(Message* message,
const FieldDescriptor* field_descriptor,
PyObject* slice, PyObject* cmessage_list);
// Sets the specified scalar value to the message.
int InternalSetScalar(CMessage* self,
const google::protobuf::FieldDescriptor* field_descriptor,
const FieldDescriptor* field_descriptor,
PyObject* value);
// Retrieves the specified scalar value from the message.
//
// Returns a new python reference.
PyObject* InternalGetScalar(CMessage* self,
const google::protobuf::FieldDescriptor* field_descriptor);
const FieldDescriptor* field_descriptor);
// Clears the message, removing all contained data. Extension dictionary and
// submessages are released first if there are remaining external references.
@ -175,8 +173,7 @@ PyObject* Clear(CMessage* self);
//
// Corresponds to reflection api method ClearField.
PyObject* ClearFieldByDescriptor(
CMessage* self,
const google::protobuf::FieldDescriptor* descriptor);
CMessage* self, const FieldDescriptor* descriptor);
// Clears the data for the given field name. The message is released if there
// are any external references.
@ -189,7 +186,7 @@ PyObject* ClearField(CMessage* self, PyObject* arg);
//
// Corresponds to reflection api method HasField
PyObject* HasFieldByDescriptor(
CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor);
CMessage* self, const FieldDescriptor* field_descriptor);
// Checks if the message has the named field.
//
@ -220,18 +217,16 @@ int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner);
int AssureWritable(CMessage* self);
DynamicMessageFactory* GetMessageFactory();
} // namespace cmessage
// Retrieve the global descriptor pool owned by the _message module.
PyDescriptorPool* GetDescriptorPool();
/* Is 64bit */
#define IS_64BIT (SIZEOF_LONG == 8)
#define FIELD_IS_REPEATED(field_descriptor) \
((field_descriptor)->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
((field_descriptor)->label() == FieldDescriptor::LABEL_REPEATED)
#define GOOGLE_CHECK_GET_INT32(arg, value, err) \
int32 value; \
@ -294,18 +289,17 @@ bool CheckAndGetDouble(PyObject* arg, double* value);
bool CheckAndGetFloat(PyObject* arg, float* value);
bool CheckAndGetBool(PyObject* arg, bool* value);
bool CheckAndSetString(
PyObject* arg, google::protobuf::Message* message,
const google::protobuf::FieldDescriptor* descriptor,
const google::protobuf::Reflection* reflection,
PyObject* arg, Message* message,
const FieldDescriptor* descriptor,
const Reflection* reflection,
bool append,
int index);
PyObject* ToStringObject(
const google::protobuf::FieldDescriptor* descriptor, string value);
PyObject* ToStringObject(const FieldDescriptor* descriptor, string value);
// Check if the passed field descriptor belongs to the given message.
// If not, return false and set a Python exception (a KeyError)
bool CheckFieldBelongsToMessage(const google::protobuf::FieldDescriptor* field_descriptor,
const google::protobuf::Message* message);
bool CheckFieldBelongsToMessage(const FieldDescriptor* field_descriptor,
const Message* message);
extern PyObject* PickleError_class;

56
python/google/protobuf/pyext/message_factory_cpp2_test.py
View file

@ -1,56 +0,0 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for google.protobuf.message_factory."""
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
# We must set the implementation version above before the google3 imports.
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
# Run all tests from the original module by putting them in our namespace.
# pylint: disable=wildcard-import
from google.protobuf.internal.message_factory_test import *
class ConfirmCppApi2Test(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
if __name__ == '__main__':
basetest.main()

94
python/google/protobuf/pyext/reflection_cpp2_generated_test.py
View file

@ -1,94 +0,0 @@
#! /usr/bin/python
# -*- coding: utf-8 -*-
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Unittest for reflection.py, which tests the generated C++ implementation."""
__author__ = 'jasonh@google.com (Jason Hsueh)'
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
from google.apputils import basetest
from google.protobuf.internal import api_implementation
from google.protobuf.internal import more_extensions_dynamic_pb2
from google.protobuf.internal import more_extensions_pb2
from google.protobuf.internal.reflection_test import *
class ReflectionCppTest(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
def testExtensionOfGeneratedTypeInDynamicFile(self):
"""Tests that a file built dynamically can extend a generated C++ type.
The C++ implementation uses a DescriptorPool that has the generated
DescriptorPool as an underlay. Typically, a type can only find
extensions in its own pool. With the python C-extension, the generated C++
extendee may be available, but not the extension. This tests that the
C-extension implements the correct special handling to make such extensions
available.
"""
pb1 = more_extensions_pb2.ExtendedMessage()
# Test that basic accessors work.
self.assertFalse(
pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension))
self.assertFalse(
pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension))
pb1.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension] = 17
pb1.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a = 24
self.assertTrue(
pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension))
self.assertTrue(
pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension))
# Now serialize the data and parse to a new message.
pb2 = more_extensions_pb2.ExtendedMessage()
pb2.MergeFromString(pb1.SerializeToString())
self.assertTrue(
pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension))
self.assertTrue(
pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension))
self.assertEqual(
17, pb2.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension])
self.assertEqual(
24,
pb2.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a)
if __name__ == '__main__':
basetest.main()

57
python/google/protobuf/pyext/repeated_composite_container.cc
View file

@ -56,8 +56,6 @@ namespace google {
namespace protobuf {
namespace python {
extern google::protobuf::DynamicMessageFactory* global_message_factory;
namespace repeated_composite_container {
// TODO(tibell): We might also want to check:
@ -120,9 +118,9 @@ static int InternalQuickSort(RepeatedCompositeContainer* self,
GOOGLE_CHECK_ATTACHED(self);
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor;
Message* message = self->message;
const Reflection* reflection = message->GetReflection();
const FieldDescriptor* descriptor = self->parent_field_descriptor;
Py_ssize_t left;
Py_ssize_t right;
@ -199,7 +197,7 @@ static int InternalQuickSort(RepeatedCompositeContainer* self,
// len()
static Py_ssize_t Length(RepeatedCompositeContainer* self) {
google::protobuf::Message* message = self->message;
Message* message = self->message;
if (message != NULL) {
return message->GetReflection()->FieldSize(*message,
self->parent_field_descriptor);
@ -221,8 +219,8 @@ static int UpdateChildMessages(RepeatedCompositeContainer* self) {
// be removed in such a way so there's no need to worry about that.
Py_ssize_t message_length = Length(self);
Py_ssize_t child_length = PyList_GET_SIZE(self->child_messages);
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
Message* message = self->message;
const Reflection* reflection = message->GetReflection();
for (Py_ssize_t i = child_length; i < message_length; ++i) {
const Message& sub_message = reflection->GetRepeatedMessage(
*(self->message), self->parent_field_descriptor, i);
@ -233,7 +231,7 @@ static int UpdateChildMessages(RepeatedCompositeContainer* self) {
return -1;
}
cmsg->owner = self->owner;
cmsg->message = const_cast<google::protobuf::Message*>(&sub_message);
cmsg->message = const_cast<Message*>(&sub_message);
cmsg->parent = self->parent;
if (PyList_Append(self->child_messages, py_cmsg) < 0) {
return -1;
@ -255,8 +253,8 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self,
}
if (cmessage::AssureWritable(self->parent) == -1)
return NULL;
google::protobuf::Message* message = self->message;
google::protobuf::Message* sub_message =
Message* message = self->message;
Message* sub_message =
message->GetReflection()->AddMessage(message,
self->parent_field_descriptor);
CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
@ -482,9 +480,9 @@ static PyObject* SortAttached(RepeatedCompositeContainer* self,
// Finally reverse the result if requested.
if (reverse) {
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor;
Message* message = self->message;
const Reflection* reflection = message->GetReflection();
const FieldDescriptor* descriptor = self->parent_field_descriptor;
// Reverse the Message array.
for (int i = 0; i < length / 2; ++i)
@ -554,6 +552,26 @@ static PyObject* Item(RepeatedCompositeContainer* self, Py_ssize_t index) {
return item;
}
static PyObject* Pop(RepeatedCompositeContainer* self,
PyObject* args) {
Py_ssize_t index = -1;
if (!PyArg_ParseTuple(args, "|n", &index)) {
return NULL;
}
PyObject* item = Item(self, index);
if (item == NULL) {
PyErr_Format(PyExc_IndexError,
"list index (%zd) out of range",
index);
return NULL;
}
ScopedPyObjectPtr py_index(PyLong_FromSsize_t(index));
if (AssignSubscript(self, py_index, NULL) < 0) {
return NULL;
}
return item;
}
// The caller takes ownership of the returned Message.
Message* ReleaseLast(const FieldDescriptor* field,
const Descriptor* type,
@ -571,7 +589,8 @@ Message* ReleaseLast(const FieldDescriptor* field,
// the latter points to the default instance via a const_cast<>, so we
// have to reset it to a new mutable object since we are taking ownership.
if (released_message == NULL) {
const Message* prototype = global_message_factory->GetPrototype(type);
const Message* prototype =
cmessage::GetMessageFactory()->GetPrototype(type);
GOOGLE_CHECK_NOTNULL(prototype);
return prototype->New();
} else {
@ -646,7 +665,7 @@ int SetOwner(RepeatedCompositeContainer* self,
// The private constructor of RepeatedCompositeContainer objects.
PyObject *NewContainer(
CMessage* parent,
const google::protobuf::FieldDescriptor* parent_field_descriptor,
const FieldDescriptor* parent_field_descriptor,
PyObject *concrete_class) {
if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
return NULL;
@ -698,6 +717,8 @@ static PyMethodDef Methods[] = {
"Adds an object to the repeated container." },
{ "extend", (PyCFunction) Extend, METH_O,
"Adds objects to the repeated container." },
{ "pop", (PyCFunction)Pop, METH_VARARGS,
"Removes an object from the repeated container and returns it." },
{ "remove", (PyCFunction) Remove, METH_O,
"Removes an object from the repeated container." },
{ "sort", (PyCFunction) Sort, METH_VARARGS | METH_KEYWORDS,
@ -711,8 +732,8 @@ static PyMethodDef Methods[] = {
PyTypeObject RepeatedCompositeContainer_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"google.protobuf.internal."
"cpp._message.RepeatedCompositeContainer", // tp_name
"google.protobuf.pyext."
"_message.RepeatedCompositeContainer", // tp_name
sizeof(RepeatedCompositeContainer), // tp_basicsize
0, // tp_itemsize
(destructor)repeated_composite_container::Dealloc, // tp_dealloc

8
python/google/protobuf/pyext/repeated_composite_container.h
View file

@ -43,7 +43,6 @@
#include <string>
#include <vector>
namespace google {
namespace protobuf {
@ -82,7 +81,7 @@ typedef struct RepeatedCompositeContainer {
// A descriptor used to modify the underlying 'message'.
// The pointer is owned by the global DescriptorPool.
const google::protobuf::FieldDescriptor* parent_field_descriptor;
const FieldDescriptor* parent_field_descriptor;
// Pointer to the C++ Message that contains this container. The
// RepeatedCompositeContainer does not own this pointer.
@ -106,7 +105,7 @@ namespace repeated_composite_container {
// field descriptor.
PyObject *NewContainer(
CMessage* parent,
const google::protobuf::FieldDescriptor* parent_field_descriptor,
const FieldDescriptor* parent_field_descriptor,
PyObject *concrete_class);
// Returns the number of items in this repeated composite container.
@ -150,8 +149,7 @@ int AssignSubscript(RepeatedCompositeContainer* self,
// Releases the messages in the container to the given message.
//
// Returns 0 on success, -1 on failure.
int ReleaseToMessage(RepeatedCompositeContainer* self,
google::protobuf::Message* new_message);
int ReleaseToMessage(RepeatedCompositeContainer* self, Message* new_message);
// Releases the messages in the container to a new message.
//

187
python/google/protobuf/pyext/repeated_scalar_container.cc
View file

@ -60,8 +60,6 @@ namespace google {
namespace protobuf {
namespace python {
extern google::protobuf::DynamicMessageFactory* global_message_factory;
namespace repeated_scalar_container {
static int InternalAssignRepeatedField(
@ -78,7 +76,7 @@ static int InternalAssignRepeatedField(
}
static Py_ssize_t Len(RepeatedScalarContainer* self) {
google::protobuf::Message* message = self->message;
Message* message = self->message;
return message->GetReflection()->FieldSize(*message,
self->parent_field_descriptor);
}
@ -87,11 +85,10 @@ static int AssignItem(RepeatedScalarContainer* self,
Py_ssize_t index,
PyObject* arg) {
cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
self->parent_field_descriptor;
Message* message = self->message;
const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
const google::protobuf::Reflection* reflection = message->GetReflection();
const Reflection* reflection = message->GetReflection();
int field_size = reflection->FieldSize(*message, field_descriptor);
if (index < 0) {
index = field_size + index;
@ -115,64 +112,68 @@ static int AssignItem(RepeatedScalarContainer* self,
}
switch (field_descriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32: {
case FieldDescriptor::CPPTYPE_INT32: {
GOOGLE_CHECK_GET_INT32(arg, value, -1);
reflection->SetRepeatedInt32(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_INT64: {
case FieldDescriptor::CPPTYPE_INT64: {
GOOGLE_CHECK_GET_INT64(arg, value, -1);
reflection->SetRepeatedInt64(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: {
case FieldDescriptor::CPPTYPE_UINT32: {
GOOGLE_CHECK_GET_UINT32(arg, value, -1);
reflection->SetRepeatedUInt32(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: {
case FieldDescriptor::CPPTYPE_UINT64: {
GOOGLE_CHECK_GET_UINT64(arg, value, -1);
reflection->SetRepeatedUInt64(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: {
case FieldDescriptor::CPPTYPE_FLOAT: {
GOOGLE_CHECK_GET_FLOAT(arg, value, -1);
reflection->SetRepeatedFloat(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: {
case FieldDescriptor::CPPTYPE_DOUBLE: {
GOOGLE_CHECK_GET_DOUBLE(arg, value, -1);
reflection->SetRepeatedDouble(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: {
case FieldDescriptor::CPPTYPE_BOOL: {
GOOGLE_CHECK_GET_BOOL(arg, value, -1);
reflection->SetRepeatedBool(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_STRING: {
case FieldDescriptor::CPPTYPE_STRING: {
if (!CheckAndSetString(
arg, message, field_descriptor, reflection, false, index)) {
return -1;
}
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: {
case FieldDescriptor::CPPTYPE_ENUM: {
GOOGLE_CHECK_GET_INT32(arg, value, -1);
const google::protobuf::EnumDescriptor* enum_descriptor =
field_descriptor->enum_type();
const google::protobuf::EnumValueDescriptor* enum_value =
enum_descriptor->FindValueByNumber(value);
if (enum_value != NULL) {
reflection->SetRepeatedEnum(message, field_descriptor, index,
enum_value);
if (reflection->SupportsUnknownEnumValues()) {
reflection->SetRepeatedEnumValue(message, field_descriptor, index,
value);
} else {
ScopedPyObjectPtr s(PyObject_Str(arg));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
PyString_AsString(s));
const EnumDescriptor* enum_descriptor = field_descriptor->enum_type();
const EnumValueDescriptor* enum_value =
enum_descriptor->FindValueByNumber(value);
if (enum_value != NULL) {
reflection->SetRepeatedEnum(message, field_descriptor, index,
enum_value);
} else {
ScopedPyObjectPtr s(PyObject_Str(arg));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
PyString_AsString(s));
}
return -1;
}
return -1;
}
break;
}
@ -186,10 +187,9 @@ static int AssignItem(RepeatedScalarContainer* self,
}
static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
self->parent_field_descriptor;
const google::protobuf::Reflection* reflection = message->GetReflection();
Message* message = self->message;
const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
const Reflection* reflection = message->GetReflection();
int field_size = reflection->FieldSize(*message, field_descriptor);
if (index < 0) {
@ -197,80 +197,80 @@ static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
}
if (index < 0 || index >= field_size) {
PyErr_Format(PyExc_IndexError,
"list assignment index (%d) out of range",
static_cast<int>(index));
"list index (%zd) out of range",
index);
return NULL;
}
PyObject* result = NULL;
switch (field_descriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32: {
case FieldDescriptor::CPPTYPE_INT32: {
int32 value = reflection->GetRepeatedInt32(
*message, field_descriptor, index);
result = PyInt_FromLong(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_INT64: {
case FieldDescriptor::CPPTYPE_INT64: {
int64 value = reflection->GetRepeatedInt64(
*message, field_descriptor, index);
result = PyLong_FromLongLong(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: {
case FieldDescriptor::CPPTYPE_UINT32: {
uint32 value = reflection->GetRepeatedUInt32(
*message, field_descriptor, index);
result = PyLong_FromLongLong(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: {
case FieldDescriptor::CPPTYPE_UINT64: {
uint64 value = reflection->GetRepeatedUInt64(
*message, field_descriptor, index);
result = PyLong_FromUnsignedLongLong(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: {
case FieldDescriptor::CPPTYPE_FLOAT: {
float value = reflection->GetRepeatedFloat(
*message, field_descriptor, index);
result = PyFloat_FromDouble(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: {
case FieldDescriptor::CPPTYPE_DOUBLE: {
double value = reflection->GetRepeatedDouble(
*message, field_descriptor, index);
result = PyFloat_FromDouble(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: {
case FieldDescriptor::CPPTYPE_BOOL: {
bool value = reflection->GetRepeatedBool(
*message, field_descriptor, index);
result = PyBool_FromLong(value ? 1 : 0);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: {
const google::protobuf::EnumValueDescriptor* enum_value =
case FieldDescriptor::CPPTYPE_ENUM: {
const EnumValueDescriptor* enum_value =
message->GetReflection()->GetRepeatedEnum(
*message, field_descriptor, index);
result = PyInt_FromLong(enum_value->number());
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_STRING: {
case FieldDescriptor::CPPTYPE_STRING: {
string value = reflection->GetRepeatedString(
*message, field_descriptor, index);
result = ToStringObject(field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: {
case FieldDescriptor::CPPTYPE_MESSAGE: {
PyObject* py_cmsg = PyObject_CallObject(reinterpret_cast<PyObject*>(
&CMessage_Type), NULL);
if (py_cmsg == NULL) {
return NULL;
}
CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
const google::protobuf::Message& msg = reflection->GetRepeatedMessage(
const Message& msg = reflection->GetRepeatedMessage(
*message, field_descriptor, index);
cmsg->owner = self->owner;
cmsg->parent = self->parent;
cmsg->message = const_cast<google::protobuf::Message*>(&msg);
cmsg->message = const_cast<Message*>(&msg);
cmsg->read_only = false;
result = reinterpret_cast<PyObject*>(py_cmsg);
break;
@ -351,69 +351,71 @@ static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) {
PyObject* Append(RepeatedScalarContainer* self, PyObject* item) {
cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
self->parent_field_descriptor;
Message* message = self->message;
const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
const google::protobuf::Reflection* reflection = message->GetReflection();
const Reflection* reflection = message->GetReflection();
switch (field_descriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32: {
case FieldDescriptor::CPPTYPE_INT32: {
GOOGLE_CHECK_GET_INT32(item, value, NULL);
reflection->AddInt32(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_INT64: {
case FieldDescriptor::CPPTYPE_INT64: {
GOOGLE_CHECK_GET_INT64(item, value, NULL);
reflection->AddInt64(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: {
case FieldDescriptor::CPPTYPE_UINT32: {
GOOGLE_CHECK_GET_UINT32(item, value, NULL);
reflection->AddUInt32(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: {
case FieldDescriptor::CPPTYPE_UINT64: {
GOOGLE_CHECK_GET_UINT64(item, value, NULL);
reflection->AddUInt64(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: {
case FieldDescriptor::CPPTYPE_FLOAT: {
GOOGLE_CHECK_GET_FLOAT(item, value, NULL);
reflection->AddFloat(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: {
case FieldDescriptor::CPPTYPE_DOUBLE: {
GOOGLE_CHECK_GET_DOUBLE(item, value, NULL);
reflection->AddDouble(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: {
case FieldDescriptor::CPPTYPE_BOOL: {
GOOGLE_CHECK_GET_BOOL(item, value, NULL);
reflection->AddBool(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_STRING: {
case FieldDescriptor::CPPTYPE_STRING: {
if (!CheckAndSetString(
item, message, field_descriptor, reflection, true, -1)) {
return NULL;
}
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: {
case FieldDescriptor::CPPTYPE_ENUM: {
GOOGLE_CHECK_GET_INT32(item, value, NULL);
const google::protobuf::EnumDescriptor* enum_descriptor =
field_descriptor->enum_type();
const google::protobuf::EnumValueDescriptor* enum_value =
enum_descriptor->FindValueByNumber(value);
if (enum_value != NULL) {
reflection->AddEnum(message, field_descriptor, enum_value);
if (reflection->SupportsUnknownEnumValues()) {
reflection->AddEnumValue(message, field_descriptor, value);
} else {
ScopedPyObjectPtr s(PyObject_Str(item));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
PyString_AsString(s));
const EnumDescriptor* enum_descriptor = field_descriptor->enum_type();
const EnumValueDescriptor* enum_value =
enum_descriptor->FindValueByNumber(value);
if (enum_value != NULL) {
reflection->AddEnum(message, field_descriptor, enum_value);
} else {
ScopedPyObjectPtr s(PyObject_Str(item));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
PyString_AsString(s));
}
return NULL;
}
return NULL;
}
break;
}
@ -438,8 +440,8 @@ static int AssSubscript(RepeatedScalarContainer* self,
bool create_list = false;
cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
Message* message = self->message;
const FieldDescriptor* field_descriptor =
self->parent_field_descriptor;
#if PY_MAJOR_VERSION < 3
@ -450,7 +452,7 @@ static int AssSubscript(RepeatedScalarContainer* self,
if (PyLong_Check(slice)) {
from = to = PyLong_AsLong(slice);
} else if (PySlice_Check(slice)) {
const google::protobuf::Reflection* reflection = message->GetReflection();
const Reflection* reflection = message->GetReflection();
length = reflection->FieldSize(*message, field_descriptor);
#if PY_MAJOR_VERSION >= 3
if (PySlice_GetIndicesEx(slice,
@ -492,9 +494,15 @@ static int AssSubscript(RepeatedScalarContainer* self,
PyObject* Extend(RepeatedScalarContainer* self, PyObject* value) {
cmessage::AssureWritable(self->parent);
if (PyObject_Not(value)) {
// TODO(ptucker): Deprecate this behavior. b/18413862
if (value == Py_None) {
Py_RETURN_NONE;
}
if ((Py_TYPE(value)->tp_as_sequence == NULL) && PyObject_Not(value)) {
Py_RETURN_NONE;
}
ScopedPyObjectPtr iter(PyObject_GetIter(value));
if (iter == NULL) {
PyErr_SetString(PyExc_TypeError, "Value must be iterable");
@ -627,9 +635,28 @@ static PyObject* Sort(RepeatedScalarContainer* self,
Py_RETURN_NONE;
}
static PyObject* Pop(RepeatedScalarContainer* self,
PyObject* args) {
Py_ssize_t index = -1;
if (!PyArg_ParseTuple(args, "|n", &index)) {
return NULL;
}
PyObject* item = Item(self, index);
if (item == NULL) {
PyErr_Format(PyExc_IndexError,
"list index (%zd) out of range",
index);
return NULL;
}
if (AssignItem(self, index, NULL) < 0) {
return NULL;
}
return item;
}
// The private constructor of RepeatedScalarContainer objects.
PyObject *NewContainer(
CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) {
CMessage* parent, const FieldDescriptor* parent_field_descriptor) {
if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
return NULL;
}
@ -663,7 +690,7 @@ static int InitializeAndCopyToParentContainer(
if (values == NULL) {
return -1;
}
google::protobuf::Message* new_message = global_message_factory->GetPrototype(
Message* new_message = cmessage::GetMessageFactory()->GetPrototype(
from->message->GetDescriptor())->New();
to->parent = NULL;
to->parent_field_descriptor = from->parent_field_descriptor;
@ -729,6 +756,8 @@ static PyMethodDef Methods[] = {
"Appends objects to the repeated container." },
{ "insert", (PyCFunction)Insert, METH_VARARGS,
"Appends objects to the repeated container." },
{ "pop", (PyCFunction)Pop, METH_VARARGS,
"Removes an object from the repeated container and returns it." },
{ "remove", (PyCFunction)Remove, METH_O,
"Removes an object from the repeated container." },
{ "sort", (PyCFunction)Sort, METH_VARARGS | METH_KEYWORDS,
@ -740,8 +769,8 @@ static PyMethodDef Methods[] = {
PyTypeObject RepeatedScalarContainer_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"google.protobuf.internal."
"cpp._message.RepeatedScalarContainer", // tp_name
"google.protobuf."
"pyext._message.RepeatedScalarContainer", // tp_name
sizeof(RepeatedScalarContainer), // tp_basicsize
0, // tp_itemsize
(destructor)repeated_scalar_container::Dealloc, // tp_dealloc

4
python/google/protobuf/pyext/repeated_scalar_container.h
View file

@ -77,7 +77,7 @@ typedef struct RepeatedScalarContainer {
// field. Used together with the parent's message when making a
// default message instance mutable.
// The pointer is owned by the global DescriptorPool.
const google::protobuf::FieldDescriptor* parent_field_descriptor;
const FieldDescriptor* parent_field_descriptor;
} RepeatedScalarContainer;
extern PyTypeObject RepeatedScalarContainer_Type;
@ -87,7 +87,7 @@ namespace repeated_scalar_container {
// Builds a RepeatedScalarContainer object, from a parent message and a
// field descriptor.
extern PyObject *NewContainer(
CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor);
CMessage* parent, const FieldDescriptor* parent_field_descriptor);
// Appends the scalar 'item' to the end of the container 'self'.
//

18
python/google/protobuf/reflection.py
View file

@ -56,18 +56,12 @@ _FieldDescriptor = descriptor_mod.FieldDescriptor
if api_implementation.Type() == 'cpp':
if api_implementation.Version() == 2:
from google.protobuf.pyext import cpp_message
_NewMessage = cpp_message.NewMessage
_InitMessage = cpp_message.InitMessage
else:
from google.protobuf.internal import cpp_message
_NewMessage = cpp_message.NewMessage
_InitMessage = cpp_message.InitMessage
from google.protobuf.pyext import cpp_message as message_impl
else:
from google.protobuf.internal import python_message
_NewMessage = python_message.NewMessage
_InitMessage = python_message.InitMessage
from google.protobuf.internal import python_message as message_impl
_NewMessage = message_impl.NewMessage
_InitMessage = message_impl.InitMessage
class GeneratedProtocolMessageType(type):
@ -127,7 +121,6 @@ class GeneratedProtocolMessageType(type):
superclass = super(GeneratedProtocolMessageType, cls)
new_class = superclass.__new__(cls, name, bases, dictionary)
setattr(descriptor, '_concrete_class', new_class)
return new_class
def __init__(cls, name, bases, dictionary):
@ -151,6 +144,7 @@ class GeneratedProtocolMessageType(type):
_InitMessage(descriptor, cls)
superclass = super(GeneratedProtocolMessageType, cls)
superclass.__init__(name, bases, dictionary)
setattr(descriptor, '_concrete_class', cls)
def ParseMessage(descriptor, byte_str):

5
python/google/protobuf/text_format.py
View file

@ -319,6 +319,11 @@ def _MergeField(tokenizer, message, allow_multiple_scalars):
ParseError: In case of ASCII parsing problems.
"""
message_descriptor = message.DESCRIPTOR
if (hasattr(message_descriptor, 'syntax') and
message_descriptor.syntax == 'proto3'):
# Proto3 doesn't represent presence so we can't test if multiple
# scalars have occurred. We have to allow them.
allow_multiple_scalars = True
if tokenizer.TryConsume('['):
name = [tokenizer.ConsumeIdentifier()]
while tokenizer.TryConsume('.'):

22
python/setup.py
View file

@ -77,6 +77,7 @@ def GenerateUnittestProtos():
generate_proto("../src/google/protobuf/unittest_import_public.proto")
generate_proto("../src/google/protobuf/unittest_mset.proto")
generate_proto("../src/google/protobuf/unittest_no_generic_services.proto")
generate_proto("../src/google/protobuf/unittest_proto3_arena.proto")
generate_proto("google/protobuf/internal/descriptor_pool_test1.proto")
generate_proto("google/protobuf/internal/descriptor_pool_test2.proto")
generate_proto("google/protobuf/internal/test_bad_identifiers.proto")
@ -90,23 +91,6 @@ def GenerateUnittestProtos():
generate_proto("google/protobuf/internal/import_test_package/outer.proto")
generate_proto("google/protobuf/pyext/python.proto")
def MakeTestSuite():
# Test C++ implementation
import unittest
import google.protobuf.pyext.descriptor_cpp2_test as descriptor_cpp2_test
import google.protobuf.pyext.message_factory_cpp2_test \
as message_factory_cpp2_test
import google.protobuf.pyext.reflection_cpp2_generated_test \
as reflection_cpp2_generated_test
loader = unittest.defaultTestLoader
suite = unittest.TestSuite()
for test in [ descriptor_cpp2_test,
message_factory_cpp2_test,
reflection_cpp2_generated_test]:
suite.addTest(loader.loadTestsFromModule(test))
return suite
class clean(_clean):
def run(self):
# Delete generated files in the code tree.
@ -152,6 +136,8 @@ if __name__ == '__main__':
ext_module_list.append(Extension(
"google.protobuf.pyext._message",
[ "google/protobuf/pyext/descriptor.cc",
"google/protobuf/pyext/descriptor_containers.cc",
"google/protobuf/pyext/descriptor_pool.cc",
"google/protobuf/pyext/message.cc",
"google/protobuf/pyext/extension_dict.cc",
"google/protobuf/pyext/repeated_scalar_container.cc",
@ -161,12 +147,12 @@ if __name__ == '__main__':
libraries = [ "protobuf" ],
library_dirs = [ '../src/.libs' ],
))
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
setup(name = 'protobuf',
version = '3.0.0-alpha-2',
packages = [ 'google' ],
namespace_packages = [ 'google' ],
test_suite = 'setup.MakeTestSuite',
google_test_dir = "google/protobuf/internal",
# Must list modules explicitly so that we don't install tests.
py_modules = [

1
src/Makefile.am
View file

@ -325,6 +325,7 @@ EXTRA_DIST = \
google/protobuf/io/gzip_stream_unittest.sh \
google/protobuf/testdata/golden_message \
google/protobuf/testdata/golden_message_oneof_implemented \
google/protobuf/testdata/golden_message_proto3 \
google/protobuf/testdata/golden_packed_fields_message \
google/protobuf/testdata/bad_utf8_string \
google/protobuf/testdata/text_format_unittest_data.txt \

25
src/google/protobuf/compiler/python/python_generator.cc
View file

@ -273,6 +273,19 @@ string StringifyDefaultValue(const FieldDescriptor& field) {
return "";
}
string StringifySyntax(FileDescriptor::Syntax syntax) {
switch (syntax) {
case FileDescriptor::SYNTAX_PROTO2:
return "proto2";
case FileDescriptor::SYNTAX_PROTO3:
return "proto3";
case FileDescriptor::SYNTAX_UNKNOWN:
default:
GOOGLE_LOG(FATAL) << "Unsupported syntax; this generator only supports proto2 "
"and proto3 syntax.";
return "";
}
}
} // namespace
@ -367,10 +380,12 @@ void Generator::PrintFileDescriptor() const {
m["descriptor_name"] = kDescriptorKey;
m["name"] = file_->name();
m["package"] = file_->package();
m["syntax"] = StringifySyntax(file_->syntax());
const char file_descriptor_template[] =
"$descriptor_name$ = _descriptor.FileDescriptor(\n"
" name='$name$',\n"
" package='$package$',\n";
" package='$package$',\n"
" syntax='$syntax$',\n";
printer_->Print(m, file_descriptor_template);
printer_->Indent();
printer_->Print(
@ -414,7 +429,7 @@ void Generator::PrintTopLevelEnums() const {
for (int j = 0; j < enum_descriptor.value_count(); ++j) {
const EnumValueDescriptor& value_descriptor = *enum_descriptor.value(j);
top_level_enum_values.push_back(
make_pair(value_descriptor.name(), value_descriptor.number()));
std::make_pair(value_descriptor.name(), value_descriptor.number()));
}
}
@ -665,10 +680,12 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const {
message_descriptor.options().SerializeToString(&options_string);
printer_->Print(
"options=$options_value$,\n"
"is_extendable=$extendable$",
"is_extendable=$extendable$,\n"
"syntax='$syntax$'",
"options_value", OptionsValue("MessageOptions", options_string),
"extendable", message_descriptor.extension_range_count() > 0 ?
"True" : "False");
"True" : "False",
"syntax", StringifySyntax(message_descriptor.file()->syntax()));
printer_->Print(",\n");
// Extension ranges

BIN
src/google/protobuf/testdata/golden_message_proto3 vendored Normal file
View file

Binary file not shown.

54
src/google/protobuf/unittest_proto3_arena.proto
View file

@ -47,9 +47,10 @@ message TestAllTypes {
}
enum NestedEnum {
FOO = 0;
BAR = 1;
BAZ = 2;
ZERO = 0;
FOO = 1;
BAR = 2;
BAZ = 3;
NEG = -1; // Intentionally negative.
}
@ -81,6 +82,11 @@ message TestAllTypes {
optional NestedEnum optional_nested_enum = 21;
optional ForeignEnum optional_foreign_enum = 22;
// Omitted (compared to unittest.proto) because proto2 enums are not allowed
// inside proto2 messages.
//
// optional protobuf_unittest_import.ImportEnum optional_import_enum = 23;
optional string optional_string_piece = 24 [ctype=STRING_PIECE];
optional string optional_cord = 25 [ctype=CORD];
@ -118,6 +124,11 @@ message TestAllTypes {
repeated NestedEnum repeated_nested_enum = 51;
repeated ForeignEnum repeated_foreign_enum = 52;
// Omitted (compared to unittest.proto) because proto2 enums are not allowed
// inside proto2 messages.
//
// repeated protobuf_unittest_import.ImportEnum repeated_import_enum = 53;
repeated string repeated_string_piece = 54 [ctype=STRING_PIECE];
repeated string repeated_cord = 55 [ctype=CORD];
@ -131,6 +142,31 @@ message TestAllTypes {
}
}
// Test messages for packed fields
message TestPackedTypes {
repeated int32 packed_int32 = 90 [packed = true];
repeated int64 packed_int64 = 91 [packed = true];
repeated uint32 packed_uint32 = 92 [packed = true];
repeated uint64 packed_uint64 = 93 [packed = true];
repeated sint32 packed_sint32 = 94 [packed = true];
repeated sint64 packed_sint64 = 95 [packed = true];
repeated fixed32 packed_fixed32 = 96 [packed = true];
repeated fixed64 packed_fixed64 = 97 [packed = true];
repeated sfixed32 packed_sfixed32 = 98 [packed = true];
repeated sfixed64 packed_sfixed64 = 99 [packed = true];
repeated float packed_float = 100 [packed = true];
repeated double packed_double = 101 [packed = true];
repeated bool packed_bool = 102 [packed = true];
repeated ForeignEnum packed_enum = 103 [packed = true];
}
// This proto includes a recusively nested message.
message NestedTestAllTypes {
NestedTestAllTypes child = 1;
TestAllTypes payload = 2;
}
// Define these after TestAllTypes to make sure the compiler can handle
// that.
message ForeignMessage {
@ -138,7 +174,13 @@ message ForeignMessage {
}
enum ForeignEnum {
FOREIGN_FOO = 0;
FOREIGN_BAR = 1;
FOREIGN_BAZ = 2;
FOREIGN_ZERO = 0;
FOREIGN_FOO = 4;
FOREIGN_BAR = 5;
FOREIGN_BAZ = 6;
}
// TestEmptyMessage is used to test behavior of unknown fields.
message TestEmptyMessage {
}