organicmaps/protobuf
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

Merge pull request #867 from pherl/master

Browse source 
Python bazel support
This commit is contained in:
Jisi Liu 2015-10-21 10:43:00 -07:00
commit 66e3a6d0b7
2 changed files with 299 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
BUILD
View file

@ -18,7 +18,13 @@ COPTS = [
# Bazel should provide portable link_opts for pthread.
LINK_OPTS = ["-lpthread"]
load("protobuf", "cc_proto_library")
load(
"protobuf",
"cc_proto_library",
"py_proto_library",
"internal_copied_filegroup",
"internal_protobuf_py_tests",
)
cc_library(
name = "protobuf_lite",
@ -126,7 +132,7 @@ objc_library(
visibility = ["//visibility:public"],
)
WELL_KNOWN_PROTOS = [
RELATIVE_WELL_KNOWN_PROTOS = [
# AUTOGEN(well_known_protos)
"google/protobuf/any.proto",
"google/protobuf/api.proto",
@ -142,12 +148,15 @@ WELL_KNOWN_PROTOS = [
"google/protobuf/wrappers.proto",
]
WELL_KNOWN_PROTOS = ["src/" + s for s in RELATIVE_WELL_KNOWN_PROTOS]
cc_proto_library(
name = "cc_wkt_protos",
srcs = ["src/" + s for s in WELL_KNOWN_PROTOS],
internal_bootstrap_hack = 1,
srcs = WELL_KNOWN_PROTOS,
include = "src",
cc_libs = [":protobuf"],
internal_bootstrap_hack = 1,
protoc = ":protoc",
)
################################################################################
@ -263,32 +272,11 @@ cc_binary(
deps = [":protoc_lib"],
)
################################################################################
# Java support
################################################################################
genrule(
name = "generate_java_descriptor_proto",
srcs = ["src/google/protobuf/descriptor.proto"],
outs = ["com/google/protobuf/DescriptorProtos.java"],
cmd = "$(location :protoc) --java_out=$(@D)/../../.. $<",
tools = [":protoc"],
)
java_library(
name = "java_proto",
srcs = glob([
"java/src/main/java/com/google/protobuf/*.java",
]) + [
":generate_java_descriptor_proto",
],
visibility = ["//visibility:public"],
)
################################################################################
# Tests
################################################################################
LITE_TEST_PROTOS = [
RELATIVE_LITE_TEST_PROTOS = [
# AUTOGEN(lite_test_protos)
"google/protobuf/map_lite_unittest.proto",
"google/protobuf/unittest_import_lite.proto",
@ -297,7 +285,9 @@ LITE_TEST_PROTOS = [
"google/protobuf/unittest_no_arena_lite.proto",
]
TEST_PROTOS = [
LITE_TEST_PROTOS = ["src/" + s for s in RELATIVE_LITE_TEST_PROTOS]
RELATIVE_TEST_PROTOS = [
# AUTOGEN(test_protos)
"google/protobuf/any_test.proto",
"google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto",
@ -337,10 +327,13 @@ TEST_PROTOS = [
"google/protobuf/util/json_format_proto3.proto",
]
TEST_PROTOS = ["src/" + s for s in RELATIVE_TEST_PROTOS]
cc_proto_library(
name = "cc_test_protos",
srcs = ["src/" + s for s in (LITE_TEST_PROTOS + TEST_PROTOS)],
srcs = LITE_TEST_PROTOS + TEST_PROTOS,
include = "src",
protoc = ":protoc",
deps = [":cc_wkt_protos"],
)
@ -445,9 +438,134 @@ cc_test(
],
linkopts = LINK_OPTS,
deps = [
":cc_test_protos",
":protobuf",
":protoc_lib",
":cc_test_protos",
"//external:gtest_main",
],
)
################################################################################
# Java support
################################################################################
genrule(
name = "generate_java_descriptor_proto",
srcs = ["src/google/protobuf/descriptor.proto"],
outs = ["com/google/protobuf/DescriptorProtos.java"],
cmd = "$(location :protoc) --java_out=$(@D)/../../.. $<",
tools = [":protoc"],
)
java_library(
name = "java_proto",
srcs = glob([
"java/src/main/java/com/google/protobuf/*.java",
]) + [
":generate_java_descriptor_proto",
],
visibility = ["//visibility:public"],
)
################################################################################
# Python support
################################################################################
# Requires: six for python 2/3 compatibility. `pip install six`
# Hack:
# protoc generated files contain imports like:
# "from google.protobuf.xxx import yyy"
# However, the sources files of the python runtime are not directly under
# "google/protobuf" (they are under python/google/protobuf). We workaround
# this by copying runtime source files into the desired location to workaround
# the import issue. Ideally py_library should support something similiar to the
# "include" attribute in cc_library to inject the PYTHON_PATH for all libraries
# that depend on the target.
#
# If you use python protobuf as a third_party library in your bazel managed
# project, please import the whole package to //google/protobuf in your
# project. Otherwise, bazel disallows generated files out of the current
# package, thus we won't be able to copy protobuf runtime files into
# //google/protobuf/.
internal_copied_filegroup(
name = "python_srcs",
srcs = glob(
[
"python/google/protobuf/*.py",
"python/google/protobuf/**/*.py",
],
exclude = [
"python/google/protobuf/internal/*_test.py",
"python/google/protobuf/internal/test_util.py",
],
),
include = "python",
)
py_proto_library(
name = "python_proto",
srcs = WELL_KNOWN_PROTOS,
include = "src",
protoc = ":protoc",
py_extra_srcs = [":python_srcs"],
visibility = ["//visibility:public"],
)
internal_copied_filegroup(
name = "python_test_srcs",
srcs = glob(
[
"python/google/protobuf/internal/*_test.py",
"python/google/protobuf/internal/test_util.py",
],
),
include = "python",
)
py_proto_library(
name = "python_common_test_protos",
srcs = LITE_TEST_PROTOS + TEST_PROTOS,
include = "src",
protoc = ":protoc",
deps = [":python_proto"],
)
py_proto_library(
name = "python_specific_test_protos",
srcs = glob(["python/google/protobuf/internal/*.proto"]),
include = "python",
protoc = ":protoc",
deps = [":python_common_test_protos"],
)
py_library(
name = "python_tests",
srcs = [":python_test_srcs"],
deps = [
":python_common_test_protos",
":python_proto",
":python_specific_test_protos",
],
)
internal_protobuf_py_tests(
name = "python_tests_batch",
modules = [
"descriptor_database_test",
"descriptor_pool_test",
"descriptor_test",
"generator_test",
"json_format_test",
"message_factory_test",
# "message_test", # failed due to testdata path
"proto_builder_test",
# "reflection_test", # failed due to testdata path
"service_reflection_test",
"symbol_database_test",
"text_encoding_test",
# "text_format_test", # failed due to testdata path
"unknown_fields_test",
"wire_format_test",
],
deps = [":python_tests"],
)

169
protobuf.bzl
View file

@ -1,28 +1,54 @@
# -*- mode: python; -*- PYTHON-PREPROCESSING-REQUIRED
def _gen_dir(ctx):
if ctx.attr.include == None:
def _GenDir(ctx):
if not ctx.attr.includes:
return ""
if not ctx.attr.include:
if not ctx.attr.includes[0]:
return ctx.label.package
if not ctx.label.package:
return ctx.attr.include
return ctx.label.package + '/' + ctx.attr.include
return ctx.attr.includes[0]
return ctx.label.package + '/' + ctx.attr.includes[0]
def _cc_outs(srcs):
def _CcOuts(srcs):
return [s[:-len(".proto")] + ".pb.h" for s in srcs] + \
[s[:-len(".proto")] + ".pb.cc" for s in srcs]
def _py_outs(srcs):
def _PyOuts(srcs):
return [s[:-len(".proto")] + "_pb2.py" for s in srcs]
def _RelativeOutputPath(path, include):
if include == None:
return path
if not path.startswith(include):
fail("Include path %s isn't part of the path %s." % (include, path))
if include and include[-1] != '/':
include = include + '/'
path = path[len(include):]
if not path.startswith(PACKAGE_NAME):
fail("The package %s is not within the path %s" % (PACKAGE_NAME, path))
if not PACKAGE_NAME:
return path
return path[len(PACKAGE_NAME)+1:]
def _proto_gen_impl(ctx):
"""General implementation for generating protos"""
srcs = ctx.files.srcs
deps = []
deps += ctx.files.srcs
gen_dir = _gen_dir(ctx)
import_flags = ["-I" + gen_dir]
gen_dir = _GenDir(ctx)
if gen_dir:
import_flags = ["-I" + gen_dir]
else:
import_flags = ["-I."]
for dep in ctx.attr.deps:
import_flags += dep.proto.import_flags
deps += dep.proto.deps
@ -53,7 +79,7 @@ _proto_gen = rule(
attrs = {
"srcs": attr.label_list(allow_files = True),
"deps": attr.label_list(providers = ["proto"]),
"include": attr.string(),
"includes": attr.string_list(),
"protoc": attr.label(
executable = True,
single_file = True,
@ -73,7 +99,7 @@ def cc_proto_library(
deps=[],
cc_libs=[],
include=None,
protoc=":protoc",
protoc="//google/protobuf:protoc",
internal_bootstrap_hack=False,
**kargs):
"""Bazel rule to create a C++ protobuf library from proto source files
@ -94,6 +120,10 @@ def cc_proto_library(
"""
includes = []
if include != None:
includes = [include]
if internal_bootstrap_hack:
# For pre-checked-in generated files, we add the internal_bootstrap_hack
# which will skip the codegen action.
@ -101,7 +131,7 @@ def cc_proto_library(
name=name + "_genproto",
srcs=srcs,
deps=[s + "_genproto" for s in deps],
include=include,
includes=includes,
protoc=protoc,
)
# An empty cc_library to make rule dependency consistent.
@ -110,20 +140,17 @@ def cc_proto_library(
**kargs)
return
outs = _cc_outs(srcs)
outs = _CcOuts(srcs)
_proto_gen(
name=name + "_genproto",
srcs=srcs,
deps=[s + "_genproto" for s in deps],
include=include,
includes=includes,
protoc=protoc,
gen_cc=1,
outs=outs,
)
includes = []
if include != None:
includes = [include]
native.cc_library(
name=name,
@ -131,3 +158,111 @@ def cc_proto_library(
deps=cc_libs + deps,
includes=includes,
**kargs)
def internal_copied_filegroup(
name,
srcs,
include,
**kargs):
"""Bazel rule to fix sources file to workaround with python path issues.
Args:
name: the name of the internal_copied_filegroup rule, which will be the
name of the generated filegroup.
srcs: the source files to be copied.
include: the expected import root of the source.
**kargs: extra arguments that will be passed into the filegroup.
"""
outs = [_RelativeOutputPath(s, include) for s in srcs]
native.genrule(
name=name+"_genrule",
srcs=srcs,
outs=outs,
cmd=";".join(["cp $(location %s) $(location %s)" % \
(s, _RelativeOutputPath(s, include)) \
for s in srcs]))
native.filegroup(
name=name,
srcs=outs,
**kargs)
def py_proto_library(
name,
srcs=[],
deps=[],
py_libs=[],
py_extra_srcs=[],
include=None,
protoc="//google/protobuf:protoc",
**kargs):
"""Bazel rule to create a Python protobuf library from proto source files
Args:
name: the name of the py_proto_library.
srcs: the .proto files of the py_proto_library.
deps: a list of dependency labels; must be py_proto_library.
py_libs: a list of other py_library targets depended by the generated
py_library.
py_extra_srcs: extra source files that will be added to the output
py_library. This attribute is used for internal bootstrapping.
include: a string indicating the include path of the .proto files.
protoc: the label of the protocol compiler to generate the sources.
**kargs: other keyword arguments that are passed to cc_library.
"""
outs = _PyOuts(srcs)
includes = []
if include != None:
includes = [include]
_proto_gen(
name=name + "_genproto",
srcs=srcs,
deps=[s + "_genproto" for s in deps],
includes=includes,
protoc=protoc,
gen_py=1,
outs=outs,
)
if include != None:
# Copy the output files to the desired location to make the import work.
internal_copied_filegroup_name=name + "_internal_copied_filegroup"
internal_copied_filegroup(
name=internal_copied_filegroup_name,
srcs=outs,
include=include)
outs=[internal_copied_filegroup_name]
native.py_library(
name=name,
srcs=outs+py_extra_srcs,
deps=py_libs+deps,
**kargs)
def internal_protobuf_py_tests(
name,
modules=[],
**kargs):
"""Bazel rules to create batch tests for protobuf internal.
Args:
name: the name of the rule.
modules: a list of modules for tests. The macro will create a py_test for
each of the parameter with the source "google/protobuf/%s.py"
kargs: extra parameters that will be passed into the py_test.
"""
for m in modules:
s = _RelativeOutputPath(
"python/google/protobuf/internal/%s.py" % m, "python")
native.py_test(
name="py_%s" % m,
srcs=[s],
main=s,
**kargs)