diff --git a/BUILD b/BUILD index 69ea6cf3..11f00b81 100644 --- a/BUILD +++ b/BUILD @@ -18,7 +18,13 @@ COPTS = [ # Bazel should provide portable link_opts for pthread. LINK_OPTS = ["-lpthread"] -load("protobuf", "cc_proto_library") +load( + "protobuf", + "cc_proto_library", + "py_proto_library", + "internal_copied_filegroup", + "internal_protobuf_py_tests", +) cc_library( name = "protobuf_lite", @@ -126,7 +132,7 @@ objc_library( visibility = ["//visibility:public"], ) -WELL_KNOWN_PROTOS = [ +RELATIVE_WELL_KNOWN_PROTOS = [ # AUTOGEN(well_known_protos) "google/protobuf/any.proto", "google/protobuf/api.proto", @@ -142,12 +148,15 @@ WELL_KNOWN_PROTOS = [ "google/protobuf/wrappers.proto", ] +WELL_KNOWN_PROTOS = ["src/" + s for s in RELATIVE_WELL_KNOWN_PROTOS] + cc_proto_library( name = "cc_wkt_protos", - srcs = ["src/" + s for s in WELL_KNOWN_PROTOS], - internal_bootstrap_hack = 1, + srcs = WELL_KNOWN_PROTOS, include = "src", cc_libs = [":protobuf"], + internal_bootstrap_hack = 1, + protoc = ":protoc", ) ################################################################################ @@ -263,32 +272,11 @@ cc_binary( deps = [":protoc_lib"], ) -################################################################################ -# Java support -################################################################################ -genrule( - name = "generate_java_descriptor_proto", - srcs = ["src/google/protobuf/descriptor.proto"], - outs = ["com/google/protobuf/DescriptorProtos.java"], - cmd = "$(location :protoc) --java_out=$(@D)/../../.. $<", - tools = [":protoc"], -) - -java_library( - name = "java_proto", - srcs = glob([ - "java/src/main/java/com/google/protobuf/*.java", - ]) + [ - ":generate_java_descriptor_proto", - ], - visibility = ["//visibility:public"], -) - ################################################################################ # Tests ################################################################################ -LITE_TEST_PROTOS = [ +RELATIVE_LITE_TEST_PROTOS = [ # AUTOGEN(lite_test_protos) "google/protobuf/map_lite_unittest.proto", "google/protobuf/unittest_import_lite.proto", @@ -297,7 +285,9 @@ LITE_TEST_PROTOS = [ "google/protobuf/unittest_no_arena_lite.proto", ] -TEST_PROTOS = [ +LITE_TEST_PROTOS = ["src/" + s for s in RELATIVE_LITE_TEST_PROTOS] + +RELATIVE_TEST_PROTOS = [ # AUTOGEN(test_protos) "google/protobuf/any_test.proto", "google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto", @@ -337,10 +327,13 @@ TEST_PROTOS = [ "google/protobuf/util/json_format_proto3.proto", ] +TEST_PROTOS = ["src/" + s for s in RELATIVE_TEST_PROTOS] + cc_proto_library( name = "cc_test_protos", - srcs = ["src/" + s for s in (LITE_TEST_PROTOS + TEST_PROTOS)], + srcs = LITE_TEST_PROTOS + TEST_PROTOS, include = "src", + protoc = ":protoc", deps = [":cc_wkt_protos"], ) @@ -445,9 +438,134 @@ cc_test( ], linkopts = LINK_OPTS, deps = [ + ":cc_test_protos", ":protobuf", ":protoc_lib", - ":cc_test_protos", "//external:gtest_main", ], ) + +################################################################################ +# Java support +################################################################################ +genrule( + name = "generate_java_descriptor_proto", + srcs = ["src/google/protobuf/descriptor.proto"], + outs = ["com/google/protobuf/DescriptorProtos.java"], + cmd = "$(location :protoc) --java_out=$(@D)/../../.. $<", + tools = [":protoc"], +) + +java_library( + name = "java_proto", + srcs = glob([ + "java/src/main/java/com/google/protobuf/*.java", + ]) + [ + ":generate_java_descriptor_proto", + ], + visibility = ["//visibility:public"], +) + +################################################################################ +# Python support +################################################################################ + +# Requires: six for python 2/3 compatibility. `pip install six` + +# Hack: +# protoc generated files contain imports like: +# "from google.protobuf.xxx import yyy" +# However, the sources files of the python runtime are not directly under +# "google/protobuf" (they are under python/google/protobuf). We workaround +# this by copying runtime source files into the desired location to workaround +# the import issue. Ideally py_library should support something similiar to the +# "include" attribute in cc_library to inject the PYTHON_PATH for all libraries +# that depend on the target. +# +# If you use python protobuf as a third_party library in your bazel managed +# project, please import the whole package to //google/protobuf in your +# project. Otherwise, bazel disallows generated files out of the current +# package, thus we won't be able to copy protobuf runtime files into +# //google/protobuf/. +internal_copied_filegroup( + name = "python_srcs", + srcs = glob( + [ + "python/google/protobuf/*.py", + "python/google/protobuf/**/*.py", + ], + exclude = [ + "python/google/protobuf/internal/*_test.py", + "python/google/protobuf/internal/test_util.py", + ], + ), + include = "python", +) + +py_proto_library( + name = "python_proto", + srcs = WELL_KNOWN_PROTOS, + include = "src", + protoc = ":protoc", + py_extra_srcs = [":python_srcs"], + visibility = ["//visibility:public"], +) + +internal_copied_filegroup( + name = "python_test_srcs", + srcs = glob( + [ + "python/google/protobuf/internal/*_test.py", + "python/google/protobuf/internal/test_util.py", + ], + ), + include = "python", +) + +py_proto_library( + name = "python_common_test_protos", + srcs = LITE_TEST_PROTOS + TEST_PROTOS, + include = "src", + protoc = ":protoc", + deps = [":python_proto"], +) + +py_proto_library( + name = "python_specific_test_protos", + srcs = glob(["python/google/protobuf/internal/*.proto"]), + include = "python", + protoc = ":protoc", + deps = [":python_common_test_protos"], +) + +py_library( + name = "python_tests", + srcs = [":python_test_srcs"], + deps = [ + ":python_common_test_protos", + ":python_proto", + ":python_specific_test_protos", + ], +) + +internal_protobuf_py_tests( + name = "python_tests_batch", + modules = [ + "descriptor_database_test", + "descriptor_pool_test", + "descriptor_test", + "generator_test", + "json_format_test", + "message_factory_test", + # "message_test", # failed due to testdata path + "proto_builder_test", + # "reflection_test", # failed due to testdata path + "service_reflection_test", + "symbol_database_test", + "text_encoding_test", + # "text_format_test", # failed due to testdata path + "unknown_fields_test", + "wire_format_test", + ], + deps = [":python_tests"], +) diff --git a/protobuf.bzl b/protobuf.bzl index b83f7f5a..2199caf1 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -1,28 +1,54 @@ # -*- mode: python; -*- PYTHON-PREPROCESSING-REQUIRED -def _gen_dir(ctx): - if ctx.attr.include == None: +def _GenDir(ctx): + if not ctx.attr.includes: return "" - if not ctx.attr.include: + if not ctx.attr.includes[0]: return ctx.label.package if not ctx.label.package: - return ctx.attr.include - return ctx.label.package + '/' + ctx.attr.include + return ctx.attr.includes[0] + return ctx.label.package + '/' + ctx.attr.includes[0] -def _cc_outs(srcs): +def _CcOuts(srcs): return [s[:-len(".proto")] + ".pb.h" for s in srcs] + \ [s[:-len(".proto")] + ".pb.cc" for s in srcs] -def _py_outs(srcs): +def _PyOuts(srcs): return [s[:-len(".proto")] + "_pb2.py" for s in srcs] +def _RelativeOutputPath(path, include): + if include == None: + return path + + if not path.startswith(include): + fail("Include path %s isn't part of the path %s." % (include, path)) + + if include and include[-1] != '/': + include = include + '/' + + path = path[len(include):] + + if not path.startswith(PACKAGE_NAME): + fail("The package %s is not within the path %s" % (PACKAGE_NAME, path)) + + if not PACKAGE_NAME: + return path + + return path[len(PACKAGE_NAME)+1:] + + + def _proto_gen_impl(ctx): """General implementation for generating protos""" srcs = ctx.files.srcs deps = [] deps += ctx.files.srcs - gen_dir = _gen_dir(ctx) - import_flags = ["-I" + gen_dir] + gen_dir = _GenDir(ctx) + if gen_dir: + import_flags = ["-I" + gen_dir] + else: + import_flags = ["-I."] + for dep in ctx.attr.deps: import_flags += dep.proto.import_flags deps += dep.proto.deps @@ -53,7 +79,7 @@ _proto_gen = rule( attrs = { "srcs": attr.label_list(allow_files = True), "deps": attr.label_list(providers = ["proto"]), - "include": attr.string(), + "includes": attr.string_list(), "protoc": attr.label( executable = True, single_file = True, @@ -73,7 +99,7 @@ def cc_proto_library( deps=[], cc_libs=[], include=None, - protoc=":protoc", + protoc="//google/protobuf:protoc", internal_bootstrap_hack=False, **kargs): """Bazel rule to create a C++ protobuf library from proto source files @@ -94,6 +120,10 @@ def cc_proto_library( """ + includes = [] + if include != None: + includes = [include] + if internal_bootstrap_hack: # For pre-checked-in generated files, we add the internal_bootstrap_hack # which will skip the codegen action. @@ -101,7 +131,7 @@ def cc_proto_library( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - include=include, + includes=includes, protoc=protoc, ) # An empty cc_library to make rule dependency consistent. @@ -110,20 +140,17 @@ def cc_proto_library( **kargs) return - outs = _cc_outs(srcs) + outs = _CcOuts(srcs) _proto_gen( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - include=include, + includes=includes, protoc=protoc, gen_cc=1, outs=outs, ) - includes = [] - if include != None: - includes = [include] native.cc_library( name=name, @@ -131,3 +158,111 @@ def cc_proto_library( deps=cc_libs + deps, includes=includes, **kargs) + + +def internal_copied_filegroup( + name, + srcs, + include, + **kargs): + """Bazel rule to fix sources file to workaround with python path issues. + + Args: + name: the name of the internal_copied_filegroup rule, which will be the + name of the generated filegroup. + srcs: the source files to be copied. + include: the expected import root of the source. + **kargs: extra arguments that will be passed into the filegroup. + """ + outs = [_RelativeOutputPath(s, include) for s in srcs] + + native.genrule( + name=name+"_genrule", + srcs=srcs, + outs=outs, + cmd=";".join(["cp $(location %s) $(location %s)" % \ + (s, _RelativeOutputPath(s, include)) \ + for s in srcs])) + + native.filegroup( + name=name, + srcs=outs, + **kargs) + + +def py_proto_library( + name, + srcs=[], + deps=[], + py_libs=[], + py_extra_srcs=[], + include=None, + protoc="//google/protobuf:protoc", + **kargs): + """Bazel rule to create a Python protobuf library from proto source files + + Args: + name: the name of the py_proto_library. + srcs: the .proto files of the py_proto_library. + deps: a list of dependency labels; must be py_proto_library. + py_libs: a list of other py_library targets depended by the generated + py_library. + py_extra_srcs: extra source files that will be added to the output + py_library. This attribute is used for internal bootstrapping. + include: a string indicating the include path of the .proto files. + protoc: the label of the protocol compiler to generate the sources. + **kargs: other keyword arguments that are passed to cc_library. + + """ + outs = _PyOuts(srcs) + + includes = [] + if include != None: + includes = [include] + + _proto_gen( + name=name + "_genproto", + srcs=srcs, + deps=[s + "_genproto" for s in deps], + includes=includes, + protoc=protoc, + gen_py=1, + outs=outs, + ) + + if include != None: + # Copy the output files to the desired location to make the import work. + internal_copied_filegroup_name=name + "_internal_copied_filegroup" + internal_copied_filegroup( + name=internal_copied_filegroup_name, + srcs=outs, + include=include) + outs=[internal_copied_filegroup_name] + + native.py_library( + name=name, + srcs=outs+py_extra_srcs, + deps=py_libs+deps, + **kargs) + +def internal_protobuf_py_tests( + name, + modules=[], + **kargs): + """Bazel rules to create batch tests for protobuf internal. + + Args: + name: the name of the rule. + modules: a list of modules for tests. The macro will create a py_test for + each of the parameter with the source "google/protobuf/%s.py" + kargs: extra parameters that will be passed into the py_test. + + """ + for m in modules: + s = _RelativeOutputPath( + "python/google/protobuf/internal/%s.py" % m, "python") + native.py_test( + name="py_%s" % m, + srcs=[s], + main=s, + **kargs)