organicmaps/protobuf
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

WIP.

Browse source
This commit is contained in:
Josh Haberman 2016-02-02 15:18:34 -08:00
parent c40f8c1f54
commit 55cc3aa987
6 changed files with 287 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
js/commonjs_export.js Normal file
View file

@ -0,0 +1,10 @@
/**
* @fileoverview Export symbols needed by generated code in CommonJS style.
*/
exports = {
Message: jspb.Message,
BinaryReader: jspb.BinaryReader,
BinaryWriter: jspb.BinaryWriter,
ExtensionFieldInfo: jspb.ExtensionFieldInfo,
};

39
js/gulpfile.js
View file

@ -1,25 +1,45 @@
var gulp = require('gulp');
var exec = require('child_process').exec;
gulp.task('genproto', function (cb) {
gulp.task('genproto_closure', function (cb) {
exec('../src/protoc --js_out=library=testproto_libs,binary:. -I ../src -I . *.proto ../src/google/protobuf/descriptor.proto',
function (err, stdout, stderr) {
console.log(stdout);
console.log(stderr);
cb(err);
});
})
});
gulp.task('deps', ['genproto'], function (cb) {
gulp.task('genproto_commonjs', function (cb) {
exec('mkdir -p commonjs_out && ../src/protoc --js_out=import_style=commonjs,binary:commonjs_out -I ../src -I . *.proto ../src/google/protobuf/descriptor.proto',
function (err, stdout, stderr) {
console.log(stdout);
console.log(stderr);
cb(err);
});
});
gulp.task('dist', function (cb) {
// TODO(haberman): minify this more aggressively.
// Will require proper externs/exports.
exec('./node_modules/google-closure-library/closure/bin/calcdeps.py -i message.js -i binary/reader.js -i binary/writer.js -p . -p node_modules/google-closure-library/closure -o compiled --compiler_jar node_modules/google-closure-compiler/compiler.jar > google-protobuf.js',
function (err, stdout, stderr) {
console.log(stdout);
console.log(stderr);
cb(err);
});
});
gulp.task('deps', ['genproto_closure'], function (cb) {
exec('./node_modules/google-closure-library/closure/bin/build/depswriter.py *.js binary/*.js > deps.js',
function (err, stdout, stderr) {
console.log(stdout);
console.log(stderr);
cb(err);
});
})
});
gulp.task('test', ['genproto', 'deps'], function (cb) {
gulp.task('test_closure', ['genproto_closure', 'deps'], function (cb) {
exec('JASMINE_CONFIG_PATH=jasmine.json ./node_modules/.bin/jasmine',
function (err, stdout, stderr) {
console.log(stdout);
@ -27,3 +47,12 @@ gulp.task('test', ['genproto', 'deps'], function (cb) {
cb(err);
});
});
gulp.task('test_commonjs', ['genproto_commonjs', 'dist'], function (cb) {
exec('JASMINE_CONFIG_PATH=jasmine.json cp jasmine_commonjs.json commonjs_out/jasmine.json && cd commonjs_out && ../node_modules/.bin/jasmine',
function (err, stdout, stderr) {
console.log(stdout);
console.log(stderr);
cb(err);
});
});

11
js/jasmine_commonjs.json Normal file
View file

@ -0,0 +1,11 @@
{
"spec_dir": "",
"spec_files": [
"*_test.js"
],
"helpers": [
"node_modules/google-closure-library/closure/goog/bootstrap/nodejs.js",
"node_loader.js",
"deps.js"
]
}

6
js/package.json
View file

@ -2,13 +2,15 @@
"name": "google-protobuf",
"version": "3.0.0-alpha.5",
"description": "Protocol Buffers for JavaScript",
"main": "debug.js",
"main": "google-protobuf.js",
"dependencies": {
"google-closure-library": "~20160125.0.0",
"gulp": "~3.9.0",
"jasmine": "~2.4.1"
},
"devDependencies": {},
"devDependencies": {
"google-closure-compiler": "~20151216.2.0"
},
"scripts": {
"test": "./node_modules/gulp/bin/gulp.js test"
},

268
src/google/protobuf/compiler/js/js_generator.cc
View file

@ -134,12 +134,37 @@ bool IsReserved(const string& ident) {
// Returns a copy of |filename| with any trailing ".protodevel" or ".proto
// suffix stripped.
// TODO(robinson): Unify with copy in compiler/cpp/internal/helpers.cc.
string StripProto(const string& filename) {
const char* suffix = HasSuffixString(filename, ".protodevel")
? ".protodevel" : ".proto";
return StripSuffixString(filename, suffix);
}
// Given a filename like foo/bar/baz.proto, returns the correspoding JavaScript
// file foo/bar/baz.js.
string GetJSFilename(const string& filename) {
const char* suffix = HasSuffixString(filename, ".protodevel")
? ".protodevel" : ".proto";
return StripSuffixString(filename, suffix) + "_pb.js";
}
// Returns the alias we assign to the module of the given .proto filename
// when importing.
string ModuleAlias(const string& filename) {
// This scheme could technically cause problems if a file includes any 2 of:
// foo/bar_baz.proto
// foo_bar_baz.proto
// foo_bar/baz.proto
//
// We'll worry about this problem if/when we actually see it. This name isn't
// exposed to users so we can change it later if we need to.
string basename = StripProto(filename);
StripString(&basename, "-", '$');
StripString(&basename, "/", '_');
return basename + "_pb";
}
// Returns the fully normalized JavaScript path for the given
// file descriptor's package.
string GetPath(const GeneratorOptions& options,
@ -215,6 +240,26 @@ string GetPath(const GeneratorOptions& options,
value_descriptor->type()) + "." + value_descriptor->name();
}
string MaybeCrossFileRef(const GeneratorOptions& options,
const FileDescriptor* from_file,
const Descriptor* to_message) {
if (options.import_style == GeneratorOptions::IMPORT_COMMONJS &&
from_file != to_message->file()) {
// Cross-file ref in CommonJS needs to use the module alias instead of
// the global name.
return ModuleAlias(to_message->file()->name()) + "." + to_message->name();
} else {
// Within a single file we use a full name.
return GetPath(options, to_message);
}
}
string SubmessageTypeRef(const GeneratorOptions& options,
const FieldDescriptor* field) {
GOOGLE_CHECK(field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE);
return MaybeCrossFileRef(options, field->file(), field->message_type());
}
// - Object field name: LOWER_UNDERSCORE -> LOWER_CAMEL, except for group fields
// (UPPER_CAMEL -> LOWER_CAMEL), with "List" (or "Map") appended if appropriate,
// and with reserved words triggering a "pb_" prefix.
@ -952,11 +997,13 @@ string RelativeTypeName(const FieldDescriptor* field) {
}
string JSExtensionsObjectName(const GeneratorOptions& options,
const FileDescriptor* from_file,
const Descriptor* desc) {
if (desc->full_name() == "google.protobuf.bridge.MessageSet") {
// TODO(haberman): fix this for the IMPORT_COMMONJS case.
return "jspb.Message.messageSetExtensions";
} else {
return GetPath(options, desc) + ".extensions";
return MaybeCrossFileRef(options, from_file, desc) + ".extensions";
}
}
@ -1113,19 +1160,24 @@ void Generator::GenerateHeader(const GeneratorOptions& options,
"\n");
}
void Generator::FindProvidesForFile(const GeneratorOptions& options,
io::Printer* printer,
const FileDescriptor* file,
std::set<string>* provided) const {
for (int i = 0; i < file->message_type_count(); i++) {
FindProvidesForMessage(options, printer, file->message_type(i), provided);
}
for (int i = 0; i < file->enum_type_count(); i++) {
FindProvidesForEnum(options, printer, file->enum_type(i), provided);
}
}
void Generator::FindProvides(const GeneratorOptions& options,
io::Printer* printer,
const vector<const FileDescriptor*>& files,
std::set<string>* provided) const {
for (int i = 0; i < files.size(); i++) {
for (int j = 0; j < files[i]->message_type_count(); j++) {
FindProvidesForMessage(options, printer, files[i]->message_type(j),
provided);
}
for (int j = 0; j < files[i]->enum_type_count(); j++) {
FindProvidesForEnum(options, printer, files[i]->enum_type(j),
provided);
}
FindProvidesForFile(options, printer, files[i], provided);
}
printer->Print("\n");
@ -1204,38 +1256,45 @@ void Generator::GenerateRequires(const GeneratorOptions& options,
io::Printer* printer,
const vector<const FileDescriptor*>& files,
std::set<string>* provided) const {
std::set<string> required;
std::set<string> forwards;
bool have_extensions = false;
bool have_message = false;
if (options.import_style == GeneratorOptions::IMPORT_BROWSER) {
return;
} else if (options.import_style == GeneratorOptions::IMPORT_CLOSURE) {
// For Closure imports we need to import every message type individually.
std::set<string> required;
std::set<string> forwards;
bool have_extensions = false;
bool have_message = false;
for (int i = 0; i < files.size(); i++) {
for (int j = 0; j < files[i]->message_type_count(); j++) {
FindRequiresForMessage(options,
files[i]->message_type(j),
&required, &forwards, &have_message);
}
if (!have_extensions && HasExtensions(files[i])) {
have_extensions = true;
for (int i = 0; i < files.size(); i++) {
for (int j = 0; j < files[i]->message_type_count(); j++) {
FindRequiresForMessage(options,
files[i]->message_type(j),
&required, &forwards, &have_message);
}
if (!have_extensions && HasExtensions(files[i])) {
have_extensions = true;
}
for (int j = 0; j < files[i]->extension_count(); j++) {
const FieldDescriptor* extension = files[i]->extension(j);
if (IgnoreField(extension)) {
continue;
}
if (extension->containing_type()->full_name() !=
"google.protobuf.bridge.MessageSet") {
required.insert(GetPath(options, extension->containing_type()));
}
FindRequiresForField(options, extension, &required, &forwards);
have_extensions = true;
}
}
for (int j = 0; j < files[i]->extension_count(); j++) {
const FieldDescriptor* extension = files[i]->extension(j);
if (IgnoreField(extension)) {
continue;
}
if (extension->containing_type()->full_name() !=
"google.protobuf.bridge.MessageSet") {
required.insert(GetPath(options, extension->containing_type()));
}
FindRequiresForField(options, extension, &required, &forwards);
have_extensions = true;
}
GenerateRequiresImpl(options, printer, &required, &forwards, provided,
/* require_jspb = */ have_message,
/* require_extension = */ have_extensions);
} else if (options.import_style == GeneratorOptions::IMPORT_COMMONJS) {
// CommonJS imports are based on files
}
GenerateRequiresImpl(options, printer, &required, &forwards, provided,
/* require_jspb = */ have_message,
/* require_extension = */ have_extensions);
}
void Generator::GenerateRequires(const GeneratorOptions& options,
@ -1406,6 +1465,19 @@ void Generator::GenerateClass(const GeneratorOptions& options,
if (IsExtendable(desc) && desc->full_name() != "google.protobuf.bridge.MessageSet") {
GenerateClassExtensionFieldInfo(options, printer, desc);
}
if (options.import_style == GeneratorOptions::IMPORT_COMMONJS) {
printer->Print(
"exports.$name$ = $fullName$;\n",
"name", desc->name(),
"fullName", GetPath(options, desc));
}
if (options.import_style != GeneratorOptions:: IMPORT_CLOSURE) {
for (int i = 0; i < desc->extension_count(); i++) {
GenerateExtension(options, printer, desc->extension(i));
}
}
}
// Recurse on nested types.
@ -1623,7 +1695,7 @@ void Generator::GenerateClassToObject(const GeneratorOptions& options,
"obj,\n"
" $extObject$, $class$.prototype.getExtension,\n"
" includeInstance);\n",
"extObject", JSExtensionsObjectName(options, desc),
"extObject", JSExtensionsObjectName(options, desc->file(), desc),
"class", GetPath(options, desc));
}
@ -1652,13 +1724,13 @@ void Generator::GenerateClassFieldToObject(const GeneratorOptions& options,
printer->Print("jspb.Message.toObjectList(msg.get$getter$(),\n"
" $type$.toObject, includeInstance)",
"getter", JSGetterName(field),
"type", GetPath(options, field->message_type()));
"type", SubmessageTypeRef(options, field));
}
} else {
printer->Print("(f = msg.get$getter$()) && "
"$type$.toObject(includeInstance, f)",
"getter", JSGetterName(field),
"type", GetPath(options, field->message_type()));
"type", SubmessageTypeRef(options, field));
}
} else {
// Simple field (singular or repeated).
@ -1723,7 +1795,7 @@ void Generator::GenerateClassFieldFromObject(
" }));\n",
"name", JSObjectFieldName(field),
"index", JSFieldIndex(field),
"fieldclass", GetPath(options, field->message_type()));
"fieldclass", SubmessageTypeRef(options, field));
}
} else {
printer->Print(
@ -1731,7 +1803,7 @@ void Generator::GenerateClassFieldFromObject(
" msg, $index$, $fieldclass$.fromObject(obj.$name$));\n",
"name", JSObjectFieldName(field),
"index", JSFieldIndex(field),
"fieldclass", GetPath(options, field->message_type()));
"fieldclass", SubmessageTypeRef(options, field));
}
} else {
// Simple (primitive) field.
@ -1815,7 +1887,7 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
/* always_singular = */ false),
"rpt", (field->is_repeated() ? "Repeated" : ""),
"index", JSFieldIndex(field),
"wrapperclass", GetPath(options, field->message_type()),
"wrapperclass", SubmessageTypeRef(options, field),
"required", (field->label() == FieldDescriptor::LABEL_REQUIRED ?
", 1" : ""));
printer->Print(
@ -2043,7 +2115,7 @@ void Generator::GenerateClassDeserializeBinary(const GeneratorOptions& options,
" $class$.prototype.getExtension,\n"
" $class$.prototype.setExtension);\n"
" break;\n",
"extobj", JSExtensionsObjectName(options, desc),
"extobj", JSExtensionsObjectName(options, desc->file(), desc),
"class", GetPath(options, desc));
} else {
printer->Print(
@ -2073,7 +2145,7 @@ void Generator::GenerateClassDeserializeBinaryField(
" var value = new $fieldclass$;\n"
" reader.read$msgOrGroup$($grpfield$value,"
"$fieldclass$.deserializeBinaryFromReader);\n",
"fieldclass", GetPath(options, field->message_type()),
"fieldclass", SubmessageTypeRef(options, field),
"msgOrGroup", (field->type() == FieldDescriptor::TYPE_GROUP) ?
"Group" : "Message",
"grpfield", (field->type() == FieldDescriptor::TYPE_GROUP) ?
@ -2149,7 +2221,7 @@ void Generator::GenerateClassSerializeBinary(const GeneratorOptions& options,
printer->Print(
" jspb.Message.serializeBinaryExtensions(this, writer, $extobj$,\n"
" $class$.prototype.getExtension);\n",
"extobj", JSExtensionsObjectName(options, desc),
"extobj", JSExtensionsObjectName(options, desc->file(), desc),
"class", GetPath(options, desc));
}
@ -2222,7 +2294,7 @@ void Generator::GenerateClassSerializeBinaryField(
printer->Print(
",\n"
" $submsg$.serializeBinaryToWriter\n",
"submsg", GetPath(options, field->message_type()));
"submsg", SubmessageTypeRef(options, field));
} else {
printer->Print("\n");
}
@ -2290,9 +2362,9 @@ void Generator::GenerateExtension(const GeneratorOptions& options,
"index", SimpleItoa(field->number()),
"name", JSObjectFieldName(field),
"ctor", (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ?
GetPath(options, field->message_type()) : string("null")),
SubmessageTypeRef(options, field) : string("null")),
"toObject", (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ?
(GetPath(options, field->message_type()) + ".toObject") :
(SubmessageTypeRef(options, field) + ".toObject") :
string("null")),
"repeated", (field->is_repeated() ? "1" : "0"));
@ -2308,11 +2380,11 @@ void Generator::GenerateExtension(const GeneratorOptions& options,
"binaryWriterFn", JSBinaryWriterMethodName(field),
"binaryMessageSerializeFn",
(field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) ?
(GetPath(options, field->message_type()) +
(SubmessageTypeRef(options, field) +
".serializeBinaryToWriter") : "null",
"binaryMessageDeserializeFn",
(field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) ?
(GetPath(options, field->message_type()) +
(SubmessageTypeRef(options, field) +
".deserializeBinaryFromReader") : "null",
"isPacked", (field->is_packed() ? "true" : "false"));
} else {
@ -2324,7 +2396,8 @@ void Generator::GenerateExtension(const GeneratorOptions& options,
"// toObject() will function correctly.\n"
"$extendName$[$index$] = $class$.$name$;\n"
"\n",
"extendName", JSExtensionsObjectName(options, field->containing_type()),
"extendName", JSExtensionsObjectName(options, field->file(),
field->containing_type()),
"index", SimpleItoa(field->number()),
"class", extension_scope,
"name", JSObjectFieldName(field));
@ -2364,6 +2437,19 @@ bool GeneratorOptions::ParseFromOptions(
namespace_prefix = options[i].second;
} else if (options[i].first == "library") {
library = options[i].second;
} else if (options[i].first == "import_style") {
if (options[i].second == "closure") {
import_style = IMPORT_CLOSURE;
} else if (options[i].second == "commonjs") {
import_style = IMPORT_COMMONJS;
} else if (options[i].second == "browser") {
import_style = IMPORT_BROWSER;
} else if (options[i].second == "es6") {
import_style = IMPORT_ES6;
} else {
*error = "Unknown import style " + options[i].second + ", expected " +
"one of: closure, commonjs, browser, es6.";
}
} else {
// Assume any other option is an output directory, as long as it is a bare
// `key` rather than a `key=value` option.
@ -2375,6 +2461,11 @@ bool GeneratorOptions::ParseFromOptions(
}
}
if (!library.empty() && import_style != IMPORT_CLOSURE) {
*error = "The library option should only be used for "
"import_style=closure";
}
return true;
}
@ -2418,6 +2509,47 @@ void Generator::GenerateFileAndDeps(
}
}
void Generator::GenerateFile(const GeneratorOptions& options,
io::Printer* printer,
const FileDescriptor* file) const {
GenerateHeader(options, printer);
// Generate "require" statements.
if (options.import_style == GeneratorOptions::IMPORT_COMMONJS) {
for (int i = 0; i < file->dependency_count(); i++) {
const std::string& name = file->dependency(i)->name();
printer->Print(
"var $alias$ = require('$file$');\n",
"alias", ModuleAlias(name),
"file", GetJSFilename(name));
}
}
// We aren't using Closure's import system, but we use goog.exportSymbol()
// to construct the expected tree of objects, eg.
//
// goog.exportSymbol('foo.bar.Baz', null, this);
//
// // Later generated code expects foo.bar = {} to exist:
// foo.bar.Baz = function() { /* ... */ }
std::set<std::string> provided;
FindProvidesForFile(options, printer, file, &provided);
//FindProvidesForFields(options, printer, extensions, &provided);
for (std::set<string>::iterator it = provided.begin();
it != provided.end(); ++it) {
printer->Print("goog.exportSymbol('$name$', null, this);\n",
"name", *it);
}
GenerateClassesAndEnums(options, printer, file);
// Extensions nested inside messages are emitted inside
// GenerateClassesAndEnums().
for (int i = 0; i < file->extension_count(); i++) {
GenerateExtension(options, printer, file->extension(i));
}
}
bool Generator::GenerateAll(const vector<const FileDescriptor*>& files,
const string& parameter,
GeneratorContext* context,
@ -2430,10 +2562,14 @@ bool Generator::GenerateAll(const vector<const FileDescriptor*>& files,
}
// We're either generating a single library file with definitions for message
// and enum types in *all* FileDescriptor inputs, or we're generating a single
// file for each type.
if (options.library != "") {
// There are three schemes for where output files go:
//
// - import_style = IMPORT_CLOSURE, library non-empty: all output in one file
// - import_style = IMPORT_CLOSURE, library empty: one output file per type
// - import_style != IMPORT_CLOSURE: one output file per .proto file
if (options.import_style == GeneratorOptions::IMPORT_CLOSURE &&
options.library != "") {
// All output should go in a single file.
string filename = options.output_dir + "/" + options.library + ".js";
google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(context->Open(filename));
GOOGLE_CHECK(output.get());
@ -2469,7 +2605,7 @@ bool Generator::GenerateAll(const vector<const FileDescriptor*>& files,
if (printer.failed()) {
return false;
}
} else {
} else if (options.import_style == GeneratorOptions::IMPORT_CLOSURE) {
// Collect all types, and print each type to a separate file. Pull out
// free-floating extensions while we make this pass.
map< string, vector<const FieldDescriptor*> > extensions_by_namespace;
@ -2611,6 +2747,24 @@ bool Generator::GenerateAll(const vector<const FileDescriptor*>& files,
}
}
}
} else {
// Generate one output file per input (.proto) file.
for (int i = 0; i < files.size(); i++) {
const google::protobuf::FileDescriptor* file = files[i];
string filename = options.output_dir + "/" + GetJSFilename(file->name());
google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(
context->Open(filename));
GOOGLE_CHECK(output.get());
io::Printer printer(output.get(), '$');
GenerateFile(options, &printer, file);
if (printer.failed()) {
return false;
}
}
}
return true;

18
src/google/protobuf/compiler/js/js_generator.h
View file

@ -67,6 +67,13 @@ struct GeneratorOptions {
bool error_on_name_conflict;
// Enable binary-format support?
bool binary;
// What style of imports should be used.
enum ImportStyle {
IMPORT_CLOSURE, // goog.require()
IMPORT_COMMONJS, // require()
IMPORT_BROWSER, // no import statements
IMPORT_ES6, // import { member } from ''
} import_style;
GeneratorOptions()
: add_require_for_enums(false),
@ -75,7 +82,8 @@ struct GeneratorOptions {
namespace_prefix(""),
library(""),
error_on_name_conflict(false),
binary(false) {}
binary(false),
import_style(IMPORT_CLOSURE) {}
bool ParseFromOptions(
const vector< pair< string, string > >& options,
@ -111,6 +119,10 @@ class LIBPROTOC_EXPORT Generator : public CodeGenerator {
io::Printer* printer,
const vector<const FileDescriptor*>& file,
std::set<string>* provided) const;
void FindProvidesForFile(const GeneratorOptions& options,
io::Printer* printer,
const FileDescriptor* file,
std::set<string>* provided) const;
void FindProvidesForMessage(const GeneratorOptions& options,
io::Printer* printer,
const Descriptor* desc,
@ -168,6 +180,10 @@ class LIBPROTOC_EXPORT Generator : public CodeGenerator {
std::set<string>* required,
std::set<string>* forwards) const;
void GenerateFile(const GeneratorOptions& options,
io::Printer* printer,
const FileDescriptor* file) const;
// Generate definitions for all message classes and enums in all files,
// processing the files in dependence order.
void GenerateFilesInDepOrder(const GeneratorOptions& options,