1 #region Copyright notice and license 2 3 // Copyright 2018 gRPC authors. 4 // 5 // Licensed under the Apache License, Version 2.0 (the "License"); 6 // you may not use this file except in compliance with the License. 7 // You may obtain a copy of the License at 8 // 9 // http://www.apache.org/licenses/LICENSE-2.0 10 // 11 // Unless required by applicable law or agreed to in writing, software 12 // distributed under the License is distributed on an "AS IS" BASIS, 13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 // See the License for the specific language governing permissions and 15 // limitations under the License. 16 17 #endregion 18 19 using System; 20 using System.Collections.Generic; 21 using System.Text; 22 using System.Text.RegularExpressions; 23 using Microsoft.Build.Framework; 24 using Microsoft.Build.Utilities; 25 26 namespace Grpc.Tools 27 { 28 /// <summary> 29 /// Run Google proto compiler (protoc). 30 /// 31 /// After a successful run, the task reads the dependency file if specified 32 /// to be saved by the compiler, and returns its output files. 33 /// 34 /// This task (unlike PrepareProtoCompile) does not attempt to guess anything 35 /// about language-specific behavior of protoc, and therefore can be used for 36 /// any language outputs. 37 /// </summary> 38 public class ProtoCompile : ToolTask 39 { 40 /* 41 42 Usage: /home/kkm/work/protobuf/src/.libs/lt-protoc [OPTION] PROTO_FILES 43 Parse PROTO_FILES and generate output based on the options given: 44 -IPATH, --proto_path=PATH Specify the directory in which to search for 45 imports. May be specified multiple times; 46 directories will be searched in order. If not 47 given, the current working directory is used. 48 --version Show version info and exit. 49 -h, --help Show this text and exit. 50 --encode=MESSAGE_TYPE Read a text-format message of the given type 51 from standard input and write it in binary 52 to standard output. The message type must 53 be defined in PROTO_FILES or their imports. 54 --decode=MESSAGE_TYPE Read a binary message of the given type from 55 standard input and write it in text format 56 to standard output. The message type must 57 be defined in PROTO_FILES or their imports. 58 --decode_raw Read an arbitrary protocol message from 59 standard input and write the raw tag/value 60 pairs in text format to standard output. No 61 PROTO_FILES should be given when using this 62 flag. 63 --descriptor_set_in=FILES Specifies a delimited list of FILES 64 each containing a FileDescriptorSet (a 65 protocol buffer defined in descriptor.proto). 66 The FileDescriptor for each of the PROTO_FILES 67 provided will be loaded from these 68 FileDescriptorSets. If a FileDescriptor 69 appears multiple times, the first occurrence 70 will be used. 71 -oFILE, Writes a FileDescriptorSet (a protocol buffer, 72 --descriptor_set_out=FILE defined in descriptor.proto) containing all of 73 the input files to FILE. 74 --include_imports When using --descriptor_set_out, also include 75 all dependencies of the input files in the 76 set, so that the set is self-contained. 77 --include_source_info When using --descriptor_set_out, do not strip 78 SourceCodeInfo from the FileDescriptorProto. 79 This results in vastly larger descriptors that 80 include information about the original 81 location of each decl in the source file as 82 well as surrounding comments. 83 --dependency_out=FILE Write a dependency output file in the format 84 expected by make. This writes the transitive 85 set of input file paths to FILE 86 --error_format=FORMAT Set the format in which to print errors. 87 FORMAT may be 'gcc' (the default) or 'msvs' 88 (Microsoft Visual Studio format). 89 --print_free_field_numbers Print the free field numbers of the messages 90 defined in the given proto files. Groups share 91 the same field number space with the parent 92 message. Extension ranges are counted as 93 occupied fields numbers. 94 95 --plugin=EXECUTABLE Specifies a plugin executable to use. 96 Normally, protoc searches the PATH for 97 plugins, but you may specify additional 98 executables not in the path using this flag. 99 Additionally, EXECUTABLE may be of the form 100 NAME=PATH, in which case the given plugin name 101 is mapped to the given executable even if 102 the executable's own name differs. 103 --cpp_out=OUT_DIR Generate C++ header and source. 104 --csharp_out=OUT_DIR Generate C# source file. 105 --java_out=OUT_DIR Generate Java source file. 106 --javanano_out=OUT_DIR Generate Java Nano source file. 107 --js_out=OUT_DIR Generate JavaScript source. 108 --objc_out=OUT_DIR Generate Objective C header and source. 109 --php_out=OUT_DIR Generate PHP source file. 110 --python_out=OUT_DIR Generate Python source file. 111 --ruby_out=OUT_DIR Generate Ruby source file. 112 @<filename> Read options and filenames from file. If a 113 relative file path is specified, the file 114 will be searched in the working directory. 115 The --proto_path option will not affect how 116 this argument file is searched. Content of 117 the file will be expanded in the position of 118 @<filename> as in the argument list. Note 119 that shell expansion is not applied to the 120 content of the file (i.e., you cannot use 121 quotes, wildcards, escapes, commands, etc.). 122 Each line corresponds to a single argument, 123 even if it contains spaces. 124 */ 125 static string[] s_supportedGenerators = new[] { "cpp", "csharp", "java", 126 "javanano", "js", "objc", 127 "php", "python", "ruby" }; 128 129 static readonly TimeSpan s_regexTimeout = TimeSpan.FromSeconds(1); 130 131 static readonly List<ErrorListFilter> s_errorListFilters = new List<ErrorListFilter>() 132 { 133 // Example warning with location 134 //../Protos/greet.proto(19) : warning in column=5 : warning : When enum name is stripped and label is PascalCased (Zero), 135 // this value label conflicts with Zero. This will make the proto fail to compile for some languages, such as C#. 136 new ErrorListFilter 137 { 138 Pattern = new Regex( 139 pattern: "^(?'FILENAME'.+?)\\((?'LINE'\\d+)\\) ?: ?warning in column=(?'COLUMN'\\d+) ?: ?(?'TEXT'.*)", 140 options: RegexOptions.Compiled | RegexOptions.IgnoreCase, 141 matchTimeout: s_regexTimeout), 142 LogAction = (log, match) => 143 { 144 int.TryParse(match.Groups["LINE"].Value, out var line); 145 int.TryParse(match.Groups["COLUMN"].Value, out var column); 146 147 log.LogWarning( 148 subcategory: null, 149 warningCode: null, 150 helpKeyword: null, 151 file: match.Groups["FILENAME"].Value, 152 lineNumber: line, 153 columnNumber: column, 154 endLineNumber: 0, 155 endColumnNumber: 0, 156 message: match.Groups["TEXT"].Value); 157 } 158 }, 159 160 // Example error with location 161 //../Protos/greet.proto(14) : error in column=10: "name" is already defined in "Greet.HelloRequest". 162 new ErrorListFilter 163 { 164 Pattern = new Regex( 165 pattern: "^(?'FILENAME'.+?)\\((?'LINE'\\d+)\\) ?: ?error in column=(?'COLUMN'\\d+) ?: ?(?'TEXT'.*)", 166 options: RegexOptions.Compiled | RegexOptions.IgnoreCase, 167 matchTimeout: s_regexTimeout), 168 LogAction = (log, match) => 169 { 170 int.TryParse(match.Groups["LINE"].Value, out var line); 171 int.TryParse(match.Groups["COLUMN"].Value, out var column); 172 173 log.LogError( 174 subcategory: null, 175 errorCode: null, 176 helpKeyword: null, 177 file: match.Groups["FILENAME"].Value, 178 lineNumber: line, 179 columnNumber: column, 180 endLineNumber: 0, 181 endColumnNumber: 0, 182 message: match.Groups["TEXT"].Value); 183 } 184 }, 185 186 // Example warning without location 187 //../Protos/greet.proto: warning: Import google/protobuf/empty.proto but not used. 188 new ErrorListFilter 189 { 190 Pattern = new Regex( 191 pattern: "^(?'FILENAME'.+?): ?warning: ?(?'TEXT'.*)", 192 options: RegexOptions.Compiled | RegexOptions.IgnoreCase, 193 matchTimeout: s_regexTimeout), 194 LogAction = (log, match) => 195 { 196 log.LogWarning( 197 subcategory: null, 198 warningCode: null, 199 helpKeyword: null, 200 file: match.Groups["FILENAME"].Value, 201 lineNumber: 0, 202 columnNumber: 0, 203 endLineNumber: 0, 204 endColumnNumber: 0, 205 message: match.Groups["TEXT"].Value); 206 } 207 }, 208 209 // Example warning from plugins that use GOOGLE_LOG 210 // [libprotobuf WARNING T:\altsrc\..\csharp_enum.cc:74] Duplicate enum value Work (originally Work) in PhoneType; adding underscore to distinguish 211 new ErrorListFilter 212 { 213 Pattern = new Regex( 214 pattern: "^\\[.+? WARNING (?'FILENAME'.+?):(?'LINE'\\d+?)\\] ?(?'TEXT'.*)", 215 options: RegexOptions.Compiled | RegexOptions.IgnoreCase, 216 matchTimeout: s_regexTimeout), 217 LogAction = (log, match) => 218 { 219 // The filename and line logged by the plugins may not be useful to the 220 // end user as they are not the location in the proto file but rather 221 // in the source code for the plugin. Log them anyway as they may help in 222 // diagnostics. 223 int.TryParse(match.Groups["LINE"].Value, out var line); 224 log.LogWarning( 225 subcategory: null, 226 warningCode: null, 227 helpKeyword: null, 228 file: match.Groups["FILENAME"].Value, 229 lineNumber: line, 230 columnNumber: 0, 231 endLineNumber: 0, 232 endColumnNumber: 0, 233 message: match.Groups["TEXT"].Value); 234 } 235 }, 236 237 // Example error from plugins that use GOOGLE_LOG 238 // [libprotobuf ERROR T:\path\...\filename:23] Some message 239 // [libprotobuf FATAL T:\path\...\filename:23] Some message 240 new ErrorListFilter 241 { 242 Pattern = new Regex( 243 pattern: "^\\[.+? (?'LEVEL'ERROR|FATAL) (?'FILENAME'.+?):(?'LINE'\\d+?)\\] ?(?'TEXT'.*)", 244 options: RegexOptions.Compiled | RegexOptions.IgnoreCase, 245 matchTimeout: s_regexTimeout), 246 LogAction = (log, match) => 247 { 248 // The filename and line logged by the plugins may not be useful to the 249 // end user as they are not the location in the proto file but rather 250 // in the source code for the plugin. Log them anyway as they may help in 251 // diagnostics. 252 int.TryParse(match.Groups["LINE"].Value, out var line); 253 log.LogError( 254 subcategory: null, 255 errorCode: null, 256 helpKeyword: null, 257 file: match.Groups["FILENAME"].Value, 258 lineNumber: line, 259 columnNumber: 0, 260 endLineNumber: 0, 261 endColumnNumber: 0, 262 message: match.Groups["LEVEL"].Value + " " + match.Groups["TEXT"].Value); 263 } 264 }, 265 266 // Example error without location 267 //../Protos/greet.proto: Import "google/protobuf/empty.proto" was listed twice. 268 new ErrorListFilter 269 { 270 Pattern = new Regex( 271 pattern: "^(?'FILENAME'.+?): ?(?'TEXT'.*)", 272 options: RegexOptions.Compiled | RegexOptions.IgnoreCase, 273 matchTimeout: s_regexTimeout), 274 LogAction = (log, match) => 275 { 276 log.LogError( 277 subcategory: null, 278 errorCode: null, 279 helpKeyword: null, 280 file: match.Groups["FILENAME"].Value, 281 lineNumber: 0, 282 columnNumber: 0, 283 endLineNumber: 0, 284 endColumnNumber: 0, 285 message: match.Groups["TEXT"].Value); 286 } 287 } 288 }; 289 290 /// <summary> 291 /// Code generator. 292 /// </summary> 293 [Required] 294 public string Generator { get; set; } 295 296 /// <summary> 297 /// Protobuf files to compile. 298 /// </summary> 299 [Required] 300 public ITaskItem[] Protobuf { get; set; } 301 302 /// <summary> 303 /// Directory where protoc dependency files are cached. If provided, dependency 304 /// output filename is autogenerated from source directory hash and file name. 305 /// Mutually exclusive with DependencyOut. 306 /// Switch: --dependency_out (with autogenerated file name). 307 /// </summary> 308 public string ProtoDepDir { get; set; } 309 310 /// <summary> 311 /// Dependency file full name. Mutually exclusive with ProtoDepDir. 312 /// Autogenerated file name is available in this property after execution. 313 /// Switch: --dependency_out. 314 /// </summary> 315 [Output] 316 public string DependencyOut { get; set; } 317 318 /// <summary> 319 /// The directories to search for imports. Directories will be searched 320 /// in order. If not given, the current working directory is used. 321 /// Switch: --proto_path. 322 /// </summary> 323 public string[] ProtoPath { get; set; } 324 325 /// <summary> 326 /// Generated code directory. The generator property determines the language. 327 /// Switch: --GEN_out= (for different generators GEN, e.g. --csharp_out). 328 /// </summary> 329 [Required] 330 public string OutputDir { get; set; } 331 332 /// <summary> 333 /// Codegen options. See also OptionsFromMetadata. 334 /// Switch: --GEN_opt= (for different generators GEN, e.g. --csharp_opt). 335 /// </summary> 336 public string[] OutputOptions { get; set; } 337 338 /// <summary> 339 /// Additional arguments that will be passed unmodified to protoc (and before any file names). 340 /// For example, "--experimental_allow_proto3_optional" 341 /// </summary> 342 public string[] AdditionalProtocArguments { get; set; } 343 344 /// <summary> 345 /// Full path to the gRPC plugin executable. If specified, gRPC generation 346 /// is enabled for the files. 347 /// Switch: --plugin=protoc-gen-grpc= 348 /// </summary> 349 public string GrpcPluginExe { get; set; } 350 351 /// <summary> 352 /// Generated gRPC directory. The generator property determines the 353 /// language. If gRPC is enabled but this is not given, OutputDir is used. 354 /// Switch: --grpc_out= 355 /// </summary> 356 public string GrpcOutputDir { get; set; } 357 358 /// <summary> 359 /// gRPC Codegen options. See also OptionsFromMetadata. 360 /// --grpc_opt=opt1,opt2=val (comma-separated). 361 /// </summary> 362 public string[] GrpcOutputOptions { get; set; } 363 364 /// <summary> 365 /// List of files written in addition to generated outputs. Includes a 366 /// single item for the dependency file if written. 367 /// </summary> 368 [Output] 369 public ITaskItem[] AdditionalFileWrites { get; private set; } 370 371 /// <summary> 372 /// List of language files generated by protoc. Empty unless DependencyOut 373 /// or ProtoDepDir is set, since the file writes are extracted from protoc 374 /// dependency output file. 375 /// </summary> 376 [Output] 377 public ITaskItem[] GeneratedFiles { get; private set; } 378 379 // Hide this property from MSBuild, we should never use a shell script. 380 private new bool UseCommandProcessor { get; set; } 381 382 protected override string ToolName => Platform.IsWindows ? "protoc.exe" : "protoc"; 383 384 // Since we never try to really locate protoc.exe somehow, just try ToolExe 385 // as the full tool location. It will be either just protoc[.exe] from 386 // ToolName above if not set by the user, or a user-supplied full path. The 387 // base class will then resolve the former using system PATH. GenerateFullPathToTool()388 protected override string GenerateFullPathToTool() => ToolExe; 389 390 // Log protoc errors with the High priority (bold white in MsBuild, 391 // printed with -v:n, and shown in the Output windows in VS). 392 protected override MessageImportance StandardErrorLoggingImportance => MessageImportance.High; 393 394 // Called by base class to validate arguments and make them consistent. ValidateParameters()395 protected override bool ValidateParameters() 396 { 397 // Part of proto command line switches, must be lowercased. 398 Generator = Generator.ToLowerInvariant(); 399 if (!System.Array.Exists(s_supportedGenerators, g => g == Generator)) 400 { 401 Log.LogError("Invalid value for Generator='{0}'. Supported generators: {1}", 402 Generator, string.Join(", ", s_supportedGenerators)); 403 } 404 405 if (ProtoDepDir != null && DependencyOut != null) 406 { 407 Log.LogError("Properties ProtoDepDir and DependencyOut may not be both specified"); 408 } 409 410 if (Protobuf.Length > 1 && (ProtoDepDir != null || DependencyOut != null)) 411 { 412 Log.LogError("Proto compiler currently allows only one input when " + 413 "--dependency_out is specified (via ProtoDepDir or DependencyOut). " + 414 "Tracking issue: https://github.com/protocolbuffers/protobuf/pull/3959"); 415 } 416 417 // Use ProtoDepDir to autogenerate DependencyOut 418 if (ProtoDepDir != null) 419 { 420 DependencyOut = DepFileUtil.GetDepFilenameForProto(ProtoDepDir, Protobuf[0].ItemSpec); 421 } 422 423 if (GrpcPluginExe == null) 424 { 425 GrpcOutputOptions = null; 426 GrpcOutputDir = null; 427 } 428 else if (GrpcOutputDir == null) 429 { 430 // Use OutputDir for gRPC output if not specified otherwise by user. 431 GrpcOutputDir = OutputDir; 432 } 433 434 return !Log.HasLoggedErrors && base.ValidateParameters(); 435 } 436 437 // Protoc chokes on BOM, naturally. I would! 438 static readonly Encoding s_utf8WithoutBom = new UTF8Encoding(false); 439 protected override Encoding ResponseFileEncoding => s_utf8WithoutBom; 440 441 // Protoc takes one argument per line from the response file, and does not 442 // require any quoting whatsoever. Otherwise, this is similar to the 443 // standard CommandLineBuilder 444 class ProtocResponseFileBuilder 445 { 446 StringBuilder _data = new StringBuilder(1000); ToString()447 public override string ToString() => _data.ToString(); 448 449 // If 'value' is not empty, append '--name=value\n'. AddSwitchMaybe(string name, string value)450 public void AddSwitchMaybe(string name, string value) 451 { 452 if (!string.IsNullOrEmpty(value)) 453 { 454 _data.Append("--").Append(name).Append("=") 455 .Append(value).Append('\n'); 456 } 457 } 458 459 // Add switch with the 'values' separated by commas, for options. AddSwitchMaybe(string name, string[] values)460 public void AddSwitchMaybe(string name, string[] values) 461 { 462 if (values?.Length > 0) 463 { 464 _data.Append("--").Append(name).Append("=") 465 .Append(string.Join(",", values)).Append('\n'); 466 } 467 } 468 469 // Add a positional argument to the file data. AddArg(string arg)470 public void AddArg(string arg) 471 { 472 _data.Append(arg).Append('\n'); 473 } 474 }; 475 476 // Called by the base ToolTask to get response file contents. GenerateResponseFileCommands()477 protected override string GenerateResponseFileCommands() 478 { 479 var cmd = new ProtocResponseFileBuilder(); 480 cmd.AddSwitchMaybe(Generator + "_out", TrimEndSlash(OutputDir)); 481 cmd.AddSwitchMaybe(Generator + "_opt", OutputOptions); 482 cmd.AddSwitchMaybe("plugin=protoc-gen-grpc", GrpcPluginExe); 483 cmd.AddSwitchMaybe("grpc_out", TrimEndSlash(GrpcOutputDir)); 484 cmd.AddSwitchMaybe("grpc_opt", GrpcOutputOptions); 485 if (ProtoPath != null) 486 { 487 foreach (string path in ProtoPath) 488 { 489 cmd.AddSwitchMaybe("proto_path", TrimEndSlash(path)); 490 } 491 } 492 cmd.AddSwitchMaybe("dependency_out", DependencyOut); 493 cmd.AddSwitchMaybe("error_format", "msvs"); 494 495 if (AdditionalProtocArguments != null) 496 { 497 foreach (var additionalProtocOption in AdditionalProtocArguments) 498 { 499 cmd.AddArg(additionalProtocOption); 500 } 501 } 502 503 foreach (var proto in Protobuf) 504 { 505 cmd.AddArg(proto.ItemSpec); 506 } 507 return cmd.ToString(); 508 } 509 510 // Protoc cannot digest trailing slashes in directory names, 511 // curiously under Linux, but not in Windows. TrimEndSlash(string dir)512 static string TrimEndSlash(string dir) 513 { 514 if (dir == null || dir.Length <= 1) 515 { 516 return dir; 517 } 518 string trim = dir.TrimEnd('/', '\\'); 519 // Do not trim the root slash, drive letter possible. 520 if (trim.Length == 0) 521 { 522 // Slashes all the way down. 523 return dir.Substring(0, 1); 524 } 525 if (trim.Length == 2 && dir.Length > 2 && trim[1] == ':') 526 { 527 // We have a drive letter and root, e. g. 'C:\' 528 return dir.Substring(0, 3); 529 } 530 return trim; 531 } 532 533 // Called by the base class to log tool's command line. 534 // 535 // Protoc command file is peculiar, with one argument per line, separated 536 // by newlines. Unwrap it for log readability into a single line, and also 537 // quote arguments, lest it look weird and so it may be copied and pasted 538 // into shell. Since this is for logging only, correct enough is correct. LogToolCommand(string cmd)539 protected override void LogToolCommand(string cmd) 540 { 541 var printer = new StringBuilder(1024); 542 543 // Print 'str' slice into 'printer', wrapping in quotes if contains some 544 // interesting characters in file names, or if empty string. The list of 545 // characters requiring quoting is not by any means exhaustive; we are 546 // just striving to be nice, not guaranteeing to be nice. 547 var quotable = new[] { ' ', '!', '$', '&', '\'', '^' }; 548 void PrintQuoting(string str, int start, int count) 549 { 550 bool wrap = count == 0 || str.IndexOfAny(quotable, start, count) >= 0; 551 if (wrap) printer.Append('"'); 552 printer.Append(str, start, count); 553 if (wrap) printer.Append('"'); 554 } 555 556 for (int ib = 0, ie; (ie = cmd.IndexOf('\n', ib)) >= 0; ib = ie + 1) 557 { 558 // First line only contains both the program name and the first switch. 559 // We can rely on at least the '--out_dir' switch being always present. 560 if (ib == 0) 561 { 562 int iep = cmd.IndexOf(" --"); 563 if (iep > 0) 564 { 565 PrintQuoting(cmd, 0, iep); 566 ib = iep + 1; 567 } 568 } 569 printer.Append(' '); 570 if (cmd[ib] == '-') 571 { 572 // Print switch unquoted, including '=' if any. 573 int iarg = cmd.IndexOf('=', ib, ie - ib); 574 if (iarg < 0) 575 { 576 // Bare switch without a '='. 577 printer.Append(cmd, ib, ie - ib); 578 continue; 579 } 580 printer.Append(cmd, ib, iarg + 1 - ib); 581 ib = iarg + 1; 582 } 583 // A positional argument or switch value. 584 PrintQuoting(cmd, ib, ie - ib); 585 } 586 587 base.LogToolCommand(printer.ToString()); 588 } 589 LogEventsFromTextOutput(string singleLine, MessageImportance messageImportance)590 protected override void LogEventsFromTextOutput(string singleLine, MessageImportance messageImportance) 591 { 592 foreach (ErrorListFilter filter in s_errorListFilters) 593 { 594 try 595 { 596 Match match = filter.Pattern.Match(singleLine); 597 598 if (match.Success) 599 { 600 filter.LogAction(Log, match); 601 return; 602 } 603 } catch (RegexMatchTimeoutException) 604 { 605 Log.LogWarning("Unable to parse output from protoc. Regex timeout."); 606 } 607 } 608 609 base.LogEventsFromTextOutput(singleLine, messageImportance); 610 } 611 612 // Main task entry point. Execute()613 public override bool Execute() 614 { 615 base.UseCommandProcessor = false; 616 617 bool ok = base.Execute(); 618 if (!ok) 619 { 620 return false; 621 } 622 623 // Read dependency output file from the compiler to retrieve the 624 // definitive list of created files. Report the dependency file 625 // itself as having been written to. 626 if (DependencyOut != null) 627 { 628 string[] outputs = DepFileUtil.ReadDependencyOutputs(DependencyOut, Log); 629 if (HasLoggedErrors) 630 { 631 return false; 632 } 633 634 GeneratedFiles = new ITaskItem[outputs.Length]; 635 for (int i = 0; i < outputs.Length; i++) 636 { 637 GeneratedFiles[i] = new TaskItem(outputs[i]); 638 } 639 AdditionalFileWrites = new ITaskItem[] { new TaskItem(DependencyOut) }; 640 } 641 642 return true; 643 } 644 645 class ErrorListFilter 646 { 647 public Regex Pattern { get; set; } 648 public Action<TaskLoggingHelper, Match> LogAction { get; set; } 649 } 650 }; 651 } 652