From 4b8c6a33e116bb84d4b1c57a4c9478cc39d6e6e4 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Thu, 7 Apr 2022 13:03:38 +0200 Subject: [PATCH 01/22] Checkpoint --- prost-build/Cargo.toml | 4 +- prost-build/build.rs | 55 +++++- prost-build/src/lib.rs | 206 +++++++++++++++----- prost-build/src/libprotoc.cpp | 348 ++++++++++++++++++++++++++++++++++ 4 files changed, 563 insertions(+), 50 deletions(-) create mode 100644 prost-build/src/libprotoc.cpp diff --git a/prost-build/Cargo.toml b/prost-build/Cargo.toml index 751cac6f7..ed52bd66e 100644 --- a/prost-build/Cargo.toml +++ b/prost-build/Cargo.toml @@ -30,9 +30,9 @@ lazy_static = "1.4.0" regex = { version = "1.5.5", default-features = false, features = ["std", "unicode-bool"] } [build-dependencies] -which = { version = "4", default-features = false } +cc = { version = "1.0", features = ["parallel"] } cfg-if = "1" -cmake = "0.1" +which = { version = "4", default-features = false } [dev-dependencies] env_logger = { version = "0.8", default-features = false } diff --git a/prost-build/build.rs b/prost-build/build.rs index 89abf0176..dc26bdf2f 100644 --- a/prost-build/build.rs +++ b/prost-build/build.rs @@ -73,15 +73,62 @@ fn vendored() -> bool { } } -/// Compile `protoc` via `cmake`. +/// Compile `protoc`. fn compile() -> Option { - let protobuf_src = bundle_path().join("protobuf").join("cmake"); + let protobuf_src = bundle_path().join("protobuf/src/google/protobuf"); println!("cargo:rerun-if-changed={}", protobuf_src.display()); - let dst = cmake::Config::new(protobuf_src).build(); + // compile our protoc wrapper lib + { + let mut build = cc::Build::new(); + build.cpp(true); + build.flag("-Wno-unused-parameter"); + + build.includes([bundle_path().join("protobuf/src")]); + + build.files( + [ + "any.cc", + "any_lite.cc", + "arena.cc", + "descriptor.cc", + "dynamic_message.cc", + "extension_set.cc", + "map_field.cc", + "message.cc", + "message_lite.cc", + "generated_message_reflection.cc", + "generated_message_util.cc", + "parse_context.cc", + "repeated_field.cc", + "repeated_ptr_field.cc", + "text_format.cc", + "unknown_field_set.cc", + "wire_format.cc", + "wire_format_lite.cc", + "io/coded_stream.cc", + "io/strtod.cc", + "io/tokenizer.cc", + "io/zero_copy_stream.cc", + "io/zero_copy_stream_impl_lite.cc", + "stubs/common.cc", + "stubs/stringpiece.cc", + "stubs/stringprintf.cc", + "stubs/structurally_valid.cc", + "stubs/strutil.cc", + ] + .iter() + .map(|fname| protobuf_src.join(fname)), + ); + + // This is our little wrapper that only does the 1 thing prost-build + // actually needs from the the bloated protoc binary + build.file("src/libprotoc.cpp"); + build.compile("protoc"); + } - Some(dst.join("bin").join("protoc")) + Some(PathBuf::from("linked")) } /// Try to find a `protoc` through a few methods. diff --git a/prost-build/src/lib.rs b/prost-build/src/lib.rs index e8defe4fb..db5818176 100644 --- a/prost-build/src/lib.rs +++ b/prost-build/src/lib.rs @@ -797,59 +797,64 @@ impl Config { // this figured out. // [1]: http://doc.crates.io/build-script.html#outputs-of-the-build-script - let tmp; - let file_descriptor_set_path = if let Some(path) = &self.file_descriptor_set_path { - path.clone() + let buf = if protoc() == Path::new("linked") { + Self::serialize_file_descriptor_set(protos, includes)? } else { - if self.skip_protoc_run { - return Err(Error::new( - ErrorKind::Other, - "file_descriptor_set_path is required with skip_protoc_run", - )); - } - tmp = tempfile::Builder::new().prefix("prost-build").tempdir()?; - tmp.path().join("prost-descriptor-set") - }; + let tmp; + let file_descriptor_set_path = if let Some(path) = &self.file_descriptor_set_path { + path.clone() + } else { + if self.skip_protoc_run { + return Err(Error::new( + ErrorKind::Other, + "file_descriptor_set_path is required with skip_protoc_run", + )); + } + tmp = tempfile::Builder::new().prefix("prost-build").tempdir()?; + tmp.path().join("prost-descriptor-set") + }; + + if !self.skip_protoc_run { + let mut cmd = Command::new(protoc()); + cmd.arg("--include_imports") + .arg("--include_source_info") + .arg("-o") + .arg(&file_descriptor_set_path); + + for include in includes { + cmd.arg("-I").arg(include.as_ref()); + } - if !self.skip_protoc_run { - let mut cmd = Command::new(protoc()); - cmd.arg("--include_imports") - .arg("--include_source_info") - .arg("-o") - .arg(&file_descriptor_set_path); + // Set the protoc include after the user includes in case the user wants to + // override one of the built-in .protos. + cmd.arg("-I").arg(protoc_include()); - for include in includes { - cmd.arg("-I").arg(include.as_ref()); - } + for arg in &self.protoc_args { + cmd.arg(arg); + } - // Set the protoc include after the user includes in case the user wants to - // override one of the built-in .protos. - cmd.arg("-I").arg(protoc_include()); + for proto in protos { + cmd.arg(proto.as_ref()); + } - for arg in &self.protoc_args { - cmd.arg(arg); - } + let output = cmd.output().map_err(|error| { + Error::new( + error.kind(), + format!("failed to invoke protoc (hint: https://docs.rs/prost-build/#sourcing-protoc): {}", error), + ) + })?; - for proto in protos { - cmd.arg(proto.as_ref()); + if !output.status.success() { + return Err(Error::new( + ErrorKind::Other, + format!("protoc failed: {}", String::from_utf8_lossy(&output.stderr)), + )); + } } - let output = cmd.output().map_err(|error| { - Error::new( - error.kind(), - format!("failed to invoke protoc (hint: https://docs.rs/prost-build/#sourcing-protoc): {}", error), - ) - })?; - - if !output.status.success() { - return Err(Error::new( - ErrorKind::Other, - format!("protoc failed: {}", String::from_utf8_lossy(&output.stderr)), - )); - } - } + fs::read(file_descriptor_set_path)? + }; - let buf = fs::read(file_descriptor_set_path)?; let file_descriptor_set = FileDescriptorSet::decode(&*buf).map_err(|error| { Error::new( ErrorKind::InvalidInput, @@ -913,6 +918,119 @@ impl Config { Ok(()) } + fn serialize_file_descriptor_set( + protos: &[impl AsRef], + includes: &[impl AsRef], + ) -> Result> { + use std::ffi::c_void; + + #[no_mangle] + unsafe extern "C" fn resize_callback(ctx: *mut c_void, size: usize) -> *mut c_void { + let vec: &mut Vec = &mut (*ctx.cast()); + vec.resize(size, 0); + vec.as_mut_ptr().cast() + } + + #[no_mangle] + unsafe extern "C" fn cap_callback(ctx: *mut c_void) -> usize { + let vec: &Vec = &(*ctx.cast()); + vec.capacity() + } + + #[repr(C)] + struct CPath { + path: *const i8, + len: usize, + } + + #[repr(C)] + struct Buffer { + resize_buffer: unsafe extern "C" fn(ctx: *mut c_void, size: usize) -> *mut c_void, + buffer_capacity: unsafe extern "C" fn(ctx: *mut c_void) -> usize, + context: *mut c_void, + } + + extern "C" { + fn write_descriptor_set( + input_files: *const CPath, + num_inputs: usize, + includes_paths: *const CPath, + num_includes: usize, + output: *mut Buffer, + ) -> i32; + } + + struct PathBufs<'s> { + #[allow(dead_code)] + storage: Vec>, + paths: Vec, + } + + fn path_bufs<'s>(paths: &'s [impl AsRef]) -> PathBufs<'s> { + let storage: Vec>; + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + storage = paths + .iter() + .map(|p| p.as_ref().as_os_str().as_bytes().into()) + .collect(); + } + + #[cfg(windows)] + { + use std::os::windows::ffi::OsStrExt; + storage = paths + .iter() + .map(|p| { + p.as_ref() + .as_os_str() + .encode_wide() + .flatten(|c| [((c & 0xff00) >> 8) as u8, c & 0xff]) + .collect::>() + .into() + }) + .collect(); + } + + let paths = storage + .iter() + .map(|sp| CPath { + path: sp.as_ref().as_ptr().cast(), + len: sp.len(), + }) + .collect(); + + PathBufs { storage, paths } + } + + let protos = path_bufs(protos); + let includes = path_bufs(includes); + + let mut output_vec = Vec::new(); + + let mut buffer = Buffer { + resize_buffer: resize_callback, + buffer_capacity: cap_callback, + context: (&mut output_vec as *mut Vec).cast(), + }; + + if unsafe { + write_descriptor_set( + protos.paths.as_ptr(), + protos.paths.len(), + includes.paths.as_ptr(), + includes.paths.len(), + &mut buffer, + ) + } == 0 + { + Ok(output_vec) + } else { + panic!("oh noes"); + } + } + fn write_includes( &self, mut entries: Vec<&Module>, diff --git a/prost-build/src/libprotoc.cpp b/prost-build/src/libprotoc.cpp new file mode 100644 index 000000000..44f7563ef --- /dev/null +++ b/prost-build/src/libprotoc.cpp @@ -0,0 +1,348 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + + + +extern "C" { + // Small and simple and FFI safe, unlike `StringPiece` + struct Path { + const char* path; + size_t len; + }; + + typedef void* (*resize)(void*, size_t); + typedef size_t (*capacity)(void*); + + // Simple buffer wrapper so that we can write directly to memory allocated + // in Rust + struct Buffer { + resize resize_buffer; + capacity buffer_capacity; + void* context; + }; +} + +using google::protobuf::FileDescriptor; +using google::protobuf::FileDescriptorSet; +using google::protobuf::DescriptorPool; +using google::protobuf::MergedDescriptorDatabase; +using google::protobuf::Message; +using google::protobuf::FileDescriptorProto; +using google::protobuf::RepeatedPtrField; + +using google::protobuf::compiler::DiskSourceTree; +using google::protobuf::compiler::SourceTreeDescriptorDatabase; + +// Port of CommandLineInterface::ParseInputFiles +bool parse_input_files( + const std::vector& input_files, + DescriptorPool* descriptor_pool, + std::vector* parsed_files +) { + for (const auto& input : input_files) { + // Import the file. + const FileDescriptor* parsed_file = descriptor_pool->FindFileByName(input); + if (parsed_file == nullptr) { + return false; + } + parsed_files->push_back(parsed_file); + } + + return true; +} + +bool make_inputs_relative(std::vector* inputs, DiskSourceTree* source_tree) { + for (auto& input_file : *inputs) { + std::string virtual_file, shadowing_disk_file; + + auto mapping = source_tree->DiskFileToVirtualFile( + input_file, + &virtual_file, + &shadowing_disk_file); + + switch (mapping) { + case DiskSourceTree::SUCCESS: { + input_file = virtual_file; + break; + } + case DiskSourceTree::SHADOWED: { + fprintf(stderr, "%s: Input is shadowed by an include in \"%s\"." + "Either use the latter file as your input or reorder the" + "includes so that the former file's location comes first.\n", + input_file.c_str(), shadowing_disk_file.c_str() + ); + return false; + } + case DiskSourceTree::CANNOT_OPEN: { + auto error_str = source_tree->GetLastErrorMessage().empty() + ? strerror(errno) + : source_tree->GetLastErrorMessage(); + fprintf(stderr, "Could not map to virtual file: %s: %s\n", input_file.c_str(), error_str.c_str()); + return false; + } + case DiskSourceTree::NO_MAPPING: { + // Try to interpret the path as a virtual path. + std::string disk_file; + if (source_tree->VirtualFileToDiskFile(input_file, &disk_file)) { + return true; + } else { + // The input file path can't be mapped to any --proto_path and it also + // can't be interpreted as a virtual path. + fprintf(stderr, "%s: File does not reside within any include path.\n", input_file.c_str()); + return false; + } + } + } + } + + return true; +} + +void get_transitive_deps( + const FileDescriptor* file, + bool include_json_name, + bool include_source_code_info, + std::set* already_seen, + RepeatedPtrField* output +) { + if (!already_seen->insert(file).second) { + return; + } + + for (int i = 0; i < file->dependency_count(); i++) { + get_transitive_deps( + file->dependency(i), + include_json_name, + include_source_code_info, + already_seen, + output + ); + } + + FileDescriptorProto* new_descriptor = output->Add(); + file->CopyTo(new_descriptor); + if (include_json_name) { + file->CopyJsonNameTo(new_descriptor); + } + if (include_source_code_info) { + file->CopySourceCodeInfoTo(new_descriptor); + } +} + +class BufferOutput : public google::protobuf::io::ZeroCopyOutputStream { +public: + Buffer* impl; + void* buffer_base = nullptr; + size_t len = 0; + + BufferOutput(Buffer* impl) : impl(impl) {} + + bool Next(void** data, int* size) final { + size_t old_size = this->len; + size_t cur_cap = this->impl->buffer_capacity(this->impl->context); + + size_t new_size; + if (old_size < cur_cap) { + new_size = cur_cap; + } else { + new_size = old_size * 2; + } + + // Avoid integer overflow in returned '*size'. + new_size = std::min(new_size, old_size + std::numeric_limits::max()); + this->buffer_base = this->impl->resize_buffer(this->impl->context, new_size); + + *data = ((uint8_t*)buffer_base + old_size); + *size = new_size - old_size; + this->len = new_size; + + return true; + } + + void BackUp(int count) final { + this->buffer_base = this->impl->resize_buffer(this->impl->context, this->len - count); + } + + int64_t ByteCount() const final { + return this->len; + } +}; + +bool write_descriptor_set(const std::vector& parsed_files, Buffer* output) { + FileDescriptorSet file_set; + + std::set already_seen; + + for (const auto& parsed : parsed_files) { + get_transitive_deps( + parsed, + true, // Include json_name + true, // Include source info, prost requires this + &already_seen, + file_set.mutable_file() + ); + } + + { + BufferOutput zero_copy_stream(output); + google::protobuf::io::CodedOutputStream coded_out(&zero_copy_stream); + + // Determinism is useful here because build outputs are sometimes checked + // into version control. + coded_out.SetSerializationDeterministic(true); + if (!file_set.SerializeToCodedStream(&coded_out)) { + return false; + } + } + + return true; +} + +// A MultiFileErrorCollector that prints errors to stderr. +class ErrorPrinter + : public google::protobuf::compiler::MultiFileErrorCollector + , public google::protobuf::io::ErrorCollector + , public DescriptorPool::ErrorCollector { +public: + ErrorPrinter(DiskSourceTree* tree = nullptr) + : tree_(tree), + found_errors_(false), + found_warnings_(false) {} + ~ErrorPrinter() {} + + // implements MultiFileErrorCollector ------------------------------ + void AddError( const std::string& filename, int line, int column, + const std::string& message) override { + found_errors_ = true; + AddErrorOrWarning(filename, line, column, message, "error", std::cerr); + } + + void AddWarning(const std::string& filename, int line, int column, + const std::string& message) override { + found_warnings_ = true; + AddErrorOrWarning(filename, line, column, message, "warning", std::clog); + } + + // implements io::ErrorCollector ----------------------------------- + void AddError(int line, int column, const std::string& message) final { + AddError("input", line, column, message); + } + + void AddWarning(int line, int column, const std::string& message) final { + AddErrorOrWarning("input", line, column, message, "warning", std::clog); + } + + // implements DescriptorPool::ErrorCollector------------------------- + void AddError( const std::string& filename, const std::string& element_name, + const Message* descriptor, ErrorLocation location, + const std::string& message) override { + AddErrorOrWarning(filename, -1, -1, message, "error", std::cerr); + } + + void AddWarning(const std::string& filename, const std::string& element_name, + const Message* descriptor, ErrorLocation location, + const std::string& message) final { + AddErrorOrWarning(filename, -1, -1, message, "warning", std::clog); + } + + bool FoundErrors() const { return found_errors_; } + + bool FoundWarnings() const { return found_warnings_; } + +private: + void AddErrorOrWarning( const std::string& filename, int line, int column, + const std::string& message, const std::string& type, + std::ostream& out) { + out << filename; + + // Users typically expect 1-based line/column numbers, so we add 1 + // to each here. + if (line != -1) { + out << ":" << (line + 1) << ":" << (column + 1); + } + + if (type == "warning") { + out << ": warning: " << message << std::endl; + } else { + out << ": " << message << std::endl; + } + } + + DiskSourceTree* tree_; + bool found_errors_; + bool found_warnings_; +}; + +extern "C" { + int write_descriptor_set( + const Path* input_files, + size_t num_inputs, + const Path* includes_paths, + size_t num_includes, + Buffer* output + ) { + // We're forced to reallocate these because some of the following APIs + // only take std::string :p + std::vector inputs(num_inputs); + // I'm sure there's some fancier way to initialize this these days but I + // prefer to keep the C++ as dumb as possible + for (size_t i = 0; i < num_inputs; ++i) { + inputs.push_back(std::string(input_files[i].path, input_files[i].len)); + } + + std::vector includes(num_includes); + for (size_t i = 0; i < num_includes; ++i) { + includes.push_back(std::string(includes_paths[i].path, includes_paths[i].len)); + } + + // Port of CommandLineInterface::InitializeDiskSourceTree + std::unique_ptr source_tree(new DiskSourceTree()); + std::unique_ptr descriptor_set_in_database; + + // Set up the source tree. Note that the way prost uses protoc, virtual + // paths would only be possible if the user is explicitly sending their + // own -I args with the ':/;' separated virtual path. I seriously doubt + // this happens in practice. (famous last words) + for (const auto& include : includes) { + source_tree->MapPath(include, ""); + } + + // Map input files to virtual paths if possible. I'm not sure if this + // is even needed since as stated prost really doesn't do virtual paths + if (!make_inputs_relative(&inputs, source_tree.get())) { + return 1; + } + + std::unique_ptr error_collector(new ErrorPrinter(source_tree.get())); + std::unique_ptr source_tree_database( + new SourceTreeDescriptorDatabase(source_tree.get(), descriptor_set_in_database.get())); + source_tree_database->RecordErrorsTo(error_collector.get()); + std::unique_ptr descriptor_pool(new DescriptorPool( + source_tree_database.get(), + source_tree_database->GetValidationErrorCollector())); + descriptor_pool->EnforceWeakDependencies(true); + + // Try to actually parse all of our inputs, if this + std::vector parsed(inputs.size()); + if (!parse_input_files(inputs, descriptor_pool.get(), &parsed)) { + return 1; + } + + if (!write_descriptor_set(parsed, output)) { + return 1; + } + + return 0; + } +} \ No newline at end of file From e160a815a67a8cbe7ce9a3d36e6c994cdf18e209 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Thu, 7 Apr 2022 17:25:26 +0200 Subject: [PATCH 02/22] Fixup --- prost-build/build.rs | 28 ++++++++++++++++++++++++++-- prost-build/src/libprotoc.cpp | 18 ++++++++++-------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/prost-build/build.rs b/prost-build/build.rs index dc26bdf2f..6727e713d 100644 --- a/prost-build/build.rs +++ b/prost-build/build.rs @@ -82,8 +82,19 @@ fn compile() -> Option { // compile our protoc wrapper lib { let mut build = cc::Build::new(); - build.cpp(true); - build.flag("-Wno-unused-parameter"); + build + .cpp(true) + // We _always_ want to build optmized, protoc code is far too slow otherwise + .opt_level_str("2"); + + // Disable all the compiler warnings for the protoc code we have no + // intention of changing + if !build.get_compiler().is_like_msvc() { + build + .flag("-Wno-unused-parameter") + .flag("-Wno-redundant-move") + .flag("-Wno-sign-compare"); + } build.includes([bundle_path().join("protobuf/src")]); @@ -92,31 +103,42 @@ fn compile() -> Option { "any.cc", "any_lite.cc", "arena.cc", + "arenastring.cc", "descriptor.cc", + "descriptor.pb.cc", + "descriptor_database.cc", "dynamic_message.cc", "extension_set.cc", + "extension_set_heavy.cc", + "implicit_weak_message.cc", + "map.cc", "map_field.cc", "message.cc", "message_lite.cc", "generated_message_reflection.cc", "generated_message_util.cc", "parse_context.cc", + "reflection_ops.cc", "repeated_field.cc", "repeated_ptr_field.cc", "text_format.cc", "unknown_field_set.cc", "wire_format.cc", "wire_format_lite.cc", + "compiler/importer.cc", + "compiler/parser.cc", "io/coded_stream.cc", "io/strtod.cc", "io/tokenizer.cc", "io/zero_copy_stream.cc", + "io/zero_copy_stream_impl.cc", "io/zero_copy_stream_impl_lite.cc", "stubs/common.cc", "stubs/stringpiece.cc", "stubs/stringprintf.cc", "stubs/structurally_valid.cc", "stubs/strutil.cc", + "stubs/substitute.cc", ] .iter() .map(|fname| protobuf_src.join(fname)), @@ -126,6 +148,8 @@ fn compile() -> Option { // actually needs from the the bloated protoc binary build.file("src/libprotoc.cpp"); build.compile("protoc"); + + println!("cargo:rerun-if-changed=src/libprotoc.cpp"); } Some(PathBuf::from("linked")) diff --git a/prost-build/src/libprotoc.cpp b/prost-build/src/libprotoc.cpp index 44f7563ef..3dc4fa6d5 100644 --- a/prost-build/src/libprotoc.cpp +++ b/prost-build/src/libprotoc.cpp @@ -161,6 +161,7 @@ class BufferOutput : public google::protobuf::io::ZeroCopyOutputStream { // Avoid integer overflow in returned '*size'. new_size = std::min(new_size, old_size + std::numeric_limits::max()); + new_size = std::max(new_size, size_t(1024)); this->buffer_base = this->impl->resize_buffer(this->impl->context, new_size); *data = ((uint8_t*)buffer_base + old_size); @@ -215,9 +216,8 @@ class ErrorPrinter , public google::protobuf::io::ErrorCollector , public DescriptorPool::ErrorCollector { public: - ErrorPrinter(DiskSourceTree* tree = nullptr) - : tree_(tree), - found_errors_(false), + ErrorPrinter() + : found_errors_(false), found_warnings_(false) {} ~ErrorPrinter() {} @@ -279,7 +279,6 @@ class ErrorPrinter } } - DiskSourceTree* tree_; bool found_errors_; bool found_warnings_; }; @@ -294,14 +293,16 @@ extern "C" { ) { // We're forced to reallocate these because some of the following APIs // only take std::string :p - std::vector inputs(num_inputs); + std::vector inputs; + inputs.reserve(num_inputs); // I'm sure there's some fancier way to initialize this these days but I // prefer to keep the C++ as dumb as possible for (size_t i = 0; i < num_inputs; ++i) { inputs.push_back(std::string(input_files[i].path, input_files[i].len)); } - std::vector includes(num_includes); + std::vector includes; + includes.reserve(num_includes); for (size_t i = 0; i < num_includes; ++i) { includes.push_back(std::string(includes_paths[i].path, includes_paths[i].len)); } @@ -324,7 +325,7 @@ extern "C" { return 1; } - std::unique_ptr error_collector(new ErrorPrinter(source_tree.get())); + std::unique_ptr error_collector(new ErrorPrinter()); std::unique_ptr source_tree_database( new SourceTreeDescriptorDatabase(source_tree.get(), descriptor_set_in_database.get())); source_tree_database->RecordErrorsTo(error_collector.get()); @@ -334,7 +335,8 @@ extern "C" { descriptor_pool->EnforceWeakDependencies(true); // Try to actually parse all of our inputs, if this - std::vector parsed(inputs.size()); + std::vector parsed; + parsed.reserve(inputs.size()); if (!parse_input_files(inputs, descriptor_pool.get(), &parsed)) { return 1; } From 454cdec335b632ae5fcc9b38a802f6484cb0124f Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Thu, 7 Apr 2022 17:40:48 +0200 Subject: [PATCH 03/22] Add deny.toml --- deny.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 deny.toml diff --git a/deny.toml b/deny.toml new file mode 100644 index 000000000..508ffd95d --- /dev/null +++ b/deny.toml @@ -0,0 +1,11 @@ +[licenses] +allow = ["Apache-2.0", "MIT", "BSD-3-Clause"] + +[bans] +deny = [{ name = "cmake" }] +skip = [ + # old csv via old criterion + { name = "itoa", version = "=0.4.8" }, + # Proptest includes 2 version :( + { name = "quick-error", version = "=1.2.3" }, +] From b2beb4106541c298520060afb8b8590a1a9a0205 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Thu, 7 Apr 2022 17:42:35 +0200 Subject: [PATCH 04/22] Add cargo-deny to CI --- .github/workflows/continuous-integration-workflow.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/continuous-integration-workflow.yaml b/.github/workflows/continuous-integration-workflow.yaml index 5efebde60..1db931bc9 100644 --- a/.github/workflows/continuous-integration-workflow.yaml +++ b/.github/workflows/continuous-integration-workflow.yaml @@ -138,3 +138,11 @@ jobs: - name: cargo check run: cd test-vendored && cargo check + deny: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - uses: EmbarkStudios/cargo-deny-action@v1 From f01ff47cf41020e070aedd111a90e22af9876f0f Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Thu, 5 May 2022 17:36:37 +0200 Subject: [PATCH 05/22] Use flat_map not flatten for windows --- prost-build/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prost-build/src/lib.rs b/prost-build/src/lib.rs index db5818176..67fb70cb9 100644 --- a/prost-build/src/lib.rs +++ b/prost-build/src/lib.rs @@ -986,7 +986,7 @@ impl Config { p.as_ref() .as_os_str() .encode_wide() - .flatten(|c| [((c & 0xff00) >> 8) as u8, c & 0xff]) + .flat_map(|c| [((c & 0xff00) >> 8) as u8, c & 0xff]) .collect::>() .into() }) From c40adce5ef826cbd3176e4f9e58be1d85c983523 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 07:11:31 +0200 Subject: [PATCH 06/22] Ignore clippy lint --- src/encoding.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/encoding.rs b/src/encoding.rs index 4953d6603..70ef62b2b 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -2,7 +2,11 @@ //! //! Meant to be used only from `Message` implementations. -#![allow(clippy::implicit_hasher, clippy::ptr_arg)] +#![allow( + clippy::implicit_hasher, + clippy::ptr_arg, + clippy::manual_range_contains +)] use alloc::collections::BTreeMap; use alloc::format; From 9d3246e133bfb9a95f8076a4ea6eb18646118be3 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 07:11:43 +0200 Subject: [PATCH 07/22] Fix path when checked out in github --- tests/single-include/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/single-include/Cargo.toml b/tests/single-include/Cargo.toml index 1d8eebebd..d95c9852c 100644 --- a/tests/single-include/Cargo.toml +++ b/tests/single-include/Cargo.toml @@ -7,7 +7,7 @@ publish = false license = "MIT" [dependencies] -prost = { path = "../../../prost" } +prost = { path = "../.." } [build-dependencies] prost-build = { path = "../../prost-build" } From a56895c5d938cd4291a8353c8651c955116c46d8 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 07:17:38 +0200 Subject: [PATCH 08/22] Fix 1.51 --- prost-build/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prost-build/build.rs b/prost-build/build.rs index 6727e713d..781f4b612 100644 --- a/prost-build/build.rs +++ b/prost-build/build.rs @@ -96,7 +96,7 @@ fn compile() -> Option { .flag("-Wno-sign-compare"); } - build.includes([bundle_path().join("protobuf/src")]); + build.includes(&[bundle_path().join("protobuf/src")]); build.files( [ From cac3683d954ed477e0a8d1f3f3ab0201f002c883 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Thu, 14 Apr 2022 08:54:15 +0200 Subject: [PATCH 09/22] Silence ubuntu warning --- prost-build/build.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prost-build/build.rs b/prost-build/build.rs index 781f4b612..6e3014da6 100644 --- a/prost-build/build.rs +++ b/prost-build/build.rs @@ -93,7 +93,8 @@ fn compile() -> Option { build .flag("-Wno-unused-parameter") .flag("-Wno-redundant-move") - .flag("-Wno-sign-compare"); + .flag("-Wno-sign-compare") + .flag("-Wno-stringop-overflow"); } build.includes(&[bundle_path().join("protobuf/src")]); From fab75537a6e0c5148debc80cef43166c6904176a Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Thu, 7 Apr 2022 21:57:05 +0200 Subject: [PATCH 10/22] Tell Mac it is in 21st century --- prost-build/build.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/prost-build/build.rs b/prost-build/build.rs index 6e3014da6..b566aa77e 100644 --- a/prost-build/build.rs +++ b/prost-build/build.rs @@ -91,6 +91,7 @@ fn compile() -> Option { // intention of changing if !build.get_compiler().is_like_msvc() { build + .flag("-std=c++11") .flag("-Wno-unused-parameter") .flag("-Wno-redundant-move") .flag("-Wno-sign-compare") From 87a6cb70706b85b244e4637da13c29e45e923c69 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 07:26:24 +0200 Subject: [PATCH 11/22] Ignore unmaintained advisory --- deny.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deny.toml b/deny.toml index 508ffd95d..978f2c9de 100644 --- a/deny.toml +++ b/deny.toml @@ -1,3 +1,9 @@ +[advisories] +ignore = [ + # serde_cbor is unmaintained, but brought in by criterion + "RUSTSEC-2021-0127", +] + [licenses] allow = ["Apache-2.0", "MIT", "BSD-3-Clause"] From b5fcf1b3b9f4e5c277ddedc9f35c9ae267294790 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 08:23:18 +0200 Subject: [PATCH 12/22] Oops, revert force push overwrite --- prost-build/build.rs | 123 ++++++++++++---------- prost-build/src/lib.rs | 192 ++++++++++++++++++++-------------- prost-build/src/libprotoc.cpp | 26 ++++- 3 files changed, 203 insertions(+), 138 deletions(-) diff --git a/prost-build/build.rs b/prost-build/build.rs index b566aa77e..31b134f61 100644 --- a/prost-build/build.rs +++ b/prost-build/build.rs @@ -16,7 +16,6 @@ //! 2. The bundled Protobuf include directory. //! -use cfg_if::cfg_if; use std::env; use std::path::PathBuf; use which::which; @@ -64,17 +63,14 @@ fn path_protoc() -> Option { /// Returns true if the vendored flag is enabled. fn vendored() -> bool { - cfg_if! { - if #[cfg(feature = "vendored")] { - true - } else { - false - } - } + env::var("CARGO_FEATURE_VENDORED").is_ok() } -/// Compile `protoc`. -fn compile() -> Option { +/// Compile `protoc`. If stub is true, this won't do anything other than emit +/// the single function we need to link, this is needed if the `vendored` +/// feature is being used, but the user has also set the `PROTOC_NO_VENDOR` flag +/// so protoc will be used at runtime +fn compile(stub: bool) -> Option { let protobuf_src = bundle_path().join("protobuf/src/google/protobuf"); println!("cargo:rerun-if-changed={}", protobuf_src.display()); @@ -100,51 +96,57 @@ fn compile() -> Option { build.includes(&[bundle_path().join("protobuf/src")]); - build.files( - [ - "any.cc", - "any_lite.cc", - "arena.cc", - "arenastring.cc", - "descriptor.cc", - "descriptor.pb.cc", - "descriptor_database.cc", - "dynamic_message.cc", - "extension_set.cc", - "extension_set_heavy.cc", - "implicit_weak_message.cc", - "map.cc", - "map_field.cc", - "message.cc", - "message_lite.cc", - "generated_message_reflection.cc", - "generated_message_util.cc", - "parse_context.cc", - "reflection_ops.cc", - "repeated_field.cc", - "repeated_ptr_field.cc", - "text_format.cc", - "unknown_field_set.cc", - "wire_format.cc", - "wire_format_lite.cc", - "compiler/importer.cc", - "compiler/parser.cc", - "io/coded_stream.cc", - "io/strtod.cc", - "io/tokenizer.cc", - "io/zero_copy_stream.cc", - "io/zero_copy_stream_impl.cc", - "io/zero_copy_stream_impl_lite.cc", - "stubs/common.cc", - "stubs/stringpiece.cc", - "stubs/stringprintf.cc", - "stubs/structurally_valid.cc", - "stubs/strutil.cc", - "stubs/substitute.cc", - ] - .iter() - .map(|fname| protobuf_src.join(fname)), - ); + if !stub { + build.files( + [ + "any.cc", + "any_lite.cc", + "arena.cc", + "arenastring.cc", + "descriptor.cc", + "descriptor.pb.cc", + "descriptor_database.cc", + "dynamic_message.cc", + "extension_set.cc", + "extension_set_heavy.cc", + "implicit_weak_message.cc", + "map.cc", + "map_field.cc", + "message.cc", + "message_lite.cc", + "generated_message_reflection.cc", + "generated_message_util.cc", + "parse_context.cc", + "reflection_ops.cc", + "repeated_field.cc", + "repeated_ptr_field.cc", + "text_format.cc", + "unknown_field_set.cc", + "wire_format.cc", + "wire_format_lite.cc", + "compiler/importer.cc", + "compiler/parser.cc", + "io/coded_stream.cc", + "io/strtod.cc", + "io/tokenizer.cc", + "io/zero_copy_stream.cc", + "io/zero_copy_stream_impl.cc", + "io/zero_copy_stream_impl_lite.cc", + "stubs/common.cc", + "stubs/stringpiece.cc", + "stubs/stringprintf.cc", + "stubs/structurally_valid.cc", + "stubs/strutil.cc", + "stubs/substitute.cc", + ] + .iter() + .map(|fname| protobuf_src.join(fname)), + ); + } + + if stub { + build.define("STUB_ONLY", "1"); + } // This is our little wrapper that only does the 1 thing prost-build // actually needs from the the bloated protoc binary @@ -162,11 +164,17 @@ fn compile() -> Option { /// Check module docs for more info. fn protoc() -> Option { if env::var_os("PROTOC_NO_VENDOR").is_some() { + compile(true); path_protoc() } else if vendored() { - compile() + compile(false) } else { - path_protoc().or_else(compile) + if let Some(path) = path_protoc() { + compile(true); + Some(path) + } else { + compile(false) + } } } @@ -188,4 +196,5 @@ fn main() { ); println!("cargo:rerun-if-env-changed=PROTOC"); println!("cargo:rerun-if-env-changed=PROTOC_INCLUDE"); + println!("cargo:rerun-if-env-changed=PROTOC_NO_VENDOR"); } diff --git a/prost-build/src/lib.rs b/prost-build/src/lib.rs index 67fb70cb9..ff6d744b9 100644 --- a/prost-build/src/lib.rs +++ b/prost-build/src/lib.rs @@ -797,63 +797,7 @@ impl Config { // this figured out. // [1]: http://doc.crates.io/build-script.html#outputs-of-the-build-script - let buf = if protoc() == Path::new("linked") { - Self::serialize_file_descriptor_set(protos, includes)? - } else { - let tmp; - let file_descriptor_set_path = if let Some(path) = &self.file_descriptor_set_path { - path.clone() - } else { - if self.skip_protoc_run { - return Err(Error::new( - ErrorKind::Other, - "file_descriptor_set_path is required with skip_protoc_run", - )); - } - tmp = tempfile::Builder::new().prefix("prost-build").tempdir()?; - tmp.path().join("prost-descriptor-set") - }; - - if !self.skip_protoc_run { - let mut cmd = Command::new(protoc()); - cmd.arg("--include_imports") - .arg("--include_source_info") - .arg("-o") - .arg(&file_descriptor_set_path); - - for include in includes { - cmd.arg("-I").arg(include.as_ref()); - } - - // Set the protoc include after the user includes in case the user wants to - // override one of the built-in .protos. - cmd.arg("-I").arg(protoc_include()); - - for arg in &self.protoc_args { - cmd.arg(arg); - } - - for proto in protos { - cmd.arg(proto.as_ref()); - } - - let output = cmd.output().map_err(|error| { - Error::new( - error.kind(), - format!("failed to invoke protoc (hint: https://docs.rs/prost-build/#sourcing-protoc): {}", error), - ) - })?; - - if !output.status.success() { - return Err(Error::new( - ErrorKind::Other, - format!("protoc failed: {}", String::from_utf8_lossy(&output.stderr)), - )); - } - } - - fs::read(file_descriptor_set_path)? - }; + let buf = self.load_file_descriptor_set(protos, includes)?; let file_descriptor_set = FileDescriptorSet::decode(&*buf).map_err(|error| { Error::new( @@ -918,6 +862,89 @@ impl Config { Ok(()) } + fn load_file_descriptor_set( + &self, + protos: &[impl AsRef], + includes: &[impl AsRef], + ) -> Result> { + if self.skip_protoc_run { + if let Some(path) = &self.file_descriptor_set_path { + return Ok(fs::read(path)?); + } else { + return Err(Error::new( + ErrorKind::Other, + "file_descriptor_set_path is required with skip_protoc_run", + )); + } + } + + if protoc() == Path::new("linked") { + let file_descriptor_set = Self::serialize_file_descriptor_set(protos, includes)?; + + // If the user has set the file descriptor path manually then also save it + // to disk in case they are counting on that behavior + if let Some(path) = &self.file_descriptor_set_path { + std::fs::write(path, &file_descriptor_set)?; + } + + Ok(file_descriptor_set) + } else { + let tmp; + let file_descriptor_set_path = if let Some(path) = &self.file_descriptor_set_path { + path.clone() + } else { + if self.skip_protoc_run { + return Err(Error::new( + ErrorKind::Other, + "file_descriptor_set_path is required with skip_protoc_run", + )); + } + tmp = tempfile::Builder::new().prefix("prost-build").tempdir()?; + tmp.path().join("prost-descriptor-set") + }; + + if !self.skip_protoc_run { + let mut cmd = Command::new(protoc()); + cmd.arg("--include_imports") + .arg("--include_source_info") + .arg("-o") + .arg(&file_descriptor_set_path); + + for include in includes { + cmd.arg("-I").arg(include.as_ref()); + } + + // Set the protoc include after the user includes in case the user wants to + // override one of the built-in .protos. + cmd.arg("-I").arg(protoc_include()); + + for arg in &self.protoc_args { + cmd.arg(arg); + } + + for proto in protos { + cmd.arg(proto.as_ref()); + } + + let output = cmd.output().map_err(|error| { + Error::new( + error.kind(), + format!("failed to invoke protoc (hint: https://docs.rs/prost-build/#sourcing-protoc): {}", error), + ) + })?; + + if !output.status.success() { + return Err(Error::new( + ErrorKind::Other, + format!("protoc failed: {}", String::from_utf8_lossy(&output.stderr)), + )); + } + } + + fs::read(file_descriptor_set_path) + } + } + fn serialize_file_descriptor_set( protos: &[impl AsRef], includes: &[impl AsRef], @@ -966,32 +993,28 @@ impl Config { paths: Vec, } - fn path_bufs<'s>(paths: &'s [impl AsRef]) -> PathBufs<'s> { - let storage: Vec>; + #[inline] + fn path_to_vec<'s>(path: &'s impl AsRef) -> std::borrow::Cow<'s, [u8]> { #[cfg(unix)] { use std::os::unix::ffi::OsStrExt; - storage = paths - .iter() - .map(|p| p.as_ref().as_os_str().as_bytes().into()) - .collect(); + path.as_ref().as_os_str().as_bytes().into() } #[cfg(windows)] { - use std::os::windows::ffi::OsStrExt; - storage = paths - .iter() - .map(|p| { - p.as_ref() - .as_os_str() - .encode_wide() - .flat_map(|c| [((c & 0xff00) >> 8) as u8, c & 0xff]) - .collect::>() - .into() - }) - .collect(); + path.as_ref() + .as_os_str() + .encode_wide() + .flat_map(|c| [((c & 0xff00) >> 8) as u8, c & 0xff]) + .collect::>() + .into() } + } + + #[inline] + fn path_bufs<'s>(paths: &'s [impl AsRef]) -> PathBufs<'s> { + let storage: Vec<_> = paths.iter().map(path_to_vec).collect(); let paths = storage .iter() @@ -1005,7 +1028,19 @@ impl Config { } let protos = path_bufs(protos); - let includes = path_bufs(includes); + let mut includes = path_bufs(includes); + + // Explicitly add the default protoc includes, these are normally added + // by the CLI, which we don't use + let protoc_include = protoc_include(); + { + let pi = path_to_vec(&protoc_include); + includes.paths.push(CPath { + path: pi.as_ref().as_ptr().cast(), + len: pi.len(), + }); + includes.storage.push(pi); + } let mut output_vec = Vec::new(); @@ -1027,7 +1062,10 @@ impl Config { { Ok(output_vec) } else { - panic!("oh noes"); + Err(Error::new( + ErrorKind::Other, + "failed to serialize file descriptor set, see stderr", + )) } } diff --git a/prost-build/src/libprotoc.cpp b/prost-build/src/libprotoc.cpp index 3dc4fa6d5..0cd14a48e 100644 --- a/prost-build/src/libprotoc.cpp +++ b/prost-build/src/libprotoc.cpp @@ -1,4 +1,6 @@ #include + +#ifndef STUB_ONLY #include #include @@ -11,8 +13,7 @@ #include #include - - +#endif // STUB_ONLY extern "C" { // Small and simple and FFI safe, unlike `StringPiece` @@ -31,8 +32,24 @@ extern "C" { capacity buffer_capacity; void* context; }; + + #ifdef STUB_ONLY + // Expose a function that always fails in case the user has configured + // the `vendored` feature, but has also set the `PROTOC_NO_VENDOR` + // env var, meaning we still need to link the lib, but it won't actually be used + int write_descriptor_set( + const Path* input_files, + size_t num_inputs, + const Path* includes_paths, + size_t num_includes, + Buffer* output + ) { + return 1; + } + #endif // STUB_ONLY } +#ifndef STUB_ONLY using google::protobuf::FileDescriptor; using google::protobuf::FileDescriptorSet; using google::protobuf::DescriptorPool; @@ -316,7 +333,7 @@ extern "C" { // own -I args with the ':/;' separated virtual path. I seriously doubt // this happens in practice. (famous last words) for (const auto& include : includes) { - source_tree->MapPath(include, ""); + source_tree->MapPath("", include); } // Map input files to virtual paths if possible. I'm not sure if this @@ -347,4 +364,5 @@ extern "C" { return 0; } -} \ No newline at end of file +} +#endif // STUB_ONLY From 350472cefa52adc9a49b5708edcc14f60f7670c3 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 08:32:49 +0200 Subject: [PATCH 13/22] Fix windows --- prost-build/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/prost-build/src/lib.rs b/prost-build/src/lib.rs index ff6d744b9..6148a456d 100644 --- a/prost-build/src/lib.rs +++ b/prost-build/src/lib.rs @@ -1003,6 +1003,7 @@ impl Config { #[cfg(windows)] { + use std::os::windows::ffi::OsStrExt; path.as_ref() .as_os_str() .encode_wide() From 4ae711d8f0fbe78bc85b24def9c9270eba8deaa2 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 08:40:26 +0200 Subject: [PATCH 14/22] Fix windows again --- prost-build/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prost-build/src/lib.rs b/prost-build/src/lib.rs index 6148a456d..dcffdff50 100644 --- a/prost-build/src/lib.rs +++ b/prost-build/src/lib.rs @@ -1007,7 +1007,7 @@ impl Config { path.as_ref() .as_os_str() .encode_wide() - .flat_map(|c| [((c & 0xff00) >> 8) as u8, c & 0xff]) + .flat_map(|c| [((c & 0xff00) >> 8) as u8, (c & 0xff) as u8]) .collect::>() .into() } From bde457ed0c5303fa9fd0893395f50d240022cb8c Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 08:53:57 +0200 Subject: [PATCH 15/22] Fix windows...again --- prost-build/build.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/prost-build/build.rs b/prost-build/build.rs index 31b134f61..c50b26687 100644 --- a/prost-build/build.rs +++ b/prost-build/build.rs @@ -142,6 +142,10 @@ fn compile(stub: bool) -> Option { .iter() .map(|fname| protobuf_src.join(fname)), ); + + if env::var("CARGO_CFG_TARGET_FAMILY").as_deref() == Ok("windows") { + build.file(protobuf_src.join("io/io_win32.cc")); + } } if stub { From 63e1e2b1121a0dfc8262709e3e463faab0cabae8 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 09:05:23 +0200 Subject: [PATCH 16/22] Fix windows...again --- prost-build/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prost-build/src/lib.rs b/prost-build/src/lib.rs index dcffdff50..1b361af66 100644 --- a/prost-build/src/lib.rs +++ b/prost-build/src/lib.rs @@ -1007,7 +1007,7 @@ impl Config { path.as_ref() .as_os_str() .encode_wide() - .flat_map(|c| [((c & 0xff00) >> 8) as u8, (c & 0xff) as u8]) + .flat_map(|c| [((c & 0xff00) >> 8) as u8, (c & 0xff) as u8].into_iter()) .collect::>() .into() } From 3a5f0375df85062f5ef718fc6b210d1d29fdfcdc Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 09:19:36 +0200 Subject: [PATCH 17/22] Don't use iterators due to 1.51 being lame --- prost-build/src/lib.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/prost-build/src/lib.rs b/prost-build/src/lib.rs index 1b361af66..c74ac99a9 100644 --- a/prost-build/src/lib.rs +++ b/prost-build/src/lib.rs @@ -1004,12 +1004,15 @@ impl Config { #[cfg(windows)] { use std::os::windows::ffi::OsStrExt; - path.as_ref() - .as_os_str() - .encode_wide() - .flat_map(|c| [((c & 0xff00) >> 8) as u8, (c & 0xff) as u8].into_iter()) - .collect::>() - .into() + let os_str = path.as_ref().as_os_str(); + let mut pv = Vec::with_capacity(os_str.len() * 2); + + for c in os_str.encode_wide() { + pv.push(((c & 0xff00) >> 8) as u8); + pv.push((c & 0xff) as u8); + } + + pv.into() } } From b25cbd5bd2eb5f90c6ee5bea251ddbbe14ec67d7 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 09:36:05 +0200 Subject: [PATCH 18/22] Ugh --- prost-build/build.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/prost-build/build.rs b/prost-build/build.rs index c50b26687..0ebb23dbf 100644 --- a/prost-build/build.rs +++ b/prost-build/build.rs @@ -144,7 +144,11 @@ fn compile(stub: bool) -> Option { ); if env::var("CARGO_CFG_TARGET_FAMILY").as_deref() == Ok("windows") { - build.file(protobuf_src.join("io/io_win32.cc")); + build.files( + ["io/io_win32.cc", "stubs/status.cc"] + .iter() + .map(|fname| protobuf_src.join(fname)), + ); } } From b3f4558da8e78c942dc5d89c4d9f292ecc40fe5c Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 09:56:33 +0200 Subject: [PATCH 19/22] C++ gonna C++ --- prost-build/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prost-build/build.rs b/prost-build/build.rs index 0ebb23dbf..c93020b6a 100644 --- a/prost-build/build.rs +++ b/prost-build/build.rs @@ -145,7 +145,7 @@ fn compile(stub: bool) -> Option { if env::var("CARGO_CFG_TARGET_FAMILY").as_deref() == Ok("windows") { build.files( - ["io/io_win32.cc", "stubs/status.cc"] + ["io/io_win32.cc", "stubs/int128.cc", "stubs/status.cc"] .iter() .map(|fname| protobuf_src.join(fname)), ); From 7e1263a4599450b390660323b397da363c38b00b Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Fri, 6 May 2022 11:01:50 +0200 Subject: [PATCH 20/22] Workaround Windows being lame until someone complains --- prost-build/src/lib.rs | 23 ++++++++++++++--------- prost-build/src/libprotoc.cpp | 2 -- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/prost-build/src/lib.rs b/prost-build/src/lib.rs index c74ac99a9..dcd97d3b7 100644 --- a/prost-build/src/lib.rs +++ b/prost-build/src/lib.rs @@ -1003,16 +1003,21 @@ impl Config { #[cfg(windows)] { - use std::os::windows::ffi::OsStrExt; - let os_str = path.as_ref().as_os_str(); - let mut pv = Vec::with_capacity(os_str.len() * 2); - - for c in os_str.encode_wide() { - pv.push(((c & 0xff00) >> 8) as u8); - pv.push((c & 0xff) as u8); + // https://internals.rust-lang.org/t/pathbuf-to-cstring/12560/19 + // protobuf is internally (incorrectly) using std::string to store + // paths, and I don't know if it's expected to work with actual + // non-ascii/utf-8 paths, so for now just cheat until someone + // complains about it not working + match path.as_ref().to_string_lossy() { + std::borrow::Cow::Owned(s) => { + log::warn!( + "non utf-8 path '{}' detected, this may fail in protoc", + path.as_ref().display() + ); + s.into_bytes().into() + } + std::borrow::Cow::Borrowed(s) => s.as_bytes().into(), } - - pv.into() } } diff --git a/prost-build/src/libprotoc.cpp b/prost-build/src/libprotoc.cpp index 0cd14a48e..1ac9f467b 100644 --- a/prost-build/src/libprotoc.cpp +++ b/prost-build/src/libprotoc.cpp @@ -336,8 +336,6 @@ extern "C" { source_tree->MapPath("", include); } - // Map input files to virtual paths if possible. I'm not sure if this - // is even needed since as stated prost really doesn't do virtual paths if (!make_inputs_relative(&inputs, source_tree.get())) { return 1; } From c864934c1e7705ae3c1991d6764baf6c9ce492d1 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Wed, 11 May 2022 16:10:38 +0200 Subject: [PATCH 21/22] Use HTTPS for protobuf submodule --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 6251abeb8..ebee18b86 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "prost-build/third-party/protobuf"] path = prost-build/third-party/protobuf - url = git@github.com:protocolbuffers/protobuf + url = https://github.com/protocolbuffers/protobuf From 52819721ac9c88f9a997bee5cea9e24d2f259aac Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Wed, 11 May 2022 18:27:55 +0200 Subject: [PATCH 22/22] Fix prost version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e8d41fc2d..1ede18270 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "prost" -version = "0.10.1" +version = "0.10.3" authors = [ "Dan Burkert ", "Tokio Contributors ",