Skip to content

Internal change #25670

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
+187 −4
Draft

Internal change #25670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 167 additions & 2 deletions src/google/protobuf/io/gzip_stream.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
// This file contains the implementation of classes GzipInputStream and
// GzipOutputStream.

#include <zlib.h>
#define HAVE_ZLIB 1

#if HAVE_ZLIB
#include "google/protobuf/io/gzip_stream.h"
Expand Down Expand Up @@ -37,6 +39,18 @@ GzipInputStream::GzipInputStream(ZeroCopyInputStream* sub_stream, Format format,
zcontext_.avail_in = 0;
zcontext_.total_in = 0;
zcontext_.msg = nullptr;
#else
zcontext_ = new z_stream;
zcontext_->state = Z_NULL;
zcontext_->zalloc = Z_NULL;
zcontext_->zfree = Z_NULL;
zcontext_->opaque = Z_NULL;
zcontext_->total_out = 0;
zcontext_->next_in = nullptr;
zcontext_->avail_in = 0;
zcontext_->total_in = 0;
zcontext_->msg = nullptr;

if (buffer_size == -1) {
output_buffer_length_ = kDefaultBufferSize;
} else {
Expand All @@ -46,11 +60,17 @@ GzipInputStream::GzipInputStream(ZeroCopyInputStream* sub_stream, Format format,
ABSL_CHECK(output_buffer_ != nullptr);
zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
zcontext_.avail_out = output_buffer_length_;
#else
zcontext_->next_out = static_cast<Bytef*>(output_buffer_);
zcontext_->avail_out = output_buffer_length_;
output_position_ = output_buffer_;
}
GzipInputStream::~GzipInputStream() {
internal::SizedDelete(output_buffer_, output_buffer_length_);
zerror_ = inflateEnd(&zcontext_);
#else
zerror_ = inflateEnd(zcontext_);
delete zcontext_;
}

static inline int internalInflateInit2(z_stream* zcontext,
Expand Down Expand Up @@ -97,12 +117,42 @@ int GzipInputStream::Inflate(int flush) {
output_position_ = output_buffer_;
int error = inflate(&zcontext_, flush);
return error;
#else
if ((zerror_ == Z_OK) && (zcontext_->avail_out == 0)) {
// previous inflate filled output buffer. don't change input params yet.
} else if (zcontext_->avail_in == 0) {
const void* in;
int in_size;
bool first = zcontext_->next_in == nullptr;
bool ok = sub_stream_->Next(&in, &in_size);
if (!ok) {
zcontext_->next_out = nullptr;
zcontext_->avail_out = 0;
return Z_STREAM_END;
}
zcontext_->next_in = static_cast<Bytef*>(const_cast<void*>(in));
zcontext_->avail_in = in_size;
if (first) {
int error = internalInflateInit2(zcontext_, format_);
if (error != Z_OK) {
return error;
}
}
}
zcontext_->next_out = static_cast<Bytef*>(output_buffer_);
zcontext_->avail_out = output_buffer_length_;
output_position_ = output_buffer_;
int error = inflate(zcontext_, flush);
return error;
}

void GzipInputStream::DoNextOutput(const void** data, int* size) {
*data = output_position_;
*size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_);
output_position_ = zcontext_.next_out;
#else
*size = ((uintptr_t)zcontext_->next_out) - ((uintptr_t)output_position_);
output_position_ = zcontext_->next_out;
}

// implements ZeroCopyInputStream ----------------------------------
Expand Down Expand Up @@ -139,6 +189,39 @@ bool GzipInputStream::Next(const void** data, int* size) {
// The underlying stream's Next returned false inside Inflate.
return false;
}
#else
bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
(zerror_ == Z_BUF_ERROR);
if ((!ok) || (zcontext_->next_out == nullptr)) {
return false;
}
if (zcontext_->next_out != output_position_) {
DoNextOutput(data, size);
return true;
}
if (zerror_ == Z_STREAM_END) {
if (zcontext_->next_out != nullptr) {
// sub_stream_ may have concatenated streams to follow
zerror_ = inflateEnd(zcontext_);
byte_count_ += zcontext_->total_out;
if (zerror_ != Z_OK) {
return false;
}
zerror_ = internalInflateInit2(zcontext_, format_);
if (zerror_ != Z_OK) {
return false;
}
} else {
*data = nullptr;
*size = 0;
return false;
}
}
zerror_ = Inflate(Z_NO_FLUSH);
if ((zerror_ == Z_STREAM_END) && (zcontext_->next_out == nullptr)) {
// The underlying stream's Next returned false inside Inflate.
return false;
}
ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
(zerror_ == Z_BUF_ERROR);
if (!ok) {
Expand Down Expand Up @@ -171,8 +254,16 @@ int64_t GzipInputStream::ByteCount() const {
reinterpret_cast<uintptr_t>(output_position_);
}
return ret;
#else
int64_t ret = byte_count_ + zcontext_->total_out;
if (zcontext_->next_out != nullptr && output_position_ != nullptr) {
ret += reinterpret_cast<uintptr_t>(zcontext_->next_out) -
reinterpret_cast<uintptr_t>(output_position_);
}
return ret;
}


// =========================================================================

GzipOutputStream::Options::Options()
Expand Down Expand Up @@ -210,6 +301,18 @@ void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream,
zcontext_.avail_in = 0;
zcontext_.total_in = 0;
zcontext_.msg = nullptr;
#else
zcontext_ = new z_stream;
zcontext_->zalloc = Z_NULL;
zcontext_->zfree = Z_NULL;
zcontext_->opaque = Z_NULL;
zcontext_->next_out = nullptr;
zcontext_->avail_out = 0;
zcontext_->total_out = 0;
zcontext_->next_in = nullptr;
zcontext_->avail_in = 0;
zcontext_->total_in = 0;
zcontext_->msg = nullptr;
// default to GZIP format
int windowBitsFormat = 16;
if (options.format == ZLIB) {
Expand All @@ -219,6 +322,11 @@ void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream,
deflateInit2(&zcontext_, options.compression_level, Z_DEFLATED,
/* windowBits */ 15 | windowBitsFormat,
/* memLevel (default) */ 8, options.compression_strategy);
#else
zerror_ =
deflateInit2(zcontext_, options.compression_level, Z_DEFLATED,
/* windowBits */ 15 | windowBitsFormat,
/* memLevel (default) */ 8, options.compression_strategy);
}

GzipOutputStream::~GzipOutputStream() {
Expand Down Expand Up @@ -252,6 +360,30 @@ int GzipOutputStream::Deflate(int flush) {
sub_data_size_ = 0;
}
return error;
#else
int error = Z_OK;
do {
if ((sub_data_ == nullptr) || (zcontext_->avail_out == 0)) {
bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_);
if (!ok) {
sub_data_ = nullptr;
sub_data_size_ = 0;
return Z_BUF_ERROR;
}
ABSL_CHECK_GT(sub_data_size_, 0);
zcontext_->next_out = static_cast<Bytef*>(sub_data_);
zcontext_->avail_out = sub_data_size_;
}
error = deflate(zcontext_, flush);
} while (error == Z_OK && zcontext_->avail_out == 0);
if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) {
// Notify lower layer of data.
sub_stream_->BackUp(zcontext_->avail_out);
// We don't own the buffer anymore.
sub_data_ = nullptr;
sub_data_size_ = 0;
}
return error;
}

// implements ZeroCopyOutputStream ---------------------------------
Expand All @@ -276,21 +408,52 @@ bool GzipOutputStream::Next(void** data, int* size) {
ABSL_DLOG(FATAL) << "Deflate left bytes unconsumed";
}
return true;
#else
if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
return false;
}
if (zcontext_->avail_in != 0) {
zerror_ = Deflate(Z_NO_FLUSH);
if (zerror_ != Z_OK) {
return false;
}
}
if (zcontext_->avail_in == 0) {
// all input was consumed. reset the buffer.
zcontext_->next_in = static_cast<Bytef*>(input_buffer_);
zcontext_->avail_in = input_buffer_length_;
*data = input_buffer_;
*size = input_buffer_length_;
} else {
// The loop in Deflate should consume all avail_in
ABSL_DLOG(FATAL) << "Deflate left bytes unconsumed";
}
return true;
}
void GzipOutputStream::BackUp(int count) {
ABSL_CHECK_GE(zcontext_.avail_in, static_cast<uInt>(count));
zcontext_.avail_in -= count;
#else
ABSL_CHECK_GE(zcontext_->avail_in, static_cast<uInt>(count));
zcontext_->avail_in -= count;
}
int64_t GzipOutputStream::ByteCount() const {
return zcontext_.total_in + zcontext_.avail_in;
#else
return zcontext_->total_in + zcontext_->avail_in;
}


bool GzipOutputStream::Flush() {
zerror_ = Deflate(Z_FULL_FLUSH);
// Return true if the flush succeeded or if it was a no-op.
return (zerror_ == Z_OK ||
(zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 &&
zcontext_.avail_out != 0);
#else
return (zerror_ == Z_OK) ||
(zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 &&
zcontext_.avail_out != 0);
(zerror_ == Z_BUF_ERROR && zcontext_->avail_in == 0 &&
zcontext_->avail_out != 0);
}

bool GzipOutputStream::Close() {
Expand All @@ -301,6 +464,8 @@ bool GzipOutputStream::Close() {
zerror_ = Deflate(Z_FINISH);
} while (zerror_ == Z_OK);
zerror_ = deflateEnd(&zcontext_);
#else
zerror_ = deflateEnd(zcontext_);
bool ok = zerror_ == Z_OK;
zerror_ = Z_STREAM_END;
return ok;
Expand Down
22 changes: 20 additions & 2 deletions src/google/protobuf/io/gzip_stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,18 @@
#include "google/protobuf/io/zero_copy_stream.h"
#include "google/protobuf/port.h"
#include <zlib.h>
// Externally we always want to include <zlib.h>, but internally we only want to
// do that for Android builds, where we get zlib for free as a shared library.
// Otherwise we have to get zlib from //third_party/zlib. For this
// reason during the migration the rust version of zlib, some classes are
// defined in the .cc file. others checks on PROTO2_OPENSOURCE were set in this
// file to implement this forward declarations only for the internal version.
// TODO - b/477521589: Remove this comment after the migration is complete.

// Must be included last.
#include "google/protobuf/port_def.inc"


namespace google {
namespace protobuf {
namespace io {
Expand Down Expand Up @@ -60,6 +68,9 @@ class PROTOBUF_FUTURE_ADD_EARLY_WARN_UNUSED PROTOBUF_EXPORT
const {
return zcontext_.msg;
}
#else
PROTOBUF_FUTURE_ADD_EARLY_NODISCARD const char* ZlibErrorMessage() const;

PROTOBUF_FUTURE_ADD_EARLY_NODISCARD inline int ZlibErrorCode() const {
return zerror_;
}
Expand All @@ -76,7 +87,9 @@ class PROTOBUF_FUTURE_ADD_EARLY_WARN_UNUSED PROTOBUF_EXPORT

ZeroCopyInputStream* sub_stream_;

z_stream zcontext_;
z_stream_s zcontext_;
#else
struct z_stream_s* zcontext_;
int zerror_;

void* output_buffer_;
Expand Down Expand Up @@ -134,6 +147,9 @@ class PROTOBUF_FUTURE_ADD_EARLY_WARN_UNUSED PROTOBUF_EXPORT
const {
return zcontext_.msg;
}
#else
PROTOBUF_FUTURE_ADD_EARLY_NODISCARD const char* ZlibErrorMessage() const;

PROTOBUF_FUTURE_ADD_EARLY_NODISCARD inline int ZlibErrorCode() const {
return zerror_;
}
Expand Down Expand Up @@ -170,7 +186,9 @@ class PROTOBUF_FUTURE_ADD_EARLY_WARN_UNUSED PROTOBUF_EXPORT
void* sub_data_;
int sub_data_size_;

z_stream zcontext_;
z_stream_s zcontext_;
#else
struct z_stream_s* zcontext_;
int zerror_;
void* input_buffer_;
size_t input_buffer_length_;
Expand Down
Loading