Media CAS Proxy SDK release: 16.5.0

ubuntu/google/protobuf/io/coded_stream.cc

@@ -0,0 +1,956 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// This implementation is heavily optimized to make reads and writes
+// of small values (especially varints) as fast as possible.  In
+// particular, we optimize for the common case that a read or a write
+// will not cross the end of the buffer, since we can avoid a lot
+// of branching in this case.
+
+#include <google/protobuf/io/coded_stream.h>
+
+#include <limits.h>
+
+#include <algorithm>
+#include <cstring>
+#include <utility>
+
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/arena.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <google/protobuf/stubs/stl_util.h>
+
+
+#include <google/protobuf/port_def.inc>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+namespace {
+
+static const int kMaxVarintBytes = 10;
+static const int kMaxVarint32Bytes = 5;
+
+
+inline bool NextNonEmpty(ZeroCopyInputStream* input, const void** data,
+                         int* size) {
+  bool success;
+  do {
+    success = input->Next(data, size);
+  } while (success && *size == 0);
+  return success;
+}
+
+}  // namespace
+
+// CodedInputStream ==================================================
+
+CodedInputStream::~CodedInputStream() {
+  if (input_ != NULL) {
+    BackUpInputToCurrentPosition();
+  }
+}
+
+// Static.
+int CodedInputStream::default_recursion_limit_ = 100;
+
+
+void CodedInputStream::BackUpInputToCurrentPosition() {
+  int backup_bytes = BufferSize() + buffer_size_after_limit_ + overflow_bytes_;
+  if (backup_bytes > 0) {
+    input_->BackUp(backup_bytes);
+
+    // total_bytes_read_ doesn't include overflow_bytes_.
+    total_bytes_read_ -= BufferSize() + buffer_size_after_limit_;
+    buffer_end_ = buffer_;
+    buffer_size_after_limit_ = 0;
+    overflow_bytes_ = 0;
+  }
+}
+
+inline void CodedInputStream::RecomputeBufferLimits() {
+  buffer_end_ += buffer_size_after_limit_;
+  int closest_limit = std::min(current_limit_, total_bytes_limit_);
+  if (closest_limit < total_bytes_read_) {
+    // The limit position is in the current buffer.  We must adjust
+    // the buffer size accordingly.
+    buffer_size_after_limit_ = total_bytes_read_ - closest_limit;
+    buffer_end_ -= buffer_size_after_limit_;
+  } else {
+    buffer_size_after_limit_ = 0;
+  }
+}
+
+CodedInputStream::Limit CodedInputStream::PushLimit(int byte_limit) {
+  // Current position relative to the beginning of the stream.
+  int current_position = CurrentPosition();
+
+  Limit old_limit = current_limit_;
+
+  // security: byte_limit is possibly evil, so check for negative values
+  // and overflow. Also check that the new requested limit is before the
+  // previous limit; otherwise we continue to enforce the previous limit.
+  if (PROTOBUF_PREDICT_TRUE(byte_limit >= 0 &&
+                            byte_limit <= INT_MAX - current_position &&
+                            byte_limit < current_limit_ - current_position)) {
+    current_limit_ = current_position + byte_limit;
+    RecomputeBufferLimits();
+  }
+
+  return old_limit;
+}
+
+void CodedInputStream::PopLimit(Limit limit) {
+  // The limit passed in is actually the *old* limit, which we returned from
+  // PushLimit().
+  current_limit_ = limit;
+  RecomputeBufferLimits();
+
+  // We may no longer be at a legitimate message end.  ReadTag() needs to be
+  // called again to find out.
+  legitimate_message_end_ = false;
+}
+
+std::pair<CodedInputStream::Limit, int>
+CodedInputStream::IncrementRecursionDepthAndPushLimit(int byte_limit) {
+  return std::make_pair(PushLimit(byte_limit), --recursion_budget_);
+}
+
+CodedInputStream::Limit CodedInputStream::ReadLengthAndPushLimit() {
+  uint32 length;
+  return PushLimit(ReadVarint32(&length) ? length : 0);
+}
+
+bool CodedInputStream::DecrementRecursionDepthAndPopLimit(Limit limit) {
+  bool result = ConsumedEntireMessage();
+  PopLimit(limit);
+  GOOGLE_DCHECK_LT(recursion_budget_, recursion_limit_);
+  ++recursion_budget_;
+  return result;
+}
+
+bool CodedInputStream::CheckEntireMessageConsumedAndPopLimit(Limit limit) {
+  bool result = ConsumedEntireMessage();
+  PopLimit(limit);
+  return result;
+}
+
+int CodedInputStream::BytesUntilLimit() const {
+  if (current_limit_ == INT_MAX) return -1;
+  int current_position = CurrentPosition();
+
+  return current_limit_ - current_position;
+}
+
+void CodedInputStream::SetTotalBytesLimit(int total_bytes_limit) {
+  // Make sure the limit isn't already past, since this could confuse other
+  // code.
+  int current_position = CurrentPosition();
+  total_bytes_limit_ = std::max(current_position, total_bytes_limit);
+  RecomputeBufferLimits();
+}
+
+int CodedInputStream::BytesUntilTotalBytesLimit() const {
+  if (total_bytes_limit_ == INT_MAX) return -1;
+  return total_bytes_limit_ - CurrentPosition();
+}
+
+void CodedInputStream::PrintTotalBytesLimitError() {
+  GOOGLE_LOG(ERROR)
+      << "A protocol message was rejected because it was too "
+         "big (more than "
+      << total_bytes_limit_
+      << " bytes).  To increase the limit (or to disable these "
+         "warnings), see CodedInputStream::SetTotalBytesLimit() "
+         "in third_party/protobuf/src/google/protobuf/io/coded_stream.h.";
+}
+
+bool CodedInputStream::SkipFallback(int count, int original_buffer_size) {
+  if (buffer_size_after_limit_ > 0) {
+    // We hit a limit inside this buffer.  Advance to the limit and fail.
+    Advance(original_buffer_size);
+    return false;
+  }
+
+  count -= original_buffer_size;
+  buffer_ = NULL;
+  buffer_end_ = buffer_;
+
+  // Make sure this skip doesn't try to skip past the current limit.
+  int closest_limit = std::min(current_limit_, total_bytes_limit_);
+  int bytes_until_limit = closest_limit - total_bytes_read_;
+  if (bytes_until_limit < count) {
+    // We hit the limit.  Skip up to it then fail.
+    if (bytes_until_limit > 0) {
+      total_bytes_read_ = closest_limit;
+      input_->Skip(bytes_until_limit);
+    }
+    return false;
+  }
+
+  if (!input_->Skip(count)) {
+    total_bytes_read_ = input_->ByteCount();
+    return false;
+  }
+  total_bytes_read_ += count;
+  return true;
+}
+
+bool CodedInputStream::GetDirectBufferPointer(const void** data, int* size) {
+  if (BufferSize() == 0 && !Refresh()) return false;
+
+  *data = buffer_;
+  *size = BufferSize();
+  return true;
+}
+
+bool CodedInputStream::ReadRaw(void* buffer, int size) {
+  int current_buffer_size;
+  while ((current_buffer_size = BufferSize()) < size) {
+    // Reading past end of buffer.  Copy what we have, then refresh.
+    memcpy(buffer, buffer_, current_buffer_size);
+    buffer = reinterpret_cast<uint8*>(buffer) + current_buffer_size;
+    size -= current_buffer_size;
+    Advance(current_buffer_size);
+    if (!Refresh()) return false;
+  }
+
+  memcpy(buffer, buffer_, size);
+  Advance(size);
+
+  return true;
+}
+
+bool CodedInputStream::ReadString(std::string* buffer, int size) {
+  if (size < 0) return false;  // security: size is often user-supplied
+
+  if (BufferSize() >= size) {
+    STLStringResizeUninitialized(buffer, size);
+    std::pair<char*, bool> z = as_string_data(buffer);
+    if (z.second) {
+      // Oddly enough, memcpy() requires its first two args to be non-NULL even
+      // if we copy 0 bytes.  So, we have ensured that z.first is non-NULL here.
+      GOOGLE_DCHECK(z.first != NULL);
+      memcpy(z.first, buffer_, size);
+      Advance(size);
+    }
+    return true;
+  }
+
+  return ReadStringFallback(buffer, size);
+}
+
+bool CodedInputStream::ReadStringFallback(std::string* buffer, int size) {
+  if (!buffer->empty()) {
+    buffer->clear();
+  }
+
+  int closest_limit = std::min(current_limit_, total_bytes_limit_);
+  if (closest_limit != INT_MAX) {
+    int bytes_to_limit = closest_limit - CurrentPosition();
+    if (bytes_to_limit > 0 && size > 0 && size <= bytes_to_limit) {
+      buffer->reserve(size);
+    }
+  }
+
+  int current_buffer_size;
+  while ((current_buffer_size = BufferSize()) < size) {
+    // Some STL implementations "helpfully" crash on buffer->append(NULL, 0).
+    if (current_buffer_size != 0) {
+      // Note:  string1.append(string2) is O(string2.size()) (as opposed to
+      //   O(string1.size() + string2.size()), which would be bad).
+      buffer->append(reinterpret_cast<const char*>(buffer_),
+                     current_buffer_size);
+    }
+    size -= current_buffer_size;
+    Advance(current_buffer_size);
+    if (!Refresh()) return false;
+  }
+
+  buffer->append(reinterpret_cast<const char*>(buffer_), size);
+  Advance(size);
+
+  return true;
+}
+
+
+bool CodedInputStream::ReadLittleEndian32Fallback(uint32* value) {
+  uint8 bytes[sizeof(*value)];
+
+  const uint8* ptr;
+  if (BufferSize() >= sizeof(*value)) {
+    // Fast path:  Enough bytes in the buffer to read directly.
+    ptr = buffer_;
+    Advance(sizeof(*value));
+  } else {
+    // Slow path:  Had to read past the end of the buffer.
+    if (!ReadRaw(bytes, sizeof(*value))) return false;
+    ptr = bytes;
+  }
+  ReadLittleEndian32FromArray(ptr, value);
+  return true;
+}
+
+bool CodedInputStream::ReadLittleEndian64Fallback(uint64* value) {
+  uint8 bytes[sizeof(*value)];
+
+  const uint8* ptr;
+  if (BufferSize() >= sizeof(*value)) {
+    // Fast path:  Enough bytes in the buffer to read directly.
+    ptr = buffer_;
+    Advance(sizeof(*value));
+  } else {
+    // Slow path:  Had to read past the end of the buffer.
+    if (!ReadRaw(bytes, sizeof(*value))) return false;
+    ptr = bytes;
+  }
+  ReadLittleEndian64FromArray(ptr, value);
+  return true;
+}
+
+namespace {
+
+// Decodes varint64 with known size, N, and returns next pointer. Knowing N at
+// compile time, compiler can generate optimal code. For example, instead of
+// subtracting 0x80 at each iteration, it subtracts properly shifted mask once.
+template <size_t N>
+const uint8* DecodeVarint64KnownSize(const uint8* buffer, uint64* value) {
+  GOOGLE_DCHECK_GT(N, 0);
+  uint64 result = static_cast<uint64>(buffer[N - 1]) << (7 * (N - 1));
+  for (int i = 0, offset = 0; i < N - 1; i++, offset += 7) {
+    result += static_cast<uint64>(buffer[i] - 0x80) << offset;
+  }
+  *value = result;
+  return buffer + N;
+}
+
+// Read a varint from the given buffer, write it to *value, and return a pair.
+// The first part of the pair is true iff the read was successful.  The second
+// part is buffer + (number of bytes read).  This function is always inlined,
+// so returning a pair is costless.
+PROTOBUF_ALWAYS_INLINE
+::std::pair<bool, const uint8*> ReadVarint32FromArray(uint32 first_byte,
+                                                      const uint8* buffer,
+                                                      uint32* value);
+inline ::std::pair<bool, const uint8*> ReadVarint32FromArray(
+    uint32 first_byte, const uint8* buffer, uint32* value) {
+  // Fast path:  We have enough bytes left in the buffer to guarantee that
+  // this read won't cross the end, so we can skip the checks.
+  GOOGLE_DCHECK_EQ(*buffer, first_byte);
+  GOOGLE_DCHECK_EQ(first_byte & 0x80, 0x80) << first_byte;
+  const uint8* ptr = buffer;
+  uint32 b;
+  uint32 result = first_byte - 0x80;
+  ++ptr;  // We just processed the first byte.  Move on to the second.
+  b = *(ptr++);
+  result += b << 7;
+  if (!(b & 0x80)) goto done;
+  result -= 0x80 << 7;
+  b = *(ptr++);
+  result += b << 14;
+  if (!(b & 0x80)) goto done;
+  result -= 0x80 << 14;
+  b = *(ptr++);
+  result += b << 21;
+  if (!(b & 0x80)) goto done;
+  result -= 0x80 << 21;
+  b = *(ptr++);
+  result += b << 28;
+  if (!(b & 0x80)) goto done;
+  // "result -= 0x80 << 28" is irrevelant.
+
+  // If the input is larger than 32 bits, we still need to read it all
+  // and discard the high-order bits.
+  for (int i = 0; i < kMaxVarintBytes - kMaxVarint32Bytes; i++) {
+    b = *(ptr++);
+    if (!(b & 0x80)) goto done;
+  }
+
+  // We have overrun the maximum size of a varint (10 bytes).  Assume
+  // the data is corrupt.
+  return std::make_pair(false, ptr);
+
+done:
+  *value = result;
+  return std::make_pair(true, ptr);
+}
+
+PROTOBUF_ALWAYS_INLINE::std::pair<bool, const uint8*> ReadVarint64FromArray(
+    const uint8* buffer, uint64* value);
+inline ::std::pair<bool, const uint8*> ReadVarint64FromArray(
+    const uint8* buffer, uint64* value) {
+  // Assumes varint64 is at least 2 bytes.
+  GOOGLE_DCHECK_GE(buffer[0], 128);
+
+  const uint8* next;
+  if (buffer[1] < 128) {
+    next = DecodeVarint64KnownSize<2>(buffer, value);
+  } else if (buffer[2] < 128) {
+    next = DecodeVarint64KnownSize<3>(buffer, value);
+  } else if (buffer[3] < 128) {
+    next = DecodeVarint64KnownSize<4>(buffer, value);
+  } else if (buffer[4] < 128) {
+    next = DecodeVarint64KnownSize<5>(buffer, value);
+  } else if (buffer[5] < 128) {
+    next = DecodeVarint64KnownSize<6>(buffer, value);
+  } else if (buffer[6] < 128) {
+    next = DecodeVarint64KnownSize<7>(buffer, value);
+  } else if (buffer[7] < 128) {
+    next = DecodeVarint64KnownSize<8>(buffer, value);
+  } else if (buffer[8] < 128) {
+    next = DecodeVarint64KnownSize<9>(buffer, value);
+  } else if (buffer[9] < 128) {
+    next = DecodeVarint64KnownSize<10>(buffer, value);
+  } else {
+    // We have overrun the maximum size of a varint (10 bytes). Assume
+    // the data is corrupt.
+    return std::make_pair(false, buffer + 11);
+  }
+
+  return std::make_pair(true, next);
+}
+
+}  // namespace
+
+bool CodedInputStream::ReadVarint32Slow(uint32* value) {
+  // Directly invoke ReadVarint64Fallback, since we already tried to optimize
+  // for one-byte varints.
+  std::pair<uint64, bool> p = ReadVarint64Fallback();
+  *value = static_cast<uint32>(p.first);
+  return p.second;
+}
+
+int64 CodedInputStream::ReadVarint32Fallback(uint32 first_byte_or_zero) {
+  if (BufferSize() >= kMaxVarintBytes ||
+      // Optimization:  We're also safe if the buffer is non-empty and it ends
+      // with a byte that would terminate a varint.
+      (buffer_end_ > buffer_ && !(buffer_end_[-1] & 0x80))) {
+    GOOGLE_DCHECK_NE(first_byte_or_zero, 0)
+        << "Caller should provide us with *buffer_ when buffer is non-empty";
+    uint32 temp;
+    ::std::pair<bool, const uint8*> p =
+        ReadVarint32FromArray(first_byte_or_zero, buffer_, &temp);
+    if (!p.first) return -1;
+    buffer_ = p.second;
+    return temp;
+  } else {
+    // Really slow case: we will incur the cost of an extra function call here,
+    // but moving this out of line reduces the size of this function, which
+    // improves the common case. In micro benchmarks, this is worth about 10-15%
+    uint32 temp;
+    return ReadVarint32Slow(&temp) ? static_cast<int64>(temp) : -1;
+  }
+}
+
+int CodedInputStream::ReadVarintSizeAsIntSlow() {
+  // Directly invoke ReadVarint64Fallback, since we already tried to optimize
+  // for one-byte varints.
+  std::pair<uint64, bool> p = ReadVarint64Fallback();
+  if (!p.second || p.first > static_cast<uint64>(INT_MAX)) return -1;
+  return p.first;
+}
+
+int CodedInputStream::ReadVarintSizeAsIntFallback() {
+  if (BufferSize() >= kMaxVarintBytes ||
+      // Optimization:  We're also safe if the buffer is non-empty and it ends
+      // with a byte that would terminate a varint.
+      (buffer_end_ > buffer_ && !(buffer_end_[-1] & 0x80))) {
+    uint64 temp;
+    ::std::pair<bool, const uint8*> p = ReadVarint64FromArray(buffer_, &temp);
+    if (!p.first || temp > static_cast<uint64>(INT_MAX)) return -1;
+    buffer_ = p.second;
+    return temp;
+  } else {
+    // Really slow case: we will incur the cost of an extra function call here,
+    // but moving this out of line reduces the size of this function, which
+    // improves the common case. In micro benchmarks, this is worth about 10-15%
+    return ReadVarintSizeAsIntSlow();
+  }
+}
+
+uint32 CodedInputStream::ReadTagSlow() {
+  if (buffer_ == buffer_end_) {
+    // Call refresh.
+    if (!Refresh()) {
+      // Refresh failed.  Make sure that it failed due to EOF, not because
+      // we hit total_bytes_limit_, which, unlike normal limits, is not a
+      // valid place to end a message.
+      int current_position = total_bytes_read_ - buffer_size_after_limit_;
+      if (current_position >= total_bytes_limit_) {
+        // Hit total_bytes_limit_.  But if we also hit the normal limit,
+        // we're still OK.
+        legitimate_message_end_ = current_limit_ == total_bytes_limit_;
+      } else {
+        legitimate_message_end_ = true;
+      }
+      return 0;
+    }
+  }
+
+  // For the slow path, just do a 64-bit read. Try to optimize for one-byte tags
+  // again, since we have now refreshed the buffer.
+  uint64 result = 0;
+  if (!ReadVarint64(&result)) return 0;
+  return static_cast<uint32>(result);
+}
+
+uint32 CodedInputStream::ReadTagFallback(uint32 first_byte_or_zero) {
+  const int buf_size = BufferSize();
+  if (buf_size >= kMaxVarintBytes ||
+      // Optimization:  We're also safe if the buffer is non-empty and it ends
+      // with a byte that would terminate a varint.
+      (buf_size > 0 && !(buffer_end_[-1] & 0x80))) {
+    GOOGLE_DCHECK_EQ(first_byte_or_zero, buffer_[0]);
+    if (first_byte_or_zero == 0) {
+      ++buffer_;
+      return 0;
+    }
+    uint32 tag;
+    ::std::pair<bool, const uint8*> p =
+        ReadVarint32FromArray(first_byte_or_zero, buffer_, &tag);
+    if (!p.first) {
+      return 0;
+    }
+    buffer_ = p.second;
+    return tag;
+  } else {
+    // We are commonly at a limit when attempting to read tags. Try to quickly
+    // detect this case without making another function call.
+    if ((buf_size == 0) &&
+        ((buffer_size_after_limit_ > 0) ||
+         (total_bytes_read_ == current_limit_)) &&
+        // Make sure that the limit we hit is not total_bytes_limit_, since
+        // in that case we still need to call Refresh() so that it prints an
+        // error.
+        total_bytes_read_ - buffer_size_after_limit_ < total_bytes_limit_) {
+      // We hit a byte limit.
+      legitimate_message_end_ = true;
+      return 0;
+    }
+    return ReadTagSlow();
+  }
+}
+
+bool CodedInputStream::ReadVarint64Slow(uint64* value) {
+  // Slow path:  This read might cross the end of the buffer, so we
+  // need to check and refresh the buffer if and when it does.
+
+  uint64 result = 0;
+  int count = 0;
+  uint32 b;
+
+  do {
+    if (count == kMaxVarintBytes) {
+      *value = 0;
+      return false;
+    }
+    while (buffer_ == buffer_end_) {
+      if (!Refresh()) {
+        *value = 0;
+        return false;
+      }
+    }
+    b = *buffer_;
+    result |= static_cast<uint64>(b & 0x7F) << (7 * count);
+    Advance(1);
+    ++count;
+  } while (b & 0x80);
+
+  *value = result;
+  return true;
+}
+
+std::pair<uint64, bool> CodedInputStream::ReadVarint64Fallback() {
+  if (BufferSize() >= kMaxVarintBytes ||
+      // Optimization:  We're also safe if the buffer is non-empty and it ends
+      // with a byte that would terminate a varint.
+      (buffer_end_ > buffer_ && !(buffer_end_[-1] & 0x80))) {
+    uint64 temp;
+    ::std::pair<bool, const uint8*> p = ReadVarint64FromArray(buffer_, &temp);
+    if (!p.first) {
+      return std::make_pair(0, false);
+    }
+    buffer_ = p.second;
+    return std::make_pair(temp, true);
+  } else {
+    uint64 temp;
+    bool success = ReadVarint64Slow(&temp);
+    return std::make_pair(temp, success);
+  }
+}
+
+bool CodedInputStream::Refresh() {
+  GOOGLE_DCHECK_EQ(0, BufferSize());
+
+  if (buffer_size_after_limit_ > 0 || overflow_bytes_ > 0 ||
+      total_bytes_read_ == current_limit_) {
+    // We've hit a limit.  Stop.
+    int current_position = total_bytes_read_ - buffer_size_after_limit_;
+
+    if (current_position >= total_bytes_limit_ &&
+        total_bytes_limit_ != current_limit_) {
+      // Hit total_bytes_limit_.
+      PrintTotalBytesLimitError();
+    }
+
+    return false;
+  }
+
+  const void* void_buffer;
+  int buffer_size;
+  if (NextNonEmpty(input_, &void_buffer, &buffer_size)) {
+    buffer_ = reinterpret_cast<const uint8*>(void_buffer);
+    buffer_end_ = buffer_ + buffer_size;
+    GOOGLE_CHECK_GE(buffer_size, 0);
+
+    if (total_bytes_read_ <= INT_MAX - buffer_size) {
+      total_bytes_read_ += buffer_size;
+    } else {
+      // Overflow.  Reset buffer_end_ to not include the bytes beyond INT_MAX.
+      // We can't get that far anyway, because total_bytes_limit_ is guaranteed
+      // to be less than it.  We need to keep track of the number of bytes
+      // we discarded, though, so that we can call input_->BackUp() to back
+      // up over them on destruction.
+
+      // The following line is equivalent to:
+      //   overflow_bytes_ = total_bytes_read_ + buffer_size - INT_MAX;
+      // except that it avoids overflows.  Signed integer overflow has
+      // undefined results according to the C standard.
+      overflow_bytes_ = total_bytes_read_ - (INT_MAX - buffer_size);
+      buffer_end_ -= overflow_bytes_;
+      total_bytes_read_ = INT_MAX;
+    }
+
+    RecomputeBufferLimits();
+    return true;
+  } else {
+    buffer_ = NULL;
+    buffer_end_ = NULL;
+    return false;
+  }
+}
+
+// CodedOutputStream =================================================
+
+void EpsCopyOutputStream::EnableAliasing(bool enabled) {
+  aliasing_enabled_ = enabled && stream_->AllowsAliasing();
+}
+
+int64 EpsCopyOutputStream::ByteCount(uint8* ptr) const {
+  // Calculate the current offset relative to the end of the stream buffer.
+  int delta = (end_ - ptr) + (buffer_end_ ? 0 : kSlopBytes);
+  return stream_->ByteCount() - delta;
+}
+
+// Flushes what's written out to the underlying ZeroCopyOutputStream buffers.
+// Returns the size remaining in the buffer and sets buffer_end_ to the start
+// of the remaining buffer, ie. [buffer_end_, buffer_end_ + return value)
+int EpsCopyOutputStream::Flush(uint8* ptr) {
+  while (buffer_end_ && ptr > end_) {
+    int overrun = ptr - end_;
+    GOOGLE_DCHECK(!had_error_);
+    GOOGLE_DCHECK(overrun <= kSlopBytes);  // NOLINT
+    ptr = Next() + overrun;
+    if (had_error_) return 0;
+  }
+  int s;
+  if (buffer_end_) {
+    std::memcpy(buffer_end_, buffer_, ptr - buffer_);
+    buffer_end_ += ptr - buffer_;
+    s = end_ - ptr;
+  } else {
+    // The stream is writing directly in the ZeroCopyOutputStream buffer.
+    s = end_ + kSlopBytes - ptr;
+    buffer_end_ = ptr;
+  }
+  GOOGLE_DCHECK(s >= 0);  // NOLINT
+  return s;
+}
+
+uint8* EpsCopyOutputStream::Trim(uint8* ptr) {
+  if (had_error_) return ptr;
+  int s = Flush(ptr);
+  if (s) stream_->BackUp(s);
+  // Reset to initial state (expecting new buffer)
+  buffer_end_ = end_ = buffer_;
+  return buffer_;
+}
+
+
+uint8* EpsCopyOutputStream::FlushAndResetBuffer(uint8* ptr) {
+  if (had_error_) return buffer_;
+  int s = Flush(ptr);
+  if (had_error_) return buffer_;
+  return SetInitialBuffer(buffer_end_, s);
+}
+
+bool EpsCopyOutputStream::Skip(int count, uint8** pp) {
+  if (count < 0) return false;
+  if (had_error_) {
+    *pp = buffer_;
+    return false;
+  }
+  int size = Flush(*pp);
+  if (had_error_) {
+    *pp = buffer_;
+    return false;
+  }
+  void* data = buffer_end_;
+  while (count > size) {
+    count -= size;
+    if (!stream_->Next(&data, &size)) {
+      *pp = Error();
+      return false;
+    }
+  }
+  *pp = SetInitialBuffer(static_cast<uint8*>(data) + count, size - count);
+  return true;
+}
+
+bool EpsCopyOutputStream::GetDirectBufferPointer(void** data, int* size,
+                                                 uint8** pp) {
+  if (had_error_) {
+    *pp = buffer_;
+    return false;
+  }
+  *size = Flush(*pp);
+  if (had_error_) {
+    *pp = buffer_;
+    return false;
+  }
+  *data = buffer_end_;
+  while (*size == 0) {
+    if (!stream_->Next(data, size)) {
+      *pp = Error();
+      return false;
+    }
+  }
+  *pp = SetInitialBuffer(*data, *size);
+  return true;
+}
+
+uint8* EpsCopyOutputStream::GetDirectBufferForNBytesAndAdvance(int size,
+                                                               uint8** pp) {
+  if (had_error_) {
+    *pp = buffer_;
+    return nullptr;
+  }
+  int s = Flush(*pp);
+  if (had_error_) {
+    *pp = buffer_;
+    return nullptr;
+  }
+  if (s >= size) {
+    auto res = buffer_end_;
+    *pp = SetInitialBuffer(buffer_end_ + size, s - size);
+    return res;
+  } else {
+    *pp = SetInitialBuffer(buffer_end_, s);
+    return nullptr;
+  }
+}
+
+uint8* EpsCopyOutputStream::Next() {
+  GOOGLE_DCHECK(!had_error_);  // NOLINT
+  if (PROTOBUF_PREDICT_FALSE(stream_ == nullptr)) return Error();
+  if (buffer_end_) {
+    // We're in the patch buffer and need to fill up the previous buffer.
+    std::memcpy(buffer_end_, buffer_, end_ - buffer_);
+    uint8* ptr;
+    int size;
+    do {
+      void* data;
+      if (PROTOBUF_PREDICT_FALSE(!stream_->Next(&data, &size))) {
+        // Stream has an error, we use the patch buffer to continue to be
+        // able to write.
+        return Error();
+      }
+      ptr = static_cast<uint8*>(data);
+    } while (size == 0);
+    if (PROTOBUF_PREDICT_TRUE(size > kSlopBytes)) {
+      std::memcpy(ptr, end_, kSlopBytes);
+      end_ = ptr + size - kSlopBytes;
+      buffer_end_ = nullptr;
+      return ptr;
+    } else {
+      GOOGLE_DCHECK(size > 0);  // NOLINT
+      // Buffer to small
+      std::memmove(buffer_, end_, kSlopBytes);
+      buffer_end_ = ptr;
+      end_ = buffer_ + size;
+      return buffer_;
+    }
+  } else {
+    std::memcpy(buffer_, end_, kSlopBytes);
+    buffer_end_ = end_;
+    end_ = buffer_ + kSlopBytes;
+    return buffer_;
+  }
+}
+
+uint8* EpsCopyOutputStream::EnsureSpaceFallback(uint8* ptr) {
+  do {
+    if (PROTOBUF_PREDICT_FALSE(had_error_)) return buffer_;
+    int overrun = ptr - end_;
+    GOOGLE_DCHECK(overrun >= 0);           // NOLINT
+    GOOGLE_DCHECK(overrun <= kSlopBytes);  // NOLINT
+    ptr = Next() + overrun;
+  } while (ptr >= end_);
+  GOOGLE_DCHECK(ptr < end_);  // NOLINT
+  return ptr;
+}
+
+uint8* EpsCopyOutputStream::WriteRawFallback(const void* data, int size,
+                                             uint8* ptr) {
+  int s = GetSize(ptr);
+  while (s < size) {
+    std::memcpy(ptr, data, s);
+    size -= s;
+    data = static_cast<const uint8*>(data) + s;
+    ptr = EnsureSpaceFallback(ptr + s);
+    s = GetSize(ptr);
+  }
+  std::memcpy(ptr, data, size);
+  return ptr + size;
+}
+
+uint8* EpsCopyOutputStream::WriteAliasedRaw(const void* data, int size,
+                                            uint8* ptr) {
+  if (size < GetSize(ptr)
+  ) {
+    return WriteRaw(data, size, ptr);
+  } else {
+    ptr = Trim(ptr);
+    if (stream_->WriteAliasedRaw(data, size)) return ptr;
+    return Error();
+  }
+}
+
+#ifndef PROTOBUF_LITTLE_ENDIAN
+uint8* EpsCopyOutputStream::WriteRawLittleEndian32(const void* data, int size,
+                                                   uint8* ptr) {
+  auto p = static_cast<const uint8*>(data);
+  auto end = p + size;
+  while (end - p >= kSlopBytes) {
+    ptr = EnsureSpace(ptr);
+    uint32 buffer[4];
+    static_assert(sizeof(buffer) == kSlopBytes, "Buffer must be kSlopBytes");
+    std::memcpy(buffer, p, kSlopBytes);
+    p += kSlopBytes;
+    for (auto x : buffer)
+      ptr = CodedOutputStream::WriteLittleEndian32ToArray(x, ptr);
+  }
+  while (p < end) {
+    ptr = EnsureSpace(ptr);
+    uint32 buffer;
+    std::memcpy(&buffer, p, 4);
+    p += 4;
+    ptr = CodedOutputStream::WriteLittleEndian32ToArray(buffer, ptr);
+  }
+  return ptr;
+}
+
+uint8* EpsCopyOutputStream::WriteRawLittleEndian64(const void* data, int size,
+                                                   uint8* ptr) {
+  auto p = static_cast<const uint8*>(data);
+  auto end = p + size;
+  while (end - p >= kSlopBytes) {
+    ptr = EnsureSpace(ptr);
+    uint64 buffer[2];
+    static_assert(sizeof(buffer) == kSlopBytes, "Buffer must be kSlopBytes");
+    std::memcpy(buffer, p, kSlopBytes);
+    p += kSlopBytes;
+    for (auto x : buffer)
+      ptr = CodedOutputStream::WriteLittleEndian64ToArray(x, ptr);
+  }
+  while (p < end) {
+    ptr = EnsureSpace(ptr);
+    uint64 buffer;
+    std::memcpy(&buffer, p, 8);
+    p += 8;
+    ptr = CodedOutputStream::WriteLittleEndian64ToArray(buffer, ptr);
+  }
+  return ptr;
+}
+#endif
+
+
+uint8* EpsCopyOutputStream::WriteStringMaybeAliasedOutline(uint32 num,
+                                                           const std::string& s,
+                                                           uint8* ptr) {
+  ptr = EnsureSpace(ptr);
+  uint32 size = s.size();
+  ptr = WriteLengthDelim(num, size, ptr);
+  return WriteRawMaybeAliased(s.data(), size, ptr);
+}
+
+uint8* EpsCopyOutputStream::WriteStringOutline(uint32 num, const std::string& s,
+                                               uint8* ptr) {
+  ptr = EnsureSpace(ptr);
+  uint32 size = s.size();
+  ptr = WriteLengthDelim(num, size, ptr);
+  return WriteRaw(s.data(), size, ptr);
+}
+
+std::atomic<bool> CodedOutputStream::default_serialization_deterministic_{
+    false};
+
+CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* stream,
+                                     bool do_eager_refresh)
+    : impl_(stream, IsDefaultSerializationDeterministic(), &cur_),
+      start_count_(stream->ByteCount()) {
+  if (do_eager_refresh) {
+    void* data;
+    int size;
+    if (!stream->Next(&data, &size) || size == 0) return;
+    cur_ = impl_.SetInitialBuffer(data, size);
+  }
+}
+
+CodedOutputStream::~CodedOutputStream() { Trim(); }
+
+
+uint8* CodedOutputStream::WriteStringWithSizeToArray(const std::string& str,
+                                                     uint8* target) {
+  GOOGLE_DCHECK_LE(str.size(), kuint32max);
+  target = WriteVarint32ToArray(str.size(), target);
+  return WriteStringToArray(str, target);
+}
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google

ubuntu/google/protobuf/io/gzip_stream.cc

@@ -0,0 +1,333 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: brianolson@google.com (Brian Olson)
+//
+// This file contains the implementation of classes GzipInputStream and
+// GzipOutputStream.
+
+
+#if HAVE_ZLIB
+#include <google/protobuf/io/gzip_stream.h>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/stubs/logging.h>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+static const int kDefaultBufferSize = 65536;
+
+GzipInputStream::GzipInputStream(ZeroCopyInputStream* sub_stream, Format format,
+                                 int buffer_size)
+    : format_(format), sub_stream_(sub_stream), zerror_(Z_OK), byte_count_(0) {
+  zcontext_.state = Z_NULL;
+  zcontext_.zalloc = Z_NULL;
+  zcontext_.zfree = Z_NULL;
+  zcontext_.opaque = Z_NULL;
+  zcontext_.total_out = 0;
+  zcontext_.next_in = NULL;
+  zcontext_.avail_in = 0;
+  zcontext_.total_in = 0;
+  zcontext_.msg = NULL;
+  if (buffer_size == -1) {
+    output_buffer_length_ = kDefaultBufferSize;
+  } else {
+    output_buffer_length_ = buffer_size;
+  }
+  output_buffer_ = operator new(output_buffer_length_);
+  GOOGLE_CHECK(output_buffer_ != NULL);
+  zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
+  zcontext_.avail_out = output_buffer_length_;
+  output_position_ = output_buffer_;
+}
+GzipInputStream::~GzipInputStream() {
+  operator delete(output_buffer_);
+  zerror_ = inflateEnd(&zcontext_);
+}
+
+static inline int internalInflateInit2(z_stream* zcontext,
+                                       GzipInputStream::Format format) {
+  int windowBitsFormat = 0;
+  switch (format) {
+    case GzipInputStream::GZIP:
+      windowBitsFormat = 16;
+      break;
+    case GzipInputStream::AUTO:
+      windowBitsFormat = 32;
+      break;
+    case GzipInputStream::ZLIB:
+      windowBitsFormat = 0;
+      break;
+  }
+  return inflateInit2(zcontext, /* windowBits */ 15 | windowBitsFormat);
+}
+
+int GzipInputStream::Inflate(int flush) {
+  if ((zerror_ == Z_OK) && (zcontext_.avail_out == 0)) {
+    // previous inflate filled output buffer. don't change input params yet.
+  } else if (zcontext_.avail_in == 0) {
+    const void* in;
+    int in_size;
+    bool first = zcontext_.next_in == NULL;
+    bool ok = sub_stream_->Next(&in, &in_size);
+    if (!ok) {
+      zcontext_.next_out = NULL;
+      zcontext_.avail_out = 0;
+      return Z_STREAM_END;
+    }
+    zcontext_.next_in = static_cast<Bytef*>(const_cast<void*>(in));
+    zcontext_.avail_in = in_size;
+    if (first) {
+      int error = internalInflateInit2(&zcontext_, format_);
+      if (error != Z_OK) {
+        return error;
+      }
+    }
+  }
+  zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
+  zcontext_.avail_out = output_buffer_length_;
+  output_position_ = output_buffer_;
+  int error = inflate(&zcontext_, flush);
+  return error;
+}
+
+void GzipInputStream::DoNextOutput(const void** data, int* size) {
+  *data = output_position_;
+  *size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_);
+  output_position_ = zcontext_.next_out;
+}
+
+// implements ZeroCopyInputStream ----------------------------------
+bool GzipInputStream::Next(const void** data, int* size) {
+  bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
+            (zerror_ == Z_BUF_ERROR);
+  if ((!ok) || (zcontext_.next_out == NULL)) {
+    return false;
+  }
+  if (zcontext_.next_out != output_position_) {
+    DoNextOutput(data, size);
+    return true;
+  }
+  if (zerror_ == Z_STREAM_END) {
+    if (zcontext_.next_out != NULL) {
+      // sub_stream_ may have concatenated streams to follow
+      zerror_ = inflateEnd(&zcontext_);
+      byte_count_ += zcontext_.total_out;
+      if (zerror_ != Z_OK) {
+        return false;
+      }
+      zerror_ = internalInflateInit2(&zcontext_, format_);
+      if (zerror_ != Z_OK) {
+        return false;
+      }
+    } else {
+      *data = NULL;
+      *size = 0;
+      return false;
+    }
+  }
+  zerror_ = Inflate(Z_NO_FLUSH);
+  if ((zerror_ == Z_STREAM_END) && (zcontext_.next_out == NULL)) {
+    // The underlying stream's Next returned false inside Inflate.
+    return false;
+  }
+  ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
+       (zerror_ == Z_BUF_ERROR);
+  if (!ok) {
+    return false;
+  }
+  DoNextOutput(data, size);
+  return true;
+}
+void GzipInputStream::BackUp(int count) {
+  output_position_ = reinterpret_cast<void*>(
+      reinterpret_cast<uintptr_t>(output_position_) - count);
+}
+bool GzipInputStream::Skip(int count) {
+  const void* data;
+  int size = 0;
+  bool ok = Next(&data, &size);
+  while (ok && (size < count)) {
+    count -= size;
+    ok = Next(&data, &size);
+  }
+  if (size > count) {
+    BackUp(size - count);
+  }
+  return ok;
+}
+int64_t GzipInputStream::ByteCount() const {
+  int64 ret = byte_count_ + zcontext_.total_out;
+  if (zcontext_.next_out != NULL && output_position_ != NULL) {
+    ret += reinterpret_cast<uintptr_t>(zcontext_.next_out) -
+           reinterpret_cast<uintptr_t>(output_position_);
+  }
+  return ret;
+}
+
+// =========================================================================
+
+GzipOutputStream::Options::Options()
+    : format(GZIP),
+      buffer_size(kDefaultBufferSize),
+      compression_level(Z_DEFAULT_COMPRESSION),
+      compression_strategy(Z_DEFAULT_STRATEGY) {}
+
+GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream) {
+  Init(sub_stream, Options());
+}
+
+GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream,
+                                   const Options& options) {
+  Init(sub_stream, options);
+}
+
+void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream,
+                            const Options& options) {
+  sub_stream_ = sub_stream;
+  sub_data_ = NULL;
+  sub_data_size_ = 0;
+
+  input_buffer_length_ = options.buffer_size;
+  input_buffer_ = operator new(input_buffer_length_);
+  GOOGLE_CHECK(input_buffer_ != NULL);
+
+  zcontext_.zalloc = Z_NULL;
+  zcontext_.zfree = Z_NULL;
+  zcontext_.opaque = Z_NULL;
+  zcontext_.next_out = NULL;
+  zcontext_.avail_out = 0;
+  zcontext_.total_out = 0;
+  zcontext_.next_in = NULL;
+  zcontext_.avail_in = 0;
+  zcontext_.total_in = 0;
+  zcontext_.msg = NULL;
+  // default to GZIP format
+  int windowBitsFormat = 16;
+  if (options.format == ZLIB) {
+    windowBitsFormat = 0;
+  }
+  zerror_ =
+      deflateInit2(&zcontext_, options.compression_level, Z_DEFLATED,
+                   /* windowBits */ 15 | windowBitsFormat,
+                   /* memLevel (default) */ 8, options.compression_strategy);
+}
+
+GzipOutputStream::~GzipOutputStream() {
+  Close();
+  operator delete(input_buffer_);
+}
+
+// private
+int GzipOutputStream::Deflate(int flush) {
+  int error = Z_OK;
+  do {
+    if ((sub_data_ == NULL) || (zcontext_.avail_out == 0)) {
+      bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_);
+      if (!ok) {
+        sub_data_ = NULL;
+        sub_data_size_ = 0;
+        return Z_BUF_ERROR;
+      }
+      GOOGLE_CHECK_GT(sub_data_size_, 0);
+      zcontext_.next_out = static_cast<Bytef*>(sub_data_);
+      zcontext_.avail_out = sub_data_size_;
+    }
+    error = deflate(&zcontext_, flush);
+  } while (error == Z_OK && zcontext_.avail_out == 0);
+  if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) {
+    // Notify lower layer of data.
+    sub_stream_->BackUp(zcontext_.avail_out);
+    // We don't own the buffer anymore.
+    sub_data_ = NULL;
+    sub_data_size_ = 0;
+  }
+  return error;
+}
+
+// implements ZeroCopyOutputStream ---------------------------------
+bool GzipOutputStream::Next(void** data, int* size) {
+  if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
+    return false;
+  }
+  if (zcontext_.avail_in != 0) {
+    zerror_ = Deflate(Z_NO_FLUSH);
+    if (zerror_ != Z_OK) {
+      return false;
+    }
+  }
+  if (zcontext_.avail_in == 0) {
+    // all input was consumed. reset the buffer.
+    zcontext_.next_in = static_cast<Bytef*>(input_buffer_);
+    zcontext_.avail_in = input_buffer_length_;
+    *data = input_buffer_;
+    *size = input_buffer_length_;
+  } else {
+    // The loop in Deflate should consume all avail_in
+    GOOGLE_LOG(DFATAL) << "Deflate left bytes unconsumed";
+  }
+  return true;
+}
+void GzipOutputStream::BackUp(int count) {
+  GOOGLE_CHECK_GE(zcontext_.avail_in, count);
+  zcontext_.avail_in -= count;
+}
+int64_t GzipOutputStream::ByteCount() const {
+  return zcontext_.total_in + zcontext_.avail_in;
+}
+
+bool GzipOutputStream::Flush() {
+  zerror_ = Deflate(Z_FULL_FLUSH);
+  // Return true if the flush succeeded or if it was a no-op.
+  return (zerror_ == Z_OK) ||
+         (zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 &&
+          zcontext_.avail_out != 0);
+}
+
+bool GzipOutputStream::Close() {
+  if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
+    return false;
+  }
+  do {
+    zerror_ = Deflate(Z_FINISH);
+  } while (zerror_ == Z_OK);
+  zerror_ = deflateEnd(&zcontext_);
+  bool ok = zerror_ == Z_OK;
+  zerror_ = Z_STREAM_END;
+  return ok;
+}
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // HAVE_ZLIB

ubuntu/google/protobuf/io/gzip_stream.h

@@ -0,0 +1,202 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: brianolson@google.com (Brian Olson)
+//
+// This file contains the definition for classes GzipInputStream and
+// GzipOutputStream.
+//
+// GzipInputStream decompresses data from an underlying
+// ZeroCopyInputStream and provides the decompressed data as a
+// ZeroCopyInputStream.
+//
+// GzipOutputStream is an ZeroCopyOutputStream that compresses data to
+// an underlying ZeroCopyOutputStream.
+
+#ifndef GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__
+#define GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__
+
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/port.h>
+#include <zlib.h>
+
+#include <google/protobuf/port_def.inc>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+// A ZeroCopyInputStream that reads compressed data through zlib
+class PROTOBUF_EXPORT GzipInputStream : public ZeroCopyInputStream {
+ public:
+  // Format key for constructor
+  enum Format {
+    // zlib will autodetect gzip header or deflate stream
+    AUTO = 0,
+
+    // GZIP streams have some extra header data for file attributes.
+    GZIP = 1,
+
+    // Simpler zlib stream format.
+    ZLIB = 2,
+  };
+
+  // buffer_size and format may be -1 for default of 64kB and GZIP format
+  explicit GzipInputStream(ZeroCopyInputStream* sub_stream,
+                           Format format = AUTO, int buffer_size = -1);
+  virtual ~GzipInputStream();
+
+  // Return last error message or NULL if no error.
+  inline const char* ZlibErrorMessage() const { return zcontext_.msg; }
+  inline int ZlibErrorCode() const { return zerror_; }
+
+  // implements ZeroCopyInputStream ----------------------------------
+  bool Next(const void** data, int* size);
+  void BackUp(int count);
+  bool Skip(int count);
+  int64_t ByteCount() const;
+
+ private:
+  Format format_;
+
+  ZeroCopyInputStream* sub_stream_;
+
+  z_stream zcontext_;
+  int zerror_;
+
+  void* output_buffer_;
+  void* output_position_;
+  size_t output_buffer_length_;
+  int64 byte_count_;
+
+  int Inflate(int flush);
+  void DoNextOutput(const void** data, int* size);
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(GzipInputStream);
+};
+
+class PROTOBUF_EXPORT GzipOutputStream : public ZeroCopyOutputStream {
+ public:
+  // Format key for constructor
+  enum Format {
+    // GZIP streams have some extra header data for file attributes.
+    GZIP = 1,
+
+    // Simpler zlib stream format.
+    ZLIB = 2,
+  };
+
+  struct PROTOBUF_EXPORT Options {
+    // Defaults to GZIP.
+    Format format;
+
+    // What size buffer to use internally.  Defaults to 64kB.
+    int buffer_size;
+
+    // A number between 0 and 9, where 0 is no compression and 9 is best
+    // compression.  Defaults to Z_DEFAULT_COMPRESSION (see zlib.h).
+    int compression_level;
+
+    // Defaults to Z_DEFAULT_STRATEGY.  Can also be set to Z_FILTERED,
+    // Z_HUFFMAN_ONLY, or Z_RLE.  See the documentation for deflateInit2 in
+    // zlib.h for definitions of these constants.
+    int compression_strategy;
+
+    Options();  // Initializes with default values.
+  };
+
+  // Create a GzipOutputStream with default options.
+  explicit GzipOutputStream(ZeroCopyOutputStream* sub_stream);
+
+  // Create a GzipOutputStream with the given options.
+  GzipOutputStream(ZeroCopyOutputStream* sub_stream, const Options& options);
+
+  virtual ~GzipOutputStream();
+
+  // Return last error message or NULL if no error.
+  inline const char* ZlibErrorMessage() const { return zcontext_.msg; }
+  inline int ZlibErrorCode() const { return zerror_; }
+
+  // Flushes data written so far to zipped data in the underlying stream.
+  // It is the caller's responsibility to flush the underlying stream if
+  // necessary.
+  // Compression may be less efficient stopping and starting around flushes.
+  // Returns true if no error.
+  //
+  // Please ensure that block size is > 6. Here is an excerpt from the zlib
+  // doc that explains why:
+  //
+  // In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that avail_out
+  // is greater than six to avoid repeated flush markers due to
+  // avail_out == 0 on return.
+  bool Flush();
+
+  // Writes out all data and closes the gzip stream.
+  // It is the caller's responsibility to close the underlying stream if
+  // necessary.
+  // Returns true if no error.
+  bool Close();
+
+  // implements ZeroCopyOutputStream ---------------------------------
+  bool Next(void** data, int* size);
+  void BackUp(int count);
+  int64_t ByteCount() const;
+
+ private:
+  ZeroCopyOutputStream* sub_stream_;
+  // Result from calling Next() on sub_stream_
+  void* sub_data_;
+  int sub_data_size_;
+
+  z_stream zcontext_;
+  int zerror_;
+  void* input_buffer_;
+  size_t input_buffer_length_;
+
+  // Shared constructor code.
+  void Init(ZeroCopyOutputStream* sub_stream, const Options& options);
+
+  // Do some compression.
+  // Takes zlib flush mode.
+  // Returns zlib error code.
+  int Deflate(int flush);
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(GzipOutputStream);
+};
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
+#endif  // GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__

ubuntu/google/protobuf/io/gzip_stream_unittest.sh

@@ -0,0 +1,44 @@
+#!/bin/sh -x
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2009 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Author: brianolson@google.com (Brian Olson)
+#
+# Test compatibility between command line gzip/gunzip binaries and
+# ZeroCopyStream versions.
+
+TESTFILE=Makefile
+
+(./zcgzip < ${TESTFILE} | gunzip | cmp - ${TESTFILE}) && \
+(gzip < ${TESTFILE} | ./zcgunzip | cmp - ${TESTFILE})
+
+# Result of "(cmd) && (cmd)" implicitly becomes result of this script
+# and thus the test.

ubuntu/google/protobuf/io/io_win32.cc

@@ -0,0 +1,470 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: laszlocsomor@google.com (Laszlo Csomor)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+// Implementation for long-path-aware open/mkdir/access/etc. on Windows, as well
+// as for the supporting utility functions.
+//
+// These functions convert the input path to an absolute Windows path
+// with "\\?\" prefix, then pass that to _wopen/_wmkdir/_waccess/etc.
+// (declared in <io.h>) respectively. This allows working with files/directories
+// whose paths are longer than MAX_PATH (260 chars).
+//
+// This file is only used on Windows, it's empty on other platforms.
+
+#if defined(_WIN32) && !defined(_XBOX_ONE)
+
+// Comment this out to fall back to using the ANSI versions (open, mkdir, ...)
+// instead of the Unicode ones (_wopen, _wmkdir, ...). Doing so can be useful to
+// debug failing tests if that's caused by the long path support.
+#define SUPPORT_LONGPATHS
+
+#include <google/protobuf/io/io_win32.h>
+
+#include <ctype.h>
+#include <direct.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <io.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <wctype.h>
+
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN 1
+#endif
+
+#include <windows.h>
+
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+namespace google {
+namespace protobuf {
+namespace io {
+namespace win32 {
+namespace {
+
+using std::string;
+using std::wstring;
+
+template <typename char_type>
+struct CharTraits {
+  static bool is_alpha(char_type ch);
+};
+
+template <>
+struct CharTraits<char> {
+  static bool is_alpha(char ch) { return isalpha(ch); }
+};
+
+template <>
+struct CharTraits<wchar_t> {
+  static bool is_alpha(wchar_t ch) { return iswalpha(ch); }
+};
+
+template <typename char_type>
+bool null_or_empty(const char_type* s) {
+  return s == nullptr || *s == 0;
+}
+
+// Returns true if the path starts with a drive letter, e.g. "c:".
+// Note that this won't check for the "\" after the drive letter, so this also
+// returns true for "c:foo" (which is "c:\${PWD}\foo").
+// This check requires that a path not have a longpath prefix ("\\?\").
+template <typename char_type>
+bool has_drive_letter(const char_type* ch) {
+  return CharTraits<char_type>::is_alpha(ch[0]) && ch[1] == ':';
+}
+
+// Returns true if the path starts with a longpath prefix ("\\?\").
+template <typename char_type>
+bool has_longpath_prefix(const char_type* path) {
+  return path[0] == '\\' && path[1] == '\\' && path[2] == '?' &&
+         path[3] == '\\';
+}
+
+template <typename char_type>
+bool is_separator(char_type c) {
+  return c == '/' || c == '\\';
+}
+
+// Returns true if the path starts with a drive specifier (e.g. "c:\").
+template <typename char_type>
+bool is_path_absolute(const char_type* path) {
+  return has_drive_letter(path) && is_separator(path[2]);
+}
+
+template <typename char_type>
+bool is_drive_relative(const char_type* path) {
+  return has_drive_letter(path) && (path[2] == 0 || !is_separator(path[2]));
+}
+
+wstring join_paths(const wstring& path1, const wstring& path2) {
+  if (path1.empty() || is_path_absolute(path2.c_str()) ||
+      has_longpath_prefix(path2.c_str())) {
+    return path2;
+  }
+  if (path2.empty()) {
+    return path1;
+  }
+
+  if (is_separator(path1[path1.size() - 1])) {
+    return is_separator(path2[0]) ? (path1 + path2.substr(1))
+                                       : (path1 + path2);
+  } else {
+    return is_separator(path2[0]) ? (path1 + path2)
+                                       : (path1 + L'\\' + path2);
+  }
+}
+
+wstring normalize(wstring path) {
+  if (has_longpath_prefix(path.c_str())) {
+    path = path.substr(4);
+  }
+
+  static const wstring dot(L".");
+  static const wstring dotdot(L"..");
+  const WCHAR* p = path.c_str();
+
+  std::vector<wstring> segments;
+  int segment_start = -1;
+  // Find the path segments in `path` (separated by "/").
+  for (int i = 0;; ++i) {
+    if (!is_separator(p[i]) && p[i] != L'\0') {
+      // The current character does not end a segment, so start one unless it's
+      // already started.
+      if (segment_start < 0) {
+        segment_start = i;
+      }
+    } else if (segment_start >= 0 && i > segment_start) {
+      // The current character is "/" or "\0", so this ends a segment.
+      // Add that to `segments` if there's anything to add; handle "." and "..".
+      wstring segment(p, segment_start, i - segment_start);
+      segment_start = -1;
+      if (segment == dotdot) {
+        if (!segments.empty() &&
+            (!has_drive_letter(segments[0].c_str()) || segments.size() > 1)) {
+          segments.pop_back();
+        }
+      } else if (segment != dot && !segment.empty()) {
+        segments.push_back(segment);
+      }
+    }
+    if (p[i] == L'\0') {
+      break;
+    }
+  }
+
+  // Handle the case when `path` is just a drive specifier (or some degenerate
+  // form of it, e.g. "c:\..").
+  if (segments.size() == 1 && segments[0].size() == 2 &&
+      has_drive_letter(segments[0].c_str())) {
+    return segments[0] + L'\\';
+  }
+
+  // Join all segments.
+  bool first = true;
+  std::wstringstream result;
+  for (int i = 0; i < segments.size(); ++i) {
+    if (!first) {
+      result << L'\\';
+    }
+    first = false;
+    result << segments[i];
+  }
+  // Preserve trailing separator if the input contained it.
+  if (!path.empty() && is_separator(p[path.size() - 1])) {
+    result << L'\\';
+  }
+  return result.str();
+}
+
+bool as_windows_path(const char* path, wstring* result) {
+  if (null_or_empty(path)) {
+    result->clear();
+    return true;
+  }
+  wstring wpath;
+  if (!strings::utf8_to_wcs(path, &wpath)) {
+    return false;
+  }
+  if (has_longpath_prefix(wpath.c_str())) {
+    *result = wpath;
+    return true;
+  }
+  if (is_separator(path[0]) || is_drive_relative(path)) {
+    return false;
+  }
+
+
+  if (!is_path_absolute(wpath.c_str())) {
+    int size = ::GetCurrentDirectoryW(0, nullptr);
+    if (size == 0 && GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
+      return false;
+    }
+    std::unique_ptr<WCHAR[]> wcwd(new WCHAR[size]);
+    ::GetCurrentDirectoryW(size, wcwd.get());
+    wpath = join_paths(wcwd.get(), wpath);
+  }
+  wpath = normalize(wpath);
+  if (!has_longpath_prefix(wpath.c_str())) {
+    // Add the "\\?\" prefix unconditionally. This way we prevent the Win32 API
+    // from processing the path and "helpfully" removing trailing dots from the
+    // path, for example.
+    // See https://github.com/bazelbuild/bazel/issues/2935
+    wpath = wstring(L"\\\\?\\") + wpath;
+  }
+  *result = wpath;
+  return true;
+}
+
+}  // namespace
+
+int open(const char* path, int flags, int mode) {
+#ifdef SUPPORT_LONGPATHS
+  wstring wpath;
+  if (!as_windows_path(path, &wpath)) {
+    errno = ENOENT;
+    return -1;
+  }
+  return ::_wopen(wpath.c_str(), flags, mode);
+#else
+  return ::_open(path, flags, mode);
+#endif
+}
+
+int mkdir(const char* path, int _mode) {
+#ifdef SUPPORT_LONGPATHS
+  wstring wpath;
+  if (!as_windows_path(path, &wpath)) {
+    errno = ENOENT;
+    return -1;
+  }
+  return ::_wmkdir(wpath.c_str());
+#else   // not SUPPORT_LONGPATHS
+  return ::_mkdir(path);
+#endif  // not SUPPORT_LONGPATHS
+}
+
+int access(const char* path, int mode) {
+#ifdef SUPPORT_LONGPATHS
+  wstring wpath;
+  if (!as_windows_path(path, &wpath)) {
+    errno = ENOENT;
+    return -1;
+  }
+  return ::_waccess(wpath.c_str(), mode);
+#else
+  return ::_access(path, mode);
+#endif
+}
+
+int chdir(const char* path) {
+#ifdef SUPPORT_LONGPATHS
+  wstring wpath;
+  if (!as_windows_path(path, &wpath)) {
+    errno = ENOENT;
+    return -1;
+  }
+  return ::_wchdir(wpath.c_str());
+#else
+  return ::_chdir(path);
+#endif
+}
+
+int stat(const char* path, struct _stat* buffer) {
+#ifdef SUPPORT_LONGPATHS
+  wstring wpath;
+  if (!as_windows_path(path, &wpath)) {
+    errno = ENOENT;
+    return -1;
+  }
+  return ::_wstat(wpath.c_str(), buffer);
+#else   // not SUPPORT_LONGPATHS
+  return ::_stat(path, buffer);
+#endif  // not SUPPORT_LONGPATHS
+}
+
+FILE* fopen(const char* path, const char* mode) {
+#ifdef SUPPORT_LONGPATHS
+  if (null_or_empty(path)) {
+    errno = EINVAL;
+    return nullptr;
+  }
+  wstring wpath;
+  if (!as_windows_path(path, &wpath)) {
+    errno = ENOENT;
+    return nullptr;
+  }
+  wstring wmode;
+  if (!strings::utf8_to_wcs(mode, &wmode)) {
+    errno = EINVAL;
+    return nullptr;
+  }
+  return ::_wfopen(wpath.c_str(), wmode.c_str());
+#else
+  return ::fopen(path, mode);
+#endif
+}
+
+int close(int fd) { return ::_close(fd); }
+
+int dup(int fd) { return ::_dup(fd); }
+
+int dup2(int fd1, int fd2) { return ::_dup2(fd1, fd2); }
+
+int read(int fd, void* buffer, size_t size) {
+  return ::_read(fd, buffer, size);
+}
+
+int setmode(int fd, int mode) { return ::_setmode(fd, mode); }
+
+int write(int fd, const void* buffer, size_t size) {
+  return ::_write(fd, buffer, size);
+}
+
+wstring testonly_utf8_to_winpath(const char* path) {
+  wstring wpath;
+  return as_windows_path(path, &wpath) ? wpath : wstring();
+}
+
+ExpandWildcardsResult ExpandWildcards(
+    const string& path, std::function<void(const string&)> consume) {
+  if (path.find_first_of("*?") == string::npos) {
+    // There are no wildcards in the path, we don't need to expand it.
+    consume(path);
+    return ExpandWildcardsResult::kSuccess;
+  }
+
+  wstring wpath;
+  if (!as_windows_path(path.c_str(), &wpath)) {
+    return ExpandWildcardsResult::kErrorInputPathConversion;
+  }
+
+  static const wstring kDot = L".";
+  static const wstring kDotDot = L"..";
+  WIN32_FIND_DATAW metadata;
+  HANDLE handle = ::FindFirstFileW(wpath.c_str(), &metadata);
+  if (handle == INVALID_HANDLE_VALUE) {
+    // The pattern does not match any files (or directories).
+    return ExpandWildcardsResult::kErrorNoMatchingFile;
+  }
+
+  string::size_type pos = path.find_last_of("\\/");
+  string dirname;
+  if (pos != string::npos) {
+    dirname = path.substr(0, pos + 1);
+  }
+
+  ExpandWildcardsResult matched = ExpandWildcardsResult::kErrorNoMatchingFile;
+  do {
+    // Ignore ".", "..", and directories.
+    if ((metadata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0 &&
+        kDot != metadata.cFileName && kDotDot != metadata.cFileName) {
+      matched = ExpandWildcardsResult::kSuccess;
+      string filename;
+      if (!strings::wcs_to_utf8(metadata.cFileName, &filename)) {
+        return ExpandWildcardsResult::kErrorOutputPathConversion;
+      }
+
+      if (dirname.empty()) {
+        consume(filename);
+      } else {
+        consume(dirname + filename);
+      }
+    }
+  } while (::FindNextFileW(handle, &metadata));
+  FindClose(handle);
+  return matched;
+}
+
+namespace strings {
+
+bool wcs_to_mbs(const WCHAR* s, string* out, bool outUtf8) {
+  if (null_or_empty(s)) {
+    out->clear();
+    return true;
+  }
+  BOOL usedDefaultChar = FALSE;
+  SetLastError(0);
+  int size = WideCharToMultiByte(
+      outUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, nullptr, 0, nullptr,
+      outUtf8 ? nullptr : &usedDefaultChar);
+  if ((size == 0 && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
+      || usedDefaultChar) {
+    return false;
+  }
+  std::unique_ptr<CHAR[]> astr(new CHAR[size]);
+  WideCharToMultiByte(
+      outUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, astr.get(), size, nullptr, nullptr);
+  out->assign(astr.get());
+  return true;
+}
+
+bool mbs_to_wcs(const char* s, wstring* out, bool inUtf8) {
+  if (null_or_empty(s)) {
+    out->clear();
+    return true;
+  }
+
+  SetLastError(0);
+  int size =
+      MultiByteToWideChar(inUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, nullptr, 0);
+  if (size == 0 && GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
+    return false;
+  }
+  std::unique_ptr<WCHAR[]> wstr(new WCHAR[size]);
+  MultiByteToWideChar(
+      inUtf8 ? CP_UTF8 : CP_ACP, 0, s, -1, wstr.get(), size + 1);
+  out->assign(wstr.get());
+  return true;
+}
+
+bool utf8_to_wcs(const char* input, wstring* out) {
+  return mbs_to_wcs(input, out, true);
+}
+
+bool wcs_to_utf8(const wchar_t* input, string* out) {
+  return wcs_to_mbs(input, out, true);
+}
+
+}  // namespace strings
+}  // namespace win32
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // defined(_WIN32)

ubuntu/google/protobuf/io/io_win32.h

@@ -0,0 +1,139 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: laszlocsomor@google.com (Laszlo Csomor)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+// This file contains the declarations for Windows implementations of
+// commonly used POSIX functions such as open(2) and access(2), as well
+// as macro definitions for flags of these functions.
+//
+// By including this file you'll redefine open/access/etc. to
+// ::google::protobuf::io::win32::{open/access/etc.}.
+// Make sure you don't include a header that attempts to redeclare or
+// redefine these functions, that'll lead to confusing compilation
+// errors. It's best to #include this file as the last one to ensure that.
+//
+// This file is only used on Windows, it's empty on other platforms.
+
+#ifndef GOOGLE_PROTOBUF_IO_IO_WIN32_H__
+#define GOOGLE_PROTOBUF_IO_IO_WIN32_H__
+
+#if defined(_WIN32)
+
+#include <functional>
+#include <string>
+
+#include <google/protobuf/port.h>
+#include <google/protobuf/port_def.inc>
+
+// Compilers on Windows other than MSVC (e.g. Cygwin, MinGW32) define the
+// following functions already, except for mkdir.
+namespace google {
+namespace protobuf {
+namespace io {
+namespace win32 {
+
+PROTOBUF_EXPORT FILE* fopen(const char* path, const char* mode);
+PROTOBUF_EXPORT int access(const char* path, int mode);
+PROTOBUF_EXPORT int chdir(const char* path);
+PROTOBUF_EXPORT int close(int fd);
+PROTOBUF_EXPORT int dup(int fd);
+PROTOBUF_EXPORT int dup2(int fd1, int fd2);
+PROTOBUF_EXPORT int mkdir(const char* path, int _mode);
+PROTOBUF_EXPORT int open(const char* path, int flags, int mode = 0);
+PROTOBUF_EXPORT int read(int fd, void* buffer, size_t size);
+PROTOBUF_EXPORT int setmode(int fd, int mode);
+PROTOBUF_EXPORT int stat(const char* path, struct _stat* buffer);
+PROTOBUF_EXPORT int write(int fd, const void* buffer, size_t size);
+PROTOBUF_EXPORT std::wstring testonly_utf8_to_winpath(const char* path);
+
+enum class ExpandWildcardsResult {
+  kSuccess = 0,
+  kErrorNoMatchingFile = 1,
+  kErrorInputPathConversion = 2,
+  kErrorOutputPathConversion = 3,
+};
+
+// Expand wildcards in a path pattern, feed the result to a consumer function.
+//
+// `path` must be a valid, Windows-style path. It may be absolute, or relative
+// to the current working directory, and it may contain wildcards ("*" and "?")
+// in the last path segment. This function passes all matching file names to
+// `consume`. The resulting paths may not be absolute nor normalized.
+//
+// The function returns a value from `ExpandWildcardsResult`.
+PROTOBUF_EXPORT ExpandWildcardsResult ExpandWildcards(
+    const std::string& path, std::function<void(const std::string&)> consume);
+
+namespace strings {
+
+// Convert from UTF-16 to Active-Code-Page-encoded or to UTF-8-encoded text.
+PROTOBUF_EXPORT bool wcs_to_mbs(const wchar_t* s, std::string* out,
+                                bool outUtf8);
+
+// Convert from Active-Code-Page-encoded or UTF-8-encoded text to UTF-16.
+PROTOBUF_EXPORT bool mbs_to_wcs(const char* s, std::wstring* out, bool inUtf8);
+
+// Convert from UTF-8-encoded text to UTF-16.
+PROTOBUF_EXPORT bool utf8_to_wcs(const char* input, std::wstring* out);
+
+// Convert from UTF-16-encoded text to UTF-8.
+PROTOBUF_EXPORT bool wcs_to_utf8(const wchar_t* input, std::string* out);
+
+}  // namespace strings
+
+}  // namespace win32
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#ifndef W_OK
+#define W_OK 02  // not defined by MSVC for whatever reason
+#endif
+
+#ifndef F_OK
+#define F_OK 00  // not defined by MSVC for whatever reason
+#endif
+
+#ifndef STDIN_FILENO
+#define STDIN_FILENO 0
+#endif
+
+#ifndef STDOUT_FILENO
+#define STDOUT_FILENO 1
+#endif
+
+#include <google/protobuf/port_undef.inc>
+
+#endif  // defined(_WIN32)
+
+#endif  // GOOGLE_PROTOBUF_IO_IO_WIN32_H__

ubuntu/google/protobuf/io/io_win32_unittest.cc

@@ -0,0 +1,631 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: laszlocsomor@google.com (Laszlo Csomor)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+// Unit tests for long-path-aware open/mkdir/access/etc. on Windows, as well as
+// for the supporting utility functions.
+//
+// This file is only used on Windows, it's empty on other platforms.
+
+#if defined(_WIN32)
+
+#define WIN32_LEAN_AND_MEAN
+#include <google/protobuf/io/io_win32.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <wchar.h>
+#include <windows.h>
+
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+namespace google {
+namespace protobuf {
+namespace io {
+namespace win32 {
+namespace {
+
+const char kUtf8Text[] = {
+    'h', 'i', ' ',
+    // utf-8: 11010000 10011111, utf-16: 100 0001 1111 = 0x041F
+    static_cast<char>(0xd0), static_cast<char>(0x9f),
+    // utf-8: 11010001 10000000, utf-16: 100 0100 0000 = 0x0440
+    static_cast<char>(0xd1), static_cast<char>(0x80),
+    // utf-8: 11010000 10111000, utf-16: 100 0011 1000 = 0x0438
+    static_cast<char>(0xd0), static_cast<char>(0xb8),
+    // utf-8: 11010000 10110010, utf-16: 100 0011 0010 = 0x0432
+    static_cast<char>(0xd0), static_cast<char>(0xb2),
+    // utf-8: 11010000 10110101, utf-16: 100 0011 0101 = 0x0435
+    static_cast<char>(0xd0), static_cast<char>(0xb5),
+    // utf-8: 11010001 10000010, utf-16: 100 0100 0010 = 0x0442
+    static_cast<char>(0xd1), static_cast<char>(0x82), 0
+};
+
+const wchar_t kUtf16Text[] = {
+  L'h', L'i', L' ',
+  L'\x41f', L'\x440', L'\x438', L'\x432', L'\x435', L'\x442', 0
+};
+
+using std::string;
+using std::vector;
+using std::wstring;
+
+class IoWin32Test : public ::testing::Test {
+ public:
+  void SetUp();
+  void TearDown();
+
+ protected:
+  bool CreateAllUnder(wstring path);
+  bool DeleteAllUnder(wstring path);
+
+  WCHAR working_directory[MAX_PATH];
+  string test_tmpdir;
+  wstring wtest_tmpdir;
+};
+
+#define ASSERT_INITIALIZED              \
+  {                                     \
+    EXPECT_FALSE(test_tmpdir.empty());  \
+    EXPECT_FALSE(wtest_tmpdir.empty()); \
+  }
+
+namespace {
+void StripTrailingSlashes(string* str) {
+  int i = str->size() - 1;
+  for (; i >= 0 && ((*str)[i] == '/' || (*str)[i] == '\\'); --i) {}
+  str->resize(i+1);
+}
+
+bool GetEnvVarAsUtf8(const WCHAR* name, string* result) {
+  DWORD size = ::GetEnvironmentVariableW(name, nullptr, 0);
+  if (size > 0 && GetLastError() != ERROR_ENVVAR_NOT_FOUND) {
+    std::unique_ptr<WCHAR[]> wcs(new WCHAR[size]);
+    ::GetEnvironmentVariableW(name, wcs.get(), size);
+    // GetEnvironmentVariableA retrieves an Active-Code-Page-encoded text which
+    // we'd first need to convert to UTF-16 then to UTF-8, because there seems
+    // to be no API function to do that conversion directly.
+    // GetEnvironmentVariableW retrieves an UTF-16-encoded text, which we need
+    // to convert to UTF-8.
+    return strings::wcs_to_utf8(wcs.get(), result);
+  } else {
+    return false;
+  }
+}
+
+bool GetCwdAsUtf8(string* result) {
+  DWORD size = ::GetCurrentDirectoryW(0, nullptr);
+  if (size > 0) {
+    std::unique_ptr<WCHAR[]> wcs(new WCHAR[size]);
+    ::GetCurrentDirectoryW(size, wcs.get());
+    // GetCurrentDirectoryA retrieves an Active-Code-Page-encoded text which
+    // we'd first need to convert to UTF-16 then to UTF-8, because there seems
+    // to be no API function to do that conversion directly.
+    // GetCurrentDirectoryW retrieves an UTF-16-encoded text, which we need
+    // to convert to UTF-8.
+    return strings::wcs_to_utf8(wcs.get(), result);
+  } else {
+    return false;
+  }
+}
+
+bool CreateEmptyFile(const wstring& path) {
+  HANDLE h = CreateFileW(path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS,
+                         FILE_ATTRIBUTE_NORMAL, NULL);
+  if (h == INVALID_HANDLE_VALUE) {
+    return false;
+  }
+  CloseHandle(h);
+  return true;
+}
+
+}  // namespace
+
+void IoWin32Test::SetUp() {
+  test_tmpdir.clear();
+  wtest_tmpdir.clear();
+  DWORD size = ::GetCurrentDirectoryW(MAX_PATH, working_directory);
+  EXPECT_GT(size, 0);
+  EXPECT_LT(size, MAX_PATH);
+
+  string tmp;
+  bool ok = false;
+  if (!ok) {
+    // Bazel sets this environment variable when it runs tests.
+    ok = GetEnvVarAsUtf8(L"TEST_TMPDIR", &tmp);
+  }
+  if (!ok) {
+    // Bazel 0.8.0 sets this environment for every build and test action.
+    ok = GetEnvVarAsUtf8(L"TEMP", &tmp);
+  }
+  if (!ok) {
+    // Bazel 0.8.0 sets this environment for every build and test action.
+    ok = GetEnvVarAsUtf8(L"TMP", &tmp);
+  }
+  if (!ok) {
+    // Fall back to using the current directory.
+    ok = GetCwdAsUtf8(&tmp);
+  }
+  if (!ok || tmp.empty()) {
+    FAIL() << "Cannot find a temp directory.";
+  }
+
+  StripTrailingSlashes(&tmp);
+  std::stringstream result;
+  // Deleting files and directories is asynchronous on Windows, and if TearDown
+  // just deleted the previous temp directory, sometimes we cannot recreate the
+  // same directory.
+  // Use a counter so every test method gets its own temp directory.
+  static unsigned int counter = 0;
+  result << tmp << "\\w32tst" << counter++ << ".tmp";
+  test_tmpdir = result.str();
+  wtest_tmpdir = testonly_utf8_to_winpath(test_tmpdir.c_str());
+  ASSERT_FALSE(wtest_tmpdir.empty());
+  ASSERT_TRUE(DeleteAllUnder(wtest_tmpdir));
+  ASSERT_TRUE(CreateAllUnder(wtest_tmpdir));
+}
+
+void IoWin32Test::TearDown() {
+  if (!wtest_tmpdir.empty()) {
+    DeleteAllUnder(wtest_tmpdir);
+  }
+  ::SetCurrentDirectoryW(working_directory);
+}
+
+bool IoWin32Test::CreateAllUnder(wstring path) {
+  // Prepend UNC prefix if the path doesn't have it already. Don't bother
+  // checking if the path is shorter than MAX_PATH, let's just do it
+  // unconditionally.
+  if (path.find(L"\\\\?\\") != 0) {
+    path = wstring(L"\\\\?\\") + path;
+  }
+  if (::CreateDirectoryW(path.c_str(), nullptr) ||
+      GetLastError() == ERROR_ALREADY_EXISTS ||
+      GetLastError() == ERROR_ACCESS_DENIED) {
+    return true;
+  }
+  if (GetLastError() == ERROR_PATH_NOT_FOUND) {
+    size_t pos = path.find_last_of(L'\\');
+    if (pos != wstring::npos) {
+      wstring parent(path, 0, pos);
+      if (CreateAllUnder(parent) && CreateDirectoryW(path.c_str(), nullptr)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+bool IoWin32Test::DeleteAllUnder(wstring path) {
+  static const wstring kDot(L".");
+  static const wstring kDotDot(L"..");
+
+  // Prepend UNC prefix if the path doesn't have it already. Don't bother
+  // checking if the path is shorter than MAX_PATH, let's just do it
+  // unconditionally.
+  if (path.find(L"\\\\?\\") != 0) {
+    path = wstring(L"\\\\?\\") + path;
+  }
+  // Append "\" if necessary.
+  if (path[path.size() - 1] != L'\\') {
+    path.push_back(L'\\');
+  }
+
+  WIN32_FIND_DATAW metadata;
+  HANDLE handle = ::FindFirstFileW((path + L"*").c_str(), &metadata);
+  if (handle == INVALID_HANDLE_VALUE) {
+    return true;  // directory doesn't exist
+  }
+
+  bool result = true;
+  do {
+    wstring childname = metadata.cFileName;
+    if (kDot != childname && kDotDot != childname) {
+      wstring childpath = path + childname;
+      if ((metadata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) {
+        // If this is not a junction, delete its contents recursively.
+        // Finally delete this directory/junction too.
+        if (((metadata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0 &&
+             !DeleteAllUnder(childpath)) ||
+            !::RemoveDirectoryW(childpath.c_str())) {
+          result = false;
+          break;
+        }
+      } else {
+        if (!::DeleteFileW(childpath.c_str())) {
+          result = false;
+          break;
+        }
+      }
+    }
+  } while (::FindNextFileW(handle, &metadata));
+  ::FindClose(handle);
+  return result;
+}
+
+TEST_F(IoWin32Test, AccessTest) {
+  ASSERT_INITIALIZED;
+
+  string path = test_tmpdir;
+  while (path.size() < MAX_PATH - 30) {
+    path += "\\accesstest";
+    EXPECT_EQ(mkdir(path.c_str(), 0644), 0);
+  }
+  string file = path + "\\file.txt";
+  int fd = open(file.c_str(), O_CREAT | O_WRONLY, 0644);
+  if (fd > 0) {
+    EXPECT_EQ(close(fd), 0);
+  } else {
+    EXPECT_TRUE(false);
+  }
+
+  EXPECT_EQ(access(test_tmpdir.c_str(), F_OK), 0);
+  EXPECT_EQ(access(path.c_str(), F_OK), 0);
+  EXPECT_EQ(access(path.c_str(), W_OK), 0);
+  EXPECT_EQ(access(file.c_str(), F_OK | W_OK), 0);
+  EXPECT_NE(access((file + ".blah").c_str(), F_OK), 0);
+  EXPECT_NE(access((file + ".blah").c_str(), W_OK), 0);
+
+  EXPECT_EQ(access(".", F_OK), 0);
+  EXPECT_EQ(access(".", W_OK), 0);
+  EXPECT_EQ(access((test_tmpdir + "/accesstest").c_str(), F_OK | W_OK), 0);
+  ASSERT_EQ(access((test_tmpdir + "/./normalize_me/.././accesstest").c_str(),
+                   F_OK | W_OK),
+            0);
+  EXPECT_NE(access("io_win32_unittest.AccessTest.nonexistent", F_OK), 0);
+  EXPECT_NE(access("io_win32_unittest.AccessTest.nonexistent", W_OK), 0);
+
+  ASSERT_EQ(access("c:bad", F_OK), -1);
+  ASSERT_EQ(errno, ENOENT);
+  ASSERT_EQ(access("/tmp/bad", F_OK), -1);
+  ASSERT_EQ(errno, ENOENT);
+  ASSERT_EQ(access("\\bad", F_OK), -1);
+  ASSERT_EQ(errno, ENOENT);
+}
+
+TEST_F(IoWin32Test, OpenTest) {
+  ASSERT_INITIALIZED;
+
+  string path = test_tmpdir;
+  while (path.size() < MAX_PATH) {
+    path += "\\opentest";
+    EXPECT_EQ(mkdir(path.c_str(), 0644), 0);
+  }
+  string file = path + "\\file.txt";
+  int fd = open(file.c_str(), O_CREAT | O_WRONLY, 0644);
+  if (fd > 0) {
+    EXPECT_EQ(write(fd, "hello", 5), 5);
+    EXPECT_EQ(close(fd), 0);
+  } else {
+    EXPECT_TRUE(false);
+  }
+
+  ASSERT_EQ(open("c:bad.txt", O_CREAT | O_WRONLY, 0644), -1);
+  ASSERT_EQ(errno, ENOENT);
+  ASSERT_EQ(open("/tmp/bad.txt", O_CREAT | O_WRONLY, 0644), -1);
+  ASSERT_EQ(errno, ENOENT);
+  ASSERT_EQ(open("\\bad.txt", O_CREAT | O_WRONLY, 0644), -1);
+  ASSERT_EQ(errno, ENOENT);
+}
+
+TEST_F(IoWin32Test, MkdirTest) {
+  ASSERT_INITIALIZED;
+
+  string path = test_tmpdir;
+  do {
+    path += "\\mkdirtest";
+    ASSERT_EQ(mkdir(path.c_str(), 0644), 0);
+  } while (path.size() <= MAX_PATH);
+
+  ASSERT_EQ(mkdir("c:bad", 0644), -1);
+  ASSERT_EQ(errno, ENOENT);
+  ASSERT_EQ(mkdir("/tmp/bad", 0644), -1);
+  ASSERT_EQ(errno, ENOENT);
+  ASSERT_EQ(mkdir("\\bad", 0644), -1);
+  ASSERT_EQ(errno, ENOENT);
+}
+
+TEST_F(IoWin32Test, MkdirTestNonAscii) {
+  ASSERT_INITIALIZED;
+
+  // Create a non-ASCII path.
+  // Ensure that we can create the directory using CreateDirectoryW.
+  EXPECT_TRUE(CreateDirectoryW((wtest_tmpdir + L"\\1").c_str(), nullptr));
+  EXPECT_TRUE(CreateDirectoryW((wtest_tmpdir + L"\\1\\" + kUtf16Text).c_str(), nullptr));
+  // Ensure that we can create a very similarly named directory using mkdir.
+  // We don't attempt to delete and recreate the same directory, because on
+  // Windows, deleting files and directories seems to be asynchronous.
+  EXPECT_EQ(mkdir((test_tmpdir + "\\2").c_str(), 0644), 0);
+  EXPECT_EQ(mkdir((test_tmpdir + "\\2\\" + kUtf8Text).c_str(), 0644), 0);
+}
+
+TEST_F(IoWin32Test, ChdirTest) {
+  string path("C:\\");
+  EXPECT_EQ(access(path.c_str(), F_OK), 0);
+  ASSERT_EQ(chdir(path.c_str()), 0);
+
+  // Do not try to chdir into the test_tmpdir, it may already contain directory
+  // names with trailing dots.
+  // Instead test here with an obviously dot-trailed path. If the win32_chdir
+  // function would not convert the path to absolute and prefix with "\\?\" then
+  // the Win32 API would ignore the trailing dot, but because of the prefixing
+  // there'll be no path processing done, so we'll actually attempt to chdir
+  // into "C:\some\path\foo."
+  path = test_tmpdir + "/foo.";
+  EXPECT_EQ(mkdir(path.c_str(), 644), 0);
+  EXPECT_EQ(access(path.c_str(), F_OK), 0);
+  ASSERT_NE(chdir(path.c_str()), 0);
+}
+
+TEST_F(IoWin32Test, ChdirTestNonAscii) {
+  ASSERT_INITIALIZED;
+
+  // Create a directory with a non-ASCII path and ensure we can cd into it.
+  wstring wNonAscii(wtest_tmpdir + L"\\" + kUtf16Text);
+  string nonAscii;
+  EXPECT_TRUE(strings::wcs_to_utf8(wNonAscii.c_str(), &nonAscii));
+  EXPECT_TRUE(CreateDirectoryW(wNonAscii.c_str(), nullptr));
+  WCHAR cwd[MAX_PATH];
+  EXPECT_TRUE(GetCurrentDirectoryW(MAX_PATH, cwd));
+  // Ensure that we can cd into the path using SetCurrentDirectoryW.
+  EXPECT_TRUE(SetCurrentDirectoryW(wNonAscii.c_str()));
+  EXPECT_TRUE(SetCurrentDirectoryW(cwd));
+  // Ensure that we can cd into the path using chdir.
+  ASSERT_EQ(chdir(nonAscii.c_str()), 0);
+  // Ensure that the GetCurrentDirectoryW returns the desired path.
+  EXPECT_TRUE(GetCurrentDirectoryW(MAX_PATH, cwd));
+  ASSERT_EQ(wNonAscii, cwd);
+}
+
+TEST_F(IoWin32Test, ExpandWildcardsInRelativePathTest) {
+  wstring wNonAscii(wtest_tmpdir + L"\\" + kUtf16Text);
+  EXPECT_TRUE(CreateDirectoryW(wNonAscii.c_str(), nullptr));
+  // Create mock files we will test pattern matching on.
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_a.proto"));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_b.proto"));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\bar.proto"));
+  // `cd` into `wtest_tmpdir`.
+  EXPECT_TRUE(SetCurrentDirectoryW(wtest_tmpdir.c_str()));
+
+  int found_a = 0;
+  int found_b = 0;
+  vector<string> found_bad;
+  // Assert matching a relative path pattern. Results should also be relative.
+  ExpandWildcardsResult result =
+      ExpandWildcards(string(kUtf8Text) + "\\foo*.proto",
+                      [&found_a, &found_b, &found_bad](const string& p) {
+                        if (p == string(kUtf8Text) + "\\foo_a.proto") {
+                          found_a++;
+                        } else if (p == string(kUtf8Text) + "\\foo_b.proto") {
+                          found_b++;
+                        } else {
+                          found_bad.push_back(p);
+                        }
+                      });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  EXPECT_EQ(found_b, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+
+  // Assert matching the exact filename.
+  found_a = 0;
+  found_bad.clear();
+  result = ExpandWildcards(string(kUtf8Text) + "\\foo_a.proto",
+                           [&found_a, &found_bad](const string& p) {
+                             if (p == string(kUtf8Text) + "\\foo_a.proto") {
+                               found_a++;
+                             } else {
+                               found_bad.push_back(p);
+                             }
+                           });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+}
+
+TEST_F(IoWin32Test, ExpandWildcardsInAbsolutePathTest) {
+  wstring wNonAscii(wtest_tmpdir + L"\\" + kUtf16Text);
+  EXPECT_TRUE(CreateDirectoryW(wNonAscii.c_str(), nullptr));
+  // Create mock files we will test pattern matching on.
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_a.proto"));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_b.proto"));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\bar.proto"));
+
+  int found_a = 0;
+  int found_b = 0;
+  vector<string> found_bad;
+  // Assert matching an absolute path. The results should also use absolute
+  // path.
+  ExpandWildcardsResult result =
+      ExpandWildcards(string(test_tmpdir) + "\\" + kUtf8Text + "\\foo*.proto",
+                      [this, &found_a, &found_b, &found_bad](const string& p) {
+                        if (p == string(this->test_tmpdir) + "\\" + kUtf8Text +
+                                     "\\foo_a.proto") {
+                          found_a++;
+                        } else if (p == string(this->test_tmpdir) + "\\" +
+                                            kUtf8Text + "\\foo_b.proto") {
+                          found_b++;
+                        } else {
+                          found_bad.push_back(p);
+                        }
+                      });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  EXPECT_EQ(found_b, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+
+  // Assert matching the exact filename.
+  found_a = 0;
+  found_bad.clear();
+  result =
+      ExpandWildcards(string(test_tmpdir) + "\\" + kUtf8Text + "\\foo_a.proto",
+                      [this, &found_a, &found_bad](const string& p) {
+                        if (p == string(this->test_tmpdir) + "\\" + kUtf8Text +
+                                     "\\foo_a.proto") {
+                          found_a++;
+                        } else {
+                          found_bad.push_back(p);
+                        }
+                      });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+}
+
+TEST_F(IoWin32Test, ExpandWildcardsIgnoresDirectoriesTest) {
+  wstring wNonAscii(wtest_tmpdir + L"\\" + kUtf16Text);
+  EXPECT_TRUE(CreateDirectoryW(wNonAscii.c_str(), nullptr));
+  // Create mock files we will test pattern matching on.
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_a.proto"));
+  EXPECT_TRUE(
+      CreateDirectoryW((wNonAscii + L"\\foo_b.proto").c_str(), nullptr));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_c.proto"));
+  // `cd` into `wtest_tmpdir`.
+  EXPECT_TRUE(SetCurrentDirectoryW(wtest_tmpdir.c_str()));
+
+  int found_a = 0;
+  int found_c = 0;
+  vector<string> found_bad;
+  // Assert that the pattern matches exactly the expected files, and using the
+  // absolute path as did the input pattern.
+  ExpandWildcardsResult result =
+      ExpandWildcards(string(kUtf8Text) + "\\foo*.proto",
+                      [&found_a, &found_c, &found_bad](const string& p) {
+                        if (p == string(kUtf8Text) + "\\foo_a.proto") {
+                          found_a++;
+                        } else if (p == string(kUtf8Text) + "\\foo_c.proto") {
+                          found_c++;
+                        } else {
+                          found_bad.push_back(p);
+                        }
+                      });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  EXPECT_EQ(found_c, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+}
+
+TEST_F(IoWin32Test, ExpandWildcardsFailsIfNoFileMatchesTest) {
+  wstring wNonAscii(wtest_tmpdir + L"\\" + kUtf16Text);
+  EXPECT_TRUE(CreateDirectoryW(wNonAscii.c_str(), nullptr));
+  // Create mock files we will test pattern matching on.
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_a.proto"));
+  // `cd` into `wtest_tmpdir`.
+  EXPECT_TRUE(SetCurrentDirectoryW(wtest_tmpdir.c_str()));
+
+  // Control test: should match foo*.proto
+  ExpandWildcardsResult result =
+      ExpandWildcards(string(kUtf8Text) + "\\foo*.proto", [](const string&) {});
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+
+  // Control test: should match foo_a.proto
+  result = ExpandWildcards(string(kUtf8Text) + "\\foo_a.proto",
+                           [](const string&) {});
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+
+  // Actual test: should not match anything.
+  result =
+      ExpandWildcards(string(kUtf8Text) + "\\bar*.proto", [](const string&) {});
+  ASSERT_EQ(result, ExpandWildcardsResult::kErrorNoMatchingFile);
+}
+
+TEST_F(IoWin32Test, AsWindowsPathTest) {
+  DWORD size = GetCurrentDirectoryW(0, nullptr);
+  std::unique_ptr<wchar_t[]> cwd_str(new wchar_t[size]);
+  EXPECT_GT(GetCurrentDirectoryW(size, cwd_str.get()), 0);
+  wstring cwd = wstring(L"\\\\?\\") + cwd_str.get();
+
+  ASSERT_EQ(testonly_utf8_to_winpath("relative_mkdirtest"),
+            cwd + L"\\relative_mkdirtest");
+  ASSERT_EQ(testonly_utf8_to_winpath("preserve//\\trailing///"),
+            cwd + L"\\preserve\\trailing\\");
+  ASSERT_EQ(testonly_utf8_to_winpath("./normalize_me\\/../blah"),
+            cwd + L"\\blah");
+  std::ostringstream relpath;
+  for (wchar_t* p = cwd_str.get(); *p; ++p) {
+    if (*p == '/' || *p == '\\') {
+      relpath << "../";
+    }
+  }
+  relpath << ".\\/../\\./beyond-toplevel";
+  ASSERT_EQ(testonly_utf8_to_winpath(relpath.str().c_str()),
+            wstring(L"\\\\?\\") + cwd_str.get()[0] + L":\\beyond-toplevel");
+
+  // Absolute unix paths lack drive letters, driveless absolute windows paths
+  // do too. Neither can be converted to a drive-specifying absolute Windows
+  // path.
+  ASSERT_EQ(testonly_utf8_to_winpath("/absolute/unix/path"), L"");
+  // Though valid on Windows, we also don't support UNC paths (\\UNC\\blah).
+  ASSERT_EQ(testonly_utf8_to_winpath("\\driveless\\absolute"), L"");
+  // Though valid in cmd.exe, drive-relative paths are not supported.
+  ASSERT_EQ(testonly_utf8_to_winpath("c:foo"), L"");
+  ASSERT_EQ(testonly_utf8_to_winpath("c:/foo"), L"\\\\?\\c:\\foo");
+  ASSERT_EQ(testonly_utf8_to_winpath("\\\\?\\C:\\foo"), L"\\\\?\\C:\\foo");
+}
+
+TEST_F(IoWin32Test, Utf8Utf16ConversionTest) {
+  string mbs;
+  wstring wcs;
+  ASSERT_TRUE(strings::utf8_to_wcs(kUtf8Text, &wcs));
+  ASSERT_TRUE(strings::wcs_to_utf8(kUtf16Text, &mbs));
+  ASSERT_EQ(wcs, kUtf16Text);
+  ASSERT_EQ(mbs, kUtf8Text);
+}
+
+}  // namespace
+}  // namespace win32
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // defined(_WIN32)
+

ubuntu/google/protobuf/io/package_info.h

@@ -0,0 +1,53 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// This file exists solely to document the google::protobuf::io namespace.
+// It is not compiled into anything, but it may be read by an automated
+// documentation generator.
+
+namespace google {
+namespace protobuf {
+
+// Auxiliary classes used for I/O.
+//
+// The Protocol Buffer library uses the classes in this package to deal with
+// I/O and encoding/decoding raw bytes.  Most users will not need to
+// deal with this package.  However, users who want to adapt the system to
+// work with their own I/O abstractions -- e.g., to allow Protocol Buffers
+// to be read from a different kind of input stream without the need for a
+// temporary buffer -- should take a closer look.
+namespace io {}
+
+}  // namespace protobuf
+}  // namespace google

ubuntu/google/protobuf/io/printer.cc

@@ -0,0 +1,400 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+#include <google/protobuf/io/printer.h>
+
+#include <cctype>
+
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+Printer::Printer(ZeroCopyOutputStream* output, char variable_delimiter)
+    : variable_delimiter_(variable_delimiter),
+      output_(output),
+      buffer_(NULL),
+      buffer_size_(0),
+      offset_(0),
+      at_start_of_line_(true),
+      failed_(false),
+      annotation_collector_(NULL) {}
+
+Printer::Printer(ZeroCopyOutputStream* output, char variable_delimiter,
+                 AnnotationCollector* annotation_collector)
+    : variable_delimiter_(variable_delimiter),
+      output_(output),
+      buffer_(NULL),
+      buffer_size_(0),
+      offset_(0),
+      at_start_of_line_(true),
+      failed_(false),
+      annotation_collector_(annotation_collector) {}
+
+Printer::~Printer() {
+  // Only BackUp() if we have called Next() at least once and never failed.
+  if (buffer_size_ > 0 && !failed_) {
+    output_->BackUp(buffer_size_);
+  }
+}
+
+bool Printer::GetSubstitutionRange(const char* varname,
+                                   std::pair<size_t, size_t>* range) {
+  std::map<std::string, std::pair<size_t, size_t> >::const_iterator iter =
+      substitutions_.find(varname);
+  if (iter == substitutions_.end()) {
+    GOOGLE_LOG(DFATAL) << " Undefined variable in annotation: " << varname;
+    return false;
+  }
+  if (iter->second.first > iter->second.second) {
+    GOOGLE_LOG(DFATAL) << " Variable used for annotation used multiple times: "
+                << varname;
+    return false;
+  }
+  *range = iter->second;
+  return true;
+}
+
+void Printer::Annotate(const char* begin_varname, const char* end_varname,
+                       const std::string& file_path,
+                       const std::vector<int>& path) {
+  if (annotation_collector_ == NULL) {
+    // Can't generate signatures with this Printer.
+    return;
+  }
+  std::pair<size_t, size_t> begin, end;
+  if (!GetSubstitutionRange(begin_varname, &begin) ||
+      !GetSubstitutionRange(end_varname, &end)) {
+    return;
+  }
+  if (begin.first > end.second) {
+    GOOGLE_LOG(DFATAL) << "  Annotation has negative length from " << begin_varname
+                << " to " << end_varname;
+  } else {
+    annotation_collector_->AddAnnotation(begin.first, end.second, file_path,
+                                         path);
+  }
+}
+
+void Printer::Print(const std::map<std::string, std::string>& variables,
+                    const char* text) {
+  int size = strlen(text);
+  int pos = 0;  // The number of bytes we've written so far.
+  substitutions_.clear();
+  line_start_variables_.clear();
+
+  for (int i = 0; i < size; i++) {
+    if (text[i] == '\n') {
+      // Saw newline.  If there is more text, we may need to insert an indent
+      // here.  So, write what we have so far, including the '\n'.
+      WriteRaw(text + pos, i - pos + 1);
+      pos = i + 1;
+
+      // Setting this true will cause the next WriteRaw() to insert an indent
+      // first.
+      at_start_of_line_ = true;
+      line_start_variables_.clear();
+
+    } else if (text[i] == variable_delimiter_) {
+      // Saw the start of a variable name.
+
+      // Write what we have so far.
+      WriteRaw(text + pos, i - pos);
+      pos = i + 1;
+
+      // Find closing delimiter.
+      const char* end = strchr(text + pos, variable_delimiter_);
+      if (end == NULL) {
+        GOOGLE_LOG(DFATAL) << " Unclosed variable name.";
+        end = text + pos;
+      }
+      int endpos = end - text;
+
+      std::string varname(text + pos, endpos - pos);
+      if (varname.empty()) {
+        // Two delimiters in a row reduce to a literal delimiter character.
+        WriteRaw(&variable_delimiter_, 1);
+      } else {
+        // Replace with the variable's value.
+        std::map<std::string, std::string>::const_iterator iter =
+            variables.find(varname);
+        if (iter == variables.end()) {
+          GOOGLE_LOG(DFATAL) << " Undefined variable: " << varname;
+        } else {
+          if (at_start_of_line_ && iter->second.empty()) {
+            line_start_variables_.push_back(varname);
+          }
+          WriteRaw(iter->second.data(), iter->second.size());
+          std::pair<std::map<std::string, std::pair<size_t, size_t> >::iterator,
+                    bool>
+              inserted = substitutions_.insert(std::make_pair(
+                  varname,
+                  std::make_pair(offset_ - iter->second.size(), offset_)));
+          if (!inserted.second) {
+            // This variable was used multiple times.  Make its span have
+            // negative length so we can detect it if it gets used in an
+            // annotation.
+            inserted.first->second = std::make_pair(1, 0);
+          }
+        }
+      }
+
+      // Advance past this variable.
+      i = endpos;
+      pos = endpos + 1;
+    }
+  }
+
+  // Write the rest.
+  WriteRaw(text + pos, size - pos);
+}
+
+void Printer::Indent() { indent_ += "  "; }
+
+void Printer::Outdent() {
+  if (indent_.empty()) {
+    GOOGLE_LOG(DFATAL) << " Outdent() without matching Indent().";
+    return;
+  }
+
+  indent_.resize(indent_.size() - 2);
+}
+
+void Printer::PrintRaw(const std::string& data) {
+  WriteRaw(data.data(), data.size());
+}
+
+void Printer::PrintRaw(const char* data) {
+  if (failed_) return;
+  WriteRaw(data, strlen(data));
+}
+
+void Printer::WriteRaw(const char* data, int size) {
+  if (failed_) return;
+  if (size == 0) return;
+
+  if (at_start_of_line_ && (size > 0) && (data[0] != '\n')) {
+    // Insert an indent.
+    at_start_of_line_ = false;
+    CopyToBuffer(indent_.data(), indent_.size());
+    if (failed_) return;
+    // Fix up empty variables (e.g., "{") that should be annotated as
+    // coming after the indent.
+    for (std::vector<std::string>::iterator i = line_start_variables_.begin();
+         i != line_start_variables_.end(); ++i) {
+      substitutions_[*i].first += indent_.size();
+      substitutions_[*i].second += indent_.size();
+    }
+  }
+
+  // If we're going to write any data, clear line_start_variables_, since
+  // we've either updated them in the block above or they no longer refer to
+  // the current line.
+  line_start_variables_.clear();
+
+  CopyToBuffer(data, size);
+}
+
+bool Printer::Next() {
+  do {
+    void* void_buffer;
+    if (!output_->Next(&void_buffer, &buffer_size_)) {
+      failed_ = true;
+      return false;
+    }
+    buffer_ = reinterpret_cast<char*>(void_buffer);
+  } while (buffer_size_ == 0);
+  return true;
+}
+
+void Printer::CopyToBuffer(const char* data, int size) {
+  if (failed_) return;
+  if (size == 0) return;
+
+  while (size > buffer_size_) {
+    // Data exceeds space in the buffer.  Copy what we can and request a
+    // new buffer.
+    if (buffer_size_ > 0) {
+      memcpy(buffer_, data, buffer_size_);
+      offset_ += buffer_size_;
+      data += buffer_size_;
+      size -= buffer_size_;
+    }
+    void* void_buffer;
+    failed_ = !output_->Next(&void_buffer, &buffer_size_);
+    if (failed_) return;
+    buffer_ = reinterpret_cast<char*>(void_buffer);
+  }
+
+  // Buffer is big enough to receive the data; copy it.
+  memcpy(buffer_, data, size);
+  buffer_ += size;
+  buffer_size_ -= size;
+  offset_ += size;
+}
+
+void Printer::IndentIfAtStart() {
+  if (at_start_of_line_) {
+    CopyToBuffer(indent_.data(), indent_.size());
+    at_start_of_line_ = false;
+  }
+}
+
+void Printer::FormatInternal(const std::vector<std::string>& args,
+                             const std::map<std::string, std::string>& vars,
+                             const char* format) {
+  auto save = format;
+  int arg_index = 0;
+  std::vector<AnnotationCollector::Annotation> annotations;
+  while (*format) {
+    char c = *format++;
+    switch (c) {
+      case '$':
+        format = WriteVariable(args, vars, format, &arg_index, &annotations);
+        continue;
+      case '\n':
+        at_start_of_line_ = true;
+        line_start_variables_.clear();
+        break;
+      default:
+        IndentIfAtStart();
+        break;
+    }
+    push_back(c);
+  }
+  if (arg_index != args.size()) {
+    GOOGLE_LOG(FATAL) << " Unused arguments. " << save;
+  }
+  if (!annotations.empty()) {
+    GOOGLE_LOG(FATAL) << " Annotation range is not-closed, expect $}$. " << save;
+  }
+}
+
+const char* Printer::WriteVariable(
+    const std::vector<std::string>& args,
+    const std::map<std::string, std::string>& vars, const char* format,
+    int* arg_index, std::vector<AnnotationCollector::Annotation>* annotations) {
+  auto start = format;
+  auto end = strchr(format, '$');
+  if (!end) {
+    GOOGLE_LOG(FATAL) << " Unclosed variable name.";
+  }
+  format = end + 1;
+  if (end == start) {
+    // "$$" is an escape for just '$'
+    IndentIfAtStart();
+    push_back('$');
+    return format;
+  }
+  if (*start == '{') {
+    GOOGLE_CHECK(std::isdigit(start[1]));
+    GOOGLE_CHECK_EQ(end - start, 2);
+    int idx = start[1] - '1';
+    if (idx < 0 || idx >= args.size()) {
+      GOOGLE_LOG(FATAL) << "Annotation ${" << idx + 1 << "$ is out of bounds.";
+    }
+    if (idx > *arg_index) {
+      GOOGLE_LOG(FATAL) << "Annotation arg must be in correct order as given. Expected"
+                 << " ${" << (*arg_index) + 1 << "$ got ${" << idx + 1 << "$.";
+    } else if (idx == *arg_index) {
+      (*arg_index)++;
+    }
+    IndentIfAtStart();
+    annotations->push_back({{offset_, 0}, args[idx]});
+    return format;
+  } else if (*start == '}') {
+    GOOGLE_CHECK(annotations);
+    if (annotations->empty()) {
+      GOOGLE_LOG(FATAL) << "Unexpected end of annotation found.";
+    }
+    auto& a = annotations->back();
+    a.first.second = offset_;
+    if (annotation_collector_) annotation_collector_->AddAnnotationNew(a);
+    annotations->pop_back();
+    return format;
+  }
+  auto start_var = start;
+  while (start_var < end && *start_var == ' ') start_var++;
+  if (start_var == end) {
+    GOOGLE_LOG(FATAL) << " Empty variable.";
+  }
+  auto end_var = end;
+  while (start_var < end_var && *(end_var - 1) == ' ') end_var--;
+  std::string var_name{
+      start_var, static_cast<std::string::size_type>(end_var - start_var)};
+  std::string sub;
+  if (std::isdigit(var_name[0])) {
+    GOOGLE_CHECK_EQ(var_name.size(), 1);  // No need for multi-digits
+    int idx = var_name[0] - '1';   // Start counting at 1
+    GOOGLE_CHECK_GE(idx, 0);
+    if (idx >= args.size()) {
+      GOOGLE_LOG(FATAL) << "Argument $" << idx + 1 << "$ is out of bounds.";
+    }
+    if (idx > *arg_index) {
+      GOOGLE_LOG(FATAL) << "Arguments must be used in same order as given. Expected $"
+                 << (*arg_index) + 1 << "$ got $" << idx + 1 << "$.";
+    } else if (idx == *arg_index) {
+      (*arg_index)++;
+    }
+    sub = args[idx];
+  } else {
+    auto it = vars.find(var_name);
+    if (it == vars.end()) {
+      GOOGLE_LOG(FATAL) << " Unknown variable: " << var_name << ".";
+    }
+    sub = it->second;
+  }
+
+  // By returning here in case of empty we also skip possible spaces inside
+  // the $...$, i.e. "void$ dllexpor$ f();" -> "void f();" in the empty case.
+  if (sub.empty()) return format;
+
+  // We're going to write something non-empty so we need a possible indent.
+  IndentIfAtStart();
+
+  // Write the possible spaces in front.
+  CopyToBuffer(start, start_var - start);
+  // Write a non-empty substituted variable.
+  CopyToBuffer(sub.c_str(), sub.size());
+  // Finish off with writing possible trailing spaces.
+  CopyToBuffer(end_var, end - end_var);
+  return format;
+}
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google

ubuntu/google/protobuf/io/printer.h

@@ -0,0 +1,385 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// Utility class for writing text to a ZeroCopyOutputStream.
+
+#ifndef GOOGLE_PROTOBUF_IO_PRINTER_H__
+#define GOOGLE_PROTOBUF_IO_PRINTER_H__
+
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/port_def.inc>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+class ZeroCopyOutputStream;  // zero_copy_stream.h
+
+// Records annotations about a Printer's output.
+class PROTOBUF_EXPORT AnnotationCollector {
+ public:
+  // Annotation is a offset range and a payload pair.
+  typedef std::pair<std::pair<size_t, size_t>, std::string> Annotation;
+
+  // Records that the bytes in file_path beginning with begin_offset and ending
+  // before end_offset are associated with the SourceCodeInfo-style path.
+  virtual void AddAnnotation(size_t begin_offset, size_t end_offset,
+                             const std::string& file_path,
+                             const std::vector<int>& path) = 0;
+
+  // TODO(gerbens) I don't see why we need virtuals here. Just a vector of
+  // range, payload pairs stored in a context should suffice.
+  virtual void AddAnnotationNew(Annotation& a) {}
+
+  virtual ~AnnotationCollector() {}
+};
+
+// Records annotations about a Printer's output to the given protocol buffer,
+// assuming that the buffer has an ::Annotation message exposing path,
+// source_file, begin and end fields.
+template <typename AnnotationProto>
+class AnnotationProtoCollector : public AnnotationCollector {
+ public:
+  // annotation_proto is the protocol buffer to which new Annotations should be
+  // added. It is not owned by the AnnotationProtoCollector.
+  explicit AnnotationProtoCollector(AnnotationProto* annotation_proto)
+      : annotation_proto_(annotation_proto) {}
+
+  // Override for AnnotationCollector::AddAnnotation.
+  virtual void AddAnnotation(size_t begin_offset, size_t end_offset,
+                             const std::string& file_path,
+                             const std::vector<int>& path) {
+    typename AnnotationProto::Annotation* annotation =
+        annotation_proto_->add_annotation();
+    for (int i = 0; i < path.size(); ++i) {
+      annotation->add_path(path[i]);
+    }
+    annotation->set_source_file(file_path);
+    annotation->set_begin(begin_offset);
+    annotation->set_end(end_offset);
+  }
+  // Override for AnnotationCollector::AddAnnotation.
+  virtual void AddAnnotationNew(Annotation& a) {
+    auto* annotation = annotation_proto_->add_annotation();
+    annotation->ParseFromString(a.second);
+    annotation->set_begin(a.first.first);
+    annotation->set_end(a.first.second);
+  }
+
+ private:
+  // The protocol buffer to which new annotations should be added.
+  AnnotationProto* const annotation_proto_;
+};
+
+// This simple utility class assists in code generation.  It basically
+// allows the caller to define a set of variables and then output some
+// text with variable substitutions.  Example usage:
+//
+//   Printer printer(output, '$');
+//   map<string, string> vars;
+//   vars["name"] = "Bob";
+//   printer.Print(vars, "My name is $name$.");
+//
+// The above writes "My name is Bob." to the output stream.
+//
+// Printer aggressively enforces correct usage, crashing (with assert failures)
+// in the case of undefined variables in debug builds. This helps greatly in
+// debugging code which uses it.
+//
+// If a Printer is constructed with an AnnotationCollector, it will provide it
+// with annotations that connect the Printer's output to paths that can identify
+// various descriptors.  In the above example, if person_ is a descriptor that
+// identifies Bob, we can associate the output string "My name is Bob." with
+// a source path pointing to that descriptor with:
+//
+//   printer.Annotate("name", person_);
+//
+// The AnnotationCollector will be sent an annotation linking the output range
+// covering "Bob" to the logical path provided by person_.  Tools may use
+// this association to (for example) link "Bob" in the output back to the
+// source file that defined the person_ descriptor identifying Bob.
+//
+// Annotate can only examine variables substituted during the last call to
+// Print.  It is invalid to refer to a variable that was used multiple times
+// in a single Print call.
+//
+// In full generality, one may specify a range of output text using a beginning
+// substitution variable and an ending variable.  The resulting annotation will
+// span from the first character of the substituted value for the beginning
+// variable to the last character of the substituted value for the ending
+// variable.  For example, the Annotate call above is equivalent to this one:
+//
+//   printer.Annotate("name", "name", person_);
+//
+// This is useful if multiple variables combine to form a single span of output
+// that should be annotated with the same source path.  For example:
+//
+//   Printer printer(output, '$');
+//   map<string, string> vars;
+//   vars["first"] = "Alice";
+//   vars["last"] = "Smith";
+//   printer.Print(vars, "My name is $first$ $last$.");
+//   printer.Annotate("first", "last", person_);
+//
+// This code would associate the span covering "Alice Smith" in the output with
+// the person_ descriptor.
+//
+// Note that the beginning variable must come before (or overlap with, in the
+// case of zero-sized substitution values) the ending variable.
+//
+// It is also sometimes useful to use variables with zero-sized values as
+// markers.  This avoids issues with multiple references to the same variable
+// and also allows annotation ranges to span literal text from the Print
+// templates:
+//
+//   Printer printer(output, '$');
+//   map<string, string> vars;
+//   vars["foo"] = "bar";
+//   vars["function"] = "call";
+//   vars["mark"] = "";
+//   printer.Print(vars, "$function$($foo$,$foo$)$mark$");
+//   printer.Annotate("function", "mark", call_);
+//
+// This code associates the span covering "call(bar,bar)" in the output with the
+// call_ descriptor.
+
+class PROTOBUF_EXPORT Printer {
+ public:
+  // Create a printer that writes text to the given output stream.  Use the
+  // given character as the delimiter for variables.
+  Printer(ZeroCopyOutputStream* output, char variable_delimiter);
+
+  // Create a printer that writes text to the given output stream.  Use the
+  // given character as the delimiter for variables.  If annotation_collector
+  // is not null, Printer will provide it with annotations about code written
+  // to the stream.  annotation_collector is not owned by Printer.
+  Printer(ZeroCopyOutputStream* output, char variable_delimiter,
+          AnnotationCollector* annotation_collector);
+
+  ~Printer();
+
+  // Link a substitution variable emitted by the last call to Print to the
+  // object described by descriptor.
+  template <typename SomeDescriptor>
+  void Annotate(const char* varname, const SomeDescriptor* descriptor) {
+    Annotate(varname, varname, descriptor);
+  }
+
+  // Link the output range defined by the substitution variables as emitted by
+  // the last call to Print to the object described by descriptor. The range
+  // begins at begin_varname's value and ends after the last character of the
+  // value substituted for end_varname.
+  template <typename SomeDescriptor>
+  void Annotate(const char* begin_varname, const char* end_varname,
+                const SomeDescriptor* descriptor) {
+    if (annotation_collector_ == NULL) {
+      // Annotations aren't turned on for this Printer, so don't pay the cost
+      // of building the location path.
+      return;
+    }
+    std::vector<int> path;
+    descriptor->GetLocationPath(&path);
+    Annotate(begin_varname, end_varname, descriptor->file()->name(), path);
+  }
+
+  // Link a substitution variable emitted by the last call to Print to the file
+  // with path file_name.
+  void Annotate(const char* varname, const std::string& file_name) {
+    Annotate(varname, varname, file_name);
+  }
+
+  // Link the output range defined by the substitution variables as emitted by
+  // the last call to Print to the file with path file_name. The range begins
+  // at begin_varname's value and ends after the last character of the value
+  // substituted for end_varname.
+  void Annotate(const char* begin_varname, const char* end_varname,
+                const std::string& file_name) {
+    if (annotation_collector_ == NULL) {
+      // Annotations aren't turned on for this Printer.
+      return;
+    }
+    std::vector<int> empty_path;
+    Annotate(begin_varname, end_varname, file_name, empty_path);
+  }
+
+  // Print some text after applying variable substitutions.  If a particular
+  // variable in the text is not defined, this will crash.  Variables to be
+  // substituted are identified by their names surrounded by delimiter
+  // characters (as given to the constructor).  The variable bindings are
+  // defined by the given map.
+  void Print(const std::map<std::string, std::string>& variables,
+             const char* text);
+
+  // Like the first Print(), except the substitutions are given as parameters.
+  template <typename... Args>
+  void Print(const char* text, const Args&... args) {
+    std::map<std::string, std::string> vars;
+    PrintInternal(&vars, text, args...);
+  }
+
+  // Indent text by two spaces.  After calling Indent(), two spaces will be
+  // inserted at the beginning of each line of text.  Indent() may be called
+  // multiple times to produce deeper indents.
+  void Indent();
+
+  // Reduces the current indent level by two spaces, or crashes if the indent
+  // level is zero.
+  void Outdent();
+
+  // Write a string to the output buffer.
+  // This method does not look for newlines to add indentation.
+  void PrintRaw(const std::string& data);
+
+  // Write a zero-delimited string to output buffer.
+  // This method does not look for newlines to add indentation.
+  void PrintRaw(const char* data);
+
+  // Write some bytes to the output buffer.
+  // This method does not look for newlines to add indentation.
+  void WriteRaw(const char* data, int size);
+
+  // FormatInternal is a helper function not meant to use directly, use
+  // compiler::cpp::Formatter instead. This function is meant to support
+  // formatting text using named variables (eq. "$foo$) from a lookup map (vars)
+  // and variables directly supplied by arguments (eq "$1$" meaning first
+  // argument which is the zero index element of args).
+  void FormatInternal(const std::vector<std::string>& args,
+                      const std::map<std::string, std::string>& vars,
+                      const char* format);
+
+  // True if any write to the underlying stream failed.  (We don't just
+  // crash in this case because this is an I/O failure, not a programming
+  // error.)
+  bool failed() const { return failed_; }
+
+ private:
+  // Link the output range defined by the substitution variables as emitted by
+  // the last call to Print to the object found at the SourceCodeInfo-style path
+  // in a file with path file_path. The range begins at the start of
+  // begin_varname's value and ends after the last character of the value
+  // substituted for end_varname. Note that begin_varname and end_varname
+  // may refer to the same variable.
+  void Annotate(const char* begin_varname, const char* end_varname,
+                const std::string& file_path, const std::vector<int>& path);
+
+  // Base case
+  void PrintInternal(std::map<std::string, std::string>* vars,
+                     const char* text) {
+    Print(*vars, text);
+  }
+
+  template <typename... Args>
+  void PrintInternal(std::map<std::string, std::string>* vars, const char* text,
+                     const char* key, const std::string& value,
+                     const Args&... args) {
+    (*vars)[key] = value;
+    PrintInternal(vars, text, args...);
+  }
+
+  // Copy size worth of bytes from data to buffer_.
+  void CopyToBuffer(const char* data, int size);
+
+  void push_back(char c) {
+    if (failed_) return;
+    if (buffer_size_ == 0) {
+      if (!Next()) return;
+    }
+    *buffer_++ = c;
+    buffer_size_--;
+    offset_++;
+  }
+
+  bool Next();
+
+  inline void IndentIfAtStart();
+  const char* WriteVariable(
+      const std::vector<std::string>& args,
+      const std::map<std::string, std::string>& vars, const char* format,
+      int* arg_index,
+      std::vector<AnnotationCollector::Annotation>* annotations);
+
+  const char variable_delimiter_;
+
+  ZeroCopyOutputStream* const output_;
+  char* buffer_;
+  int buffer_size_;
+  // The current position, in bytes, in the output stream.  This is equivalent
+  // to the total number of bytes that have been written so far.  This value is
+  // used to calculate annotation ranges in the substitutions_ map below.
+  size_t offset_;
+
+  std::string indent_;
+  bool at_start_of_line_;
+  bool failed_;
+
+  // A map from variable name to [start, end) offsets in the output buffer.
+  // These refer to the offsets used for a variable after the last call to
+  // Print.  If a variable was used more than once, the entry used in
+  // this map is set to a negative-length span.  For singly-used variables, the
+  // start offset is the beginning of the substitution; the end offset is the
+  // last byte of the substitution plus one (such that (end - start) is the
+  // length of the substituted string).
+  std::map<std::string, std::pair<size_t, size_t> > substitutions_;
+
+  // Keeps track of the keys in substitutions_ that need to be updated when
+  // indents are inserted. These are keys that refer to the beginning of the
+  // current line.
+  std::vector<std::string> line_start_variables_;
+
+  // Returns true and sets range to the substitution range in the output for
+  // varname if varname was used once in the last call to Print. If varname
+  // was not used, or if it was used multiple times, returns false (and
+  // fails a debug assertion).
+  bool GetSubstitutionRange(const char* varname,
+                            std::pair<size_t, size_t>* range);
+
+  // If non-null, annotation_collector_ is used to store annotations about
+  // generated code.
+  AnnotationCollector* const annotation_collector_;
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Printer);
+};
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
+#endif  // GOOGLE_PROTOBUF_IO_PRINTER_H__

ubuntu/google/protobuf/io/printer_unittest.cc

@@ -0,0 +1,735 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+#include <google/protobuf/io/printer.h>
+
+#include <vector>
+
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/descriptor.pb.h>
+#include <google/protobuf/testing/googletest.h>
+#include <gtest/gtest.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+// Each test repeats over several block sizes in order to test both cases
+// where particular writes cross a buffer boundary and cases where they do
+// not.
+
+TEST(Printer, EmptyPrinter) {
+  char buffer[8192];
+  const int block_size = 100;
+  ArrayOutputStream output(buffer, GOOGLE_ARRAYSIZE(buffer), block_size);
+  Printer printer(&output, '\0');
+  EXPECT_TRUE(!printer.failed());
+}
+
+TEST(Printer, BasicPrinting) {
+  char buffer[8192];
+
+  for (int block_size = 1; block_size < 512; block_size *= 2) {
+    ArrayOutputStream output(buffer, sizeof(buffer), block_size);
+
+    {
+      Printer printer(&output, '\0');
+
+      printer.Print("Hello World!");
+      printer.Print("  This is the same line.\n");
+      printer.Print("But this is a new one.\nAnd this is another one.");
+
+      EXPECT_FALSE(printer.failed());
+    }
+
+    buffer[output.ByteCount()] = '\0';
+
+    EXPECT_STREQ(
+        "Hello World!  This is the same line.\n"
+        "But this is a new one.\n"
+        "And this is another one.",
+        buffer);
+  }
+}
+
+TEST(Printer, WriteRaw) {
+  char buffer[8192];
+
+  for (int block_size = 1; block_size < 512; block_size *= 2) {
+    ArrayOutputStream output(buffer, sizeof(buffer), block_size);
+
+    {
+      std::string string_obj = "From an object\n";
+      Printer printer(&output, '$');
+      printer.WriteRaw("Hello World!", 12);
+      printer.PrintRaw("  This is the same line.\n");
+      printer.PrintRaw("But this is a new one.\nAnd this is another one.");
+      printer.WriteRaw("\n", 1);
+      printer.PrintRaw(string_obj);
+      EXPECT_FALSE(printer.failed());
+    }
+
+    buffer[output.ByteCount()] = '\0';
+
+    EXPECT_STREQ(
+        "Hello World!  This is the same line.\n"
+        "But this is a new one.\n"
+        "And this is another one."
+        "\n"
+        "From an object\n",
+        buffer);
+  }
+}
+
+TEST(Printer, VariableSubstitution) {
+  char buffer[8192];
+
+  for (int block_size = 1; block_size < 512; block_size *= 2) {
+    ArrayOutputStream output(buffer, sizeof(buffer), block_size);
+
+    {
+      Printer printer(&output, '$');
+      std::map<std::string, std::string> vars;
+
+      vars["foo"] = "World";
+      vars["bar"] = "$foo$";
+      vars["abcdefg"] = "1234";
+
+      printer.Print(vars, "Hello $foo$!\nbar = $bar$\n");
+      printer.PrintRaw("RawBit\n");
+      printer.Print(vars, "$abcdefg$\nA literal dollar sign:  $$");
+
+      vars["foo"] = "blah";
+      printer.Print(vars, "\nNow foo = $foo$.");
+
+      EXPECT_FALSE(printer.failed());
+    }
+
+    buffer[output.ByteCount()] = '\0';
+
+    EXPECT_STREQ(
+        "Hello World!\n"
+        "bar = $foo$\n"
+        "RawBit\n"
+        "1234\n"
+        "A literal dollar sign:  $\n"
+        "Now foo = blah.",
+        buffer);
+  }
+}
+
+TEST(Printer, InlineVariableSubstitution) {
+  char buffer[8192];
+
+  ArrayOutputStream output(buffer, sizeof(buffer));
+
+  {
+    Printer printer(&output, '$');
+    printer.Print("Hello $foo$!\n", "foo", "World");
+    printer.PrintRaw("RawBit\n");
+    printer.Print("$foo$ $bar$\n", "foo", "one", "bar", "two");
+    EXPECT_FALSE(printer.failed());
+  }
+
+  buffer[output.ByteCount()] = '\0';
+
+  EXPECT_STREQ(
+      "Hello World!\n"
+      "RawBit\n"
+      "one two\n",
+      buffer);
+}
+
+// MockDescriptorFile defines only those members that Printer uses to write out
+// annotations.
+class MockDescriptorFile {
+ public:
+  explicit MockDescriptorFile(const std::string& file) : file_(file) {}
+
+  // The mock filename for this file.
+  const std::string& name() const { return file_; }
+
+ private:
+  std::string file_;
+};
+
+// MockDescriptor defines only those members that Printer uses to write out
+// annotations.
+class MockDescriptor {
+ public:
+  MockDescriptor(const std::string& file, const std::vector<int>& path)
+      : file_(file), path_(path) {}
+
+  // The mock file in which this descriptor was defined.
+  const MockDescriptorFile* file() const { return &file_; }
+
+ private:
+  // Allows access to GetLocationPath.
+  friend class Printer;
+
+  // Copies the pre-stored path to output.
+  void GetLocationPath(std::vector<int>* output) const { *output = path_; }
+
+  MockDescriptorFile file_;
+  std::vector<int> path_;
+};
+
+TEST(Printer, AnnotateMap) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    std::map<std::string, std::string> vars;
+    vars["foo"] = "3";
+    vars["bar"] = "5";
+    printer.Print(vars, "012$foo$4$bar$\n");
+    std::vector<int> path_1;
+    path_1.push_back(33);
+    std::vector<int> path_2;
+    path_2.push_back(11);
+    path_2.push_back(22);
+    MockDescriptor descriptor_1("path_1", path_1);
+    MockDescriptor descriptor_2("path_2", path_2);
+    printer.Annotate("foo", "foo", &descriptor_1);
+    printer.Annotate("bar", "bar", &descriptor_2);
+  }
+  buffer[output.ByteCount()] = '\0';
+  EXPECT_STREQ("012345\n", buffer);
+  ASSERT_EQ(2, info.annotation_size());
+  const GeneratedCodeInfo::Annotation* foo = info.annotation(0).path_size() == 1
+                                                 ? &info.annotation(0)
+                                                 : &info.annotation(1);
+  const GeneratedCodeInfo::Annotation* bar = info.annotation(0).path_size() == 1
+                                                 ? &info.annotation(1)
+                                                 : &info.annotation(0);
+  ASSERT_EQ(1, foo->path_size());
+  ASSERT_EQ(2, bar->path_size());
+  EXPECT_EQ(33, foo->path(0));
+  EXPECT_EQ(11, bar->path(0));
+  EXPECT_EQ(22, bar->path(1));
+  EXPECT_EQ("path_1", foo->source_file());
+  EXPECT_EQ("path_2", bar->source_file());
+  EXPECT_EQ(3, foo->begin());
+  EXPECT_EQ(4, foo->end());
+  EXPECT_EQ(5, bar->begin());
+  EXPECT_EQ(6, bar->end());
+}
+
+TEST(Printer, AnnotateInline) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    printer.Print("012$foo$4$bar$\n", "foo", "3", "bar", "5");
+    std::vector<int> path_1;
+    path_1.push_back(33);
+    std::vector<int> path_2;
+    path_2.push_back(11);
+    path_2.push_back(22);
+    MockDescriptor descriptor_1("path_1", path_1);
+    MockDescriptor descriptor_2("path_2", path_2);
+    printer.Annotate("foo", "foo", &descriptor_1);
+    printer.Annotate("bar", "bar", &descriptor_2);
+  }
+  buffer[output.ByteCount()] = '\0';
+  EXPECT_STREQ("012345\n", buffer);
+  ASSERT_EQ(2, info.annotation_size());
+  const GeneratedCodeInfo::Annotation* foo = info.annotation(0).path_size() == 1
+                                                 ? &info.annotation(0)
+                                                 : &info.annotation(1);
+  const GeneratedCodeInfo::Annotation* bar = info.annotation(0).path_size() == 1
+                                                 ? &info.annotation(1)
+                                                 : &info.annotation(0);
+  ASSERT_EQ(1, foo->path_size());
+  ASSERT_EQ(2, bar->path_size());
+  EXPECT_EQ(33, foo->path(0));
+  EXPECT_EQ(11, bar->path(0));
+  EXPECT_EQ(22, bar->path(1));
+  EXPECT_EQ("path_1", foo->source_file());
+  EXPECT_EQ("path_2", bar->source_file());
+  EXPECT_EQ(3, foo->begin());
+  EXPECT_EQ(4, foo->end());
+  EXPECT_EQ(5, bar->begin());
+  EXPECT_EQ(6, bar->end());
+}
+
+TEST(Printer, AnnotateRange) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    printer.Print("012$foo$4$bar$\n", "foo", "3", "bar", "5");
+    std::vector<int> path;
+    path.push_back(33);
+    MockDescriptor descriptor("path", path);
+    printer.Annotate("foo", "bar", &descriptor);
+  }
+  buffer[output.ByteCount()] = '\0';
+  EXPECT_STREQ("012345\n", buffer);
+  ASSERT_EQ(1, info.annotation_size());
+  const GeneratedCodeInfo::Annotation* foobar = &info.annotation(0);
+  ASSERT_EQ(1, foobar->path_size());
+  EXPECT_EQ(33, foobar->path(0));
+  EXPECT_EQ("path", foobar->source_file());
+  EXPECT_EQ(3, foobar->begin());
+  EXPECT_EQ(6, foobar->end());
+}
+
+TEST(Printer, AnnotateEmptyRange) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    printer.Print("012$foo$4$baz$$bam$$bar$\n", "foo", "3", "bar", "5", "baz",
+                  "", "bam", "");
+    std::vector<int> path;
+    path.push_back(33);
+    MockDescriptor descriptor("path", path);
+    printer.Annotate("baz", "bam", &descriptor);
+  }
+  buffer[output.ByteCount()] = '\0';
+  EXPECT_STREQ("012345\n", buffer);
+  ASSERT_EQ(1, info.annotation_size());
+  const GeneratedCodeInfo::Annotation* bazbam = &info.annotation(0);
+  ASSERT_EQ(1, bazbam->path_size());
+  EXPECT_EQ(33, bazbam->path(0));
+  EXPECT_EQ("path", bazbam->source_file());
+  EXPECT_EQ(5, bazbam->begin());
+  EXPECT_EQ(5, bazbam->end());
+}
+
+TEST(Printer, AnnotateDespiteUnrelatedMultipleUses) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    printer.Print("012$foo$4$foo$$bar$\n", "foo", "3", "bar", "5");
+    std::vector<int> path;
+    path.push_back(33);
+    MockDescriptor descriptor("path", path);
+    printer.Annotate("bar", "bar", &descriptor);
+  }
+  buffer[output.ByteCount()] = '\0';
+  EXPECT_STREQ("0123435\n", buffer);
+  ASSERT_EQ(1, info.annotation_size());
+  const GeneratedCodeInfo::Annotation* bar = &info.annotation(0);
+  ASSERT_EQ(1, bar->path_size());
+  EXPECT_EQ(33, bar->path(0));
+  EXPECT_EQ("path", bar->source_file());
+  EXPECT_EQ(6, bar->begin());
+  EXPECT_EQ(7, bar->end());
+}
+
+TEST(Printer, AnnotateIndent) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    printer.Print("0\n");
+    printer.Indent();
+    printer.Print("$foo$", "foo", "4");
+    std::vector<int> path;
+    path.push_back(44);
+    MockDescriptor descriptor("path", path);
+    printer.Annotate("foo", &descriptor);
+    printer.Print(",\n");
+    printer.Print("$bar$", "bar", "9");
+    path[0] = 99;
+    MockDescriptor descriptor_two("path", path);
+    printer.Annotate("bar", &descriptor_two);
+    printer.Print("\n${$$D$$}$\n", "{", "", "}", "", "D", "d");
+    path[0] = 1313;
+    MockDescriptor descriptor_three("path", path);
+    printer.Annotate("{", "}", &descriptor_three);
+    printer.Outdent();
+    printer.Print("\n");
+  }
+  buffer[output.ByteCount()] = '\0';
+  EXPECT_STREQ("0\n  4,\n  9\n  d\n\n", buffer);
+  ASSERT_EQ(3, info.annotation_size());
+  const GeneratedCodeInfo::Annotation* foo = &info.annotation(0);
+  ASSERT_EQ(1, foo->path_size());
+  EXPECT_EQ(44, foo->path(0));
+  EXPECT_EQ("path", foo->source_file());
+  EXPECT_EQ(4, foo->begin());
+  EXPECT_EQ(5, foo->end());
+  const GeneratedCodeInfo::Annotation* bar = &info.annotation(1);
+  ASSERT_EQ(1, bar->path_size());
+  EXPECT_EQ(99, bar->path(0));
+  EXPECT_EQ("path", bar->source_file());
+  EXPECT_EQ(9, bar->begin());
+  EXPECT_EQ(10, bar->end());
+  const GeneratedCodeInfo::Annotation* braces = &info.annotation(2);
+  ASSERT_EQ(1, braces->path_size());
+  EXPECT_EQ(1313, braces->path(0));
+  EXPECT_EQ("path", braces->source_file());
+  EXPECT_EQ(13, braces->begin());
+  EXPECT_EQ(14, braces->end());
+}
+
+TEST(Printer, AnnotateIndentNewline) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    printer.Indent();
+    printer.Print("$A$$N$$B$C\n", "A", "", "N", "\nz", "B", "");
+    std::vector<int> path;
+    path.push_back(0);
+    MockDescriptor descriptor("path", path);
+    printer.Annotate("A", "B", &descriptor);
+    printer.Outdent();
+    printer.Print("\n");
+  }
+  buffer[output.ByteCount()] = '\0';
+  EXPECT_STREQ("\nz  C\n\n", buffer);
+  ASSERT_EQ(1, info.annotation_size());
+  const GeneratedCodeInfo::Annotation* ab = &info.annotation(0);
+  ASSERT_EQ(1, ab->path_size());
+  EXPECT_EQ(0, ab->path(0));
+  EXPECT_EQ("path", ab->source_file());
+  EXPECT_EQ(0, ab->begin());
+  EXPECT_EQ(4, ab->end());
+}
+
+TEST(Printer, Indenting) {
+  char buffer[8192];
+
+  for (int block_size = 1; block_size < 512; block_size *= 2) {
+    ArrayOutputStream output(buffer, sizeof(buffer), block_size);
+
+    {
+      Printer printer(&output, '$');
+      std::map<std::string, std::string> vars;
+
+      vars["newline"] = "\n";
+
+      printer.Print("This is not indented.\n");
+      printer.Indent();
+      printer.Print("This is indented\nAnd so is this\n");
+      printer.Outdent();
+      printer.Print("But this is not.");
+      printer.Indent();
+      printer.Print(
+          "  And this is still the same line.\n"
+          "But this is indented.\n");
+      printer.PrintRaw("RawBit has indent at start\n");
+      printer.PrintRaw("but not after a raw newline\n");
+      printer.Print(vars,
+                    "Note that a newline in a variable will break "
+                    "indenting, as we see$newline$here.\n");
+      printer.Indent();
+      printer.Print("And this");
+      printer.Outdent();
+      printer.Outdent();
+      printer.Print(" is double-indented\nBack to normal.");
+
+      EXPECT_FALSE(printer.failed());
+    }
+
+    buffer[output.ByteCount()] = '\0';
+
+    EXPECT_STREQ(
+        "This is not indented.\n"
+        "  This is indented\n"
+        "  And so is this\n"
+        "But this is not.  And this is still the same line.\n"
+        "  But this is indented.\n"
+        "  RawBit has indent at start\n"
+        "but not after a raw newline\n"
+        "Note that a newline in a variable will break indenting, as we see\n"
+        "here.\n"
+        "    And this is double-indented\n"
+        "Back to normal.",
+        buffer);
+  }
+}
+
+// Death tests do not work on Windows as of yet.
+#ifdef PROTOBUF_HAS_DEATH_TEST
+TEST(Printer, Death) {
+  char buffer[8192];
+
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  Printer printer(&output, '$');
+
+  EXPECT_DEBUG_DEATH(printer.Print("$nosuchvar$"), "Undefined variable");
+  EXPECT_DEBUG_DEATH(printer.Print("$unclosed"), "Unclosed variable name");
+  EXPECT_DEBUG_DEATH(printer.Outdent(), "without matching Indent");
+}
+
+TEST(Printer, AnnotateMultipleUsesDeath) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    printer.Print("012$foo$4$foo$\n", "foo", "3");
+    std::vector<int> path;
+    path.push_back(33);
+    MockDescriptor descriptor("path", path);
+    EXPECT_DEBUG_DEATH(printer.Annotate("foo", "foo", &descriptor), "multiple");
+  }
+}
+
+TEST(Printer, AnnotateNegativeLengthDeath) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    printer.Print("012$foo$4$bar$\n", "foo", "3", "bar", "5");
+    std::vector<int> path;
+    path.push_back(33);
+    MockDescriptor descriptor("path", path);
+    EXPECT_DEBUG_DEATH(printer.Annotate("bar", "foo", &descriptor), "negative");
+  }
+}
+
+TEST(Printer, AnnotateUndefinedDeath) {
+  char buffer[8192];
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  GeneratedCodeInfo info;
+  AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+  {
+    Printer printer(&output, '$', &info_collector);
+    printer.Print("012$foo$4$foo$\n", "foo", "3");
+    std::vector<int> path;
+    path.push_back(33);
+    MockDescriptor descriptor("path", path);
+    EXPECT_DEBUG_DEATH(printer.Annotate("bar", "bar", &descriptor),
+                       "Undefined");
+  }
+}
+#endif  // PROTOBUF_HAS_DEATH_TEST
+
+TEST(Printer, WriteFailurePartial) {
+  char buffer[17];
+
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  Printer printer(&output, '$');
+
+  // Print 16 bytes to almost fill the buffer (should not fail).
+  printer.Print("0123456789abcdef");
+  EXPECT_FALSE(printer.failed());
+
+  // Try to print 2 chars. Only one fits.
+  printer.Print("<>");
+  EXPECT_TRUE(printer.failed());
+
+  // Anything else should fail too.
+  printer.Print(" ");
+  EXPECT_TRUE(printer.failed());
+  printer.Print("blah");
+  EXPECT_TRUE(printer.failed());
+
+  // Buffer should contain the first 17 bytes written.
+  EXPECT_EQ("0123456789abcdef<", std::string(buffer, sizeof(buffer)));
+}
+
+TEST(Printer, WriteFailureExact) {
+  char buffer[16];
+
+  ArrayOutputStream output(buffer, sizeof(buffer));
+  Printer printer(&output, '$');
+
+  // Print 16 bytes to fill the buffer exactly (should not fail).
+  printer.Print("0123456789abcdef");
+  EXPECT_FALSE(printer.failed());
+
+  // Try to print one more byte (should fail).
+  printer.Print(" ");
+  EXPECT_TRUE(printer.failed());
+
+  // Should not crash
+  printer.Print("blah");
+  EXPECT_TRUE(printer.failed());
+
+  // Buffer should contain the first 16 bytes written.
+  EXPECT_EQ("0123456789abcdef", std::string(buffer, sizeof(buffer)));
+}
+
+TEST(Printer, FormatInternal) {
+  std::vector<std::string> args{"arg1", "arg2"};
+  std::map<std::string, std::string> vars{
+      {"foo", "bar"}, {"baz", "bla"}, {"empty", ""}};
+  // Substitution tests
+  {
+    // Direct arg substitution
+    std::string s;
+    {
+      StringOutputStream output(&s);
+      Printer printer(&output, '$');
+      printer.FormatInternal(args, vars, "$1$ $2$");
+    }
+    EXPECT_EQ("arg1 arg2", s);
+  }
+  {
+    // Variable substitution including spaces left
+    std::string s;
+    {
+      StringOutputStream output(&s);
+      Printer printer(&output, '$');
+      printer.FormatInternal({}, vars, "$foo$$ baz$$ empty$");
+    }
+    EXPECT_EQ("bar bla", s);
+  }
+  {
+    // Variable substitution including spaces right
+    std::string s;
+    {
+      StringOutputStream output(&s);
+      Printer printer(&output, '$');
+      printer.FormatInternal({}, vars, "$empty $$foo $$baz$");
+    }
+    EXPECT_EQ("bar bla", s);
+  }
+  {
+    // Mixed variable substitution
+    std::string s;
+    {
+      StringOutputStream output(&s);
+      Printer printer(&output, '$');
+      printer.FormatInternal(args, vars, "$empty $$1$ $foo $$2$ $baz$");
+    }
+    EXPECT_EQ("arg1 bar arg2 bla", s);
+  }
+
+  // Indentation tests
+  {
+    // Empty lines shouldn't indent.
+    std::string s;
+    {
+      StringOutputStream output(&s);
+      Printer printer(&output, '$');
+      printer.Indent();
+      printer.FormatInternal(args, vars, "$empty $\n\n$1$ $foo $$2$\n$baz$");
+      printer.Outdent();
+    }
+    EXPECT_EQ("\n\n  arg1 bar arg2\n  bla", s);
+  }
+  {
+    // Annotations should respect indentation.
+    std::string s;
+    GeneratedCodeInfo info;
+    {
+      StringOutputStream output(&s);
+      AnnotationProtoCollector<GeneratedCodeInfo> info_collector(&info);
+      Printer printer(&output, '$', &info_collector);
+      printer.Indent();
+      GeneratedCodeInfo::Annotation annotation;
+      annotation.set_source_file("file.proto");
+      annotation.add_path(33);
+      std::vector<std::string> args{annotation.SerializeAsString(), "arg1",
+                                    "arg2"};
+      printer.FormatInternal(args, vars, "$empty $\n\n${1$$2$$}$ $3$\n$baz$");
+      printer.Outdent();
+    }
+    EXPECT_EQ("\n\n  arg1 arg2\n  bla", s);
+    ASSERT_EQ(1, info.annotation_size());
+    const GeneratedCodeInfo::Annotation* arg1 = &info.annotation(0);
+    ASSERT_EQ(1, arg1->path_size());
+    EXPECT_EQ(33, arg1->path(0));
+    EXPECT_EQ("file.proto", arg1->source_file());
+    EXPECT_EQ(4, arg1->begin());
+    EXPECT_EQ(8, arg1->end());
+  }
+#ifdef PROTOBUF_HAS_DEATH_TEST
+  // Death tests in case of illegal format strings.
+  {
+    // Unused arguments
+    std::string s;
+    StringOutputStream output(&s);
+    Printer printer(&output, '$');
+    EXPECT_DEATH(printer.FormatInternal(args, vars, "$empty $$1$"), "Unused");
+  }
+  {
+    // Wrong order arguments
+    std::string s;
+    StringOutputStream output(&s);
+    Printer printer(&output, '$');
+    EXPECT_DEATH(printer.FormatInternal(args, vars, "$2$ $1$"), "order");
+  }
+  {
+    // Zero is illegal argument
+    std::string s;
+    StringOutputStream output(&s);
+    Printer printer(&output, '$');
+    EXPECT_DEATH(printer.FormatInternal(args, vars, "$0$"), "failed");
+  }
+  {
+    // Argument out of bounds
+    std::string s;
+    StringOutputStream output(&s);
+    Printer printer(&output, '$');
+    EXPECT_DEATH(printer.FormatInternal(args, vars, "$1$ $2$ $3$"), "bounds");
+  }
+  {
+    // Unknown variable
+    std::string s;
+    StringOutputStream output(&s);
+    Printer printer(&output, '$');
+    EXPECT_DEATH(printer.FormatInternal(args, vars, "$huh$ $1$$2$"), "Unknown");
+  }
+  {
+    // Illegal variable
+    std::string s;
+    StringOutputStream output(&s);
+    Printer printer(&output, '$');
+    EXPECT_DEATH(printer.FormatInternal({}, vars, "$ $"), "Empty");
+  }
+#endif  // PROTOBUF_HAS_DEATH_TEST
+}
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google

ubuntu/google/protobuf/io/strtod.cc

@@ -0,0 +1,82 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <google/protobuf/io/strtod.h>
+
+#include <cstdio>
+#include <cstring>
+#include <limits>
+#include <string>
+
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+
+#include <google/protobuf/stubs/strutil.h>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+// This approximately 0x1.ffffffp127, but we don't use 0x1.ffffffp127 because
+// it won't compile in MSVC.
+const double MAX_FLOAT_AS_DOUBLE_ROUNDED = 3.4028235677973366e+38;
+
+float SafeDoubleToFloat(double value) {
+  // static_cast<float> on a number larger than float can result in illegal
+  // instruction error, so we need to manually convert it to infinity or max.
+  if (value > std::numeric_limits<float>::max()) {
+    // Max float value is about 3.4028234664E38 when represented as a double.
+    // However, when printing float as text, it will be rounded as
+    // 3.4028235e+38. If we parse the value of 3.4028235e+38 from text and
+    // compare it to 3.4028234664E38, we may think that it is larger, but
+    // actually, any number between these two numbers could only be represented
+    // as the same max float number in float, so we should treat them the same
+    // as max float.
+    if (value <= MAX_FLOAT_AS_DOUBLE_ROUNDED) {
+      return std::numeric_limits<float>::max();
+    }
+    return std::numeric_limits<float>::infinity();
+  } else if (value < -std::numeric_limits<float>::max()) {
+    if (value >= -MAX_FLOAT_AS_DOUBLE_ROUNDED) {
+      return -std::numeric_limits<float>::max();
+    }
+    return -std::numeric_limits<float>::infinity();
+  } else {
+    return static_cast<float>(value);
+  }
+}
+
+double NoLocaleStrtod(const char* str, char** endptr) {
+  return google::protobuf::internal::NoLocaleStrtod(str, endptr);
+}
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google

ubuntu/google/protobuf/io/strtod.h

@@ -0,0 +1,55 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A locale-independent version of strtod(), used to parse floating
+// point default values in .proto files, where the decimal separator
+// is always a dot.
+
+#ifndef GOOGLE_PROTOBUF_IO_STRTOD_H__
+#define GOOGLE_PROTOBUF_IO_STRTOD_H__
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+// A locale-independent version of the standard strtod(), which always
+// uses a dot as the decimal separator.
+double NoLocaleStrtod(const char* str, char** endptr);
+
+// Casts a double value to a float value. If the value is outside of the
+// representable range of float, it will be converted to positive or negative
+// infinity.
+float SafeDoubleToFloat(double value);
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_IO_STRTOD_H__

ubuntu/google/protobuf/io/tokenizer.h

@@ -0,0 +1,413 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// Class for parsing tokenized text from a ZeroCopyInputStream.
+
+#ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__
+#define GOOGLE_PROTOBUF_IO_TOKENIZER_H__
+
+
+#include <string>
+#include <vector>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/port_def.inc>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+class ZeroCopyInputStream;  // zero_copy_stream.h
+
+// Defined in this file.
+class ErrorCollector;
+class Tokenizer;
+
+// By "column number", the proto compiler refers to a count of the number
+// of bytes before a given byte, except that a tab character advances to
+// the next multiple of 8 bytes.  Note in particular that column numbers
+// are zero-based, while many user interfaces use one-based column numbers.
+typedef int ColumnNumber;
+
+// Abstract interface for an object which collects the errors that occur
+// during parsing.  A typical implementation might simply print the errors
+// to stdout.
+class PROTOBUF_EXPORT ErrorCollector {
+ public:
+  inline ErrorCollector() {}
+  virtual ~ErrorCollector();
+
+  // Indicates that there was an error in the input at the given line and
+  // column numbers.  The numbers are zero-based, so you may want to add
+  // 1 to each before printing them.
+  virtual void AddError(int line, ColumnNumber column,
+                        const std::string& message) = 0;
+
+  // Indicates that there was a warning in the input at the given line and
+  // column numbers.  The numbers are zero-based, so you may want to add
+  // 1 to each before printing them.
+  virtual void AddWarning(int line, ColumnNumber column,
+                          const std::string& message) {}
+
+ private:
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ErrorCollector);
+};
+
+// This class converts a stream of raw text into a stream of tokens for
+// the protocol definition parser to parse.  The tokens recognized are
+// similar to those that make up the C language; see the TokenType enum for
+// precise descriptions.  Whitespace and comments are skipped.  By default,
+// C- and C++-style comments are recognized, but other styles can be used by
+// calling set_comment_style().
+class PROTOBUF_EXPORT Tokenizer {
+ public:
+  // Construct a Tokenizer that reads and tokenizes text from the given
+  // input stream and writes errors to the given error_collector.
+  // The caller keeps ownership of input and error_collector.
+  Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector);
+  ~Tokenizer();
+
+  enum TokenType {
+    TYPE_START,  // Next() has not yet been called.
+    TYPE_END,    // End of input reached.  "text" is empty.
+
+    TYPE_IDENTIFIER,  // A sequence of letters, digits, and underscores, not
+                      // starting with a digit.  It is an error for a number
+                      // to be followed by an identifier with no space in
+                      // between.
+    TYPE_INTEGER,     // A sequence of digits representing an integer.  Normally
+                      // the digits are decimal, but a prefix of "0x" indicates
+                      // a hex number and a leading zero indicates octal, just
+                      // like with C numeric literals.  A leading negative sign
+                      // is NOT included in the token; it's up to the parser to
+                      // interpret the unary minus operator on its own.
+    TYPE_FLOAT,       // A floating point literal, with a fractional part and/or
+                      // an exponent.  Always in decimal.  Again, never
+                      // negative.
+    TYPE_STRING,      // A quoted sequence of escaped characters.  Either single
+                      // or double quotes can be used, but they must match.
+                      // A string literal cannot cross a line break.
+    TYPE_SYMBOL,      // Any other printable character, like '!' or '+'.
+                      // Symbols are always a single character, so "!+$%" is
+                      // four tokens.
+  };
+
+  // Structure representing a token read from the token stream.
+  struct Token {
+    TokenType type;
+    std::string text;  // The exact text of the token as it appeared in
+                       // the input.  e.g. tokens of TYPE_STRING will still
+                       // be escaped and in quotes.
+
+    // "line" and "column" specify the position of the first character of
+    // the token within the input stream.  They are zero-based.
+    int line;
+    ColumnNumber column;
+    ColumnNumber end_column;
+  };
+
+  // Get the current token.  This is updated when Next() is called.  Before
+  // the first call to Next(), current() has type TYPE_START and no contents.
+  const Token& current();
+
+  // Return the previous token -- i.e. what current() returned before the
+  // previous call to Next().
+  const Token& previous();
+
+  // Advance to the next token.  Returns false if the end of the input is
+  // reached.
+  bool Next();
+
+  // Like Next(), but also collects comments which appear between the previous
+  // and next tokens.
+  //
+  // Comments which appear to be attached to the previous token are stored
+  // in *prev_tailing_comments.  Comments which appear to be attached to the
+  // next token are stored in *next_leading_comments.  Comments appearing in
+  // between which do not appear to be attached to either will be added to
+  // detached_comments.  Any of these parameters can be NULL to simply discard
+  // the comments.
+  //
+  // A series of line comments appearing on consecutive lines, with no other
+  // tokens appearing on those lines, will be treated as a single comment.
+  //
+  // Only the comment content is returned; comment markers (e.g. //) are
+  // stripped out.  For block comments, leading whitespace and an asterisk will
+  // be stripped from the beginning of each line other than the first.  Newlines
+  // are included in the output.
+  //
+  // Examples:
+  //
+  //   optional int32 foo = 1;  // Comment attached to foo.
+  //   // Comment attached to bar.
+  //   optional int32 bar = 2;
+  //
+  //   optional string baz = 3;
+  //   // Comment attached to baz.
+  //   // Another line attached to baz.
+  //
+  //   // Comment attached to qux.
+  //   //
+  //   // Another line attached to qux.
+  //   optional double qux = 4;
+  //
+  //   // Detached comment.  This is not attached to qux or corge
+  //   // because there are blank lines separating it from both.
+  //
+  //   optional string corge = 5;
+  //   /* Block comment attached
+  //    * to corge.  Leading asterisks
+  //    * will be removed. */
+  //   /* Block comment attached to
+  //    * grault. */
+  //   optional int32 grault = 6;
+  bool NextWithComments(std::string* prev_trailing_comments,
+                        std::vector<std::string>* detached_comments,
+                        std::string* next_leading_comments);
+
+  // Parse helpers ---------------------------------------------------
+
+  // Parses a TYPE_FLOAT token.  This never fails, so long as the text actually
+  // comes from a TYPE_FLOAT token parsed by Tokenizer.  If it doesn't, the
+  // result is undefined (possibly an assert failure).
+  static double ParseFloat(const std::string& text);
+
+  // Parses a TYPE_STRING token.  This never fails, so long as the text actually
+  // comes from a TYPE_STRING token parsed by Tokenizer.  If it doesn't, the
+  // result is undefined (possibly an assert failure).
+  static void ParseString(const std::string& text, std::string* output);
+
+  // Identical to ParseString, but appends to output.
+  static void ParseStringAppend(const std::string& text, std::string* output);
+
+  // Parses a TYPE_INTEGER token.  Returns false if the result would be
+  // greater than max_value.  Otherwise, returns true and sets *output to the
+  // result.  If the text is not from a Token of type TYPE_INTEGER originally
+  // parsed by a Tokenizer, the result is undefined (possibly an assert
+  // failure).
+  static bool ParseInteger(const std::string& text, uint64 max_value,
+                           uint64* output);
+
+  // Options ---------------------------------------------------------
+
+  // Set true to allow floats to be suffixed with the letter 'f'.  Tokens
+  // which would otherwise be integers but which have the 'f' suffix will be
+  // forced to be interpreted as floats.  For all other purposes, the 'f' is
+  // ignored.
+  void set_allow_f_after_float(bool value) { allow_f_after_float_ = value; }
+
+  // Valid values for set_comment_style().
+  enum CommentStyle {
+    // Line comments begin with "//", block comments are delimited by "/*" and
+    // "*/".
+    CPP_COMMENT_STYLE,
+    // Line comments begin with "#".  No way to write block comments.
+    SH_COMMENT_STYLE
+  };
+
+  // Sets the comment style.
+  void set_comment_style(CommentStyle style) { comment_style_ = style; }
+
+  // Whether to require whitespace between a number and a field name.
+  // Default is true. Do not use this; for Google-internal cleanup only.
+  void set_require_space_after_number(bool require) {
+    require_space_after_number_ = require;
+  }
+
+  // Whether to allow string literals to span multiple lines. Default is false.
+  // Do not use this; for Google-internal cleanup only.
+  void set_allow_multiline_strings(bool allow) {
+    allow_multiline_strings_ = allow;
+  }
+
+  // External helper: validate an identifier.
+  static bool IsIdentifier(const std::string& text);
+
+  // -----------------------------------------------------------------
+ private:
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer);
+
+  Token current_;   // Returned by current().
+  Token previous_;  // Returned by previous().
+
+  ZeroCopyInputStream* input_;
+  ErrorCollector* error_collector_;
+
+  char current_char_;   // == buffer_[buffer_pos_], updated by NextChar().
+  const char* buffer_;  // Current buffer returned from input_.
+  int buffer_size_;     // Size of buffer_.
+  int buffer_pos_;      // Current position within the buffer.
+  bool read_error_;     // Did we previously encounter a read error?
+
+  // Line and column number of current_char_ within the whole input stream.
+  int line_;
+  ColumnNumber column_;
+
+  // String to which text should be appended as we advance through it.
+  // Call RecordTo(&str) to start recording and StopRecording() to stop.
+  // E.g. StartToken() calls RecordTo(&current_.text).  record_start_ is the
+  // position within the current buffer where recording started.
+  std::string* record_target_;
+  int record_start_;
+
+  // Options.
+  bool allow_f_after_float_;
+  CommentStyle comment_style_;
+  bool require_space_after_number_;
+  bool allow_multiline_strings_;
+
+  // Since we count columns we need to interpret tabs somehow.  We'll take
+  // the standard 8-character definition for lack of any way to do better.
+  // This must match the documentation of ColumnNumber.
+  static const int kTabWidth = 8;
+
+  // -----------------------------------------------------------------
+  // Helper methods.
+
+  // Consume this character and advance to the next one.
+  void NextChar();
+
+  // Read a new buffer from the input.
+  void Refresh();
+
+  inline void RecordTo(std::string* target);
+  inline void StopRecording();
+
+  // Called when the current character is the first character of a new
+  // token (not including whitespace or comments).
+  inline void StartToken();
+  // Called when the current character is the first character after the
+  // end of the last token.  After this returns, current_.text will
+  // contain all text consumed since StartToken() was called.
+  inline void EndToken();
+
+  // Convenience method to add an error at the current line and column.
+  void AddError(const std::string& message) {
+    error_collector_->AddError(line_, column_, message);
+  }
+
+  // -----------------------------------------------------------------
+  // The following four methods are used to consume tokens of specific
+  // types.  They are actually used to consume all characters *after*
+  // the first, since the calling function consumes the first character
+  // in order to decide what kind of token is being read.
+
+  // Read and consume a string, ending when the given delimiter is
+  // consumed.
+  void ConsumeString(char delimiter);
+
+  // Read and consume a number, returning TYPE_FLOAT or TYPE_INTEGER
+  // depending on what was read.  This needs to know if the first
+  // character was a zero in order to correctly recognize hex and octal
+  // numbers.
+  // It also needs to know if the first character was a . to parse floating
+  // point correctly.
+  TokenType ConsumeNumber(bool started_with_zero, bool started_with_dot);
+
+  // Consume the rest of a line.
+  void ConsumeLineComment(std::string* content);
+  // Consume until "*/".
+  void ConsumeBlockComment(std::string* content);
+
+  enum NextCommentStatus {
+    // Started a line comment.
+    LINE_COMMENT,
+
+    // Started a block comment.
+    BLOCK_COMMENT,
+
+    // Consumed a slash, then realized it wasn't a comment.  current_ has
+    // been filled in with a slash token.  The caller should return it.
+    SLASH_NOT_COMMENT,
+
+    // We do not appear to be starting a comment here.
+    NO_COMMENT
+  };
+
+  // If we're at the start of a new comment, consume it and return what kind
+  // of comment it is.
+  NextCommentStatus TryConsumeCommentStart();
+
+  // -----------------------------------------------------------------
+  // These helper methods make the parsing code more readable.  The
+  // "character classes" referred to are defined at the top of the .cc file.
+  // Basically it is a C++ class with one method:
+  //   static bool InClass(char c);
+  // The method returns true if c is a member of this "class", like "Letter"
+  // or "Digit".
+
+  // Returns true if the current character is of the given character
+  // class, but does not consume anything.
+  template <typename CharacterClass>
+  inline bool LookingAt();
+
+  // If the current character is in the given class, consume it and return
+  // true.  Otherwise return false.
+  // e.g. TryConsumeOne<Letter>()
+  template <typename CharacterClass>
+  inline bool TryConsumeOne();
+
+  // Like above, but try to consume the specific character indicated.
+  inline bool TryConsume(char c);
+
+  // Consume zero or more of the given character class.
+  template <typename CharacterClass>
+  inline void ConsumeZeroOrMore();
+
+  // Consume one or more of the given character class or log the given
+  // error message.
+  // e.g. ConsumeOneOrMore<Digit>("Expected digits.");
+  template <typename CharacterClass>
+  inline void ConsumeOneOrMore(const char* error);
+};
+
+// inline methods ====================================================
+inline const Tokenizer::Token& Tokenizer::current() { return current_; }
+
+inline const Tokenizer::Token& Tokenizer::previous() { return previous_; }
+
+inline void Tokenizer::ParseString(const std::string& text,
+                                   std::string* output) {
+  output->clear();
+  ParseStringAppend(text, output);
+}
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
+#endif  // GOOGLE_PROTOBUF_IO_TOKENIZER_H__

ubuntu/google/protobuf/io/tokenizer_unittest.cc

@@ -0,0 +1,962 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+#include <google/protobuf/io/tokenizer.h>
+
+#include <limits.h>
+#include <math.h>
+
+#include <vector>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/strutil.h>
+#include <google/protobuf/stubs/substitute.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+#include <google/protobuf/testing/googletest.h>
+#include <gtest/gtest.h>
+
+namespace google {
+namespace protobuf {
+namespace io {
+namespace {
+
+// ===================================================================
+// Data-Driven Test Infrastructure
+
+// TODO(kenton):  This is copied from coded_stream_unittest.  This is
+//   temporary until these features are integrated into gTest itself.
+
+// TEST_1D and TEST_2D are macros I'd eventually like to see added to
+// gTest.  These macros can be used to declare tests which should be
+// run multiple times, once for each item in some input array.  TEST_1D
+// tests all cases in a single input array.  TEST_2D tests all
+// combinations of cases from two arrays.  The arrays must be statically
+// defined such that the GOOGLE_ARRAYSIZE() macro works on them.  Example:
+//
+// int kCases[] = {1, 2, 3, 4}
+// TEST_1D(MyFixture, MyTest, kCases) {
+//   EXPECT_GT(kCases_case, 0);
+// }
+//
+// This test iterates through the numbers 1, 2, 3, and 4 and tests that
+// they are all grater than zero.  In case of failure, the exact case
+// which failed will be printed.  The case type must be printable using
+// ostream::operator<<.
+
+#define TEST_1D(FIXTURE, NAME, CASES)                             \
+  class FIXTURE##_##NAME##_DD : public FIXTURE {                  \
+   protected:                                                     \
+    template <typename CaseType>                                  \
+    void DoSingleCase(const CaseType& CASES##_case);              \
+  };                                                              \
+                                                                  \
+  TEST_F(FIXTURE##_##NAME##_DD, NAME) {                           \
+    for (int i = 0; i < GOOGLE_ARRAYSIZE(CASES); i++) {                  \
+      SCOPED_TRACE(testing::Message()                             \
+                   << #CASES " case #" << i << ": " << CASES[i]); \
+      DoSingleCase(CASES[i]);                                     \
+    }                                                             \
+  }                                                               \
+                                                                  \
+  template <typename CaseType>                                    \
+  void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType& CASES##_case)
+
+#define TEST_2D(FIXTURE, NAME, CASES1, CASES2)                              \
+  class FIXTURE##_##NAME##_DD : public FIXTURE {                            \
+   protected:                                                               \
+    template <typename CaseType1, typename CaseType2>                       \
+    void DoSingleCase(const CaseType1& CASES1##_case,                       \
+                      const CaseType2& CASES2##_case);                      \
+  };                                                                        \
+                                                                            \
+  TEST_F(FIXTURE##_##NAME##_DD, NAME) {                                     \
+    for (int i = 0; i < GOOGLE_ARRAYSIZE(CASES1); i++) {                           \
+      for (int j = 0; j < GOOGLE_ARRAYSIZE(CASES2); j++) {                         \
+        SCOPED_TRACE(testing::Message()                                     \
+                     << #CASES1 " case #" << i << ": " << CASES1[i] << ", " \
+                     << #CASES2 " case #" << j << ": " << CASES2[j]);       \
+        DoSingleCase(CASES1[i], CASES2[j]);                                 \
+      }                                                                     \
+    }                                                                       \
+  }                                                                         \
+                                                                            \
+  template <typename CaseType1, typename CaseType2>                         \
+  void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType1& CASES1##_case,  \
+                                           const CaseType2& CASES2##_case)
+
+// -------------------------------------------------------------------
+
+// An input stream that is basically like an ArrayInputStream but sometimes
+// returns empty buffers, just to throw us off.
+class TestInputStream : public ZeroCopyInputStream {
+ public:
+  TestInputStream(const void* data, int size, int block_size)
+      : array_stream_(data, size, block_size), counter_(0) {}
+  ~TestInputStream() {}
+
+  // implements ZeroCopyInputStream ----------------------------------
+  bool Next(const void** data, int* size) override {
+    // We'll return empty buffers starting with the first buffer, and every
+    // 3 and 5 buffers after that.
+    if (counter_ % 3 == 0 || counter_ % 5 == 0) {
+      *data = NULL;
+      *size = 0;
+      ++counter_;
+      return true;
+    } else {
+      ++counter_;
+      return array_stream_.Next(data, size);
+    }
+  }
+
+  void BackUp(int count) override { return array_stream_.BackUp(count); }
+  bool Skip(int count) override { return array_stream_.Skip(count); }
+  int64_t ByteCount() const override { return array_stream_.ByteCount(); }
+
+ private:
+  ArrayInputStream array_stream_;
+  int counter_;
+};
+
+// -------------------------------------------------------------------
+
+// An error collector which simply concatenates all its errors into a big
+// block of text which can be checked.
+class TestErrorCollector : public ErrorCollector {
+ public:
+  TestErrorCollector() {}
+  ~TestErrorCollector() {}
+
+  std::string text_;
+
+  // implements ErrorCollector ---------------------------------------
+  void AddError(int line, int column, const std::string& message) {
+    strings::SubstituteAndAppend(&text_, "$0:$1: $2\n", line, column, message);
+  }
+};
+
+// -------------------------------------------------------------------
+
+// We test each operation over a variety of block sizes to insure that
+// we test cases where reads cross buffer boundaries as well as cases
+// where they don't.  This is sort of a brute-force approach to this,
+// but it's easy to write and easy to understand.
+const int kBlockSizes[] = {1, 2, 3, 5, 7, 13, 32, 1024};
+
+class TokenizerTest : public testing::Test {
+ protected:
+  // For easy testing.
+  uint64 ParseInteger(const std::string& text) {
+    uint64 result;
+    EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result));
+    return result;
+  }
+};
+
+// ===================================================================
+
+// These tests causes gcc 3.3.5 (and earlier?) to give the cryptic error:
+//   "sorry, unimplemented: `method_call_expr' not supported by dump_expr"
+#if !defined(__GNUC__) || __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3)
+
+// In each test case, the entire input text should parse as a single token
+// of the given type.
+struct SimpleTokenCase {
+  std::string input;
+  Tokenizer::TokenType type;
+};
+
+inline std::ostream& operator<<(std::ostream& out,
+                                const SimpleTokenCase& test_case) {
+  return out << CEscape(test_case.input);
+}
+
+SimpleTokenCase kSimpleTokenCases[] = {
+    // Test identifiers.
+    {"hello", Tokenizer::TYPE_IDENTIFIER},
+
+    // Test integers.
+    {"123", Tokenizer::TYPE_INTEGER},
+    {"0xab6", Tokenizer::TYPE_INTEGER},
+    {"0XAB6", Tokenizer::TYPE_INTEGER},
+    {"0X1234567", Tokenizer::TYPE_INTEGER},
+    {"0x89abcdef", Tokenizer::TYPE_INTEGER},
+    {"0x89ABCDEF", Tokenizer::TYPE_INTEGER},
+    {"01234567", Tokenizer::TYPE_INTEGER},
+
+    // Test floats.
+    {"123.45", Tokenizer::TYPE_FLOAT},
+    {"1.", Tokenizer::TYPE_FLOAT},
+    {"1e3", Tokenizer::TYPE_FLOAT},
+    {"1E3", Tokenizer::TYPE_FLOAT},
+    {"1e-3", Tokenizer::TYPE_FLOAT},
+    {"1e+3", Tokenizer::TYPE_FLOAT},
+    {"1.e3", Tokenizer::TYPE_FLOAT},
+    {"1.2e3", Tokenizer::TYPE_FLOAT},
+    {".1", Tokenizer::TYPE_FLOAT},
+    {".1e3", Tokenizer::TYPE_FLOAT},
+    {".1e-3", Tokenizer::TYPE_FLOAT},
+    {".1e+3", Tokenizer::TYPE_FLOAT},
+
+    // Test strings.
+    {"'hello'", Tokenizer::TYPE_STRING},
+    {"\"foo\"", Tokenizer::TYPE_STRING},
+    {"'a\"b'", Tokenizer::TYPE_STRING},
+    {"\"a'b\"", Tokenizer::TYPE_STRING},
+    {"'a\\'b'", Tokenizer::TYPE_STRING},
+    {"\"a\\\"b\"", Tokenizer::TYPE_STRING},
+    {"'\\xf'", Tokenizer::TYPE_STRING},
+    {"'\\0'", Tokenizer::TYPE_STRING},
+
+    // Test symbols.
+    {"+", Tokenizer::TYPE_SYMBOL},
+    {".", Tokenizer::TYPE_SYMBOL},
+};
+
+TEST_2D(TokenizerTest, SimpleTokens, kSimpleTokenCases, kBlockSizes) {
+  // Set up the tokenizer.
+  TestInputStream input(kSimpleTokenCases_case.input.data(),
+                        kSimpleTokenCases_case.input.size(), kBlockSizes_case);
+  TestErrorCollector error_collector;
+  Tokenizer tokenizer(&input, &error_collector);
+
+  // Before Next() is called, the initial token should always be TYPE_START.
+  EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
+  EXPECT_EQ("", tokenizer.current().text);
+  EXPECT_EQ(0, tokenizer.current().line);
+  EXPECT_EQ(0, tokenizer.current().column);
+  EXPECT_EQ(0, tokenizer.current().end_column);
+
+  // Parse the token.
+  ASSERT_TRUE(tokenizer.Next());
+
+  // Check that it has the right type.
+  EXPECT_EQ(kSimpleTokenCases_case.type, tokenizer.current().type);
+  // Check that it contains the complete input text.
+  EXPECT_EQ(kSimpleTokenCases_case.input, tokenizer.current().text);
+  // Check that it is located at the beginning of the input
+  EXPECT_EQ(0, tokenizer.current().line);
+  EXPECT_EQ(0, tokenizer.current().column);
+  EXPECT_EQ(kSimpleTokenCases_case.input.size(),
+            tokenizer.current().end_column);
+
+  // There should be no more input.
+  EXPECT_FALSE(tokenizer.Next());
+
+  // After Next() returns false, the token should have type TYPE_END.
+  EXPECT_EQ(Tokenizer::TYPE_END, tokenizer.current().type);
+  EXPECT_EQ("", tokenizer.current().text);
+  EXPECT_EQ(0, tokenizer.current().line);
+  EXPECT_EQ(kSimpleTokenCases_case.input.size(), tokenizer.current().column);
+  EXPECT_EQ(kSimpleTokenCases_case.input.size(),
+            tokenizer.current().end_column);
+
+  // There should be no errors.
+  EXPECT_TRUE(error_collector.text_.empty());
+}
+
+TEST_1D(TokenizerTest, FloatSuffix, kBlockSizes) {
+  // Test the "allow_f_after_float" option.
+
+  // Set up the tokenizer.
+  const char* text = "1f 2.5f 6e3f 7F";
+  TestInputStream input(text, strlen(text), kBlockSizes_case);
+  TestErrorCollector error_collector;
+  Tokenizer tokenizer(&input, &error_collector);
+  tokenizer.set_allow_f_after_float(true);
+
+  // Advance through tokens and check that they are parsed as expected.
+  ASSERT_TRUE(tokenizer.Next());
+  EXPECT_EQ(tokenizer.current().text, "1f");
+  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
+  ASSERT_TRUE(tokenizer.Next());
+  EXPECT_EQ(tokenizer.current().text, "2.5f");
+  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
+  ASSERT_TRUE(tokenizer.Next());
+  EXPECT_EQ(tokenizer.current().text, "6e3f");
+  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
+  ASSERT_TRUE(tokenizer.Next());
+  EXPECT_EQ(tokenizer.current().text, "7F");
+  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
+
+  // There should be no more input.
+  EXPECT_FALSE(tokenizer.Next());
+  // There should be no errors.
+  EXPECT_TRUE(error_collector.text_.empty());
+}
+
+#endif
+
+// -------------------------------------------------------------------
+
+// In each case, the input is parsed to produce a list of tokens.  The
+// last token in "output" must have type TYPE_END.
+struct MultiTokenCase {
+  std::string input;
+  Tokenizer::Token output[10];  // The compiler wants a constant array
+                                // size for initialization to work.  There
+                                // is no reason this can't be increased if
+                                // needed.
+};
+
+inline std::ostream& operator<<(std::ostream& out,
+                                const MultiTokenCase& test_case) {
+  return out << CEscape(test_case.input);
+}
+
+MultiTokenCase kMultiTokenCases[] = {
+    // Test empty input.
+    {"",
+     {
+         {Tokenizer::TYPE_END, "", 0, 0, 0},
+     }},
+
+    // Test all token types at the same time.
+    {"foo 1 1.2 + 'bar'",
+     {
+         {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
+         {Tokenizer::TYPE_INTEGER, "1", 0, 4, 5},
+         {Tokenizer::TYPE_FLOAT, "1.2", 0, 6, 9},
+         {Tokenizer::TYPE_SYMBOL, "+", 0, 10, 11},
+         {Tokenizer::TYPE_STRING, "'bar'", 0, 12, 17},
+         {Tokenizer::TYPE_END, "", 0, 17, 17},
+     }},
+
+    // Test that consecutive symbols are parsed as separate tokens.
+    {"!@+%",
+     {
+         {Tokenizer::TYPE_SYMBOL, "!", 0, 0, 1},
+         {Tokenizer::TYPE_SYMBOL, "@", 0, 1, 2},
+         {Tokenizer::TYPE_SYMBOL, "+", 0, 2, 3},
+         {Tokenizer::TYPE_SYMBOL, "%", 0, 3, 4},
+         {Tokenizer::TYPE_END, "", 0, 4, 4},
+     }},
+
+    // Test that newlines affect line numbers correctly.
+    {"foo bar\nrab oof",
+     {
+         {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
+         {Tokenizer::TYPE_IDENTIFIER, "bar", 0, 4, 7},
+         {Tokenizer::TYPE_IDENTIFIER, "rab", 1, 0, 3},
+         {Tokenizer::TYPE_IDENTIFIER, "oof", 1, 4, 7},
+         {Tokenizer::TYPE_END, "", 1, 7, 7},
+     }},
+
+    // Test that tabs affect column numbers correctly.
+    {"foo\tbar  \tbaz",
+     {
+         {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
+         {Tokenizer::TYPE_IDENTIFIER, "bar", 0, 8, 11},
+         {Tokenizer::TYPE_IDENTIFIER, "baz", 0, 16, 19},
+         {Tokenizer::TYPE_END, "", 0, 19, 19},
+     }},
+
+    // Test that tabs in string literals affect column numbers correctly.
+    {"\"foo\tbar\" baz",
+     {
+         {Tokenizer::TYPE_STRING, "\"foo\tbar\"", 0, 0, 12},
+         {Tokenizer::TYPE_IDENTIFIER, "baz", 0, 13, 16},
+         {Tokenizer::TYPE_END, "", 0, 16, 16},
+     }},
+
+    // Test that line comments are ignored.
+    {"foo // This is a comment\n"
+     "bar // This is another comment",
+     {
+         {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
+         {Tokenizer::TYPE_IDENTIFIER, "bar", 1, 0, 3},
+         {Tokenizer::TYPE_END, "", 1, 30, 30},
+     }},
+
+    // Test that block comments are ignored.
+    {"foo /* This is a block comment */ bar",
+     {
+         {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
+         {Tokenizer::TYPE_IDENTIFIER, "bar", 0, 34, 37},
+         {Tokenizer::TYPE_END, "", 0, 37, 37},
+     }},
+
+    // Test that sh-style comments are not ignored by default.
+    {"foo # bar\n"
+     "baz",
+     {
+         {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
+         {Tokenizer::TYPE_SYMBOL, "#", 0, 4, 5},
+         {Tokenizer::TYPE_IDENTIFIER, "bar", 0, 6, 9},
+         {Tokenizer::TYPE_IDENTIFIER, "baz", 1, 0, 3},
+         {Tokenizer::TYPE_END, "", 1, 3, 3},
+     }},
+
+    // Test all whitespace chars
+    {"foo\n\t\r\v\fbar",
+     {
+         {Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3},
+         {Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11, 14},
+         {Tokenizer::TYPE_END, "", 1, 14, 14},
+     }},
+};
+
+TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {
+  // Set up the tokenizer.
+  TestInputStream input(kMultiTokenCases_case.input.data(),
+                        kMultiTokenCases_case.input.size(), kBlockSizes_case);
+  TestErrorCollector error_collector;
+  Tokenizer tokenizer(&input, &error_collector);
+
+  // Before Next() is called, the initial token should always be TYPE_START.
+  EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
+  EXPECT_EQ("", tokenizer.current().text);
+  EXPECT_EQ(0, tokenizer.current().line);
+  EXPECT_EQ(0, tokenizer.current().column);
+  EXPECT_EQ(0, tokenizer.current().end_column);
+
+  // Loop through all expected tokens.
+  int i = 0;
+  Tokenizer::Token token;
+  do {
+    token = kMultiTokenCases_case.output[i++];
+
+    SCOPED_TRACE(testing::Message() << "Token #" << i << ": " << token.text);
+
+    Tokenizer::Token previous = tokenizer.current();
+
+    // Next() should only return false when it hits the end token.
+    if (token.type != Tokenizer::TYPE_END) {
+      ASSERT_TRUE(tokenizer.Next());
+    } else {
+      ASSERT_FALSE(tokenizer.Next());
+    }
+
+    // Check that the previous token is set correctly.
+    EXPECT_EQ(previous.type, tokenizer.previous().type);
+    EXPECT_EQ(previous.text, tokenizer.previous().text);
+    EXPECT_EQ(previous.line, tokenizer.previous().line);
+    EXPECT_EQ(previous.column, tokenizer.previous().column);
+    EXPECT_EQ(previous.end_column, tokenizer.previous().end_column);
+
+    // Check that the token matches the expected one.
+    EXPECT_EQ(token.type, tokenizer.current().type);
+    EXPECT_EQ(token.text, tokenizer.current().text);
+    EXPECT_EQ(token.line, tokenizer.current().line);
+    EXPECT_EQ(token.column, tokenizer.current().column);
+    EXPECT_EQ(token.end_column, tokenizer.current().end_column);
+
+  } while (token.type != Tokenizer::TYPE_END);
+
+  // There should be no errors.
+  EXPECT_TRUE(error_collector.text_.empty());
+}
+
+// This test causes gcc 3.3.5 (and earlier?) to give the cryptic error:
+//   "sorry, unimplemented: `method_call_expr' not supported by dump_expr"
+#if !defined(__GNUC__) || __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3)
+
+TEST_1D(TokenizerTest, ShCommentStyle, kBlockSizes) {
+  // Test the "comment_style" option.
+
+  const char* text =
+      "foo # bar\n"
+      "baz // qux\n"
+      "corge /* grault */\n"
+      "garply";
+  const char* const kTokens[] = {"foo",  // "# bar" is ignored
+                                 "baz", "/",      "/", "qux", "corge", "/",
+                                 "*",   "grault", "*", "/",   "garply"};
+
+  // Set up the tokenizer.
+  TestInputStream input(text, strlen(text), kBlockSizes_case);
+  TestErrorCollector error_collector;
+  Tokenizer tokenizer(&input, &error_collector);
+  tokenizer.set_comment_style(Tokenizer::SH_COMMENT_STYLE);
+
+  // Advance through tokens and check that they are parsed as expected.
+  for (int i = 0; i < GOOGLE_ARRAYSIZE(kTokens); i++) {
+    EXPECT_TRUE(tokenizer.Next());
+    EXPECT_EQ(tokenizer.current().text, kTokens[i]);
+  }
+
+  // There should be no more input.
+  EXPECT_FALSE(tokenizer.Next());
+  // There should be no errors.
+  EXPECT_TRUE(error_collector.text_.empty());
+}
+
+#endif
+
+// -------------------------------------------------------------------
+
+// In each case, the input is expected to have two tokens named "prev" and
+// "next" with comments in between.
+struct DocCommentCase {
+  std::string input;
+
+  const char* prev_trailing_comments;
+  const char* detached_comments[10];
+  const char* next_leading_comments;
+};
+
+inline std::ostream& operator<<(std::ostream& out,
+                                const DocCommentCase& test_case) {
+  return out << CEscape(test_case.input);
+}
+
+DocCommentCase kDocCommentCases[] = {
+    {"prev next",
+
+     "",
+     {},
+     ""},
+
+    {"prev /* ignored */ next",
+
+     "",
+     {},
+     ""},
+
+    {"prev // trailing comment\n"
+     "next",
+
+     " trailing comment\n",
+     {},
+     ""},
+
+    {"prev\n"
+     "// leading comment\n"
+     "// line 2\n"
+     "next",
+
+     "",
+     {},
+     " leading comment\n"
+     " line 2\n"},
+
+    {"prev\n"
+     "// trailing comment\n"
+     "// line 2\n"
+     "\n"
+     "next",
+
+     " trailing comment\n"
+     " line 2\n",
+     {},
+     ""},
+
+    {"prev // trailing comment\n"
+     "// leading comment\n"
+     "// line 2\n"
+     "next",
+
+     " trailing comment\n",
+     {},
+     " leading comment\n"
+     " line 2\n"},
+
+    {"prev /* trailing block comment */\n"
+     "/* leading block comment\n"
+     " * line 2\n"
+     " * line 3 */"
+     "next",
+
+     " trailing block comment ",
+     {},
+     " leading block comment\n"
+     " line 2\n"
+     " line 3 "},
+
+    {"prev\n"
+     "/* trailing block comment\n"
+     " * line 2\n"
+     " * line 3\n"
+     " */\n"
+     "/* leading block comment\n"
+     " * line 2\n"
+     " * line 3 */"
+     "next",
+
+     " trailing block comment\n"
+     " line 2\n"
+     " line 3\n",
+     {},
+     " leading block comment\n"
+     " line 2\n"
+     " line 3 "},
+
+    {"prev\n"
+     "// trailing comment\n"
+     "\n"
+     "// detached comment\n"
+     "// line 2\n"
+     "\n"
+     "// second detached comment\n"
+     "/* third detached comment\n"
+     " * line 2 */\n"
+     "// leading comment\n"
+     "next",
+
+     " trailing comment\n",
+     {" detached comment\n"
+      " line 2\n",
+      " second detached comment\n",
+      " third detached comment\n"
+      " line 2 "},
+     " leading comment\n"},
+
+    {"prev /**/\n"
+     "\n"
+     "// detached comment\n"
+     "\n"
+     "// leading comment\n"
+     "next",
+
+     "",
+     {" detached comment\n"},
+     " leading comment\n"},
+
+    {"prev /**/\n"
+     "// leading comment\n"
+     "next",
+
+     "",
+     {},
+     " leading comment\n"},
+};
+
+TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
+  // Set up the tokenizer.
+  TestInputStream input(kDocCommentCases_case.input.data(),
+                        kDocCommentCases_case.input.size(), kBlockSizes_case);
+  TestErrorCollector error_collector;
+  Tokenizer tokenizer(&input, &error_collector);
+
+  // Set up a second tokenizer where we'll pass all NULLs to NextWithComments().
+  TestInputStream input2(kDocCommentCases_case.input.data(),
+                         kDocCommentCases_case.input.size(), kBlockSizes_case);
+  Tokenizer tokenizer2(&input2, &error_collector);
+
+  tokenizer.Next();
+  tokenizer2.Next();
+
+  EXPECT_EQ("prev", tokenizer.current().text);
+  EXPECT_EQ("prev", tokenizer2.current().text);
+
+  std::string prev_trailing_comments;
+  std::vector<std::string> detached_comments;
+  std::string next_leading_comments;
+  tokenizer.NextWithComments(&prev_trailing_comments, &detached_comments,
+                             &next_leading_comments);
+  tokenizer2.NextWithComments(NULL, NULL, NULL);
+  EXPECT_EQ("next", tokenizer.current().text);
+  EXPECT_EQ("next", tokenizer2.current().text);
+
+  EXPECT_EQ(kDocCommentCases_case.prev_trailing_comments,
+            prev_trailing_comments);
+
+  for (int i = 0; i < detached_comments.size(); i++) {
+    ASSERT_LT(i, GOOGLE_ARRAYSIZE(kDocCommentCases));
+    ASSERT_TRUE(kDocCommentCases_case.detached_comments[i] != NULL);
+    EXPECT_EQ(kDocCommentCases_case.detached_comments[i], detached_comments[i]);
+  }
+
+  // Verify that we matched all the detached comments.
+  EXPECT_EQ(NULL,
+            kDocCommentCases_case.detached_comments[detached_comments.size()]);
+
+  EXPECT_EQ(kDocCommentCases_case.next_leading_comments, next_leading_comments);
+}
+
+// -------------------------------------------------------------------
+
+// Test parse helpers.  It's not really worth setting up a full data-driven
+// test here.
+TEST_F(TokenizerTest, ParseInteger) {
+  EXPECT_EQ(0, ParseInteger("0"));
+  EXPECT_EQ(123, ParseInteger("123"));
+  EXPECT_EQ(0xabcdef12u, ParseInteger("0xabcdef12"));
+  EXPECT_EQ(0xabcdef12u, ParseInteger("0xABCDEF12"));
+  EXPECT_EQ(kuint64max, ParseInteger("0xFFFFFFFFFFFFFFFF"));
+  EXPECT_EQ(01234567, ParseInteger("01234567"));
+  EXPECT_EQ(0X123, ParseInteger("0X123"));
+
+  // Test invalid integers that may still be tokenized as integers.
+  EXPECT_EQ(0, ParseInteger("0x"));
+
+  uint64 i;
+
+  // Test invalid integers that will never be tokenized as integers.
+  EXPECT_FALSE(Tokenizer::ParseInteger("zxy", kuint64max, &i));
+  EXPECT_FALSE(Tokenizer::ParseInteger("1.2", kuint64max, &i));
+  EXPECT_FALSE(Tokenizer::ParseInteger("08", kuint64max, &i));
+  EXPECT_FALSE(Tokenizer::ParseInteger("0xg", kuint64max, &i));
+  EXPECT_FALSE(Tokenizer::ParseInteger("-1", kuint64max, &i));
+
+  // Test overflows.
+  EXPECT_TRUE(Tokenizer::ParseInteger("0", 0, &i));
+  EXPECT_FALSE(Tokenizer::ParseInteger("1", 0, &i));
+  EXPECT_TRUE(Tokenizer::ParseInteger("1", 1, &i));
+  EXPECT_TRUE(Tokenizer::ParseInteger("12345", 12345, &i));
+  EXPECT_FALSE(Tokenizer::ParseInteger("12346", 12345, &i));
+  EXPECT_TRUE(Tokenizer::ParseInteger("0xFFFFFFFFFFFFFFFF", kuint64max, &i));
+  EXPECT_FALSE(Tokenizer::ParseInteger("0x10000000000000000", kuint64max, &i));
+}
+
+TEST_F(TokenizerTest, ParseFloat) {
+  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1."));
+  EXPECT_DOUBLE_EQ(1e3, Tokenizer::ParseFloat("1e3"));
+  EXPECT_DOUBLE_EQ(1e3, Tokenizer::ParseFloat("1E3"));
+  EXPECT_DOUBLE_EQ(1.5e3, Tokenizer::ParseFloat("1.5e3"));
+  EXPECT_DOUBLE_EQ(.1, Tokenizer::ParseFloat(".1"));
+  EXPECT_DOUBLE_EQ(.25, Tokenizer::ParseFloat(".25"));
+  EXPECT_DOUBLE_EQ(.1e3, Tokenizer::ParseFloat(".1e3"));
+  EXPECT_DOUBLE_EQ(.25e3, Tokenizer::ParseFloat(".25e3"));
+  EXPECT_DOUBLE_EQ(.1e+3, Tokenizer::ParseFloat(".1e+3"));
+  EXPECT_DOUBLE_EQ(.1e-3, Tokenizer::ParseFloat(".1e-3"));
+  EXPECT_DOUBLE_EQ(5, Tokenizer::ParseFloat("5"));
+  EXPECT_DOUBLE_EQ(6e-12, Tokenizer::ParseFloat("6e-12"));
+  EXPECT_DOUBLE_EQ(1.2, Tokenizer::ParseFloat("1.2"));
+  EXPECT_DOUBLE_EQ(1.e2, Tokenizer::ParseFloat("1.e2"));
+
+  // Test invalid integers that may still be tokenized as integers.
+  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e"));
+  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e-"));
+  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.e"));
+
+  // Test 'f' suffix.
+  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1f"));
+  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.0f"));
+  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1F"));
+
+  // These should parse successfully even though they are out of range.
+  // Overflows become infinity and underflows become zero.
+  EXPECT_EQ(0.0, Tokenizer::ParseFloat("1e-9999999999999999999999999999"));
+  EXPECT_EQ(HUGE_VAL, Tokenizer::ParseFloat("1e+9999999999999999999999999999"));
+
+#ifdef PROTOBUF_HAS_DEATH_TEST  // death tests do not work on Windows yet
+  // Test invalid integers that will never be tokenized as integers.
+  EXPECT_DEBUG_DEATH(
+      Tokenizer::ParseFloat("zxy"),
+      "passed text that could not have been tokenized as a float");
+  EXPECT_DEBUG_DEATH(
+      Tokenizer::ParseFloat("1-e0"),
+      "passed text that could not have been tokenized as a float");
+  EXPECT_DEBUG_DEATH(
+      Tokenizer::ParseFloat("-1.0"),
+      "passed text that could not have been tokenized as a float");
+#endif  // PROTOBUF_HAS_DEATH_TEST
+}
+
+TEST_F(TokenizerTest, ParseString) {
+  std::string output;
+  Tokenizer::ParseString("'hello'", &output);
+  EXPECT_EQ("hello", output);
+  Tokenizer::ParseString("\"blah\\nblah2\"", &output);
+  EXPECT_EQ("blah\nblah2", output);
+  Tokenizer::ParseString("'\\1x\\1\\123\\739\\52\\334n\\3'", &output);
+  EXPECT_EQ("\1x\1\123\739\52\334n\3", output);
+  Tokenizer::ParseString("'\\x20\\x4'", &output);
+  EXPECT_EQ("\x20\x4", output);
+
+  // Test invalid strings that may still be tokenized as strings.
+  Tokenizer::ParseString("\"\\a\\l\\v\\t", &output);  // \l is invalid
+  EXPECT_EQ("\a?\v\t", output);
+  Tokenizer::ParseString("'", &output);
+  EXPECT_EQ("", output);
+  Tokenizer::ParseString("'\\", &output);
+  EXPECT_EQ("\\", output);
+
+  // Experiment with Unicode escapes. Here are one-, two- and three-byte Unicode
+  // characters.
+  Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\U00024b62XX'", &output);
+  EXPECT_EQ("$¢€𤭢XX", output);
+  // Same thing encoded using UTF16.
+  Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\ud852\\udf62XX'", &output);
+  EXPECT_EQ("$¢€𤭢XX", output);
+  // Here's some broken UTF16; there's a head surrogate with no tail surrogate.
+  // We just output this as if it were UTF8; it's not a defined code point, but
+  // it has a defined encoding.
+  Tokenizer::ParseString("'\\ud852XX'", &output);
+  EXPECT_EQ("\xed\xa1\x92XX", output);
+  // Malformed escape: Demons may fly out of the nose.
+  Tokenizer::ParseString("'\\u0'", &output);
+  EXPECT_EQ("u0", output);
+  // Beyond the range of valid UTF-32 code units.
+  Tokenizer::ParseString("'\\U00110000\\U00200000\\UFFFFFFFF'", &output);
+  EXPECT_EQ("\\U00110000\\U00200000\\Uffffffff", output);
+
+  // Test invalid strings that will never be tokenized as strings.
+#ifdef PROTOBUF_HAS_DEATH_TEST  // death tests do not work on Windows yet
+  EXPECT_DEBUG_DEATH(
+      Tokenizer::ParseString("", &output),
+      "passed text that could not have been tokenized as a string");
+#endif  // PROTOBUF_HAS_DEATH_TEST
+}
+
+TEST_F(TokenizerTest, ParseStringAppend) {
+  // Check that ParseString and ParseStringAppend differ.
+  std::string output("stuff+");
+  Tokenizer::ParseStringAppend("'hello'", &output);
+  EXPECT_EQ("stuff+hello", output);
+  Tokenizer::ParseString("'hello'", &output);
+  EXPECT_EQ("hello", output);
+}
+
+// -------------------------------------------------------------------
+
+// Each case parses some input text, ignoring the tokens produced, and
+// checks that the error output matches what is expected.
+struct ErrorCase {
+  std::string input;
+  bool recoverable;  // True if the tokenizer should be able to recover and
+                     // parse more tokens after seeing this error.  Cases
+                     // for which this is true must end with "foo" as
+                     // the last token, which the test will check for.
+  const char* errors;
+};
+
+inline std::ostream& operator<<(std::ostream& out, const ErrorCase& test_case) {
+  return out << CEscape(test_case.input);
+}
+
+ErrorCase kErrorCases[] = {
+    // String errors.
+    {"'\\l' foo", true, "0:2: Invalid escape sequence in string literal.\n"},
+    {"'\\X' foo", true, "0:2: Invalid escape sequence in string literal.\n"},
+    {"'\\x' foo", true, "0:3: Expected hex digits for escape sequence.\n"},
+    {"'foo", false, "0:4: Unexpected end of string.\n"},
+    {"'bar\nfoo", true, "0:4: String literals cannot cross line boundaries.\n"},
+    {"'\\u01' foo", true,
+     "0:5: Expected four hex digits for \\u escape sequence.\n"},
+    {"'\\u01' foo", true,
+     "0:5: Expected four hex digits for \\u escape sequence.\n"},
+    {"'\\uXYZ' foo", true,
+     "0:3: Expected four hex digits for \\u escape sequence.\n"},
+
+    // Integer errors.
+    {"123foo", true, "0:3: Need space between number and identifier.\n"},
+
+    // Hex/octal errors.
+    {"0x foo", true, "0:2: \"0x\" must be followed by hex digits.\n"},
+    {"0541823 foo", true,
+     "0:4: Numbers starting with leading zero must be in octal.\n"},
+    {"0x123z foo", true, "0:5: Need space between number and identifier.\n"},
+    {"0x123.4 foo", true, "0:5: Hex and octal numbers must be integers.\n"},
+    {"0123.4 foo", true, "0:4: Hex and octal numbers must be integers.\n"},
+
+    // Float errors.
+    {"1e foo", true, "0:2: \"e\" must be followed by exponent.\n"},
+    {"1e- foo", true, "0:3: \"e\" must be followed by exponent.\n"},
+    {"1.2.3 foo", true,
+     "0:3: Already saw decimal point or exponent; can't have another one.\n"},
+    {"1e2.3 foo", true,
+     "0:3: Already saw decimal point or exponent; can't have another one.\n"},
+    {"a.1 foo", true,
+     "0:1: Need space between identifier and decimal point.\n"},
+    // allow_f_after_float not enabled, so this should be an error.
+    {"1.0f foo", true, "0:3: Need space between number and identifier.\n"},
+
+    // Block comment errors.
+    {"/*", false,
+     "0:2: End-of-file inside block comment.\n"
+     "0:0:   Comment started here.\n"},
+    {"/*/*/ foo", true,
+     "0:3: \"/*\" inside block comment.  Block comments cannot be nested.\n"},
+
+    // Control characters.  Multiple consecutive control characters should only
+    // produce one error.
+    {"\b foo", true, "0:0: Invalid control characters encountered in text.\n"},
+    {"\b\b foo", true,
+     "0:0: Invalid control characters encountered in text.\n"},
+
+    // Check that control characters at end of input don't result in an
+    // infinite loop.
+    {"\b", false, "0:0: Invalid control characters encountered in text.\n"},
+
+    // Check recovery from '\0'.  We have to explicitly specify the length of
+    // these strings because otherwise the string constructor will just call
+    // strlen() which will see the first '\0' and think that is the end of the
+    // string.
+    {std::string("\0foo", 4), true,
+     "0:0: Invalid control characters encountered in text.\n"},
+    {std::string("\0\0foo", 5), true,
+     "0:0: Invalid control characters encountered in text.\n"},
+
+    // Check error from high order bits set
+    {"\300foo", true, "0:0: Interpreting non ascii codepoint 192.\n"},
+};
+
+TEST_2D(TokenizerTest, Errors, kErrorCases, kBlockSizes) {
+  // Set up the tokenizer.
+  TestInputStream input(kErrorCases_case.input.data(),
+                        kErrorCases_case.input.size(), kBlockSizes_case);
+  TestErrorCollector error_collector;
+  Tokenizer tokenizer(&input, &error_collector);
+
+  // Ignore all input, except remember if the last token was "foo".
+  bool last_was_foo = false;
+  while (tokenizer.Next()) {
+    last_was_foo = tokenizer.current().text == "foo";
+  }
+
+  // Check that the errors match what was expected.
+  EXPECT_EQ(kErrorCases_case.errors, error_collector.text_);
+
+  // If the error was recoverable, make sure we saw "foo" after it.
+  if (kErrorCases_case.recoverable) {
+    EXPECT_TRUE(last_was_foo);
+  }
+}
+
+// -------------------------------------------------------------------
+
+TEST_1D(TokenizerTest, BackUpOnDestruction, kBlockSizes) {
+  std::string text = "foo bar";
+  TestInputStream input(text.data(), text.size(), kBlockSizes_case);
+
+  // Create a tokenizer, read one token, then destroy it.
+  {
+    TestErrorCollector error_collector;
+    Tokenizer tokenizer(&input, &error_collector);
+
+    tokenizer.Next();
+  }
+
+  // Only "foo" should have been read.
+  EXPECT_EQ(strlen("foo"), input.ByteCount());
+}
+
+
+}  // namespace
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google

ubuntu/google/protobuf/io/zero_copy_stream.cc

@@ -0,0 +1,55 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+#include <google/protobuf/io/zero_copy_stream.h>
+
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+
+bool ZeroCopyOutputStream::WriteAliasedRaw(const void* /* data */,
+                                           int /* size */) {
+  GOOGLE_LOG(FATAL) << "This ZeroCopyOutputStream doesn't support aliasing. "
+                "Reaching here usually means a ZeroCopyOutputStream "
+                "implementation bug.";
+  return false;
+}
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google

ubuntu/google/protobuf/io/zero_copy_stream.h

@@ -0,0 +1,253 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// This file contains the ZeroCopyInputStream and ZeroCopyOutputStream
+// interfaces, which represent abstract I/O streams to and from which
+// protocol buffers can be read and written.  For a few simple
+// implementations of these interfaces, see zero_copy_stream_impl.h.
+//
+// These interfaces are different from classic I/O streams in that they
+// try to minimize the amount of data copying that needs to be done.
+// To accomplish this, responsibility for allocating buffers is moved to
+// the stream object, rather than being the responsibility of the caller.
+// So, the stream can return a buffer which actually points directly into
+// the final data structure where the bytes are to be stored, and the caller
+// can interact directly with that buffer, eliminating an intermediate copy
+// operation.
+//
+// As an example, consider the common case in which you are reading bytes
+// from an array that is already in memory (or perhaps an mmap()ed file).
+// With classic I/O streams, you would do something like:
+//   char buffer[BUFFER_SIZE];
+//   input->Read(buffer, BUFFER_SIZE);
+//   DoSomething(buffer, BUFFER_SIZE);
+// Then, the stream basically just calls memcpy() to copy the data from
+// the array into your buffer.  With a ZeroCopyInputStream, you would do
+// this instead:
+//   const void* buffer;
+//   int size;
+//   input->Next(&buffer, &size);
+//   DoSomething(buffer, size);
+// Here, no copy is performed.  The input stream returns a pointer directly
+// into the backing array, and the caller ends up reading directly from it.
+//
+// If you want to be able to read the old-fashion way, you can create
+// a CodedInputStream or CodedOutputStream wrapping these objects and use
+// their ReadRaw()/WriteRaw() methods.  These will, of course, add a copy
+// step, but Coded*Stream will handle buffering so at least it will be
+// reasonably efficient.
+//
+// ZeroCopyInputStream example:
+//   // Read in a file and print its contents to stdout.
+//   int fd = open("myfile", O_RDONLY);
+//   ZeroCopyInputStream* input = new FileInputStream(fd);
+//
+//   const void* buffer;
+//   int size;
+//   while (input->Next(&buffer, &size)) {
+//     cout.write(buffer, size);
+//   }
+//
+//   delete input;
+//   close(fd);
+//
+// ZeroCopyOutputStream example:
+//   // Copy the contents of "infile" to "outfile", using plain read() for
+//   // "infile" but a ZeroCopyOutputStream for "outfile".
+//   int infd = open("infile", O_RDONLY);
+//   int outfd = open("outfile", O_WRONLY);
+//   ZeroCopyOutputStream* output = new FileOutputStream(outfd);
+//
+//   void* buffer;
+//   int size;
+//   while (output->Next(&buffer, &size)) {
+//     int bytes = read(infd, buffer, size);
+//     if (bytes < size) {
+//       // Reached EOF.
+//       output->BackUp(size - bytes);
+//       break;
+//     }
+//   }
+//
+//   delete output;
+//   close(infd);
+//   close(outfd);
+
+#ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__
+#define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__
+
+
+#include <string>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/port_def.inc>
+
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+// Defined in this file.
+class ZeroCopyInputStream;
+class ZeroCopyOutputStream;
+
+// Abstract interface similar to an input stream but designed to minimize
+// copying.
+class PROTOBUF_EXPORT ZeroCopyInputStream {
+ public:
+  ZeroCopyInputStream() {}
+  virtual ~ZeroCopyInputStream() {}
+
+  // Obtains a chunk of data from the stream.
+  //
+  // Preconditions:
+  // * "size" and "data" are not NULL.
+  //
+  // Postconditions:
+  // * If the returned value is false, there is no more data to return or
+  //   an error occurred.  All errors are permanent.
+  // * Otherwise, "size" points to the actual number of bytes read and "data"
+  //   points to a pointer to a buffer containing these bytes.
+  // * Ownership of this buffer remains with the stream, and the buffer
+  //   remains valid only until some other method of the stream is called
+  //   or the stream is destroyed.
+  // * It is legal for the returned buffer to have zero size, as long
+  //   as repeatedly calling Next() eventually yields a buffer with non-zero
+  //   size.
+  virtual bool Next(const void** data, int* size) = 0;
+
+  // Backs up a number of bytes, so that the next call to Next() returns
+  // data again that was already returned by the last call to Next().  This
+  // is useful when writing procedures that are only supposed to read up
+  // to a certain point in the input, then return.  If Next() returns a
+  // buffer that goes beyond what you wanted to read, you can use BackUp()
+  // to return to the point where you intended to finish.
+  //
+  // Preconditions:
+  // * The last method called must have been Next().
+  // * count must be less than or equal to the size of the last buffer
+  //   returned by Next().
+  //
+  // Postconditions:
+  // * The last "count" bytes of the last buffer returned by Next() will be
+  //   pushed back into the stream.  Subsequent calls to Next() will return
+  //   the same data again before producing new data.
+  virtual void BackUp(int count) = 0;
+
+  // Skips a number of bytes.  Returns false if the end of the stream is
+  // reached or some input error occurred.  In the end-of-stream case, the
+  // stream is advanced to the end of the stream (so ByteCount() will return
+  // the total size of the stream).
+  virtual bool Skip(int count) = 0;
+
+  // Returns the total number of bytes read since this object was created.
+  virtual int64_t ByteCount() const = 0;
+
+
+ private:
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ZeroCopyInputStream);
+};
+
+// Abstract interface similar to an output stream but designed to minimize
+// copying.
+class PROTOBUF_EXPORT ZeroCopyOutputStream {
+ public:
+  ZeroCopyOutputStream() {}
+  virtual ~ZeroCopyOutputStream() {}
+
+  // Obtains a buffer into which data can be written.  Any data written
+  // into this buffer will eventually (maybe instantly, maybe later on)
+  // be written to the output.
+  //
+  // Preconditions:
+  // * "size" and "data" are not NULL.
+  //
+  // Postconditions:
+  // * If the returned value is false, an error occurred.  All errors are
+  //   permanent.
+  // * Otherwise, "size" points to the actual number of bytes in the buffer
+  //   and "data" points to the buffer.
+  // * Ownership of this buffer remains with the stream, and the buffer
+  //   remains valid only until some other method of the stream is called
+  //   or the stream is destroyed.
+  // * Any data which the caller stores in this buffer will eventually be
+  //   written to the output (unless BackUp() is called).
+  // * It is legal for the returned buffer to have zero size, as long
+  //   as repeatedly calling Next() eventually yields a buffer with non-zero
+  //   size.
+  virtual bool Next(void** data, int* size) = 0;
+
+  // Backs up a number of bytes, so that the end of the last buffer returned
+  // by Next() is not actually written.  This is needed when you finish
+  // writing all the data you want to write, but the last buffer was bigger
+  // than you needed.  You don't want to write a bunch of garbage after the
+  // end of your data, so you use BackUp() to back up.
+  //
+  // Preconditions:
+  // * The last method called must have been Next().
+  // * count must be less than or equal to the size of the last buffer
+  //   returned by Next().
+  // * The caller must not have written anything to the last "count" bytes
+  //   of that buffer.
+  //
+  // Postconditions:
+  // * The last "count" bytes of the last buffer returned by Next() will be
+  //   ignored.
+  virtual void BackUp(int count) = 0;
+
+  // Returns the total number of bytes written since this object was created.
+  virtual int64_t ByteCount() const = 0;
+
+  // Write a given chunk of data to the output.  Some output streams may
+  // implement this in a way that avoids copying. Check AllowsAliasing() before
+  // calling WriteAliasedRaw(). It will GOOGLE_CHECK fail if WriteAliasedRaw() is
+  // called on a stream that does not allow aliasing.
+  //
+  // NOTE: It is caller's responsibility to ensure that the chunk of memory
+  // remains live until all of the data has been consumed from the stream.
+  virtual bool WriteAliasedRaw(const void* data, int size);
+  virtual bool AllowsAliasing() const { return false; }
+
+
+ private:
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ZeroCopyOutputStream);
+};
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
+#endif  // GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__

ubuntu/google/protobuf/io/zero_copy_stream_impl.cc

@@ -0,0 +1,366 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+#ifndef _MSC_VER
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#endif
+#include <errno.h>
+
+#include <algorithm>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/io/io_win32.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+#include <google/protobuf/stubs/stl_util.h>
+
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+#ifdef _WIN32
+// Win32 lseek is broken:  If invoked on a non-seekable file descriptor, its
+// return value is undefined.  We re-define it to always produce an error.
+#define lseek(fd, offset, origin) ((off_t)-1)
+// DO NOT include <io.h>, instead create functions in io_win32.{h,cc} and import
+// them like we do below.
+using google::protobuf::io::win32::access;
+using google::protobuf::io::win32::close;
+using google::protobuf::io::win32::open;
+using google::protobuf::io::win32::read;
+using google::protobuf::io::win32::write;
+#endif
+
+namespace {
+
+// EINTR sucks.
+int close_no_eintr(int fd) {
+  int result;
+  do {
+    result = close(fd);
+  } while (result < 0 && errno == EINTR);
+  return result;
+}
+
+}  // namespace
+
+// ===================================================================
+
+FileInputStream::FileInputStream(int file_descriptor, int block_size)
+    : copying_input_(file_descriptor), impl_(&copying_input_, block_size) {}
+
+bool FileInputStream::Close() { return copying_input_.Close(); }
+
+bool FileInputStream::Next(const void** data, int* size) {
+  return impl_.Next(data, size);
+}
+
+void FileInputStream::BackUp(int count) { impl_.BackUp(count); }
+
+bool FileInputStream::Skip(int count) { return impl_.Skip(count); }
+
+int64_t FileInputStream::ByteCount() const { return impl_.ByteCount(); }
+
+FileInputStream::CopyingFileInputStream::CopyingFileInputStream(
+    int file_descriptor)
+    : file_(file_descriptor),
+      close_on_delete_(false),
+      is_closed_(false),
+      errno_(0),
+      previous_seek_failed_(false) {}
+
+FileInputStream::CopyingFileInputStream::~CopyingFileInputStream() {
+  if (close_on_delete_) {
+    if (!Close()) {
+      GOOGLE_LOG(ERROR) << "close() failed: " << strerror(errno_);
+    }
+  }
+}
+
+bool FileInputStream::CopyingFileInputStream::Close() {
+  GOOGLE_CHECK(!is_closed_);
+
+  is_closed_ = true;
+  if (close_no_eintr(file_) != 0) {
+    // The docs on close() do not specify whether a file descriptor is still
+    // open after close() fails with EIO.  However, the glibc source code
+    // seems to indicate that it is not.
+    errno_ = errno;
+    return false;
+  }
+
+  return true;
+}
+
+int FileInputStream::CopyingFileInputStream::Read(void* buffer, int size) {
+  GOOGLE_CHECK(!is_closed_);
+
+  int result;
+  do {
+    result = read(file_, buffer, size);
+  } while (result < 0 && errno == EINTR);
+
+  if (result < 0) {
+    // Read error (not EOF).
+    errno_ = errno;
+  }
+
+  return result;
+}
+
+int FileInputStream::CopyingFileInputStream::Skip(int count) {
+  GOOGLE_CHECK(!is_closed_);
+
+  if (!previous_seek_failed_ && lseek(file_, count, SEEK_CUR) != (off_t)-1) {
+    // Seek succeeded.
+    return count;
+  } else {
+    // Failed to seek.
+
+    // Note to self:  Don't seek again.  This file descriptor doesn't
+    // support it.
+    previous_seek_failed_ = true;
+
+    // Use the default implementation.
+    return CopyingInputStream::Skip(count);
+  }
+}
+
+// ===================================================================
+
+FileOutputStream::FileOutputStream(int file_descriptor, int block_size)
+    : CopyingOutputStreamAdaptor(&copying_output_),
+      copying_output_(file_descriptor) {}
+
+bool FileOutputStream::Close() {
+  bool flush_succeeded = Flush();
+  return copying_output_.Close() && flush_succeeded;
+}
+
+FileOutputStream::CopyingFileOutputStream::CopyingFileOutputStream(
+    int file_descriptor)
+    : file_(file_descriptor),
+      close_on_delete_(false),
+      is_closed_(false),
+      errno_(0) {}
+
+FileOutputStream::~FileOutputStream() { Flush(); }
+
+FileOutputStream::CopyingFileOutputStream::~CopyingFileOutputStream() {
+  if (close_on_delete_) {
+    if (!Close()) {
+      GOOGLE_LOG(ERROR) << "close() failed: " << strerror(errno_);
+    }
+  }
+}
+
+bool FileOutputStream::CopyingFileOutputStream::Close() {
+  GOOGLE_CHECK(!is_closed_);
+
+  is_closed_ = true;
+  if (close_no_eintr(file_) != 0) {
+    // The docs on close() do not specify whether a file descriptor is still
+    // open after close() fails with EIO.  However, the glibc source code
+    // seems to indicate that it is not.
+    errno_ = errno;
+    return false;
+  }
+
+  return true;
+}
+
+bool FileOutputStream::CopyingFileOutputStream::Write(const void* buffer,
+                                                      int size) {
+  GOOGLE_CHECK(!is_closed_);
+  int total_written = 0;
+
+  const uint8* buffer_base = reinterpret_cast<const uint8*>(buffer);
+
+  while (total_written < size) {
+    int bytes;
+    do {
+      bytes = write(file_, buffer_base + total_written, size - total_written);
+    } while (bytes < 0 && errno == EINTR);
+
+    if (bytes <= 0) {
+      // Write error.
+
+      // FIXME(kenton):  According to the man page, if write() returns zero,
+      //   there was no error; write() simply did not write anything.  It's
+      //   unclear under what circumstances this might happen, but presumably
+      //   errno won't be set in this case.  I am confused as to how such an
+      //   event should be handled.  For now I'm treating it as an error, since
+      //   retrying seems like it could lead to an infinite loop.  I suspect
+      //   this never actually happens anyway.
+
+      if (bytes < 0) {
+        errno_ = errno;
+      }
+      return false;
+    }
+    total_written += bytes;
+  }
+
+  return true;
+}
+
+// ===================================================================
+
+IstreamInputStream::IstreamInputStream(std::istream* input, int block_size)
+    : copying_input_(input), impl_(&copying_input_, block_size) {}
+
+bool IstreamInputStream::Next(const void** data, int* size) {
+  return impl_.Next(data, size);
+}
+
+void IstreamInputStream::BackUp(int count) { impl_.BackUp(count); }
+
+bool IstreamInputStream::Skip(int count) { return impl_.Skip(count); }
+
+int64_t IstreamInputStream::ByteCount() const { return impl_.ByteCount(); }
+
+IstreamInputStream::CopyingIstreamInputStream::CopyingIstreamInputStream(
+    std::istream* input)
+    : input_(input) {}
+
+IstreamInputStream::CopyingIstreamInputStream::~CopyingIstreamInputStream() {}
+
+int IstreamInputStream::CopyingIstreamInputStream::Read(void* buffer,
+                                                        int size) {
+  input_->read(reinterpret_cast<char*>(buffer), size);
+  int result = input_->gcount();
+  if (result == 0 && input_->fail() && !input_->eof()) {
+    return -1;
+  }
+  return result;
+}
+
+// ===================================================================
+
+OstreamOutputStream::OstreamOutputStream(std::ostream* output, int block_size)
+    : copying_output_(output), impl_(&copying_output_, block_size) {}
+
+OstreamOutputStream::~OstreamOutputStream() { impl_.Flush(); }
+
+bool OstreamOutputStream::Next(void** data, int* size) {
+  return impl_.Next(data, size);
+}
+
+void OstreamOutputStream::BackUp(int count) { impl_.BackUp(count); }
+
+int64_t OstreamOutputStream::ByteCount() const { return impl_.ByteCount(); }
+
+OstreamOutputStream::CopyingOstreamOutputStream::CopyingOstreamOutputStream(
+    std::ostream* output)
+    : output_(output) {}
+
+OstreamOutputStream::CopyingOstreamOutputStream::~CopyingOstreamOutputStream() {
+}
+
+bool OstreamOutputStream::CopyingOstreamOutputStream::Write(const void* buffer,
+                                                            int size) {
+  output_->write(reinterpret_cast<const char*>(buffer), size);
+  return output_->good();
+}
+
+// ===================================================================
+
+ConcatenatingInputStream::ConcatenatingInputStream(
+    ZeroCopyInputStream* const streams[], int count)
+    : streams_(streams), stream_count_(count), bytes_retired_(0) {
+}
+
+bool ConcatenatingInputStream::Next(const void** data, int* size) {
+  while (stream_count_ > 0) {
+    if (streams_[0]->Next(data, size)) return true;
+
+    // That stream is done.  Advance to the next one.
+    bytes_retired_ += streams_[0]->ByteCount();
+    ++streams_;
+    --stream_count_;
+  }
+
+  // No more streams.
+  return false;
+}
+
+void ConcatenatingInputStream::BackUp(int count) {
+  if (stream_count_ > 0) {
+    streams_[0]->BackUp(count);
+  } else {
+    GOOGLE_LOG(DFATAL) << "Can't BackUp() after failed Next().";
+  }
+}
+
+bool ConcatenatingInputStream::Skip(int count) {
+  while (stream_count_ > 0) {
+    // Assume that ByteCount() can be used to find out how much we actually
+    // skipped when Skip() fails.
+    int64 target_byte_count = streams_[0]->ByteCount() + count;
+    if (streams_[0]->Skip(count)) return true;
+
+    // Hit the end of the stream.  Figure out how many more bytes we still have
+    // to skip.
+    int64 final_byte_count = streams_[0]->ByteCount();
+    GOOGLE_DCHECK_LT(final_byte_count, target_byte_count);
+    count = target_byte_count - final_byte_count;
+
+    // That stream is done.  Advance to the next one.
+    bytes_retired_ += final_byte_count;
+    ++streams_;
+    --stream_count_;
+  }
+
+  return false;
+}
+
+int64_t ConcatenatingInputStream::ByteCount() const {
+  if (stream_count_ == 0) {
+    return bytes_retired_;
+  } else {
+    return bytes_retired_ + streams_[0]->ByteCount();
+  }
+}
+
+
+// ===================================================================
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google

ubuntu/google/protobuf/io/zero_copy_stream_impl.h

@@ -0,0 +1,327 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// This file contains common implementations of the interfaces defined in
+// zero_copy_stream.h which are only included in the full (non-lite)
+// protobuf library.  These implementations include Unix file descriptors
+// and C++ iostreams.  See also:  zero_copy_stream_impl_lite.h
+
+#ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__
+#define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__
+
+
+#include <iosfwd>
+#include <string>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+
+#include <google/protobuf/port_def.inc>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+// ===================================================================
+
+// A ZeroCopyInputStream which reads from a file descriptor.
+//
+// FileInputStream is preferred over using an ifstream with IstreamInputStream.
+// The latter will introduce an extra layer of buffering, harming performance.
+// Also, it's conceivable that FileInputStream could someday be enhanced
+// to use zero-copy file descriptors on OSs which support them.
+class PROTOBUF_EXPORT FileInputStream : public ZeroCopyInputStream {
+ public:
+  // Creates a stream that reads from the given Unix file descriptor.
+  // If a block_size is given, it specifies the number of bytes that
+  // should be read and returned with each call to Next().  Otherwise,
+  // a reasonable default is used.
+  explicit FileInputStream(int file_descriptor, int block_size = -1);
+
+  // Flushes any buffers and closes the underlying file.  Returns false if
+  // an error occurs during the process; use GetErrno() to examine the error.
+  // Even if an error occurs, the file descriptor is closed when this returns.
+  bool Close();
+
+  // By default, the file descriptor is not closed when the stream is
+  // destroyed.  Call SetCloseOnDelete(true) to change that.  WARNING:
+  // This leaves no way for the caller to detect if close() fails.  If
+  // detecting close() errors is important to you, you should arrange
+  // to close the descriptor yourself.
+  void SetCloseOnDelete(bool value) { copying_input_.SetCloseOnDelete(value); }
+
+  // If an I/O error has occurred on this file descriptor, this is the
+  // errno from that error.  Otherwise, this is zero.  Once an error
+  // occurs, the stream is broken and all subsequent operations will
+  // fail.
+  int GetErrno() const { return copying_input_.GetErrno(); }
+
+  // implements ZeroCopyInputStream ----------------------------------
+  bool Next(const void** data, int* size) override;
+  void BackUp(int count) override;
+  bool Skip(int count) override;
+  int64_t ByteCount() const override;
+
+ private:
+  class PROTOBUF_EXPORT CopyingFileInputStream : public CopyingInputStream {
+   public:
+    CopyingFileInputStream(int file_descriptor);
+    ~CopyingFileInputStream() override;
+
+    bool Close();
+    void SetCloseOnDelete(bool value) { close_on_delete_ = value; }
+    int GetErrno() const { return errno_; }
+
+    // implements CopyingInputStream ---------------------------------
+    int Read(void* buffer, int size) override;
+    int Skip(int count) override;
+
+   private:
+    // The file descriptor.
+    const int file_;
+    bool close_on_delete_;
+    bool is_closed_;
+
+    // The errno of the I/O error, if one has occurred.  Otherwise, zero.
+    int errno_;
+
+    // Did we try to seek once and fail?  If so, we assume this file descriptor
+    // doesn't support seeking and won't try again.
+    bool previous_seek_failed_;
+
+    GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingFileInputStream);
+  };
+
+  CopyingFileInputStream copying_input_;
+  CopyingInputStreamAdaptor impl_;
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FileInputStream);
+};
+
+// ===================================================================
+
+// A ZeroCopyOutputStream which writes to a file descriptor.
+//
+// FileOutputStream is preferred over using an ofstream with
+// OstreamOutputStream.  The latter will introduce an extra layer of buffering,
+// harming performance.  Also, it's conceivable that FileOutputStream could
+// someday be enhanced to use zero-copy file descriptors on OSs which
+// support them.
+class PROTOBUF_EXPORT FileOutputStream : public CopyingOutputStreamAdaptor {
+ public:
+  // Creates a stream that writes to the given Unix file descriptor.
+  // If a block_size is given, it specifies the size of the buffers
+  // that should be returned by Next().  Otherwise, a reasonable default
+  // is used.
+  explicit FileOutputStream(int file_descriptor, int block_size = -1);
+
+  ~FileOutputStream() override;
+
+  // Flushes any buffers and closes the underlying file.  Returns false if
+  // an error occurs during the process; use GetErrno() to examine the error.
+  // Even if an error occurs, the file descriptor is closed when this returns.
+  bool Close();
+
+  // By default, the file descriptor is not closed when the stream is
+  // destroyed.  Call SetCloseOnDelete(true) to change that.  WARNING:
+  // This leaves no way for the caller to detect if close() fails.  If
+  // detecting close() errors is important to you, you should arrange
+  // to close the descriptor yourself.
+  void SetCloseOnDelete(bool value) { copying_output_.SetCloseOnDelete(value); }
+
+  // If an I/O error has occurred on this file descriptor, this is the
+  // errno from that error.  Otherwise, this is zero.  Once an error
+  // occurs, the stream is broken and all subsequent operations will
+  // fail.
+  int GetErrno() const { return copying_output_.GetErrno(); }
+
+ private:
+  class PROTOBUF_EXPORT CopyingFileOutputStream : public CopyingOutputStream {
+   public:
+    CopyingFileOutputStream(int file_descriptor);
+    ~CopyingFileOutputStream() override;
+
+    bool Close();
+    void SetCloseOnDelete(bool value) { close_on_delete_ = value; }
+    int GetErrno() const { return errno_; }
+
+    // implements CopyingOutputStream --------------------------------
+    bool Write(const void* buffer, int size) override;
+
+   private:
+    // The file descriptor.
+    const int file_;
+    bool close_on_delete_;
+    bool is_closed_;
+
+    // The errno of the I/O error, if one has occurred.  Otherwise, zero.
+    int errno_;
+
+    GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingFileOutputStream);
+  };
+
+  CopyingFileOutputStream copying_output_;
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FileOutputStream);
+};
+
+// ===================================================================
+
+// A ZeroCopyInputStream which reads from a C++ istream.
+//
+// Note that for reading files (or anything represented by a file descriptor),
+// FileInputStream is more efficient.
+class PROTOBUF_EXPORT IstreamInputStream : public ZeroCopyInputStream {
+ public:
+  // Creates a stream that reads from the given C++ istream.
+  // If a block_size is given, it specifies the number of bytes that
+  // should be read and returned with each call to Next().  Otherwise,
+  // a reasonable default is used.
+  explicit IstreamInputStream(std::istream* stream, int block_size = -1);
+
+  // implements ZeroCopyInputStream ----------------------------------
+  bool Next(const void** data, int* size) override;
+  void BackUp(int count) override;
+  bool Skip(int count) override;
+  int64_t ByteCount() const override;
+
+ private:
+  class PROTOBUF_EXPORT CopyingIstreamInputStream : public CopyingInputStream {
+   public:
+    CopyingIstreamInputStream(std::istream* input);
+    ~CopyingIstreamInputStream() override;
+
+    // implements CopyingInputStream ---------------------------------
+    int Read(void* buffer, int size) override;
+    // (We use the default implementation of Skip().)
+
+   private:
+    // The stream.
+    std::istream* input_;
+
+    GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingIstreamInputStream);
+  };
+
+  CopyingIstreamInputStream copying_input_;
+  CopyingInputStreamAdaptor impl_;
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(IstreamInputStream);
+};
+
+// ===================================================================
+
+// A ZeroCopyOutputStream which writes to a C++ ostream.
+//
+// Note that for writing files (or anything represented by a file descriptor),
+// FileOutputStream is more efficient.
+class PROTOBUF_EXPORT OstreamOutputStream : public ZeroCopyOutputStream {
+ public:
+  // Creates a stream that writes to the given C++ ostream.
+  // If a block_size is given, it specifies the size of the buffers
+  // that should be returned by Next().  Otherwise, a reasonable default
+  // is used.
+  explicit OstreamOutputStream(std::ostream* stream, int block_size = -1);
+  ~OstreamOutputStream() override;
+
+  // implements ZeroCopyOutputStream ---------------------------------
+  bool Next(void** data, int* size) override;
+  void BackUp(int count) override;
+  int64_t ByteCount() const override;
+
+ private:
+  class PROTOBUF_EXPORT CopyingOstreamOutputStream
+      : public CopyingOutputStream {
+   public:
+    CopyingOstreamOutputStream(std::ostream* output);
+    ~CopyingOstreamOutputStream() override;
+
+    // implements CopyingOutputStream --------------------------------
+    bool Write(const void* buffer, int size) override;
+
+   private:
+    // The stream.
+    std::ostream* output_;
+
+    GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingOstreamOutputStream);
+  };
+
+  CopyingOstreamOutputStream copying_output_;
+  CopyingOutputStreamAdaptor impl_;
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(OstreamOutputStream);
+};
+
+// ===================================================================
+
+// A ZeroCopyInputStream which reads from several other streams in sequence.
+// ConcatenatingInputStream is unable to distinguish between end-of-stream
+// and read errors in the underlying streams, so it assumes any errors mean
+// end-of-stream.  So, if the underlying streams fail for any other reason,
+// ConcatenatingInputStream may do odd things.  It is suggested that you do
+// not use ConcatenatingInputStream on streams that might produce read errors
+// other than end-of-stream.
+class PROTOBUF_EXPORT ConcatenatingInputStream : public ZeroCopyInputStream {
+ public:
+  // All streams passed in as well as the array itself must remain valid
+  // until the ConcatenatingInputStream is destroyed.
+  ConcatenatingInputStream(ZeroCopyInputStream* const streams[], int count);
+  ~ConcatenatingInputStream() override = default;
+
+  // implements ZeroCopyInputStream ----------------------------------
+  bool Next(const void** data, int* size) override;
+  void BackUp(int count) override;
+  bool Skip(int count) override;
+  int64_t ByteCount() const override;
+
+
+ private:
+  // As streams are retired, streams_ is incremented and count_ is
+  // decremented.
+  ZeroCopyInputStream* const* streams_;
+  int stream_count_;
+  int64 bytes_retired_;  // Bytes read from previous streams.
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ConcatenatingInputStream);
+};
+
+// ===================================================================
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
+#endif  // GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__

ubuntu/google/protobuf/io/zero_copy_stream_impl_lite.cc

@@ -0,0 +1,467 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+
+#include <algorithm>
+#include <limits>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/casts.h>
+#include <google/protobuf/stubs/stl_util.h>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+namespace {
+
+// Default block size for Copying{In,Out}putStreamAdaptor.
+static const int kDefaultBlockSize = 8192;
+
+}  // namespace
+
+// ===================================================================
+
+ArrayInputStream::ArrayInputStream(const void* data, int size, int block_size)
+    : data_(reinterpret_cast<const uint8*>(data)),
+      size_(size),
+      block_size_(block_size > 0 ? block_size : size),
+      position_(0),
+      last_returned_size_(0) {}
+
+bool ArrayInputStream::Next(const void** data, int* size) {
+  if (position_ < size_) {
+    last_returned_size_ = std::min(block_size_, size_ - position_);
+    *data = data_ + position_;
+    *size = last_returned_size_;
+    position_ += last_returned_size_;
+    return true;
+  } else {
+    // We're at the end of the array.
+    last_returned_size_ = 0;  // Don't let caller back up.
+    return false;
+  }
+}
+
+void ArrayInputStream::BackUp(int count) {
+  GOOGLE_CHECK_GT(last_returned_size_, 0)
+      << "BackUp() can only be called after a successful Next().";
+  GOOGLE_CHECK_LE(count, last_returned_size_);
+  GOOGLE_CHECK_GE(count, 0);
+  position_ -= count;
+  last_returned_size_ = 0;  // Don't let caller back up further.
+}
+
+bool ArrayInputStream::Skip(int count) {
+  GOOGLE_CHECK_GE(count, 0);
+  last_returned_size_ = 0;  // Don't let caller back up.
+  if (count > size_ - position_) {
+    position_ = size_;
+    return false;
+  } else {
+    position_ += count;
+    return true;
+  }
+}
+
+int64_t ArrayInputStream::ByteCount() const { return position_; }
+
+
+// ===================================================================
+
+ArrayOutputStream::ArrayOutputStream(void* data, int size, int block_size)
+    : data_(reinterpret_cast<uint8*>(data)),
+      size_(size),
+      block_size_(block_size > 0 ? block_size : size),
+      position_(0),
+      last_returned_size_(0) {}
+
+bool ArrayOutputStream::Next(void** data, int* size) {
+  if (position_ < size_) {
+    last_returned_size_ = std::min(block_size_, size_ - position_);
+    *data = data_ + position_;
+    *size = last_returned_size_;
+    position_ += last_returned_size_;
+    return true;
+  } else {
+    // We're at the end of the array.
+    last_returned_size_ = 0;  // Don't let caller back up.
+    return false;
+  }
+}
+
+void ArrayOutputStream::BackUp(int count) {
+  GOOGLE_CHECK_GT(last_returned_size_, 0)
+      << "BackUp() can only be called after a successful Next().";
+  GOOGLE_CHECK_LE(count, last_returned_size_);
+  GOOGLE_CHECK_GE(count, 0);
+  position_ -= count;
+  last_returned_size_ = 0;  // Don't let caller back up further.
+}
+
+int64_t ArrayOutputStream::ByteCount() const { return position_; }
+
+// ===================================================================
+
+StringOutputStream::StringOutputStream(std::string* target) : target_(target) {}
+
+bool StringOutputStream::Next(void** data, int* size) {
+  GOOGLE_CHECK(target_ != NULL);
+  size_t old_size = target_->size();
+
+  // Grow the string.
+  size_t new_size;
+  if (old_size < target_->capacity()) {
+    // Resize the string to match its capacity, since we can get away
+    // without a memory allocation this way.
+    new_size = target_->capacity();
+  } else {
+    // Size has reached capacity, try to double it.
+    new_size = old_size * 2;
+  }
+  // Avoid integer overflow in returned '*size'.
+  new_size = std::min(new_size, old_size + std::numeric_limits<int>::max());
+  // Increase the size, also make sure that it is at least kMinimumSize.
+  STLStringResizeUninitialized(
+      target_,
+      std::max(new_size,
+               kMinimumSize + 0));  // "+ 0" works around GCC4 weirdness.
+
+  *data = mutable_string_data(target_) + old_size;
+  *size = target_->size() - old_size;
+  return true;
+}
+
+void StringOutputStream::BackUp(int count) {
+  GOOGLE_CHECK_GE(count, 0);
+  GOOGLE_CHECK(target_ != NULL);
+  GOOGLE_CHECK_LE(count, target_->size());
+  target_->resize(target_->size() - count);
+}
+
+int64_t StringOutputStream::ByteCount() const {
+  GOOGLE_CHECK(target_ != NULL);
+  return target_->size();
+}
+
+// ===================================================================
+
+int CopyingInputStream::Skip(int count) {
+  char junk[4096];
+  int skipped = 0;
+  while (skipped < count) {
+    int bytes = Read(junk, std::min(count - skipped,
+                                    implicit_cast<int>(sizeof(junk))));
+    if (bytes <= 0) {
+      // EOF or read error.
+      return skipped;
+    }
+    skipped += bytes;
+  }
+  return skipped;
+}
+
+CopyingInputStreamAdaptor::CopyingInputStreamAdaptor(
+    CopyingInputStream* copying_stream, int block_size)
+    : copying_stream_(copying_stream),
+      owns_copying_stream_(false),
+      failed_(false),
+      position_(0),
+      buffer_size_(block_size > 0 ? block_size : kDefaultBlockSize),
+      buffer_used_(0),
+      backup_bytes_(0) {}
+
+CopyingInputStreamAdaptor::~CopyingInputStreamAdaptor() {
+  if (owns_copying_stream_) {
+    delete copying_stream_;
+  }
+}
+
+bool CopyingInputStreamAdaptor::Next(const void** data, int* size) {
+  if (failed_) {
+    // Already failed on a previous read.
+    return false;
+  }
+
+  AllocateBufferIfNeeded();
+
+  if (backup_bytes_ > 0) {
+    // We have data left over from a previous BackUp(), so just return that.
+    *data = buffer_.get() + buffer_used_ - backup_bytes_;
+    *size = backup_bytes_;
+    backup_bytes_ = 0;
+    return true;
+  }
+
+  // Read new data into the buffer.
+  buffer_used_ = copying_stream_->Read(buffer_.get(), buffer_size_);
+  if (buffer_used_ <= 0) {
+    // EOF or read error.  We don't need the buffer anymore.
+    if (buffer_used_ < 0) {
+      // Read error (not EOF).
+      failed_ = true;
+    }
+    FreeBuffer();
+    return false;
+  }
+  position_ += buffer_used_;
+
+  *size = buffer_used_;
+  *data = buffer_.get();
+  return true;
+}
+
+void CopyingInputStreamAdaptor::BackUp(int count) {
+  GOOGLE_CHECK(backup_bytes_ == 0 && buffer_.get() != NULL)
+      << " BackUp() can only be called after Next().";
+  GOOGLE_CHECK_LE(count, buffer_used_)
+      << " Can't back up over more bytes than were returned by the last call"
+         " to Next().";
+  GOOGLE_CHECK_GE(count, 0) << " Parameter to BackUp() can't be negative.";
+
+  backup_bytes_ = count;
+}
+
+bool CopyingInputStreamAdaptor::Skip(int count) {
+  GOOGLE_CHECK_GE(count, 0);
+
+  if (failed_) {
+    // Already failed on a previous read.
+    return false;
+  }
+
+  // First skip any bytes left over from a previous BackUp().
+  if (backup_bytes_ >= count) {
+    // We have more data left over than we're trying to skip.  Just chop it.
+    backup_bytes_ -= count;
+    return true;
+  }
+
+  count -= backup_bytes_;
+  backup_bytes_ = 0;
+
+  int skipped = copying_stream_->Skip(count);
+  position_ += skipped;
+  return skipped == count;
+}
+
+int64_t CopyingInputStreamAdaptor::ByteCount() const {
+  return position_ - backup_bytes_;
+}
+
+void CopyingInputStreamAdaptor::AllocateBufferIfNeeded() {
+  if (buffer_.get() == NULL) {
+    buffer_.reset(new uint8[buffer_size_]);
+  }
+}
+
+void CopyingInputStreamAdaptor::FreeBuffer() {
+  GOOGLE_CHECK_EQ(backup_bytes_, 0);
+  buffer_used_ = 0;
+  buffer_.reset();
+}
+
+// ===================================================================
+
+CopyingOutputStreamAdaptor::CopyingOutputStreamAdaptor(
+    CopyingOutputStream* copying_stream, int block_size)
+    : copying_stream_(copying_stream),
+      owns_copying_stream_(false),
+      failed_(false),
+      position_(0),
+      buffer_size_(block_size > 0 ? block_size : kDefaultBlockSize),
+      buffer_used_(0) {}
+
+CopyingOutputStreamAdaptor::~CopyingOutputStreamAdaptor() {
+  WriteBuffer();
+  if (owns_copying_stream_) {
+    delete copying_stream_;
+  }
+}
+
+bool CopyingOutputStreamAdaptor::Flush() { return WriteBuffer(); }
+
+bool CopyingOutputStreamAdaptor::Next(void** data, int* size) {
+  if (buffer_used_ == buffer_size_) {
+    if (!WriteBuffer()) return false;
+  }
+
+  AllocateBufferIfNeeded();
+
+  *data = buffer_.get() + buffer_used_;
+  *size = buffer_size_ - buffer_used_;
+  buffer_used_ = buffer_size_;
+  return true;
+}
+
+void CopyingOutputStreamAdaptor::BackUp(int count) {
+  GOOGLE_CHECK_GE(count, 0);
+  GOOGLE_CHECK_EQ(buffer_used_, buffer_size_)
+      << " BackUp() can only be called after Next().";
+  GOOGLE_CHECK_LE(count, buffer_used_)
+      << " Can't back up over more bytes than were returned by the last call"
+         " to Next().";
+
+  buffer_used_ -= count;
+}
+
+int64_t CopyingOutputStreamAdaptor::ByteCount() const {
+  return position_ + buffer_used_;
+}
+
+bool CopyingOutputStreamAdaptor::WriteAliasedRaw(const void* data, int size) {
+  if (size >= buffer_size_) {
+    if (!Flush() || !copying_stream_->Write(data, size)) {
+      return false;
+    }
+    GOOGLE_DCHECK_EQ(buffer_used_, 0);
+    position_ += size;
+    return true;
+  }
+
+  void* out;
+  int out_size;
+  while (true) {
+    if (!Next(&out, &out_size)) {
+      return false;
+    }
+
+    if (size <= out_size) {
+      std::memcpy(out, data, size);
+      BackUp(out_size - size);
+      return true;
+    }
+
+    std::memcpy(out, data, out_size);
+    data = static_cast<const char*>(data) + out_size;
+    size -= out_size;
+  }
+  return true;
+}
+
+
+bool CopyingOutputStreamAdaptor::WriteBuffer() {
+  if (failed_) {
+    // Already failed on a previous write.
+    return false;
+  }
+
+  if (buffer_used_ == 0) return true;
+
+  if (copying_stream_->Write(buffer_.get(), buffer_used_)) {
+    position_ += buffer_used_;
+    buffer_used_ = 0;
+    return true;
+  } else {
+    failed_ = true;
+    FreeBuffer();
+    return false;
+  }
+}
+
+void CopyingOutputStreamAdaptor::AllocateBufferIfNeeded() {
+  if (buffer_ == NULL) {
+    buffer_.reset(new uint8[buffer_size_]);
+  }
+}
+
+void CopyingOutputStreamAdaptor::FreeBuffer() {
+  buffer_used_ = 0;
+  buffer_.reset();
+}
+
+// ===================================================================
+
+LimitingInputStream::LimitingInputStream(ZeroCopyInputStream* input,
+                                         int64 limit)
+    : input_(input), limit_(limit) {
+  prior_bytes_read_ = input_->ByteCount();
+}
+
+LimitingInputStream::~LimitingInputStream() {
+  // If we overshot the limit, back up.
+  if (limit_ < 0) input_->BackUp(-limit_);
+}
+
+bool LimitingInputStream::Next(const void** data, int* size) {
+  if (limit_ <= 0) return false;
+  if (!input_->Next(data, size)) return false;
+
+  limit_ -= *size;
+  if (limit_ < 0) {
+    // We overshot the limit.  Reduce *size to hide the rest of the buffer.
+    *size += limit_;
+  }
+  return true;
+}
+
+void LimitingInputStream::BackUp(int count) {
+  if (limit_ < 0) {
+    input_->BackUp(count - limit_);
+    limit_ = count;
+  } else {
+    input_->BackUp(count);
+    limit_ += count;
+  }
+}
+
+bool LimitingInputStream::Skip(int count) {
+  if (count > limit_) {
+    if (limit_ < 0) return false;
+    input_->Skip(limit_);
+    limit_ = 0;
+    return false;
+  } else {
+    if (!input_->Skip(count)) return false;
+    limit_ -= count;
+    return true;
+  }
+}
+
+int64_t LimitingInputStream::ByteCount() const {
+  if (limit_ < 0) {
+    return input_->ByteCount() + limit_ - prior_bytes_read_;
+  } else {
+    return input_->ByteCount() - prior_bytes_read_;
+  }
+}
+
+
+// ===================================================================
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google

ubuntu/google/protobuf/io/zero_copy_stream_impl_lite.h

@@ -0,0 +1,408 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// This file contains common implementations of the interfaces defined in
+// zero_copy_stream.h which are included in the "lite" protobuf library.
+// These implementations cover I/O on raw arrays and strings, as well as
+// adaptors which make it easy to implement streams based on traditional
+// streams.  Of course, many users will probably want to write their own
+// implementations of these interfaces specific to the particular I/O
+// abstractions they prefer to use, but these should cover the most common
+// cases.
+
+#ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__
+#define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__
+
+
+#include <iosfwd>
+#include <memory>
+#include <string>
+
+#include <google/protobuf/stubs/callback.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/stubs/stl_util.h>
+
+
+#include <google/protobuf/port_def.inc>
+
+namespace google {
+namespace protobuf {
+namespace io {
+
+// ===================================================================
+
+// A ZeroCopyInputStream backed by an in-memory array of bytes.
+class PROTOBUF_EXPORT ArrayInputStream : public ZeroCopyInputStream {
+ public:
+  // Create an InputStream that returns the bytes pointed to by "data".
+  // "data" remains the property of the caller but must remain valid until
+  // the stream is destroyed.  If a block_size is given, calls to Next()
+  // will return data blocks no larger than the given size.  Otherwise, the
+  // first call to Next() returns the entire array.  block_size is mainly
+  // useful for testing; in production you would probably never want to set
+  // it.
+  ArrayInputStream(const void* data, int size, int block_size = -1);
+  ~ArrayInputStream() override = default;
+
+  // implements ZeroCopyInputStream ----------------------------------
+  bool Next(const void** data, int* size) override;
+  void BackUp(int count) override;
+  bool Skip(int count) override;
+  int64_t ByteCount() const override;
+
+
+ private:
+  const uint8* const data_;  // The byte array.
+  const int size_;           // Total size of the array.
+  const int block_size_;     // How many bytes to return at a time.
+
+  int position_;
+  int last_returned_size_;  // How many bytes we returned last time Next()
+                            // was called (used for error checking only).
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayInputStream);
+};
+
+// ===================================================================
+
+// A ZeroCopyOutputStream backed by an in-memory array of bytes.
+class PROTOBUF_EXPORT ArrayOutputStream : public ZeroCopyOutputStream {
+ public:
+  // Create an OutputStream that writes to the bytes pointed to by "data".
+  // "data" remains the property of the caller but must remain valid until
+  // the stream is destroyed.  If a block_size is given, calls to Next()
+  // will return data blocks no larger than the given size.  Otherwise, the
+  // first call to Next() returns the entire array.  block_size is mainly
+  // useful for testing; in production you would probably never want to set
+  // it.
+  ArrayOutputStream(void* data, int size, int block_size = -1);
+  ~ArrayOutputStream() override = default;
+
+  // implements ZeroCopyOutputStream ---------------------------------
+  bool Next(void** data, int* size) override;
+  void BackUp(int count) override;
+  int64_t ByteCount() const override;
+
+ private:
+  uint8* const data_;     // The byte array.
+  const int size_;        // Total size of the array.
+  const int block_size_;  // How many bytes to return at a time.
+
+  int position_;
+  int last_returned_size_;  // How many bytes we returned last time Next()
+                            // was called (used for error checking only).
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayOutputStream);
+};
+
+// ===================================================================
+
+// A ZeroCopyOutputStream which appends bytes to a string.
+class PROTOBUF_EXPORT StringOutputStream : public ZeroCopyOutputStream {
+ public:
+  // Create a StringOutputStream which appends bytes to the given string.
+  // The string remains property of the caller, but it is mutated in arbitrary
+  // ways and MUST NOT be accessed in any way until you're done with the
+  // stream. Either be sure there's no further usage, or (safest) destroy the
+  // stream before using the contents.
+  //
+  // Hint:  If you call target->reserve(n) before creating the stream,
+  //   the first call to Next() will return at least n bytes of buffer
+  //   space.
+  explicit StringOutputStream(std::string* target);
+  ~StringOutputStream() override = default;
+
+  // implements ZeroCopyOutputStream ---------------------------------
+  bool Next(void** data, int* size) override;
+  void BackUp(int count) override;
+  int64_t ByteCount() const override;
+
+ private:
+  static constexpr size_t kMinimumSize = 16;
+
+  std::string* target_;
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringOutputStream);
+};
+
+// Note:  There is no StringInputStream.  Instead, just create an
+// ArrayInputStream as follows:
+//   ArrayInputStream input(str.data(), str.size());
+
+// ===================================================================
+
+// A generic traditional input stream interface.
+//
+// Lots of traditional input streams (e.g. file descriptors, C stdio
+// streams, and C++ iostreams) expose an interface where every read
+// involves copying bytes into a buffer.  If you want to take such an
+// interface and make a ZeroCopyInputStream based on it, simply implement
+// CopyingInputStream and then use CopyingInputStreamAdaptor.
+//
+// CopyingInputStream implementations should avoid buffering if possible.
+// CopyingInputStreamAdaptor does its own buffering and will read data
+// in large blocks.
+class PROTOBUF_EXPORT CopyingInputStream {
+ public:
+  virtual ~CopyingInputStream() {}
+
+  // Reads up to "size" bytes into the given buffer.  Returns the number of
+  // bytes read.  Read() waits until at least one byte is available, or
+  // returns zero if no bytes will ever become available (EOF), or -1 if a
+  // permanent read error occurred.
+  virtual int Read(void* buffer, int size) = 0;
+
+  // Skips the next "count" bytes of input.  Returns the number of bytes
+  // actually skipped.  This will always be exactly equal to "count" unless
+  // EOF was reached or a permanent read error occurred.
+  //
+  // The default implementation just repeatedly calls Read() into a scratch
+  // buffer.
+  virtual int Skip(int count);
+};
+
+// A ZeroCopyInputStream which reads from a CopyingInputStream.  This is
+// useful for implementing ZeroCopyInputStreams that read from traditional
+// streams.  Note that this class is not really zero-copy.
+//
+// If you want to read from file descriptors or C++ istreams, this is
+// already implemented for you:  use FileInputStream or IstreamInputStream
+// respectively.
+class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream {
+ public:
+  // Creates a stream that reads from the given CopyingInputStream.
+  // If a block_size is given, it specifies the number of bytes that
+  // should be read and returned with each call to Next().  Otherwise,
+  // a reasonable default is used.  The caller retains ownership of
+  // copying_stream unless SetOwnsCopyingStream(true) is called.
+  explicit CopyingInputStreamAdaptor(CopyingInputStream* copying_stream,
+                                     int block_size = -1);
+  ~CopyingInputStreamAdaptor() override;
+
+  // Call SetOwnsCopyingStream(true) to tell the CopyingInputStreamAdaptor to
+  // delete the underlying CopyingInputStream when it is destroyed.
+  void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; }
+
+  // implements ZeroCopyInputStream ----------------------------------
+  bool Next(const void** data, int* size) override;
+  void BackUp(int count) override;
+  bool Skip(int count) override;
+  int64_t ByteCount() const override;
+
+ private:
+  // Insures that buffer_ is not NULL.
+  void AllocateBufferIfNeeded();
+  // Frees the buffer and resets buffer_used_.
+  void FreeBuffer();
+
+  // The underlying copying stream.
+  CopyingInputStream* copying_stream_;
+  bool owns_copying_stream_;
+
+  // True if we have seen a permanent error from the underlying stream.
+  bool failed_;
+
+  // The current position of copying_stream_, relative to the point where
+  // we started reading.
+  int64 position_;
+
+  // Data is read into this buffer.  It may be NULL if no buffer is currently
+  // in use.  Otherwise, it points to an array of size buffer_size_.
+  std::unique_ptr<uint8[]> buffer_;
+  const int buffer_size_;
+
+  // Number of valid bytes currently in the buffer (i.e. the size last
+  // returned by Next()).  0 <= buffer_used_ <= buffer_size_.
+  int buffer_used_;
+
+  // Number of bytes in the buffer which were backed up over by a call to
+  // BackUp().  These need to be returned again.
+  // 0 <= backup_bytes_ <= buffer_used_
+  int backup_bytes_;
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingInputStreamAdaptor);
+};
+
+// ===================================================================
+
+// A generic traditional output stream interface.
+//
+// Lots of traditional output streams (e.g. file descriptors, C stdio
+// streams, and C++ iostreams) expose an interface where every write
+// involves copying bytes from a buffer.  If you want to take such an
+// interface and make a ZeroCopyOutputStream based on it, simply implement
+// CopyingOutputStream and then use CopyingOutputStreamAdaptor.
+//
+// CopyingOutputStream implementations should avoid buffering if possible.
+// CopyingOutputStreamAdaptor does its own buffering and will write data
+// in large blocks.
+class PROTOBUF_EXPORT CopyingOutputStream {
+ public:
+  virtual ~CopyingOutputStream() {}
+
+  // Writes "size" bytes from the given buffer to the output.  Returns true
+  // if successful, false on a write error.
+  virtual bool Write(const void* buffer, int size) = 0;
+};
+
+// A ZeroCopyOutputStream which writes to a CopyingOutputStream.  This is
+// useful for implementing ZeroCopyOutputStreams that write to traditional
+// streams.  Note that this class is not really zero-copy.
+//
+// If you want to write to file descriptors or C++ ostreams, this is
+// already implemented for you:  use FileOutputStream or OstreamOutputStream
+// respectively.
+class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream {
+ public:
+  // Creates a stream that writes to the given Unix file descriptor.
+  // If a block_size is given, it specifies the size of the buffers
+  // that should be returned by Next().  Otherwise, a reasonable default
+  // is used.
+  explicit CopyingOutputStreamAdaptor(CopyingOutputStream* copying_stream,
+                                      int block_size = -1);
+  ~CopyingOutputStreamAdaptor() override;
+
+  // Writes all pending data to the underlying stream.  Returns false if a
+  // write error occurred on the underlying stream.  (The underlying
+  // stream itself is not necessarily flushed.)
+  bool Flush();
+
+  // Call SetOwnsCopyingStream(true) to tell the CopyingOutputStreamAdaptor to
+  // delete the underlying CopyingOutputStream when it is destroyed.
+  void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; }
+
+  // implements ZeroCopyOutputStream ---------------------------------
+  bool Next(void** data, int* size) override;
+  void BackUp(int count) override;
+  int64_t ByteCount() const override;
+  bool WriteAliasedRaw(const void* data, int size) override;
+  bool AllowsAliasing() const override { return true; }
+
+ private:
+  // Write the current buffer, if it is present.
+  bool WriteBuffer();
+  // Insures that buffer_ is not NULL.
+  void AllocateBufferIfNeeded();
+  // Frees the buffer.
+  void FreeBuffer();
+
+  // The underlying copying stream.
+  CopyingOutputStream* copying_stream_;
+  bool owns_copying_stream_;
+
+  // True if we have seen a permanent error from the underlying stream.
+  bool failed_;
+
+  // The current position of copying_stream_, relative to the point where
+  // we started writing.
+  int64 position_;
+
+  // Data is written from this buffer.  It may be NULL if no buffer is
+  // currently in use.  Otherwise, it points to an array of size buffer_size_.
+  std::unique_ptr<uint8[]> buffer_;
+  const int buffer_size_;
+
+  // Number of valid bytes currently in the buffer (i.e. the size last
+  // returned by Next()).  When BackUp() is called, we just reduce this.
+  // 0 <= buffer_used_ <= buffer_size_.
+  int buffer_used_;
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingOutputStreamAdaptor);
+};
+
+// ===================================================================
+
+// A ZeroCopyInputStream which wraps some other stream and limits it to
+// a particular byte count.
+class PROTOBUF_EXPORT LimitingInputStream : public ZeroCopyInputStream {
+ public:
+  LimitingInputStream(ZeroCopyInputStream* input, int64 limit);
+  ~LimitingInputStream() override;
+
+  // implements ZeroCopyInputStream ----------------------------------
+  bool Next(const void** data, int* size) override;
+  void BackUp(int count) override;
+  bool Skip(int count) override;
+  int64_t ByteCount() const override;
+
+
+ private:
+  ZeroCopyInputStream* input_;
+  int64 limit_;  // Decreases as we go, becomes negative if we overshoot.
+  int64 prior_bytes_read_;  // Bytes read on underlying stream at construction
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(LimitingInputStream);
+};
+
+
+// ===================================================================
+
+// mutable_string_data() and as_string_data() are workarounds to improve
+// the performance of writing new data to an existing string.  Unfortunately
+// the methods provided by the string class are suboptimal, and using memcpy()
+// is mildly annoying because it requires its pointer args to be non-NULL even
+// if we ask it to copy 0 bytes.  Furthermore, string_as_array() has the
+// property that it always returns NULL if its arg is the empty string, exactly
+// what we want to avoid if we're using it in conjunction with memcpy()!
+// With C++11, the desired memcpy() boils down to memcpy(..., &(*s)[0], size),
+// where s is a string*.  Without C++11, &(*s)[0] is not guaranteed to be safe,
+// so we use string_as_array(), and live with the extra logic that tests whether
+// *s is empty.
+
+// Return a pointer to mutable characters underlying the given string.  The
+// return value is valid until the next time the string is resized.  We
+// trust the caller to treat the return value as an array of length s->size().
+inline char* mutable_string_data(std::string* s) {
+  // This should be simpler & faster than string_as_array() because the latter
+  // is guaranteed to return NULL when *s is empty, so it has to check for that.
+  return &(*s)[0];
+}
+
+// as_string_data(s) is equivalent to
+//  ({ char* p = mutable_string_data(s); make_pair(p, p != NULL); })
+// Sometimes it's faster: in some scenarios p cannot be NULL, and then the
+// code can avoid that check.
+inline std::pair<char*, bool> as_string_data(std::string* s) {
+  char* p = mutable_string_data(s);
+  return std::make_pair(p, true);
+}
+
+}  // namespace io
+}  // namespace protobuf
+}  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
+#endif  // GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__