Implement go/streaming-proto native libs, part 2
Implement ProtoOutputStream.h to write protobuf format data.
Usage of dumping proto:
ProtoOutputStream proto(fd);
proto.write(fieldId, value1); // dump a single value
// start to dump a message:
long long token = proto.start(messageFieldId);
proto.write(nestedField1, nestedValue1);
...
proto.end(token);
fieldId will be generated by protoc-cpp plugin(TBD). It is an encoded uint64_t
value, with 0 - 32 bits as its proto number, 33 - 40 bits reserved for
field type, int32, bool, string, message, etc. and 41 - 43 bits for
single, repeated or packed type. Currently packed field is not
supported.
Bug: 65641021
Test: N/A, need to wait for protoc-cpp plugin and will test in
incident_helper
Change-Id: Ic188615b950235aae0edeee4876b78d31feb5619
diff --git a/cmds/incidentd/src/PrivacyBuffer.cpp b/cmds/incidentd/src/PrivacyBuffer.cpp
index 07a064cf..37f6ed7 100644
--- a/cmds/incidentd/src/PrivacyBuffer.cpp
+++ b/cmds/incidentd/src/PrivacyBuffer.cpp
@@ -33,18 +33,18 @@
{
EncodedBuffer::Pointer snapshot = iter->rp()->copy();
size_t bytesToWrite = 0;
- uint32_t varint = 0;
+ uint64_t varint = 0;
switch (wireType) {
case WIRE_TYPE_VARINT:
varint = iter->readRawVarint();
- if(!skip) return buf->writeRawVarint(varint);
+ if(!skip) return buf->writeRawVarint64(varint);
break;
case WIRE_TYPE_FIXED64:
bytesToWrite = 8;
break;
case WIRE_TYPE_LENGTH_DELIMITED:
bytesToWrite = iter->readRawVarint();
- if(!skip) buf->writeRawVarint(bytesToWrite);
+ if(!skip) buf->writeRawVarint32(bytesToWrite);
break;
case WIRE_TYPE_FIXED32:
bytesToWrite = 4;
@@ -76,7 +76,6 @@
uint8_t wireType = read_wire_type(varint);
uint32_t fieldId = read_field_id(varint);
const Privacy* policy = parentPolicy->lookup(fieldId);
-
if (policy == NULL || !policy->IsMessageType() || !policy->HasChildren()) {
bool skip = !spec.CheckPremission(policy);
size_t amt = buf->size();
@@ -99,7 +98,7 @@
}
buf->writeHeader(fieldId, wireType);
- buf->writeRawVarint(finalSize);
+ buf->writeRawVarint32(finalSize);
while (!q.empty()) {
EncodedBuffer* subField = q.front();
EncodedBuffer::iterator it = subField->begin();
diff --git a/libs/protoutil/Android.mk b/libs/protoutil/Android.mk
index a534816..2a2b087 100644
--- a/libs/protoutil/Android.mk
+++ b/libs/protoutil/Android.mk
@@ -22,15 +22,15 @@
-Wall -Werror -Wno-missing-field-initializers -Wno-unused-variable -Wunused-parameter
LOCAL_SHARED_LIBRARIES := \
- libbinder \
+ libcutils \
liblog \
- libutils
LOCAL_C_INCLUDES := \
$(LOCAL_PATH)/include
LOCAL_SRC_FILES := \
src/EncodedBuffer.cpp \
+ src/ProtoOutputStream.cpp \
src/protobuf.cpp \
LOCAL_EXPORT_C_INCLUDE_DIRS := $(LOCAL_PATH)/include
diff --git a/libs/protoutil/include/android/util/EncodedBuffer.h b/libs/protoutil/include/android/util/EncodedBuffer.h
index cf09609..e568e4c 100644
--- a/libs/protoutil/include/android/util/EncodedBuffer.h
+++ b/libs/protoutil/include/android/util/EncodedBuffer.h
@@ -52,10 +52,10 @@
size_t index() const;
size_t offset() const;
- void move(size_t amt);
- inline void move() { move(1); };
+ Pointer* move(size_t amt);
+ inline Pointer* move() { return move(1); };
+ Pointer* rewind();
- void rewind();
Pointer copy() const;
private:
@@ -88,15 +88,71 @@
size_t currentToWrite();
/**
- * Write a varint into a vector. Return the size of the varint.
+ * Write a single byte to the buffer.
*/
- size_t writeRawVarint(uint32_t val);
+ void writeRawByte(uint8_t val);
+
+ /**
+ * Write a varint32 into the buffer. Return the size of the varint.
+ */
+ size_t writeRawVarint32(uint32_t val);
+
+ /**
+ * Write a varint64 into the buffer. Return the size of the varint.
+ */
+ size_t writeRawVarint64(uint64_t val);
+
+ /**
+ * Write Fixed32 into the buffer.
+ */
+ void writeRawFixed32(uint32_t val);
+
+ /**
+ * Write Fixed64 into the buffer.
+ */
+ void writeRawFixed64(uint64_t val);
/**
* Write a protobuf header. Return the size of the header.
*/
size_t writeHeader(uint32_t fieldId, uint8_t wireType);
+ /********************************* Edit APIs ************************************************/
+ /**
+ * Returns the edit pointer.
+ */
+ Pointer* ep();
+
+ /**
+ * Read a single byte at ep, and move ep to next byte;
+ */
+ uint8_t readRawByte();
+
+ /**
+ * Read varint starting at ep, ep will move to pos of next byte.
+ */
+ uint64_t readRawVarint();
+
+ /**
+ * Read 4 bytes starting at ep, ep will move to pos of next byte.
+ */
+ uint32_t readRawFixed32();
+
+ /**
+ * Read 8 bytes starting at ep, ep will move to pos of next byte.
+ */
+ uint64_t readRawFixed64();
+
+ /**
+ * Edit 4 bytes starting at pos.
+ */
+ void editRawFixed32(size_t pos, uint32_t val);
+
+ /**
+ * Copy _size_ bytes of data starting at __srcPos__ to wp.
+ */
+ void copy(size_t srcPos, size_t size);
+
/********************************* Read APIs ************************************************/
class iterator;
friend class iterator;
@@ -141,9 +197,8 @@
/**
* Read varint from iterator, the iterator will point to next available byte.
- * Return the number of bytes of the varint.
*/
- uint32_t readRawVarint();
+ uint64_t readRawVarint();
private:
const EncodedBuffer& mData;
@@ -160,6 +215,7 @@
vector<uint8_t*> mBuffers;
Pointer mWp;
+ Pointer mEp;
inline uint8_t* at(const Pointer& p) const; // helper function to get value
};
@@ -167,4 +223,5 @@
} // util
} // android
-#endif // ANDROID_UTIL_ENCODED_BUFFER_H
\ No newline at end of file
+#endif // ANDROID_UTIL_ENCODED_BUFFER_H
+
diff --git a/libs/protoutil/include/android/util/ProtoOutputStream.h b/libs/protoutil/include/android/util/ProtoOutputStream.h
new file mode 100644
index 0000000..49ec169
--- /dev/null
+++ b/libs/protoutil/include/android/util/ProtoOutputStream.h
@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_UTIL_PROTOOUTPUT_STREAM_H
+#define ANDROID_UTIL_PROTOOUTPUT_STREAM_H
+
+#include <android/util/EncodedBuffer.h>
+
+#include <stdint.h>
+#include <string>
+
+namespace android {
+namespace util {
+
+/**
+ * Class to write to a protobuf stream.
+ *
+ * Each write method takes an ID code from the protoc generated classes
+ * and the value to write. To make a nested object, call start
+ * and then end when you are done.
+ *
+ * See the java version implementation (ProtoOutputStream.java) for more infos.
+ */
+class ProtoOutputStream
+{
+public:
+ ProtoOutputStream(int fd);
+ ~ProtoOutputStream();
+
+ /**
+ * Write APIs for dumping protobuf data. Returns true if the write succeeds.
+ */
+ bool write(uint64_t fieldId, double val);
+ bool write(uint64_t fieldId, float val);
+ bool write(uint64_t fieldId, int val);
+ bool write(uint64_t fieldId, long long val);
+ bool write(uint64_t fieldId, bool val);
+ bool write(uint64_t fieldId, std::string val);
+ bool write(uint64_t fieldId, const char* val);
+
+ /**
+ * Starts a sub-message write session.
+ * Returns a token of this write session.
+ * Must call end(token) when finish write this sub-message.
+ */
+ long long start(uint64_t fieldId);
+ void end(long long token);
+
+ /**
+ * Flushes the protobuf data out.
+ */
+ bool flush();
+
+private:
+ EncodedBuffer mBuffer;
+ int mFd;
+ size_t mCopyBegin;
+ bool mCompact;
+ int mDepth;
+ int mObjectId;
+ long long mExpectedObjectToken;
+
+ inline void writeDoubleImpl(uint32_t id, double val);
+ inline void writeFloatImpl(uint32_t id, float val);
+ inline void writeInt64Impl(uint32_t id, long long val);
+ inline void writeInt32Impl(uint32_t id, int val);
+ inline void writeUint64Impl(uint32_t id, uint64_t val);
+ inline void writeUint32Impl(uint32_t id, uint32_t val);
+ inline void writeFixed64Impl(uint32_t id, uint64_t val);
+ inline void writeFixed32Impl(uint32_t id, uint32_t val);
+ inline void writeSFixed64Impl(uint32_t id, long long val);
+ inline void writeSFixed32Impl(uint32_t id, int val);
+ inline void writeZigzagInt64Impl(uint32_t id, long long val);
+ inline void writeZigzagInt32Impl(uint32_t id, int val);
+ inline void writeEnumImpl(uint32_t id, int val);
+ inline void writeBoolImpl(uint32_t id, bool val);
+ inline void writeUtf8StringImpl(uint32_t id, const char* val, size_t size);
+
+ bool compact();
+ size_t editEncodedSize(size_t rawSize);
+ bool compactSize(size_t rawSize);
+};
+
+}
+}
+
+#endif // ANDROID_UTIL_PROTOOUTPUT_STREAM_H
\ No newline at end of file
diff --git a/libs/protoutil/include/android/util/protobuf.h b/libs/protoutil/include/android/util/protobuf.h
index f4e8d09..ca45e26 100644
--- a/libs/protoutil/include/android/util/protobuf.h
+++ b/libs/protoutil/include/android/util/protobuf.h
@@ -24,6 +24,9 @@
using namespace std;
+const int FIELD_ID_SHIFT = 3;
+const uint8_t WIRE_TYPE_MASK = (1 << FIELD_ID_SHIFT) - 1;
+
const uint8_t WIRE_TYPE_VARINT = 0;
const uint8_t WIRE_TYPE_FIXED64 = 1;
const uint8_t WIRE_TYPE_LENGTH_DELIMITED = 2;
@@ -35,16 +38,20 @@
uint8_t read_wire_type(uint32_t varint);
/**
- * read field id from varint, it is varint >> 3;
+ * Read field id from varint, it is varint >> 3;
*/
uint32_t read_field_id(uint32_t varint);
/**
- * Write a varint into the buffer. Return the next position to write at.
- * There must be 10 bytes in the buffer. The same as
- * EncodedBuffer.writeRawVarint32
+ * Get the size of a varint.
*/
-uint8_t* write_raw_varint(uint8_t* buf, uint32_t val);
+size_t get_varint_size(uint64_t varint);
+
+/**
+ * Write a varint into the buffer. Return the next position to write at.
+ * There must be 10 bytes in the buffer.
+ */
+uint8_t* write_raw_varint(uint8_t* buf, uint64_t val);
/**
* Write a protobuf WIRE_TYPE_LENGTH_DELIMITED header. Return the next position
diff --git a/libs/protoutil/src/EncodedBuffer.cpp b/libs/protoutil/src/EncodedBuffer.cpp
index 84dc5b6..435ae88 100644
--- a/libs/protoutil/src/EncodedBuffer.cpp
+++ b/libs/protoutil/src/EncodedBuffer.cpp
@@ -15,6 +15,7 @@
*/
#include <android/util/EncodedBuffer.h>
+#include <android/util/protobuf.h>
#include <stdlib.h>
@@ -52,19 +53,21 @@
return mOffset;
}
-void
+EncodedBuffer::Pointer*
EncodedBuffer::Pointer::move(size_t amt)
{
size_t newOffset = mOffset + amt;
mIndex += newOffset / mChunkSize;
mOffset = newOffset % mChunkSize;
+ return this;
}
-void
+EncodedBuffer::Pointer*
EncodedBuffer::Pointer::rewind()
{
mIndex = 0;
mOffset = 0;
+ return this;
}
EncodedBuffer::Pointer
@@ -86,6 +89,7 @@
{
mChunkSize = chunkSize == 0 ? BUFFER_SIZE : chunkSize;
mWp = Pointer(mChunkSize);
+ mEp = Pointer(mChunkSize);
}
EncodedBuffer::~EncodedBuffer()
@@ -137,28 +141,136 @@
return mChunkSize - mWp.offset();
}
+void
+EncodedBuffer::writeRawByte(uint8_t val)
+{
+ *writeBuffer() = val;
+ mWp.move();
+}
+
size_t
-EncodedBuffer::writeRawVarint(uint32_t val)
+EncodedBuffer::writeRawVarint64(uint64_t val)
{
size_t size = 0;
while (true) {
size++;
if ((val & ~0x7F) == 0) {
- *writeBuffer() = (uint8_t) val;
- mWp.move();
+ writeRawByte((uint8_t) val);
return size;
} else {
- *writeBuffer() = (uint8_t)((val & 0x7F) | 0x80);
- mWp.move();
+ writeRawByte((uint8_t)((val & 0x7F) | 0x80));
val >>= 7;
}
}
}
size_t
+EncodedBuffer::writeRawVarint32(uint32_t val)
+{
+ uint64_t v =(uint64_t)val;
+ return writeRawVarint64(v);
+}
+
+void
+EncodedBuffer::writeRawFixed32(uint32_t val)
+{
+ writeRawByte((uint8_t) val);
+ writeRawByte((uint8_t) (val>>8));
+ writeRawByte((uint8_t) (val>>16));
+ writeRawByte((uint8_t) (val>>24));
+}
+
+void
+EncodedBuffer::writeRawFixed64(uint64_t val)
+{
+ writeRawByte((uint8_t) val);
+ writeRawByte((uint8_t) (val>>8));
+ writeRawByte((uint8_t) (val>>16));
+ writeRawByte((uint8_t) (val>>24));
+ writeRawByte((uint8_t) (val>>32));
+ writeRawByte((uint8_t) (val>>40));
+ writeRawByte((uint8_t) (val>>48));
+ writeRawByte((uint8_t) (val>>56));
+}
+
+size_t
EncodedBuffer::writeHeader(uint32_t fieldId, uint8_t wireType)
{
- return writeRawVarint((fieldId << 3) | wireType);
+ return writeRawVarint32((fieldId << FIELD_ID_SHIFT) | wireType);
+}
+
+/******************************** Edit APIs ************************************************/
+EncodedBuffer::Pointer*
+EncodedBuffer::ep()
+{
+ return &mEp;
+}
+
+uint8_t
+EncodedBuffer::readRawByte()
+{
+ uint8_t val = *at(mEp);
+ mEp.move();
+ return val;
+}
+
+uint64_t
+EncodedBuffer::readRawVarint()
+{
+ uint64_t val = 0, shift = 0;
+ size_t start = mEp.pos();
+ while (true) {
+ uint8_t byte = readRawByte();
+ val += (byte & 0x7F) << shift;
+ if ((byte & 0x80) == 0) break;
+ shift += 7;
+ }
+ return val;
+}
+
+uint32_t
+EncodedBuffer::readRawFixed32()
+{
+ uint32_t val = 0;
+ for (auto i=0; i<32; i+=8) {
+ val += (uint32_t)readRawByte() << i;
+ }
+ return val;
+}
+
+uint64_t
+EncodedBuffer::readRawFixed64()
+{
+ uint64_t val = 0;
+ for (auto i=0; i<64; i+=8) {
+ val += (uint64_t)readRawByte() << i;
+ }
+ return val;
+}
+
+void
+EncodedBuffer::editRawFixed32(size_t pos, uint32_t val)
+{
+ size_t oldPos = mEp.pos();
+ mEp.rewind()->move(pos);
+ for (auto i=0; i<32; i+=8) {
+ *at(mEp) = (uint8_t) (val >> i);
+ mEp.move();
+ }
+ mEp.rewind()->move(oldPos);
+}
+
+void
+EncodedBuffer::copy(size_t srcPos, size_t size)
+{
+ if (size == 0) return;
+ Pointer cp(mChunkSize);
+ cp.move(srcPos);
+
+ while (cp.pos() < srcPos + size) {
+ writeRawByte(*at(cp));
+ cp.move();
+ }
}
/********************************* Read APIs ************************************************/
@@ -220,10 +332,10 @@
return res;
}
-uint32_t
+uint64_t
EncodedBuffer::iterator::readRawVarint()
{
- uint32_t val = 0, shift = 0;
+ uint64_t val = 0, shift = 0;
while (true) {
uint8_t byte = next();
val += (byte & 0x7F) << shift;
diff --git a/libs/protoutil/src/ProtoOutputStream.cpp b/libs/protoutil/src/ProtoOutputStream.cpp
new file mode 100644
index 0000000..e9ca0dc
--- /dev/null
+++ b/libs/protoutil/src/ProtoOutputStream.cpp
@@ -0,0 +1,652 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#define LOG_TAG "libprotoutil"
+
+#include <android/util/protobuf.h>
+#include <android/util/ProtoOutputStream.h>
+#include <cutils/log.h>
+#include <cstring>
+
+namespace android {
+namespace util {
+
+/**
+ * Position of the field type in a (long long) fieldId.
+ */
+const uint64_t FIELD_TYPE_SHIFT = 32;
+
+/**
+ * Mask for the field types stored in a fieldId. Leaves a whole
+ * byte for future expansion, even though there are currently only 17 types.
+ */
+const uint64_t FIELD_TYPE_MASK = 0x0ffULL << FIELD_TYPE_SHIFT;
+
+const uint64_t FIELD_TYPE_UNKNOWN = 0;
+const uint64_t TYPE_DOUBLE = 1ULL << FIELD_TYPE_SHIFT; // double, exactly eight bytes on the wire.
+const uint64_t TYPE_FLOAT = 2ULL << FIELD_TYPE_SHIFT; // float, exactly four bytes on the wire.
+const uint64_t TYPE_INT64 = 3ULL << FIELD_TYPE_SHIFT; // int64, varint on the wire. Negative numbers
+ // take 10 bytes. Use TYPE_SINT64 if negative
+ // values are likely.
+const uint64_t TYPE_UINT64 = 4ULL << FIELD_TYPE_SHIFT; // uint64, varint on the wire.
+const uint64_t TYPE_INT32 = 5ULL << FIELD_TYPE_SHIFT; // int32, varint on the wire. Negative numbers
+ // take 10 bytes. Use TYPE_SINT32 if negative
+ // values are likely.
+const uint64_t TYPE_FIXED64 = 6ULL << FIELD_TYPE_SHIFT; // uint64, exactly eight bytes on the wire.
+const uint64_t TYPE_FIXED32 = 7ULL << FIELD_TYPE_SHIFT; // uint32, exactly four bytes on the wire.
+const uint64_t TYPE_BOOL = 8ULL << FIELD_TYPE_SHIFT; // bool, varint on the wire.
+const uint64_t TYPE_STRING = 9ULL << FIELD_TYPE_SHIFT; // UTF-8 text.
+const uint64_t TYPE_GROUP = 10ULL << FIELD_TYPE_SHIFT; // Tag-delimited message. Deprecated.
+const uint64_t TYPE_MESSAGE = 11ULL << FIELD_TYPE_SHIFT; // Length-delimited message.
+
+const uint64_t TYPE_BYTES = 12ULL << FIELD_TYPE_SHIFT; // Arbitrary byte array.
+const uint64_t TYPE_UINT32 = 13ULL << FIELD_TYPE_SHIFT; // uint32, varint on the wire
+const uint64_t TYPE_ENUM = 14ULL << FIELD_TYPE_SHIFT; // Enum, varint on the wire
+const uint64_t TYPE_SFIXED32 = 15ULL << FIELD_TYPE_SHIFT; // int32, exactly four bytes on the wire
+const uint64_t TYPE_SFIXED64 = 16ULL << FIELD_TYPE_SHIFT; // int64, exactly eight bytes on the wire
+const uint64_t TYPE_SINT32 = 17ULL << FIELD_TYPE_SHIFT; // int32, ZigZag-encoded varint on the wire
+const uint64_t TYPE_SINT64 = 18ULL << FIELD_TYPE_SHIFT; // int64, ZigZag-encoded varint on the wire
+
+//
+// FieldId flags for whether the field is single, repeated or packed.
+// TODO: packed is not supported yet.
+//
+const uint64_t FIELD_COUNT_SHIFT = 40;
+const uint64_t FIELD_COUNT_MASK = 0x0fULL << FIELD_COUNT_SHIFT;
+const uint64_t FIELD_COUNT_UNKNOWN = 0;
+const uint64_t FIELD_COUNT_SINGLE = 1ULL << FIELD_COUNT_SHIFT;
+const uint64_t FIELD_COUNT_REPEATED = 2ULL << FIELD_COUNT_SHIFT;
+const uint64_t FIELD_COUNT_PACKED = 4ULL << FIELD_COUNT_SHIFT;
+
+ProtoOutputStream::ProtoOutputStream(int fd)
+ :mBuffer(),
+ mFd(fd),
+ mCopyBegin(0),
+ mCompact(false),
+ mDepth(0),
+ mObjectId(0),
+ mExpectedObjectToken(0LL)
+{
+}
+
+ProtoOutputStream::~ProtoOutputStream()
+{
+}
+
+bool
+ProtoOutputStream::write(uint64_t fieldId, double val)
+{
+ if (mCompact) return false;
+ const uint32_t id = (uint32_t)fieldId;
+ switch (fieldId & FIELD_TYPE_MASK) {
+ case TYPE_DOUBLE: writeDoubleImpl(id, (double)val); break;
+ case TYPE_FLOAT: writeFloatImpl(id, (float)val); break;
+ case TYPE_INT64: writeInt64Impl(id, (long long)val); break;
+ case TYPE_UINT64: writeUint64Impl(id, (uint64_t)val); break;
+ case TYPE_INT32: writeInt32Impl(id, (int)val); break;
+ case TYPE_FIXED64: writeFixed64Impl(id, (uint64_t)val); break;
+ case TYPE_FIXED32: writeFixed32Impl(id, (uint32_t)val); break;
+ case TYPE_UINT32: writeUint32Impl(id, (uint32_t)val); break;
+ case TYPE_SFIXED32: writeSFixed32Impl(id, (int)val); break;
+ case TYPE_SFIXED64: writeSFixed64Impl(id, (long long)val); break;
+ case TYPE_SINT32: writeZigzagInt32Impl(id, (int)val); break;
+ case TYPE_SINT64: writeZigzagInt64Impl(id, (long long)val); break;
+ default:
+ ALOGW("Field type %d is not supported when writing double val.",
+ (int)((fieldId & FIELD_TYPE_MASK) >> FIELD_TYPE_SHIFT));
+ return false;
+ }
+ return true;
+}
+
+bool
+ProtoOutputStream::write(uint64_t fieldId, float val)
+{
+ if (mCompact) return false;
+ const uint32_t id = (uint32_t)fieldId;
+ switch (fieldId & FIELD_TYPE_MASK) {
+ case TYPE_DOUBLE: writeDoubleImpl(id, (double)val); break;
+ case TYPE_FLOAT: writeFloatImpl(id, (float)val); break;
+ case TYPE_INT64: writeInt64Impl(id, (long long)val); break;
+ case TYPE_UINT64: writeUint64Impl(id, (uint64_t)val); break;
+ case TYPE_INT32: writeInt32Impl(id, (int)val); break;
+ case TYPE_FIXED64: writeFixed64Impl(id, (uint64_t)val); break;
+ case TYPE_FIXED32: writeFixed32Impl(id, (uint32_t)val); break;
+ case TYPE_UINT32: writeUint32Impl(id, (uint32_t)val); break;
+ case TYPE_SFIXED32: writeSFixed32Impl(id, (int)val); break;
+ case TYPE_SFIXED64: writeSFixed64Impl(id, (long long)val); break;
+ case TYPE_SINT32: writeZigzagInt32Impl(id, (int)val); break;
+ case TYPE_SINT64: writeZigzagInt64Impl(id, (long long)val); break;
+ default:
+ ALOGW("Field type %d is not supported when writing float val.",
+ (int)((fieldId & FIELD_TYPE_MASK) >> FIELD_TYPE_SHIFT));
+ return false;
+ }
+ return true;
+}
+
+bool
+ProtoOutputStream::write(uint64_t fieldId, int val)
+{
+ if (mCompact) return false;
+ const uint32_t id = (uint32_t)fieldId;
+ switch (fieldId & FIELD_TYPE_MASK) {
+ case TYPE_DOUBLE: writeDoubleImpl(id, (double)val); break;
+ case TYPE_FLOAT: writeFloatImpl(id, (float)val); break;
+ case TYPE_INT64: writeInt64Impl(id, (long long)val); break;
+ case TYPE_UINT64: writeUint64Impl(id, (uint64_t)val); break;
+ case TYPE_INT32: writeInt32Impl(id, (int)val); break;
+ case TYPE_FIXED64: writeFixed64Impl(id, (uint64_t)val); break;
+ case TYPE_FIXED32: writeFixed32Impl(id, (uint32_t)val); break;
+ case TYPE_UINT32: writeUint32Impl(id, (uint32_t)val); break;
+ case TYPE_SFIXED32: writeSFixed32Impl(id, (int)val); break;
+ case TYPE_SFIXED64: writeSFixed64Impl(id, (long long)val); break;
+ case TYPE_SINT32: writeZigzagInt32Impl(id, (int)val); break;
+ case TYPE_SINT64: writeZigzagInt64Impl(id, (long long)val); break;
+ case TYPE_ENUM: writeEnumImpl(id, (int)val); break;
+ case TYPE_BOOL: writeBoolImpl(id, val != 0); break;
+ default:
+ ALOGW("Field type %d is not supported when writing int val.",
+ (int)((fieldId & FIELD_TYPE_MASK) >> FIELD_TYPE_SHIFT));
+ return false;
+ }
+ return true;
+}
+
+bool
+ProtoOutputStream::write(uint64_t fieldId, long long val)
+{
+ if (mCompact) return false;
+ const uint32_t id = (uint32_t)fieldId;
+ switch (fieldId & FIELD_TYPE_MASK) {
+ case TYPE_DOUBLE: writeDoubleImpl(id, (double)val); break;
+ case TYPE_FLOAT: writeFloatImpl(id, (float)val); break;
+ case TYPE_INT64: writeInt64Impl(id, (long long)val); break;
+ case TYPE_UINT64: writeUint64Impl(id, (uint64_t)val); break;
+ case TYPE_INT32: writeInt32Impl(id, (int)val); break;
+ case TYPE_FIXED64: writeFixed64Impl(id, (uint64_t)val); break;
+ case TYPE_FIXED32: writeFixed32Impl(id, (uint32_t)val); break;
+ case TYPE_UINT32: writeUint32Impl(id, (uint32_t)val); break;
+ case TYPE_SFIXED32: writeSFixed32Impl(id, (int)val); break;
+ case TYPE_SFIXED64: writeSFixed64Impl(id, (long long)val); break;
+ case TYPE_SINT32: writeZigzagInt32Impl(id, (int)val); break;
+ case TYPE_SINT64: writeZigzagInt64Impl(id, (long long)val); break;
+ case TYPE_ENUM: writeEnumImpl(id, (int)val); break;
+ case TYPE_BOOL: writeBoolImpl(id, val != 0); break;
+ default:
+ ALOGW("Field type %d is not supported when writing long long val.",
+ (int)((fieldId & FIELD_TYPE_MASK) >> FIELD_TYPE_SHIFT));
+ return false;
+ }
+ return true;
+}
+
+bool
+ProtoOutputStream::write(uint64_t fieldId, bool val)
+{
+ if (mCompact) return false;
+ const uint32_t id = (uint32_t)fieldId;
+ switch (fieldId & FIELD_TYPE_MASK) {
+ case TYPE_BOOL:
+ writeBoolImpl(id, val);
+ return true;
+ default:
+ ALOGW("Field type %d is not supported when writing bool val.",
+ (int)((fieldId & FIELD_TYPE_MASK) >> FIELD_TYPE_SHIFT));
+ return false;
+ }
+}
+
+bool
+ProtoOutputStream::write(uint64_t fieldId, string val)
+{
+ if (mCompact) return false;
+ const uint32_t id = (uint32_t)fieldId;
+ switch (fieldId & FIELD_TYPE_MASK) {
+ case TYPE_STRING:
+ writeUtf8StringImpl(id, val.c_str(), val.size());
+ return true;
+ default:
+ ALOGW("Field type %d is not supported when writing string val.",
+ (int)((fieldId & FIELD_TYPE_MASK) >> FIELD_TYPE_SHIFT));
+ return false;
+ }
+}
+
+bool
+ProtoOutputStream::write(uint64_t fieldId, const char* val)
+{
+ if (mCompact) return false;
+ const uint32_t id = (uint32_t)fieldId;
+ int size = 0;
+ while (val[size] != '\0') size++;
+ switch (fieldId & FIELD_TYPE_MASK) {
+ case TYPE_STRING:
+ writeUtf8StringImpl(id, val, size);
+ return true;
+ default:
+ ALOGW("Field type %d is not supported when writing char[] val.",
+ (int)((fieldId & FIELD_TYPE_MASK) >> FIELD_TYPE_SHIFT));
+ return false;
+ }
+}
+
+/**
+ * Make a token.
+ * Bits 61-63 - tag size (So we can go backwards later if the object had not data)
+ * - 3 bits, max value 7, max value needed 5
+ * Bit 60 - true if the object is repeated
+ * Bits 59-51 - depth (For error checking)
+ * - 9 bits, max value 512, when checking, value is masked (if we really
+ * are more than 512 levels deep)
+ * Bits 32-50 - objectId (For error checking)
+ * - 19 bits, max value 524,288. that's a lot of objects. IDs will wrap
+ * because of the overflow, and only the tokens are compared.
+ * Bits 0-31 - offset of the first size field in the buffer.
+ */
+long long
+makeToken(int tagSize, bool repeated, int depth, int objectId, int sizePos) {
+ return ((0x07L & (long long)tagSize) << 61)
+ | (repeated ? (1LL << 60) : 0)
+ | (0x01ffL & (long long)depth) << 51
+ | (0x07ffffL & (long long)objectId) << 32
+ | (0x0ffffffffL & (long long)sizePos);
+}
+
+/**
+ * Get the encoded tag size from the token.
+ */
+static int getTagSizeFromToken(long long token) {
+ return (int)(0x7 & (token >> 61));
+}
+
+/**
+ * Get the nesting depth of startObject calls from the token.
+ */
+static int getDepthFromToken(long long token) {
+ return (int)(0x01ff & (token >> 51));
+}
+
+/**
+ * Get the location of the childRawSize (the first 32 bit size field) in this object.
+ */
+static int getSizePosFromToken(long long token) {
+ return (int)token;
+}
+
+long long
+ProtoOutputStream::start(uint64_t fieldId)
+{
+ if ((fieldId & FIELD_TYPE_MASK) != TYPE_MESSAGE) {
+ ALOGE("Can't call start for non-message type field: 0x%llx", (long long)fieldId);
+ return 0;
+ }
+
+ uint32_t id = (uint32_t)fieldId;
+ mBuffer.writeHeader(id, WIRE_TYPE_LENGTH_DELIMITED);
+
+ size_t sizePos = mBuffer.wp()->pos();
+
+ mDepth++;
+ mObjectId++;
+ mBuffer.writeRawFixed64(mExpectedObjectToken); // push previous token into stack.
+
+ mExpectedObjectToken = makeToken(get_varint_size(id),
+ (bool)(fieldId & FIELD_COUNT_REPEATED), mDepth, mObjectId, sizePos);
+ return mExpectedObjectToken;
+}
+
+void
+ProtoOutputStream::end(long long token)
+{
+ if (token != mExpectedObjectToken) {
+ ALOGE("Unexpected token: 0x%llx, should be 0x%llx", token, mExpectedObjectToken);
+ return;
+ }
+
+ int depth = getDepthFromToken(token);
+ if (depth != (mDepth & 0x01ff)) {
+ ALOGE("Unexpected depth: %d, should be %d", depth, mDepth);
+ return;
+ }
+ mDepth--;
+
+ int sizePos = getSizePosFromToken(token);
+ // number of bytes written in this start-end session.
+ int childRawSize = mBuffer.wp()->pos() - sizePos - 8;
+
+ // retrieve the old token from stack.
+ mBuffer.ep()->rewind()->move(sizePos);
+ mExpectedObjectToken = mBuffer.readRawFixed64();
+
+ // If raw size is larger than 0, write the negative value here to indicate a compact is needed.
+ if (childRawSize > 0) {
+ mBuffer.editRawFixed32(sizePos, -childRawSize);
+ mBuffer.editRawFixed32(sizePos+4, -1);
+ } else {
+ // reset wp which erase the header tag of the message when its size is 0.
+ mBuffer.wp()->rewind()->move(sizePos - getTagSizeFromToken(token));
+ }
+}
+
+bool
+ProtoOutputStream::compact() {
+ if (mCompact) return true;
+ if (mDepth != 0) {
+ ALOGE("Can't compact when depth(%d) is not zero. Missing calls to end.", mDepth);
+ return false;
+ }
+ // record the size of the original buffer.
+ size_t rawBufferSize = mBuffer.size();
+ if (rawBufferSize == 0) return true; // nothing to do if the buffer is empty;
+
+ // reset edit pointer and recursively compute encoded size of messages.
+ mBuffer.ep()->rewind();
+ if (editEncodedSize(rawBufferSize) == 0) {
+ ALOGE("Failed to editEncodedSize.");
+ return false;
+ }
+
+ // reset both edit pointer and write pointer, and compact recursively.
+ mBuffer.ep()->rewind();
+ mBuffer.wp()->rewind();
+ if (!compactSize(rawBufferSize)) {
+ ALOGE("Failed to compactSize.");
+ return false;
+ }
+ // copy the reset to the buffer.
+ if (mCopyBegin < rawBufferSize) {
+ mBuffer.copy(mCopyBegin, rawBufferSize - mCopyBegin);
+ }
+
+ // mark true means it is not legal to write to this ProtoOutputStream anymore
+ mCompact = true;
+ return true;
+}
+
+/**
+ * First compaction pass. Iterate through the data, and fill in the
+ * nested object sizes so the next pass can compact them.
+ */
+size_t
+ProtoOutputStream::editEncodedSize(size_t rawSize)
+{
+ size_t objectStart = mBuffer.ep()->pos();
+ size_t objectEnd = objectStart + rawSize;
+ size_t encodedSize = 0;
+ int childRawSize, childEncodedSize;
+ size_t childEncodedSizePos;
+
+ while (mBuffer.ep()->pos() < objectEnd) {
+ uint32_t tag = (uint32_t)mBuffer.readRawVarint();
+ encodedSize += get_varint_size(tag);
+ switch (read_wire_type(tag)) {
+ case WIRE_TYPE_VARINT:
+ do {
+ encodedSize++;
+ } while ((mBuffer.readRawByte() & 0x80) != 0);
+ break;
+ case WIRE_TYPE_FIXED64:
+ encodedSize += 8;
+ mBuffer.ep()->move(8);
+ break;
+ case WIRE_TYPE_LENGTH_DELIMITED:
+ childRawSize = (int)mBuffer.readRawFixed32();
+ childEncodedSizePos = mBuffer.ep()->pos();
+ childEncodedSize = (int)mBuffer.readRawFixed32();
+ if (childRawSize >= 0 && childRawSize == childEncodedSize) {
+ mBuffer.ep()->move(childRawSize);
+ } else if (childRawSize < 0 && childEncodedSize == -1){
+ childEncodedSize = editEncodedSize(-childRawSize);
+ mBuffer.editRawFixed32(childEncodedSizePos, childEncodedSize);
+ } else {
+ ALOGE("Bad raw or encoded values: raw=%d, encoded=%d at %zu",
+ childRawSize, childEncodedSize, childEncodedSizePos);
+ return 0;
+ }
+ encodedSize += get_varint_size(childEncodedSize) + childEncodedSize;
+ break;
+ case WIRE_TYPE_FIXED32:
+ encodedSize += 4;
+ mBuffer.ep()->move(4);
+ break;
+ default:
+ ALOGE("Unexpected wire type %d in editEncodedSize at [%zu, %zu]",
+ read_wire_type(tag), objectStart, objectEnd);
+ return 0;
+ }
+ }
+ return encodedSize;
+}
+
+/**
+ * Second compaction pass. Iterate through the data, and copy the data
+ * forward in the buffer, converting the pairs of uint32s into a single
+ * unsigned varint of the size.
+ */
+bool
+ProtoOutputStream::compactSize(size_t rawSize)
+{
+ size_t objectStart = mBuffer.ep()->pos();
+ size_t objectEnd = objectStart + rawSize;
+ int childRawSize, childEncodedSize;
+
+ while (mBuffer.ep()->pos() < objectEnd) {
+ uint32_t tag = (uint32_t)mBuffer.readRawVarint();
+ switch (read_wire_type(tag)) {
+ case WIRE_TYPE_VARINT:
+ while ((mBuffer.readRawByte() & 0x80) != 0) {}
+ break;
+ case WIRE_TYPE_FIXED64:
+ mBuffer.ep()->move(8);
+ break;
+ case WIRE_TYPE_LENGTH_DELIMITED:
+ mBuffer.copy(mCopyBegin, mBuffer.ep()->pos() - mCopyBegin);
+
+ childRawSize = (int)mBuffer.readRawFixed32();
+ childEncodedSize = (int)mBuffer.readRawFixed32();
+ mCopyBegin = mBuffer.ep()->pos();
+
+ // write encoded size to buffer.
+ mBuffer.writeRawVarint32(childEncodedSize);
+ if (childRawSize >= 0 && childRawSize == childEncodedSize) {
+ mBuffer.ep()->move(childEncodedSize);
+ } else if (childRawSize < 0){
+ if (!compactSize(-childRawSize)) return false;
+ } else {
+ ALOGE("Bad raw or encoded values: raw=%d, encoded=%d",
+ childRawSize, childEncodedSize);
+ return false;
+ }
+ break;
+ case WIRE_TYPE_FIXED32:
+ mBuffer.ep()->move(4);
+ break;
+ default:
+ ALOGE("Unexpected wire type %d in compactSize at [%zu, %zu]",
+ read_wire_type(tag), objectStart, objectEnd);
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool write_all(int fd, uint8_t const* buf, size_t size)
+{
+ while (size > 0) {
+ ssize_t amt = ::write(fd, buf, size);
+ if (amt < 0) {
+ return false;
+ }
+ size -= amt;
+ buf += amt;
+ }
+ return true;
+}
+
+bool
+ProtoOutputStream::flush()
+{
+ if (mFd < 0) return false;
+ if (!compact()) return false;
+
+ EncodedBuffer::iterator it = mBuffer.begin();
+ while (it.readBuffer() != NULL) {
+ if (!write_all(mFd, it.readBuffer(), it.currentToRead())) return false;
+ it.rp()->move(it.currentToRead());
+ }
+ return true;
+}
+
+
+// =========================================================================
+// Private functions
+
+/**
+ * bit_cast
+ */
+template <class From, class To>
+inline To bit_cast(From const &from) {
+ To to;
+ memcpy(&to, &from, sizeof(to));
+ return to;
+}
+
+inline void
+ProtoOutputStream::writeDoubleImpl(uint32_t id, double val)
+{
+ if (val == 0.0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_FIXED64);
+ mBuffer.writeRawFixed64(bit_cast<double, uint64_t>(val));
+}
+
+inline void
+ProtoOutputStream::writeFloatImpl(uint32_t id, float val)
+{
+ if (val == 0.0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_FIXED32);
+ mBuffer.writeRawFixed32(bit_cast<float, uint32_t>(val));
+}
+
+inline void
+ProtoOutputStream::writeInt64Impl(uint32_t id, long long val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_VARINT);
+ mBuffer.writeRawVarint64((uint64_t)val);
+}
+
+inline void
+ProtoOutputStream::writeInt32Impl(uint32_t id, int val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_VARINT);
+ mBuffer.writeRawVarint32((uint32_t)val);
+}
+
+inline void
+ProtoOutputStream::writeUint64Impl(uint32_t id, uint64_t val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_VARINT);
+ mBuffer.writeRawVarint64(val);
+}
+
+inline void
+ProtoOutputStream::writeUint32Impl(uint32_t id, uint32_t val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_VARINT);
+ mBuffer.writeRawVarint32(val);
+}
+
+inline void
+ProtoOutputStream::writeFixed64Impl(uint32_t id, uint64_t val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_FIXED64);
+ mBuffer.writeRawFixed64(val);
+}
+
+inline void
+ProtoOutputStream::writeFixed32Impl(uint32_t id, uint32_t val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_FIXED32);
+ mBuffer.writeRawFixed32(val);
+}
+
+inline void
+ProtoOutputStream::writeSFixed64Impl(uint32_t id, long long val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_FIXED64);
+ mBuffer.writeRawFixed64((uint64_t)val);
+}
+
+inline void
+ProtoOutputStream::writeSFixed32Impl(uint32_t id, int val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_FIXED32);
+ mBuffer.writeRawFixed32((uint32_t)val);
+}
+
+inline void
+ProtoOutputStream::writeZigzagInt64Impl(uint32_t id, long long val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_VARINT);
+ mBuffer.writeRawVarint64((val << 1) ^ (val >> 63));
+}
+
+inline void
+ProtoOutputStream::writeZigzagInt32Impl(uint32_t id, int val)
+{
+ if (val == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_VARINT);
+ mBuffer.writeRawVarint32((val << 1) ^ (val >> 31));
+}
+
+inline void
+ProtoOutputStream::writeEnumImpl(uint32_t id, int val)
+{
+ mBuffer.writeHeader(id, WIRE_TYPE_VARINT);
+ mBuffer.writeRawVarint32((uint32_t) val);
+}
+
+inline void
+ProtoOutputStream::writeBoolImpl(uint32_t id, bool val)
+{
+ if (!val) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_VARINT);
+ mBuffer.writeRawVarint32(val ? 1 : 0);
+}
+
+inline void
+ProtoOutputStream::writeUtf8StringImpl(uint32_t id, const char* val, size_t size)
+{
+ if (val == NULL || size == 0) return;
+ mBuffer.writeHeader(id, WIRE_TYPE_LENGTH_DELIMITED);
+ mBuffer.writeRawFixed32(size);
+ mBuffer.writeRawFixed32(size);
+ for (size_t i=0; i<size; i++) {
+ mBuffer.writeRawByte((uint8_t)val[i]);
+ }
+}
+
+} // util
+} // android
+
diff --git a/libs/protoutil/src/protobuf.cpp b/libs/protoutil/src/protobuf.cpp
index ec5325c..1c7eef9 100644
--- a/libs/protoutil/src/protobuf.cpp
+++ b/libs/protoutil/src/protobuf.cpp
@@ -22,17 +22,28 @@
uint8_t
read_wire_type(uint32_t varint)
{
- return (uint8_t) (varint & 0x07);
+ return (uint8_t) (varint & WIRE_TYPE_MASK);
}
uint32_t
read_field_id(uint32_t varint)
{
- return varint >> 3;
+ return varint >> FIELD_ID_SHIFT;
+}
+
+size_t
+get_varint_size(uint64_t varint)
+{
+ size_t size = 1;
+ while ((varint & ~0x7F)) {
+ size++;
+ varint >>= 7;
+ }
+ return size;
}
uint8_t*
-write_raw_varint(uint8_t* buf, uint32_t val)
+write_raw_varint(uint8_t* buf, uint64_t val)
{
uint8_t* p = buf;
while (true) {
@@ -49,7 +60,7 @@
uint8_t*
write_length_delimited_tag_header(uint8_t* buf, uint32_t fieldId, size_t size)
{
- buf = write_raw_varint(buf, (fieldId << 3) | 2);
+ buf = write_raw_varint(buf, (fieldId << FIELD_ID_SHIFT) | WIRE_TYPE_LENGTH_DELIMITED);
buf = write_raw_varint(buf, size);
return buf;
}
