3 * Copyright 2016 gRPC authors.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
19 #ifndef GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H
20 #define GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H
22 #include <grpc/support/port_platform.h>
24 #include <grpc/support/log.h>
26 #include <grpc/slice.h>
27 #include <grpc/slice_buffer.h>
30 #include "src/core/lib/gpr/murmur_hash.h"
31 #include "src/core/lib/gprpp/memory.h"
32 #include "src/core/lib/gprpp/ref_counted.h"
33 #include "src/core/lib/slice/slice_utils.h"
34 #include "src/core/lib/transport/static_metadata.h"
36 // Interned slices have specific fast-path operations for hashing. To inline
37 // these operations, we need to forward declare them here.
38 extern uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
39 extern uint32_t g_hash_seed;
41 // grpc_slice_refcount : A reference count for grpc_slice.
43 // Non-inlined grpc_slice objects are refcounted. Historically this was
44 // implemented via grpc_slice_refcount, a C-style polymorphic class using a
45 // manually managed vtable of operations. Subclasses would define their own
46 // vtable; the 'virtual' methods (ref, unref, equals and hash) would simply call
47 // the function pointers in the vtable as necessary.
49 // Unfortunately, this leads to some inefficiencies in the generated code that
50 // can be improved upon. For example, equality checking for interned slices is a
51 // simple equality check on the refcount pointer. With the vtable approach, this
52 // would translate to roughly the following (high-level) instructions:
54 // grpc_slice_equals(slice1, slice2):
55 // load vtable->eq -> eq_func
56 // call eq_func(slice1, slice2)
58 // interned_slice_equals(slice1, slice2)
59 // load slice1.ref -> r1
60 // load slice2.ref -> r2
61 // cmp r1, r2 -> retval
64 // This leads to a function call for a function defined in another translation
65 // unit, which imposes memory barriers, which reduces the compiler's ability to
66 // optimize (in addition to the added overhead of call/ret). Additionally, it
67 // may be harder to reason about branch prediction when we're jumping to
68 // essentially arbitrarily provided function pointers.
70 // In addition, it is arguable that while virtualization was helpful for
71 // Equals()/Hash() methods, that it was fundamentally unnecessary for
74 // Instead, grpc_slice_refcount provides the same functionality as the C-style
75 // virtual class, but in a de-virtualized manner - Eq(), Hash(), Ref() and
76 // Unref() are provided within this header file. Fastpaths for Eq()/Hash()
77 // (interned and static metadata slices), as well as the Ref() operation, can
78 // all be inlined without any memory barriers.
81 // 1. Using grpc_core::RefCount<> (header-only) for Ref/Unref. Two special cases
82 // need support: No-op ref/unref (eg. static metadata slices) and stream
83 // slice references (where all the slices share the streamref). This is in
84 // addition to the normal case of '1 slice, 1 ref'.
85 // To support these cases, we explicitly track a nullable pointer to the
86 // underlying RefCount<>. No-op ref/unref is used by checking the pointer for
87 // null, and doing nothing if it is. Both stream slice refs and 'normal'
88 // slices use the same path for Ref/Unref (by targeting the non-null
91 // 2. introducing the notion of grpc_slice_refcount::Type. This describes if a
92 // slice ref is used by a static metadata slice, an interned slice, or other
93 // slices. We switch on the slice ref type in order to provide fastpaths for
94 // Equals() and Hash().
96 // In total, this saves us roughly 1-2% latency for unary calls, with smaller
97 // calls benefitting. The effect is present, but not as useful, for larger calls
98 // where the cost of sending the data dominates.
99 // TODO(arjunroy): Investigate if this can be removed with strongly typed
101 struct grpc_slice_refcount {
104 STATIC, // Refcount for a static metadata slice.
105 INTERNED, // Refcount for an interned slice.
107 REGULAR // Refcount for non-static-metadata, non-interned slices.
109 typedef void (*DestroyerFn)(void*);
111 grpc_slice_refcount() = default;
113 explicit grpc_slice_refcount(Type t) : ref_type_(t) {}
115 explicit grpc_slice_refcount(grpc_slice_refcount* sub) : sub_refcount_(sub) {}
116 // Regular constructor for grpc_slice_refcount.
119 // 1. grpc_slice_refcount::Type type
120 // Whether we are the refcount for a static
121 // metadata slice, an interned slice, or any other kind of slice.
124 // The pointer to the actual underlying grpc_core::RefCount. Rather than
125 // performing struct offset computations as in the original implementation to
126 // get to the refcount, which requires a virtual method, we devirtualize by
127 // using a nullable pointer to allow a single pair of Ref/Unref methods.
129 // 3. DestroyerFn destroyer_fn
130 // Called when the refcount goes to 0, with destroyer_arg as parameter.
132 // 4. void* destroyer_arg
133 // Argument for the virtualized destructor.
135 // 5. grpc_slice_refcount* sub
136 // Argument used for interned slices.
137 grpc_slice_refcount(grpc_slice_refcount::Type type, grpc_core::RefCount* ref,
138 DestroyerFn destroyer_fn, void* destroyer_arg,
139 grpc_slice_refcount* sub)
143 dest_fn_(destroyer_fn),
144 destroy_fn_arg_(destroyer_arg) {}
145 // Initializer for static refcounts.
146 grpc_slice_refcount(grpc_slice_refcount* sub, Type type)
147 : ref_type_(type), sub_refcount_(sub) {}
149 Type GetType() const { return ref_type_; }
151 int Eq(const grpc_slice& a, const grpc_slice& b);
153 uint32_t Hash(const grpc_slice& slice);
155 if (ref_ == nullptr) return;
159 if (ref_ == nullptr) return;
161 dest_fn_(destroy_fn_arg_);
165 grpc_slice_refcount* sub_refcount() const { return sub_refcount_; }
168 grpc_core::RefCount* ref_ = nullptr;
169 const Type ref_type_ = Type::REGULAR;
170 grpc_slice_refcount* sub_refcount_ = this;
171 DestroyerFn dest_fn_ = nullptr;
172 void* destroy_fn_arg_ = nullptr;
175 namespace grpc_core {
177 struct StaticSliceRefcount {
178 static grpc_slice_refcount kStaticSubRefcount;
180 StaticSliceRefcount(uint32_t index)
181 : base(&kStaticSubRefcount, grpc_slice_refcount::Type::STATIC),
184 grpc_slice_refcount base;
185 const uint32_t index;
188 extern grpc_slice_refcount kNoopRefcount;
190 struct InternedSliceRefcount {
191 static void Destroy(void* arg) {
192 auto* rc = static_cast<InternedSliceRefcount*>(arg);
193 rc->~InternedSliceRefcount();
197 InternedSliceRefcount(size_t length, uint32_t hash,
198 InternedSliceRefcount* bucket_next)
199 : base(grpc_slice_refcount::Type::INTERNED, &refcnt, Destroy, this, &sub),
200 sub(grpc_slice_refcount::Type::REGULAR, &refcnt, Destroy, this, &sub),
203 bucket_next(bucket_next) {}
205 ~InternedSliceRefcount();
207 grpc_slice_refcount base;
208 grpc_slice_refcount sub;
212 InternedSliceRefcount* bucket_next;
215 } // namespace grpc_core
217 inline size_t grpc_refcounted_slice_length(const grpc_slice& slice) {
218 GPR_DEBUG_ASSERT(slice.refcount != nullptr);
219 return slice.data.refcounted.length;
222 inline const uint8_t* grpc_refcounted_slice_data(const grpc_slice& slice) {
223 GPR_DEBUG_ASSERT(slice.refcount != nullptr);
224 return slice.data.refcounted.bytes;
227 inline int grpc_slice_refcount::Eq(const grpc_slice& a, const grpc_slice& b) {
228 GPR_DEBUG_ASSERT(a.refcount != nullptr);
229 GPR_DEBUG_ASSERT(a.refcount == this);
233 (GRPC_STATIC_METADATA_INDEX(a) == GRPC_STATIC_METADATA_INDEX(b)) ==
234 (a.refcount == b.refcount));
236 return a.refcount == b.refcount;
241 if (grpc_refcounted_slice_length(a) != GRPC_SLICE_LENGTH(b)) return false;
242 if (grpc_refcounted_slice_length(a) == 0) return true;
243 return 0 == memcmp(grpc_refcounted_slice_data(a), GRPC_SLICE_START_PTR(b),
244 grpc_refcounted_slice_length(a));
247 inline uint32_t grpc_slice_refcount::Hash(const grpc_slice& slice) {
248 GPR_DEBUG_ASSERT(slice.refcount != nullptr);
249 GPR_DEBUG_ASSERT(slice.refcount == this);
252 return ::grpc_static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(
255 return reinterpret_cast<grpc_core::InternedSliceRefcount*>(slice.refcount)
261 return gpr_murmur_hash3(grpc_refcounted_slice_data(slice),
262 grpc_refcounted_slice_length(slice), g_hash_seed);
265 inline const grpc_slice& grpc_slice_ref_internal(const grpc_slice& slice) {
266 if (slice.refcount) {
267 slice.refcount->Ref();
272 inline void grpc_slice_unref_internal(const grpc_slice& slice) {
273 if (slice.refcount) {
274 slice.refcount->Unref();
278 void grpc_slice_buffer_reset_and_unref_internal(grpc_slice_buffer* sb);
279 void grpc_slice_buffer_partial_unref_internal(grpc_slice_buffer* sb,
281 void grpc_slice_buffer_destroy_internal(grpc_slice_buffer* sb);
283 // Returns a pointer to the first slice in the slice buffer without giving
284 // ownership to or a reference count on that slice.
285 inline grpc_slice* grpc_slice_buffer_peek_first(grpc_slice_buffer* sb) {
286 GPR_DEBUG_ASSERT(sb->count > 0);
287 return &sb->slices[0];
290 // Removes the first slice from the slice buffer.
291 void grpc_slice_buffer_remove_first(grpc_slice_buffer* sb);
293 // Calls grpc_slice_sub with the given parameters on the first slice.
294 void grpc_slice_buffer_sub_first(grpc_slice_buffer* sb, size_t begin,
297 /* Check if a slice is interned */
298 bool grpc_slice_is_interned(const grpc_slice& slice);
299 inline bool grpc_slice_is_interned(const grpc_slice& slice) {
300 return (slice.refcount &&
301 (slice.refcount->GetType() == grpc_slice_refcount::Type::INTERNED ||
302 slice.refcount->GetType() == grpc_slice_refcount::Type::STATIC));
305 inline bool grpc_slice_static_interned_equal(const grpc_slice& a,
306 const grpc_slice& b) {
307 GPR_DEBUG_ASSERT(grpc_slice_is_interned(a) && grpc_slice_is_interned(b));
308 return a.refcount == b.refcount;
311 void grpc_slice_intern_init(void);
312 void grpc_slice_intern_shutdown(void);
313 void grpc_test_only_set_slice_hash_seed(uint32_t key);
314 // if slice matches a static slice, returns the static slice
315 // otherwise returns the passed in slice (without reffing it)
316 // used for surface boundaries where we might receive an un-interned static
318 grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
319 bool* returned_slice_is_different);
320 uint32_t grpc_static_slice_hash(grpc_slice s);
321 int grpc_static_slice_eq(grpc_slice a, grpc_slice b);
323 inline uint32_t grpc_slice_hash_refcounted(const grpc_slice& s) {
324 GPR_DEBUG_ASSERT(s.refcount != nullptr);
325 return s.refcount->Hash(s);
328 inline uint32_t grpc_slice_default_hash_internal(const grpc_slice& s) {
329 return gpr_murmur_hash3(GRPC_SLICE_START_PTR(s), GRPC_SLICE_LENGTH(s),
333 inline uint32_t grpc_slice_hash_internal(const grpc_slice& s) {
334 return s.refcount == nullptr ? grpc_slice_default_hash_internal(s)
335 : grpc_slice_hash_refcounted(s);
338 grpc_slice grpc_slice_from_moved_buffer(grpc_core::UniquePtr<char> p,
340 grpc_slice grpc_slice_from_moved_string(grpc_core::UniquePtr<char> p);
342 // Returns the memory used by this slice, not counting the slice structure
343 // itself. This means that inlined and slices from static strings will return
344 // 0. All other slices will return the size of the allocated chars.
345 size_t grpc_slice_memory_usage(grpc_slice s);
347 grpc_core::UnmanagedMemorySlice grpc_slice_sub_no_ref(
348 const grpc_core::UnmanagedMemorySlice& source, size_t begin, size_t end);
350 #endif /* GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H */