GRPC C++  1.26.0
slice_internal.h
Go to the documentation of this file.
1 /*
2  *
3  * Copyright 2016 gRPC authors.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  *
17  */
18 
19 #ifndef GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H
20 #define GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H
21 
23 
24 #include <grpc/support/log.h>
25 
26 #include <grpc/slice.h>
27 #include <grpc/slice_buffer.h>
28 #include <string.h>
29 
35 
36 // Interned slices have specific fast-path operations for hashing. To inline
37 // these operations, we need to forward declare them here.
39 
40 // grpc_slice_refcount : A reference count for grpc_slice.
41 //
42 // Non-inlined grpc_slice objects are refcounted. Historically this was
43 // implemented via grpc_slice_refcount, a C-style polymorphic class using a
44 // manually managed vtable of operations. Subclasses would define their own
45 // vtable; the 'virtual' methods (ref, unref, equals and hash) would simply call
46 // the function pointers in the vtable as necessary.
47 //
48 // Unfortunately, this leads to some inefficiencies in the generated code that
49 // can be improved upon. For example, equality checking for interned slices is a
50 // simple equality check on the refcount pointer. With the vtable approach, this
51 // would translate to roughly the following (high-level) instructions:
52 //
53 // grpc_slice_equals(slice1, slice2):
54 // load vtable->eq -> eq_func
55 // call eq_func(slice1, slice2)
56 //
57 // interned_slice_equals(slice1, slice2)
58 // load slice1.ref -> r1
59 // load slice2.ref -> r2
60 // cmp r1, r2 -> retval
61 // ret retval
62 //
63 // This leads to a function call for a function defined in another translation
64 // unit, which imposes memory barriers, which reduces the compiler's ability to
65 // optimize (in addition to the added overhead of call/ret). Additionally, it
66 // may be harder to reason about branch prediction when we're jumping to
67 // essentially arbitrarily provided function pointers.
68 //
69 // In addition, it is arguable that while virtualization was helpful for
70 // Equals()/Hash() methods, that it was fundamentally unnecessary for
71 // Ref()/Unref().
72 //
73 // Instead, grpc_slice_refcount provides the same functionality as the C-style
74 // virtual class, but in a de-virtualized manner - Eq(), Hash(), Ref() and
75 // Unref() are provided within this header file. Fastpaths for Eq()/Hash()
76 // (interned and static metadata slices), as well as the Ref() operation, can
77 // all be inlined without any memory barriers.
78 //
79 // It does this by:
80 // 1. Using grpc_core::RefCount<> (header-only) for Ref/Unref. Two special cases
81 // need support: No-op ref/unref (eg. static metadata slices) and stream
82 // slice references (where all the slices share the streamref). This is in
83 // addition to the normal case of '1 slice, 1 ref'.
84 // To support these cases, we explicitly track a nullable pointer to the
85 // underlying RefCount<>. No-op ref/unref is used by checking the pointer for
86 // null, and doing nothing if it is. Both stream slice refs and 'normal'
87 // slices use the same path for Ref/Unref (by targeting the non-null
88 // pointer).
89 //
90 // 2. introducing the notion of grpc_slice_refcount::Type. This describes if a
91 // slice ref is used by a static metadata slice, an interned slice, or other
92 // slices. We switch on the slice ref type in order to provide fastpaths for
93 // Equals() and Hash().
94 //
95 // In total, this saves us roughly 1-2% latency for unary calls, with smaller
96 // calls benefitting. The effect is present, but not as useful, for larger calls
97 // where the cost of sending the data dominates.
98 // TODO(arjunroy): Investigate if this can be removed with strongly typed
99 // grpc_slices.
101  public:
102  enum class Type {
103  STATIC, // Refcount for a static metadata slice.
104  INTERNED, // Refcount for an interned slice.
105  NOP, // No-Op
106  REGULAR // Refcount for non-static-metadata, non-interned slices.
107  };
108  typedef void (*DestroyerFn)(void*);
109 
110  grpc_slice_refcount() = default;
111 
112  explicit grpc_slice_refcount(Type t) : ref_type_(t) {}
113 
114  explicit grpc_slice_refcount(grpc_slice_refcount* sub) : sub_refcount_(sub) {}
115  // Regular constructor for grpc_slice_refcount.
116  //
117  // Parameters:
118  // 1. grpc_slice_refcount::Type type
119  // Whether we are the refcount for a static
120  // metadata slice, an interned slice, or any other kind of slice.
121  //
122  // 2. RefCount* ref
123  // The pointer to the actual underlying grpc_core::RefCount. Rather than
124  // performing struct offset computations as in the original implementation to
125  // get to the refcount, which requires a virtual method, we devirtualize by
126  // using a nullable pointer to allow a single pair of Ref/Unref methods.
127  //
128  // 3. DestroyerFn destroyer_fn
129  // Called when the refcount goes to 0, with destroyer_arg as parameter.
130  //
131  // 4. void* destroyer_arg
132  // Argument for the virtualized destructor.
133  //
134  // 5. grpc_slice_refcount* sub
135  // Argument used for interned slices.
137  DestroyerFn destroyer_fn, void* destroyer_arg,
138  grpc_slice_refcount* sub)
139  : ref_(ref),
140  ref_type_(type),
141  sub_refcount_(sub),
142  dest_fn_(destroyer_fn),
143  destroy_fn_arg_(destroyer_arg) {}
144  // Initializer for static refcounts.
146  : ref_type_(type), sub_refcount_(sub) {}
147 
148  Type GetType() const { return ref_type_; }
149 
150  int Eq(const grpc_slice& a, const grpc_slice& b);
151 
152  uint32_t Hash(const grpc_slice& slice);
153  void Ref() {
154  if (ref_ == nullptr) return;
155  ref_->RefNonZero();
156  }
157  void Unref() {
158  if (ref_ == nullptr) return;
159  if (ref_->Unref()) {
160  dest_fn_(destroy_fn_arg_);
161  }
162  }
163 
164  grpc_slice_refcount* sub_refcount() const { return sub_refcount_; }
165 
166  private:
167  grpc_core::RefCount* ref_ = nullptr;
168  const Type ref_type_ = Type::REGULAR;
169  grpc_slice_refcount* sub_refcount_ = this;
170  DestroyerFn dest_fn_ = nullptr;
171  void* destroy_fn_arg_ = nullptr;
172 };
173 
174 namespace grpc_core {
175 
178 
180  : base(&kStaticSubRefcount, grpc_slice_refcount::Type::STATIC),
181  index(index) {}
182 
184  const uint32_t index;
185 };
186 
188 
190  static void Destroy(void* arg) {
191  auto* rc = static_cast<InternedSliceRefcount*>(arg);
193  gpr_free(rc);
194  }
195 
198  : base(grpc_slice_refcount::Type::INTERNED, &refcnt, Destroy, this, &sub),
199  sub(grpc_slice_refcount::Type::REGULAR, &refcnt, Destroy, this, &sub),
200  length(length),
201  hash(hash),
203 
205 
208  const size_t length;
210  const uint32_t hash;
212 };
213 
214 } // namespace grpc_core
215 
216 inline size_t grpc_refcounted_slice_length(const grpc_slice& slice) {
217  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
218  return slice.data.refcounted.length;
219 }
220 
221 inline const uint8_t* grpc_refcounted_slice_data(const grpc_slice& slice) {
222  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
223  return slice.data.refcounted.bytes;
224 }
225 
226 inline int grpc_slice_refcount::Eq(const grpc_slice& a, const grpc_slice& b) {
227  GPR_DEBUG_ASSERT(a.refcount != nullptr);
228  GPR_DEBUG_ASSERT(a.refcount == this);
229  switch (ref_type_) {
230  case Type::STATIC:
233  (a.refcount == b.refcount));
234  case Type::INTERNED:
235  return a.refcount == b.refcount;
236  case Type::NOP:
237  case Type::REGULAR:
238  break;
239  }
240  if (grpc_refcounted_slice_length(a) != GRPC_SLICE_LENGTH(b)) return false;
241  if (grpc_refcounted_slice_length(a) == 0) return true;
242  return 0 == memcmp(grpc_refcounted_slice_data(a), GRPC_SLICE_START_PTR(b),
244 }
245 
246 inline uint32_t grpc_slice_refcount::Hash(const grpc_slice& slice) {
247  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
248  GPR_DEBUG_ASSERT(slice.refcount == this);
249  switch (ref_type_) {
250  case Type::STATIC:
252  slice)];
253  case Type::INTERNED:
254  return reinterpret_cast<grpc_core::InternedSliceRefcount*>(slice.refcount)
255  ->hash;
256  case Type::NOP:
257  case Type::REGULAR:
258  break;
259  }
263 }
264 
265 inline const grpc_slice& grpc_slice_ref_internal(const grpc_slice& slice) {
266  if (slice.refcount) {
267  slice.refcount->Ref();
268  }
269  return slice;
270 }
271 
272 inline void grpc_slice_unref_internal(const grpc_slice& slice) {
273  if (slice.refcount) {
274  slice.refcount->Unref();
275  }
276 }
277 
280  size_t idx);
282 
283 // Returns a pointer to the first slice in the slice buffer without giving
284 // ownership to or a reference count on that slice.
286  GPR_DEBUG_ASSERT(sb->count > 0);
287  return &sb->slices[0];
288 }
289 
290 // Removes the first slice from the slice buffer.
292 
293 // Calls grpc_slice_sub with the given parameters on the first slice.
294 void grpc_slice_buffer_sub_first(grpc_slice_buffer* sb, size_t begin,
295  size_t end);
296 
297 /* Check if a slice is interned */
298 bool grpc_slice_is_interned(const grpc_slice& slice);
299 inline bool grpc_slice_is_interned(const grpc_slice& slice) {
300  return (slice.refcount &&
303 }
304 
306  const grpc_slice& b) {
308  return a.refcount == b.refcount;
309 }
310 
311 void grpc_slice_intern_init(void);
312 void grpc_slice_intern_shutdown(void);
313 void grpc_test_only_set_slice_hash_seed(uint32_t key);
314 // if slice matches a static slice, returns the static slice
315 // otherwise returns the passed in slice (without reffing it)
316 // used for surface boundaries where we might receive an un-interned static
317 // string
319  bool* returned_slice_is_different);
322 
323 inline uint32_t grpc_slice_hash_refcounted(const grpc_slice& s) {
324  GPR_DEBUG_ASSERT(s.refcount != nullptr);
325  return s.refcount->Hash(s);
326 }
327 
328 inline uint32_t grpc_slice_default_hash_internal(const grpc_slice& s) {
331 }
332 
333 inline uint32_t grpc_slice_hash_internal(const grpc_slice& s) {
334  return s.refcount == nullptr ? grpc_slice_default_hash_internal(s)
336 }
337 
339  size_t len);
341 
342 // Returns the memory used by this slice, not counting the slice structure
343 // itself. This means that inlined and slices from static strings will return
344 // 0. All other slices will return the size of the allocated chars.
346 
348  const grpc_core::UnmanagedMemorySlice& source, size_t begin, size_t end);
349 
350 #endif /* GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H */
Definition: slice_internal.h:100
grpc_slice grpc_slice_from_moved_buffer(grpc_core::UniquePtr< char > p, size_t len)
void grpc_slice_buffer_remove_first(grpc_slice_buffer *sb)
grpc_slice_refcount(Type t)
Definition: slice_internal.h:112
GPRAPI void gpr_free(void *ptr)
free
const uint32_t index
Definition: slice_internal.h:184
uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed)
const uint32_t hash
Definition: slice_internal.h:210
void grpc_slice_buffer_sub_first(grpc_slice_buffer *sb, size_t begin, size_t end)
RefCount refcnt
Definition: slice_internal.h:209
void grpc_slice_intern_shutdown(void)
grpc_slice_refcount base
Definition: slice_internal.h:206
grpc_slice_refcount kNoopRefcount
Definition: slice_utils.h:143
void RefNonZero()
Definition: ref_counted.h:117
StaticSliceRefcount(uint32_t index)
Definition: slice_internal.h:179
A grpc_slice s, if initialized, represents the byte range s.bytes[0..s.length-1]. ...
Definition: slice.h:60
void Unref()
Definition: slice_internal.h:157
uint32_t Hash(const grpc_slice &slice)
Definition: slice_internal.h:246
void grpc_slice_buffer_partial_unref_internal(grpc_slice_buffer *sb, size_t idx)
size_t grpc_slice_memory_usage(grpc_slice s)
grpc_slice * slices
slices in the array (Points to the first valid grpc_slice in the array)
Definition: slice.h:84
const size_t length
Definition: slice_internal.h:208
#define GRPC_SLICE_START_PTR(slice)
Definition: slice.h:96
Represents an expandable array of slices, to be interpreted as a single item.
Definition: slice.h:78
int grpc_static_slice_eq(grpc_slice a, grpc_slice b)
Definition: ref_counted.h:62
struct grpc_slice::grpc_slice_data::grpc_slice_refcounted refcounted
uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT]
Internal thread interface.
Definition: backoff.h:26
grpc_slice_refcount()=default
Type GetType() const
Definition: slice_internal.h:148
const grpc_slice & grpc_slice_ref_internal(const grpc_slice &slice)
Definition: slice_internal.h:265
#define GRPC_STATIC_METADATA_INDEX(static_slice)
Definition: static_metadata.h:299
void grpc_slice_unref_internal(const grpc_slice &slice)
Definition: slice_internal.h:272
const uint8_t * grpc_refcounted_slice_data(const grpc_slice &slice)
Definition: slice_internal.h:221
grpc_slice_refcount sub
Definition: slice_internal.h:207
void grpc_slice_intern_init(void)
int Eq(const grpc_slice &a, const grpc_slice &b)
Definition: slice_internal.h:226
grpc_slice_refcount * sub_refcount() const
Definition: slice_internal.h:164
struct grpc_slice_refcount * refcount
Definition: slice.h:61
bool grpc_slice_is_interned(const grpc_slice &slice)
Definition: slice_internal.h:299
uint32_t g_hash_seed
Definition: slice_internal.h:176
static void Destroy(void *arg)
Definition: slice_internal.h:190
uint32_t grpc_slice_hash_refcounted(const grpc_slice &s)
Definition: slice_internal.h:323
grpc_slice_refcount(grpc_slice_refcount *sub)
Definition: slice_internal.h:114
#define GRPC_STATIC_MDSTR_COUNT
Definition: static_metadata.h:39
bool Unref()
Definition: ref_counted.h:174
std::unique_ptr< T, DefaultDeleteChar > UniquePtr
Definition: memory.h:45
grpc_core::UnmanagedMemorySlice grpc_slice_sub_no_ref(const grpc_core::UnmanagedMemorySlice &source, size_t begin, size_t end)
grpc_slice_refcount base
Definition: slice_internal.h:183
Definition: slice_utils.h:122
union grpc_slice::grpc_slice_data data
uint32_t grpc_slice_hash_internal(const grpc_slice &s)
Definition: slice_internal.h:333
void(* DestroyerFn)(void *)
Definition: slice_internal.h:108
grpc_slice * grpc_slice_buffer_peek_first(grpc_slice_buffer *sb)
Definition: slice_internal.h:285
static grpc_slice_refcount kStaticSubRefcount
Definition: slice_internal.h:177
bool grpc_slice_static_interned_equal(const grpc_slice &a, const grpc_slice &b)
Definition: slice_internal.h:305
grpc_slice_refcount(grpc_slice_refcount::Type type, grpc_core::RefCount *ref, DestroyerFn destroyer_fn, void *destroyer_arg, grpc_slice_refcount *sub)
Definition: slice_internal.h:136
grpc_slice_refcount(grpc_slice_refcount *sub, Type type)
Definition: slice_internal.h:145
size_t count
the number of slices in the array
Definition: slice.h:86
size_t grpc_refcounted_slice_length(const grpc_slice &slice)
Definition: slice_internal.h:216
InternedSliceRefcount * bucket_next
Definition: slice_internal.h:211
#define GPR_DEBUG_ASSERT(x)
Definition: log.h:103
void Ref()
Definition: slice_internal.h:153
grpc_slice grpc_slice_from_moved_string(grpc_core::UniquePtr< char > p)
void grpc_test_only_set_slice_hash_seed(uint32_t key)
Definition: slice_internal.h:189
uint32_t grpc_static_slice_hash(grpc_slice s)
#define GRPC_SLICE_LENGTH(slice)
Definition: slice.h:99
uint32_t grpc_slice_default_hash_internal(const grpc_slice &s)
Definition: slice_internal.h:328
grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice, bool *returned_slice_is_different)
void grpc_slice_buffer_reset_and_unref_internal(grpc_slice_buffer *sb)
InternedSliceRefcount(size_t length, uint32_t hash, InternedSliceRefcount *bucket_next)
Definition: slice_internal.h:196
Type
Definition: slice_internal.h:102
void grpc_slice_buffer_destroy_internal(grpc_slice_buffer *sb)