1 | // Copyright (c) 2005, Google Inc.
|
---|
2 | // All rights reserved.
|
---|
3 | //
|
---|
4 | // Redistribution and use in source and binary forms, with or without
|
---|
5 | // modification, are permitted provided that the following conditions are
|
---|
6 | // met:
|
---|
7 | //
|
---|
8 | // * Redistributions of source code must retain the above copyright
|
---|
9 | // notice, this list of conditions and the following disclaimer.
|
---|
10 | // * Redistributions in binary form must reproduce the above
|
---|
11 | // copyright notice, this list of conditions and the following disclaimer
|
---|
12 | // in the documentation and/or other materials provided with the
|
---|
13 | // distribution.
|
---|
14 | // * Neither the name of Google Inc. nor the names of its
|
---|
15 | // contributors may be used to endorse or promote products derived from
|
---|
16 | // this software without specific prior written permission.
|
---|
17 | //
|
---|
18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
---|
19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
---|
20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
---|
21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
---|
22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
---|
23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
---|
24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
---|
25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
---|
26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
---|
27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
---|
28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
---|
29 |
|
---|
30 | // ---
|
---|
31 | // Author: Sanjay Ghemawat <[email protected]>
|
---|
32 | //
|
---|
33 | // A data structure used by the caching malloc. It maps from page# to
|
---|
34 | // a pointer that contains info about that page. We use two
|
---|
35 | // representations: one for 32-bit addresses, and another for 64 bit
|
---|
36 | // addresses. Both representations provide the same interface. The
|
---|
37 | // first representation is implemented as a flat array, the seconds as
|
---|
38 | // a three-level radix tree that strips away approximately 1/3rd of
|
---|
39 | // the bits every time.
|
---|
40 | //
|
---|
41 | // The BITS parameter should be the number of bits required to hold
|
---|
42 | // a page number. E.g., with 32 bit pointers and 4K pages (i.e.,
|
---|
43 | // page offset fits in lower 12 bits), BITS == 20.
|
---|
44 |
|
---|
45 | #ifndef TCMALLOC_PAGEMAP_H__
|
---|
46 | #define TCMALLOC_PAGEMAP_H__
|
---|
47 |
|
---|
48 | #include "config.h"
|
---|
49 |
|
---|
50 | #if HAVE(STDINT_H)
|
---|
51 | #include <stdint.h>
|
---|
52 | #elif HAVE(INTTYPES_H)
|
---|
53 | #include <inttypes.h>
|
---|
54 | #else
|
---|
55 | #include <sys/types.h>
|
---|
56 | #endif
|
---|
57 |
|
---|
58 | #include <string.h>
|
---|
59 |
|
---|
60 | #include "Assertions.h"
|
---|
61 |
|
---|
62 | // Single-level array
|
---|
63 | template <int BITS>
|
---|
64 | class TCMalloc_PageMap1 {
|
---|
65 | private:
|
---|
66 | void** array_;
|
---|
67 |
|
---|
68 | public:
|
---|
69 | typedef uintptr_t Number;
|
---|
70 |
|
---|
71 | explicit TCMalloc_PageMap1(void* (*allocator)(size_t)) {
|
---|
72 | array_ = reinterpret_cast<void**>((*allocator)(sizeof(void*) << BITS));
|
---|
73 | memset(array_, 0, sizeof(void*) << BITS);
|
---|
74 | }
|
---|
75 |
|
---|
76 | // Ensure that the map contains initialized entries "x .. x+n-1".
|
---|
77 | // Returns true if successful, false if we could not allocate memory.
|
---|
78 | bool Ensure(Number x, size_t n) {
|
---|
79 | // Nothing to do since flat array was allocate at start
|
---|
80 | return true;
|
---|
81 | }
|
---|
82 |
|
---|
83 | // REQUIRES "k" is in range "[0,2^BITS-1]".
|
---|
84 | // REQUIRES "k" has been ensured before.
|
---|
85 | //
|
---|
86 | // Return the current value for KEY. Returns "Value()" if not
|
---|
87 | // yet set.
|
---|
88 | void* get(Number k) const {
|
---|
89 | return array_[k];
|
---|
90 | }
|
---|
91 |
|
---|
92 | // REQUIRES "k" is in range "[0,2^BITS-1]".
|
---|
93 | // REQUIRES "k" has been ensured before.
|
---|
94 | //
|
---|
95 | // Sets the value for KEY.
|
---|
96 | void set(Number k, void* v) {
|
---|
97 | array_[k] = v;
|
---|
98 | }
|
---|
99 | };
|
---|
100 |
|
---|
101 | // Two-level radix tree
|
---|
102 | template <int BITS>
|
---|
103 | class TCMalloc_PageMap2 {
|
---|
104 | private:
|
---|
105 | // Put 32 entries in the root and (2^BITS)/32 entries in each leaf.
|
---|
106 | static const int ROOT_BITS = 5;
|
---|
107 | static const int ROOT_LENGTH = 1 << ROOT_BITS;
|
---|
108 |
|
---|
109 | static const int LEAF_BITS = BITS - ROOT_BITS;
|
---|
110 | static const int LEAF_LENGTH = 1 << LEAF_BITS;
|
---|
111 |
|
---|
112 | // Leaf node
|
---|
113 | struct Leaf {
|
---|
114 | void* values[LEAF_LENGTH];
|
---|
115 | };
|
---|
116 |
|
---|
117 | Leaf* root_[ROOT_LENGTH]; // Pointers to 32 child nodes
|
---|
118 | void* (*allocator_)(size_t); // Memory allocator
|
---|
119 |
|
---|
120 | public:
|
---|
121 | typedef uintptr_t Number;
|
---|
122 |
|
---|
123 | explicit TCMalloc_PageMap2(void* (*allocator)(size_t)) {
|
---|
124 | allocator_ = allocator;
|
---|
125 | memset(root_, 0, sizeof(root_));
|
---|
126 | }
|
---|
127 |
|
---|
128 | void* get(Number k) const {
|
---|
129 | ASSERT(k >> BITS == 0);
|
---|
130 | const Number i1 = k >> LEAF_BITS;
|
---|
131 | const Number i2 = k & (LEAF_LENGTH-1);
|
---|
132 | return root_[i1]->values[i2];
|
---|
133 | }
|
---|
134 |
|
---|
135 | void set(Number k, void* v) {
|
---|
136 | ASSERT(k >> BITS == 0);
|
---|
137 | const Number i1 = k >> LEAF_BITS;
|
---|
138 | const Number i2 = k & (LEAF_LENGTH-1);
|
---|
139 | root_[i1]->values[i2] = v;
|
---|
140 | }
|
---|
141 |
|
---|
142 | bool Ensure(Number start, size_t n) {
|
---|
143 | for (Number key = start; key <= start + n - 1; ) {
|
---|
144 | const Number i1 = key >> LEAF_BITS;
|
---|
145 |
|
---|
146 | // Make 2nd level node if necessary
|
---|
147 | if (root_[i1] == NULL) {
|
---|
148 | Leaf* leaf = reinterpret_cast<Leaf*>((*allocator_)(sizeof(Leaf)));
|
---|
149 | if (leaf == NULL) return false;
|
---|
150 | memset(leaf, 0, sizeof(*leaf));
|
---|
151 | root_[i1] = leaf;
|
---|
152 | }
|
---|
153 |
|
---|
154 | // Advance key past whatever is covered by this leaf node
|
---|
155 | key = ((key >> LEAF_BITS) + 1) << LEAF_BITS;
|
---|
156 | }
|
---|
157 | return true;
|
---|
158 | }
|
---|
159 | };
|
---|
160 |
|
---|
161 | // Three-level radix tree
|
---|
162 | template <int BITS>
|
---|
163 | class TCMalloc_PageMap3 {
|
---|
164 | private:
|
---|
165 | // How many bits should we consume at each interior level
|
---|
166 | static const int INTERIOR_BITS = (BITS + 2) / 3; // Round-up
|
---|
167 | static const int INTERIOR_LENGTH = 1 << INTERIOR_BITS;
|
---|
168 |
|
---|
169 | // How many bits should we consume at leaf level
|
---|
170 | static const int LEAF_BITS = BITS - 2*INTERIOR_BITS;
|
---|
171 | static const int LEAF_LENGTH = 1 << LEAF_BITS;
|
---|
172 |
|
---|
173 | // Interior node
|
---|
174 | struct Node {
|
---|
175 | Node* ptrs[INTERIOR_LENGTH];
|
---|
176 | };
|
---|
177 |
|
---|
178 | // Leaf node
|
---|
179 | struct Leaf {
|
---|
180 | void* values[LEAF_LENGTH];
|
---|
181 | };
|
---|
182 |
|
---|
183 | Node* root_; // Root of radix tree
|
---|
184 | void* (*allocator_)(size_t); // Memory allocator
|
---|
185 |
|
---|
186 | Node* NewNode() {
|
---|
187 | Node* result = reinterpret_cast<Node*>((*allocator_)(sizeof(Node)));
|
---|
188 | if (result != NULL) {
|
---|
189 | memset(result, 0, sizeof(*result));
|
---|
190 | }
|
---|
191 | return result;
|
---|
192 | }
|
---|
193 |
|
---|
194 | public:
|
---|
195 | typedef uintptr_t Number;
|
---|
196 |
|
---|
197 | explicit TCMalloc_PageMap3(void* (*allocator)(size_t)) {
|
---|
198 | allocator_ = allocator;
|
---|
199 | root_ = NewNode();
|
---|
200 | }
|
---|
201 |
|
---|
202 | void* get(Number k) const {
|
---|
203 | ASSERT(k >> BITS == 0);
|
---|
204 | const Number i1 = k >> (LEAF_BITS + INTERIOR_BITS);
|
---|
205 | const Number i2 = (k >> LEAF_BITS) & (INTERIOR_LENGTH-1);
|
---|
206 | const Number i3 = k & (LEAF_LENGTH-1);
|
---|
207 | return reinterpret_cast<Leaf*>(root_->ptrs[i1]->ptrs[i2])->values[i3];
|
---|
208 | }
|
---|
209 |
|
---|
210 | void set(Number k, void* v) {
|
---|
211 | ASSERT(k >> BITS == 0);
|
---|
212 | const Number i1 = k >> (LEAF_BITS + INTERIOR_BITS);
|
---|
213 | const Number i2 = (k >> LEAF_BITS) & (INTERIOR_LENGTH-1);
|
---|
214 | const Number i3 = k & (LEAF_LENGTH-1);
|
---|
215 | reinterpret_cast<Leaf*>(root_->ptrs[i1]->ptrs[i2])->values[i3] = v;
|
---|
216 | }
|
---|
217 |
|
---|
218 | bool Ensure(Number start, size_t n) {
|
---|
219 | for (Number key = start; key <= start + n - 1; ) {
|
---|
220 | const Number i1 = key >> (LEAF_BITS + INTERIOR_BITS);
|
---|
221 | const Number i2 = (key >> LEAF_BITS) & (INTERIOR_LENGTH-1);
|
---|
222 |
|
---|
223 | // Make 2nd level node if necessary
|
---|
224 | if (root_->ptrs[i1] == NULL) {
|
---|
225 | Node* n = NewNode();
|
---|
226 | if (n == NULL) return false;
|
---|
227 | root_->ptrs[i1] = n;
|
---|
228 | }
|
---|
229 |
|
---|
230 | // Make leaf node if necessary
|
---|
231 | if (root_->ptrs[i1]->ptrs[i2] == NULL) {
|
---|
232 | Leaf* leaf = reinterpret_cast<Leaf*>((*allocator_)(sizeof(Leaf)));
|
---|
233 | if (leaf == NULL) return false;
|
---|
234 | memset(leaf, 0, sizeof(*leaf));
|
---|
235 | root_->ptrs[i1]->ptrs[i2] = reinterpret_cast<Node*>(leaf);
|
---|
236 | }
|
---|
237 |
|
---|
238 | // Advance key past whatever is covered by this leaf node
|
---|
239 | key = ((key >> LEAF_BITS) + 1) << LEAF_BITS;
|
---|
240 | }
|
---|
241 | return true;
|
---|
242 | }
|
---|
243 | };
|
---|
244 |
|
---|
245 | #endif // TCMALLOC_PAGEMAP_H__
|
---|