1 : // Copyright 2012 Google Inc. All Rights Reserved.
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 : //
15 : // Implements the BasicBlockEntryHookTransform class.
16 :
17 : #include "syzygy/instrument/transforms/basic_block_entry_hook_transform.h"
18 :
19 : #include "base/logging.h"
20 : #include "base/string_util.h"
21 : #include "base/stringprintf.h"
22 : #include "syzygy/block_graph/block_builder.h"
23 : #include "syzygy/block_graph/block_util.h"
24 : #include "syzygy/common/defs.h"
25 : #include "syzygy/common/indexed_frequency_data.h"
26 : #include "syzygy/instrument/transforms/entry_thunk_transform.h"
27 : #include "syzygy/pe/block_util.h"
28 : #include "syzygy/pe/pe_utils.h"
29 : #include "syzygy/pe/transforms/add_imports_transform.h"
30 :
31 : namespace instrument {
32 : namespace transforms {
33 :
34 : namespace {
35 :
36 : using block_graph::BasicBlock;
37 : using block_graph::BasicCodeBlock;
38 : using block_graph::BasicBlockAssembler;
39 : using block_graph::BasicBlockReference;
40 : using block_graph::BlockBuilder;
41 : using block_graph::BlockGraph;
42 : using block_graph::Displacement;
43 : using block_graph::Immediate;
44 : using block_graph::Operand;
45 : using block_graph::Successor;
46 : using common::kBasicBlockEntryAgentId;
47 : using pe::transforms::AddImportsTransform;
48 :
49 : typedef AddImportsTransform::ImportedModule ImportedModule;
50 : typedef BasicBlockEntryHookTransform::RelativeAddressRange RelativeAddressRange;
51 :
52 : const char kDefaultModuleName[] = "basic_block_entry_client.dll";
53 : const char kBasicBlockEnter[] = "_increment_indexed_freq_data";
54 : const char kGetRawFrequencyData[] = "GetRawFrequencyData";
55 :
56 : // Compares two relative address ranges to see if they overlap. Assumes they
57 : // are already sorted. This is used to validate basic-block ranges.
58 : struct RelativeAddressRangesOverlapFunctor {
59 : bool operator()(const RelativeAddressRange& r1,
60 E : const RelativeAddressRange& r2) const {
61 E : DCHECK_LT(r1.start(), r2.start());
62 :
63 E : if (r1.end() > r2.start())
64 i : return true;
65 :
66 E : return false;
67 E : }
68 : };
69 :
70 : // Sets up the basic-block entry and the frequency data hooks import.
71 : bool SetupEntryHooks(BlockGraph* block_graph,
72 : BlockGraph::Block* header_block,
73 : const std::string& module_name,
74 : BlockGraph::Reference* basic_block_enter,
75 E : BlockGraph::Reference* get_raw_frequency_data) {
76 E : DCHECK(block_graph != NULL);
77 E : DCHECK(header_block != NULL);
78 E : DCHECK(basic_block_enter != NULL);
79 E : DCHECK(get_raw_frequency_data != NULL);
80 :
81 : // Setup the import module.
82 E : ImportedModule module(module_name);
83 : size_t bb_index = module.AddSymbol(kBasicBlockEnter,
84 E : ImportedModule::kAlwaysImport);
85 :
86 : size_t fd_index = module.AddSymbol(kGetRawFrequencyData,
87 E : ImportedModule::kAlwaysImport);
88 :
89 : // Setup the add-imports transform.
90 E : AddImportsTransform add_imports;
91 E : add_imports.AddModule(&module);
92 :
93 : // Add the imports to the block-graph.
94 E : if (!ApplyBlockGraphTransform(&add_imports, block_graph, header_block)) {
95 i : LOG(ERROR) << "Unable to add import entry hook functions.";
96 i : return false;
97 : }
98 :
99 : // Get a reference to the entry-hook function.
100 E : if (!module.GetSymbolReference(bb_index, basic_block_enter)) {
101 i : LOG(ERROR) << "Unable to get " << kBasicBlockEnter << ".";
102 i : return false;
103 : }
104 E : DCHECK(basic_block_enter->IsValid());
105 :
106 : // Get a reference to the frequency-hook function.
107 E : if (!module.GetSymbolReference(fd_index, get_raw_frequency_data)) {
108 i : LOG(ERROR) << "Unable to get " << kGetRawFrequencyData << ".";
109 i : return false;
110 : }
111 E : DCHECK(get_raw_frequency_data->IsValid());
112 :
113 E : return true;
114 E : }
115 :
116 : void AddSuccessorBetween(Successor::Condition condition,
117 : BasicCodeBlock* from,
118 E : BasicCodeBlock* to) {
119 : from->successors().push_back(
120 : Successor(condition,
121 : BasicBlockReference(BlockGraph::RELATIVE_REF,
122 : BlockGraph::Reference::kMaximumSize,
123 : to),
124 E : 0));
125 E : }
126 :
127 : } // namespace
128 :
129 : const char BasicBlockEntryHookTransform::kTransformName[] =
130 : "BasicBlockEntryHookTransform";
131 :
132 : BasicBlockEntryHookTransform::BasicBlockEntryHookTransform()
133 : : add_frequency_data_(kBasicBlockEntryAgentId,
134 : "Basic-Block Frequency Data",
135 : common::kBasicBlockFrequencyDataVersion,
136 : common::IndexedFrequencyData::BASIC_BLOCK_ENTRY),
137 : thunk_section_(NULL),
138 : instrument_dll_name_(kDefaultModuleName),
139 : set_src_ranges_for_thunks_(false),
140 E : set_inline_fast_path_(false) {
141 E : }
142 :
143 : bool BasicBlockEntryHookTransform::PreBlockGraphIteration(
144 : BlockGraph* block_graph,
145 E : BlockGraph::Block* header_block) {
146 E : DCHECK(block_graph != NULL);
147 E : DCHECK(header_block != NULL);
148 :
149 : // Setup basic block entry and the frequency data hooks.
150 : if (!SetupEntryHooks(block_graph,
151 : header_block,
152 : instrument_dll_name_,
153 : &bb_entry_hook_ref_,
154 E : &fd_entry_hook_ref_)) {
155 i : return false;
156 : }
157 :
158 : // Add the static basic-block frequency data.
159 : if (!ApplyBlockGraphTransform(
160 E : &add_frequency_data_, block_graph, header_block)) {
161 i : LOG(ERROR) << "Failed to insert basic-block frequency data.";
162 i : return false;
163 : }
164 :
165 : // Find or create the section we put our thunks in.
166 : thunk_section_ = block_graph->FindOrAddSection(common::kThunkSectionName,
167 E : pe::kCodeCharacteristics);
168 E : DCHECK(thunk_section_ != NULL);
169 :
170 : // Create basic block entry thunk, called when using the fast path.
171 E : if (set_inline_fast_path_) {
172 E : VLOG(1) << "Creating an inlined fast-path.";
173 E : if (!CreateBasicBlockEntryThunk(block_graph, &fast_bb_entry_block_))
174 i : return false;
175 : }
176 :
177 E : return true;
178 E : }
179 :
180 : bool BasicBlockEntryHookTransform::OnBlock(BlockGraph* block_graph,
181 E : BlockGraph::Block* block) {
182 E : DCHECK(block_graph != NULL);
183 E : DCHECK(block != NULL);
184 E : DCHECK(thunk_section_ != NULL);
185 :
186 : // Skip blocks created by this transform.
187 E : if (block->section() == thunk_section_->id())
188 E : return true;
189 :
190 E : if (block->type() != BlockGraph::CODE_BLOCK)
191 E : return true;
192 :
193 E : if (!pe::CodeBlockIsBasicBlockDecomposable(block)) {
194 E : if (!ThunkNonDecomposableCodeBlock(block_graph, block))
195 i : return false;
196 E : return true;
197 : }
198 :
199 E : if (!ApplyBasicBlockSubGraphTransform(this, block_graph, block, NULL))
200 i : return false;
201 :
202 E : return true;
203 E : }
204 :
205 : bool BasicBlockEntryHookTransform::TransformBasicBlockSubGraph(
206 E : BlockGraph* block_graph , BasicBlockSubGraph* subgraph) {
207 : // TODO(rogerm): A lot of this is boilerplate that can be hoisted to an
208 : // IterativeBasicBlockSubgraphTransform (or some such). In particular,
209 : // iterating the subgraph, dispatch on code/data basic block, and the
210 : // bb_ranges_ are duplicated in the coverage transform.
211 E : DCHECK(block_graph != NULL);
212 E : DCHECK(subgraph != NULL);
213 E : DCHECK(bb_entry_hook_ref_.IsValid());
214 E : DCHECK(fd_entry_hook_ref_.IsValid());
215 E : DCHECK(add_frequency_data_.frequency_data_block() != NULL);
216 :
217 : // Insert a call to the basic-block entry hook at the top of each code
218 : // basic-block.
219 : BasicBlockSubGraph::BBCollection::iterator it =
220 E : subgraph->basic_blocks().begin();
221 E : for (; it != subgraph->basic_blocks().end(); ++it) {
222 E : BasicCodeBlock* bb = BasicCodeBlock::Cast(*it);
223 E : if (bb == NULL || bb->is_padding())
224 E : continue;
225 :
226 : // Find the source range associated with this basic-block.
227 E : BlockGraph::Block::SourceRange source_range;
228 E : if (!GetBasicBlockSourceRange(*bb, &source_range)) {
229 i : LOG(ERROR) << "Unable to get source range for basic block '"
230 : << bb->name() << "'";
231 i : return false;
232 : }
233 :
234 : // We use the location/index in the bb_ranges vector of the current
235 : // basic-block range as the basic_block_id, and we pass a pointer to
236 : // the frequency data block as the module_data parameter. We then make
237 : // a memory indirect call to the bb_entry_hook.
238 E : Immediate basic_block_id(bb_ranges_.size(), core::kSize32Bit);
239 E : Immediate module_data(add_frequency_data_.frequency_data_block(), 0);
240 : Operand bb_entry_hook(Displacement(bb_entry_hook_ref_.referenced(),
241 E : bb_entry_hook_ref_.offset()));
242 :
243 : // Assemble entry hook instrumentation into the instruction stream.
244 E : BasicBlockAssembler bb_asm(bb->instructions().begin(), &bb->instructions());
245 :
246 E : if (set_inline_fast_path_) {
247 : // Inline fast-path: call to local hook.
248 E : DCHECK(fast_bb_entry_block_ != NULL);
249 E : bb_asm.push(basic_block_id);
250 E : bb_asm.call(Immediate(fast_bb_entry_block_, 0));
251 E : } else {
252 : // Fallback path: call to agent hook.
253 E : bb_asm.push(basic_block_id);
254 E : bb_asm.push(module_data);
255 E : bb_asm.call(bb_entry_hook);
256 : }
257 :
258 E : bb_ranges_.push_back(source_range);
259 E : }
260 :
261 E : return true;
262 E : }
263 :
264 : bool BasicBlockEntryHookTransform::PostBlockGraphIteration(
265 E : BlockGraph* block_graph, BlockGraph::Block* header_block) {
266 E : DCHECK(block_graph != NULL);
267 E : DCHECK(header_block != NULL);
268 :
269 E : size_t num_basic_blocks = bb_ranges_.size();
270 E : if (num_basic_blocks == 0) {
271 i : LOG(WARNING) << "Encountered no basic code blocks during instrumentation.";
272 i : return true;
273 : }
274 :
275 : if (!add_frequency_data_.ConfigureFrequencyDataBuffer(num_basic_blocks,
276 E : sizeof(uint32))) {
277 i : LOG(ERROR) << "Failed to configure frequency data buffer.";
278 i : return false;
279 : }
280 :
281 : // Add the module entry thunks.
282 E : EntryThunkTransform add_thunks;
283 E : add_thunks.set_only_instrument_module_entry(true);
284 E : add_thunks.set_instrument_dll_name(instrument_dll_name_);
285 E : add_thunks.set_src_ranges_for_thunks(set_src_ranges_for_thunks_);
286 :
287 E : Immediate module_data(add_frequency_data_.frequency_data_block(), 0);
288 E : if (!add_thunks.SetEntryThunkParameter(module_data)) {
289 i : LOG(ERROR) << "Failed to configure the entry thunks with the module_data "
290 : << "parameter.";
291 i : return false;
292 : }
293 :
294 E : if (!ApplyBlockGraphTransform(&add_thunks, block_graph, header_block)) {
295 i : LOG(ERROR) << "Unable to thunk module entry points.";
296 i : return false;
297 : }
298 E : thunk_section_ = add_thunks.thunk_section();
299 E : DCHECK(thunk_section_ != NULL);
300 :
301 : #ifndef NDEBUG
302 : // If we're in debug mode then sanity check the basic block ranges. When
303 : // sorted, they should not overlap.
304 E : RelativeAddressRangeVector bb_ranges_copy(bb_ranges_);
305 E : std::sort(bb_ranges_copy.begin(), bb_ranges_copy.end());
306 : DCHECK(std::adjacent_find(bb_ranges_copy.begin(),
307 : bb_ranges_copy.end(),
308 : RelativeAddressRangesOverlapFunctor()) ==
309 E : bb_ranges_copy.end());
310 : #endif
311 :
312 E : return true;
313 E : }
314 :
315 : bool BasicBlockEntryHookTransform::ThunkNonDecomposableCodeBlock(
316 E : BlockGraph* block_graph, BlockGraph::Block* code_block) {
317 E : DCHECK(block_graph != NULL);
318 E : DCHECK(code_block != NULL);
319 E : DCHECK(!pe::CodeBlockIsBasicBlockDecomposable(code_block));
320 :
321 : // Typedef for the thunk block map. The key is the offset within the callee
322 : // block and the value is the thunk block that forwards to the callee at that
323 : // offset.
324 : typedef std::map<BlockGraph::Offset, BlockGraph::Block*> ThunkBlockMap;
325 :
326 : // We keep a cache of thunks we've already created (by target offset of the
327 : // entry-point into the block) so that we only create one thunk per entry
328 : // point.
329 E : ThunkBlockMap thunk_block_map;
330 :
331 : // Iterate through all the block's referrers, creating thunks as we go.
332 : // We copy the referrer set for simplicity, as it's potentially mutated
333 : // in the loop.
334 E : BlockGraph::Block::ReferrerSet referrers = code_block->referrers();
335 E : BlockGraph::Block::ReferrerSet::const_iterator referrer_it(referrers.begin());
336 E : for (; referrer_it != referrers.end(); ++referrer_it) {
337 E : const BlockGraph::Block::Referrer& referrer = *referrer_it;
338 : if (!EnsureReferrerIsThunked(
339 E : referrer, block_graph, code_block, &thunk_block_map)) {
340 i : return false;
341 : }
342 E : }
343 :
344 E : return true;
345 E : }
346 :
347 : bool BasicBlockEntryHookTransform::EnsureReferrerIsThunked(
348 : const BlockGraph::Block::Referrer& referrer,
349 : BlockGraph* block_graph,
350 : BlockGraph::Block* code_block,
351 E : ThunkBlockMap* thunk_block_map) {
352 E : DCHECK(block_graph != NULL);
353 E : DCHECK(code_block != NULL);
354 E : DCHECK(thunk_block_map != NULL);
355 E : DCHECK(!pe::CodeBlockIsBasicBlockDecomposable(code_block));
356 :
357 : // Get the reference.
358 E : BlockGraph::Reference ref;
359 E : if (!referrer.first->GetReference(referrer.second, &ref)) {
360 i : LOG(ERROR) << "Unable to get reference from referrer.";
361 i : return false;
362 : }
363 E : DCHECK_EQ(code_block, ref.referenced());
364 :
365 : // Skip self-references, except long references to the start of the block.
366 : // Note: This may currently miss important cases. Notably if a block contains
367 : // more than one function, and the functions are mutually recursive, we'll
368 : // only record the original entry to the block, but will miss the internal
369 : // recursion. As-is, this does work for the common case where a block
370 : // contains one self-recursive function, however.
371 E : if (referrer.first == code_block) {
372 : // Skip short references.
373 E : if (ref.size() < sizeof(core::AbsoluteAddress))
374 E : return true;
375 :
376 : // Skip interior references. The block is not bb-decomposable so there is
377 : // nothing for us to do with them.
378 E : if (ref.offset() != 0)
379 E : return true;
380 : }
381 :
382 : // Get a thunk for the referenced offset from the thunk block map, creating
383 : // a new one if one does not already exist.
384 E : BlockGraph::Block* thunk_block = NULL;
385 : if (!FindOrCreateThunk(block_graph, thunk_block_map, code_block, ref.offset(),
386 E : &thunk_block)) {
387 i : LOG(ERROR) << "Unable to create thunk block.";
388 i : return false;
389 : }
390 E : DCHECK(thunk_block != NULL);
391 :
392 : // Update the referrer to point to the thunk.
393 : BlockGraph::Reference new_ref(ref.type(),
394 : ref.size(),
395 : thunk_block,
396 E : 0, 0);
397 E : referrer.first->SetReference(referrer.second, new_ref);
398 :
399 E : return true;
400 E : }
401 :
402 : bool BasicBlockEntryHookTransform::FindOrCreateThunk(
403 : BlockGraph* block_graph,
404 : ThunkBlockMap* thunk_block_map,
405 : BlockGraph::Block* code_block,
406 : BlockGraph::Offset offset,
407 E : BlockGraph::Block** thunk) {
408 E : DCHECK(block_graph != NULL);
409 E : DCHECK(thunk_block_map != NULL);
410 E : DCHECK(code_block != NULL);
411 E : DCHECK(thunk != NULL);
412 E : DCHECK_EQ(BlockGraph::CODE_BLOCK, code_block->type());
413 :
414 : // Do we already have a thunk defined for this offset? If so, return it.
415 E : ThunkBlockMap::const_iterator thunk_it = thunk_block_map->find(offset);
416 E : if (thunk_it != thunk_block_map->end()) {
417 E : *thunk = thunk_it->second;
418 E : return true;
419 : }
420 :
421 E : *thunk = NULL;
422 :
423 : // Determine the name for this thunk.
424 E : std::string name;
425 E : if (offset == 0) {
426 : name = base::StringPrintf("%s%s",
427 : code_block->name().c_str(),
428 E : common::kThunkSuffix);
429 E : } else {
430 : name = base::StringPrintf("%s%s+%d",
431 : code_block->name().c_str(),
432 : common::kThunkSuffix,
433 E : offset);
434 : }
435 :
436 : // Set up a basic block subgraph containing a single block description, with
437 : // that block description containing a single empty basic block, and get an
438 : // assembler writing into that basic block.
439 E : BasicBlockSubGraph subgraph;
440 E : BasicCodeBlock* bb = subgraph.AddBasicCodeBlock(name);
441 : BasicBlockSubGraph::BlockDescription* desc = subgraph.AddBlockDescription(
442 E : name, BlockGraph::CODE_BLOCK, thunk_section_->id(), 1, 0);
443 E : desc->basic_block_order.push_back(bb);
444 :
445 : // Find the source range associated with this block.
446 E : BlockGraph::Block::SourceRange source_range;
447 E : if (!code_block->source_ranges().empty())
448 E : source_range = code_block->source_ranges().range_pair(0).second;
449 :
450 : // Make sure we only push the source range if we have not already created
451 : // a source range mapping for this block (i.e., if the non-decomposable
452 : // block has multiple entry points, we want them to share an id). We can do
453 : // this because we handle one block at a time; so, all of a block's thunks
454 : // will be created as a group. This assertion is sanity checked in the
455 : // PostBlockGraphIteration function's check for overlapping source ranges.
456 E : if (bb_ranges_.empty() || source_range != bb_ranges_.back())
457 E : bb_ranges_.push_back(source_range);
458 :
459 : // We use the location/index in the bb_ranges vector of the current
460 : // basic-block range as the basic_block_id, and we pass a pointer to
461 : // the frequency data block as the module_data parameter. We then make
462 : // a memory indirect call to the bb_entry_hook.
463 E : Immediate basic_block_id(bb_ranges_.size()-1, core::kSize32Bit);
464 E : Immediate module_data(add_frequency_data_.frequency_data_block(), 0);
465 E : Immediate original_function(Displacement(code_block, offset));
466 : Operand bb_entry_hook(Displacement(bb_entry_hook_ref_.referenced(),
467 E : bb_entry_hook_ref_.offset()));
468 :
469 : // Assemble entry hook instrumentation into the thunk's instruction stream.
470 : // Note that we turn this into a simulated call, so that the return from
471 : // the bb entry hook continues from the thunked function.
472 E : BasicBlockAssembler bb_asm(bb->instructions().begin(), &bb->instructions());
473 E : bb_asm.push(basic_block_id);
474 E : bb_asm.push(module_data);
475 E : bb_asm.push(original_function);
476 E : bb_asm.jmp(bb_entry_hook);
477 :
478 : // Condense the whole mess into a block.
479 E : BlockBuilder block_builder(block_graph);
480 E : if (!block_builder.Merge(&subgraph)) {
481 i : LOG(ERROR) << "Failed to build thunk block.";
482 i : return false;
483 : }
484 :
485 : // Exactly one new block should have been created.
486 E : DCHECK_EQ(1u, block_builder.new_blocks().size());
487 E : *thunk = block_builder.new_blocks().front();
488 E : (*thunk_block_map)[offset] = *thunk;
489 :
490 E : return true;
491 E : }
492 :
493 : // This function injects into the instrumented application a fast hook to
494 : // improve data collection by avoiding repeated indirect calls from the
495 : // application to the agent, and by keeping a per-thread pointer to the
496 : // frequency data in a TLS slot accessible via the FS segment (fs:[0x700]
497 : // Reserved for user application).
498 : // See: http://en.wikipedia.org/wiki/Win32_Thread_Information_Block.
499 : //
500 : // The hook should be invoked like this:
501 : // _asm {
502 : // push block_id
503 : // call fast_path_hook
504 : // }
505 : //
506 : // This is the assembly code for the hook:
507 : // _asm {
508 : // bb1:
509 : // push eax ; Save flags and registers.
510 : // lahf
511 : // seto eax
512 : // push eax
513 : // push edx
514 : // mov edx, [esp + 16] ; Load block_id.
515 : // mov eax, fs:[0x700] ; Load Data Frequency Pointer.
516 : // test eax, eax ; Test pointer valid, otherwise load it.
517 : // je bbs ; Jump to slow path.
518 : // bb2:
519 : // add [eax + edx*4], 1 ; Increment Basic Block counter.
520 : // jz bbo ; Check if an overflow occurred.
521 : // bb3:
522 : // pop edx ; Restore flags and registers.
523 : // pop eax
524 : // add al, 0x7F
525 : // sahf
526 : // pop eax
527 : // ret 4
528 : //
529 : // bbo: ; Overflow.
530 : // sub [eax + edx*4], 1
531 : // jmp bb3
532 : //
533 : // bbs: ; Slow path, perform a call to the agent hook.
534 : // push ecx
535 : // push edx
536 : // pushfd
537 : //
538 : // push module_data
539 : // call fd_entry_hook
540 : // mov fs:[0x700], eax ; Store the frequency_data_ pointer in TLS slot.
541 : //
542 : // popfd
543 : // pop edx
544 : // pop ecx
545 : // jmp bb2
546 : // }
547 : bool BasicBlockEntryHookTransform::CreateBasicBlockEntryThunk(
548 : BlockGraph* block_graph,
549 E : BlockGraph::Block** fast_path_block) {
550 E : DCHECK(block_graph != NULL);
551 E : DCHECK(fast_path_block != NULL);
552 E : DCHECK(thunk_section_ != NULL);
553 :
554 : Operand bb_entry_hook(Displacement(bb_entry_hook_ref_.referenced(),
555 E : bb_entry_hook_ref_.offset()));
556 :
557 : // Determine the name for this thunk.
558 E : std::string name = base::StringPrintf("bb_entry_%s", common::kThunkSuffix);
559 :
560 : // Set up a basic block subgraph containing a single block description, with
561 : // that block description containing the fast path.
562 E : block_graph::BasicBlockSubGraph subgraph;
563 : block_graph::BasicBlockSubGraph::BlockDescription* desc =
564 : subgraph.AddBlockDescription(
565 E : name, BlockGraph::CODE_BLOCK, thunk_section_->id(), 1, 0);
566 :
567 E : BasicCodeBlock* bb1 = subgraph.AddBasicCodeBlock("bb1");
568 E : desc->basic_block_order.push_back(bb1);
569 : BasicBlockAssembler bb1_asm(bb1->instructions().begin(),
570 E : &bb1->instructions());
571 :
572 E : BasicCodeBlock* bb2 = subgraph.AddBasicCodeBlock("bb2");
573 E : desc->basic_block_order.push_back(bb2);
574 : BasicBlockAssembler bb2_asm(bb2->instructions().begin(),
575 E : &bb2->instructions());
576 :
577 E : BasicCodeBlock* bb3 = subgraph.AddBasicCodeBlock("bb3");
578 E : desc->basic_block_order.push_back(bb3);
579 : BasicBlockAssembler bb3_asm(bb3->instructions().begin(),
580 E : &bb3->instructions());
581 :
582 E : BasicCodeBlock* bbo = subgraph.AddBasicCodeBlock("overflow");
583 E : desc->basic_block_order.push_back(bbo);
584 : BasicBlockAssembler bbo_asm(bbo->instructions().begin(),
585 E : &bbo->instructions());
586 :
587 E : BasicCodeBlock* bbs = subgraph.AddBasicCodeBlock("slowpath");
588 : BasicBlockAssembler bbs_asm(bbs->instructions().begin(),
589 E : &bbs->instructions());
590 E : desc->basic_block_order.push_back(bbs);
591 :
592 : // Assemble instrumentation into the instruction stream.
593 E : Immediate module_data(add_frequency_data_.frequency_data_block(), 0);
594 : Operand fd_entry_hook(Displacement(fd_entry_hook_ref_.referenced(),
595 E : fd_entry_hook_ref_.offset()));
596 E : Operand fd_slot(Displacement(0x700, core::kSize32Bit));
597 :
598 E : bb1_asm.push(core::eax);
599 E : bb1_asm.lahf();
600 E : bb1_asm.set(core::kOverflow, core::eax);
601 E : bb1_asm.push(core::eax);
602 E : bb1_asm.push(core::edx);
603 E : bb1_asm.mov(core::edx, Operand(core::esp, Displacement(16)));
604 E : bb1_asm.mov_fs(core::eax, Operand(Displacement(0x700)));
605 E : bb1_asm.test(core::eax, core::eax);
606 :
607 : // Equivalent to: je slowpath.
608 E : AddSuccessorBetween(Successor::kConditionEqual, bb1, bbs);
609 E : AddSuccessorBetween(Successor::kConditionNotEqual, bb1, bb2);
610 :
611 : bb2_asm.add(Operand(core::eax, core::edx, core::kTimes4),
612 E : Immediate(1, core::kSize8Bit));
613 :
614 : // Equivalent to: jz overflow.
615 E : AddSuccessorBetween(Successor::kConditionEqual, bb2, bbo);
616 E : AddSuccessorBetween(Successor::kConditionNotEqual, bb2, bb3);
617 :
618 E : bb3_asm.pop(core::edx);
619 E : bb3_asm.pop(core::eax);
620 E : bb3_asm.add_b(core::eax, Immediate(0x7F, core::kSize8Bit));
621 E : bb3_asm.sahf();
622 E : bb3_asm.pop(core::eax);
623 E : bb3_asm.ret(4);
624 :
625 : // Overflow block. Call each time a counter has an overflow.
626 : // TODO(etienneb): Accumulate 32-bit overflow in a 64-bit external counter?
627 : bbo_asm.sub(Operand(core::eax, core::edx, core::kTimes4),
628 E : Immediate(1, core::kSize8Bit));
629 E : AddSuccessorBetween(Successor::kConditionTrue, bbo, bb3);
630 :
631 : // Slow path block. Should be called once by thread.
632 E : bbs_asm.push(core::ecx);
633 E : bbs_asm.push(core::edx);
634 E : bbs_asm.pushfd();
635 :
636 E : bbs_asm.push(module_data);
637 E : bbs_asm.call(fd_entry_hook);
638 E : bbs_asm.mov_fs(fd_slot, core::eax);
639 :
640 E : bbs_asm.popfd();
641 E : bbs_asm.pop(core::edx);
642 E : bbs_asm.pop(core::ecx);
643 E : AddSuccessorBetween(Successor::kConditionTrue, bbs, bb2);
644 :
645 : // Condense the whole mess into a block.
646 E : BlockBuilder block_builder(block_graph);
647 E : if (!block_builder.Merge(&subgraph)) {
648 i : LOG(ERROR) << "Failed to build thunk block.";
649 i : return false;
650 : }
651 :
652 : // Exactly one new block should have been created.
653 E : DCHECK_EQ(1u, block_builder.new_blocks().size());
654 E : *fast_path_block = block_builder.new_blocks().front();
655 :
656 E : return true;
657 E : }
658 :
659 : } // namespace transforms
660 : } // namespace instrument
|