1 : // Copyright 2012 Google Inc. All Rights Reserved.
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 : //
15 : // Implements the BasicBlockEntryHookTransform class.
16 :
17 : #include "syzygy/instrument/transforms/basic_block_entry_hook_transform.h"
18 :
19 : #include "base/logging.h"
20 : #include "base/string_util.h"
21 : #include "base/stringprintf.h"
22 : #include "syzygy/block_graph/block_builder.h"
23 : #include "syzygy/block_graph/block_util.h"
24 : #include "syzygy/common/defs.h"
25 : #include "syzygy/common/indexed_frequency_data.h"
26 : #include "syzygy/instrument/transforms/entry_thunk_transform.h"
27 : #include "syzygy/pe/block_util.h"
28 : #include "syzygy/pe/pe_utils.h"
29 : #include "syzygy/pe/transforms/add_imports_transform.h"
30 :
31 : namespace instrument {
32 : namespace transforms {
33 :
34 : namespace {
35 :
36 : using block_graph::BasicBlock;
37 : using block_graph::BasicCodeBlock;
38 : using block_graph::BasicBlockAssembler;
39 : using block_graph::BasicBlockReference;
40 : using block_graph::BlockBuilder;
41 : using block_graph::BlockGraph;
42 : using block_graph::Displacement;
43 : using block_graph::Immediate;
44 : using block_graph::Operand;
45 : using block_graph::Successor;
46 : using common::kBasicBlockEntryAgentId;
47 : using pe::transforms::AddImportsTransform;
48 :
49 : typedef AddImportsTransform::ImportedModule ImportedModule;
50 : typedef BasicBlockEntryHookTransform::RelativeAddressRange RelativeAddressRange;
51 :
52 : const char kDefaultModuleName[] = "basic_block_entry_client.dll";
53 : const char kBasicBlockEnter[] = "_basic_block_enter";
54 : const char kGetRawFrequencyData[] = "GetRawFrequencyData";
55 :
56 : // Compares two relative address ranges to see if they overlap. Assumes they
57 : // are already sorted. This is used to validate basic-block ranges.
58 : struct RelativeAddressRangesOverlapFunctor {
59 : bool operator()(const RelativeAddressRange& r1,
60 E : const RelativeAddressRange& r2) const {
61 E : DCHECK_LT(r1.start(), r2.start());
62 :
63 E : if (r1.end() > r2.start())
64 i : return true;
65 :
66 E : return false;
67 E : }
68 : };
69 :
70 : // Sets up the basic-block entry and the frequency data hooks import.
71 : bool SetupEntryHooks(BlockGraph* block_graph,
72 : BlockGraph::Block* header_block,
73 : const std::string& module_name,
74 : BlockGraph::Reference* basic_block_enter,
75 E : BlockGraph::Reference* get_raw_frequency_data) {
76 E : DCHECK(block_graph != NULL);
77 E : DCHECK(header_block != NULL);
78 E : DCHECK(basic_block_enter != NULL);
79 E : DCHECK(get_raw_frequency_data != NULL);
80 :
81 : // Setup the import module.
82 E : ImportedModule module(module_name);
83 : size_t bb_index = module.AddSymbol(kBasicBlockEnter,
84 E : ImportedModule::kAlwaysImport);
85 :
86 : size_t fd_index = module.AddSymbol(kGetRawFrequencyData,
87 E : ImportedModule::kAlwaysImport);
88 :
89 : // Setup the add-imports transform.
90 E : AddImportsTransform add_imports;
91 E : add_imports.AddModule(&module);
92 :
93 : // Add the imports to the block-graph.
94 E : if (!ApplyBlockGraphTransform(&add_imports, block_graph, header_block)) {
95 i : LOG(ERROR) << "Unable to add import entry hook functions.";
96 i : return false;
97 : }
98 :
99 : // Get a reference to the entry-hook function.
100 E : if (!module.GetSymbolReference(bb_index, basic_block_enter)) {
101 i : LOG(ERROR) << "Unable to get " << kBasicBlockEnter << ".";
102 i : return false;
103 : }
104 E : DCHECK(basic_block_enter->IsValid());
105 :
106 : // Get a reference to the frequency-hook function.
107 E : if (!module.GetSymbolReference(fd_index, get_raw_frequency_data)) {
108 i : LOG(ERROR) << "Unable to get " << kGetRawFrequencyData << ".";
109 i : return false;
110 : }
111 E : DCHECK(get_raw_frequency_data->IsValid());
112 :
113 E : return true;
114 E : }
115 :
116 : void AddSuccessorBetween(Successor::Condition condition,
117 : BasicCodeBlock* from,
118 E : BasicCodeBlock* to) {
119 : from->successors().push_back(
120 : Successor(condition,
121 : BasicBlockReference(BlockGraph::RELATIVE_REF,
122 : BlockGraph::Reference::kMaximumSize,
123 : to),
124 E : 0));
125 E : }
126 :
127 : } // namespace
128 :
129 : const char BasicBlockEntryHookTransform::kTransformName[] =
130 : "BasicBlockEntryHookTransform";
131 :
132 : BasicBlockEntryHookTransform::BasicBlockEntryHookTransform()
133 : : add_frequency_data_(kBasicBlockEntryAgentId,
134 : "Basic-Block Frequency Data",
135 : common::kBasicBlockFrequencyDataVersion),
136 : thunk_section_(NULL),
137 : instrument_dll_name_(kDefaultModuleName),
138 : set_src_ranges_for_thunks_(false),
139 E : set_inline_fast_path_(false) {
140 E : }
141 :
142 : bool BasicBlockEntryHookTransform::PreBlockGraphIteration(
143 : BlockGraph* block_graph,
144 E : BlockGraph::Block* header_block) {
145 E : DCHECK(block_graph != NULL);
146 E : DCHECK(header_block != NULL);
147 :
148 : // Setup basic block entry and the frequency data hooks.
149 : if (!SetupEntryHooks(block_graph,
150 : header_block,
151 : instrument_dll_name_,
152 : &bb_entry_hook_ref_,
153 E : &fd_entry_hook_ref_)) {
154 i : return false;
155 : }
156 :
157 : // Add the static basic-block frequency data.
158 : if (!ApplyBlockGraphTransform(
159 E : &add_frequency_data_, block_graph, header_block)) {
160 i : LOG(ERROR) << "Failed to insert basic-block frequency data.";
161 i : return false;
162 : }
163 :
164 : // Find or create the section we put our thunks in.
165 : thunk_section_ = block_graph->FindOrAddSection(common::kThunkSectionName,
166 E : pe::kCodeCharacteristics);
167 E : DCHECK(thunk_section_ != NULL);
168 :
169 : // Create basic block entry thunk, called when using the fast path.
170 E : if (set_inline_fast_path_) {
171 E : VLOG(1) << "Creating an inlined fast-path.";
172 E : if (!CreateBasicBlockEntryThunk(block_graph, &fast_bb_entry_block_))
173 i : return false;
174 : }
175 :
176 E : return true;
177 E : }
178 :
179 : bool BasicBlockEntryHookTransform::OnBlock(BlockGraph* block_graph,
180 E : BlockGraph::Block* block) {
181 E : DCHECK(block_graph != NULL);
182 E : DCHECK(block != NULL);
183 E : DCHECK(thunk_section_ != NULL);
184 :
185 : // Skip blocks created by this transform.
186 E : if (block->section() == thunk_section_->id())
187 E : return true;
188 :
189 E : if (block->type() != BlockGraph::CODE_BLOCK)
190 E : return true;
191 :
192 E : if (!pe::CodeBlockIsBasicBlockDecomposable(block)) {
193 E : if (!ThunkNonDecomposableCodeBlock(block_graph, block))
194 i : return false;
195 E : return true;
196 : }
197 :
198 E : if (!ApplyBasicBlockSubGraphTransform(this, block_graph, block, NULL))
199 i : return false;
200 :
201 E : return true;
202 E : }
203 :
204 : bool BasicBlockEntryHookTransform::TransformBasicBlockSubGraph(
205 E : BlockGraph* block_graph , BasicBlockSubGraph* subgraph) {
206 : // TODO(rogerm): A lot of this is boilerplate that can be hoisted to an
207 : // IterativeBasicBlockSubgraphTransform (or some such). In particular,
208 : // iterating the subgraph, dispatch on code/data basic block, and the
209 : // bb_ranges_ are duplicated in the coverage transform.
210 E : DCHECK(block_graph != NULL);
211 E : DCHECK(subgraph != NULL);
212 E : DCHECK(bb_entry_hook_ref_.IsValid());
213 E : DCHECK(fd_entry_hook_ref_.IsValid());
214 E : DCHECK(add_frequency_data_.frequency_data_block() != NULL);
215 :
216 : // Insert a call to the basic-block entry hook at the top of each code
217 : // basic-block.
218 : BasicBlockSubGraph::BBCollection::iterator it =
219 E : subgraph->basic_blocks().begin();
220 E : for (; it != subgraph->basic_blocks().end(); ++it) {
221 E : BasicCodeBlock* bb = BasicCodeBlock::Cast(*it);
222 E : if (bb == NULL || bb->is_padding())
223 E : continue;
224 :
225 : // Find the source range associated with this basic-block.
226 E : BlockGraph::Block::SourceRange source_range;
227 E : if (!GetBasicBlockSourceRange(*bb, &source_range)) {
228 i : LOG(ERROR) << "Unable to get source range for basic block '"
229 : << bb->name() << "'";
230 i : return false;
231 : }
232 :
233 : // We use the location/index in the bb_ranges vector of the current
234 : // basic-block range as the basic_block_id, and we pass a pointer to
235 : // the frequency data block as the module_data parameter. We then make
236 : // a memory indirect call to the bb_entry_hook.
237 E : Immediate basic_block_id(bb_ranges_.size(), core::kSize32Bit);
238 E : Immediate module_data(add_frequency_data_.frequency_data_block(), 0);
239 : Operand bb_entry_hook(Displacement(bb_entry_hook_ref_.referenced(),
240 E : bb_entry_hook_ref_.offset()));
241 :
242 : // Assemble entry hook instrumentation into the instruction stream.
243 E : BasicBlockAssembler bb_asm(bb->instructions().begin(), &bb->instructions());
244 :
245 E : if (set_inline_fast_path_) {
246 : // Inline fast-path: call to local hook.
247 E : DCHECK(fast_bb_entry_block_ != NULL);
248 E : bb_asm.push(basic_block_id);
249 E : bb_asm.call(Immediate(fast_bb_entry_block_, 0));
250 E : } else {
251 : // Fallback path: call to agent hook.
252 E : bb_asm.push(basic_block_id);
253 E : bb_asm.push(module_data);
254 E : bb_asm.call(bb_entry_hook);
255 : }
256 :
257 E : bb_ranges_.push_back(source_range);
258 E : }
259 :
260 E : return true;
261 E : }
262 :
263 : bool BasicBlockEntryHookTransform::PostBlockGraphIteration(
264 E : BlockGraph* block_graph, BlockGraph::Block* header_block) {
265 E : DCHECK(block_graph != NULL);
266 E : DCHECK(header_block != NULL);
267 :
268 E : size_t num_basic_blocks = bb_ranges_.size();
269 E : if (num_basic_blocks == 0) {
270 i : LOG(WARNING) << "Encountered no basic code blocks during instrumentation.";
271 i : return true;
272 : }
273 :
274 : if (!add_frequency_data_.ConfigureFrequencyDataBuffer(num_basic_blocks,
275 E : sizeof(uint32))) {
276 i : LOG(ERROR) << "Failed to configure frequency data buffer.";
277 i : return false;
278 : }
279 :
280 : // Add the module entry thunks.
281 E : EntryThunkTransform add_thunks;
282 E : add_thunks.set_only_instrument_module_entry(true);
283 E : add_thunks.set_instrument_dll_name(instrument_dll_name_);
284 E : add_thunks.set_src_ranges_for_thunks(set_src_ranges_for_thunks_);
285 :
286 E : Immediate module_data(add_frequency_data_.frequency_data_block(), 0);
287 E : if (!add_thunks.SetEntryThunkParameter(module_data)) {
288 i : LOG(ERROR) << "Failed to configure the entry thunks with the module_data "
289 : << "parameter.";
290 i : return false;
291 : }
292 :
293 E : if (!ApplyBlockGraphTransform(&add_thunks, block_graph, header_block)) {
294 i : LOG(ERROR) << "Unable to thunk module entry points.";
295 i : return false;
296 : }
297 E : thunk_section_ = add_thunks.thunk_section();
298 E : DCHECK(thunk_section_ != NULL);
299 :
300 : #ifndef NDEBUG
301 : // If we're in debug mode then sanity check the basic block ranges. When
302 : // sorted, they should not overlap.
303 E : RelativeAddressRangeVector bb_ranges_copy(bb_ranges_);
304 E : std::sort(bb_ranges_copy.begin(), bb_ranges_copy.end());
305 : DCHECK(std::adjacent_find(bb_ranges_copy.begin(),
306 : bb_ranges_copy.end(),
307 : RelativeAddressRangesOverlapFunctor()) ==
308 E : bb_ranges_copy.end());
309 : #endif
310 :
311 E : return true;
312 E : }
313 :
314 : bool BasicBlockEntryHookTransform::ThunkNonDecomposableCodeBlock(
315 E : BlockGraph* block_graph, BlockGraph::Block* code_block) {
316 E : DCHECK(block_graph != NULL);
317 E : DCHECK(code_block != NULL);
318 E : DCHECK(!pe::CodeBlockIsBasicBlockDecomposable(code_block));
319 :
320 : // Typedef for the thunk block map. The key is the offset within the callee
321 : // block and the value is the thunk block that forwards to the callee at that
322 : // offset.
323 : typedef std::map<BlockGraph::Offset, BlockGraph::Block*> ThunkBlockMap;
324 :
325 : // We keep a cache of thunks we've already created (by target offset of the
326 : // entry-point into the block) so that we only create one thunk per entry
327 : // point.
328 E : ThunkBlockMap thunk_block_map;
329 :
330 : // Iterate through all the block's referrers, creating thunks as we go.
331 : // We copy the referrer set for simplicity, as it's potentially mutated
332 : // in the loop.
333 E : BlockGraph::Block::ReferrerSet referrers = code_block->referrers();
334 E : BlockGraph::Block::ReferrerSet::const_iterator referrer_it(referrers.begin());
335 E : for (; referrer_it != referrers.end(); ++referrer_it) {
336 E : const BlockGraph::Block::Referrer& referrer = *referrer_it;
337 : if (!EnsureReferrerIsThunked(
338 E : referrer, block_graph, code_block, &thunk_block_map)) {
339 i : return false;
340 : }
341 E : }
342 :
343 E : return true;
344 E : }
345 :
346 : bool BasicBlockEntryHookTransform::EnsureReferrerIsThunked(
347 : const BlockGraph::Block::Referrer& referrer,
348 : BlockGraph* block_graph,
349 : BlockGraph::Block* code_block,
350 E : ThunkBlockMap* thunk_block_map) {
351 E : DCHECK(block_graph != NULL);
352 E : DCHECK(code_block != NULL);
353 E : DCHECK(thunk_block_map != NULL);
354 E : DCHECK(!pe::CodeBlockIsBasicBlockDecomposable(code_block));
355 :
356 : // Get the reference.
357 E : BlockGraph::Reference ref;
358 E : if (!referrer.first->GetReference(referrer.second, &ref)) {
359 i : LOG(ERROR) << "Unable to get reference from referrer.";
360 i : return false;
361 : }
362 E : DCHECK_EQ(code_block, ref.referenced());
363 :
364 : // Skip self-references, except long references to the start of the block.
365 : // Note: This may currently miss important cases. Notably if a block contains
366 : // more than one function, and the functions are mutually recursive, we'll
367 : // only record the original entry to the block, but will miss the internal
368 : // recursion. As-is, this does work for the common case where a block
369 : // contains one self-recursive function, however.
370 E : if (referrer.first == code_block) {
371 : // Skip short references.
372 E : if (ref.size() < sizeof(core::AbsoluteAddress))
373 E : return true;
374 :
375 : // Skip interior references. The block is not bb-decomposable so there is
376 : // nothing for us to do with them.
377 E : if (ref.offset() != 0)
378 E : return true;
379 : }
380 :
381 : // Get a thunk for the referenced offset from the thunk block map, creating
382 : // a new one if one does not already exist.
383 E : BlockGraph::Block* thunk_block = NULL;
384 : if (!FindOrCreateThunk(block_graph, thunk_block_map, code_block, ref.offset(),
385 E : &thunk_block)) {
386 i : LOG(ERROR) << "Unable to create thunk block.";
387 i : return false;
388 : }
389 E : DCHECK(thunk_block != NULL);
390 :
391 : // Update the referrer to point to the thunk.
392 : BlockGraph::Reference new_ref(ref.type(),
393 : ref.size(),
394 : thunk_block,
395 E : 0, 0);
396 E : referrer.first->SetReference(referrer.second, new_ref);
397 :
398 E : return true;
399 E : }
400 :
401 : bool BasicBlockEntryHookTransform::FindOrCreateThunk(
402 : BlockGraph* block_graph,
403 : ThunkBlockMap* thunk_block_map,
404 : BlockGraph::Block* code_block,
405 : BlockGraph::Offset offset,
406 E : BlockGraph::Block** thunk) {
407 E : DCHECK(block_graph != NULL);
408 E : DCHECK(thunk_block_map != NULL);
409 E : DCHECK(code_block != NULL);
410 E : DCHECK(thunk != NULL);
411 E : DCHECK_EQ(BlockGraph::CODE_BLOCK, code_block->type());
412 :
413 : // Do we already have a thunk defined for this offset? If so, return it.
414 E : ThunkBlockMap::const_iterator thunk_it = thunk_block_map->find(offset);
415 E : if (thunk_it != thunk_block_map->end()) {
416 E : *thunk = thunk_it->second;
417 E : return true;
418 : }
419 :
420 E : *thunk = NULL;
421 :
422 : // Determine the name for this thunk.
423 E : std::string name;
424 E : if (offset == 0) {
425 : name = base::StringPrintf("%s%s",
426 : code_block->name().c_str(),
427 E : common::kThunkSuffix);
428 E : } else {
429 : name = base::StringPrintf("%s%s+%d",
430 : code_block->name().c_str(),
431 : common::kThunkSuffix,
432 E : offset);
433 : }
434 :
435 : // Set up a basic block subgraph containing a single block description, with
436 : // that block description containing a single empty basic block, and get an
437 : // assembler writing into that basic block.
438 E : BasicBlockSubGraph subgraph;
439 E : BasicCodeBlock* bb = subgraph.AddBasicCodeBlock(name);
440 : BasicBlockSubGraph::BlockDescription* desc = subgraph.AddBlockDescription(
441 E : name, BlockGraph::CODE_BLOCK, thunk_section_->id(), 1, 0);
442 E : desc->basic_block_order.push_back(bb);
443 :
444 : // Find the source range associated with this block.
445 E : BlockGraph::Block::SourceRange source_range;
446 E : if (!code_block->source_ranges().empty())
447 E : source_range = code_block->source_ranges().range_pair(0).second;
448 :
449 : // Make sure we only push the source range if we have not already created
450 : // a source range mapping for this block (i.e., if the non-decomposable
451 : // block has multiple entry points, we want them to share an id). We can do
452 : // this because we handle one block at a time; so, all of a block's thunks
453 : // will be created as a group. This assertion is sanity checked in the
454 : // PostBlockGraphIteration function's check for overlapping source ranges.
455 E : if (bb_ranges_.empty() || source_range != bb_ranges_.back())
456 E : bb_ranges_.push_back(source_range);
457 :
458 : // We use the location/index in the bb_ranges vector of the current
459 : // basic-block range as the basic_block_id, and we pass a pointer to
460 : // the frequency data block as the module_data parameter. We then make
461 : // a memory indirect call to the bb_entry_hook.
462 E : Immediate basic_block_id(bb_ranges_.size()-1, core::kSize32Bit);
463 E : Immediate module_data(add_frequency_data_.frequency_data_block(), 0);
464 E : Immediate original_function(Displacement(code_block, offset));
465 : Operand bb_entry_hook(Displacement(bb_entry_hook_ref_.referenced(),
466 E : bb_entry_hook_ref_.offset()));
467 :
468 : // Assemble entry hook instrumentation into the thunk's instruction stream.
469 : // Note that we turn this into a simulated call, so that the return from
470 : // the bb entry hook continues from the thunked function.
471 E : BasicBlockAssembler bb_asm(bb->instructions().begin(), &bb->instructions());
472 E : bb_asm.push(basic_block_id);
473 E : bb_asm.push(module_data);
474 E : bb_asm.push(original_function);
475 E : bb_asm.jmp(bb_entry_hook);
476 :
477 : // Condense the whole mess into a block.
478 E : BlockBuilder block_builder(block_graph);
479 E : if (!block_builder.Merge(&subgraph)) {
480 i : LOG(ERROR) << "Failed to build thunk block.";
481 i : return false;
482 : }
483 :
484 : // Exactly one new block should have been created.
485 E : DCHECK_EQ(1u, block_builder.new_blocks().size());
486 E : *thunk = block_builder.new_blocks().front();
487 E : (*thunk_block_map)[offset] = *thunk;
488 :
489 E : return true;
490 E : }
491 :
492 : // This function injects into the instrumented application a fast hook to
493 : // improve data collection by avoiding repeated indirect calls from the
494 : // application to the agent, and by keeping a per-thread pointer to the
495 : // frequency data in a TLS slot accessible via the FS segment (fs:[0x700]
496 : // Reserved for user application).
497 : // See: http://en.wikipedia.org/wiki/Win32_Thread_Information_Block.
498 : //
499 : // The hook should be invoked like this:
500 : // _asm {
501 : // push block_id
502 : // call fast_path_hook
503 : // }
504 : //
505 : // This is the assembly code for the hook:
506 : // _asm {
507 : // bb1:
508 : // push eax ; Save flags and registers.
509 : // lahf
510 : // seto eax
511 : // push eax
512 : // push edx
513 : // mov edx, [esp + 16] ; Load block_id.
514 : // mov eax, fs:[0x700] ; Load Data Frequency Pointer.
515 : // test eax, eax ; Test pointer valid, otherwise load it.
516 : // je bbs ; Jump to slow path.
517 : // bb2:
518 : // add [eax + edx*4], 1 ; Increment Basic Block counter.
519 : // jz bbo ; Check if an overflow occurred.
520 : // bb3:
521 : // pop edx ; Restore flags and registers.
522 : // pop eax
523 : // add al, 0x7F
524 : // sahf
525 : // pop eax
526 : // ret 4
527 : //
528 : // bbo: ; Overflow.
529 : // sub [eax + edx*4], 1
530 : // jmp bb3
531 : //
532 : // bbs: ; Slow path, perform a call to the agent hook.
533 : // push ecx
534 : // push edx
535 : // pushfd
536 : //
537 : // push module_data
538 : // call fd_entry_hook
539 : // mov fs:[0x700], eax ; Store the frequency_data_ pointer in TLS slot.
540 : //
541 : // popfd
542 : // pop edx
543 : // pop ecx
544 : // jmp bb2
545 : // }
546 : bool BasicBlockEntryHookTransform::CreateBasicBlockEntryThunk(
547 : BlockGraph* block_graph,
548 E : BlockGraph::Block** fast_path_block) {
549 E : DCHECK(block_graph != NULL);
550 E : DCHECK(fast_path_block != NULL);
551 E : DCHECK(thunk_section_ != NULL);
552 :
553 : Operand bb_entry_hook(Displacement(bb_entry_hook_ref_.referenced(),
554 E : bb_entry_hook_ref_.offset()));
555 :
556 : // Determine the name for this thunk.
557 E : std::string name = base::StringPrintf("bb_entry_%s", common::kThunkSuffix);
558 :
559 : // Set up a basic block subgraph containing a single block description, with
560 : // that block description containing the fast path.
561 E : block_graph::BasicBlockSubGraph subgraph;
562 : block_graph::BasicBlockSubGraph::BlockDescription* desc =
563 : subgraph.AddBlockDescription(
564 E : name, BlockGraph::CODE_BLOCK, thunk_section_->id(), 1, 0);
565 :
566 E : BasicCodeBlock* bb1 = subgraph.AddBasicCodeBlock("bb1");
567 E : desc->basic_block_order.push_back(bb1);
568 : BasicBlockAssembler bb1_asm(bb1->instructions().begin(),
569 E : &bb1->instructions());
570 :
571 E : BasicCodeBlock* bb2 = subgraph.AddBasicCodeBlock("bb2");
572 E : desc->basic_block_order.push_back(bb2);
573 : BasicBlockAssembler bb2_asm(bb2->instructions().begin(),
574 E : &bb2->instructions());
575 :
576 E : BasicCodeBlock* bb3 = subgraph.AddBasicCodeBlock("bb3");
577 E : desc->basic_block_order.push_back(bb3);
578 : BasicBlockAssembler bb3_asm(bb3->instructions().begin(),
579 E : &bb3->instructions());
580 :
581 E : BasicCodeBlock* bbo = subgraph.AddBasicCodeBlock("overflow");
582 E : desc->basic_block_order.push_back(bbo);
583 : BasicBlockAssembler bbo_asm(bbo->instructions().begin(),
584 E : &bbo->instructions());
585 :
586 E : BasicCodeBlock* bbs = subgraph.AddBasicCodeBlock("slowpath");
587 : BasicBlockAssembler bbs_asm(bbs->instructions().begin(),
588 E : &bbs->instructions());
589 E : desc->basic_block_order.push_back(bbs);
590 :
591 : // Assemble instrumentation into the instruction stream.
592 E : Immediate module_data(add_frequency_data_.frequency_data_block(), 0);
593 : Operand fd_entry_hook(Displacement(fd_entry_hook_ref_.referenced(),
594 E : fd_entry_hook_ref_.offset()));
595 E : Operand fd_slot(Displacement(0x700, core::kSize32Bit));
596 :
597 E : bb1_asm.push(core::eax);
598 E : bb1_asm.lahf();
599 E : bb1_asm.set(core::kOverflow, core::eax);
600 E : bb1_asm.push(core::eax);
601 E : bb1_asm.push(core::edx);
602 E : bb1_asm.mov(core::edx, Operand(core::esp, Displacement(16)));
603 E : bb1_asm.mov_fs(core::eax, Operand(Displacement(0x700)));
604 E : bb1_asm.test(core::eax, core::eax);
605 :
606 : // Equivalent to: je slowpath.
607 E : AddSuccessorBetween(Successor::kConditionEqual, bb1, bbs);
608 E : AddSuccessorBetween(Successor::kConditionNotEqual, bb1, bb2);
609 :
610 : bb2_asm.add(Operand(core::eax, core::edx, core::kTimes4),
611 E : Immediate(1, core::kSize8Bit));
612 :
613 : // Equivalent to: jz overflow.
614 E : AddSuccessorBetween(Successor::kConditionEqual, bb2, bbo);
615 E : AddSuccessorBetween(Successor::kConditionNotEqual, bb2, bb3);
616 :
617 E : bb3_asm.pop(core::edx);
618 E : bb3_asm.pop(core::eax);
619 E : bb3_asm.add_b(core::eax, Immediate(0x7F, core::kSize8Bit));
620 E : bb3_asm.sahf();
621 E : bb3_asm.pop(core::eax);
622 E : bb3_asm.ret(4);
623 :
624 : // Overflow block. Call each time a counter has an overflow.
625 : // TODO(etienneb): Accumulate 32-bit overflow in a 64-bit external counter?
626 : bbo_asm.sub(Operand(core::eax, core::edx, core::kTimes4),
627 E : Immediate(1, core::kSize8Bit));
628 E : AddSuccessorBetween(Successor::kConditionTrue, bbo, bb3);
629 :
630 : // Slow path block. Should be called once by thread.
631 E : bbs_asm.push(core::ecx);
632 E : bbs_asm.push(core::edx);
633 E : bbs_asm.pushfd();
634 :
635 E : bbs_asm.push(module_data);
636 E : bbs_asm.call(fd_entry_hook);
637 E : bbs_asm.mov_fs(fd_slot, core::eax);
638 :
639 E : bbs_asm.popfd();
640 E : bbs_asm.pop(core::edx);
641 E : bbs_asm.pop(core::ecx);
642 E : AddSuccessorBetween(Successor::kConditionTrue, bbs, bb2);
643 :
644 : // Condense the whole mess into a block.
645 E : BlockBuilder block_builder(block_graph);
646 E : if (!block_builder.Merge(&subgraph)) {
647 i : LOG(ERROR) << "Failed to build thunk block.";
648 i : return false;
649 : }
650 :
651 : // Exactly one new block should have been created.
652 E : DCHECK_EQ(1u, block_builder.new_blocks().size());
653 E : *fast_path_block = block_builder.new_blocks().front();
654 :
655 E : return true;
656 E : }
657 :
658 : } // namespace transforms
659 : } // namespace instrument
|