# ========
# captured on: Tue Sep 22 15:32:19 2015
# hostname : aphost2
# os release : 2.6.32-358.23.2.el6.x86_64
# perf version : 2.6.32-358.23.2.el6.x86_64.debug
# arch : x86_64
# nrcpus online : 16
# nrcpus avail : 16
# cpudesc : Intel(R) Xeon(R) CPU E5-4650 0 @ 2.70GHz
# cpuid : GenuineIntel,6,45,7
# total memory : 264414484 kB
# cmdline : /usr/bin/perf record -g -p 93618 
# event : name = cycles, type = 0, config = 0x0, config1 = 0x0, config2 = 0x0, excl_usr = 0, excl_kern = 0, excl_host = 0, excl_guest = 1, precise_ip = 0, id = { 24 }
# HEADER_CPU_TOPOLOGY info available, use -I to display
# HEADER_NUMA_TOPOLOGY info available, use -I to display
# ========
#
# Samples: 17K of event 'cycles'
# Event count (approx.): 12516291301
#
# Overhead   Command                  Shared Object                                   Symbol
# ........  ........  .............................  .......................................
#
    18.65%  postgres  postgres                       [.] HeapTupleSatisfiesMVCC             
            |
            --- HeapTupleSatisfiesMVCC

    12.78%  postgres  postgres                       [.] bcTruelen                          
            |
            --- bcTruelen

     8.86%  postgres  postgres                       [.] varstr_cmp                         
            |
            --- varstr_cmp

     6.03%  postgres  postgres                       [.] slot_deform_tuple                  
            |
            --- slot_deform_tuple

     4.49%  postgres  postgres                       [.] heap_getnext                       
            |
            --- heap_getnext

     4.23%  postgres  postgres                       [.] ExecMakeFunctionResultNoSets       
            |
            --- ExecMakeFunctionResultNoSets

     4.17%  postgres  postgres                       [.] hash_search_with_hash_value        
            |
            --- hash_search_with_hash_value
               |          
               |--0.14%-- 0xd7fc3000000000
                --99.86%-- [...]

     3.75%  postgres  postgres                       [.] heap_page_prune_opt                
            |
            --- heap_page_prune_opt

     3.49%  postgres  postgres                       [.] PinBuffer                          
            |
            --- PinBuffer

     3.02%  postgres  postgres                       [.] slot_getattr                       
            |
            --- slot_getattr

     2.71%  postgres  postgres                       [.] LWLockAcquire                      
            |
            --- LWLockAcquire

     2.66%  postgres  postgres                       [.] heap_parallelscan_nextpage         
            |
            --- heap_parallelscan_nextpage

     2.57%  postgres  postgres                       [.] bpcharle                           
            |
            --- bpcharle

     2.55%  postgres  postgres                       [.] ExecScan                           
            |
            --- ExecScan

     1.75%  postgres  postgres                       [.] heapgetpage                        
            |
            --- heapgetpage

     1.50%  postgres  postgres                       [.] ExecEvalScalarVarFast              
            |
            --- ExecEvalScalarVarFast

     1.46%  postgres  postgres                       [.] ExecQual                           
            |
            --- ExecQual

     1.45%  postgres  postgres                       [.] UnpinBuffer.clone.0                
            |
            --- UnpinBuffer.clone.0

     1.31%  postgres  postgres                       [.] ExecStoreTuple                     
            |
            --- ExecStoreTuple

     1.21%  postgres  postgres                       [.] ExecEvalConst                      
            |
            --- ExecEvalConst

     0.86%  postgres  postgres                       [.] PartialSeqNext                     
            |
            --- PartialSeqNext

     0.81%  postgres  postgres                       [.] pg_detoast_datum_packed            
            |
            --- pg_detoast_datum_packed

     0.62%  postgres  postgres                       [.] check_stack_depth                  
            |
            --- check_stack_depth

     0.59%  postgres  postgres                       [.] LWLockRelease                      
            |
            --- LWLockRelease

     0.57%  postgres  postgres                       [.] pgstat_end_function_usage          
            |
            --- pgstat_end_function_usage
               |          
               |--1.01%-- 0x4b2bf00497ac1
               |          
               |--1.01%-- 0x1b402300497ac1
               |          
               |--1.01%-- 0x2292bc00497ac1
               |          
               |--1.01%-- 0x134fe200497ac1
               |          
               |--1.01%-- 0xb7c5000497ac1
               |          
               |--1.01%-- 0x29fd900497ac1
               |          
               |--1.01%-- 0x23affd00497ac1
               |          
               |--1.01%-- 0x161c5500497ac1
               |          
               |--1.01%-- 0x1ffcb800497ac1
               |          
               |--1.01%-- 0x19263400497ac1
               |          
               |--1.01%-- 0xe01c600497ac1
               |          
               |--1.01%-- 0xe1b1e00497ac1
               |          
               |--1.01%-- 0x12704b00497ac1
               |          
               |--1.01%-- 0x22fdd900497ac1
               |          
               |--1.01%-- 0xfdc1000497ac1
               |          
               |--1.01%-- 0xc8b5800497ac1
               |          
               |--1.01%-- 0x10206c00497ac1
               |          
               |--1.01%-- 0x1f156800497ac1
               |          
               |--1.01%-- 0x6bcf000497ac1
               |          
               |--1.01%-- 0xd9e0d00497ac1
               |          
               |--1.01%-- 0x19a5c00497ac1
               |          
               |--1.01%-- 0x90d3b00497ac1
               |          
               |--1.01%-- 0xe8f000497ac1
               |          
               |--1.01%-- 0x9579300497ac1
               |          
               |--1.01%-- 0xb33bd00497ac1
               |          
               |--1.01%-- 0xce05d00497ac1
               |          
               |--1.01%-- 0x5213800497ac1
               |          
               |--1.01%-- 0x1eff9600497ac1
               |          
               |--1.01%-- 0x206cf600497ac1
               |          
               |--1.01%-- 0xc027200497ac1
               |          
               |--1.01%-- 0x1b951800497ac1
               |          
               |--1.01%-- 0x1dcb6400497ac1
               |          
               |--1.01%-- 0x1b6fe900497ac1
               |          
               |--1.01%-- 0x14bfdb00497ac1
               |          
               |--1.01%-- 0x781c600497ac1
               |          
               |--1.01%-- 0x20162700497ac1
               |          
               |--1.01%-- 0x1901b00497ac1
               |          
               |--1.01%-- 0xa40a200497ac1
               |          
               |--1.01%-- 0x174c8c00497ac1
               |          
               |--1.01%-- 0x63f0200497ac1
               |          
               |--1.01%-- 0x1bab7600497ac1
               |          
               |--1.01%-- 0x145eab00497ac1
               |          
               |--1.01%-- 0x6f0ee00497ac1
               |          
               |--1.01%-- 0x1cf24400497ac1
               |          
               |--1.01%-- 0x14337c00497ac1
               |          
               |--1.01%-- 0x2483000497ac1
               |          
               |--1.01%-- 0x15b39800497ac1
               |          
               |--1.01%-- 0x1d51a900497ac1
               |          
               |--1.01%-- 0x7fc7f00497ac1
               |          
               |--1.01%-- 0x4527a00497ac1
               |          
               |--1.01%-- 0x35a4a00497ac1
               |          
               |--1.01%-- 0x1de01100497ac1
               |          
               |--0.91%-- 0x20e7cb00497ac1
                --46.51%-- [...]

     0.53%  postgres  postgres                       [.] pgstat_init_function_usage         
            |
            --- pgstat_init_function_usage

     0.52%  postgres  postgres                       [.] XidInMVCCSnapshot                  
            |
            --- XidInMVCCSnapshot

     0.50%  postgres  postgres                       [.] CheckForSerializableConflictOut    
            |
            --- CheckForSerializableConflictOut

     0.50%  postgres  postgres                       [.] hash_any                           
            |
            --- hash_any

     0.40%  postgres  postgres                       [.] lc_collate_is_c                    
            |
            --- lc_collate_is_c

     0.31%  postgres  postgres                       [.] MemoryContextReset                 
            |
            --- MemoryContextReset

     0.31%  postgres  postgres                       [.] GetPrivateRefCountEntry            
            |
            --- GetPrivateRefCountEntry

     0.31%  postgres  postgres                       [.] LockBuffer                         
            |
            --- LockBuffer

     0.28%  postgres  postgres                       [.] ReadBuffer_common                  
            |
            --- ReadBuffer_common

     0.24%  postgres  postgres                       [.] bpchareq                           
            |
            --- bpchareq

     0.23%  postgres  libc-2.12.so                   [.] memcpy                             
            |
            --- memcpy
               |          
               |--87.50%-- networkjoinsel
                --12.50%-- [...]

     0.21%  postgres  postgres                       [.] TransactionIdPrecedes              
            |
            --- TransactionIdPrecedes

     0.17%  postgres  [kernel.kallsyms].vsyscall_fn  [.] vread_tsc                          
            |
            --- vread_tsc
               |          
               |--79.31%-- 0x7fff2dfff814
               |          0x7fff2dfff8bc
               |          __gettimeofday
               |          
                --20.69%-- 0x7fff2dfff8bc
                          __gettimeofday

     0.17%  postgres  libc-2.12.so                   [.] __memcmp_sse4_1                    
            |
            --- __memcmp_sse4_1

     0.16%  postgres  postgres                       [.] shm_mq_receive                     
            |
            --- shm_mq_receive

     0.15%  postgres  postgres                       [.] ExecProject                        
            |
            --- ExecProject

     0.14%  postgres  postgres                       [.] ResourceOwnerForgetBuffer          
            |
            --- ResourceOwnerForgetBuffer

     0.12%  postgres  postgres                       [.] shm_mq_receive_bytes               
            |
            --- shm_mq_receive_bytes

     0.12%  postgres  [vdso]                         [.] 0x00007fff2dfff826                 
            |          
            |--20.00%-- 0x7fff2dfff8a4
            |          __gettimeofday
            |          
            |--20.00%-- 0x7fff2dfff830
            |          0x7fff2dfff8bc
            |          __gettimeofday
            |          
            |--10.00%-- 0x7fff2dfff8d4
            |          __gettimeofday
            |          
            |--5.00%-- 0x7fff2dfff826
            |          0x7fff2dfff8bc
            |          __gettimeofday
            |          
            |--5.00%-- 0x7fff2dfff8b0
            |          __gettimeofday
            |          
            |--5.00%-- 0x7fff2dfff81f
            |          0x7fff2dfff8bc
            |          __gettimeofday
            |          
            |--5.00%-- 0x7fff2dfff8db
            |          __gettimeofday
            |          
            |--5.00%-- 0x7fff2dfff7d1
            |          __gettimeofday
            |          
            |--5.00%-- 0x7fff2dfff8e9
            |          __gettimeofday
            |          
            |--5.00%-- 0x7fff2dfff8bc
            |          __gettimeofday
            |          
            |--5.00%-- 0x7fff2dfff890
            |          
            |--5.00%-- 0x7fff2dfff7d7
            |          0x7fff2dfff8bc
            |          __gettimeofday
            |          
             --5.00%-- 0x7fff2dfff8d8
                       __gettimeofday

     0.10%  postgres  postgres                       [.] make_result                        
            |
            --- make_result

     0.10%  postgres  postgres                       [.] LWLockConditionalAcquire           
            |
            --- LWLockConditionalAcquire

     0.09%  postgres  postgres                       [.] ExecClearTuple                     
            |
            --- ExecClearTuple

     0.09%  postgres  postgres                       [.] add_abs                            
            |
            --- add_abs

     0.09%  postgres  postgres                       [.] AllocSetAlloc                      
            |
            --- AllocSetAlloc

     0.09%  postgres  postgres                       [.] ResourceOwnerRememberBuffer        
            |
            --- ResourceOwnerRememberBuffer

     0.07%  postgres  postgres                       [.] ReadBufferExtended                 
            |
            --- ReadBufferExtended

     0.07%  postgres  postgres                       [.] s_lock                             
            |
            --- s_lock

     0.06%  postgres  postgres                       [.] ReleaseBuffer                      
            |
            --- ReleaseBuffer

     0.06%  postgres  postgres                       [.] do_numeric_accum                   
            |
            --- do_numeric_accum
               |          
               |--90.00%-- 0x0
                --10.00%-- [...]

     0.06%  postgres  postgres                       [.] advance_aggregates                 
            |
            --- advance_aggregates

     0.06%  postgres  postgres                       [.] IncrBufferRefCount                 
            |
            --- IncrBufferRefCount

     0.05%  postgres  postgres                       [.] ExecAgg                            
            |
            --- ExecAgg

     0.05%  postgres  postgres                       [.] BufTableLookup                     
            |
            --- BufTableLookup

     0.05%  postgres  postgres                       [.] isTempToastNamespace               
            |
            --- isTempToastNamespace

     0.05%  postgres  postgres                       [.] ExecProcNode                       
            |
            --- ExecProcNode

     0.05%  postgres  postgres                       [.] get_hash_value                     
            |
            --- get_hash_value

     0.04%  postgres  [kernel.kallsyms]              [k] rb_end_commit                      
            |
            --- rb_end_commit
                ring_buffer_unlock_commit
                trace_nowake_buffer_unlock_commit
               |          
               |--85.71%-- ftrace_raw_event_id_softirq
               |          |          
               |          |--49.99%-- ftrace_raw_event_softirq_raise
               |          |          raise_softirq
               |          |          run_local_timers
               |          |          update_process_times
               |          |          tick_sched_timer
               |          |          __run_hrtimer
               |          |          hrtimer_interrupt
               |          |          smp_apic_timer_interrupt
               |          |          apic_timer_interrupt
               |          |          |          
               |          |          |--66.66%-- ExecMakeFunctionResultNoSets
               |          |          |          
               |          |           --33.34%-- varstr_cmp
               |          |          
               |          |--33.35%-- ftrace_raw_event_softirq_exit
               |          |          __do_softirq
               |          |          call_softirq
               |          |          do_softirq
               |          |          irq_exit
               |          |          smp_apic_timer_interrupt
               |          |          apic_timer_interrupt
               |          |          HeapTupleSatisfiesMVCC
               |          |          
               |           --16.67%-- ftrace_raw_event_softirq_entry
               |                     __do_softirq
               |                     call_softirq
               |                     do_softirq
               |                     irq_exit
               |                     smp_apic_timer_interrupt
               |                     apic_timer_interrupt
               |                     HeapTupleSatisfiesMVCC
               |          
                --14.29%-- ftrace_raw_event_sched_stat_runtime
                          update_curr
                          task_tick_fair
                          scheduler_tick
                          update_process_times
                          tick_sched_timer
                          __run_hrtimer
                          hrtimer_interrupt
                          smp_apic_timer_interrupt
                          apic_timer_interrupt
                          ExecScan

     0.03%  postgres  postgres                       [.] TupleQueueFunnelNext               
            |
            --- TupleQueueFunnelNext

     0.03%  postgres  postgres                       [.] tag_hash                           
            |
            --- tag_hash

     0.03%  postgres  postgres                       [.] TupleHashTableHash                 
            |
            --- TupleHashTableHash
               |          
               |--16.67%-- 0xdb2510
                --83.33%-- [...]

     0.03%  postgres  postgres                       [.] int64_to_numericvar                
            |
            --- int64_to_numericvar

     0.03%  postgres  postgres                       [.] RecoveryInProgress                 
            |
            --- RecoveryInProgress

     0.03%  postgres  postgres                       [.] AllocSetFree                       
            |
            --- AllocSetFree

     0.03%  postgres  postgres                       [.] ResourceOwnerEnlargeBuffers        
            |
            --- ResourceOwnerEnlargeBuffers

     0.03%  postgres  postgres                       [.] palloc                             
            |
            --- palloc
               |          
               |--20.00%-- networkjoinsel
                --80.00%-- [...]

     0.03%  postgres  postgres                       [.] advance_transition_function        
            |
            --- advance_transition_function

     0.03%  postgres  postgres                       [.] DirectFunctionCall1Coll            
            |
            --- DirectFunctionCall1Coll

     0.03%  postgres  postgres                       [.] int8_avg_accum                     
            |
            --- int8_avg_accum

     0.03%  postgres  postgres                       [.] shm_mq_inc_bytes_read              
            |
            --- shm_mq_inc_bytes_read

     0.03%  postgres  [kernel.kallsyms]              [k] rb_reserve_next_event              
            |
            --- rb_reserve_next_event
                ring_buffer_lock_reserve
                trace_buffer_lock_reserve
                trace_current_buffer_lock_reserve
                ftrace_raw_event_id_softirq
               |          
               |--40.00%-- ftrace_raw_event_softirq_exit
               |          __do_softirq
               |          call_softirq
               |          do_softirq
               |          irq_exit
               |          smp_apic_timer_interrupt
               |          apic_timer_interrupt
               |          |          
               |          |--50.01%-- ExecMakeFunctionResultNoSets
               |          |          
               |           --49.99%-- hash_search_with_hash_value
               |          
               |--40.00%-- ftrace_raw_event_softirq_entry
               |          __do_softirq
               |          call_softirq
               |          do_softirq
               |          irq_exit
               |          smp_apic_timer_interrupt
               |          apic_timer_interrupt
               |          HeapTupleSatisfiesMVCC
               |          
                --19.99%-- ftrace_raw_event_softirq_raise
                          raise_softirq
                          run_local_timers
                          update_process_times
                          tick_sched_timer
                          __run_hrtimer
                          hrtimer_interrupt
                          smp_apic_timer_interrupt
                          apic_timer_interrupt
                          ExecScan

     0.03%  postgres  [kernel.kallsyms]              [k] trace_clock_local                  
            |
            --- trace_clock_local
               |          
               |--60.00%-- rb_reserve_next_event
               |          ring_buffer_lock_reserve
               |          trace_buffer_lock_reserve
               |          trace_current_buffer_lock_reserve
               |          ftrace_raw_event_id_softirq
               |          |          
               |          |--66.67%-- ftrace_raw_event_softirq_entry
               |          |          __do_softirq
               |          |          call_softirq
               |          |          do_softirq
               |          |          irq_exit
               |          |          smp_apic_timer_interrupt
               |          |          apic_timer_interrupt
               |          |          |          
               |          |          |--50.01%-- LWLockAcquire
               |          |          |          
               |          |           --49.99%-- HeapTupleSatisfiesMVCC
               |          |          
               |           --33.33%-- ftrace_raw_event_softirq_raise
               |                     raise_softirq
               |                     rcu_check_callbacks
               |                     update_process_times
               |                     tick_sched_timer
               |                     __run_hrtimer
               |                     hrtimer_interrupt
               |                     smp_apic_timer_interrupt
               |                     apic_timer_interrupt
               |                     HeapTupleSatisfiesMVCC
               |          
                --40.00%-- ring_buffer_lock_reserve
                          trace_buffer_lock_reserve
                          trace_current_buffer_lock_reserve
                          ftrace_raw_event_sched_stat_runtime
                          update_curr
                          task_tick_fair
                          scheduler_tick
                          update_process_times
                          tick_sched_timer
                          __run_hrtimer
                          hrtimer_interrupt
                          smp_apic_timer_interrupt
                          apic_timer_interrupt
                          |          
                          |--50.00%-- hash_search_with_hash_value
                          |          
                           --50.00%-- heap_getnext

     0.03%  postgres  [kernel.kallsyms]              [k] _spin_lock_irqsave                 
            |
            --- _spin_lock_irqsave
                ntp_tick_length
                do_timer
                tick_do_update_jiffies64
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--40.00%-- HeapTupleSatisfiesMVCC
               |          
               |--20.01%-- heap_getnext
               |          
               |--20.00%-- hash_any
               |          
                --20.00%-- bcTruelen

     0.03%  postgres  [kernel.kallsyms]              [k] __do_softirq                       
            |
            --- __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--40.02%-- HeapTupleSatisfiesMVCC
               |          
               |--20.00%-- LWLockRelease
               |          
               |--20.00%-- heap_getnext
               |          
                --19.97%-- bcTruelen

     0.02%  postgres  postgres                       [.] LookupTupleHashEntry               
            |
            --- LookupTupleHashEntry
                0xd7fc3000000000

     0.02%  postgres  postgres                       [.] execTuplesMatch                    
            |
            --- execTuplesMatch

     0.02%  postgres  postgres                       [.] int4_sum                           
            |
            --- int4_sum

     0.02%  postgres  postgres                       [.] slot_getsomeattrs                  
            |
            --- slot_getsomeattrs

     0.02%  postgres  postgres                       [.] init_var_from_num                  
            |
            --- init_var_from_num
               |          
               |--74.99%-- 0x0
               |          
                --25.01%-- slot_getsomeattrs

     0.02%  postgres  postgres                       [.] fetch_input_tuple                  
            |
            --- fetch_input_tuple

     0.02%  postgres  postgres                       [.] pfree                              
            |
            --- pfree

     0.02%  postgres  libc-2.12.so                   [.] __gettimeofday                     
            |
            --- __gettimeofday

     0.02%  postgres  [kernel.kallsyms]              [k] account_user_time                  
            |
            --- account_user_time
               |          
               |--50.00%-- account_process_tick
               |          update_process_times
               |          tick_sched_timer
               |          __run_hrtimer
               |          hrtimer_interrupt
               |          smp_apic_timer_interrupt
               |          apic_timer_interrupt
               |          |          
               |          |--50.02%-- bcTruelen
               |          |          
               |           --49.98%-- pg_detoast_datum_packed
               |          
                --50.00%-- update_process_times
                          tick_sched_timer
                          __run_hrtimer
                          hrtimer_interrupt
                          smp_apic_timer_interrupt
                          apic_timer_interrupt
                          |          
                          |--50.01%-- ExecEvalScalarVarFast
                          |          
                           --49.99%-- pgstat_end_function_usage

     0.02%  postgres  postgres                       [.] AllocSetReset                      
            |
            --- AllocSetReset

     0.02%  postgres  postgres                       [.] ExecFunnel                         
            |
            --- ExecFunnel

     0.02%  postgres  postgres                       [.] hash_search                        
            |
            --- hash_search
               |          
               |--33.34%-- 0xd83710
               |          
               |--33.34%-- 0xdb2510
               |          
                --33.33%-- 0xd7fc3000000000

     0.02%  postgres  postgres                       [.] FunctionCall1Coll                  
            |
            --- FunctionCall1Coll

     0.02%  postgres  postgres                       [.] InstrStopNode                      
            |
            --- InstrStopNode

     0.02%  postgres  postgres                       [.] int8_numeric                       
            |
            --- int8_numeric

     0.02%  postgres  postgres                       [.] alloc_var                          
            |
            --- alloc_var

     0.02%  postgres  [kernel.kallsyms]              [k] update_curr                        
            |
            --- update_curr
                task_tick_fair
                scheduler_tick
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--66.66%-- bcTruelen
               |          
                --33.34%-- heapgetpage

     0.02%  postgres  [kernel.kallsyms]              [k] tracing_generic_entry_update       
            |
            --- tracing_generic_entry_update
                trace_buffer_lock_reserve
                trace_current_buffer_lock_reserve
                ftrace_raw_event_id_softirq
               |          
               |--33.34%-- ftrace_raw_event_softirq_entry
               |          __do_softirq
               |          call_softirq
               |          do_softirq
               |          irq_exit
               |          smp_apic_timer_interrupt
               |          apic_timer_interrupt
               |          bcTruelen
               |          
               |--33.33%-- ftrace_raw_event_softirq_exit
               |          __do_softirq
               |          call_softirq
               |          do_softirq
               |          irq_exit
               |          smp_apic_timer_interrupt
               |          apic_timer_interrupt
               |          HeapTupleSatisfiesMVCC
               |          
                --33.33%-- ftrace_raw_event_softirq_raise
                          raise_softirq
                          rcu_check_callbacks
                          update_process_times
                          tick_sched_timer
                          __run_hrtimer
                          hrtimer_interrupt
                          smp_apic_timer_interrupt
                          apic_timer_interrupt
                          ExecStoreTuple

     0.02%  postgres  [kernel.kallsyms]              [k] scheduler_tick                     
            |
            --- scheduler_tick
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--33.34%-- ExecMakeFunctionResultNoSets
               |          
               |--33.33%-- varstr_cmp
               |          
                --33.33%-- HeapTupleSatisfiesMVCC

     0.02%  postgres  [kernel.kallsyms]              [k] raise_softirq                      
            |
            --- raise_softirq
               |          
               |--66.66%-- run_local_timers
               |          update_process_times
               |          tick_sched_timer
               |          __run_hrtimer
               |          hrtimer_interrupt
               |          smp_apic_timer_interrupt
               |          apic_timer_interrupt
               |          |          
               |          |--50.01%-- bcTruelen
               |          |          
               |           --49.99%-- heap_page_prune_opt
               |          
                --33.34%-- update_process_times
                          tick_sched_timer
                          __run_hrtimer
                          hrtimer_interrupt
                          smp_apic_timer_interrupt
                          apic_timer_interrupt
                          varstr_cmp

     0.02%  postgres  postgres                       [.] ss_report_location                 
            |
            --- ss_report_location

     0.01%  postgres  postgres                       [.] ExecInitAgg                        
            |
            --- ExecInitAgg

     0.01%  postgres  postgres                       [.] add_var                            
            |
            --- add_var
                (nil)

     0.01%  postgres  postgres                       [.] ExecStoreMinimalTuple              
            |
            --- ExecStoreMinimalTuple

     0.01%  postgres  postgres                       [.] hashbpchar                         
            |
            --- hashbpchar

     0.01%  postgres  [kernel.kallsyms]              [k] native_write_msr_safe              
            |
            --- native_write_msr_safe
               |          
               |--99.95%-- intel_pmu_disable_all
               |          x86_pmu_disable
               |          perf_pmu_disable
               |          perf_adjust_freq_unthr_context
               |          perf_event_task_tick
               |          scheduler_tick
               |          update_process_times
               |          tick_sched_timer
               |          __run_hrtimer
               |          hrtimer_interrupt
               |          smp_apic_timer_interrupt
               |          apic_timer_interrupt
               |          |          
               |          |--50.00%-- PinBuffer
               |          |          
               |           --50.00%-- varstr_cmp
                --0.05%-- [...]

     0.01%  postgres  [kernel.kallsyms]              [k] _spin_lock                         
            |
            --- _spin_lock
                scheduler_tick
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--50.01%-- bcTruelen
               |          
                --49.99%-- pgstat_end_function_usage

     0.01%  postgres  [kernel.kallsyms]              [k] ktime_get                          
            |
            --- ktime_get
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--50.01%-- hash_search_with_hash_value
               |          
                --49.99%-- HeapTupleSatisfiesMVCC

     0.01%  postgres  postgres                       [.] IsToastNamespace                   
            |
            --- IsToastNamespace

     0.01%  postgres  [kernel.kallsyms]              [k] rcu_process_gp_end                 
            |
            --- rcu_process_gp_end
                __rcu_process_callbacks
                rcu_process_callbacks
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--50.01%-- heap_parallelscan_nextpage
               |          
                --49.99%-- HeapTupleSatisfiesMVCC

     0.01%  postgres  [kernel.kallsyms]              [k] run_posix_cpu_timers               
            |
            --- run_posix_cpu_timers
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--50.00%-- varstr_cmp
               |          
                --50.00%-- MemoryContextReset

     0.01%  postgres  postgres                       [.] ReservePrivateRefCountEntry        
            |
            --- ReservePrivateRefCountEntry

     0.01%  postgres  [kernel.kallsyms]              [k] trace_recursive_unlock             
            |
            --- trace_recursive_unlock
                trace_nowake_buffer_unlock_commit
                ftrace_raw_event_id_softirq
                ftrace_raw_event_softirq_raise
                raise_softirq
                run_local_timers
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--50.01%-- heap_parallelscan_nextpage
               |          
                --49.99%-- HeapTupleSatisfiesMVCC

     0.01%  postgres  [kernel.kallsyms]              [k] __rcu_process_callbacks            
            |
            --- __rcu_process_callbacks
                rcu_process_callbacks
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
               |          
               |--50.01%-- varstr_cmp
               |          
                --49.99%-- heap_page_prune_opt

     0.01%  postgres  [kernel.kallsyms]              [k] ring_buffer_unlock_commit          
            |
            --- ring_buffer_unlock_commit
               |          
               |--50.03%-- trace_nowake_buffer_unlock_commit
               |          ftrace_raw_event_id_softirq
               |          ftrace_raw_event_softirq_raise
               |          raise_softirq
               |          run_local_timers
               |          update_process_times
               |          tick_sched_timer
               |          __run_hrtimer
               |          hrtimer_interrupt
               |          smp_apic_timer_interrupt
               |          apic_timer_interrupt
               |          bcTruelen
               |          
                --49.97%-- ftrace_raw_event_id_softirq
                          ftrace_raw_event_softirq_raise
                          raise_softirq
                          run_local_timers
                          update_process_times
                          tick_sched_timer
                          __run_hrtimer
                          hrtimer_interrupt
                          smp_apic_timer_interrupt
                          apic_timer_interrupt
                          heap_page_prune_opt

     0.01%  postgres  [kernel.kallsyms]              [k] call_softirq                       
            |
            --- call_softirq
               |          
               |--50.07%-- do_softirq
               |          irq_exit
               |          smp_apic_timer_interrupt
               |          apic_timer_interrupt
               |          ExecMakeFunctionResultNoSets
               |          
                --49.93%-- irq_exit
                          smp_apic_timer_interrupt
                          apic_timer_interrupt
                          HeapTupleSatisfiesMVCC

     0.01%  postgres  [kernel.kallsyms]              [k] apic_timer_interrupt               
            |
            --- apic_timer_interrupt
               |          
               |--50.09%-- bcTruelen
               |          
                --49.91%-- slot_deform_tuple

     0.01%  postgres  postgres                       [.] ScanKeywordLookup                  
            |
            --- ScanKeywordLookup

     0.01%  postgres  postgres                       [.] FunctionCall2Coll                  
            |
            --- FunctionCall2Coll

     0.01%  postgres  postgres                       [.] free_var                           
            |
            --- free_var

     0.01%  postgres  postgres                       [.] ExecPartialSeqScan                 
            |
            --- ExecPartialSeqScan

     0.01%  postgres  [kernel.kallsyms]              [k] rcu_bh_qs                          
            |
            --- rcu_bh_qs
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                ExecProject

     0.01%  postgres  postgres                       [.] SetLatch                           
            |
            --- SetLatch

     0.01%  postgres  [kernel.kallsyms]              [k] sched_clock                        
            |
            --- sched_clock
                rb_reserve_next_event
                ring_buffer_lock_reserve
                trace_buffer_lock_reserve
                trace_current_buffer_lock_reserve
                ftrace_raw_event_id_softirq
                ftrace_raw_event_softirq_raise
                raise_softirq
                rcu_check_callbacks
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                HeapTupleSatisfiesMVCC

     0.01%  postgres  postgres                       [.] InstrStartNode                     
            |
            --- InstrStartNode

     0.01%  postgres  postgres                       [.] BufTableHashCode                   
            |
            --- BufTableHashCode

     0.01%  postgres  postgres                       [.] ExecStoreVirtualTuple              
            |
            --- ExecStoreVirtualTuple

     0.01%  postgres  postgres                       [.] TupleHashTableMatch                
            |
            --- TupleHashTableMatch

     0.01%  postgres  [kernel.kallsyms]              [k] rcu_irq_exit                       
            |
            --- rcu_irq_exit
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                varstr_cmp

     0.01%  postgres  [kernel.kallsyms]              [k] save_args                          
            |
            --- save_args
                heap_getnext

     0.01%  postgres  [kernel.kallsyms]              [k] trace_current_buffer_lock_reserve  
            |
            --- trace_current_buffer_lock_reserve
                ftrace_raw_event_softirq_exit
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                LWLockAcquire

     0.01%  postgres  [kernel.kallsyms]              [k] x86_pmu_disable                    
            |
            --- x86_pmu_disable
                perf_pmu_disable
                perf_adjust_freq_unthr_context
                perf_event_task_tick
                scheduler_tick
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                ExecEvalConst

     0.01%  postgres  [kernel.kallsyms]              [k] update_rq_clock                    
            |
            --- update_rq_clock
                enqueue_task
                activate_task
                try_to_wake_up
                wake_up_process
                hrtimer_wakeup
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                varstr_cmp

     0.01%  postgres  [kernel.kallsyms]              [k] ip_send_skb                        
            |
            --- ip_send_skb
                ip_push_pending_frames
                icmp_push_reply
                icmp_reply
                icmp_echo
                icmp_rcv
                ip_local_deliver_finish
                ip_local_deliver
                ip_rcv_finish
                ip_rcv
                __netif_receive_skb
                process_backlog
                net_rx_action
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                do_IRQ
                ret_from_intr
                bcTruelen

     0.01%  postgres  [kernel.kallsyms]              [k] mod_timer                          
            |
            --- mod_timer
                0xffffffffa0016ff4
                0xffffffffa0018ea5
                run_timer_softirq
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                ExecQual

     0.01%  postgres  [kernel.kallsyms]              [k] read_tsc                           
            |
            --- read_tsc
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                heap_page_prune_opt

     0.01%  postgres  [kernel.kallsyms]              [k] filter_current_check_discard       
            |
            --- filter_current_check_discard
                ftrace_raw_event_softirq_exit
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                bcTruelen

     0.01%  postgres  [kernel.kallsyms]              [k] perf_pmu_disable                   
            |
            --- perf_pmu_disable
                perf_event_task_tick
                scheduler_tick
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                ExecStoreTuple

     0.01%  postgres  [kernel.kallsyms]              [k] account_process_tick               
            |
            --- account_process_tick
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                HeapTupleSatisfiesMVCC

     0.01%  postgres  postgres                       [.] MemoryContextResetOnly             
            |
            --- MemoryContextResetOnly

     0.01%  postgres  [kernel.kallsyms]              [k] task_tick_fair                     
            |
            --- task_tick_fair
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                HeapTupleSatisfiesMVCC

     0.01%  postgres  [kernel.kallsyms]              [k] force_quiescent_state              
            |
            --- force_quiescent_state
                __rcu_process_callbacks
                rcu_process_callbacks
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                bcTruelen

     0.01%  postgres  postgres                       [.] IsCatalogRelation                  
            |
            --- IsCatalogRelation

     0.01%  postgres  [kernel.kallsyms]              [k] perf_adjust_freq_unthr_context     
            |
            --- perf_adjust_freq_unthr_context
                perf_event_task_tick
                scheduler_tick
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                PinBuffer

     0.01%  postgres  [kernel.kallsyms]              [k] do_timer                           
            |
            --- do_timer
                tick_do_update_jiffies64
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                heapgetpage

     0.01%  postgres  [kernel.kallsyms]              [k] __rcu_pending                      
            |
            --- __rcu_pending
                rcu_check_callbacks
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                heap_parallelscan_nextpage

     0.01%  postgres  [kernel.kallsyms]              [k] run_timer_softirq                  
            |
            --- run_timer_softirq
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                pgstat_end_function_usage

     0.01%  postgres  [kernel.kallsyms]              [k] tcp_delack_timer                   
            |
            --- tcp_delack_timer
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                varstr_cmp

     0.01%  postgres  [kernel.kallsyms]              [k] kmem_cache_alloc_node              
            |
            --- kmem_cache_alloc_node
                __alloc_skb
                __netdev_alloc_skb
                ixgbe_poll
                net_rx_action
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                do_IRQ
                ret_from_intr
                MemoryContextReset

     0.01%  postgres  [kernel.kallsyms]              [k] ftrace_raw_event_sched_stat_runtime
            |
            --- ftrace_raw_event_sched_stat_runtime
                update_curr
                task_tick_fair
                scheduler_tick
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                PinBuffer

     0.01%  postgres  [kernel.kallsyms]              [k] clockevents_program_event          
            |
            --- clockevents_program_event
                tick_dev_program_event
                tick_program_event
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                bcTruelen

     0.01%  postgres  [kernel.kallsyms]              [k] ring_buffer_lock_reserve           
            |
            --- ring_buffer_lock_reserve
                trace_buffer_lock_reserve
                trace_current_buffer_lock_reserve
                ftrace_raw_event_id_softirq
                ftrace_raw_event_softirq_exit
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                ExecMakeFunctionResultNoSets

     0.01%  postgres  [kernel.kallsyms]              [k] update_cpu_load                    
            |
            --- update_cpu_load
                scheduler_tick
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                ExecScan

     0.01%  postgres  [kernel.kallsyms]              [k] native_sched_clock                 
            |
            --- native_sched_clock
                sched_clock
                trace_clock_local
                rb_reserve_next_event
                ring_buffer_lock_reserve
                trace_buffer_lock_reserve
                trace_current_buffer_lock_reserve
                ftrace_raw_event_id_softirq
                ftrace_raw_event_softirq_raise
                raise_softirq
                run_local_timers
                update_process_times
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                LWLockAcquire

     0.01%  postgres  [kernel.kallsyms]              [k] hrtimer_forward                    
            |
            --- hrtimer_forward
                tick_sched_timer
                __run_hrtimer
                hrtimer_interrupt
                smp_apic_timer_interrupt
                apic_timer_interrupt
                HeapTupleSatisfiesMVCC

     0.01%  postgres  [kernel.kallsyms]              [k] strcpy                             
            |
            --- strcpy
                ftrace_raw_event_net_dev_xmit
                dev_hard_start_xmit
                sch_direct_xmit
                dev_queue_xmit
                ip_finish_output
                ip_output
                ip_local_out
                ip_queue_xmit
                tcp_transmit_skb
                tcp_send_ack
                __tcp_ack_snd_check
                tcp_rcv_established
                tcp_v4_do_rcv
                tcp_v4_rcv
                ip_local_deliver_finish
                ip_local_deliver
                ip_rcv_finish
                ip_rcv
                __netif_receive_skb
                process_backlog
                net_rx_action
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                do_IRQ
                ret_from_intr
                ExecScan

     0.01%  postgres  [kernel.kallsyms]              [k] __rb_reserve_next                  
            |
            --- __rb_reserve_next
                rb_reserve_next_event
                ring_buffer_lock_reserve
                trace_buffer_lock_reserve
                trace_current_buffer_lock_reserve
                ftrace_raw_event_id_softirq
                ftrace_raw_event_softirq_entry
                __do_softirq
                call_softirq
                do_softirq
                irq_exit
                smp_apic_timer_interrupt
                apic_timer_interrupt
                hash_search_with_hash_value

     0.01%  postgres  [kernel.kallsyms]              [k] rw_verify_area                     
            |
            --- rw_verify_area
                vfs_write
                sys_write
                system_call
                __write_nocancel

     0.00%  postgres  [kernel.kallsyms]              [k] sock_poll                          
            |
            --- sock_poll
                do_sys_poll
                sys_poll
                system_call
                __poll
                0x4cf570b62e62

     0.00%  postgres  [kernel.kallsyms]              [k] perf_pmu_rotate_start              
            |
            --- perf_pmu_rotate_start
                perf_event_context_sched_in
                __perf_event_task_sched_in
                finish_task_switch
                thread_return
                schedule_hrtimeout_range
                poll_schedule_timeout
                do_sys_poll
                sys_poll
                system_call
                __poll
                0x4cf570b62e62



#
# (For a higher level overview, try: perf report --sort comm,dso)
#
