gpu events in PajeTrace

This commit is contained in:
Matthias Hochsteger 2023-02-22 14:07:08 +01:00
parent 714b064666
commit c0f57369e5
3 changed files with 42 additions and 3 deletions

View File

@ -65,6 +65,7 @@ namespace ngcore
jobs.reserve(reserve_size);
timer_events.reserve(reserve_size);
gpu_events.reserve(reserve_size);
memory_events.reserve(1024*1024);
// sync start time when running in parallel
@ -95,6 +96,9 @@ namespace ngcore
for(auto & event : timer_events)
event.time -= start_time;
for(auto & event : gpu_events)
event.time -= start_time;
for(auto & llink : links)
for(auto & link : llink)
link.time -= start_time;
@ -114,6 +118,9 @@ namespace ngcore
for(auto & event : timer_events)
event.timer_id += NgProfiler::SIZE*comm.Rank();
for(auto & event : gpu_events)
event.timer_id += NgProfiler::SIZE*comm.Rank();
if(comm.Rank() == MPI_PAJE_WRITER)
Write(tracefile_name);
else
@ -148,7 +155,7 @@ namespace ngcore
else if (x<5*d)
r=6*(x-4*d), g=0,b=1;
else
r=1, g=0,b=1-5*(x-d);
r=1, g=0,b=1-6*(x-5*d);
};
int alias_counter;
@ -504,7 +511,8 @@ namespace ngcore
for (int i=0; i<nthreads; i++)
{
auto name = "Thread " + ToString(i);
thread_aliases.emplace_back( paje.CreateContainer( container_type_thread, container_nodes[i*num_nodes/nthreads], name ) );
if(tasks[i].size())
thread_aliases.emplace_back( paje.CreateContainer( container_type_thread, container_nodes[i*num_nodes/nthreads], name ) );
}
}
@ -556,6 +564,8 @@ namespace ngcore
for(auto & event : timer_events)
timer_ids.insert(event.timer_id);
for(auto & event : gpu_events)
timer_ids.insert(event.timer_id);
// Timer names
for(auto & vtasks : tasks)
@ -623,6 +633,18 @@ namespace ngcore
paje.PopState( event.time, state_type_timer, timer_container_aliases[--timerdepth] );
}
if(gpu_events.size())
{
auto gpu_container = paje.CreateContainer( container_type_timer, container_task_manager, "GPU" );
for(auto & event : gpu_events)
{
if(event.is_start)
paje.PushState( event.time, state_type_timer, gpu_container, timer_aliases[event.timer_id] );
else
paje.PopState( event.time, state_type_timer, gpu_container);
}
}
for(auto & vtasks : tasks)
{
for (Task & t : vtasks) {

View File

@ -119,6 +119,7 @@ namespace ngcore
std::vector<std::vector<Task> > tasks;
std::vector<Job> jobs;
std::vector<TimerEvent> timer_events;
std::vector<TimerEvent> gpu_events;
std::vector<std::vector<ThreadLink> > links;
NGCORE_API static std::vector<MemoryEvent> memory_events;
@ -134,6 +135,22 @@ namespace ngcore
void operator=(const PajeTrace &) = delete;
void operator=(PajeTrace &&) = delete;
void StartGPU(int timer_id = 0)
{
if(!tracing_enabled) return;
if(unlikely(gpu_events.size() == max_num_events_per_thread))
StopTracing();
gpu_events.push_back(TimerEvent{timer_id, GetTimeCounter(), true});
}
void StopGPU(int timer_id)
{
if(!tracing_enabled) return;
if(unlikely(gpu_events.size() == max_num_events_per_thread))
StopTracing();
gpu_events.push_back(TimerEvent{timer_id, GetTimeCounter(), false});
}
void StartTimer(int timer_id)
{
if(!tracing_enabled) return;

View File

@ -247,7 +247,7 @@ namespace ngcore
double GetMFlops ()
{ return NgProfiler::GetFlops(timernr)
/ NgProfiler::GetTime(timernr) * 1e-6; }
operator int () { return timernr; }
operator int () const { return timernr; }
};