ForkMaster/netgen
1
0
Fork 0
mirror of https://github.com/NGSolve/netgen.git synced 2024-12-24 21:10:33 +05:00
Code Issues Packages Projects Releases Wiki Activity

first attempt on memory tracing

Browse Source
This commit is contained in:
Matthias Hochsteger 2020-11-18 20:20:35 +01:00
parent c58659883d
commit 1a93fb3fa5
7 changed files with 183 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
libsrc/core/array.hpp
View File

@ -11,6 +11,7 @@
#include "archive.hpp"
#include "exception.hpp"
#include "localheap.hpp"
#include "profiler.hpp"
#include "utils.hpp"
namespace ngcore
@ -654,6 +655,8 @@ namespace ngcore
/// that's the data we have to delete, nullptr for not owning the memory
T * mem_to_delete;
int mem_tracing_id = 0;
using FlatArray<T,IndexType>::size;
using FlatArray<T,IndexType>::data;
using FlatArray<T,IndexType>::BASE;
@ -1038,6 +1041,18 @@ namespace ngcore
ngcore::Swap (data, b.data);
ngcore::Swap (allocsize, b.allocsize);
ngcore::Swap (mem_to_delete, b.mem_to_delete);
ngcore::Swap (mem_tracing_id, b.mem_tracing_id);
}
NETGEN_INLINE void SetMemoryTracing (int mem_id)
{
if(!mem_tracing_id && mem_id)
TraceMemoryAlloc(mem_id, sizeof(T)*allocsize);
if(mem_tracing_id && !mem_id)
TraceMemoryFree(mem_tracing_id, sizeof(T)*allocsize);
mem_tracing_id = mem_id;
}
private:
@ -1053,6 +1068,8 @@ namespace ngcore
{
size_t nsize = 2 * allocsize;
if (nsize < minsize) nsize = minsize;
TraceMemoryAlloc(mem_tracing_id, sizeof(T)*nsize );
T * hdata = data;
data = new T[nsize];
@ -1069,6 +1086,7 @@ namespace ngcore
for (size_t i = 0; i < mins; i++) data[i] = std::move(hdata[i]);
#endif
delete [] mem_to_delete;
TraceMemoryFree( mem_tracing_id, sizeof(T)*allocsize );
}
mem_to_delete = data;

29
libsrc/core/paje_trace.cpp
View File

@ -36,6 +36,7 @@ namespace ngcore
// increases trace by a factor of two
bool PajeTrace::trace_thread_counter = false;
bool PajeTrace::trace_threads = true;
bool PajeTrace::mem_tracing_enabled = true;
PajeTrace :: PajeTrace(int anthreads, std::string aname)
{
@ -62,6 +63,7 @@ namespace ngcore
jobs.reserve(reserve_size);
timer_events.reserve(reserve_size);
memory_events.reserve(1024*1024);
// sync start time when running in parallel
#ifdef PARALLEL
@ -72,6 +74,7 @@ namespace ngcore
start_time = GetTimeCounter();
tracing_enabled = true;
mem_tracing_enabled = true;
}
PajeTrace :: ~PajeTrace()
@ -94,6 +97,9 @@ namespace ngcore
for(auto & link : llink)
link.time -= start_time;
for(auto & m : memory_events)
m.time -= start_time;
NgMPI_Comm comm(MPI_COMM_WORLD);
if(comm.Size()==1)
@ -426,6 +432,7 @@ namespace ngcore
const int container_type_thread = paje.DefineContainerType( container_type_task_manager, "Thread");
const int container_type_timer = container_type_thread; //paje.DefineContainerType( container_type_task_manager, "Timers");
const int container_type_jobs = paje.DefineContainerType( container_type_task_manager, "Jobs");
const int container_type_memory = paje.DefineContainerType( container_type_task_manager, "Memory usage");
const int state_type_job = paje.DefineStateType( container_type_jobs, "Job" );
const int state_type_task = paje.DefineStateType( container_type_thread, "Task" );
@ -433,12 +440,20 @@ namespace ngcore
int variable_type_active_threads = 0;
if(trace_thread_counter)
paje.DefineVariableType( container_type_jobs, "Active threads" );
variable_type_active_threads = paje.DefineVariableType( container_type_jobs, "Active threads" );
const int container_task_manager = paje.CreateContainer( container_type_task_manager, 0, "The task manager" );
const int container_jobs = paje.CreateContainer( container_type_jobs, container_task_manager, "Jobs" );
if(trace_thread_counter)
paje.SetVariable( 0, variable_type_active_threads, container_jobs, 0.0 );
int variable_type_memory = 0;
if(mem_tracing_enabled)
{
variable_type_memory = paje.DefineVariableType( container_type_task_manager, "Memory [MB]" );
paje.SetVariable( 0, variable_type_memory, container_type_memory, 0.0 );
}
const int container_memory = paje.CreateContainer( container_type_memory, container_task_manager, "Memory" );
int num_nodes = 1; //task_manager ? task_manager->GetNumNodes() : 1;
std::vector <int> thread_aliases;
@ -509,6 +524,14 @@ namespace ngcore
paje.PopState( j.stop_time, state_type_job, container_jobs );
}
for(const auto & m : memory_events)
{
if(m.is_alloc)
paje.AddVariable( m.time, variable_type_memory, container_memory, 1.0*m.size / (1024*1024));
else
paje.SubVariable( m.time, variable_type_memory, container_memory, 1.0*m.size / (1024*1024));
}
std::set<int> timer_ids;
std::map<int,int> timer_aliases;
std::map<int,std::string> timer_names;

38
libsrc/core/paje_trace.hpp
View File

@ -23,6 +23,7 @@ namespace ngcore
NGCORE_API static size_t max_tracefile_size;
NGCORE_API static bool trace_thread_counter;
NGCORE_API static bool trace_threads;
NGCORE_API static bool mem_tracing_enabled;
bool tracing_enabled;
TTimePoint start_time;
@ -35,6 +36,11 @@ namespace ngcore
// be stopped if any thread reaches this number of events
unsigned int max_num_events_per_thread;
static void SetTraceMemory( bool trace_memory )
{
mem_tracing_enabled = trace_memory;
}
static void SetTraceThreads( bool atrace_threads )
{
trace_threads = atrace_threads;
@ -96,10 +102,21 @@ namespace ngcore
bool operator < (const ThreadLink & other) const { return time < other.time; }
};
struct MemoryEvent
{
TTimePoint time;
size_t size;
int region_id;
bool is_alloc;
bool operator < (const MemoryEvent & other) const { return time < other.time; }
};
std::vector<std::vector<Task> > tasks;
std::vector<Job> jobs;
std::vector<TimerEvent> timer_events;
std::vector<std::vector<ThreadLink> > links;
std::vector<MemoryEvent> memory_events;
public:
NGCORE_API void StopTracing();
@ -129,6 +146,27 @@ namespace ngcore
timer_events.push_back(TimerEvent{timer_id, GetTimeCounter(), false});
}
void AllocMemory(int id, size_t size)
{
if(!mem_tracing_enabled) return;
memory_events.push_back(MemoryEvent{GetTimeCounter(), size, id, true});
}
void FreeMemory(int id, size_t size)
{
if(!mem_tracing_enabled) return;
memory_events.push_back(MemoryEvent{GetTimeCounter(), size, id, false});
}
void ChangeMemory(int id, long long size)
{
if(size>0)
AllocMemory(id, size);
if(size<0)
FreeMemory(id, -size);
}
NETGEN_INLINE int StartTask(int thread_id, int id, int id_type = Task::ID_NONE, int additional_value = -1)
{
if(!tracing_enabled) return -1;

2
libsrc/core/profiler.cpp
View File

@ -113,5 +113,7 @@ namespace ngcore
NgProfiler prof; // NOLINT
std::vector<std::string> MemoryTracer::names{"root"};
std::map< int, std::vector<int> > MemoryTracer::tree;
} // namespace ngcore

87
libsrc/core/profiler.hpp
View File

@ -3,6 +3,7 @@
#include <array>
#include <chrono>
#include <functional>
#include <string>
#include "logging.hpp"
@ -299,6 +300,92 @@ namespace ngcore
return tres;
}
class MemoryTracer
{
NGCORE_API static std::vector<std::string> names;
static int GetId(std::string name)
{
int id = names.size();
names.push_back(name);
if(id==10*NgProfiler::SIZE)
std::cerr << "Allocated " << id << " MemoryTracer objects" << std::endl;
return id;
}
NGCORE_API static std::map< int, std::vector<int> > tree;
int id;
std::vector<std::function<void()>> tracks;
public:
MemoryTracer( std::string name )
{
id = GetId(name);
}
template <typename... TRest>
MemoryTracer( std::string name, TRest & ... rest )
{
id = GetId(name);
Track(rest...);
}
template <typename T1, typename... TRest>
void Track( T1 & obj, std::string name, TRest & ... rest )
{
Track(obj, name);
Track(rest...);
}
template<typename T>
void Track( T & obj, std::string name )
{
int child_id = GetId(name);
tree[id].push_back(child_id);
obj.SetMemoryTracing(child_id);
tracks.push_back( [&obj] () { obj.SetMemoryTracing(0); } );
}
template<typename T>
void Track( T & obj )
{
auto & mt = obj.GetMemoryTracer();
int child_id = mt.id;
tree[id].push_back(child_id);
}
void StopTracking()
{
for(auto & f : tracks)
f();
tracks.clear();
}
static std::string GetName(int id)
{
return names[id];
}
~MemoryTracer()
{
StopTracking();
}
};
NETGEN_INLINE void TraceMemoryAlloc( int mem_id, size_t size )
{
if(mem_id && trace)
trace->AllocMemory(mem_id, size);
}
NETGEN_INLINE void TraceMemoryFree( int mem_id, size_t size )
{
if(mem_id && trace)
trace->FreeMemory(mem_id, size);
}
} // namespace ngcore
// Helper macro to easily add multiple timers in a function for profiling

4
libsrc/core/python_ngcore_export.cpp
View File

@ -247,15 +247,17 @@ threads : int
;
py::class_<PajeTrace>(m, "PajeTrace")
.def(py::init( [] (string filename, size_t size_mb, bool threads, bool thread_counter)
.def(py::init( [] (string filename, size_t size_mb, bool threads, bool thread_counter, bool memory)
{
PajeTrace::SetMaxTracefileSize(size_mb*1014*1024);
PajeTrace::SetTraceThreads(threads);
PajeTrace::SetTraceMemory(memory);
PajeTrace::SetTraceThreadCounter(thread_counter);
trace = new PajeTrace(TaskManager::GetMaxThreads(), filename);
return trace;
}), py::arg("filename")="ng.trace", py::arg("size")=1000,
py::arg("threads")=true, py::arg("thread_counter")=false,
py::arg("memory")=true,
"size in Megabytes"
)
.def("__enter__", [](PajeTrace & self) { })

9
libsrc/meshing/meshclass.hpp
View File

@ -925,6 +925,15 @@ namespace netgen
shared_ptr<Mesh> Mirror( netgen::Point<3> p, Vec<3> n );
private:
MemoryTracer mem_tracer = {"Mesh",
points, "points",
segments, "segments",
surfelements, "surfelements",
volelements, "volelements"
};
public:
const MemoryTracer & GetMemoryTracer() { return mem_tracer; }
};
inline ostream& operator<<(ostream& ost, const Mesh& mesh)