Add LocalHeap, some NGCORE_API

libsrc/core/CMakeLists.txt

@@ -1,5 +1,13 @@
 
-add_library(ngcore SHARED archive.cpp logging.cpp paje_trace.cpp utils.cpp profiler.cpp taskmanager.cpp)
+add_library(ngcore SHARED
+  archive.cpp
+  localheap.cpp
+  logging.cpp
+  paje_trace.cpp
+  profiler.cpp
+  taskmanager.cpp
+  utils.cpp
+  )
 
 target_compile_definitions(ngcore PRIVATE NGCORE_EXPORTS)
 if(NOT WIN32)
@@ -34,7 +42,7 @@ endif(USE_PYTHON)
 
 install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp logging.hpp
   exception.hpp symboltable.hpp paje_trace.hpp utils.hpp profiler.hpp mpi_wrapper.hpp
-  array.hpp taskmanager.hpp concurrentqueue.h
+  array.hpp taskmanager.hpp concurrentqueue.h localheap.hpp
   DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel)
 
 if(ENABLE_CPP_CORE_GUIDELINES_CHECK)

libsrc/core/array.hpp

@@ -8,6 +8,8 @@
 /**************************************************************************/
 
 
+#include "archive.hpp"
+#include "localheap.hpp"
 #include "utils.hpp"
 
 #ifdef DEBUG
@@ -415,14 +417,14 @@ namespace ngcore
     NETGEN_INLINE FlatArray (size_t asize, T * adata) 
       : size(asize), data(adata) { ; }
     
-//     /// memory from local heap
+    /// memory from local heap
-//     NETGEN_INLINE FlatArray(size_t asize, Allocator & lh)
+    NETGEN_INLINE FlatArray(size_t asize, Allocator & lh)
-//       : size(asize), data(new (lh) T[asize])
+      : size(asize), data(new (lh) T[asize])
-//     { ; }
+    { ; }
-// 
+
-//     NETGEN_INLINE FlatArray(size_t asize, LocalHeap & lh)
+    NETGEN_INLINE FlatArray(size_t asize, LocalHeap & lh)
-//       : size(asize), data (lh.Alloc<T> (asize))
+      : size(asize), data (lh.Alloc<T> (asize))
-//     { ; }
+    { ; }
 
     /// the size
     NETGEN_INLINE size_t Size() const { return size; }
@@ -475,13 +477,13 @@ namespace ngcore
       return *this;
     }
 
-//     /// assigns memory from local heap
+    /// assigns memory from local heap
-//     NETGEN_INLINE const FlatArray & Assign (size_t asize, LocalHeap & lh)
+    NETGEN_INLINE const FlatArray & Assign (size_t asize, LocalHeap & lh)
-//     {
+    {
-//       size = asize;
+      size = asize;
-//       data = lh.Alloc<T> (asize);
+      data = lh.Alloc<T> (asize);
-//       return *this;
+      return *this;
-//     }
+    }
 
     /// Access array. range check by macro CHECK_RANGE
     NETGEN_INLINE T & operator[] (size_t i) const
@@ -721,20 +723,20 @@ namespace ngcore
       delete [] mem_to_delete;
     }
 
-//     // Only provide this function if T is archivable
+    // Only provide this function if T is archivable
-//     template<typename T2=T>
+    template<typename T2=T>
-//     auto DoArchive(Archive& archive) -> typename std::enable_if_t<is_archivable<T2>, void>
+    auto DoArchive(Archive& archive) -> typename std::enable_if_t<is_archivable<T2>, void>
-//     {
+    {
-//       if(archive.Output())
+      if(archive.Output())
-//         archive << size;
+        archive << size;
-//       else
+      else
-//         {
+        {
-//           size_t s;
+          size_t s;
-//           archive & s;
+          archive & s;
-//           SetSize(s);
+          SetSize(s);
-//         }
+        }
-//       archive.Do(data, size);
+      archive.Do(data, size);
-//     }
+    }
 
     /// we tell the compiler that there is no need for deleting the array ..
     NETGEN_INLINE void NothingToDelete () 
@@ -769,15 +771,15 @@ namespace ngcore
     }
 
 
-//     /// assigns memory from local heap
+    /// assigns memory from local heap
-//     NETGEN_INLINE const Array & Assign (size_t asize, LocalHeap & lh)
+    NETGEN_INLINE const Array & Assign (size_t asize, LocalHeap & lh)
-//     {
+    {
-//       delete [] mem_to_delete;
+      delete [] mem_to_delete;
-//       size = allocsize = asize;
+      size = allocsize = asize;
-//       data = lh.Alloc<T> (asize);
+      data = lh.Alloc<T> (asize);
-//       mem_to_delete = nullptr;
+      mem_to_delete = nullptr;
-//       return *this;
+      return *this;
-//     }
+    }
 
     /// Add element at end of array. reallocation if necessary.
     NETGEN_INLINE size_t Append (const T & el)
@@ -803,7 +805,7 @@ namespace ngcore
     {
       if (size == allocsize) 
         ReSize (size+1);
-      data[size] = move(el);
+      data[size] = std::move(el);
       size++;
       return size;
     }
@@ -1011,12 +1013,12 @@ namespace ngcore
       {
         size_t mins = (nsize < size) ? nsize : size;
 #if defined(__GNUG__) && __GNUC__ < 5 && !defined(__clang__)
-        for (size_t i = 0; i < mins; i++) data[i] = move(hdata[i]);
+        for (size_t i = 0; i < mins; i++) data[i] = std::move(hdata[i]);
 #else
         if (std::is_trivially_copyable<T>::value)
           memcpy ((void*)data, hdata, sizeof(T)*mins);
         else
-          for (size_t i = 0; i < mins; i++) data[i] = move(hdata[i]);
+          for (size_t i = 0; i < mins; i++) data[i] = std::move(hdata[i]);
 #endif
         delete [] mem_to_delete;
       }

libsrc/core/localheap.cpp

@@ -0,0 +1,72 @@
+/**************************************************************************/
+/* File:   localheap.cpp                                                  */
+/* Author: Joachim Schoeberl                                              */
+/* Date:   19. Apr. 2002                                                  */
+/**************************************************************************/
+
+#include <exception>
+#include <string>
+
+#include "localheap.hpp"
+#include "taskmanager.hpp"
+
+namespace ngcore
+{
+
+  LocalHeap :: LocalHeap (size_t asize, const char * aname, bool mult_by_threads)
+  {
+    if (mult_by_threads)
+      asize *= TaskManager::GetMaxThreads();
+    totsize = asize;
+    try
+      {
+        data = new char[asize];
+      }
+    catch (std::exception & e)
+      {
+        throw Exception (ToString ("Could not allocate localheap, heapsize = ") + ToString(asize));
+      }
+
+    next = data + totsize;
+    p = data;
+    owner = true;
+    name = aname;
+    CleanUp();   // align pointer
+  }
+
+  LocalHeap LocalHeap :: Split() const
+  {
+    int pieces = TaskManager::GetNumThreads();
+    int i = TaskManager::GetThreadId();
+    size_t freemem = totsize - (p - data);
+    size_t size_of_piece = freemem / pieces;
+    return LocalHeap (p + i * size_of_piece, size_of_piece, name);
+  }
+
+  void LocalHeap :: ThrowException() // throw (LocalHeapOverflow)
+  {
+    /*
+    cout << "allocated: " << (p-data) << endl;
+    cout << "throw LocalHeapOverflow, totsize = "<< totsize << endl;
+    cout << "heap name = " << name << endl;
+    */
+    throw LocalHeapOverflow(totsize);
+  }
+
+
+  LocalHeapOverflow :: LocalHeapOverflow (size_t size) 
+    : Exception("Local Heap overflow\n")
+  {
+    std::stringstream str;
+    str << "Current heapsize is " << size << '\n';
+    Append (str.str());
+    // Append ("please use 'define constant heapsize = xxx' with larger value\n");
+  }
+  
+  LocalHeapOverflow :: ~LocalHeapOverflow ()
+  {
+    ; 
+  }
+
+}
+

libsrc/core/localheap.hpp

@@ -0,0 +1,318 @@
+#ifndef NETGEN_CORE_LOCALHEAP_HPP
+#define NETGEN_CORE_LOCALHEAP_HPP
+
+/**************************************************************************/
+/* File:   localheap.hpp                                                  */
+/* Author: Joachim Schoeberl                                              */
+/* Date:   19. Apr. 2000                                                  */
+/**************************************************************************/
+
+#include <cstdlib>
+
+#include "exception.hpp"
+#include "ngcore_api.hpp"
+#include "utils.hpp"
+
+namespace ngcore
+{
+
+class Allocator
+{
+public:
+  virtual ~Allocator() {}
+  virtual void * Alloc (size_t size)
+  {
+    return new char[size];
+  }
+  virtual void Delete(void* p)
+  {
+    delete (char*) p;
+  }
+  virtual void ArrayDelete(void* p)
+  {
+      delete [] (char*) p;
+  }
+};
+static Allocator global_alloc;
+
+/**
+   Exception on heap overflow.
+   Thrown by allocation on LocalHeap.
+*/
+class NGCORE_API LocalHeapOverflow : public Exception
+{
+public:
+  LocalHeapOverflow (size_t size);
+  virtual ~LocalHeapOverflow ();
+};
+
+
+
+/**
+   Optimized memory handler.
+   One block of data is organized as stack memory. 
+   One can allocate memory out of it. This increases the stack pointer.
+   With \Ref{CleanUp}, the pointer is reset to the beginning or to a
+   specific position. 
+*/
+class LocalHeap : public Allocator
+{
+  char * data;
+  char * next;
+  char * p;
+  size_t totsize;
+public:
+  bool owner;
+  const char * name;
+
+#if defined(__MIC__) || defined (__AVX512F__)
+  enum { ALIGN = 64 };
+#else
+  enum { ALIGN = 32 };
+#endif  
+
+public:
+  /// Allocate one block of size asize.
+  NGCORE_API LocalHeap (size_t asize, 
+                            const char * aname = "noname",
+                            bool mult_by_threads = false);
+
+  /// Use provided memory for the LocalHeap
+  NETGEN_INLINE LocalHeap (char * adata, size_t asize, 
+                    const char  * aname = "noname") throw ()
+  {
+    totsize = asize;
+    data = adata;
+    next = data + totsize;
+    owner = 0;
+    // p = data;
+    name = aname;
+    CleanUp();
+  }
+
+  /*
+  /// Use provided memory for the LocalHeap
+  NETGEN_INLINE LocalHeap (const LocalHeap & lh2)
+    : data(lh2.data), p(lh2.p), totsize(lh2.totsize), owner(false),
+      name(lh2.name)
+  {
+    next = data + totsize;
+  }
+  */
+  NETGEN_INLINE LocalHeap (const LocalHeap & lh2) = delete;
+
+  NETGEN_INLINE LocalHeap (LocalHeap && lh2)
+    : data(lh2.data), p(lh2.p), totsize(lh2.totsize), owner(lh2.owner),
+      name(lh2.name)
+  {
+    next = data + totsize;
+    lh2.owner = false;
+  }
+  
+  NETGEN_INLINE LocalHeap Borrow() 
+  {
+    return LocalHeap (p, Available());
+  }
+
+
+  NETGEN_INLINE LocalHeap & operator= (LocalHeap && lh2)
+  {
+    if (owner)
+      delete [] data;
+    
+    data = lh2.data;
+    p = lh2.p;
+    totsize = lh2.totsize;
+    owner = lh2.owner;
+    name = lh2.name;
+
+    next = data + totsize;
+    lh2.owner = false;
+    return *this;
+  }
+
+  NETGEN_INLINE LocalHeap ()
+    : data(nullptr), next(nullptr), p(nullptr), totsize(0), owner(false) { ; }
+
+
+  /// free memory
+  NETGEN_INLINE virtual ~LocalHeap ()
+  {
+    if (owner)
+      delete [] data;
+  }
+
+  /// delete all memory on local heap
+  NETGEN_INLINE void CleanUp() throw ()
+  {
+    p = data;
+    // p += (16 - (long(p) & 15) );
+    p += (ALIGN - (size_t(p) & (ALIGN-1) ) );
+  }
+
+  /// returns heap-pointer
+  NETGEN_INLINE void * GetPointer () throw ()
+  {
+    return p;
+  }
+
+  /// deletes memory back to heap-pointer
+  NETGEN_INLINE void CleanUp (void * addr) throw ()
+  {
+    p = (char*)addr;
+  }
+
+  /// allocates size bytes of memory from local heap
+  NETGEN_INLINE void * Alloc (size_t size) final // throw (LocalHeapOverflow)
+  {
+    char * oldp = p;
+  
+    // 16 byte alignment
+    size += (ALIGN - size % ALIGN);
+    p += size;
+
+    // if ( size_t(p - data) >= totsize )
+#ifndef FULLSPEED
+    if (likely(p >= next))
+        ThrowException();
+#endif
+      return oldp;
+    }
+
+    /// allocates size objects of type T on local heap
+    template <typename T>
+    NETGEN_INLINE T * Alloc (size_t size) // throw (LocalHeapOverflow)
+    {
+      char * oldp = p;
+      size *= sizeof (T);
+
+      // 16 byte alignment
+      size += (ALIGN - size % ALIGN);
+      p += size;
+
+#ifndef FULLSPEED
+      if (likely(p >= next))
+	ThrowException();
+#endif
+
+      return reinterpret_cast<T*> (oldp);
+    }
+
+    virtual void Delete(void* p) {}
+
+    virtual void ArrayDelete(void* p) {}
+  private: 
+    ///
+#ifndef __CUDA_ARCH__
+    [[noreturn]] NGCORE_API void ThrowException(); 
+#else
+    NETGEN_INLINE void ThrowException() { ; }
+#endif
+
+  public:
+    /// free memory (dummy function)
+    NETGEN_INLINE void Free (void * data) throw () 
+    {
+      ;
+    }
+
+    /// available memory on LocalHeap
+    NETGEN_INLINE size_t Available () const throw () { return (totsize - (p-data)); }
+
+    /// Split free memory on heap into pieces for each thread
+    NGCORE_API LocalHeap Split () const;
+
+    /// Split free memory on heap into pieces
+    NETGEN_INLINE LocalHeap Split (int partnr, int nparts) const
+    {
+      int pieces = nparts;
+      int i = partnr;
+
+      size_t freemem = totsize - (p - data);
+      size_t size_of_piece = freemem / pieces;
+      return LocalHeap (p + i * size_of_piece, size_of_piece, name);
+    }
+
+
+    NETGEN_INLINE void ClearValues ()
+    {
+      for (size_t i = 0; i < totsize; i++) data[i] = 47;
+    }
+
+    NETGEN_INLINE size_t UsedSize ()
+    {
+      for (size_t i = totsize-1; i != 0; i--)
+        if (data[i] != 47) return i;
+      return 0;
+    }
+  };
+
+
+
+  /**
+     Optimized memory handler.
+     Provides static memory for the local heap. The template argument specifies the size in number of chars.
+  */
+  template <int S>
+  class LocalHeapMem : public LocalHeap
+  {
+    char mem[S];
+  public:
+    NETGEN_INLINE LocalHeapMem (const char * aname) throw () : LocalHeap (mem, S, aname) { ; }
+  };
+
+
+
+
+
+
+
+
+  /**
+     A reset for the heap-pointer of a LocalHeap..
+     The constructor stores the heap-pointer, the constructor at the end of the regions resets the heap-pointer.
+  */
+  class HeapReset
+  {
+    LocalHeap & lh;
+    void * pointer;
+  public:
+    ///
+    NETGEN_INLINE HeapReset (LocalHeap & alh) 
+      : lh(alh), pointer (alh.GetPointer()) { ; }
+    
+    ///
+    NETGEN_INLINE ~HeapReset () 
+    {
+      lh.CleanUp (pointer);
+    }
+  };
+
+}
+
+
+
+NETGEN_INLINE void * operator new (size_t size, ngcore::Allocator & alloc)  
+{
+  return alloc.Alloc(size);
+}
+
+NETGEN_INLINE void * operator new [] (size_t size, ngcore::Allocator & alloc)  
+{
+  return alloc.Alloc(size);
+}
+
+
+NETGEN_INLINE void operator delete (void * p, ngcore::Allocator & lh)  
+{
+  lh.Delete(p);
+}
+
+NETGEN_INLINE void operator delete [] (void * p, ngcore::Allocator & lh)  
+{
+  lh.ArrayDelete(p);
+}
+
+
+
+#endif // NETGEN_CORE_LOCALHEAP_HPP

libsrc/core/ngcore.hpp

@@ -4,6 +4,7 @@
 #include "archive.hpp"
 #include "array.hpp"
 #include "exception.hpp"
+#include "localheap.hpp"
 #include "logging.hpp"
 #include "mpi_wrapper.hpp"
 #include "profiler.hpp"

libsrc/core/taskmanager.hpp

@@ -47,21 +47,21 @@ namespace ngcore
       atomic<int> participate{0};
     };
     
-    static const function<void(TaskInfo&)> * func;
+    NGCORE_API static const function<void(TaskInfo&)> * func;
-    static const function<void()> * startup_function;
+    NGCORE_API static const function<void()> * startup_function;
-    static const function<void()> * cleanup_function;
+    NGCORE_API static const function<void()> * cleanup_function;
-    static atomic<int> ntasks;
+    NGCORE_API static atomic<int> ntasks;
-    static Exception * ex;
+    NGCORE_API static Exception * ex;
 
-    static atomic<int> jobnr;
+    NGCORE_API static atomic<int> jobnr;
 
     static atomic<int> complete[8];   // max nodes
     static atomic<int> done;
     static atomic<int> active_workers;
     static atomic<int> workers_on_node[8];   // max nodes
     // Array<atomic<int>*> sync;
-    static int sleep_usecs;
+    NGCORE_API static int sleep_usecs;
-    static bool sleep;
+    NGCORE_API static bool sleep;
 
     static NodeData *nodedata[8];
 
@@ -80,12 +80,12 @@ namespace ngcore
     NGCORE_API static bool use_paje_trace;
   public:
     
-    TaskManager();
+    NGCORE_API TaskManager();
-    ~TaskManager();
+    NGCORE_API ~TaskManager();
 
 
-    void StartWorkers();
+    NGCORE_API void StartWorkers();
-    void StopWorkers();
+    NGCORE_API void StopWorkers();
 
     void SuspendWorkers(int asleep_usecs = 1000 )
       {
@@ -94,12 +94,12 @@ namespace ngcore
       }
     void ResumeWorkers() { sleep = false; }
 
-    static void SetNumThreads(int amax_threads);
+    NGCORE_API static void SetNumThreads(int amax_threads);
-    static int GetMaxThreads() { return max_threads; }
+    NGCORE_API static int GetMaxThreads() { return max_threads; }
     // static int GetNumThreads() { return task_manager ? task_manager->num_threads : 1; }
-    static int GetNumThreads() { return num_threads; }
+    NGCORE_API static int GetNumThreads() { return num_threads; }
-    static int GetThreadId() { return thread_id; } 
+    NGCORE_API static int GetThreadId() { return thread_id; } 
-    int GetNumNodes() const { return num_nodes; }
+    NGCORE_API int GetNumNodes() const { return num_nodes; }
 
     static void SetPajeTrace (bool use)  { use_paje_trace = use; }
     
@@ -112,9 +112,9 @@ namespace ngcore
     static void SetCleanupFunction () { cleanup_function = nullptr; }    
 
     void Done() { done = true; }
-    void Loop(int thread_num);
+    NGCORE_API void Loop(int thread_num);
 
-    static std::list<std::tuple<std::string,double>> Timing ();
+    NGCORE_API static std::list<std::tuple<std::string,double>> Timing ();
   };
 
 
@@ -128,8 +128,8 @@ namespace ngcore
   NGCORE_API void RunWithTaskManager (function<void()> alg);
 
   // For Python context manager
-  int  EnterTaskManager ();
+  NGCORE_API int  EnterTaskManager ();
-  void ExitTaskManager (int num_threads);
+  NGCORE_API void ExitTaskManager (int num_threads);
 
   NETGEN_INLINE int TasksPerThread (int tpt)
   {
@@ -888,7 +888,7 @@ public:
 
   void SetSize (size_t size)
   {
-    cerr << "************************* NumaDistArray::SetSize not overloaded" << endl;
+    std::cerr << "************************* NumaDistArray::SetSize not overloaded" << std::endl;
     Array<T>::SetSize(size);
   }
 };
@@ -946,7 +946,7 @@ public:
 
   void SetSize (size_t size)
   {
-    cerr << "************************* NumaDistArray::SetSize not overloaded" << endl;
+    std::cerr << "************************* NumaDistArray::SetSize not overloaded" << std::endl;
     Array<T>::SetSize(size);
   }
 };
@@ -995,7 +995,7 @@ public:
 
   void SetSize (size_t size)
   {
-    cerr << "************************* NumaDistArray::SetSize not overloaded" << endl;
+    std::cerr << "************************* NumaDistArray::SetSize not overloaded" << std::endl;
     Array<T>::SetSize(size);
   }
 };